commit: f1bc56c99bac05dccb01c1b68ef778eb08fbcb71
parent 64e419bd7386b2a16a3c2e7ac5da30427afe856d
Author: Sergey M․ <dstftw@gmail.com>
Date: Tue, 29 Dec 2020 02:11:48 +0700
[youtube:tab] Restore retry on browse requests (closes #27313, closes #27564)
Diffstat:
1 file changed, 19 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
@@ -16,6 +16,7 @@ from ..jsinterp import JSInterpreter
from ..swfinterp import SWFInterpreter
from ..compat import (
compat_chr,
+ compat_HTTPError,
compat_parse_qs,
compat_urllib_parse_unquote,
compat_urllib_parse_unquote_plus,
@@ -3009,10 +3010,24 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
for page_num in itertools.count(1):
if not continuation:
break
- browse = self._download_json(
- 'https://www.youtube.com/browse_ajax', None,
- 'Downloading page %d' % page_num,
- headers=headers, query=continuation, fatal=False)
+ count = 0
+ retries = 3
+ while count <= retries:
+ try:
+ # Downloading page may result in intermittent 5xx HTTP error
+ # that is usually worked around with a retry
+ browse = self._download_json(
+ 'https://www.youtube.com/browse_ajax', None,
+ 'Downloading page %d%s'
+ % (page_num, ' (retry #%d)' % count if count else ''),
+ headers=headers, query=continuation)
+ break
+ except ExtractorError as e:
+ if isinstance(e.cause, compat_HTTPError) and e.cause.code in (500, 503):
+ count += 1
+ if count <= retries:
+ continue
+ raise
if not browse:
break
response = try_get(browse, lambda x: x[1]['response'], dict)
