logo

youtube-dl

[mirror] Download/Watch videos from video hostersgit clone https://hacktivis.me/git/mirror/youtube-dl.git

youtube.py (169316B)

    

  1. # coding: utf-8
  3. from __future__ import unicode_literals
  5. import collections
  6. import itertools
  7. import json
  8. import os.path
  9. import random
  10. import re
  11. import traceback
  13. from .common import InfoExtractor, SearchInfoExtractor
  14. from ..compat import (
  15.     compat_chr,
  16.     compat_HTTPError,
  17.     compat_map as map,
  18.     compat_str,
  19.     compat_urllib_parse,
  20.     compat_urllib_parse_parse_qs as compat_parse_qs,
  21.     compat_urllib_parse_unquote_plus,
  22.     compat_urllib_parse_urlparse,
  23.     compat_zip as zip,
  24. )
  25. from ..jsinterp import JSInterpreter
  26. from ..utils import (
  27.     clean_html,
  28.     dict_get,
  29.     error_to_compat_str,
  30.     ExtractorError,
  31.     float_or_none,
  32.     extract_attributes,
  33.     get_element_by_attribute,
  34.     int_or_none,
  35.     join_nonempty,
  36.     js_to_json,
  37.     LazyList,
  38.     merge_dicts,
  39.     mimetype2ext,
  40.     NO_DEFAULT,
  41.     parse_codecs,
  42.     parse_count,
  43.     parse_duration,
  44.     parse_qs,
  45.     qualities,
  46.     remove_start,
  47.     smuggle_url,
  48.     str_or_none,
  49.     str_to_int,
  50.     T,
  51.     traverse_obj,
  52.     try_call,
  53.     try_get,
  54.     txt_or_none,
  55.     unescapeHTML,
  56.     unified_strdate,
  57.     unsmuggle_url,
  58.     update_url,
  59.     update_url_query,
  60.     url_or_none,
  61.     urlencode_postdata,
  62.     urljoin,
  63. )
  66. class YoutubeBaseInfoExtractor(InfoExtractor):
  67.     """Provide base functions for Youtube extractors"""
  68.     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
  69.     _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
  71.     _LOOKUP_URL = 'https://accounts.google.com/_/signin/sl/lookup'
  72.     _CHALLENGE_URL = 'https://accounts.google.com/_/signin/sl/challenge'
  73.     _TFA_URL = 'https://accounts.google.com/_/signin/challenge?hl=en&TL={0}'
  75.     _NETRC_MACHINE = 'youtube'
  76.     # If True it will raise an error if no login info is provided
  77.     _LOGIN_REQUIRED = False
  79.     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM)'
  81.     def _login(self):
  82.         """
  83.         Attempt to log in to YouTube.
  84.         True is returned if successful or skipped.
  85.         False is returned if login failed.
  87.         If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
  88.         """
  89.         username, password = self._get_login_info()
  90.         # No authentication to be performed
  91.         if username is None:
  92.             if self._LOGIN_REQUIRED and self._downloader.params.get('cookiefile') is None:
  93.                 raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
  94.             return True
  96.         login_page = self._download_webpage(
  97.             self._LOGIN_URL, None,
  98.             note='Downloading login page',
  99.             errnote='unable to fetch login page', fatal=False)
  100.         if login_page is False:
  101.             return
  103.         login_form = self._hidden_inputs(login_page)
  105.         def req(url, f_req, note, errnote):
  106.             data = login_form.copy()
  107.             data.update({
  108.                 'pstMsg': 1,
  109.                 'checkConnection': 'youtube',
  110.                 'checkedDomains': 'youtube',
  111.                 'hl': 'en',
  112.                 'deviceinfo': '[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]',
  113.                 'f.req': json.dumps(f_req),
  114.                 'flowName': 'GlifWebSignIn',
  115.                 'flowEntry': 'ServiceLogin',
  116.                 # TODO: reverse actual botguard identifier generation algo
  117.                 'bgRequest': '["identifier",""]',
  118.             })
  119.             return self._download_json(
  120.                 url, None, note=note, errnote=errnote,
  121.                 transform_source=lambda s: re.sub(r'^[^[]*', '', s),
  122.                 fatal=False,
  123.                 data=urlencode_postdata(data), headers={
  124.                     'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8',
  125.                     'Google-Accounts-XSRF': 1,
  126.                 })
  128.         def warn(message):
  129.             self._downloader.report_warning(message)
  131.         lookup_req = [
  132.             username,
  133.             None, [], None, 'US', None, None, 2, False, True,
  134.             [
  135.                 None, None,
  136.                 [2, 1, None, 1,
  137.                  'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn',
  138.                  None, [], 4],
  139.                 1, [None, None, []], None, None, None, True
  140.             ],
  141.             username,
  142.         ]
  144.         lookup_results = req(
  145.             self._LOOKUP_URL, lookup_req,
  146.             'Looking up account info', 'Unable to look up account info')
  148.         if lookup_results is False:
  149.             return False
  151.         user_hash = try_get(lookup_results, lambda x: x[0][2], compat_str)
  152.         if not user_hash:
  153.             warn('Unable to extract user hash')
  154.             return False
  156.         challenge_req = [
  157.             user_hash,
  158.             None, 1, None, [1, None, None, None, [password, None, True]],
  159.             [
  160.                 None, None, [2, 1, None, 1, 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn', None, [], 4],
  161.                 1, [None, None, []], None, None, None, True
  162.             ]]
  164.         challenge_results = req(
  165.             self._CHALLENGE_URL, challenge_req,
  166.             'Logging in', 'Unable to log in')
  168.         if challenge_results is False:
  169.             return
  171.         login_res = try_get(challenge_results, lambda x: x[0][5], list)
  172.         if login_res:
  173.             login_msg = try_get(login_res, lambda x: x[5], compat_str)
  174.             warn(
  175.                 'Unable to login: %s' % 'Invalid password'
  176.                 if login_msg == 'INCORRECT_ANSWER_ENTERED' else login_msg)
  177.             return False
  179.         res = try_get(challenge_results, lambda x: x[0][-1], list)
  180.         if not res:
  181.             warn('Unable to extract result entry')
  182.             return False
  184.         login_challenge = try_get(res, lambda x: x[0][0], list)
  185.         if login_challenge:
  186.             challenge_str = try_get(login_challenge, lambda x: x[2], compat_str)
  187.             if challenge_str == 'TWO_STEP_VERIFICATION':
  188.                 # SEND_SUCCESS - TFA code has been successfully sent to phone
  189.                 # QUOTA_EXCEEDED - reached the limit of TFA codes
  190.                 status = try_get(login_challenge, lambda x: x[5], compat_str)
  191.                 if status == 'QUOTA_EXCEEDED':
  192.                     warn('Exceeded the limit of TFA codes, try later')
  193.                     return False
  195.                 tl = try_get(challenge_results, lambda x: x[1][2], compat_str)
  196.                 if not tl:
  197.                     warn('Unable to extract TL')
  198.                     return False
  200.                 tfa_code = self._get_tfa_info('2-step verification code')
  202.                 if not tfa_code:
  203.                     warn(
  204.                         'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
  205.                         '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
  206.                     return False
  208.                 tfa_code = remove_start(tfa_code, 'G-')
  210.                 tfa_req = [
  211.                     user_hash, None, 2, None,
  212.                     [
  213.                         9, None, None, None, None, None, None, None,
  214.                         [None, tfa_code, True, 2]
  215.                     ]]
  217.                 tfa_results = req(
  218.                     self._TFA_URL.format(tl), tfa_req,
  219.                     'Submitting TFA code', 'Unable to submit TFA code')
  221.                 if tfa_results is False:
  222.                     return False
  224.                 tfa_res = try_get(tfa_results, lambda x: x[0][5], list)
  225.                 if tfa_res:
  226.                     tfa_msg = try_get(tfa_res, lambda x: x[5], compat_str)
  227.                     warn(
  228.                         'Unable to finish TFA: %s' % 'Invalid TFA code'
  229.                         if tfa_msg == 'INCORRECT_ANSWER_ENTERED' else tfa_msg)
  230.                     return False
  232.                 check_cookie_url = try_get(
  233.                     tfa_results, lambda x: x[0][-1][2], compat_str)
  234.             else:
  235.                 CHALLENGES = {
  236.                     'LOGIN_CHALLENGE': "This device isn't recognized. For your security, Google wants to make sure it's really you.",
  237.                     'USERNAME_RECOVERY': 'Please provide additional information to aid in the recovery process.',
  238.                     'REAUTH': "There is something unusual about your activity. For your security, Google wants to make sure it's really you.",
  239.                 }
  240.                 challenge = CHALLENGES.get(
  241.                     challenge_str,
  242.                     '%s returned error %s.' % (self.IE_NAME, challenge_str))
  243.                 warn('%s\nGo to https://accounts.google.com/, login and solve a challenge.' % challenge)
  244.                 return False
  245.         else:
  246.             check_cookie_url = try_get(res, lambda x: x[2], compat_str)
  248.         if not check_cookie_url:
  249.             warn('Unable to extract CheckCookie URL')
  250.             return False
  252.         check_cookie_results = self._download_webpage(
  253.             check_cookie_url, None, 'Checking cookie', fatal=False)
  255.         if check_cookie_results is False:
  256.             return False
  258.         if 'https://myaccount.google.com/' not in check_cookie_results:
  259.             warn('Unable to log in')
  260.             return False
  262.         return True
  264.     def _initialize_consent(self):
  265.         cookies = self._get_cookies('https://www.youtube.com/')
  266.         if cookies.get('__Secure-3PSID'):
  267.             return
  268.         socs = cookies.get('SOCS')
  269.         if socs and not socs.value.startswith('CAA'):  # not consented
  270.             return
  271.         self._set_cookie('.youtube.com', 'SOCS', 'CAI', secure=True)  # accept all (required for mixes)
  273.     def _real_initialize(self):
  274.         self._initialize_consent()
  275.         if self._downloader is None:
  276.             return
  277.         if not self._login():
  278.             return
  280.     _DEFAULT_API_DATA = {
  281.         'context': {
  282.             'client': {
  283.                 'clientName': 'WEB',
  284.                 'clientVersion': '2.20201021.03.00',
  285.             }
  286.         },
  287.     }
  289.     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*=\s*({.+?})\s*;'
  290.     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
  291.     _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
  293.     def _call_api(self, ep, query, video_id, fatal=True, headers=None):
  294.         data = self._DEFAULT_API_DATA.copy()
  295.         data.update(query)
  296.         real_headers = {'content-type': 'application/json'}
  297.         if headers:
  298.             real_headers.update(headers)
  300.         return self._download_json(
  301.             'https://www.youtube.com/youtubei/v1/%s' % ep, video_id=video_id,
  302.             note='Downloading API JSON', errnote='Unable to download API page',
  303.             data=json.dumps(data).encode('utf8'), fatal=fatal,
  304.             headers=real_headers,
  305.             query={'key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'})
  307.     def _extract_yt_initial_data(self, video_id, webpage):
  308.         return self._parse_json(
  309.             self._search_regex(
  310.                 (r'%s\s*%s' % (self._YT_INITIAL_DATA_RE, self._YT_INITIAL_BOUNDARY_RE),
  311.                  self._YT_INITIAL_DATA_RE), webpage, 'yt initial data'),
  312.             video_id)
  314.     def _extract_ytcfg(self, video_id, webpage):
  315.         return self._parse_json(
  316.             self._search_regex(
  317.                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
  318.                 default='{}'), video_id, fatal=False) or {}
  320.     def _extract_video(self, renderer):
  321.         video_id = renderer['videoId']
  322.         title = try_get(
  323.             renderer,
  324.             (lambda x: x['title']['runs'][0]['text'],
  325.              lambda x: x['title']['simpleText'],
  326.              lambda x: x['headline']['simpleText']), compat_str)
  327.         description = try_get(
  328.             renderer, lambda x: x['descriptionSnippet']['runs'][0]['text'],
  329.             compat_str)
  330.         duration = parse_duration(try_get(
  331.             renderer, lambda x: x['lengthText']['simpleText'], compat_str))
  332.         view_count_text = try_get(
  333.             renderer, lambda x: x['viewCountText']['simpleText'], compat_str) or ''
  334.         view_count = str_to_int(self._search_regex(
  335.             r'^([\d,]+)', re.sub(r'\s', '', view_count_text),
  336.             'view count', default=None))
  337.         uploader = try_get(
  338.             renderer,
  339.             (lambda x: x['ownerText']['runs'][0]['text'],
  340.              lambda x: x['shortBylineText']['runs'][0]['text']), compat_str)
  341.         return {
  342.             '_type': 'url',
  343.             'ie_key': YoutubeIE.ie_key(),
  344.             'id': video_id,
  345.             'url': video_id,
  346.             'title': title,
  347.             'description': description,
  348.             'duration': duration,
  349.             'view_count': view_count,
  350.             'uploader': uploader,
  351.         }
  353.     def _search_results(self, query, params):
  354.         data = {
  355.             'context': {
  356.                 'client': {
  357.                     'clientName': 'WEB',
  358.                     'clientVersion': '2.20201021.03.00',
  359.                 }
  360.             },
  361.             'query': query,
  362.         }
  363.         if params:
  364.             data['params'] = params
  365.         for page_num in itertools.count(1):
  366.             search = self._download_json(
  367.                 'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  368.                 video_id='query "%s"' % query,
  369.                 note='Downloading page %s' % page_num,
  370.                 errnote='Unable to download API page', fatal=False,
  371.                 data=json.dumps(data).encode('utf8'),
  372.                 headers={'content-type': 'application/json'})
  373.             if not search:
  374.                 break
  375.             slr_contents = try_get(
  376.                 search,
  377.                 (lambda x: x['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents'],
  378.                  lambda x: x['onResponseReceivedCommands'][0]['appendContinuationItemsAction']['continuationItems']),
  379.                 list)
  380.             if not slr_contents:
  381.                 break
  382.             for slr_content in slr_contents:
  383.                 isr_contents = try_get(
  384.                     slr_content,
  385.                     lambda x: x['itemSectionRenderer']['contents'],
  386.                     list)
  387.                 if not isr_contents:
  388.                     continue
  389.                 for content in isr_contents:
  390.                     if not isinstance(content, dict):
  391.                         continue
  392.                     video = content.get('videoRenderer')
  393.                     if not isinstance(video, dict):
  394.                         continue
  395.                     video_id = video.get('videoId')
  396.                     if not video_id:
  397.                         continue
  398.                     yield self._extract_video(video)
  399.             token = try_get(
  400.                 slr_contents,
  401.                 lambda x: x[-1]['continuationItemRenderer']['continuationEndpoint']['continuationCommand']['token'],
  402.                 compat_str)
  403.             if not token:
  404.                 break
  405.             data['continuation'] = token
  407.     @staticmethod
  408.     def _owner_endpoints_path():
  409.         return [
  410.             Ellipsis,
  411.             lambda k, _: k.endswith('SecondaryInfoRenderer'),
  412.             ('owner', 'videoOwner'), 'videoOwnerRenderer', 'title',
  413.             'runs', Ellipsis]
  415.     def _extract_channel_id(self, webpage, videodetails={}, metadata={}, renderers=[]):
  416.         channel_id = None
  417.         if any((videodetails, metadata, renderers)):
  418.             channel_id = (
  419.                 traverse_obj(videodetails, 'channelId')
  420.                 or traverse_obj(metadata, 'externalChannelId', 'externalId')
  421.                 or traverse_obj(renderers,
  422.                                 self._owner_endpoints_path() + [
  423.                                     'navigationEndpoint', 'browseEndpoint', 'browseId'],
  424.                                 get_all=False)
  425.             )
  426.         return channel_id or self._html_search_meta(
  427.             'channelId', webpage, 'channel id', default=None)
  429.     def _extract_author_var(self, webpage, var_name,
  430.                             videodetails={}, metadata={}, renderers=[]):
  431.         result = None
  432.         paths = {
  433.             #       (HTML, videodetails, metadata, renderers)
  434.             'name': ('content', 'author', (('ownerChannelName', None), 'title'), ['text']),
  435.             'url': ('href', 'ownerProfileUrl', 'vanityChannelUrl',
  436.                     ['navigationEndpoint', 'browseEndpoint', 'canonicalBaseUrl'])
  437.         }
  438.         if any((videodetails, metadata, renderers)):
  439.             result = (
  440.                 traverse_obj(videodetails, paths[var_name][1], get_all=False)
  441.                 or traverse_obj(metadata, paths[var_name][2], get_all=False)
  442.                 or traverse_obj(renderers,
  443.                                 self._owner_endpoints_path() + paths[var_name][3],
  444.                                 get_all=False)
  445.             )
  446.         return result or traverse_obj(
  447.             extract_attributes(self._search_regex(
  448.                 r'''(?s)(<link\b[^>]+\bitemprop\s*=\s*("|')%s\2[^>]*>)'''
  449.                 % re.escape(var_name),
  450.                 get_element_by_attribute('itemprop', 'author', webpage or '') or '',
  451.                 'author link', default='')),
  452.             paths[var_name][0])
  454.     @staticmethod
  455.     def _yt_urljoin(url_or_path):
  456.         return urljoin('https://www.youtube.com', url_or_path)
  458.     def _extract_uploader_id(self, uploader_url):
  459.         return self._search_regex(
  460.             r'/(?:(?:channel|user)/|(?=@))([^/?&#]+)', uploader_url or '',
  461.             'uploader id', default=None)
  464. class YoutubeIE(YoutubeBaseInfoExtractor):
  465.     IE_DESC = 'YouTube.com'
  466.     _INVIDIOUS_SITES = (
  467.         # invidious-redirect websites
  468.         r'(?:www\.)?redirect\.invidious\.io',
  469.         r'(?:(?:www|dev)\.)?invidio\.us',
  470.         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/Invidious-Instances.md
  471.         r'(?:(?:www|no)\.)?invidiou\.sh',
  472.         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
  473.         r'(?:www\.)?invidious\.kabi\.tk',
  474.         r'(?:www\.)?invidious\.13ad\.de',
  475.         r'(?:www\.)?invidious\.mastodon\.host',
  476.         r'(?:www\.)?invidious\.zapashcanon\.fr',
  477.         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
  478.         r'(?:www\.)?invidious\.tinfoil-hat\.net',
  479.         r'(?:www\.)?invidious\.himiko\.cloud',
  480.         r'(?:www\.)?invidious\.reallyancient\.tech',
  481.         r'(?:www\.)?invidious\.tube',
  482.         r'(?:www\.)?invidiou\.site',
  483.         r'(?:www\.)?invidious\.site',
  484.         r'(?:www\.)?invidious\.xyz',
  485.         r'(?:www\.)?invidious\.nixnet\.xyz',
  486.         r'(?:www\.)?invidious\.048596\.xyz',
  487.         r'(?:www\.)?invidious\.drycat\.fr',
  488.         r'(?:www\.)?inv\.skyn3t\.in',
  489.         r'(?:www\.)?tube\.poal\.co',
  490.         r'(?:www\.)?tube\.connect\.cafe',
  491.         r'(?:www\.)?vid\.wxzm\.sx',
  492.         r'(?:www\.)?vid\.mint\.lgbt',
  493.         r'(?:www\.)?vid\.puffyan\.us',
  494.         r'(?:www\.)?yewtu\.be',
  495.         r'(?:www\.)?yt\.elukerio\.org',
  496.         r'(?:www\.)?yt\.lelux\.fi',
  497.         r'(?:www\.)?invidious\.ggc-project\.de',
  498.         r'(?:www\.)?yt\.maisputain\.ovh',
  499.         r'(?:www\.)?ytprivate\.com',
  500.         r'(?:www\.)?invidious\.13ad\.de',
  501.         r'(?:www\.)?invidious\.toot\.koeln',
  502.         r'(?:www\.)?invidious\.fdn\.fr',
  503.         r'(?:www\.)?watch\.nettohikari\.com',
  504.         r'(?:www\.)?invidious\.namazso\.eu',
  505.         r'(?:www\.)?invidious\.silkky\.cloud',
  506.         r'(?:www\.)?invidious\.exonip\.de',
  507.         r'(?:www\.)?invidious\.riverside\.rocks',
  508.         r'(?:www\.)?invidious\.blamefran\.net',
  509.         r'(?:www\.)?invidious\.moomoo\.de',
  510.         r'(?:www\.)?ytb\.trom\.tf',
  511.         r'(?:www\.)?yt\.cyberhost\.uk',
  512.         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
  513.         r'(?:www\.)?qklhadlycap4cnod\.onion',
  514.         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
  515.         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
  516.         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
  517.         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
  518.         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
  519.         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
  520.         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
  521.         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
  522.         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
  523.         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
  524.     )
  525.     _VALID_URL = r"""(?x)^
  526.                      (
  527.                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
  528.                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
  529.                             (?:www\.)?deturl\.com/www\.youtube\.com|
  530.                             (?:www\.)?pwnyoutube\.com|
  531.                             (?:www\.)?hooktube\.com|
  532.                             (?:www\.)?yourepeat\.com|
  533.                             tube\.majestyc\.net|
  534.                             %(invidious)s|
  535.                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
  536.                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
  537.                          (?:                                                  # the various things that can precede the ID:
  538.                              (?:(?:v|embed|e)/(?!videoseries))                # v/ or embed/ or e/
  539.                              |shorts/
  540.                              |(?:                                             # or the v= param in all its forms
  541.                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
  542.                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
  543.                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
  544.                                  v=
  545.                              )
  546.                          ))
  547.                          |(?:
  548.                             youtu\.be|                                        # just youtu.be/xxxx
  549.                             vid\.plus|                                        # or vid.plus/xxxx
  550.                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
  551.                             %(invidious)s
  552.                          )/
  553.                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
  554.                          )
  555.                      )?                                                       # all until now is optional -> you can pass the naked ID
  556.                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
  557.                      (?(1).+)?                                                # if we found the ID, everything can follow
  558.                      $""" % {
  559.         'invidious': '|'.join(_INVIDIOUS_SITES),
  560.     }
  561.     _PLAYER_INFO_RE = (
  562.         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
  563.         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
  564.         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
  565.     )
  566.     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
  568.     _GEO_BYPASS = False
  570.     IE_NAME = 'youtube'
  571.     _TESTS = [
  572.         {
  573.             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
  574.             'info_dict': {
  575.                 'id': 'BaW_jenozKc',
  576.                 'ext': 'mp4',
  577.                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
  578.                 'uploader': 'Philipp Hagemeister',
  579.                 'uploader_id': '@PhilippHagemeister',
  580.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@PhilippHagemeister',
  581.                 'channel': 'Philipp Hagemeister',
  582.                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
  583.                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
  584.                 'upload_date': '20121002',
  585.                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
  586.                 'categories': ['Science & Technology'],
  587.                 'tags': ['youtube-dl'],
  588.                 'duration': 10,
  589.                 'view_count': int,
  590.                 'like_count': int,
  591.                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
  592.                 'start_time': 1,
  593.                 'end_time': 9,
  594.             },
  595.         },
  596.         {
  597.             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
  598.             'note': 'Embed-only video (#1746)',
  599.             'info_dict': {
  600.                 'id': 'yZIXLfi8CZQ',
  601.                 'ext': 'mp4',
  602.                 'upload_date': '20120608',
  603.                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
  604.                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
  605.                 'uploader': 'SET India',
  606.                 'uploader_id': 'setindia',
  607.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
  608.                 'age_limit': 18,
  609.             },
  610.             'skip': 'Private video',
  611.         },
  612.         {
  613.             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
  614.             'note': 'Use the first video ID in the URL',
  615.             'info_dict': {
  616.                 'id': 'BaW_jenozKc',
  617.                 'ext': 'mp4',
  618.                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
  619.                 'uploader': 'Philipp Hagemeister',
  620.                 'uploader_id': '@PhilippHagemeister',
  621.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@PhilippHagemeister',
  622.                 'upload_date': '20121002',
  623.                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
  624.                 'categories': ['Science & Technology'],
  625.                 'tags': ['youtube-dl'],
  626.                 'duration': 10,
  627.                 'view_count': int,
  628.                 'like_count': int,
  629.             },
  630.             'params': {
  631.                 'skip_download': True,
  632.             },
  633.         },
  634.         {
  635.             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
  636.             'note': '256k DASH audio (format 141) via DASH manifest',
  637.             'info_dict': {
  638.                 'id': 'a9LDPn-MO4I',
  639.                 'ext': 'm4a',
  640.                 'upload_date': '20121002',
  641.                 'uploader_id': '8KVIDEO',
  642.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
  643.                 'description': '',
  644.                 'uploader': '8KVIDEO',
  645.                 'title': 'UHDTV TEST 8K VIDEO.mp4'
  646.             },
  647.             'params': {
  648.                 'youtube_include_dash_manifest': True,
  649.                 'format': '141',
  650.             },
  651.             'skip': 'format 141 not served any more',
  652.         },
  653.         # DASH manifest with encrypted signature
  654.         {
  655.             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
  656.             'info_dict': {
  657.                 'id': 'IB3lcPjvWLA',
  658.                 'ext': 'm4a',
  659.                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
  660.                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
  661.                 'duration': 244,
  662.                 'uploader': 'AfrojackVEVO',
  663.                 'uploader_id': '@AfrojackVEVO',
  664.                 'upload_date': '20131011',
  665.                 'abr': 129.495,
  666.             },
  667.             'params': {
  668.                 'youtube_include_dash_manifest': True,
  669.                 'format': '141/bestaudio[ext=m4a]',
  670.             },
  671.         },
  672.         # Controversy video
  673.         {
  674.             'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8',
  675.             'info_dict': {
  676.                 'id': 'T4XJQO3qol8',
  677.                 'ext': 'mp4',
  678.                 'duration': 219,
  679.                 'upload_date': '20100909',
  680.                 'uploader': 'Amazing Atheist',
  681.                 'uploader_id': '@theamazingatheist',
  682.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@theamazingatheist',
  683.                 'title': 'Burning Everyone\'s Koran',
  684.                 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms \r\n\r\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
  685.             }
  686.         },
  687.         # Age-gated videos
  688.         {
  689.             'note': 'Age-gated video (No vevo, embed allowed)',
  690.             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
  691.             'info_dict': {
  692.                 'id': 'HtVdAasjOgU',
  693.                 'ext': 'mp4',
  694.                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
  695.                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
  696.                 'duration': 142,
  697.                 'uploader': 'The Witcher',
  698.                 'uploader_id': '@thewitcher',
  699.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@thewitcher',
  700.                 'upload_date': '20140605',
  701.                 'thumbnail': 'https://i.ytimg.com/vi/HtVdAasjOgU/maxresdefault.jpg',
  702.                 'age_limit': 18,
  703.                 'categories': ['Gaming'],
  704.                 'tags': 'count:17',
  705.                 'channel': 'The Witcher',
  706.                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
  707.                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
  708.                 'view_count': int,
  709.                 'like_count': int,
  710.             },
  711.         },
  712.         {
  713.             'note': 'Age-gated video with embed allowed in public site',
  714.             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
  715.             'info_dict': {
  716.                 'id': 'HsUATh_Nc2U',
  717.                 'ext': 'mp4',
  718.                 'title': 'Godzilla 2 (Official Video)',
  719.                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
  720.                 'duration': 177,
  721.                 'uploader': 'FlyingKitty',
  722.                 'uploader_id': '@FlyingKitty900',
  723.                 'upload_date': '20200408',
  724.                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
  725.                 'age_limit': 18,
  726.                 'categories': ['Entertainment'],
  727.                 'tags': ['Flyingkitty', 'godzilla 2'],
  728.                 'channel': 'FlyingKitty',
  729.                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
  730.                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
  731.                 'view_count': int,
  732.                 'like_count': int,
  733.             },
  734.         },
  735.         {
  736.             'note': 'Age-gated video embeddable only with clientScreen=EMBED',
  737.             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
  738.             'info_dict': {
  739.                 'id': 'Tq92D6wQ1mg',
  740.                 'ext': 'mp4',
  741.                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
  742.                 'description': 'md5:17eccca93a786d51bc67646756894066',
  743.                 'duration': 106,
  744.                 'uploader': 'Projekt Melody',
  745.                 'uploader_id': '@ProjektMelody',
  746.                 'upload_date': '20191227',
  747.                 'age_limit': 18,
  748.                 'thumbnail': 'https://i.ytimg.com/vi/Tq92D6wQ1mg/sddefault.jpg',
  749.                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
  750.                 'categories': ['Entertainment'],
  751.                 'channel': 'Projekt Melody',
  752.                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
  753.                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
  754.                 'view_count': int,
  755.                 'like_count': int,
  756.             },
  757.         },
  758.         {
  759.             'note': 'Non-Age-gated non-embeddable video',
  760.             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
  761.             'info_dict': {
  762.                 'id': 'MeJVWBSsPAY',
  763.                 'ext': 'mp4',
  764.                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
  765.                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
  766.                 'duration': 210,
  767.                 'upload_date': '20130730',
  768.                 'uploader': 'Herr Lurik',
  769.                 'uploader_id': '@HerrLurik',
  770.                 'uploader_url': 'http://www.youtube.com/@HerrLurik',
  771.                 'age_limit': 0,
  772.                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/hqdefault.jpg',
  773.                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
  774.                 'categories': ['Music'],
  775.                 'channel': 'Herr Lurik',
  776.                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
  777.                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
  778.                 'artist': 'OOMPH!',
  779.                 'view_count': int,
  780.                 'like_count': int,
  781.             },
  782.         },
  783.         {
  784.             'note': 'Non-bypassable age-gated video',
  785.             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
  786.             'only_matching': True,
  787.         },
  788.         {
  789.             'note': 'Age-gated video only available with authentication (not via embed workaround)',
  790.             'url': 'XgnwCQzjau8',
  791.             'only_matching': True,
  792.             'skip': '''This video has been removed for violating YouTube's Community Guidelines''',
  793.         },
  794.         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
  795.         # YouTube Red ad is not captured for creator
  796.         {
  797.             'url': '__2ABJjxzNo',
  798.             'info_dict': {
  799.                 'id': '__2ABJjxzNo',
  800.                 'ext': 'mp4',
  801.                 'duration': 266,
  802.                 'upload_date': '20100430',
  803.                 'uploader_id': '@deadmau5',
  804.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@deadmau5',
  805.                 'creator': 'deadmau5',
  806.                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
  807.                 'uploader': 'deadmau5',
  808.                 'title': 'Deadmau5 - Some Chords (HD)',
  809.                 'alt_title': 'Some Chords',
  810.             },
  811.             'expected_warnings': [
  812.                 'DASH manifest missing',
  813.             ]
  814.         },
  815.         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
  816.         {
  817.             'url': 'lqQg6PlCWgI',
  818.             'info_dict': {
  819.                 'id': 'lqQg6PlCWgI',
  820.                 'ext': 'mp4',
  821.                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
  822.                 'description': r're:(?s)(?:.+\s)?HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games\s*',
  823.                 'duration': 6085,
  824.                 'upload_date': '20150827',
  825.                 'uploader_id': '@Olympics',
  826.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@Olympics',
  827.                 'uploader': r're:Olympics?',
  828.                 'age_limit': 0,
  829.                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
  830.                 'categories': ['Sports'],
  831.                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
  832.                 'channel': 'Olympics',
  833.                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
  834.                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
  835.                 'view_count': int,
  836.                 'like_count': int,
  837.             },
  838.         },
  839.         # Non-square pixels
  840.         {
  841.             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
  842.             'info_dict': {
  843.                 'id': '_b-2C3KPAM0',
  844.                 'ext': 'mp4',
  845.                 'stretched_ratio': 16 / 9.,
  846.                 'duration': 85,
  847.                 'upload_date': '20110310',
  848.                 'uploader_id': '@AllenMeow',
  849.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@AllenMeow',
  850.                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
  851.                 'uploader': '孫ᄋᄅ',
  852.                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
  853.             },
  854.         },
  855.         # url_encoded_fmt_stream_map is empty string
  856.         {
  857.             'url': 'qEJwOuvDf7I',
  858.             'info_dict': {
  859.                 'id': 'qEJwOuvDf7I',
  860.                 'ext': 'webm',
  861.                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
  862.                 'description': '',
  863.                 'upload_date': '20150404',
  864.                 'uploader_id': 'spbelect',
  865.                 'uploader': 'Наблюдатели Петербурга',
  866.             },
  867.             'params': {
  868.                 'skip_download': 'requires avconv',
  869.             },
  870.             'skip': 'This live event has ended.',
  871.         },
  872.         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
  873.         {
  874.             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
  875.             'info_dict': {
  876.                 'id': 'FIl7x6_3R5Y',
  877.                 'ext': 'webm',
  878.                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
  879.                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
  880.                 'duration': 220,
  881.                 'upload_date': '20150625',
  882.                 'uploader_id': 'dorappi2000',
  883.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
  884.                 'uploader': 'dorappi2000',
  885.                 'formats': 'mincount:31',
  886.             },
  887.             'skip': 'not actual any more',
  888.         },
  889.         # DASH manifest with segment_list
  890.         {
  891.             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
  892.             'md5': '8ce563a1d667b599d21064e982ab9e31',
  893.             'info_dict': {
  894.                 'id': 'CsmdDsKjzN8',
  895.                 'ext': 'mp4',
  896.                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
  897.                 'uploader': 'Airtek',
  898.                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
  899.                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
  900.                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
  901.             },
  902.             'params': {
  903.                 'youtube_include_dash_manifest': True,
  904.                 'format': '135',  # bestvideo
  905.             },
  906.             'skip': 'This live event has ended.',
  907.         },
  908.         {
  909.             # Multifeed videos (multiple cameras), URL is for Main Camera
  910.             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
  911.             'info_dict': {
  912.                 'id': 'jvGDaLqkpTg',
  913.                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
  914.                 'description': 'md5:e03b909557865076822aa169218d6a5d',
  915.             },
  916.             'playlist': [{
  917.                 'info_dict': {
  918.                     'id': 'jvGDaLqkpTg',
  919.                     'ext': 'mp4',
  920.                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
  921.                     'description': 'md5:e03b909557865076822aa169218d6a5d',
  922.                     'duration': 10643,
  923.                     'upload_date': '20161111',
  924.                     'uploader': 'Team PGP',
  925.                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
  926.                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
  927.                 },
  928.             }, {
  929.                 'info_dict': {
  930.                     'id': '3AKt1R1aDnw',
  931.                     'ext': 'mp4',
  932.                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
  933.                     'description': 'md5:e03b909557865076822aa169218d6a5d',
  934.                     'duration': 10991,
  935.                     'upload_date': '20161111',
  936.                     'uploader': 'Team PGP',
  937.                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
  938.                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
  939.                 },
  940.             }, {
  941.                 'info_dict': {
  942.                     'id': 'RtAMM00gpVc',
  943.                     'ext': 'mp4',
  944.                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
  945.                     'description': 'md5:e03b909557865076822aa169218d6a5d',
  946.                     'duration': 10995,
  947.                     'upload_date': '20161111',
  948.                     'uploader': 'Team PGP',
  949.                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
  950.                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
  951.                 },
  952.             }, {
  953.                 'info_dict': {
  954.                     'id': '6N2fdlP3C5U',
  955.                     'ext': 'mp4',
  956.                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
  957.                     'description': 'md5:e03b909557865076822aa169218d6a5d',
  958.                     'duration': 10990,
  959.                     'upload_date': '20161111',
  960.                     'uploader': 'Team PGP',
  961.                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
  962.                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
  963.                 },
  964.             }],
  965.             'params': {
  966.                 'skip_download': True,
  967.             },
  968.             'skip': 'Not multifeed any more',
  969.         },
  970.         {
  971.             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
  972.             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
  973.             'info_dict': {
  974.                 'id': 'gVfLd0zydlo',
  975.                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
  976.             },
  977.             'playlist_count': 2,
  978.             'skip': 'Not multifeed any more',
  979.         },
  980.         {
  981.             'url': 'https://vid.plus/FlRa-iH7PGw',
  982.             'only_matching': True,
  983.         },
  984.         {
  985.             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
  986.             'only_matching': True,
  987.         },
  988.         {
  989.             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
  990.             # Also tests cut-off URL expansion in video description (see
  991.             # https://github.com/ytdl-org/youtube-dl/issues/1892,
  992.             # https://github.com/ytdl-org/youtube-dl/issues/8164)
  993.             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
  994.             'info_dict': {
  995.                 'id': 'lsguqyKfVQg',
  996.                 'ext': 'mp4',
  997.                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
  998.                 'alt_title': 'Dark Walk',
  999.                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
  1000.                 'duration': 133,
  1001.                 'upload_date': '20151119',
  1002.                 'uploader_id': '@IronSoulElf',
  1003.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@IronSoulElf',
  1004.                 'uploader': 'IronSoulElf',
  1005.                 'creator': r're:Todd Haberman[;,]\s+Daniel Law Heath and Aaron Kaplan',
  1006.                 'track': 'Dark Walk',
  1007.                 'artist': r're:Todd Haberman[;,]\s+Daniel Law Heath and Aaron Kaplan',
  1008.                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
  1009.             },
  1010.             'params': {
  1011.                 'skip_download': True,
  1012.             },
  1013.         },
  1014.         {
  1015.             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
  1016.             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
  1017.             'only_matching': True,
  1018.         },
  1019.         {
  1020.             # Video with yt:stretch=17:0
  1021.             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
  1022.             'info_dict': {
  1023.                 'id': 'Q39EVAstoRM',
  1024.                 'ext': 'mp4',
  1025.                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
  1026.                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
  1027.                 'upload_date': '20151107',
  1028.                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
  1029.                 'uploader': 'CH GAMER DROID',
  1030.             },
  1031.             'params': {
  1032.                 'skip_download': True,
  1033.             },
  1034.             'skip': 'This video does not exist.',
  1035.         },
  1036.         {
  1037.             # Video with incomplete 'yt:stretch=16:'
  1038.             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
  1039.             'only_matching': True,
  1040.         },
  1041.         {
  1042.             # Video licensed under Creative Commons
  1043.             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
  1044.             'info_dict': {
  1045.                 'id': 'M4gD1WSo5mA',
  1046.                 'ext': 'mp4',
  1047.                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
  1048.                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
  1049.                 'duration': 721,
  1050.                 'upload_date': '20150127',
  1051.                 'uploader_id': '@BKCHarvard',
  1052.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@BKCHarvard',
  1053.                 'uploader': 'The Berkman Klein Center for Internet & Society',
  1054.                 'license': 'Creative Commons Attribution license (reuse allowed)',
  1055.             },
  1056.             'params': {
  1057.                 'skip_download': True,
  1058.             },
  1059.         },
  1060.         {
  1061.             # Channel-like uploader_url
  1062.             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
  1063.             'info_dict': {
  1064.                 'id': 'eQcmzGIKrzg',
  1065.                 'ext': 'mp4',
  1066.                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
  1067.                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
  1068.                 'duration': 4060,
  1069.                 'upload_date': '20151119',
  1070.                 'uploader': 'Bernie Sanders',
  1071.                 'uploader_id': '@BernieSanders',
  1072.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@BernieSanders',
  1073.                 'license': 'Creative Commons Attribution license (reuse allowed)',
  1074.             },
  1075.             'params': {
  1076.                 'skip_download': True,
  1077.             },
  1078.         },
  1079.         {
  1080.             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
  1081.             'only_matching': True,
  1082.         },
  1083.         {
  1084.             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
  1085.             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
  1086.             'only_matching': True,
  1087.         },
  1088.         {
  1089.             # Rental video preview
  1090.             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
  1091.             'info_dict': {
  1092.                 'id': 'uGpuVWrhIzE',
  1093.                 'ext': 'mp4',
  1094.                 'title': 'Piku - Trailer',
  1095.                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
  1096.                 'upload_date': '20150811',
  1097.                 'uploader': 'FlixMatrix',
  1098.                 'uploader_id': 'FlixMatrixKaravan',
  1099.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
  1100.                 'license': 'Standard YouTube License',
  1101.             },
  1102.             'params': {
  1103.                 'skip_download': True,
  1104.             },
  1105.             'skip': 'This video is not available.',
  1106.         },
  1107.         {
  1108.             # YouTube Red video with episode data
  1109.             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
  1110.             'info_dict': {
  1111.                 'id': 'iqKdEhx-dD4',
  1112.                 'ext': 'mp4',
  1113.                 'title': 'Isolation - Mind Field (Ep 1)',
  1114.                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
  1115.                 'duration': 2085,
  1116.                 'upload_date': '20170118',
  1117.                 'uploader': 'Vsauce',
  1118.                 'uploader_id': '@Vsauce',
  1119.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@Vsauce',
  1120.                 'series': 'Mind Field',
  1121.                 'season_number': 1,
  1122.                 'episode_number': 1,
  1123.             },
  1124.             'params': {
  1125.                 'skip_download': True,
  1126.             },
  1127.             'expected_warnings': [
  1128.                 'Skipping DASH manifest',
  1129.             ],
  1130.         },
  1131.         {
  1132.             # The following content has been identified by the YouTube community
  1133.             # as inappropriate or offensive to some audiences.
  1134.             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
  1135.             'info_dict': {
  1136.                 'id': '6SJNVb0GnPI',
  1137.                 'ext': 'mp4',
  1138.                 'title': 'Race Differences in Intelligence',
  1139.                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
  1140.                 'duration': 965,
  1141.                 'upload_date': '20140124',
  1142.                 'uploader': 'New Century Foundation',
  1143.                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
  1144.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
  1145.             },
  1146.             'params': {
  1147.                 'skip_download': True,
  1148.             },
  1149.             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
  1150.         },
  1151.         {
  1152.             # itag 212
  1153.             'url': '1t24XAntNCY',
  1154.             'only_matching': True,
  1155.         },
  1156.         {
  1157.             # geo restricted to JP
  1158.             'url': 'sJL6WA-aGkQ',
  1159.             'only_matching': True,
  1160.         },
  1161.         {
  1162.             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
  1163.             'only_matching': True,
  1164.         },
  1165.         {
  1166.             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
  1167.             'only_matching': True,
  1168.         },
  1169.         {
  1170.             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
  1171.             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
  1172.             'only_matching': True,
  1173.         },
  1174.         {
  1175.             # DRM protected
  1176.             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
  1177.             'only_matching': True,
  1178.         },
  1179.         {
  1180.             # Video with unsupported adaptive stream type formats
  1181.             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
  1182.             'info_dict': {
  1183.                 'id': 'Z4Vy8R84T1U',
  1184.                 'ext': 'mp4',
  1185.                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
  1186.                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
  1187.                 'duration': 433,
  1188.                 'upload_date': '20130923',
  1189.                 'uploader': 'Amelia Putri Harwita',
  1190.                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
  1191.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
  1192.                 'formats': 'maxcount:10',
  1193.             },
  1194.             'params': {
  1195.                 'skip_download': True,
  1196.                 'youtube_include_dash_manifest': False,
  1197.             },
  1198.             'skip': 'not actual any more',
  1199.         },
  1200.         {
  1201.             # Youtube Music Auto-generated description
  1202.             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
  1203.             'info_dict': {
  1204.                 'id': 'MgNrAu2pzNs',
  1205.                 'ext': 'mp4',
  1206.                 'title': 'Voyeur Girl',
  1207.                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
  1208.                 'upload_date': '20190312',
  1209.                 'uploader': 'Stephen - Topic',
  1210.                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
  1211.                 'artist': 'Stephen',
  1212.                 'track': 'Voyeur Girl',
  1213.                 'album': 'it\'s too much love to know my dear',
  1214.                 'release_date': '20190313',
  1215.                 'release_year': 2019,
  1216.             },
  1217.             'params': {
  1218.                 'skip_download': True,
  1219.             },
  1220.         },
  1221.         {
  1222.             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
  1223.             'only_matching': True,
  1224.         },
  1225.         {
  1226.             # invalid -> valid video id redirection
  1227.             'url': 'DJztXj2GPfl',
  1228.             'info_dict': {
  1229.                 'id': 'DJztXj2GPfk',
  1230.                 'ext': 'mp4',
  1231.                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
  1232.                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
  1233.                 'upload_date': '20090125',
  1234.                 'uploader': 'Prochorowka',
  1235.                 'uploader_id': 'Prochorowka',
  1236.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
  1237.                 'artist': 'Panjabi MC',
  1238.                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
  1239.                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
  1240.             },
  1241.             'params': {
  1242.                 'skip_download': True,
  1243.             },
  1244.             'skip': 'Video unavailable',
  1245.         },
  1246.         {
  1247.             # empty description results in an empty string
  1248.             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
  1249.             'info_dict': {
  1250.                 'id': 'x41yOUIvK2k',
  1251.                 'ext': 'mp4',
  1252.                 'title': 'IMG 3456',
  1253.                 'description': '',
  1254.                 'upload_date': '20170613',
  1255.                 'uploader': "l'Or Vert asbl",
  1256.                 'uploader_id': '@ElevageOrVert',
  1257.             },
  1258.             'params': {
  1259.                 'skip_download': True,
  1260.             },
  1261.         },
  1262.         {
  1263.             # with '};' inside yt initial data (see [1])
  1264.             # see [2] for an example with '};' inside ytInitialPlayerResponse
  1265.             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
  1266.             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
  1267.             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
  1268.             'info_dict': {
  1269.                 'id': 'CHqg6qOn4no',
  1270.                 'ext': 'mp4',
  1271.                 'title': 'Part 77   Sort a list of simple types in c#',
  1272.                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
  1273.                 'upload_date': '20130831',
  1274.                 'uploader': 'kudvenkat',
  1275.                 'uploader_id': '@Csharp-video-tutorialsBlogspot',
  1276.             },
  1277.             'params': {
  1278.                 'skip_download': True,
  1279.             },
  1280.         },
  1281.         {
  1282.             # another example of '};' in ytInitialData
  1283.             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
  1284.             'only_matching': True,
  1285.         },
  1286.         {
  1287.             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
  1288.             'only_matching': True,
  1289.         },
  1290.         {
  1291.             # https://github.com/ytdl-org/youtube-dl/pull/28094
  1292.             'url': 'OtqTfy26tG0',
  1293.             'info_dict': {
  1294.                 'id': 'OtqTfy26tG0',
  1295.                 'ext': 'mp4',
  1296.                 'title': 'Burn Out',
  1297.                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
  1298.                 'upload_date': '20141120',
  1299.                 'uploader': 'The Cinematic Orchestra - Topic',
  1300.                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
  1301.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
  1302.                 'artist': 'The Cinematic Orchestra',
  1303.                 'track': 'Burn Out',
  1304.                 'album': 'Every Day',
  1305.                 'release_data': None,
  1306.                 'release_year': None,
  1307.             },
  1308.             'params': {
  1309.                 'skip_download': True,
  1310.             },
  1311.         },
  1312.         {
  1313.             # controversial video, only works with bpctr when authenticated with cookies
  1314.             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
  1315.             'only_matching': True,
  1316.         },
  1317.         {
  1318.             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
  1319.             'url': 'cBvYw8_A0vQ',
  1320.             'info_dict': {
  1321.                 'id': 'cBvYw8_A0vQ',
  1322.                 'ext': 'mp4',
  1323.                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
  1324.                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
  1325.                 'upload_date': '20201120',
  1326.                 'uploader': 'Walk around Japan',
  1327.                 'uploader_id': '@walkaroundjapan7124',
  1328.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@walkaroundjapan7124',
  1329.             },
  1330.             'params': {
  1331.                 'skip_download': True,
  1332.             },
  1333.         },
  1334.         {
  1335.             # YT 'Shorts'
  1336.             'url': 'https://youtube.com/shorts/4L2J27mJ3Dc',
  1337.             'info_dict': {
  1338.                 'id': '4L2J27mJ3Dc',
  1339.                 'ext': 'mp4',
  1340.                 'title': 'Midwest Squid Game #Shorts',
  1341.                 'description': 'md5:976512b8a29269b93bbd8a61edc45a6d',
  1342.                 'upload_date': '20211025',
  1343.                 'uploader': 'Charlie Berens',
  1344.                 'uploader_id': '@CharlieBerens',
  1345.             },
  1346.             'params': {
  1347.                 'skip_download': True,
  1348.             },
  1349.         },
  1350.     ]
  1351.     _formats = {
  1352.         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
  1353.         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
  1354.         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
  1355.         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
  1356.         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
  1357.         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
  1358.         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
  1359.         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
  1360.         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
  1361.         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
  1362.         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
  1363.         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
  1364.         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
  1365.         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
  1366.         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
  1367.         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
  1368.         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
  1369.         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
  1372.         # 3D videos
  1373.         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
  1374.         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
  1375.         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
  1376.         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
  1377.         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
  1378.         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
  1379.         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
  1381.         # Apple HTTP Live Streaming
  1382.         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
  1383.         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
  1384.         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
  1385.         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
  1386.         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
  1387.         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
  1388.         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
  1389.         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
  1391.         # DASH mp4 video
  1392.         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
  1393.         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
  1394.         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
  1395.         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
  1396.         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
  1397.         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
  1398.         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
  1399.         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
  1400.         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
  1401.         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
  1402.         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
  1403.         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
  1405.         # Dash mp4 audio
  1406.         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
  1407.         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
  1408.         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
  1409.         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
  1410.         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
  1411.         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
  1412.         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
  1414.         # Dash webm
  1415.         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
  1416.         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
  1417.         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
  1418.         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
  1419.         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
  1420.         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
  1421.         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
  1422.         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1423.         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1424.         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1425.         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1426.         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1427.         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1428.         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1429.         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1430.         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
  1431.         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1432.         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
  1433.         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
  1434.         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
  1435.         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1436.         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
  1438.         # Dash webm audio
  1439.         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
  1440.         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
  1442.         # Dash webm audio with opus inside
  1443.         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
  1444.         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
  1445.         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
  1447.         # RTMP (unnamed)
  1448.         '_rtmp': {'protocol': 'rtmp'},
  1450.         # av01 video only formats sometimes served with "unknown" codecs
  1451.         '394': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
  1452.         '395': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
  1453.         '396': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
  1454.         '397': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
  1455.     }
  1457.     @classmethod
  1458.     def suitable(cls, url):
  1459.         if parse_qs(url).get('list', [None])[0]:
  1460.             return False
  1461.         return super(YoutubeIE, cls).suitable(url)
  1463.     def __init__(self, *args, **kwargs):
  1464.         super(YoutubeIE, self).__init__(*args, **kwargs)
  1465.         self._code_cache = {}
  1466.         self._player_cache = {}
  1468.     # *ytcfgs, webpage=None
  1469.     def _extract_player_url(self, *ytcfgs, **kw_webpage):
  1470.         if ytcfgs and not isinstance(ytcfgs[0], dict):
  1471.             webpage = kw_webpage.get('webpage') or ytcfgs[0]
  1472.         if webpage:
  1473.             player_url = self._search_regex(
  1474.                 r'"(?:PLAYER_JS_URL|jsUrl)"\s*:\s*"([^"]+)"',
  1475.                 webpage or '', 'player URL', fatal=False)
  1476.             if player_url:
  1477.                 ytcfgs = ytcfgs + ({'PLAYER_JS_URL': player_url},)
  1478.         return traverse_obj(
  1479.             ytcfgs, (Ellipsis, 'PLAYER_JS_URL'), (Ellipsis, 'WEB_PLAYER_CONTEXT_CONFIGS', Ellipsis, 'jsUrl'),
  1480.             get_all=False, expected_type=lambda u: urljoin('https://www.youtube.com', u))
  1482.     def _download_player_url(self, video_id, fatal=False):
  1483.         res = self._download_webpage(
  1484.             'https://www.youtube.com/iframe_api',
  1485.             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
  1486.         player_version = self._search_regex(
  1487.             r'player\\?/([0-9a-fA-F]{8})\\?/', res or '', 'player version', fatal=fatal,
  1488.             default=NO_DEFAULT if res else None)
  1489.         if player_version:
  1490.             return 'https://www.youtube.com/s/player/{0}/player_ias.vflset/en_US/base.js'.format(player_version)
  1492.     def _signature_cache_id(self, example_sig):
  1493.         """ Return a string representation of a signature """
  1494.         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
  1496.     @classmethod
  1497.     def _extract_player_info(cls, player_url):
  1498.         for player_re in cls._PLAYER_INFO_RE:
  1499.             id_m = re.search(player_re, player_url)
  1500.             if id_m:
  1501.                 break
  1502.         else:
  1503.             raise ExtractorError('Cannot identify player %r' % player_url)
  1504.         return id_m.group('id')
  1506.     def _load_player(self, video_id, player_url, fatal=True, player_id=None):
  1507.         if not player_id:
  1508.             player_id = self._extract_player_info(player_url)
  1509.         if player_id not in self._code_cache:
  1510.             code = self._download_webpage(
  1511.                 player_url, video_id, fatal=fatal,
  1512.                 note='Downloading player ' + player_id,
  1513.                 errnote='Download of %s failed' % player_url)
  1514.             if code:
  1515.                 self._code_cache[player_id] = code
  1516.         return self._code_cache[player_id] if fatal else self._code_cache.get(player_id)
  1518.     def _extract_signature_function(self, video_id, player_url, example_sig):
  1519.         player_id = self._extract_player_info(player_url)
  1521.         # Read from filesystem cache
  1522.         func_id = 'js_{0}_{1}'.format(
  1523.             player_id, self._signature_cache_id(example_sig))
  1524.         assert os.path.basename(func_id) == func_id
  1526.         self.write_debug('Extracting signature function {0}'.format(func_id))
  1527.         cache_spec, code = self.cache.load('youtube-sigfuncs', func_id), None
  1529.         if not cache_spec:
  1530.             code = self._load_player(video_id, player_url, player_id)
  1531.         if code:
  1532.             res = self._parse_sig_js(code)
  1533.             test_string = ''.join(map(compat_chr, range(len(example_sig))))
  1534.             cache_spec = [ord(c) for c in res(test_string)]
  1535.             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
  1537.         return lambda s: ''.join(s[i] for i in cache_spec)
  1539.     def _print_sig_code(self, func, example_sig):
  1540.         if not self.get_param('youtube_print_sig_code'):
  1541.             return
  1543.         def gen_sig_code(idxs):
  1544.             def _genslice(start, end, step):
  1545.                 starts = '' if start == 0 else str(start)
  1546.                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
  1547.                 steps = '' if step == 1 else (':%d' % step)
  1548.                 return 's[{0}{1}{2}]'.format(starts, ends, steps)
  1550.             step = None
  1551.             # Quelch pyflakes warnings - start will be set when step is set
  1552.             start = '(Never used)'
  1553.             for i, prev in zip(idxs[1:], idxs[:-1]):
  1554.                 if step is not None:
  1555.                     if i - prev == step:
  1556.                         continue
  1557.                     yield _genslice(start, prev, step)
  1558.                     step = None
  1559.                     continue
  1560.                 if i - prev in [-1, 1]:
  1561.                     step = i - prev
  1562.                     start = prev
  1563.                     continue
  1564.                 else:
  1565.                     yield 's[%d]' % prev
  1566.             if step is None:
  1567.                 yield 's[%d]' % i
  1568.             else:
  1569.                 yield _genslice(start, i, step)
  1571.         test_string = ''.join(map(compat_chr, range(len(example_sig))))
  1572.         cache_res = func(test_string)
  1573.         cache_spec = [ord(c) for c in cache_res]
  1574.         expr_code = ' + '.join(gen_sig_code(cache_spec))
  1575.         signature_id_tuple = '(%s)' % (
  1576.             ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
  1577.         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
  1578.                 '    return %s\n') % (signature_id_tuple, expr_code)
  1579.         self.to_screen('Extracted signature function:\n' + code)
  1581.     def _parse_sig_js(self, jscode):
  1582.         funcname = self._search_regex(
  1583.             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
  1584.              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
  1585.              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
  1586.              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
  1587.              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\))?',
  1588.              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
  1589.              # Obsolete patterns
  1590.              r'("|\')signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
  1591.              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
  1592.              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
  1593.              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
  1594.              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
  1595.              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
  1596.             jscode, 'Initial JS player signature function name', group='sig')
  1598.         jsi = JSInterpreter(jscode)
  1599.         initial_function = jsi.extract_function(funcname)
  1600.         return lambda s: initial_function([s])
  1602.     def _cached(self, func, *cache_id):
  1603.         def inner(*args, **kwargs):
  1604.             if cache_id not in self._player_cache:
  1605.                 try:
  1606.                     self._player_cache[cache_id] = func(*args, **kwargs)
  1607.                 except ExtractorError as e:
  1608.                     self._player_cache[cache_id] = e
  1609.                 except Exception as e:
  1610.                     self._player_cache[cache_id] = ExtractorError(traceback.format_exc(), cause=e)
  1612.             ret = self._player_cache[cache_id]
  1613.             if isinstance(ret, Exception):
  1614.                 raise ret
  1615.             return ret
  1616.         return inner
  1618.     def _decrypt_signature(self, s, video_id, player_url):
  1619.         """Turn the encrypted s field into a working signature"""
  1620.         extract_sig = self._cached(
  1621.             self._extract_signature_function, 'sig', player_url, self._signature_cache_id(s))
  1622.         func = extract_sig(video_id, player_url, s)
  1623.         self._print_sig_code(func, s)
  1624.         return func(s)
  1626.     # from yt-dlp
  1627.     # See also:
  1628.     # 1. https://github.com/ytdl-org/youtube-dl/issues/29326#issuecomment-894619419
  1629.     # 2. https://code.videolan.org/videolan/vlc/-/blob/4fb284e5af69aa9ac2100ccbdd3b88debec9987f/share/lua/playlist/youtube.lua#L116
  1630.     # 3. https://github.com/ytdl-org/youtube-dl/issues/30097#issuecomment-950157377
  1631.     def _decrypt_nsig(self, n, video_id, player_url):
  1632.         """Turn the encrypted n field into a working signature"""
  1633.         if player_url is None:
  1634.             raise ExtractorError('Cannot decrypt nsig without player_url')
  1636.         try:
  1637.             jsi, player_id, func_code = self._extract_n_function_code(video_id, player_url)
  1638.         except ExtractorError as e:
  1639.             raise ExtractorError('Unable to extract nsig jsi, player_id, func_codefunction code', cause=e)
  1640.         if self.get_param('youtube_print_sig_code'):
  1641.             self.to_screen('Extracted nsig function from {0}:\n{1}\n'.format(
  1642.                 player_id, func_code[1]))
  1644.         try:
  1645.             extract_nsig = self._cached(self._extract_n_function_from_code, 'nsig func', player_url)
  1646.             ret = extract_nsig(jsi, func_code)(n)
  1647.         except JSInterpreter.Exception as e:
  1648.             self.report_warning(
  1649.                 '%s (%s %s)' % (
  1650.                     self.__ie_msg(
  1651.                         'Unable to decode n-parameter: download likely to be throttled'),
  1652.                     error_to_compat_str(e),
  1653.                     traceback.format_exc()))
  1654.             return
  1656.         self.write_debug('Decrypted nsig {0} => {1}'.format(n, ret))
  1657.         return ret
  1659.     def _extract_n_function_name(self, jscode):
  1660.         func_name, idx = self._search_regex(
  1661.             r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z_$][\w$]*)(?:\[(?P<idx>\d+)\])?\([\w$]+\)',
  1662.             jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
  1663.         if not idx:
  1664.             return func_name
  1666.         return self._parse_json(self._search_regex(
  1667.             r'var {0}\s*=\s*(\[.+?\])\s*[,;]'.format(re.escape(func_name)), jscode,
  1668.             'Initial JS player n function list ({0}.{1})'.format(func_name, idx)),
  1669.             func_name, transform_source=js_to_json)[int(idx)]
  1671.     def _extract_n_function_code(self, video_id, player_url):
  1672.         player_id = self._extract_player_info(player_url)
  1673.         func_code = self.cache.load('youtube-nsig', player_id)
  1674.         jscode = func_code or self._load_player(video_id, player_url)
  1675.         jsi = JSInterpreter(jscode)
  1677.         if func_code:
  1678.             return jsi, player_id, func_code
  1680.         func_name = self._extract_n_function_name(jscode)
  1682.         # For redundancy
  1683.         func_code = self._search_regex(
  1684.             r'''(?xs)%s\s*=\s*function\s*\((?P<var>[\w$]+)\)\s*
  1685.                      # NB: The end of the regex is intentionally kept strict
  1686.                      {(?P<code>.+?}\s*return\ [\w$]+.join\(""\))};''' % func_name,
  1687.             jscode, 'nsig function', group=('var', 'code'), default=None)
  1688.         if func_code:
  1689.             func_code = ([func_code[0]], func_code[1])
  1690.         else:
  1691.             self.write_debug('Extracting nsig function with jsinterp')
  1692.             func_code = jsi.extract_function_code(func_name)
  1694.         self.cache.store('youtube-nsig', player_id, func_code)
  1695.         return jsi, player_id, func_code
  1697.     def _extract_n_function_from_code(self, jsi, func_code):
  1698.         func = jsi.extract_function_from_code(*func_code)
  1700.         def extract_nsig(s):
  1701.             try:
  1702.                 ret = func([s])
  1703.             except JSInterpreter.Exception:
  1704.                 raise
  1705.             except Exception as e:
  1706.                 raise JSInterpreter.Exception(traceback.format_exc(), cause=e)
  1708.             if ret.startswith('enhanced_except_'):
  1709.                 raise JSInterpreter.Exception('Signature function returned an exception')
  1710.             return ret
  1712.         return extract_nsig
  1714.     def _unthrottle_format_urls(self, video_id, player_url, *formats):
  1716.         def decrypt_nsig(n):
  1717.             return self._cached(self._decrypt_nsig, 'nsig', n, player_url)
  1719.         for fmt in formats:
  1720.             parsed_fmt_url = compat_urllib_parse.urlparse(fmt['url'])
  1721.             n_param = compat_parse_qs(parsed_fmt_url.query).get('n')
  1722.             if not n_param:
  1723.                 continue
  1724.             n_param = n_param[-1]
  1725.             n_response = decrypt_nsig(n_param)(n_param, video_id, player_url)
  1726.             if n_response is None:
  1727.                 # give up if descrambling failed
  1728.                 break
  1729.             fmt['url'] = update_url_query(fmt['url'], {'n': n_response})
  1731.     # from yt-dlp, with tweaks
  1732.     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
  1733.         """
  1734.         Extract signatureTimestamp (sts)
  1735.         Required to tell API what sig/player version is in use.
  1736.         """
  1737.         sts = traverse_obj(ytcfg, 'STS', expected_type=int)
  1738.         if not sts:
  1739.             # Attempt to extract from player
  1740.             if player_url is None:
  1741.                 error_msg = 'Cannot extract signature timestamp without player_url.'
  1742.                 if fatal:
  1743.                     raise ExtractorError(error_msg)
  1744.                 self.report_warning(error_msg)
  1745.                 return
  1746.             code = self._load_player(video_id, player_url, fatal=fatal)
  1747.             sts = int_or_none(self._search_regex(
  1748.                 r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code or '',
  1749.                 'JS player signature timestamp', group='sts', fatal=fatal))
  1750.         return sts
  1752.     def _mark_watched(self, video_id, player_response):
  1753.         playback_url = url_or_none(try_get(
  1754.             player_response,
  1755.             lambda x: x['playbackTracking']['videostatsPlaybackUrl']['baseUrl']))
  1756.         if not playback_url:
  1757.             return
  1759.         # cpn generation algorithm is reverse engineered from base.js.
  1760.         # In fact it works even with dummy cpn.
  1761.         CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
  1762.         cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
  1764.         # more consistent results setting it to right before the end
  1765.         qs = parse_qs(playback_url)
  1766.         video_length = '{0}'.format(float((qs.get('len') or ['1.5'])[0]) - 1)
  1768.         playback_url = update_url_query(
  1769.             playback_url, {
  1770.                 'ver': '2',
  1771.                 'cpn': cpn,
  1772.                 'cmt': video_length,
  1773.                 'el': 'detailpage',  # otherwise defaults to "shorts"
  1774.             })
  1776.         self._download_webpage(
  1777.             playback_url, video_id, 'Marking watched',
  1778.             'Unable to mark watched', fatal=False)
  1780.     @staticmethod
  1781.     def _extract_urls(webpage):
  1782.         # Embedded YouTube player
  1783.         entries = [
  1784.             unescapeHTML(mobj.group('url'))
  1785.             for mobj in re.finditer(r'''(?x)
  1786.             (?:
  1787.                 <iframe[^>]+?src=|
  1788.                 data-video-url=|
  1789.                 <embed[^>]+?src=|
  1790.                 embedSWF\(?:\s*|
  1791.                 <object[^>]+data=|
  1792.                 new\s+SWFObject\(
  1793.             )
  1794.             (["\'])
  1795.                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
  1796.                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
  1797.             \1''', webpage)]
  1799.         # lazyYT YouTube embed
  1800.         entries.extend(list(map(
  1801.             unescapeHTML,
  1802.             re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
  1804.         # Wordpress "YouTube Video Importer" plugin
  1805.         matches = re.findall(r'''(?x)<div[^>]+
  1806.             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
  1807.             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
  1808.         entries.extend(m[-1] for m in matches)
  1810.         return entries
  1812.     @staticmethod
  1813.     def _extract_url(webpage):
  1814.         urls = YoutubeIE._extract_urls(webpage)
  1815.         return urls[0] if urls else None
  1817.     @classmethod
  1818.     def extract_id(cls, url):
  1819.         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
  1820.         if mobj is None:
  1821.             raise ExtractorError('Invalid URL: %s' % url)
  1822.         video_id = mobj.group(2)
  1823.         return video_id
  1825.     def _extract_chapters_from_json(self, data, video_id, duration):
  1826.         chapters_list = try_get(
  1827.             data,
  1828.             lambda x: x['playerOverlays']
  1829.                        ['playerOverlayRenderer']
  1830.                        ['decoratedPlayerBarRenderer']
  1831.                        ['decoratedPlayerBarRenderer']
  1832.                        ['playerBar']
  1833.                        ['chapteredPlayerBarRenderer']
  1834.                        ['chapters'],
  1835.             list)
  1836.         if not chapters_list:
  1837.             return
  1839.         def chapter_time(chapter):
  1840.             return float_or_none(
  1841.                 try_get(
  1842.                     chapter,
  1843.                     lambda x: x['chapterRenderer']['timeRangeStartMillis'],
  1844.                     int),
  1845.                 scale=1000)
  1846.         chapters = []
  1847.         for next_num, chapter in enumerate(chapters_list, start=1):
  1848.             start_time = chapter_time(chapter)
  1849.             if start_time is None:
  1850.                 continue
  1851.             end_time = (chapter_time(chapters_list[next_num])
  1852.                         if next_num < len(chapters_list) else duration)
  1853.             if end_time is None:
  1854.                 continue
  1855.             title = try_get(
  1856.                 chapter, lambda x: x['chapterRenderer']['title']['simpleText'],
  1857.                 compat_str)
  1858.             chapters.append({
  1859.                 'start_time': start_time,
  1860.                 'end_time': end_time,
  1861.                 'title': title,
  1862.             })
  1863.         return chapters
  1865.     def _extract_yt_initial_variable(self, webpage, regex, video_id, name):
  1866.         return self._parse_json(self._search_regex(
  1867.             (r'%s\s*%s' % (regex, self._YT_INITIAL_BOUNDARY_RE),
  1868.              regex), webpage, name, default='{}'), video_id, fatal=False)
  1870.     def _real_extract(self, url):
  1871.         url, smuggled_data = unsmuggle_url(url, {})
  1872.         video_id = self._match_id(url)
  1873.         base_url = self.http_scheme() + '//www.youtube.com/'
  1874.         webpage_url = base_url + 'watch?v=' + video_id
  1875.         webpage = self._download_webpage(
  1876.             webpage_url + '&bpctr=9999999999&has_verified=1', video_id, fatal=False)
  1878.         player_response = None
  1879.         player_url = None
  1880.         if webpage:
  1881.             player_response = self._extract_yt_initial_variable(
  1882.                 webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE,
  1883.                 video_id, 'initial player response')
  1884.         if not player_response:
  1885.             player_response = self._call_api(
  1886.                 'player', {'videoId': video_id}, video_id)
  1888.         def is_agegated(playability):
  1889.             if not isinstance(playability, dict):
  1890.                 return
  1892.             if playability.get('desktopLegacyAgeGateReason'):
  1893.                 return True
  1895.             reasons = filter(None, (playability.get(r) for r in ('status', 'reason')))
  1896.             AGE_GATE_REASONS = (
  1897.                 'confirm your age', 'age-restricted', 'inappropriate',  # reason
  1898.                 'age_verification_required', 'age_check_required',  # status
  1899.             )
  1900.             return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
  1902.         def get_playability_status(response):
  1903.             return try_get(response, lambda x: x['playabilityStatus'], dict) or {}
  1905.         playability_status = get_playability_status(player_response)
  1906.         if (is_agegated(playability_status)
  1907.                 and int_or_none(self._downloader.params.get('age_limit'), default=18) >= 18):
  1909.             self.report_age_confirmation()
  1911.             # Thanks: https://github.com/yt-dlp/yt-dlp/pull/3233
  1912.             pb_context = {'html5Preference': 'HTML5_PREF_WANTS'}
  1914.             # Use signatureTimestamp if available
  1915.             # Thanks https://github.com/ytdl-org/youtube-dl/issues/31034#issuecomment-1160718026
  1916.             player_url = self._extract_player_url(webpage)
  1917.             ytcfg = self._extract_ytcfg(video_id, webpage)
  1918.             sts = self._extract_signature_timestamp(video_id, player_url, ytcfg)
  1919.             if sts:
  1920.                 pb_context['signatureTimestamp'] = sts
  1922.             query = {
  1923.                 'playbackContext': {'contentPlaybackContext': pb_context},
  1924.                 'contentCheckOk': True,
  1925.                 'racyCheckOk': True,
  1926.                 'context': {
  1927.                     'client': {'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER', 'clientVersion': '2.0', 'hl': 'en', 'clientScreen': 'EMBED'},
  1928.                     'thirdParty': {'embedUrl': 'https://google.com'},
  1929.                 },
  1930.                 'videoId': video_id,
  1931.             }
  1932.             headers = {
  1933.                 'X-YouTube-Client-Name': '85',
  1934.                 'X-YouTube-Client-Version': '2.0',
  1935.                 'Origin': 'https://www.youtube.com'
  1936.             }
  1938.             video_info = self._call_api('player', query, video_id, fatal=False, headers=headers)
  1939.             age_gate_status = get_playability_status(video_info)
  1940.             if age_gate_status.get('status') == 'OK':
  1941.                 player_response = video_info
  1942.                 playability_status = age_gate_status
  1944.         trailer_video_id = try_get(
  1945.             playability_status,
  1946.             lambda x: x['errorScreen']['playerLegacyDesktopYpcTrailerRenderer']['trailerVideoId'],
  1947.             compat_str)
  1948.         if trailer_video_id:
  1949.             return self.url_result(
  1950.                 trailer_video_id, self.ie_key(), trailer_video_id)
  1952.         def get_text(x):
  1953.             if not x:
  1954.                 return
  1955.             text = x.get('simpleText')
  1956.             if text and isinstance(text, compat_str):
  1957.                 return text
  1958.             runs = x.get('runs')
  1959.             if not isinstance(runs, list):
  1960.                 return
  1961.             return ''.join([r['text'] for r in runs if isinstance(r.get('text'), compat_str)])
  1963.         search_meta = (
  1964.             lambda x: self._html_search_meta(x, webpage, default=None)) \
  1965.             if webpage else lambda x: None
  1967.         video_details = player_response.get('videoDetails') or {}
  1968.         microformat = try_get(
  1969.             player_response,
  1970.             lambda x: x['microformat']['playerMicroformatRenderer'],
  1971.             dict) or {}
  1972.         video_title = video_details.get('title') \
  1973.             or get_text(microformat.get('title')) \
  1974.             or search_meta(['og:title', 'twitter:title', 'title'])
  1975.         video_description = video_details.get('shortDescription')
  1977.         if not smuggled_data.get('force_singlefeed', False):
  1978.             if not self._downloader.params.get('noplaylist'):
  1979.                 multifeed_metadata_list = try_get(
  1980.                     player_response,
  1981.                     lambda x: x['multicamera']['playerLegacyMulticameraRenderer']['metadataList'],
  1982.                     compat_str)
  1983.                 if multifeed_metadata_list:
  1984.                     entries = []
  1985.                     feed_ids = []
  1986.                     for feed in multifeed_metadata_list.split(','):
  1987.                         # Unquote should take place before split on comma (,) since textual
  1988.                         # fields may contain comma as well (see
  1989.                         # https://github.com/ytdl-org/youtube-dl/issues/8536)
  1990.                         feed_data = compat_parse_qs(
  1991.                             compat_urllib_parse_unquote_plus(feed))
  1993.                         def feed_entry(name):
  1994.                             return try_get(
  1995.                                 feed_data, lambda x: x[name][0], compat_str)
  1997.                         feed_id = feed_entry('id')
  1998.                         if not feed_id:
  1999.                             continue
  2000.                         feed_title = feed_entry('title')
  2001.                         title = video_title
  2002.                         if feed_title:
  2003.                             title += ' (%s)' % feed_title
  2004.                         entries.append({
  2005.                             '_type': 'url_transparent',
  2006.                             'ie_key': 'Youtube',
  2007.                             'url': smuggle_url(
  2008.                                 base_url + 'watch?v=' + feed_data['id'][0],
  2009.                                 {'force_singlefeed': True}),
  2010.                             'title': title,
  2011.                         })
  2012.                         feed_ids.append(feed_id)
  2013.                     self.to_screen(
  2014.                         'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
  2015.                         % (', '.join(feed_ids), video_id))
  2016.                     return self.playlist_result(
  2017.                         entries, video_id, video_title, video_description)
  2018.             else:
  2019.                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
  2021.         if not player_url:
  2022.             player_url = self._extract_player_url(webpage)
  2024.         formats = []
  2025.         itags = collections.defaultdict(set)
  2026.         itag_qualities = {}
  2027.         q = qualities(['tiny', 'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'])
  2028.         CHUNK_SIZE = 10 << 20
  2030.         streaming_data = player_response.get('streamingData') or {}
  2031.         streaming_formats = streaming_data.get('formats') or []
  2032.         streaming_formats.extend(streaming_data.get('adaptiveFormats') or [])
  2034.         def build_fragments(f):
  2035.             return LazyList({
  2036.                 'url': update_url_query(f['url'], {
  2037.                     'range': '{0}-{1}'.format(range_start, min(range_start + CHUNK_SIZE - 1, f['filesize']))
  2038.                 })
  2039.             } for range_start in range(0, f['filesize'], CHUNK_SIZE))
  2041.         lower = lambda s: s.lower()
  2043.         for fmt in streaming_formats:
  2044.             if fmt.get('targetDurationSec'):
  2045.                 continue
  2047.             itag = str_or_none(fmt.get('itag'))
  2048.             audio_track = traverse_obj(fmt, ('audioTrack', T(dict))) or {}
  2050.             quality = traverse_obj(fmt, ((
  2051.                 # The 3gp format (17) in android client has a quality of "small",
  2052.                 # but is actually worse than other formats
  2053.                 T(lambda _: 'tiny' if itag == 17 else None),
  2054.                 ('quality', T(lambda q: q if q and q != 'tiny' else None)),
  2055.                 ('audioQuality', T(lower)),
  2056.                 'quality'), T(txt_or_none)), get_all=False)
  2057.             if quality and itag:
  2058.                 itag_qualities[itag] = quality
  2059.             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
  2060.             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
  2061.             # number of fragments that would subsequently be requested with (`&sq=N`)
  2062.             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
  2063.                 continue
  2065.             fmt_url = fmt.get('url')
  2066.             if not fmt_url:
  2067.                 sc = compat_parse_qs(fmt.get('signatureCipher'))
  2068.                 fmt_url = traverse_obj(sc, ('url', -1, T(url_or_none)))
  2069.                 encrypted_sig = traverse_obj(sc, ('s', -1))
  2070.                 if not (fmt_url and encrypted_sig):
  2071.                     continue
  2072.                 player_url = player_url or self._extract_player_url(webpage)
  2073.                 if not player_url:
  2074.                     continue
  2075.                 try:
  2076.                     fmt_url = update_url_query(fmt_url, {
  2077.                         traverse_obj(sc, ('sp', -1)) or 'signature':
  2078.                             [self._decrypt_signature(encrypted_sig, video_id, player_url)],
  2079.                     })
  2080.                 except ExtractorError as e:
  2081.                     self.report_warning('Signature extraction failed: Some formats may be missing',
  2082.                                         video_id=video_id, only_once=True)
  2083.                     self.write_debug(error_to_compat_str(e), only_once=True)
  2084.                     continue
  2086.             language_preference = (
  2087.                 10 if audio_track.get('audioIsDefault')
  2088.                 else -10 if 'descriptive' in (traverse_obj(audio_track, ('displayName', T(lower))) or '')
  2089.                 else -1)
  2090.             name = (
  2091.                 traverse_obj(fmt, ('qualityLabel', T(txt_or_none)))
  2092.                 or quality.replace('audio_quality_', ''))
  2093.             dct = {
  2094.                 'format_id': join_nonempty(itag, fmt.get('isDrc') and 'drc'),
  2095.                 'url': fmt_url,
  2096.                 # Format 22 is likely to be damaged: see https://github.com/yt-dlp/yt-dlp/issues/3372
  2097.                 'source_preference': ((-5 if itag == '22' else -1)
  2098.                                       + (100 if 'Premium' in name else 0)),
  2099.                 'quality': q(quality),
  2100.                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
  2101.                                           'desc' if language_preference < -1 else '') or None,
  2102.                 'language_preference': language_preference,
  2103.                 # Strictly de-prioritize 3gp formats
  2104.                 'preference': -2 if itag == '17' else None,
  2105.             }
  2106.             if itag:
  2107.                 itags[itag].add(('https', dct.get('language')))
  2108.             self._unthrottle_format_urls(video_id, player_url, dct)
  2109.             dct.update(traverse_obj(fmt, {
  2110.                 'asr': ('audioSampleRate', T(int_or_none)),
  2111.                 'filesize': ('contentLength', T(int_or_none)),
  2112.                 'format_note': ('qualityLabel', T(lambda x: x or quality)),
  2113.                 # for some formats, fps is wrongly returned as 1
  2114.                 'fps': ('fps', T(int_or_none), T(lambda f: f if f > 1 else None)),
  2115.                 'audio_channels': ('audioChannels', T(int_or_none)),
  2116.                 'height': ('height', T(int_or_none)),
  2117.                 'has_drm': ('drmFamilies', T(bool)),
  2118.                 'tbr': (('averageBitrate', 'bitrate'), T(lambda t: float_or_none(t, 1000))),
  2119.                 'width': ('width', T(int_or_none)),
  2120.                 '_duration_ms': ('approxDurationMs', T(int_or_none)),
  2121.             }, get_all=False))
  2122.             mime_mobj = re.match(
  2123.                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
  2124.             if mime_mobj:
  2125.                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
  2126.                 dct.update(parse_codecs(mime_mobj.group(2)))
  2127.             single_stream = 'none' in (dct.get(c) for c in ('acodec', 'vcodec'))
  2128.             if single_stream and dct.get('ext'):
  2129.                 dct['container'] = dct['ext'] + '_dash'
  2130.             if single_stream or itag == '17':
  2131.                 # avoid Youtube throttling
  2132.                 dct.update({
  2133.                     'protocol': 'http_dash_segments',
  2134.                     'fragments': build_fragments(dct),
  2135.                 } if dct['filesize'] else {
  2136.                     'downloader_options': {'http_chunk_size': CHUNK_SIZE}  # No longer useful?
  2137.                 })
  2139.             formats.append(dct)
  2141.         def process_manifest_format(f, proto, client_name, itag, all_formats=False):
  2142.             key = (proto, f.get('language'))
  2143.             if not all_formats and key in itags[itag]:
  2144.                 return False
  2145.             itags[itag].add(key)
  2147.             if itag:
  2148.                 f['format_id'] = (
  2149.                     '{0}-{1}'.format(itag, proto)
  2150.                     if all_formats or any(p != proto for p, _ in itags[itag])
  2151.                     else itag)
  2153.             if f.get('source_preference') is None:
  2154.                 f['source_preference'] = -1
  2156.             if itag in ('616', '235'):
  2157.                 f['format_note'] = join_nonempty(f.get('format_note'), 'Premium', delim=' ')
  2158.                 f['source_preference'] += 100
  2160.             f['quality'] = q(traverse_obj(f, (
  2161.                 'format_id', T(lambda s: itag_qualities[s.split('-')[0]])), default=-1))
  2162.             if try_call(lambda: f['fps'] <= 1):
  2163.                 del f['fps']
  2165.             if proto == 'hls' and f.get('has_drm'):
  2166.                 f['has_drm'] = 'maybe'
  2167.                 f['source_preference'] -= 5
  2168.             return True
  2170.         hls_manifest_url = streaming_data.get('hlsManifestUrl')
  2171.         if hls_manifest_url:
  2172.             for f in self._extract_m3u8_formats(
  2173.                     hls_manifest_url, video_id, 'mp4', fatal=False):
  2174.                 if process_manifest_format(
  2175.                         f, 'hls', None, self._search_regex(
  2176.                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
  2177.                     formats.append(f)
  2179.         if self._downloader.params.get('youtube_include_dash_manifest', True):
  2180.             dash_manifest_url = streaming_data.get('dashManifestUrl')
  2181.             if dash_manifest_url:
  2182.                 for f in self._extract_mpd_formats(
  2183.                         dash_manifest_url, video_id, fatal=False):
  2184.                     if process_manifest_format(
  2185.                             f, 'dash', None, f['format_id']):
  2186.                         f['filesize'] = traverse_obj(f, (
  2187.                             ('fragment_base_url', 'url'), T(lambda u: self._search_regex(
  2188.                                 r'/clen/(\d+)', u, 'file size', default=None)),
  2189.                             T(int_or_none)), get_all=False)
  2190.                         formats.append(f)
  2192.         playable_formats = [f for f in formats if not f.get('has_drm')]
  2193.         if formats and not playable_formats:
  2194.             # If there are no formats that definitely don't have DRM, all have DRM
  2195.             self.report_drm(video_id)
  2196.         formats[:] = playable_formats
  2198.         if not formats:
  2199.             if streaming_data.get('licenseInfos'):
  2200.                 raise ExtractorError(
  2201.                     'This video is DRM protected.', expected=True)
  2202.             pemr = try_get(
  2203.                 playability_status,
  2204.                 lambda x: x['errorScreen']['playerErrorMessageRenderer'],
  2205.                 dict) or {}
  2206.             reason = get_text(pemr.get('reason')) or playability_status.get('reason')
  2207.             subreason = pemr.get('subreason')
  2208.             if subreason:
  2209.                 subreason = clean_html(get_text(subreason))
  2210.                 if subreason == 'The uploader has not made this video available in your country.':
  2211.                     countries = microformat.get('availableCountries')
  2212.                     if not countries:
  2213.                         regions_allowed = search_meta('regionsAllowed')
  2214.                         countries = regions_allowed.split(',') if regions_allowed else None
  2215.                     self.raise_geo_restricted(
  2216.                         subreason, countries)
  2217.                 reason += '\n' + subreason
  2218.             if reason:
  2219.                 raise ExtractorError(reason, expected=True)
  2221.         self._sort_formats(formats)
  2223.         keywords = video_details.get('keywords') or []
  2224.         if not keywords and webpage:
  2225.             keywords = [
  2226.                 unescapeHTML(m.group('content'))
  2227.                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
  2228.         for keyword in keywords:
  2229.             if keyword.startswith('yt:stretch='):
  2230.                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
  2231.                 if mobj:
  2232.                     # NB: float is intentional for forcing float division
  2233.                     w, h = (float(v) for v in mobj.groups())
  2234.                     if w > 0 and h > 0:
  2235.                         ratio = w / h
  2236.                         for f in formats:
  2237.                             if f.get('vcodec') != 'none':
  2238.                                 f['stretched_ratio'] = ratio
  2239.                         break
  2241.         thumbnails = []
  2242.         for container in (video_details, microformat):
  2243.             for thumbnail in try_get(
  2244.                     container,
  2245.                     lambda x: x['thumbnail']['thumbnails'], list) or []:
  2246.                 thumbnail_url = url_or_none(thumbnail.get('url'))
  2247.                 if not thumbnail_url:
  2248.                     continue
  2249.                 thumbnails.append({
  2250.                     'height': int_or_none(thumbnail.get('height')),
  2251.                     'url': update_url(thumbnail_url, query=None, fragment=None),
  2252.                     'width': int_or_none(thumbnail.get('width')),
  2253.                 })
  2254.             if thumbnails:
  2255.                 break
  2256.         else:
  2257.             thumbnail = search_meta(['og:image', 'twitter:image'])
  2258.             if thumbnail:
  2259.                 thumbnails = [{'url': thumbnail}]
  2261.         category = microformat.get('category') or search_meta('genre')
  2262.         channel_id = self._extract_channel_id(
  2263.             webpage, videodetails=video_details, metadata=microformat)
  2264.         duration = int_or_none(
  2265.             video_details.get('lengthSeconds')
  2266.             or microformat.get('lengthSeconds')) \
  2267.             or parse_duration(search_meta('duration'))
  2269.         for f in formats:
  2270.             # Some formats may have much smaller duration than others (possibly damaged during encoding)
  2271.             # but avoid false positives with small duration differences.
  2272.             # Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
  2273.             if try_call(lambda x: float(x.pop('_duration_ms')) / duration < 500, args=(f,)):
  2274.                 self.report_warning(
  2275.                     '{0}: Some possibly damaged formats will be deprioritized'.format(video_id), only_once=True)
  2276.                 # Strictly de-prioritize damaged formats
  2277.                 f['preference'] = -10
  2279.         is_live = video_details.get('isLive')
  2281.         owner_profile_url = self._yt_urljoin(self._extract_author_var(
  2282.             webpage, 'url', videodetails=video_details, metadata=microformat))
  2284.         uploader = self._extract_author_var(
  2285.             webpage, 'name', videodetails=video_details, metadata=microformat)
  2287.         info = {
  2288.             'id': video_id,
  2289.             'title': self._live_title(video_title) if is_live else video_title,
  2290.             'formats': formats,
  2291.             'thumbnails': thumbnails,
  2292.             'description': video_description,
  2293.             'upload_date': unified_strdate(
  2294.                 microformat.get('uploadDate')
  2295.                 or search_meta('uploadDate')),
  2296.             'uploader': uploader,
  2297.             'channel_id': channel_id,
  2298.             'duration': duration,
  2299.             'view_count': int_or_none(
  2300.                 video_details.get('viewCount')
  2301.                 or microformat.get('viewCount')
  2302.                 or search_meta('interactionCount')),
  2303.             'average_rating': float_or_none(video_details.get('averageRating')),
  2304.             'age_limit': 18 if (
  2305.                 microformat.get('isFamilySafe') is False
  2306.                 or search_meta('isFamilyFriendly') == 'false'
  2307.                 or search_meta('og:restrictions:age') == '18+') else 0,
  2308.             'webpage_url': webpage_url,
  2309.             'categories': [category] if category else None,
  2310.             'tags': keywords,
  2311.             'is_live': is_live,
  2312.         }
  2314.         pctr = try_get(
  2315.             player_response,
  2316.             lambda x: x['captions']['playerCaptionsTracklistRenderer'], dict)
  2317.         if pctr:
  2318.             def process_language(container, base_url, lang_code, query):
  2319.                 lang_subs = []
  2320.                 for fmt in self._SUBTITLE_FORMATS:
  2321.                     query.update({
  2322.                         'fmt': fmt,
  2323.                     })
  2324.                     lang_subs.append({
  2325.                         'ext': fmt,
  2326.                         'url': update_url_query(base_url, query),
  2327.                     })
  2328.                 container[lang_code] = lang_subs
  2330.             subtitles = {}
  2331.             for caption_track in (pctr.get('captionTracks') or []):
  2332.                 base_url = caption_track.get('baseUrl')
  2333.                 if not base_url:
  2334.                     continue
  2335.                 if caption_track.get('kind') != 'asr':
  2336.                     lang_code = caption_track.get('languageCode')
  2337.                     if not lang_code:
  2338.                         continue
  2339.                     process_language(
  2340.                         subtitles, base_url, lang_code, {})
  2341.                     continue
  2342.                 automatic_captions = {}
  2343.                 for translation_language in (pctr.get('translationLanguages') or []):
  2344.                     translation_language_code = translation_language.get('languageCode')
  2345.                     if not translation_language_code:
  2346.                         continue
  2347.                     process_language(
  2348.                         automatic_captions, base_url, translation_language_code,
  2349.                         {'tlang': translation_language_code})
  2350.                 info['automatic_captions'] = automatic_captions
  2351.             info['subtitles'] = subtitles
  2353.         parsed_url = compat_urllib_parse_urlparse(url)
  2354.         for component in [parsed_url.fragment, parsed_url.query]:
  2355.             query = compat_parse_qs(component)
  2356.             for k, v in query.items():
  2357.                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
  2358.                     d_k += '_time'
  2359.                     if d_k not in info and k in s_ks:
  2360.                         info[d_k] = parse_duration(query[k][0])
  2362.         if video_description:
  2363.             # Youtube Music Auto-generated description
  2364.             mobj = re.search(r'(?s)(?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?.+\nAuto-generated by YouTube\.\s*$', video_description)
  2365.             if mobj:
  2366.                 release_year = mobj.group('release_year')
  2367.                 release_date = mobj.group('release_date')
  2368.                 if release_date:
  2369.                     release_date = release_date.replace('-', '')
  2370.                     if not release_year:
  2371.                         release_year = release_date[:4]
  2372.                 info.update({
  2373.                     'album': mobj.group('album'.strip()),
  2374.                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
  2375.                     'track': mobj.group('track').strip(),
  2376.                     'release_date': release_date,
  2377.                     'release_year': int_or_none(release_year),
  2378.                 })
  2380.         initial_data = None
  2381.         if webpage:
  2382.             initial_data = self._extract_yt_initial_variable(
  2383.                 webpage, self._YT_INITIAL_DATA_RE, video_id,
  2384.                 'yt initial data')
  2385.         if not initial_data:
  2386.             initial_data = self._call_api(
  2387.                 'next', {'videoId': video_id}, video_id, fatal=False)
  2389.         if initial_data:
  2390.             chapters = self._extract_chapters_from_json(
  2391.                 initial_data, video_id, duration)
  2392.             if not chapters:
  2393.                 for engagment_pannel in (initial_data.get('engagementPanels') or []):
  2394.                     contents = try_get(
  2395.                         engagment_pannel, lambda x: x['engagementPanelSectionListRenderer']['content']['macroMarkersListRenderer']['contents'],
  2396.                         list)
  2397.                     if not contents:
  2398.                         continue
  2400.                     def chapter_time(mmlir):
  2401.                         return parse_duration(
  2402.                             get_text(mmlir.get('timeDescription')))
  2404.                     chapters = []
  2405.                     for next_num, content in enumerate(contents, start=1):
  2406.                         mmlir = content.get('macroMarkersListItemRenderer') or {}
  2407.                         start_time = chapter_time(mmlir)
  2408.                         end_time = chapter_time(try_get(
  2409.                             contents, lambda x: x[next_num]['macroMarkersListItemRenderer'])) \
  2410.                             if next_num < len(contents) else duration
  2411.                         if start_time is None or end_time is None:
  2412.                             continue
  2413.                         chapters.append({
  2414.                             'start_time': start_time,
  2415.                             'end_time': end_time,
  2416.                             'title': get_text(mmlir.get('title')),
  2417.                         })
  2418.                     if chapters:
  2419.                         break
  2420.             if chapters:
  2421.                 info['chapters'] = chapters
  2423.             contents = try_get(
  2424.                 initial_data,
  2425.                 lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'],
  2426.                 list) or []
  2427.             if not info['channel_id']:
  2428.                 channel_id = self._extract_channel_id('', renderers=contents)
  2429.             if not info['uploader']:
  2430.                 info['uploader'] = self._extract_author_var('', 'name', renderers=contents)
  2431.             if not owner_profile_url:
  2432.                 owner_profile_url = self._yt_urljoin(self._extract_author_var('', 'url', renderers=contents))
  2434.             for content in contents:
  2435.                 vpir = content.get('videoPrimaryInfoRenderer')
  2436.                 if vpir:
  2437.                     stl = vpir.get('superTitleLink')
  2438.                     if stl:
  2439.                         stl = get_text(stl)
  2440.                         if try_get(
  2441.                                 vpir,
  2442.                                 lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
  2443.                             info['location'] = stl
  2444.                         else:
  2445.                             # •? doesn't match, but [•]? does; \xa0 = non-breaking space
  2446.                             mobj = re.search(r'([^\xa0\s].*?)[\xa0\s]*S(\d+)[\xa0\s]*[•]?[\xa0\s]*E(\d+)', stl)
  2447.                             if mobj:
  2448.                                 info.update({
  2449.                                     'series': mobj.group(1),
  2450.                                     'season_number': int(mobj.group(2)),
  2451.                                     'episode_number': int(mobj.group(3)),
  2452.                                 })
  2453.                     for tlb in (try_get(
  2454.                             vpir,
  2455.                             lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
  2456.                             list) or []):
  2457.                         tbr = traverse_obj(tlb, ('segmentedLikeDislikeButtonRenderer', 'likeButton', 'toggleButtonRenderer'), 'toggleButtonRenderer') or {}
  2458.                         for getter, regex in [(
  2459.                                 lambda x: x['defaultText']['accessibility']['accessibilityData'],
  2460.                                 r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
  2461.                                     lambda x: x['accessibility'],
  2462.                                     lambda x: x['accessibilityData']['accessibilityData'],
  2463.                                 ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
  2464.                             label = (try_get(tbr, getter, dict) or {}).get('label')
  2465.                             if label:
  2466.                                 mobj = re.match(regex, label)
  2467.                                 if mobj:
  2468.                                     info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
  2469.                                     break
  2470.                     sbr_tooltip = try_get(
  2471.                         vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
  2472.                     if sbr_tooltip:
  2473.                         # however dislike_count was hidden by YT, as if there could ever be dislikable content on YT
  2474.                         like_count, dislike_count = sbr_tooltip.split(' / ')
  2475.                         info.update({
  2476.                             'like_count': str_to_int(like_count),
  2477.                             'dislike_count': str_to_int(dislike_count),
  2478.                         })
  2479.                     else:
  2480.                         info['like_count'] = traverse_obj(vpir, (
  2481.                             'videoActions', 'menuRenderer', 'topLevelButtons', Ellipsis,
  2482.                             'segmentedLikeDislikeButtonViewModel', 'likeButtonViewModel', 'likeButtonViewModel',
  2483.                             'toggleButtonViewModel', 'toggleButtonViewModel', 'defaultButtonViewModel',
  2484.                             'buttonViewModel', (('title', ('accessibilityText', T(lambda s: s.split()), Ellipsis))), T(parse_count)),
  2485.                             get_all=False)
  2487.                 vsir = content.get('videoSecondaryInfoRenderer')
  2488.                 if vsir:
  2489.                     rows = try_get(
  2490.                         vsir,
  2491.                         lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
  2492.                         list) or []
  2493.                     multiple_songs = False
  2494.                     for row in rows:
  2495.                         if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
  2496.                             multiple_songs = True
  2497.                             break
  2498.                     for row in rows:
  2499.                         mrr = row.get('metadataRowRenderer') or {}
  2500.                         mrr_title = mrr.get('title')
  2501.                         if not mrr_title:
  2502.                             continue
  2503.                         mrr_title = get_text(mrr['title'])
  2504.                         mrr_contents_text = get_text(mrr['contents'][0])
  2505.                         if mrr_title == 'License':
  2506.                             info['license'] = mrr_contents_text
  2507.                         elif not multiple_songs:
  2508.                             if mrr_title == 'Album':
  2509.                                 info['album'] = mrr_contents_text
  2510.                             elif mrr_title == 'Artist':
  2511.                                 info['artist'] = mrr_contents_text
  2512.                             elif mrr_title == 'Song':
  2513.                                 info['track'] = mrr_contents_text
  2515.             # this is not extraction but spelunking!
  2516.             carousel_lockups = traverse_obj(
  2517.                 initial_data,
  2518.                 ('engagementPanels', Ellipsis, 'engagementPanelSectionListRenderer',
  2519.                  'content', 'structuredDescriptionContentRenderer', 'items', Ellipsis,
  2520.                  'videoDescriptionMusicSectionRenderer', 'carouselLockups', Ellipsis),
  2521.                 expected_type=dict) or []
  2522.             # try to reproduce logic from metadataRowContainerRenderer above (if it still is)
  2523.             fields = (('ALBUM', 'album'), ('ARTIST', 'artist'), ('SONG', 'track'), ('LICENSES', 'license'))
  2524.             # multiple_songs ?
  2525.             if len(carousel_lockups) > 1:
  2526.                 fields = fields[-1:]
  2527.             for info_row in traverse_obj(
  2528.                     carousel_lockups,
  2529.                     (0, 'carouselLockupRenderer', 'infoRows', Ellipsis, 'infoRowRenderer'),
  2530.                     expected_type=dict):
  2531.                 row_title = traverse_obj(info_row, ('title', 'simpleText'))
  2532.                 row_text = traverse_obj(info_row, 'defaultMetadata', 'expandedMetadata', expected_type=get_text)
  2533.                 if not row_text:
  2534.                     continue
  2535.                 for name, field in fields:
  2536.                     if name == row_title and not info.get(field):
  2537.                         info[field] = row_text
  2539.         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
  2540.             v = info.get(s_k)
  2541.             if v:
  2542.                 info[d_k] = v
  2544.         self.mark_watched(video_id, player_response)
  2546.         return merge_dicts(
  2547.             info, {
  2548.                 'uploader_id': self._extract_uploader_id(owner_profile_url),
  2549.                 'uploader_url': owner_profile_url,
  2550.                 'channel_id': channel_id,
  2551.                 'channel_url': channel_id and self._yt_urljoin('/channel/' + channel_id),
  2552.                 'channel': info['uploader'],
  2553.             })
  2556. class YoutubeTabIE(YoutubeBaseInfoExtractor):
  2557.     IE_DESC = 'YouTube.com tab'
  2558.     _VALID_URL = r'''(?x)
  2559.                     https?://
  2560.                         (?:\w+\.)?
  2561.                         (?:
  2562.                             youtube(?:kids)?\.com|
  2563.                             invidio\.us
  2564.                         )/
  2565.                         (?:
  2566.                             (?:channel|c|user|feed|hashtag)/|
  2567.                             (?:playlist|watch)\?.*?\blist=|
  2568.                             (?!(?:watch|embed|v|e|results)\b)
  2569.                         )
  2570.                         (?P<id>[^/?\#&]+)
  2571.                     '''
  2572.     IE_NAME = 'youtube:tab'
  2574.     _TESTS = [{
  2575.         # Shorts
  2576.         'url': 'https://www.youtube.com/@SuperCooperShorts/shorts',
  2577.         'playlist_mincount': 5,
  2578.         'info_dict': {
  2579.             'description': 'Short clips from Super Cooper Sundays!',
  2580.             'id': 'UCKMA8kHZ8bPYpnMNaUSxfEQ',
  2581.             'title': 'Super Cooper Shorts - Shorts',
  2582.             'uploader': 'Super Cooper Shorts',
  2583.             'uploader_id': '@SuperCooperShorts',
  2584.         }
  2585.     }, {
  2586.         # Channel that does not have a Shorts tab. Test should just download videos on Home tab instead
  2587.         'url': 'https://www.youtube.com/@emergencyawesome/shorts',
  2588.         'info_dict': {
  2589.             'description': 'md5:592c080c06fef4de3c902c4a8eecd850',
  2590.             'id': 'UCDiFRMQWpcp8_KD4vwIVicw',
  2591.             'title': 'Emergency Awesome - Home',
  2592.         },
  2593.         'playlist_mincount': 5,
  2594.         'skip': 'new test page needed to replace `Emergency Awesome - Shorts`',
  2595.     }, {
  2596.         # playlists, multipage
  2597.         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
  2598.         'playlist_mincount': 94,
  2599.         'info_dict': {
  2600.             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
  2601.             'title': r're:Igor Kleiner(?: Ph\.D\.)? - Playlists',
  2602.             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
  2603.             'uploader': 'Igor Kleiner',
  2604.             'uploader_id': '@IgorDataScience',
  2605.         },
  2606.     }, {
  2607.         # playlists, multipage, different order
  2608.         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
  2609.         'playlist_mincount': 94,
  2610.         'info_dict': {
  2611.             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
  2612.             'title': r're:Igor Kleiner(?: Ph\.D\.)? - Playlists',
  2613.             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
  2614.             'uploader': 'Igor Kleiner',
  2615.             'uploader_id': '@IgorDataScience',
  2616.         },
  2617.     }, {
  2618.         # playlists, series
  2619.         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
  2620.         'playlist_mincount': 5,
  2621.         'info_dict': {
  2622.             'id': 'UCYO_jab_esuFRV4b17AJtAw',
  2623.             'title': '3Blue1Brown - Playlists',
  2624.             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
  2625.             'uploader': '3Blue1Brown',
  2626.             'uploader_id': '@3blue1brown',
  2627.         },
  2628.     }, {
  2629.         # playlists, singlepage
  2630.         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
  2631.         'playlist_mincount': 4,
  2632.         'info_dict': {
  2633.             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
  2634.             'title': 'ThirstForScience - Playlists',
  2635.             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
  2636.             'uploader': 'ThirstForScience',
  2637.             'uploader_id': '@ThirstForScience',
  2638.         }
  2639.     }, {
  2640.         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
  2641.         'only_matching': True,
  2642.     }, {
  2643.         # basic, single video playlist
  2644.         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
  2645.         'info_dict': {
  2646.             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
  2647.             'title': 'youtube-dl public playlist',
  2648.             'uploader': 'Sergey M.',
  2649.             'uploader_id': '@sergeym.6173',
  2650.             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
  2651.         },
  2652.         'playlist_count': 1,
  2653.     }, {
  2654.         # empty playlist
  2655.         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
  2656.         'info_dict': {
  2657.             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
  2658.             'title': 'youtube-dl empty playlist',
  2659.             'uploader': 'Sergey M.',
  2660.             'uploader_id': '@sergeym.6173',
  2661.             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
  2662.         },
  2663.         'playlist_count': 0,
  2664.     }, {
  2665.         # Home tab
  2666.         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
  2667.         'info_dict': {
  2668.             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
  2669.             'title': 'lex will - Home',
  2670.             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
  2671.             'uploader': 'lex will',
  2672.             'uploader_id': '@lexwill718',
  2673.         },
  2674.         'playlist_mincount': 2,
  2675.     }, {
  2676.         # Videos tab
  2677.         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
  2678.         'info_dict': {
  2679.             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
  2680.             'title': 'lex will - Videos',
  2681.             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
  2682.             'uploader': 'lex will',
  2683.             'uploader_id': '@lexwill718',
  2684.         },
  2685.         'playlist_mincount': 975,
  2686.     }, {
  2687.         # Videos tab, sorted by popular
  2688.         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
  2689.         'info_dict': {
  2690.             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
  2691.             'title': 'lex will - Videos',
  2692.             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
  2693.             'uploader': 'lex will',
  2694.             'uploader_id': '@lexwill718',
  2695.         },
  2696.         'playlist_mincount': 199,
  2697.     }, {
  2698.         # Playlists tab
  2699.         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
  2700.         'info_dict': {
  2701.             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
  2702.             'title': 'lex will - Playlists',
  2703.             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
  2704.             'uploader': 'lex will',
  2705.             'uploader_id': '@lexwill718',
  2706.         },
  2707.         'playlist_mincount': 17,
  2708.     }, {
  2709.         # Community tab
  2710.         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
  2711.         'info_dict': {
  2712.             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
  2713.             'title': 'lex will - Community',
  2714.             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
  2715.             'uploader': 'lex will',
  2716.             'uploader_id': '@lexwill718',
  2717.         },
  2718.         'playlist_mincount': 18,
  2719.     }, {
  2720.         # Channels tab
  2721.         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
  2722.         'info_dict': {
  2723.             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
  2724.             'title': r're:lex will - (?:Home|Channels)',
  2725.             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
  2726.             'uploader': 'lex will',
  2727.             'uploader_id': '@lexwill718',
  2728.         },
  2729.         'playlist_mincount': 75,
  2730.     }, {
  2731.         # Releases tab
  2732.         'url': 'https://www.youtube.com/@daftpunk/releases',
  2733.         'info_dict': {
  2734.             'id': 'UC_kRDKYrUlrbtrSiyu5Tflg',
  2735.             'title': 'Daft Punk - Releases',
  2736.             'description': 'Daft Punk (1993 - 2021) - Official YouTube Channel',
  2737.             'uploader_id': '@daftpunk',
  2738.             'uploader': 'Daft Punk',
  2739.         },
  2740.         'playlist_mincount': 36,
  2741.     }, {
  2742.         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
  2743.         'only_matching': True,
  2744.     }, {
  2745.         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
  2746.         'only_matching': True,
  2747.     }, {
  2748.         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
  2749.         'only_matching': True,
  2750.     }, {
  2751.         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
  2752.         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
  2753.         'info_dict': {
  2754.             'title': '29C3: Not my department',
  2755.             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
  2756.             'uploader': 'Christiaan008',
  2757.             'uploader_id': '@ChRiStIaAn008',
  2758.             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
  2759.         },
  2760.         'playlist_count': 96,
  2761.     }, {
  2762.         'note': 'Large playlist',
  2763.         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
  2764.         'info_dict': {
  2765.             'title': 'Uploads from Cauchemar',
  2766.             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
  2767.             'uploader': 'Cauchemar',
  2768.             'uploader_id': '@Cauchemar89',
  2769.             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
  2770.         },
  2771.         'playlist_mincount': 1123,
  2772.     }, {
  2773.         # even larger playlist, 8832 videos
  2774.         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
  2775.         'only_matching': True,
  2776.     }, {
  2777.         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
  2778.         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
  2779.         'info_dict': {
  2780.             'title': 'Uploads from Interstellar Movie',
  2781.             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
  2782.             'uploader': 'Interstellar Movie',
  2783.             'uploader_id': '@InterstellarMovie',
  2784.             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
  2785.         },
  2786.         'playlist_mincount': 21,
  2787.     }, {
  2788.         # https://github.com/ytdl-org/youtube-dl/issues/21844
  2789.         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
  2790.         'info_dict': {
  2791.             'title': 'Data Analysis with Dr Mike Pound',
  2792.             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
  2793.             'uploader': 'Computerphile',
  2794.             'uploader_id': '@Computerphile',
  2795.             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
  2796.         },
  2797.         'playlist_mincount': 11,
  2798.     }, {
  2799.         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
  2800.         'only_matching': True,
  2801.     }, {
  2802.         # Playlist URL that does not actually serve a playlist
  2803.         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
  2804.         'info_dict': {
  2805.             'id': 'FqZTN594JQw',
  2806.             'ext': 'webm',
  2807.             'title': "Smiley's People 01 detective, Adventure Series, Action",
  2808.             'uploader': 'STREEM',
  2809.             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
  2810.             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
  2811.             'upload_date': '20150526',
  2812.             'license': 'Standard YouTube License',
  2813.             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
  2814.             'categories': ['People & Blogs'],
  2815.             'tags': list,
  2816.             'view_count': int,
  2817.             'like_count': int,
  2818.         },
  2819.         'params': {
  2820.             'skip_download': True,
  2821.         },
  2822.         'skip': 'This video is not available.',
  2823.         'add_ie': [YoutubeIE.ie_key()],
  2824.     }, {
  2825.         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
  2826.         'only_matching': True,
  2827.     }, {
  2828.         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
  2829.         'only_matching': True,
  2830.     }, {
  2831.         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
  2832.         'info_dict': {
  2833.             'id': r're:[\da-zA-Z_-]{8,}',
  2834.             'ext': 'mp4',
  2835.             'title': r're:(?s)[A-Z].{20,}',
  2836.             'uploader': 'Sky News',
  2837.             'uploader_id': '@SkyNews',
  2838.             'uploader_url': r're:https?://(?:www\.)?youtube\.com/@SkyNews',
  2839.             'upload_date': r're:\d{8}',
  2840.             'description': r're:(?s)(?:.*\n)+SUBSCRIBE to our YouTube channel for more videos: http://www\.youtube\.com/skynews *\n.*',
  2841.             'categories': ['News & Politics'],
  2842.             'tags': list,
  2843.             'like_count': int,
  2844.         },
  2845.         'params': {
  2846.             'skip_download': True,
  2847.         },
  2848.     }, {
  2849.         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
  2850.         'info_dict': {
  2851.             'id': 'a48o2S1cPoo',
  2852.             'ext': 'mp4',
  2853.             'title': 'The Young Turks - Live Main Show',
  2854.             'uploader': 'The Young Turks',
  2855.             'uploader_id': 'TheYoungTurks',
  2856.             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
  2857.             'upload_date': '20150715',
  2858.             'license': 'Standard YouTube License',
  2859.             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
  2860.             'categories': ['News & Politics'],
  2861.             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
  2862.             'like_count': int,
  2863.         },
  2864.         'params': {
  2865.             'skip_download': True,
  2866.         },
  2867.         'only_matching': True,
  2868.     }, {
  2869.         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
  2870.         'only_matching': True,
  2871.     }, {
  2872.         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
  2873.         'only_matching': True,
  2874.     }, {
  2875.         'url': 'https://www.youtube.com/feed/trending',
  2876.         'only_matching': True,
  2877.     }, {
  2878.         # needs auth
  2879.         'url': 'https://www.youtube.com/feed/library',
  2880.         'only_matching': True,
  2881.     }, {
  2882.         # needs auth
  2883.         'url': 'https://www.youtube.com/feed/history',
  2884.         'only_matching': True,
  2885.     }, {
  2886.         # needs auth
  2887.         'url': 'https://www.youtube.com/feed/subscriptions',
  2888.         'only_matching': True,
  2889.     }, {
  2890.         # needs auth
  2891.         'url': 'https://www.youtube.com/feed/watch_later',
  2892.         'only_matching': True,
  2893.     }, {
  2894.         # no longer available?
  2895.         'url': 'https://www.youtube.com/feed/recommended',
  2896.         'only_matching': True,
  2897.     }, {
  2898.         # inline playlist with not always working continuations
  2899.         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
  2900.         'only_matching': True,
  2901.     }, {
  2902.         'url': 'https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8',
  2903.         'only_matching': True,
  2904.     }, {
  2905.         'url': 'https://www.youtube.com/course',
  2906.         'only_matching': True,
  2907.     }, {
  2908.         'url': 'https://www.youtube.com/zsecurity',
  2909.         'only_matching': True,
  2910.     }, {
  2911.         'url': 'http://www.youtube.com/NASAgovVideo/videos',
  2912.         'only_matching': True,
  2913.     }, {
  2914.         'url': 'https://www.youtube.com/TheYoungTurks/live',
  2915.         'only_matching': True,
  2916.     }, {
  2917.         'url': 'https://www.youtube.com/hashtag/cctv9',
  2918.         'info_dict': {
  2919.             'id': 'cctv9',
  2920.             'title': '#cctv9',
  2921.         },
  2922.         'playlist_mincount': 350,
  2923.     }, {
  2924.         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
  2925.         'only_matching': True,
  2926.     }, {
  2927.         'note': 'Search tab',
  2928.         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
  2929.         'playlist_mincount': 20,
  2930.         'info_dict': {
  2931.             'id': 'UCYO_jab_esuFRV4b17AJtAw',
  2932.             'title': '3Blue1Brown - Search - linear algebra',
  2933.             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
  2934.             'uploader': '3Blue1Brown',
  2935.             'uploader_id': '@3blue1brown',
  2936.             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
  2937.         }
  2938.     }]
  2940.     @classmethod
  2941.     def suitable(cls, url):
  2942.         return not YoutubeIE.suitable(url) and super(
  2943.             YoutubeTabIE, cls).suitable(url)
  2945.     @staticmethod
  2946.     def _extract_grid_item_renderer(item):
  2947.         assert isinstance(item, dict)
  2948.         for key, renderer in item.items():
  2949.             if not key.startswith('grid') or not key.endswith('Renderer'):
  2950.                 continue
  2951.             if not isinstance(renderer, dict):
  2952.                 continue
  2953.             return renderer
  2955.     @staticmethod
  2956.     def _get_text(r, k):
  2957.         return traverse_obj(
  2958.             r, (k, 'runs', 0, 'text'), (k, 'simpleText'),
  2959.             expected_type=txt_or_none)
  2961.     def _grid_entries(self, grid_renderer):
  2962.         for item in grid_renderer['items']:
  2963.             if not isinstance(item, dict):
  2964.                 continue
  2965.             renderer = self._extract_grid_item_renderer(item)
  2966.             if not isinstance(renderer, dict):
  2967.                 continue
  2968.             title = self._get_text(renderer, 'title')
  2969.             # playlist
  2970.             playlist_id = renderer.get('playlistId')
  2971.             if playlist_id:
  2972.                 yield self.url_result(
  2973.                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
  2974.                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
  2975.                     video_title=title)
  2976.                 continue
  2977.             # video
  2978.             video_id = renderer.get('videoId')
  2979.             if video_id:
  2980.                 yield self._extract_video(renderer)
  2981.                 continue
  2982.             # channel
  2983.             channel_id = renderer.get('channelId')
  2984.             if channel_id:
  2985.                 title = self._get_text(renderer, 'title')
  2986.                 yield self.url_result(
  2987.                     'https://www.youtube.com/channel/%s' % channel_id,
  2988.                     ie=YoutubeTabIE.ie_key(), video_title=title)
  2989.                 continue
  2990.             # generic endpoint URL support
  2991.             ep_url = urljoin('https://www.youtube.com/', try_get(
  2992.                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
  2993.                 compat_str))
  2994.             if ep_url:
  2995.                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
  2996.                     if ie.suitable(ep_url):
  2997.                         yield self.url_result(
  2998.                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
  2999.                         break
  3001.     def _shelf_entries_from_content(self, shelf_renderer):
  3002.         content = shelf_renderer.get('content')
  3003.         if not isinstance(content, dict):
  3004.             return
  3005.         renderer = content.get('gridRenderer')
  3006.         if renderer:
  3007.             # TODO: add support for nested playlists so each shelf is processed
  3008.             # as separate playlist
  3009.             # TODO: this includes only first N items
  3010.             for entry in self._grid_entries(renderer):
  3011.                 yield entry
  3012.         renderer = content.get('horizontalListRenderer')
  3013.         if renderer:
  3014.             # TODO
  3015.             pass
  3017.     def _shelf_entries(self, shelf_renderer, skip_channels=False):
  3018.         ep = try_get(
  3019.             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
  3020.             compat_str)
  3021.         shelf_url = urljoin('https://www.youtube.com', ep)
  3022.         if shelf_url:
  3023.             # Skipping links to another channels, note that checking for
  3024.             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
  3025.             # will not work
  3026.             if skip_channels and '/channels?' in shelf_url:
  3027.                 return
  3028.             title = try_get(
  3029.                 shelf_renderer, lambda x: x['title']['runs'][0]['text'], compat_str)
  3030.             yield self.url_result(shelf_url, video_title=title)
  3031.         # Shelf may not contain shelf URL, fallback to extraction from content
  3032.         for entry in self._shelf_entries_from_content(shelf_renderer):
  3033.             yield entry
  3035.     def _playlist_entries(self, video_list_renderer):
  3036.         for content in video_list_renderer['contents']:
  3037.             if not isinstance(content, dict):
  3038.                 continue
  3039.             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
  3040.             if not isinstance(renderer, dict):
  3041.                 continue
  3042.             video_id = renderer.get('videoId')
  3043.             if not video_id:
  3044.                 continue
  3045.             yield self._extract_video(renderer)
  3047.     def _video_entry(self, video_renderer):
  3048.         video_id = video_renderer.get('videoId')
  3049.         if video_id:
  3050.             return self._extract_video(video_renderer)
  3052.     def _post_thread_entries(self, post_thread_renderer):
  3053.         post_renderer = try_get(
  3054.             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
  3055.         if not post_renderer:
  3056.             return
  3057.         # video attachment
  3058.         video_renderer = try_get(
  3059.             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict)
  3060.         video_id = None
  3061.         if video_renderer:
  3062.             entry = self._video_entry(video_renderer)
  3063.             if entry:
  3064.                 yield entry
  3065.         # inline video links
  3066.         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
  3067.         for run in runs:
  3068.             if not isinstance(run, dict):
  3069.                 continue
  3070.             ep_url = try_get(
  3071.                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], compat_str)
  3072.             if not ep_url:
  3073.                 continue
  3074.             if not YoutubeIE.suitable(ep_url):
  3075.                 continue
  3076.             ep_video_id = YoutubeIE._match_id(ep_url)
  3077.             if video_id == ep_video_id:
  3078.                 continue
  3079.             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=video_id)
  3081.     def _post_thread_continuation_entries(self, post_thread_continuation):
  3082.         contents = post_thread_continuation.get('contents')
  3083.         if not isinstance(contents, list):
  3084.             return
  3085.         for content in contents:
  3086.             renderer = content.get('backstagePostThreadRenderer')
  3087.             if not isinstance(renderer, dict):
  3088.                 continue
  3089.             for entry in self._post_thread_entries(renderer):
  3090.                 yield entry
  3092.     def _rich_grid_entries(self, contents):
  3093.         for content in contents:
  3094.             content = traverse_obj(
  3095.                 content, ('richItemRenderer', 'content'),
  3096.                 expected_type=dict) or {}
  3097.             video_renderer = traverse_obj(
  3098.                 content, 'videoRenderer', 'reelItemRenderer',
  3099.                 expected_type=dict)
  3100.             if video_renderer:
  3101.                 entry = self._video_entry(video_renderer)
  3102.                 if entry:
  3103.                     yield entry
  3104.             # playlist
  3105.             renderer = traverse_obj(
  3106.                 content, 'playlistRenderer', expected_type=dict) or {}
  3107.             title = self._get_text(renderer, 'title')
  3108.             playlist_id = renderer.get('playlistId')
  3109.             if playlist_id:
  3110.                 yield self.url_result(
  3111.                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
  3112.                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
  3113.                     video_title=title)
  3115.     @staticmethod
  3116.     def _build_continuation_query(continuation, ctp=None):
  3117.         query = {
  3118.             'ctoken': continuation,
  3119.             'continuation': continuation,
  3120.         }
  3121.         if ctp:
  3122.             query['itct'] = ctp
  3123.         return query
  3125.     @staticmethod
  3126.     def _extract_next_continuation_data(renderer):
  3127.         next_continuation = try_get(
  3128.             renderer, lambda x: x['continuations'][0]['nextContinuationData'], dict)
  3129.         if not next_continuation:
  3130.             return
  3131.         continuation = next_continuation.get('continuation')
  3132.         if not continuation:
  3133.             return
  3134.         ctp = next_continuation.get('clickTrackingParams')
  3135.         return YoutubeTabIE._build_continuation_query(continuation, ctp)
  3137.     @classmethod
  3138.     def _extract_continuation(cls, renderer):
  3139.         next_continuation = cls._extract_next_continuation_data(renderer)
  3140.         if next_continuation:
  3141.             return next_continuation
  3142.         contents = []
  3143.         for key in ('contents', 'items'):
  3144.             contents.extend(try_get(renderer, lambda x: x[key], list) or [])
  3145.         for content in contents:
  3146.             if not isinstance(content, dict):
  3147.                 continue
  3148.             continuation_ep = try_get(
  3149.                 content, lambda x: x['continuationItemRenderer']['continuationEndpoint'],
  3150.                 dict)
  3151.             if not continuation_ep:
  3152.                 continue
  3153.             continuation = try_get(
  3154.                 continuation_ep, lambda x: x['continuationCommand']['token'], compat_str)
  3155.             if not continuation:
  3156.                 continue
  3157.             ctp = continuation_ep.get('clickTrackingParams')
  3158.             return YoutubeTabIE._build_continuation_query(continuation, ctp)
  3160.     def _entries(self, tab, item_id, webpage):
  3161.         tab_content = try_get(tab, lambda x: x['content'], dict)
  3162.         if not tab_content:
  3163.             return
  3164.         slr_renderer = try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
  3165.         if slr_renderer:
  3166.             is_channels_tab = tab.get('title') == 'Channels'
  3167.             continuation = None
  3168.             slr_contents = try_get(slr_renderer, lambda x: x['contents'], list) or []
  3169.             for slr_content in slr_contents:
  3170.                 if not isinstance(slr_content, dict):
  3171.                     continue
  3172.                 is_renderer = try_get(slr_content, lambda x: x['itemSectionRenderer'], dict)
  3173.                 if not is_renderer:
  3174.                     continue
  3175.                 isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
  3176.                 for isr_content in isr_contents:
  3177.                     if not isinstance(isr_content, dict):
  3178.                         continue
  3179.                     renderer = isr_content.get('playlistVideoListRenderer')
  3180.                     if renderer:
  3181.                         for entry in self._playlist_entries(renderer):
  3182.                             yield entry
  3183.                         continuation = self._extract_continuation(renderer)
  3184.                         continue
  3185.                     renderer = isr_content.get('gridRenderer')
  3186.                     if renderer:
  3187.                         for entry in self._grid_entries(renderer):
  3188.                             yield entry
  3189.                         continuation = self._extract_continuation(renderer)
  3190.                         continue
  3191.                     renderer = isr_content.get('shelfRenderer')
  3192.                     if renderer:
  3193.                         for entry in self._shelf_entries(renderer, not is_channels_tab):
  3194.                             yield entry
  3195.                         continue
  3196.                     renderer = isr_content.get('backstagePostThreadRenderer')
  3197.                     if renderer:
  3198.                         for entry in self._post_thread_entries(renderer):
  3199.                             yield entry
  3200.                         continuation = self._extract_continuation(renderer)
  3201.                         continue
  3202.                     renderer = isr_content.get('videoRenderer')
  3203.                     if renderer:
  3204.                         entry = self._video_entry(renderer)
  3205.                         if entry:
  3206.                             yield entry
  3208.                 if not continuation:
  3209.                     continuation = self._extract_continuation(is_renderer)
  3210.             if not continuation:
  3211.                 continuation = self._extract_continuation(slr_renderer)
  3212.         else:
  3213.             rich_grid_renderer = tab_content.get('richGridRenderer')
  3214.             if not rich_grid_renderer:
  3215.                 return
  3216.             for entry in self._rich_grid_entries(rich_grid_renderer.get('contents') or []):
  3217.                 yield entry
  3219.             continuation = self._extract_continuation(rich_grid_renderer)
  3221.         ytcfg = self._extract_ytcfg(item_id, webpage)
  3222.         client_version = try_get(
  3223.             ytcfg, lambda x: x['INNERTUBE_CLIENT_VERSION'], compat_str) or '2.20210407.08.00'
  3225.         headers = {
  3226.             'x-youtube-client-name': '1',
  3227.             'x-youtube-client-version': client_version,
  3228.             'content-type': 'application/json',
  3229.         }
  3231.         context = try_get(ytcfg, lambda x: x['INNERTUBE_CONTEXT'], dict) or {
  3232.             'client': {
  3233.                 'clientName': 'WEB',
  3234.                 'clientVersion': client_version,
  3235.             }
  3236.         }
  3237.         visitor_data = try_get(context, lambda x: x['client']['visitorData'], compat_str)
  3239.         identity_token = self._extract_identity_token(ytcfg, webpage)
  3240.         if identity_token:
  3241.             headers['x-youtube-identity-token'] = identity_token
  3243.         data = {
  3244.             'context': context,
  3245.         }
  3247.         for page_num in itertools.count(1):
  3248.             if not continuation:
  3249.                 break
  3250.             if visitor_data:
  3251.                 headers['x-goog-visitor-id'] = visitor_data
  3252.             data['continuation'] = continuation['continuation']
  3253.             data['clickTracking'] = {
  3254.                 'clickTrackingParams': continuation['itct']
  3255.             }
  3256.             count = 0
  3257.             retries = 3
  3258.             while count <= retries:
  3259.                 try:
  3260.                     # Downloading page may result in intermittent 5xx HTTP error
  3261.                     # that is usually worked around with a retry
  3262.                     response = self._download_json(
  3263.                         'https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  3264.                         None, 'Downloading page %d%s' % (page_num, ' (retry #%d)' % count if count else ''),
  3265.                         headers=headers, data=json.dumps(data).encode('utf8'))
  3266.                     break
  3267.                 except ExtractorError as e:
  3268.                     if isinstance(e.cause, compat_HTTPError) and e.cause.code in (500, 503):
  3269.                         count += 1
  3270.                         if count <= retries:
  3271.                             continue
  3272.                     raise
  3273.             if not response:
  3274.                 break
  3276.             visitor_data = try_get(
  3277.                 response, lambda x: x['responseContext']['visitorData'], compat_str) or visitor_data
  3279.             continuation_contents = try_get(
  3280.                 response, lambda x: x['continuationContents'], dict)
  3281.             if continuation_contents:
  3282.                 continuation_renderer = continuation_contents.get('playlistVideoListContinuation')
  3283.                 if continuation_renderer:
  3284.                     for entry in self._playlist_entries(continuation_renderer):
  3285.                         yield entry
  3286.                     continuation = self._extract_continuation(continuation_renderer)
  3287.                     continue
  3288.                 continuation_renderer = continuation_contents.get('gridContinuation')
  3289.                 if continuation_renderer:
  3290.                     for entry in self._grid_entries(continuation_renderer):
  3291.                         yield entry
  3292.                     continuation = self._extract_continuation(continuation_renderer)
  3293.                     continue
  3294.                 continuation_renderer = continuation_contents.get('itemSectionContinuation')
  3295.                 if continuation_renderer:
  3296.                     for entry in self._post_thread_continuation_entries(continuation_renderer):
  3297.                         yield entry
  3298.                     continuation = self._extract_continuation(continuation_renderer)
  3299.                     continue
  3301.             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
  3302.             continuation_items = try_get(
  3303.                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
  3304.             if continuation_items:
  3305.                 continuation_item = continuation_items[0]
  3306.                 if not isinstance(continuation_item, dict):
  3307.                     continue
  3308.                 renderer = self._extract_grid_item_renderer(continuation_item)
  3309.                 if renderer:
  3310.                     grid_renderer = {'items': continuation_items}
  3311.                     for entry in self._grid_entries(grid_renderer):
  3312.                         yield entry
  3313.                     continuation = self._extract_continuation(grid_renderer)
  3314.                     continue
  3315.                 renderer = continuation_item.get('playlistVideoRenderer') or continuation_item.get('itemSectionRenderer')
  3316.                 if renderer:
  3317.                     video_list_renderer = {'contents': continuation_items}
  3318.                     for entry in self._playlist_entries(video_list_renderer):
  3319.                         yield entry
  3320.                     continuation = self._extract_continuation(video_list_renderer)
  3321.                     continue
  3322.                 renderer = continuation_item.get('backstagePostThreadRenderer')
  3323.                 if renderer:
  3324.                     continuation_renderer = {'contents': continuation_items}
  3325.                     for entry in self._post_thread_continuation_entries(continuation_renderer):
  3326.                         yield entry
  3327.                     continuation = self._extract_continuation(continuation_renderer)
  3328.                     continue
  3329.                 renderer = continuation_item.get('richItemRenderer')
  3330.                 if renderer:
  3331.                     for entry in self._rich_grid_entries(continuation_items):
  3332.                         yield entry
  3333.                     continuation = self._extract_continuation({'contents': continuation_items})
  3334.                     continue
  3336.             break
  3338.     @staticmethod
  3339.     def _extract_selected_tab(tabs):
  3340.         for tab in tabs:
  3341.             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
  3342.             if renderer.get('selected') is True:
  3343.                 return renderer
  3344.         else:
  3345.             raise ExtractorError('Unable to find selected tab')
  3347.     def _extract_uploader(self, metadata, data):
  3348.         uploader = {}
  3349.         renderers = traverse_obj(data,
  3350.                                  ('sidebar', 'playlistSidebarRenderer', 'items'))
  3351.         uploader['channel_id'] = self._extract_channel_id('', metadata=metadata, renderers=renderers)
  3352.         uploader['uploader'] = (
  3353.             self._extract_author_var('', 'name', renderers=renderers)
  3354.             or self._extract_author_var('', 'name', metadata=metadata))
  3355.         uploader['uploader_url'] = self._yt_urljoin(
  3356.             self._extract_author_var('', 'url', metadata=metadata, renderers=renderers))
  3357.         uploader['uploader_id'] = self._extract_uploader_id(uploader['uploader_url'])
  3358.         uploader['channel'] = uploader['uploader']
  3359.         return uploader
  3361.     @classmethod
  3362.     def _extract_alert(cls, data):
  3363.         alerts = []
  3364.         for alert in traverse_obj(data, ('alerts', Ellipsis), expected_type=dict):
  3365.             alert_text = traverse_obj(
  3366.                 alert, (None, lambda x: x['alertRenderer']['text']), get_all=False)
  3367.             if not alert_text:
  3368.                 continue
  3369.             text = cls._get_text(alert_text, 'text')
  3370.             if text:
  3371.                 alerts.append(text)
  3372.         return '\n'.join(alerts)
  3374.     def _extract_from_tabs(self, item_id, webpage, data, tabs):
  3375.         selected_tab = self._extract_selected_tab(tabs)
  3376.         renderer = traverse_obj(data, ('metadata', 'channelMetadataRenderer'),
  3377.                                 expected_type=dict) or {}
  3378.         playlist_id = item_id
  3379.         title = description = None
  3380.         if renderer:
  3381.             channel_title = txt_or_none(renderer.get('title')) or item_id
  3382.             tab_title = txt_or_none(selected_tab.get('title'))
  3383.             title = join_nonempty(
  3384.                 channel_title or item_id, tab_title,
  3385.                 txt_or_none(selected_tab.get('expandedText')),
  3386.                 delim=' - ')
  3387.             description = txt_or_none(renderer.get('description'))
  3388.             playlist_id = txt_or_none(renderer.get('externalId')) or playlist_id
  3389.         else:
  3390.             renderer = traverse_obj(data,
  3391.                                     ('metadata', 'playlistMetadataRenderer'),
  3392.                                     ('header', 'hashtagHeaderRenderer'),
  3393.                                     expected_type=dict) or {}
  3394.             title = traverse_obj(renderer, 'title', ('hashtag', 'simpleText'),
  3395.                                  expected_type=txt_or_none)
  3396.         playlist = self.playlist_result(
  3397.             self._entries(selected_tab, item_id, webpage),
  3398.             playlist_id=playlist_id, playlist_title=title,
  3399.             playlist_description=description)
  3400.         return merge_dicts(playlist, self._extract_uploader(renderer, data))
  3402.     def _extract_from_playlist(self, item_id, url, data, playlist):
  3403.         title = traverse_obj((playlist, data),
  3404.                              (0, 'title'), (1, 'titleText', 'simpleText'),
  3405.                              expected_type=txt_or_none)
  3406.         playlist_id = txt_or_none(playlist.get('playlistId')) or item_id
  3407.         # Inline playlist rendition continuation does not always work
  3408.         # at Youtube side, so delegating regular tab-based playlist URL
  3409.         # processing whenever possible.
  3410.         playlist_url = urljoin(url, traverse_obj(
  3411.             playlist, ('endpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
  3412.             expected_type=url_or_none))
  3413.         if playlist_url and playlist_url != url:
  3414.             return self.url_result(
  3415.                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
  3416.                 video_title=title)
  3417.         return self.playlist_result(
  3418.             self._playlist_entries(playlist), playlist_id=playlist_id,
  3419.             playlist_title=title)
  3421.     def _extract_identity_token(self, ytcfg, webpage):
  3422.         if ytcfg:
  3423.             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], compat_str)
  3424.             if token:
  3425.                 return token
  3426.         return self._search_regex(
  3427.             r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
  3428.             'identity token', default=None)
  3430.     def _real_extract(self, url):
  3431.         item_id = self._match_id(url)
  3432.         url = update_url(url, netloc='www.youtube.com')
  3433.         # Handle both video/playlist URLs
  3434.         qs = parse_qs(url)
  3435.         video_id = qs.get('v', [None])[0]
  3436.         playlist_id = qs.get('list', [None])[0]
  3437.         if video_id and playlist_id:
  3438.             if self._downloader.params.get('noplaylist'):
  3439.                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
  3440.                 return self.url_result(video_id, ie=YoutubeIE.ie_key(), video_id=video_id)
  3441.             self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
  3442.         webpage = self._download_webpage(url, item_id)
  3443.         data = self._extract_yt_initial_data(item_id, webpage)
  3444.         tabs = try_get(
  3445.             data, lambda x: x['contents']['twoColumnBrowseResultsRenderer']['tabs'], list)
  3446.         if tabs:
  3447.             return self._extract_from_tabs(item_id, webpage, data, tabs)
  3448.         playlist = try_get(
  3449.             data, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
  3450.         if playlist:
  3451.             return self._extract_from_playlist(item_id, url, data, playlist)
  3452.         # Fallback to video extraction if no playlist alike page is recognized.
  3453.         # First check for the current video then try the v attribute of URL query.
  3454.         video_id = try_get(
  3455.             data, lambda x: x['currentVideoEndpoint']['watchEndpoint']['videoId'],
  3456.             compat_str) or video_id
  3457.         if video_id:
  3458.             return self.url_result(video_id, ie=YoutubeIE.ie_key(), video_id=video_id)
  3459.         # Capture and output alerts
  3460.         alert = self._extract_alert(data)
  3461.         if alert:
  3462.             raise ExtractorError(alert, expected=True)
  3463.         # Failed to recognize
  3464.         raise ExtractorError('Unable to recognize tab page')
  3467. class YoutubePlaylistIE(InfoExtractor):
  3468.     IE_DESC = 'YouTube.com playlists'
  3469.     _VALID_URL = r'''(?x)(?:
  3470.                         (?:https?://)?
  3471.                         (?:\w+\.)?
  3472.                         (?:
  3473.                             (?:
  3474.                                 youtube(?:kids)?\.com|
  3475.                                 invidio\.us
  3476.                             )
  3477.                             /.*?\?.*?\blist=
  3478.                         )?
  3479.                         (?P<id>%(playlist_id)s)
  3480.                      )''' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
  3481.     IE_NAME = 'youtube:playlist'
  3482.     _TESTS = [{
  3483.         'note': 'issue #673',
  3484.         'url': 'PLBB231211A4F62143',
  3485.         'info_dict': {
  3486.             'title': '[OLD]Team Fortress 2 (Class-based LP)',
  3487.             'id': 'PLBB231211A4F62143',
  3488.             'uploader': 'Wickman',
  3489.             'uploader_id': '@WickmanVT',
  3490.             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
  3491.         },
  3492.         'playlist_mincount': 29,
  3493.     }, {
  3494.         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
  3495.         'info_dict': {
  3496.             'title': 'YDL_safe_search',
  3497.             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
  3498.         },
  3499.         'playlist_count': 2,
  3500.         'skip': 'This playlist is private',
  3501.     }, {
  3502.         'note': 'embedded',
  3503.         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
  3504.         # TODO: full playlist requires _reload_with_unavailable_videos()
  3505.         # 'playlist_count': 4,
  3506.         'playlist_mincount': 1,
  3507.         'info_dict': {
  3508.             'title': 'JODA15',
  3509.             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
  3510.             'uploader': 'milan',
  3511.             'uploader_id': '@milan5503',
  3512.             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
  3513.         }
  3514.     }, {
  3515.         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
  3516.         'playlist_mincount': 455,
  3517.         'info_dict': {
  3518.             'title': '2018 Chinese New Singles (11/6 updated)',
  3519.             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
  3520.             'uploader': 'LBK',
  3521.             'uploader_id': '@music_king',
  3522.             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
  3523.         }
  3524.     }, {
  3525.         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
  3526.         'only_matching': True,
  3527.     }, {
  3528.         # music album playlist
  3529.         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
  3530.         'only_matching': True,
  3531.     }]
  3533.     @classmethod
  3534.     def suitable(cls, url):
  3535.         if YoutubeTabIE.suitable(url):
  3536.             return False
  3537.         if parse_qs(url).get('v', [None])[0]:
  3538.             return False
  3539.         return super(YoutubePlaylistIE, cls).suitable(url)
  3541.     def _real_extract(self, url):
  3542.         playlist_id = self._match_id(url)
  3543.         qs = parse_qs(url)
  3544.         if not qs:
  3545.             qs = {'list': playlist_id}
  3546.         return self.url_result(
  3547.             update_url_query('https://www.youtube.com/playlist', qs),
  3548.             ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
  3551. class YoutubeYtBeIE(InfoExtractor):
  3552.     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
  3553.     _TESTS = [{
  3554.         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
  3555.         'info_dict': {
  3556.             'id': 'yeWKywCrFtk',
  3557.             'ext': 'mp4',
  3558.             'title': 'Small Scale Baler and Braiding Rugs',
  3559.             'uploader': 'Backus-Page House Museum',
  3560.             'uploader_id': '@backuspagemuseum',
  3561.             'uploader_url': r're:https?://(?:www\.)?youtube\.com/@backuspagemuseum',
  3562.             'upload_date': '20161008',
  3563.             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
  3564.             'categories': ['Nonprofits & Activism'],
  3565.             'tags': list,
  3566.             'like_count': int,
  3567.         },
  3568.         'params': {
  3569.             'noplaylist': True,
  3570.             'skip_download': True,
  3571.         },
  3572.     }, {
  3573.         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
  3574.         'only_matching': True,
  3575.     }]
  3577.     def _real_extract(self, url):
  3578.         mobj = re.match(self._VALID_URL, url)
  3579.         video_id = mobj.group('id')
  3580.         playlist_id = mobj.group('playlist_id')
  3581.         return self.url_result(
  3582.             update_url_query('https://www.youtube.com/watch', {
  3583.                 'v': video_id,
  3584.                 'list': playlist_id,
  3585.                 'feature': 'youtu.be',
  3586.             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
  3589. class YoutubeYtUserIE(InfoExtractor):
  3590.     _VALID_URL = r'ytuser:(?P<id>.+)'
  3591.     _TESTS = [{
  3592.         'url': 'ytuser:phihag',
  3593.         'only_matching': True,
  3594.     }]
  3596.     def _real_extract(self, url):
  3597.         user_id = self._match_id(url)
  3598.         return self.url_result(
  3599.             'https://www.youtube.com/user/%s' % user_id,
  3600.             ie=YoutubeTabIE.ie_key(), video_id=user_id)
  3603. class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
  3604.     IE_NAME = 'youtube:favorites'
  3605.     IE_DESC = 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'
  3606.     _VALID_URL = r'https?://(?:www\.)?youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
  3607.     _LOGIN_REQUIRED = True
  3608.     _TESTS = [{
  3609.         'url': ':ytfav',
  3610.         'only_matching': True,
  3611.     }, {
  3612.         'url': ':ytfavorites',
  3613.         'only_matching': True,
  3614.     }]
  3616.     def _real_extract(self, url):
  3617.         return self.url_result(
  3618.             'https://www.youtube.com/playlist?list=LL',
  3619.             ie=YoutubeTabIE.ie_key())
  3622. class YoutubeSearchIE(SearchInfoExtractor, YoutubeBaseInfoExtractor):
  3623.     IE_DESC = 'YouTube.com searches'
  3624.     IE_NAME = 'youtube:search'
  3625.     _SEARCH_KEY = 'ytsearch'
  3626.     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
  3627.     _MAX_RESULTS = float('inf')
  3628.     _TESTS = [{
  3629.         'url': 'ytsearch10:youtube-dl test video',
  3630.         'playlist_count': 10,
  3631.         'info_dict': {
  3632.             'id': 'youtube-dl test video',
  3633.             'title': 'youtube-dl test video',
  3634.         }
  3635.     }]
  3637.     def _get_n_results(self, query, n):
  3638.         """Get a specified number of results for a query"""
  3639.         entries = itertools.islice(self._search_results(query, self._SEARCH_PARAMS), 0, None if n == float('inf') else n)
  3640.         return self.playlist_result(entries, query, query)
  3643. class YoutubeSearchDateIE(YoutubeSearchIE):
  3644.     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
  3645.     _SEARCH_KEY = 'ytsearchdate'
  3646.     IE_DESC = 'YouTube.com searches, newest videos first'
  3647.     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
  3648.     _TESTS = [{
  3649.         'url': 'ytsearchdate10:youtube-dl test video',
  3650.         'playlist_count': 10,
  3651.         'info_dict': {
  3652.             'id': 'youtube-dl test video',
  3653.             'title': 'youtube-dl test video',
  3654.         }
  3655.     }]
  3658. class YoutubeSearchURLIE(YoutubeBaseInfoExtractor):
  3659.     IE_DESC = 'YouTube search URLs with sorting and filter support'
  3660.     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
  3661.     _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?:[^&]+)(?:[&]|$)'
  3662.     _TESTS = [{
  3663.         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
  3664.         'playlist_mincount': 5,
  3665.         'info_dict': {
  3666.             'id': 'youtube-dl test video',
  3667.             'title': 'youtube-dl test video',
  3668.         },
  3669.         'params': {'playlistend': 5}
  3670.     }, {
  3671.         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
  3672.         'only_matching': True,
  3673.     }]
  3675.     def _real_extract(self, url):
  3676.         qs = parse_qs(url)
  3677.         query = (qs.get('search_query') or qs.get('q'))[-1]
  3678.         params = qs.get('sp', ('',))[-1]
  3679.         return self.playlist_result(self._search_results(query, params), query, query)
  3682. class YoutubeFeedsInfoExtractor(YoutubeTabIE):
  3683.     """
  3684.     Base class for feed extractors
  3685.     Subclasses must define the _FEED_NAME property.
  3686.     """
  3687.     _LOGIN_REQUIRED = True
  3689.     @property
  3690.     def IE_NAME(self):
  3691.         return 'youtube:%s' % self._FEED_NAME
  3693.     def _real_initialize(self):
  3694.         self._login()
  3696.     def _real_extract(self, url):
  3697.         return self.url_result(
  3698.             'https://www.youtube.com/feed/%s' % self._FEED_NAME,
  3699.             ie=YoutubeTabIE.ie_key())
  3702. class YoutubeWatchLaterIE(InfoExtractor):
  3703.     IE_NAME = 'youtube:watchlater'
  3704.     IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
  3705.     _VALID_URL = r':ytwatchlater'
  3706.     _TESTS = [{
  3707.         'url': ':ytwatchlater',
  3708.         'only_matching': True,
  3709.     }]
  3711.     def _real_extract(self, url):
  3712.         return self.url_result(
  3713.             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
  3716. class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
  3717.     IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
  3718.     _VALID_URL = r':ytrec(?:ommended)?'
  3719.     _FEED_NAME = 'recommended'
  3720.     _TESTS = [{
  3721.         'url': ':ytrec',
  3722.         'only_matching': True,
  3723.     }, {
  3724.         'url': ':ytrecommended',
  3725.         'only_matching': True,
  3726.     }]
  3729. class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
  3730.     IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
  3731.     _VALID_URL = r':ytsubs(?:criptions)?'
  3732.     _FEED_NAME = 'subscriptions'
  3733.     _TESTS = [{
  3734.         'url': ':ytsubs',
  3735.         'only_matching': True,
  3736.     }, {
  3737.         'url': ':ytsubscriptions',
  3738.         'only_matching': True,
  3739.     }]
  3742. class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
  3743.     IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
  3744.     _VALID_URL = r':ythistory'
  3745.     _FEED_NAME = 'history'
  3746.     _TESTS = [{
  3747.         'url': ':ythistory',
  3748.         'only_matching': True,
  3749.     }]
  3752. class YoutubeTruncatedURLIE(InfoExtractor):
  3753.     IE_NAME = 'youtube:truncated_url'
  3754.     IE_DESC = False  # Do not list
  3755.     _VALID_URL = r'''(?x)
  3756.         (?:https?://)?
  3757.         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
  3758.         (?:watch\?(?:
  3759.             feature=[a-z_]+|
  3760.             annotation_id=annotation_[^&]+|
  3761.             x-yt-cl=[0-9]+|
  3762.             hl=[^&]*|
  3763.             t=[0-9]+
  3764.         )?
  3765.         |
  3766.             attribution_link\?a=[^&]+
  3767.         )
  3768.         $
  3769.     '''
  3771.     _TESTS = [{
  3772.         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
  3773.         'only_matching': True,
  3774.     }, {
  3775.         'url': 'https://www.youtube.com/watch?',
  3776.         'only_matching': True,
  3777.     }, {
  3778.         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
  3779.         'only_matching': True,
  3780.     }, {
  3781.         'url': 'https://www.youtube.com/watch?feature=foo',
  3782.         'only_matching': True,
  3783.     }, {
  3784.         'url': 'https://www.youtube.com/watch?hl=en-GB',
  3785.         'only_matching': True,
  3786.     }, {
  3787.         'url': 'https://www.youtube.com/watch?t=2372',
  3788.         'only_matching': True,
  3789.     }]
  3791.     def _real_extract(self, url):
  3792.         raise ExtractorError(
  3793.             'Did you forget to quote the URL? Remember that & is a meta '
  3794.             'character in most shells, so you want to put the URL in quotes, '
  3795.             'like  youtube-dl '
  3796.             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
  3797.             ' or simply  youtube-dl BaW_jenozKc  .',
  3798.             expected=True)
  3801. class YoutubeTruncatedIDIE(InfoExtractor):
  3802.     IE_NAME = 'youtube:truncated_id'
  3803.     IE_DESC = False  # Do not list
  3804.     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
  3806.     _TESTS = [{
  3807.         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
  3808.         'only_matching': True,
  3809.     }]
  3811.     def _real_extract(self, url):
  3812.         video_id = self._match_id(url)
  3813.         raise ExtractorError(
  3814.             'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id, url),
  3815.             expected=True)