logo

youtube-dl

[mirror] Download/Watch videos from video hostersgit clone https://hacktivis.me/git/mirror/youtube-dl.git

youtube.py (185719B)

    

  1. # coding: utf-8
  3. from __future__ import unicode_literals
  5. import collections
  6. import hashlib
  7. import itertools
  8. import json
  9. import os.path
  10. import random
  11. import re
  12. import string
  13. import time
  14. import traceback
  16. from .common import InfoExtractor, SearchInfoExtractor
  17. from ..compat import (
  18.     compat_chr,
  19.     compat_HTTPError,
  20.     compat_map as map,
  21.     compat_str,
  22.     compat_urllib_parse,
  23.     compat_urllib_parse_parse_qs as compat_parse_qs,
  24.     compat_urllib_parse_unquote_plus,
  25.     compat_urllib_parse_urlparse,
  26.     compat_zip as zip,
  27. )
  28. from ..jsinterp import JSInterpreter
  29. from ..utils import (
  30.     bug_reports_message,
  31.     clean_html,
  32.     dict_get,
  33.     error_to_compat_str,
  34.     ExtractorError,
  35.     filter_dict,
  36.     float_or_none,
  37.     get_first,
  38.     extract_attributes,
  39.     get_element_by_attribute,
  40.     int_or_none,
  41.     join_nonempty,
  42.     js_to_json,
  43.     LazyList,
  44.     merge_dicts,
  45.     mimetype2ext,
  46.     NO_DEFAULT,
  47.     parse_codecs,
  48.     parse_count,
  49.     parse_duration,
  50.     parse_qs,
  51.     qualities,
  52.     remove_start,
  53.     smuggle_url,
  54.     str_or_none,
  55.     str_to_int,
  56.     T,
  57.     traverse_obj,
  58.     try_call,
  59.     try_get,
  60.     txt_or_none,
  61.     unescapeHTML,
  62.     unified_strdate,
  63.     unsmuggle_url,
  64.     update_url,
  65.     update_url_query,
  66.     url_or_none,
  67.     urlencode_postdata,
  68.     urljoin,
  69.     variadic,
  70. )
  73. class YoutubeBaseInfoExtractor(InfoExtractor):
  74.     """Provide base functions for Youtube extractors"""
  76.     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
  77.     _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
  79.     _LOOKUP_URL = 'https://accounts.google.com/_/signin/sl/lookup'
  80.     _CHALLENGE_URL = 'https://accounts.google.com/_/signin/sl/challenge'
  81.     _TFA_URL = 'https://accounts.google.com/_/signin/challenge?hl=en&TL={0}'
  83.     _NETRC_MACHINE = 'youtube'
  84.     # If True it will raise an error if no login info is provided
  85.     _LOGIN_REQUIRED = False
  87.     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM)'
  89.     _INNERTUBE_CLIENTS = {
  90.         'ios': {
  91.             'INNERTUBE_CONTEXT': {
  92.                 'client': {
  93.                     'clientName': 'IOS',
  94.                     'clientVersion': '20.10.4',
  95.                     'deviceMake': 'Apple',
  96.                     'deviceModel': 'iPhone16,2',
  97.                     'userAgent': 'com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)',
  98.                     'osName': 'iPhone',
  99.                     'osVersion': '18.3.2.22D82',
  100.                 },
  101.             },
  102.             'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
  103.             'REQUIRE_JS_PLAYER': False,
  104.             'REQUIRE_PO_TOKEN': True,
  105.         },
  106.         # mweb has 'ultralow' formats
  107.         # See: https://github.com/yt-dlp/yt-dlp/pull/557
  108.         'mweb': {
  109.             'INNERTUBE_CONTEXT': {
  110.                 'client': {
  111.                     'clientName': 'MWEB',
  112.                     'clientVersion': '2.20250311.03.00',
  113.                     # mweb previously did not require PO Token with this UA
  114.                     'userAgent': 'Mozilla/5.0 (iPad; CPU OS 16_7_10 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1,gzip(gfe)',
  115.                 },
  116.             },
  117.             'INNERTUBE_CONTEXT_CLIENT_NAME': 2,
  118.             'REQUIRE_PO_TOKEN': True,
  119.             'SUPPORTS_COOKIES': True,
  120.         },
  121.         'tv': {
  122.             'INNERTUBE_CONTEXT': {
  123.                 'client': {
  124.                     'clientName': 'TVHTML5',
  125.                     'clientVersion': '7.20250312.16.00',
  126.                     'userAgent': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/Version',
  127.                 },
  128.             },
  129.             'INNERTUBE_CONTEXT_CLIENT_NAME': 7,
  130.             'SUPPORTS_COOKIES': True,
  131.         },
  132.         'web': {
  133.             'INNERTUBE_CONTEXT': {
  134.                 'client': {
  135.                     'clientName': 'WEB',
  136.                     'clientVersion': '2.20250312.04.00',
  137.                 },
  138.             },
  139.             'INNERTUBE_CONTEXT_CLIENT_NAME': 1,
  140.             'REQUIRE_PO_TOKEN': True,
  141.             'SUPPORTS_COOKIES': True,
  142.         },
  143.     }
  145.     def _login(self):
  146.         """
  147.         Attempt to log in to YouTube.
  149.         True is returned if successful or skipped.
  150.         False is returned if login failed.
  152.         If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
  153.         """
  154.         username, password = self._get_login_info()
  155.         # No authentication to be performed
  156.         if username is None:
  157.             if self._LOGIN_REQUIRED and self._downloader.params.get('cookiefile') is None:
  158.                 raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
  159.             return True
  161.         login_page = self._download_webpage(
  162.             self._LOGIN_URL, None,
  163.             note='Downloading login page',
  164.             errnote='unable to fetch login page', fatal=False)
  165.         if login_page is False:
  166.             return
  168.         login_form = self._hidden_inputs(login_page)
  170.         def req(url, f_req, note, errnote):
  171.             data = login_form.copy()
  172.             data.update({
  173.                 'pstMsg': 1,
  174.                 'checkConnection': 'youtube',
  175.                 'checkedDomains': 'youtube',
  176.                 'hl': 'en',
  177.                 'deviceinfo': '[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]',
  178.                 'f.req': json.dumps(f_req),
  179.                 'flowName': 'GlifWebSignIn',
  180.                 'flowEntry': 'ServiceLogin',
  181.                 # TODO: reverse actual botguard identifier generation algo
  182.                 'bgRequest': '["identifier",""]',
  183.             })
  184.             return self._download_json(
  185.                 url, None, note=note, errnote=errnote,
  186.                 transform_source=lambda s: re.sub(r'^[^[]*', '', s),
  187.                 fatal=False,
  188.                 data=urlencode_postdata(data), headers={
  189.                     'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8',
  190.                     'Google-Accounts-XSRF': 1,
  191.                 })
  193.         def warn(message):
  194.             self._downloader.report_warning(message)
  196.         lookup_req = [
  197.             username,
  198.             None, [], None, 'US', None, None, 2, False, True,
  199.             [
  200.                 None, None,
  201.                 [2, 1, None, 1,
  202.                  'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn',
  203.                  None, [], 4],
  204.                 1, [None, None, []], None, None, None, True,
  205.             ],
  206.             username,
  207.         ]
  209.         lookup_results = req(
  210.             self._LOOKUP_URL, lookup_req,
  211.             'Looking up account info', 'Unable to look up account info')
  213.         if lookup_results is False:
  214.             return False
  216.         user_hash = try_get(lookup_results, lambda x: x[0][2], compat_str)
  217.         if not user_hash:
  218.             warn('Unable to extract user hash')
  219.             return False
  221.         challenge_req = [
  222.             user_hash,
  223.             None, 1, None, [1, None, None, None, [password, None, True]],
  224.             [
  225.                 None, None, [2, 1, None, 1, 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn', None, [], 4],
  226.                 1, [None, None, []], None, None, None, True,
  227.             ]]
  229.         challenge_results = req(
  230.             self._CHALLENGE_URL, challenge_req,
  231.             'Logging in', 'Unable to log in')
  233.         if challenge_results is False:
  234.             return
  236.         login_res = try_get(challenge_results, lambda x: x[0][5], list)
  237.         if login_res:
  238.             login_msg = try_get(login_res, lambda x: x[5], compat_str)
  239.             warn(
  240.                 'Unable to login: %s' % 'Invalid password'
  241.                 if login_msg == 'INCORRECT_ANSWER_ENTERED' else login_msg)
  242.             return False
  244.         res = try_get(challenge_results, lambda x: x[0][-1], list)
  245.         if not res:
  246.             warn('Unable to extract result entry')
  247.             return False
  249.         login_challenge = try_get(res, lambda x: x[0][0], list)
  250.         if login_challenge:
  251.             challenge_str = try_get(login_challenge, lambda x: x[2], compat_str)
  252.             if challenge_str == 'TWO_STEP_VERIFICATION':
  253.                 # SEND_SUCCESS - TFA code has been successfully sent to phone
  254.                 # QUOTA_EXCEEDED - reached the limit of TFA codes
  255.                 status = try_get(login_challenge, lambda x: x[5], compat_str)
  256.                 if status == 'QUOTA_EXCEEDED':
  257.                     warn('Exceeded the limit of TFA codes, try later')
  258.                     return False
  260.                 tl = try_get(challenge_results, lambda x: x[1][2], compat_str)
  261.                 if not tl:
  262.                     warn('Unable to extract TL')
  263.                     return False
  265.                 tfa_code = self._get_tfa_info('2-step verification code')
  267.                 if not tfa_code:
  268.                     warn(
  269.                         'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
  270.                         '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
  271.                     return False
  273.                 tfa_code = remove_start(tfa_code, 'G-')
  275.                 tfa_req = [
  276.                     user_hash, None, 2, None,
  277.                     [
  278.                         9, None, None, None, None, None, None, None,
  279.                         [None, tfa_code, True, 2],
  280.                     ]]
  282.                 tfa_results = req(
  283.                     self._TFA_URL.format(tl), tfa_req,
  284.                     'Submitting TFA code', 'Unable to submit TFA code')
  286.                 if tfa_results is False:
  287.                     return False
  289.                 tfa_res = try_get(tfa_results, lambda x: x[0][5], list)
  290.                 if tfa_res:
  291.                     tfa_msg = try_get(tfa_res, lambda x: x[5], compat_str)
  292.                     warn(
  293.                         'Unable to finish TFA: %s' % 'Invalid TFA code'
  294.                         if tfa_msg == 'INCORRECT_ANSWER_ENTERED' else tfa_msg)
  295.                     return False
  297.                 check_cookie_url = try_get(
  298.                     tfa_results, lambda x: x[0][-1][2], compat_str)
  299.             else:
  300.                 CHALLENGES = {
  301.                     'LOGIN_CHALLENGE': "This device isn't recognized. For your security, Google wants to make sure it's really you.",
  302.                     'USERNAME_RECOVERY': 'Please provide additional information to aid in the recovery process.',
  303.                     'REAUTH': "There is something unusual about your activity. For your security, Google wants to make sure it's really you.",
  304.                 }
  305.                 challenge = CHALLENGES.get(
  306.                     challenge_str,
  307.                     '%s returned error %s.' % (self.IE_NAME, challenge_str))
  308.                 warn('%s\nGo to https://accounts.google.com/, login and solve a challenge.' % challenge)
  309.                 return False
  310.         else:
  311.             check_cookie_url = try_get(res, lambda x: x[2], compat_str)
  313.         if not check_cookie_url:
  314.             warn('Unable to extract CheckCookie URL')
  315.             return False
  317.         check_cookie_results = self._download_webpage(
  318.             check_cookie_url, None, 'Checking cookie', fatal=False)
  320.         if check_cookie_results is False:
  321.             return False
  323.         if 'https://myaccount.google.com/' not in check_cookie_results:
  324.             warn('Unable to log in')
  325.             return False
  327.         return True
  329.     def _initialize_consent(self):
  330.         cookies = self._get_cookies('https://www.youtube.com/')
  331.         if cookies.get('__Secure-3PSID'):
  332.             return
  333.         socs = cookies.get('SOCS')
  334.         if socs and not socs.value.startswith('CAA'):  # not consented
  335.             return
  336.         self._set_cookie('.youtube.com', 'SOCS', 'CAI', secure=True)  # accept all (required for mixes)
  338.     def _real_initialize(self):
  339.         self._initialize_consent()
  340.         if self._downloader is None:
  341.             return
  342.         if not self._login():
  343.             return
  345.     _DEFAULT_API_DATA = {'context': _INNERTUBE_CLIENTS['web']['INNERTUBE_CONTEXT']}
  347.     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*=\s*({.+?})\s*;'
  348.     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
  349.     _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
  351.     _SAPISID = None
  353.     def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
  354.         time_now = round(time.time())
  355.         if self._SAPISID is None:
  356.             yt_cookies = self._get_cookies('https://www.youtube.com')
  357.             # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
  358.             # See: https://github.com/yt-dlp/yt-dlp/issues/393
  359.             sapisid_cookie = dict_get(
  360.                 yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
  361.             if sapisid_cookie and sapisid_cookie.value:
  362.                 self._SAPISID = sapisid_cookie.value
  363.                 self.write_debug('Extracted SAPISID cookie')
  364.                 # SAPISID cookie is required if not already present
  365.                 if not yt_cookies.get('SAPISID'):
  366.                     self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
  367.                     self._set_cookie(
  368.                         '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
  369.             else:
  370.                 self._SAPISID = False
  371.         if not self._SAPISID:
  372.             return None
  373.         # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
  374.         sapisidhash = hashlib.sha1(
  375.             '{0} {1} {2}'.format(time_now, self._SAPISID, origin).encode('utf-8')).hexdigest()
  376.         return 'SAPISIDHASH {0}_{1}'.format(time_now, sapisidhash)
  378.     def _call_api(self, ep, query, video_id, fatal=True, headers=None,
  379.                   note='Downloading API JSON'):
  380.         data = self._DEFAULT_API_DATA.copy()
  381.         data.update(query)
  382.         real_headers = {'content-type': 'application/json'}
  383.         if headers:
  384.             real_headers.update(headers)
  386.         # was: 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'
  387.         api_key = self.get_param('youtube_innertube_key')
  388.         return self._download_json(
  389.             'https://www.youtube.com/youtubei/v1/%s' % ep, video_id=video_id,
  390.             note=note, errnote='Unable to download API page',
  391.             data=json.dumps(data).encode('utf8'), fatal=fatal,
  392.             headers=real_headers, query=filter_dict({
  393.                 'key': api_key,
  394.                 'prettyPrint': 'false',
  395.             }))
  397.     def _extract_yt_initial_data(self, video_id, webpage):
  398.         return self._parse_json(
  399.             self._search_regex(
  400.                 (r'%s\s*%s' % (self._YT_INITIAL_DATA_RE, self._YT_INITIAL_BOUNDARY_RE),
  401.                  self._YT_INITIAL_DATA_RE), webpage, 'yt initial data'),
  402.             video_id)
  404.     def _extract_visitor_data(self, *args):
  405.         """
  406.         Extract visitorData from an API response or ytcfg
  408.         Appears to be used to track session state
  409.         """
  410.         visitor_data = self.get_param('youtube_visitor_data')
  411.         if visitor_data:
  412.             return visitor_data
  414.         return get_first(
  415.             args, (('VISITOR_DATA',
  416.                     ('INNERTUBE_CONTEXT', 'client', 'visitorData'),
  417.                     ('responseContext', 'visitorData')),
  418.                    T(compat_str)))
  420.     def _extract_ytcfg(self, video_id, webpage):
  421.         return self._parse_json(
  422.             self._search_regex(
  423.                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
  424.                 default='{}'), video_id, fatal=False) or {}
  426.     def _extract_video(self, renderer):
  427.         video_id = renderer['videoId']
  428.         title = try_get(
  429.             renderer,
  430.             (lambda x: x['title']['runs'][0]['text'],
  431.              lambda x: x['title']['simpleText'],
  432.              lambda x: x['headline']['simpleText']), compat_str)
  433.         description = try_get(
  434.             renderer, lambda x: x['descriptionSnippet']['runs'][0]['text'],
  435.             compat_str)
  436.         duration = parse_duration(try_get(
  437.             renderer, lambda x: x['lengthText']['simpleText'], compat_str))
  438.         view_count_text = try_get(
  439.             renderer, lambda x: x['viewCountText']['simpleText'], compat_str) or ''
  440.         view_count = str_to_int(self._search_regex(
  441.             r'^([\d,]+)', re.sub(r'\s', '', view_count_text),
  442.             'view count', default=None))
  443.         uploader = try_get(
  444.             renderer,
  445.             (lambda x: x['ownerText']['runs'][0]['text'],
  446.              lambda x: x['shortBylineText']['runs'][0]['text']), compat_str)
  447.         return {
  448.             '_type': 'url',
  449.             'ie_key': YoutubeIE.ie_key(),
  450.             'id': video_id,
  451.             'url': video_id,
  452.             'title': title,
  453.             'description': description,
  454.             'duration': duration,
  455.             'view_count': view_count,
  456.             'uploader': uploader,
  457.         }
  459.     @staticmethod
  460.     def _extract_thumbnails(data, *path_list, **kw_final_key):
  461.         """
  462.         Extract thumbnails from thumbnails dict
  463.         @param path_list: path list to level that contains 'thumbnails' key
  464.         """
  465.         final_key = kw_final_key.get('final_key', 'thumbnails')
  467.         return traverse_obj(data, ((
  468.             tuple(variadic(path) + (final_key, Ellipsis)
  469.                   for path in path_list or [()])), {
  470.             'url': ('url', T(url_or_none),
  471.                     # Sometimes youtube gives a wrong thumbnail URL. See:
  472.                     # https://github.com/yt-dlp/yt-dlp/issues/233
  473.                     # https://github.com/ytdl-org/youtube-dl/issues/28023
  474.                     T(lambda u: update_url(u, query=None) if u and 'maxresdefault' in u else u)),
  475.             'height': ('height', T(int_or_none)),
  476.             'width': ('width', T(int_or_none)),
  477.         }, T(lambda t: t if t.get('url') else None)))
  479.     def _search_results(self, query, params):
  480.         data = {
  481.             'context': {
  482.                 'client': {
  483.                     'clientName': 'WEB',
  484.                     'clientVersion': '2.20201021.03.00',
  485.                 },
  486.             },
  487.             'query': query,
  488.         }
  489.         if params:
  490.             data['params'] = params
  491.         for page_num in itertools.count(1):
  492.             search = self._download_json(
  493.                 'https://www.youtube.com/youtubei/v1/search',
  494.                 video_id='query "%s"' % query,
  495.                 note='Downloading page %s' % page_num,
  496.                 errnote='Unable to download API page', fatal=False,
  497.                 data=json.dumps(data).encode('utf8'),
  498.                 query={
  499.                     # 'key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  500.                     'prettyPrint': 'false',
  501.                 },
  502.                 headers={'content-type': 'application/json'})
  503.             if not search:
  504.                 break
  505.             slr_contents = try_get(
  506.                 search,
  507.                 (lambda x: x['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents'],
  508.                  lambda x: x['onResponseReceivedCommands'][0]['appendContinuationItemsAction']['continuationItems']),
  509.                 list)
  510.             if not slr_contents:
  511.                 break
  512.             for slr_content in slr_contents:
  513.                 isr_contents = try_get(
  514.                     slr_content,
  515.                     lambda x: x['itemSectionRenderer']['contents'],
  516.                     list)
  517.                 if not isr_contents:
  518.                     continue
  519.                 for content in isr_contents:
  520.                     if not isinstance(content, dict):
  521.                         continue
  522.                     video = content.get('videoRenderer')
  523.                     if not isinstance(video, dict):
  524.                         continue
  525.                     video_id = video.get('videoId')
  526.                     if not video_id:
  527.                         continue
  528.                     yield self._extract_video(video)
  529.             token = try_get(
  530.                 slr_contents,
  531.                 lambda x: x[-1]['continuationItemRenderer']['continuationEndpoint']['continuationCommand']['token'],
  532.                 compat_str)
  533.             if not token:
  534.                 break
  535.             data['continuation'] = token
  537.     @staticmethod
  538.     def _owner_endpoints_path():
  539.         return [
  540.             Ellipsis,
  541.             lambda k, _: k.endswith('SecondaryInfoRenderer'),
  542.             ('owner', 'videoOwner'), 'videoOwnerRenderer', 'title',
  543.             'runs', Ellipsis]
  545.     def _extract_channel_id(self, webpage, videodetails={}, metadata={}, renderers=[]):
  546.         channel_id = None
  547.         if any((videodetails, metadata, renderers)):
  548.             channel_id = (
  549.                 traverse_obj(videodetails, 'channelId')
  550.                 or traverse_obj(metadata, 'externalChannelId', 'externalId')
  551.                 or traverse_obj(renderers,
  552.                                 self._owner_endpoints_path() + [
  553.                                     'navigationEndpoint', 'browseEndpoint', 'browseId'],
  554.                                 get_all=False)
  555.             )
  556.         return channel_id or self._html_search_meta(
  557.             'channelId', webpage, 'channel id', default=None)
  559.     def _extract_author_var(self, webpage, var_name,
  560.                             videodetails={}, metadata={}, renderers=[]):
  561.         result = None
  562.         paths = {
  563.             #       (HTML, videodetails, metadata, renderers)
  564.             'name': ('content', 'author', (('ownerChannelName', None), 'title'), ['text']),
  565.             'url': ('href', 'ownerProfileUrl', 'vanityChannelUrl',
  566.                     ['navigationEndpoint', 'browseEndpoint', 'canonicalBaseUrl']),
  567.         }
  568.         if any((videodetails, metadata, renderers)):
  569.             result = (
  570.                 traverse_obj(videodetails, paths[var_name][1], get_all=False)
  571.                 or traverse_obj(metadata, paths[var_name][2], get_all=False)
  572.                 or traverse_obj(renderers,
  573.                                 self._owner_endpoints_path() + paths[var_name][3],
  574.                                 get_all=False)
  575.             )
  576.         return result or traverse_obj(
  577.             extract_attributes(self._search_regex(
  578.                 r'''(?s)(<link\b[^>]+\bitemprop\s*=\s*("|')%s\2[^>]*>)'''
  579.                 % re.escape(var_name),
  580.                 get_element_by_attribute('itemprop', 'author', webpage or '') or '',
  581.                 'author link', default='')),
  582.             paths[var_name][0])
  584.     @staticmethod
  585.     def _yt_urljoin(url_or_path):
  586.         return urljoin('https://www.youtube.com', url_or_path)
  588.     def _extract_uploader_id(self, uploader_url):
  589.         return self._search_regex(
  590.             r'/(?:(?:channel|user)/|(?=@))([^/?&#]+)', uploader_url or '',
  591.             'uploader id', default=None)
  594. class YoutubeIE(YoutubeBaseInfoExtractor):
  595.     IE_DESC = 'YouTube.com'
  596.     _INVIDIOUS_SITES = (
  597.         # invidious-redirect websites
  598.         r'(?:www\.)?redirect\.invidious\.io',
  599.         r'(?:(?:www|dev)\.)?invidio\.us',
  600.         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/Invidious-Instances.md
  601.         r'(?:(?:www|no)\.)?invidiou\.sh',
  602.         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
  603.         r'(?:www\.)?invidious\.kabi\.tk',
  604.         r'(?:www\.)?invidious\.13ad\.de',
  605.         r'(?:www\.)?invidious\.mastodon\.host',
  606.         r'(?:www\.)?invidious\.zapashcanon\.fr',
  607.         r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
  608.         r'(?:www\.)?invidious\.tinfoil-hat\.net',
  609.         r'(?:www\.)?invidious\.himiko\.cloud',
  610.         r'(?:www\.)?invidious\.reallyancient\.tech',
  611.         r'(?:www\.)?invidious\.tube',
  612.         r'(?:www\.)?invidiou\.site',
  613.         r'(?:www\.)?invidious\.site',
  614.         r'(?:www\.)?invidious\.xyz',
  615.         r'(?:www\.)?invidious\.nixnet\.xyz',
  616.         r'(?:www\.)?invidious\.048596\.xyz',
  617.         r'(?:www\.)?invidious\.drycat\.fr',
  618.         r'(?:www\.)?inv\.skyn3t\.in',
  619.         r'(?:www\.)?tube\.poal\.co',
  620.         r'(?:www\.)?tube\.connect\.cafe',
  621.         r'(?:www\.)?vid\.wxzm\.sx',
  622.         r'(?:www\.)?vid\.mint\.lgbt',
  623.         r'(?:www\.)?vid\.puffyan\.us',
  624.         r'(?:www\.)?yewtu\.be',
  625.         r'(?:www\.)?yt\.elukerio\.org',
  626.         r'(?:www\.)?yt\.lelux\.fi',
  627.         r'(?:www\.)?invidious\.ggc-project\.de',
  628.         r'(?:www\.)?yt\.maisputain\.ovh',
  629.         r'(?:www\.)?ytprivate\.com',
  630.         r'(?:www\.)?invidious\.13ad\.de',
  631.         r'(?:www\.)?invidious\.toot\.koeln',
  632.         r'(?:www\.)?invidious\.fdn\.fr',
  633.         r'(?:www\.)?watch\.nettohikari\.com',
  634.         r'(?:www\.)?invidious\.namazso\.eu',
  635.         r'(?:www\.)?invidious\.silkky\.cloud',
  636.         r'(?:www\.)?invidious\.exonip\.de',
  637.         r'(?:www\.)?invidious\.riverside\.rocks',
  638.         r'(?:www\.)?invidious\.blamefran\.net',
  639.         r'(?:www\.)?invidious\.moomoo\.de',
  640.         r'(?:www\.)?ytb\.trom\.tf',
  641.         r'(?:www\.)?yt\.cyberhost\.uk',
  642.         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
  643.         r'(?:www\.)?qklhadlycap4cnod\.onion',
  644.         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
  645.         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
  646.         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
  647.         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
  648.         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
  649.         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
  650.         r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
  651.         r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
  652.         r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
  653.         r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
  654.     )
  655.     _VALID_URL = r"""(?x)^
  656.                      (
  657.                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
  658.                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
  659.                             (?:www\.)?deturl\.com/www\.youtube\.com|
  660.                             (?:www\.)?pwnyoutube\.com|
  661.                             (?:www\.)?hooktube\.com|
  662.                             (?:www\.)?yourepeat\.com|
  663.                             tube\.majestyc\.net|
  664.                             %(invidious)s|
  665.                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
  666.                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
  667.                          (?:                                                  # the various things that can precede the ID:
  668.                              (?:(?:v|embed|e)/(?!videoseries))                # v/ or embed/ or e/
  669.                              |shorts/
  670.                              |(?:                                             # or the v= param in all its forms
  671.                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
  672.                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
  673.                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
  674.                                  v=
  675.                              )
  676.                          ))
  677.                          |(?:
  678.                             youtu\.be|                                        # just youtu.be/xxxx
  679.                             vid\.plus|                                        # or vid.plus/xxxx
  680.                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
  681.                             %(invidious)s
  682.                          )/
  683.                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
  684.                          )
  685.                      )?                                                       # all until now is optional -> you can pass the naked ID
  686.                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
  687.                      (?(1).+)?                                                # if we found the ID, everything can follow
  688.                      $""" % {
  689.         'invidious': '|'.join(_INVIDIOUS_SITES),
  690.     }
  691.     _PLAYER_INFO_RE = (
  692.         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/(?:tv-)?player',
  693.         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias(?:_tce)?\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
  694.         r'\b(?P<id>vfl[a-zA-Z0-9_-]{6,})\b.*?\.js$',
  695.     )
  696.     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
  698.     _GEO_BYPASS = False
  700.     IE_NAME = 'youtube'
  701.     _TESTS = [
  702.         {
  703.             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
  704.             'info_dict': {
  705.                 'id': 'BaW_jenozKc',
  706.                 'ext': 'mp4',
  707.                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
  708.                 'uploader': 'Philipp Hagemeister',
  709.                 'uploader_id': '@PhilippHagemeister',
  710.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@PhilippHagemeister',
  711.                 'channel': 'Philipp Hagemeister',
  712.                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
  713.                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
  714.                 'upload_date': '20121002',
  715.                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
  716.                 'categories': ['Science & Technology'],
  717.                 'tags': ['youtube-dl'],
  718.                 'duration': 10,
  719.                 'view_count': int,
  720.                 'like_count': int,
  721.                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
  722.                 'start_time': 1,
  723.                 'end_time': 9,
  724.             },
  725.         },
  726.         {
  727.             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
  728.             'note': 'Embed-only video (#1746)',
  729.             'info_dict': {
  730.                 'id': 'yZIXLfi8CZQ',
  731.                 'ext': 'mp4',
  732.                 'upload_date': '20120608',
  733.                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
  734.                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
  735.                 'uploader': 'SET India',
  736.                 'uploader_id': 'setindia',
  737.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
  738.                 'age_limit': 18,
  739.             },
  740.             'skip': 'Private video',
  741.         },
  742.         {
  743.             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
  744.             'note': 'Use the first video ID in the URL',
  745.             'info_dict': {
  746.                 'id': 'BaW_jenozKc',
  747.                 'ext': 'mp4',
  748.                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
  749.                 'uploader': 'Philipp Hagemeister',
  750.                 'uploader_id': '@PhilippHagemeister',
  751.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@PhilippHagemeister',
  752.                 'upload_date': '20121002',
  753.                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
  754.                 'categories': ['Science & Technology'],
  755.                 'tags': ['youtube-dl'],
  756.                 'duration': 10,
  757.                 'view_count': int,
  758.                 'like_count': int,
  759.             },
  760.             'params': {
  761.                 'skip_download': True,
  762.             },
  763.         },
  764.         {
  765.             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
  766.             'note': '256k DASH audio (format 141) via DASH manifest',
  767.             'info_dict': {
  768.                 'id': 'a9LDPn-MO4I',
  769.                 'ext': 'm4a',
  770.                 'upload_date': '20121002',
  771.                 'uploader_id': '8KVIDEO',
  772.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
  773.                 'description': '',
  774.                 'uploader': '8KVIDEO',
  775.                 'title': 'UHDTV TEST 8K VIDEO.mp4',
  776.             },
  777.             'params': {
  778.                 'youtube_include_dash_manifest': True,
  779.                 'format': '141',
  780.             },
  781.             'skip': 'format 141 not served any more',
  782.         },
  783.         # DASH manifest with encrypted signature
  784.         {
  785.             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
  786.             'info_dict': {
  787.                 'id': 'IB3lcPjvWLA',
  788.                 'ext': 'm4a',
  789.                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
  790.                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
  791.                 'duration': 244,
  792.                 'uploader': 'AfrojackVEVO',
  793.                 'uploader_id': '@AfrojackVEVO',
  794.                 'upload_date': '20131011',
  795.                 'abr': 129.495,
  796.             },
  797.             'params': {
  798.                 'youtube_include_dash_manifest': True,
  799.                 'format': '141/bestaudio[ext=m4a]',
  800.             },
  801.         },
  802.         # Controversy video
  803.         {
  804.             'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8',
  805.             'info_dict': {
  806.                 'id': 'T4XJQO3qol8',
  807.                 'ext': 'mp4',
  808.                 'duration': 219,
  809.                 'upload_date': '20100909',
  810.                 'uploader': 'Amazing Atheist',
  811.                 'uploader_id': '@theamazingatheist',
  812.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@theamazingatheist',
  813.                 'title': 'Burning Everyone\'s Koran',
  814.                 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms \r\n\r\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
  815.             },
  816.         },
  817.         # Age-gated videos
  818.         {
  819.             'note': 'Age-gated video (No vevo, embed allowed)',
  820.             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
  821.             'info_dict': {
  822.                 'id': 'HtVdAasjOgU',
  823.                 'ext': 'mp4',
  824.                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
  825.                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
  826.                 'duration': 142,
  827.                 'uploader': 'The Witcher',
  828.                 'uploader_id': '@thewitcher',
  829.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@thewitcher',
  830.                 'upload_date': '20140605',
  831.                 'thumbnail': 'https://i.ytimg.com/vi/HtVdAasjOgU/maxresdefault.jpg',
  832.                 'age_limit': 18,
  833.                 'categories': ['Gaming'],
  834.                 'tags': 'count:17',
  835.                 'channel': 'The Witcher',
  836.                 'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
  837.                 'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
  838.                 'view_count': int,
  839.                 'like_count': int,
  840.             },
  841.         },
  842.         {
  843.             'note': 'Age-gated video with embed allowed in public site',
  844.             'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
  845.             'info_dict': {
  846.                 'id': 'HsUATh_Nc2U',
  847.                 'ext': 'mp4',
  848.                 'title': 'Godzilla 2 (Official Video)',
  849.                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
  850.                 'duration': 177,
  851.                 'uploader': 'FlyingKitty',
  852.                 'uploader_id': '@FlyingKitty900',
  853.                 'upload_date': '20200408',
  854.                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
  855.                 'age_limit': 18,
  856.                 'categories': ['Entertainment'],
  857.                 'tags': ['Flyingkitty', 'godzilla 2'],
  858.                 'channel': 'FlyingKitty',
  859.                 'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
  860.                 'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
  861.                 'view_count': int,
  862.                 'like_count': int,
  863.             },
  864.         },
  865.         {
  866.             'note': 'Age-gated video embeddable only with clientScreen=EMBED',
  867.             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
  868.             'info_dict': {
  869.                 'id': 'Tq92D6wQ1mg',
  870.                 'ext': 'mp4',
  871.                 'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
  872.                 'description': 'md5:17eccca93a786d51bc67646756894066',
  873.                 'duration': 106,
  874.                 'uploader': 'Projekt Melody',
  875.                 'uploader_id': '@ProjektMelody',
  876.                 'upload_date': '20191227',
  877.                 'age_limit': 18,
  878.                 'thumbnail': 'https://i.ytimg.com/vi/Tq92D6wQ1mg/sddefault.jpg',
  879.                 'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
  880.                 'categories': ['Entertainment'],
  881.                 'channel': 'Projekt Melody',
  882.                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
  883.                 'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
  884.                 'view_count': int,
  885.                 'like_count': int,
  886.             },
  887.         },
  888.         {
  889.             'note': 'Non-Age-gated non-embeddable video',
  890.             'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
  891.             'info_dict': {
  892.                 'id': 'MeJVWBSsPAY',
  893.                 'ext': 'mp4',
  894.                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
  895.                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
  896.                 'duration': 210,
  897.                 'upload_date': '20130730',
  898.                 'uploader': 'Herr Lurik',
  899.                 'uploader_id': '@HerrLurik',
  900.                 'uploader_url': 'http://www.youtube.com/@HerrLurik',
  901.                 'age_limit': 0,
  902.                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/hqdefault.jpg',
  903.                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
  904.                 'categories': ['Music'],
  905.                 'channel': 'Herr Lurik',
  906.                 'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
  907.                 'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
  908.                 'artist': 'OOMPH!',
  909.                 'view_count': int,
  910.                 'like_count': int,
  911.             },
  912.         },
  913.         {
  914.             'note': 'Non-bypassable age-gated video',
  915.             'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
  916.             'only_matching': True,
  917.         },
  918.         {
  919.             'note': 'Age-gated video only available with authentication (not via embed workaround)',
  920.             'url': 'XgnwCQzjau8',
  921.             'only_matching': True,
  922.             'skip': '''This video has been removed for violating YouTube's Community Guidelines''',
  923.         },
  924.         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
  925.         # YouTube Red ad is not captured for creator
  926.         {
  927.             'url': '__2ABJjxzNo',
  928.             'info_dict': {
  929.                 'id': '__2ABJjxzNo',
  930.                 'ext': 'mp4',
  931.                 'duration': 266,
  932.                 'upload_date': '20100430',
  933.                 'uploader_id': '@deadmau5',
  934.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@deadmau5',
  935.                 'creator': 'deadmau5',
  936.                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
  937.                 'uploader': 'deadmau5',
  938.                 'title': 'Deadmau5 - Some Chords (HD)',
  939.                 'alt_title': 'Some Chords',
  940.             },
  941.             'expected_warnings': [
  942.                 'DASH manifest missing',
  943.             ],
  944.         },
  945.         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
  946.         {
  947.             'url': 'lqQg6PlCWgI',
  948.             'info_dict': {
  949.                 'id': 'lqQg6PlCWgI',
  950.                 'ext': 'mp4',
  951.                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
  952.                 'description': r're:(?s)(?:.+\s)?HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games\s*',
  953.                 'duration': 6085,
  954.                 'upload_date': '20150827',
  955.                 'uploader_id': '@Olympics',
  956.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@Olympics',
  957.                 'uploader': r're:Olympics?',
  958.                 'age_limit': 0,
  959.                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
  960.                 'categories': ['Sports'],
  961.                 'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
  962.                 'channel': 'Olympics',
  963.                 'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
  964.                 'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
  965.                 'view_count': int,
  966.                 'like_count': int,
  967.             },
  968.         },
  969.         # Non-square pixels
  970.         {
  971.             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
  972.             'info_dict': {
  973.                 'id': '_b-2C3KPAM0',
  974.                 'ext': 'mp4',
  975.                 'stretched_ratio': 16 / 9.,
  976.                 'duration': 85,
  977.                 'upload_date': '20110310',
  978.                 'uploader_id': '@AllenMeow',
  979.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@AllenMeow',
  980.                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
  981.                 'uploader': '孫ᄋᄅ',
  982.                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
  983.             },
  984.         },
  985.         # url_encoded_fmt_stream_map is empty string
  986.         {
  987.             'url': 'qEJwOuvDf7I',
  988.             'info_dict': {
  989.                 'id': 'qEJwOuvDf7I',
  990.                 'ext': 'webm',
  991.                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
  992.                 'description': '',
  993.                 'upload_date': '20150404',
  994.                 'uploader_id': 'spbelect',
  995.                 'uploader': 'Наблюдатели Петербурга',
  996.             },
  997.             'params': {
  998.                 'skip_download': 'requires avconv',
  999.             },
  1000.             'skip': 'This live event has ended.',
  1001.         },
  1002.         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
  1003.         {
  1004.             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
  1005.             'info_dict': {
  1006.                 'id': 'FIl7x6_3R5Y',
  1007.                 'ext': 'webm',
  1008.                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
  1009.                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
  1010.                 'duration': 220,
  1011.                 'upload_date': '20150625',
  1012.                 'uploader_id': 'dorappi2000',
  1013.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
  1014.                 'uploader': 'dorappi2000',
  1015.                 'formats': 'mincount:31',
  1016.             },
  1017.             'skip': 'not actual any more',
  1018.         },
  1019.         # DASH manifest with segment_list
  1020.         {
  1021.             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
  1022.             'md5': '8ce563a1d667b599d21064e982ab9e31',
  1023.             'info_dict': {
  1024.                 'id': 'CsmdDsKjzN8',
  1025.                 'ext': 'mp4',
  1026.                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
  1027.                 'uploader': 'Airtek',
  1028.                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
  1029.                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
  1030.                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
  1031.             },
  1032.             'params': {
  1033.                 'youtube_include_dash_manifest': True,
  1034.                 'format': '135',  # bestvideo
  1035.             },
  1036.             'skip': 'This live event has ended.',
  1037.         },
  1038.         {
  1039.             # Multifeed videos (multiple cameras), URL is for Main Camera
  1040.             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
  1041.             'info_dict': {
  1042.                 'id': 'jvGDaLqkpTg',
  1043.                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
  1044.                 'description': 'md5:e03b909557865076822aa169218d6a5d',
  1045.             },
  1046.             'playlist': [{
  1047.                 'info_dict': {
  1048.                     'id': 'jvGDaLqkpTg',
  1049.                     'ext': 'mp4',
  1050.                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
  1051.                     'description': 'md5:e03b909557865076822aa169218d6a5d',
  1052.                     'duration': 10643,
  1053.                     'upload_date': '20161111',
  1054.                     'uploader': 'Team PGP',
  1055.                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
  1056.                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
  1057.                 },
  1058.             }, {
  1059.                 'info_dict': {
  1060.                     'id': '3AKt1R1aDnw',
  1061.                     'ext': 'mp4',
  1062.                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
  1063.                     'description': 'md5:e03b909557865076822aa169218d6a5d',
  1064.                     'duration': 10991,
  1065.                     'upload_date': '20161111',
  1066.                     'uploader': 'Team PGP',
  1067.                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
  1068.                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
  1069.                 },
  1070.             }, {
  1071.                 'info_dict': {
  1072.                     'id': 'RtAMM00gpVc',
  1073.                     'ext': 'mp4',
  1074.                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
  1075.                     'description': 'md5:e03b909557865076822aa169218d6a5d',
  1076.                     'duration': 10995,
  1077.                     'upload_date': '20161111',
  1078.                     'uploader': 'Team PGP',
  1079.                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
  1080.                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
  1081.                 },
  1082.             }, {
  1083.                 'info_dict': {
  1084.                     'id': '6N2fdlP3C5U',
  1085.                     'ext': 'mp4',
  1086.                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
  1087.                     'description': 'md5:e03b909557865076822aa169218d6a5d',
  1088.                     'duration': 10990,
  1089.                     'upload_date': '20161111',
  1090.                     'uploader': 'Team PGP',
  1091.                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
  1092.                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
  1093.                 },
  1094.             }],
  1095.             'params': {
  1096.                 'skip_download': True,
  1097.             },
  1098.             'skip': 'Not multifeed any more',
  1099.         },
  1100.         {
  1101.             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
  1102.             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
  1103.             'info_dict': {
  1104.                 'id': 'gVfLd0zydlo',
  1105.                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
  1106.             },
  1107.             'playlist_count': 2,
  1108.             'skip': 'Not multifeed any more',
  1109.         },
  1110.         {
  1111.             'url': 'https://vid.plus/FlRa-iH7PGw',
  1112.             'only_matching': True,
  1113.         },
  1114.         {
  1115.             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
  1116.             'only_matching': True,
  1117.         },
  1118.         {
  1119.             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
  1120.             # Also tests cut-off URL expansion in video description (see
  1121.             # https://github.com/ytdl-org/youtube-dl/issues/1892,
  1122.             # https://github.com/ytdl-org/youtube-dl/issues/8164)
  1123.             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
  1124.             'info_dict': {
  1125.                 'id': 'lsguqyKfVQg',
  1126.                 'ext': 'mp4',
  1127.                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
  1128.                 'alt_title': 'Dark Walk',
  1129.                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
  1130.                 'duration': 133,
  1131.                 'upload_date': '20151119',
  1132.                 'uploader_id': '@IronSoulElf',
  1133.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@IronSoulElf',
  1134.                 'uploader': 'IronSoulElf',
  1135.                 'creator': r're:Todd Haberman[;,]\s+Daniel Law Heath and Aaron Kaplan',
  1136.                 'track': 'Dark Walk',
  1137.                 'artist': r're:Todd Haberman[;,]\s+Daniel Law Heath and Aaron Kaplan',
  1138.                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
  1139.             },
  1140.             'params': {
  1141.                 'skip_download': True,
  1142.             },
  1143.         },
  1144.         {
  1145.             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
  1146.             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
  1147.             'only_matching': True,
  1148.         },
  1149.         {
  1150.             # Video with yt:stretch=17:0
  1151.             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
  1152.             'info_dict': {
  1153.                 'id': 'Q39EVAstoRM',
  1154.                 'ext': 'mp4',
  1155.                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
  1156.                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
  1157.                 'upload_date': '20151107',
  1158.                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
  1159.                 'uploader': 'CH GAMER DROID',
  1160.             },
  1161.             'params': {
  1162.                 'skip_download': True,
  1163.             },
  1164.             'skip': 'This video does not exist.',
  1165.         },
  1166.         {
  1167.             # Video with incomplete 'yt:stretch=16:'
  1168.             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
  1169.             'only_matching': True,
  1170.         },
  1171.         {
  1172.             # Video licensed under Creative Commons
  1173.             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
  1174.             'info_dict': {
  1175.                 'id': 'M4gD1WSo5mA',
  1176.                 'ext': 'mp4',
  1177.                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
  1178.                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
  1179.                 'duration': 721,
  1180.                 'upload_date': '20150127',
  1181.                 'uploader_id': '@BKCHarvard',
  1182.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@BKCHarvard',
  1183.                 'uploader': 'The Berkman Klein Center for Internet & Society',
  1184.                 'license': 'Creative Commons Attribution license (reuse allowed)',
  1185.             },
  1186.             'params': {
  1187.                 'skip_download': True,
  1188.             },
  1189.         },
  1190.         {
  1191.             # Channel-like uploader_url
  1192.             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
  1193.             'info_dict': {
  1194.                 'id': 'eQcmzGIKrzg',
  1195.                 'ext': 'mp4',
  1196.                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
  1197.                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
  1198.                 'duration': 4060,
  1199.                 'upload_date': '20151119',
  1200.                 'uploader': 'Bernie Sanders',
  1201.                 'uploader_id': '@BernieSanders',
  1202.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@BernieSanders',
  1203.                 'license': 'Creative Commons Attribution license (reuse allowed)',
  1204.             },
  1205.             'params': {
  1206.                 'skip_download': True,
  1207.             },
  1208.         },
  1209.         {
  1210.             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
  1211.             'only_matching': True,
  1212.         },
  1213.         {
  1214.             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
  1215.             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
  1216.             'only_matching': True,
  1217.         },
  1218.         {
  1219.             # Rental video preview
  1220.             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
  1221.             'info_dict': {
  1222.                 'id': 'uGpuVWrhIzE',
  1223.                 'ext': 'mp4',
  1224.                 'title': 'Piku - Trailer',
  1225.                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
  1226.                 'upload_date': '20150811',
  1227.                 'uploader': 'FlixMatrix',
  1228.                 'uploader_id': 'FlixMatrixKaravan',
  1229.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
  1230.                 'license': 'Standard YouTube License',
  1231.             },
  1232.             'params': {
  1233.                 'skip_download': True,
  1234.             },
  1235.             'skip': 'This video is not available.',
  1236.         },
  1237.         {
  1238.             # YouTube Red video with episode data
  1239.             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
  1240.             'info_dict': {
  1241.                 'id': 'iqKdEhx-dD4',
  1242.                 'ext': 'mp4',
  1243.                 'title': 'Isolation - Mind Field (Ep 1)',
  1244.                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
  1245.                 'duration': 2085,
  1246.                 'upload_date': '20170118',
  1247.                 'uploader': 'Vsauce',
  1248.                 'uploader_id': '@Vsauce',
  1249.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@Vsauce',
  1250.                 'series': 'Mind Field',
  1251.                 'season_number': 1,
  1252.                 'episode_number': 1,
  1253.             },
  1254.             'params': {
  1255.                 'skip_download': True,
  1256.             },
  1257.             'expected_warnings': [
  1258.                 'Skipping DASH manifest',
  1259.             ],
  1260.         },
  1261.         {
  1262.             # The following content has been identified by the YouTube community
  1263.             # as inappropriate or offensive to some audiences.
  1264.             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
  1265.             'info_dict': {
  1266.                 'id': '6SJNVb0GnPI',
  1267.                 'ext': 'mp4',
  1268.                 'title': 'Race Differences in Intelligence',
  1269.                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
  1270.                 'duration': 965,
  1271.                 'upload_date': '20140124',
  1272.                 'uploader': 'New Century Foundation',
  1273.                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
  1274.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
  1275.             },
  1276.             'params': {
  1277.                 'skip_download': True,
  1278.             },
  1279.             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
  1280.         },
  1281.         {
  1282.             # itag 212
  1283.             'url': '1t24XAntNCY',
  1284.             'only_matching': True,
  1285.         },
  1286.         {
  1287.             # geo restricted to JP
  1288.             'url': 'sJL6WA-aGkQ',
  1289.             'only_matching': True,
  1290.         },
  1291.         {
  1292.             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
  1293.             'only_matching': True,
  1294.         },
  1295.         {
  1296.             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
  1297.             'only_matching': True,
  1298.         },
  1299.         {
  1300.             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
  1301.             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
  1302.             'only_matching': True,
  1303.         },
  1304.         {
  1305.             # DRM protected
  1306.             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
  1307.             'only_matching': True,
  1308.         },
  1309.         {
  1310.             # Video with unsupported adaptive stream type formats
  1311.             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
  1312.             'info_dict': {
  1313.                 'id': 'Z4Vy8R84T1U',
  1314.                 'ext': 'mp4',
  1315.                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
  1316.                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
  1317.                 'duration': 433,
  1318.                 'upload_date': '20130923',
  1319.                 'uploader': 'Amelia Putri Harwita',
  1320.                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
  1321.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
  1322.                 'formats': 'maxcount:10',
  1323.             },
  1324.             'params': {
  1325.                 'skip_download': True,
  1326.                 'youtube_include_dash_manifest': False,
  1327.             },
  1328.             'skip': 'not actual any more',
  1329.         },
  1330.         {
  1331.             # Youtube Music Auto-generated description
  1332.             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
  1333.             'info_dict': {
  1334.                 'id': 'MgNrAu2pzNs',
  1335.                 'ext': 'mp4',
  1336.                 'title': 'Voyeur Girl',
  1337.                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
  1338.                 'upload_date': '20190312',
  1339.                 'uploader': 'Stephen - Topic',
  1340.                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
  1341.                 'artist': 'Stephen',
  1342.                 'track': 'Voyeur Girl',
  1343.                 'album': 'it\'s too much love to know my dear',
  1344.                 'release_date': '20190313',
  1345.                 'release_year': 2019,
  1346.             },
  1347.             'params': {
  1348.                 'skip_download': True,
  1349.             },
  1350.         },
  1351.         {
  1352.             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
  1353.             'only_matching': True,
  1354.         },
  1355.         {
  1356.             # invalid -> valid video id redirection
  1357.             'url': 'DJztXj2GPfl',
  1358.             'info_dict': {
  1359.                 'id': 'DJztXj2GPfk',
  1360.                 'ext': 'mp4',
  1361.                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
  1362.                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
  1363.                 'upload_date': '20090125',
  1364.                 'uploader': 'Prochorowka',
  1365.                 'uploader_id': 'Prochorowka',
  1366.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
  1367.                 'artist': 'Panjabi MC',
  1368.                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
  1369.                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
  1370.             },
  1371.             'params': {
  1372.                 'skip_download': True,
  1373.             },
  1374.             'skip': 'Video unavailable',
  1375.         },
  1376.         {
  1377.             # empty description results in an empty string
  1378.             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
  1379.             'info_dict': {
  1380.                 'id': 'x41yOUIvK2k',
  1381.                 'ext': 'mp4',
  1382.                 'title': 'IMG 3456',
  1383.                 'description': '',
  1384.                 'upload_date': '20170613',
  1385.                 'uploader': "l'Or Vert asbl",
  1386.                 'uploader_id': '@ElevageOrVert',
  1387.             },
  1388.             'params': {
  1389.                 'skip_download': True,
  1390.             },
  1391.         },
  1392.         {
  1393.             # with '};' inside yt initial data (see [1])
  1394.             # see [2] for an example with '};' inside ytInitialPlayerResponse
  1395.             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
  1396.             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
  1397.             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
  1398.             'info_dict': {
  1399.                 'id': 'CHqg6qOn4no',
  1400.                 'ext': 'mp4',
  1401.                 'title': 'Part 77   Sort a list of simple types in c#',
  1402.                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
  1403.                 'upload_date': '20130831',
  1404.                 'uploader': 'kudvenkat',
  1405.                 'uploader_id': '@Csharp-video-tutorialsBlogspot',
  1406.             },
  1407.             'params': {
  1408.                 'skip_download': True,
  1409.             },
  1410.         },
  1411.         {
  1412.             # another example of '};' in ytInitialData
  1413.             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
  1414.             'only_matching': True,
  1415.         },
  1416.         {
  1417.             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
  1418.             'only_matching': True,
  1419.         },
  1420.         {
  1421.             # https://github.com/ytdl-org/youtube-dl/pull/28094
  1422.             'url': 'OtqTfy26tG0',
  1423.             'info_dict': {
  1424.                 'id': 'OtqTfy26tG0',
  1425.                 'ext': 'mp4',
  1426.                 'title': 'Burn Out',
  1427.                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
  1428.                 'upload_date': '20141120',
  1429.                 'uploader': 'The Cinematic Orchestra - Topic',
  1430.                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
  1431.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
  1432.                 'artist': 'The Cinematic Orchestra',
  1433.                 'track': 'Burn Out',
  1434.                 'album': 'Every Day',
  1435.                 'release_data': None,
  1436.                 'release_year': None,
  1437.             },
  1438.             'params': {
  1439.                 'skip_download': True,
  1440.             },
  1441.         },
  1442.         {
  1443.             # controversial video, only works with bpctr when authenticated with cookies
  1444.             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
  1445.             'only_matching': True,
  1446.         },
  1447.         {
  1448.             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
  1449.             'url': 'cBvYw8_A0vQ',
  1450.             'info_dict': {
  1451.                 'id': 'cBvYw8_A0vQ',
  1452.                 'ext': 'mp4',
  1453.                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
  1454.                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
  1455.                 'upload_date': '20201120',
  1456.                 'uploader': 'Walk around Japan',
  1457.                 'uploader_id': '@walkaroundjapan7124',
  1458.                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@walkaroundjapan7124',
  1459.             },
  1460.             'params': {
  1461.                 'skip_download': True,
  1462.             },
  1463.         },
  1464.         {
  1465.             # YT 'Shorts'
  1466.             'url': 'https://youtube.com/shorts/4L2J27mJ3Dc',
  1467.             'info_dict': {
  1468.                 'id': '4L2J27mJ3Dc',
  1469.                 'ext': 'mp4',
  1470.                 'title': 'Midwest Squid Game #Shorts',
  1471.                 'description': 'md5:976512b8a29269b93bbd8a61edc45a6d',
  1472.                 'upload_date': '20211025',
  1473.                 'uploader': 'Charlie Berens',
  1474.                 'uploader_id': '@CharlieBerens',
  1475.             },
  1476.             'params': {
  1477.                 'skip_download': True,
  1478.             },
  1479.         },
  1480.     ]
  1481.     _formats = {
  1482.         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
  1483.         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
  1484.         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
  1485.         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
  1486.         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
  1487.         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
  1488.         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
  1489.         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
  1490.         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
  1491.         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
  1492.         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
  1493.         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
  1494.         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
  1495.         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
  1496.         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
  1497.         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
  1498.         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
  1499.         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
  1502.         # 3D videos
  1503.         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
  1504.         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
  1505.         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
  1506.         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
  1507.         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
  1508.         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
  1509.         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
  1511.         # Apple HTTP Live Streaming
  1512.         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
  1513.         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
  1514.         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
  1515.         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
  1516.         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
  1517.         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
  1518.         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
  1519.         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
  1521.         # DASH mp4 video
  1522.         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
  1523.         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
  1524.         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
  1525.         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
  1526.         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
  1527.         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
  1528.         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
  1529.         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
  1530.         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
  1531.         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
  1532.         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
  1533.         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
  1535.         # Dash mp4 audio
  1536.         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
  1537.         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
  1538.         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
  1539.         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
  1540.         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
  1541.         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
  1542.         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
  1544.         # Dash webm
  1545.         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
  1546.         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
  1547.         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
  1548.         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
  1549.         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
  1550.         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
  1551.         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
  1552.         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1553.         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1554.         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1555.         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1556.         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1557.         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1558.         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1559.         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1560.         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
  1561.         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1562.         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
  1563.         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
  1564.         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
  1565.         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
  1566.         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
  1568.         # Dash webm audio
  1569.         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
  1570.         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
  1572.         # Dash webm audio with opus inside
  1573.         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
  1574.         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
  1575.         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
  1577.         # RTMP (unnamed)
  1578.         '_rtmp': {'protocol': 'rtmp'},
  1580.         # av01 video only formats sometimes served with "unknown" codecs
  1581.         '394': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
  1582.         '395': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
  1583.         '396': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
  1584.         '397': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
  1585.     }
  1587.     @classmethod
  1588.     def suitable(cls, url):
  1589.         if parse_qs(url).get('list', [None])[0]:
  1590.             return False
  1591.         return super(YoutubeIE, cls).suitable(url)
  1593.     def __init__(self, *args, **kwargs):
  1594.         super(YoutubeIE, self).__init__(*args, **kwargs)
  1595.         self._code_cache = {}
  1596.         self._player_cache = {}
  1598.     # *ytcfgs, webpage=None
  1599.     def _extract_player_url(self, *ytcfgs, **kw_webpage):
  1600.         if ytcfgs and not isinstance(ytcfgs[0], dict):
  1601.             webpage = kw_webpage.get('webpage') or ytcfgs[0]
  1602.         if webpage:
  1603.             player_url = self._search_regex(
  1604.                 r'"(?:PLAYER_JS_URL|jsUrl)"\s*:\s*"([^"]+)"',
  1605.                 webpage or '', 'player URL', fatal=False)
  1606.             if player_url:
  1607.                 ytcfgs = ytcfgs + ({'PLAYER_JS_URL': player_url},)
  1608.         return traverse_obj(
  1609.             ytcfgs, (Ellipsis, 'PLAYER_JS_URL'), (Ellipsis, 'WEB_PLAYER_CONTEXT_CONFIGS', Ellipsis, 'jsUrl'),
  1610.             get_all=False, expected_type=lambda u: urljoin('https://www.youtube.com', u))
  1612.     def _download_player_url(self, video_id, fatal=False):
  1613.         res = self._download_webpage(
  1614.             'https://www.youtube.com/iframe_api',
  1615.             note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
  1616.         player_version = self._search_regex(
  1617.             r'player\\?/([0-9a-fA-F]{8})\\?/', res or '', 'player version', fatal=fatal,
  1618.             default=NO_DEFAULT if res else None)
  1619.         if player_version:
  1620.             return 'https://www.youtube.com/s/player/{0}/player_ias.vflset/en_US/base.js'.format(player_version)
  1622.     def _signature_cache_id(self, example_sig):
  1623.         """ Return a string representation of a signature """
  1624.         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
  1626.     def _extract_player_info(self, player_url):
  1627.         try:
  1628.             return self._search_regex(
  1629.                 self._PLAYER_INFO_RE, player_url, 'player info', group='id')
  1630.         except ExtractorError as e:
  1631.             raise ExtractorError(
  1632.                 'Cannot identify player %r' % (player_url,), cause=e)
  1634.     def _load_player(self, video_id, player_url, fatal=True, player_id=None):
  1635.         if not player_id:
  1636.             player_id = self._extract_player_info(player_url)
  1637.         if player_id not in self._code_cache:
  1638.             code = self._download_webpage(
  1639.                 player_url, video_id, fatal=fatal,
  1640.                 note='Downloading player ' + player_id,
  1641.                 errnote='Download of %s failed' % player_url)
  1642.             if code:
  1643.                 self._code_cache[player_id] = code
  1644.         return self._code_cache[player_id] if fatal else self._code_cache.get(player_id)
  1646.     def _extract_signature_function(self, video_id, player_url, example_sig):
  1647.         player_id = self._extract_player_info(player_url)
  1649.         # Read from filesystem cache
  1650.         func_id = 'js_{0}_{1}'.format(
  1651.             player_id, self._signature_cache_id(example_sig))
  1652.         assert os.path.basename(func_id) == func_id
  1654.         self.write_debug('Extracting signature function {0}'.format(func_id))
  1655.         cache_spec, code = self.cache.load('youtube-sigfuncs', func_id, min_ver='2025.04.07'), None
  1657.         if not cache_spec:
  1658.             code = self._load_player(video_id, player_url, player_id)
  1659.         if code:
  1660.             res = self._parse_sig_js(code)
  1661.             test_string = ''.join(map(compat_chr, range(len(example_sig))))
  1662.             cache_spec = [ord(c) for c in res(test_string)]
  1663.             self.cache.store('youtube-sigfuncs', func_id, cache_spec)
  1665.         return lambda s: ''.join(s[i] for i in cache_spec)
  1667.     def _print_sig_code(self, func, example_sig):
  1668.         if not self.get_param('youtube_print_sig_code'):
  1669.             return
  1671.         def gen_sig_code(idxs):
  1672.             def _genslice(start, end, step):
  1673.                 starts = '' if start == 0 else str(start)
  1674.                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
  1675.                 steps = '' if step == 1 else (':%d' % step)
  1676.                 return 's[{0}{1}{2}]'.format(starts, ends, steps)
  1678.             step = None
  1679.             # Quelch pyflakes warnings - start will be set when step is set
  1680.             start = '(Never used)'
  1681.             for i, prev in zip(idxs[1:], idxs[:-1]):
  1682.                 if step is not None:
  1683.                     if i - prev == step:
  1684.                         continue
  1685.                     yield _genslice(start, prev, step)
  1686.                     step = None
  1687.                     continue
  1688.                 if i - prev in [-1, 1]:
  1689.                     step = i - prev
  1690.                     start = prev
  1691.                     continue
  1692.                 else:
  1693.                     yield 's[%d]' % prev
  1694.             if step is None:
  1695.                 yield 's[%d]' % i
  1696.             else:
  1697.                 yield _genslice(start, i, step)
  1699.         test_string = ''.join(map(compat_chr, range(len(example_sig))))
  1700.         cache_res = func(test_string)
  1701.         cache_spec = [ord(c) for c in cache_res]
  1702.         expr_code = ' + '.join(gen_sig_code(cache_spec))
  1703.         signature_id_tuple = '(%s)' % (
  1704.             ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
  1705.         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
  1706.                 '    return %s\n') % (signature_id_tuple, expr_code)
  1707.         self.to_screen('Extracted signature function:\n' + code)
  1709.     def _extract_sig_fn(self, jsi, funcname):
  1710.         var_ay = self._search_regex(
  1711.             r'''(?x)
  1712.                 (?:\*/|\{|\n|^)\s*(?:'[^']+'\s*;\s*)
  1713.                     (var\s*[\w$]+\s*=\s*(?:
  1714.                         ('|")(?:\\\2|(?!\2).)+\2\s*\.\s*split\(\s*('|")\W+\3\s*\)|
  1715.                         \[\s*(?:('|")(?:\\\4|(?!\4).)*\4\s*(?:(?=\])|,\s*))+\]
  1716.                     ))(?=\s*[,;])
  1717.             ''', jsi.code, 'useful values', default='')
  1719.         sig_fn = jsi.extract_function_code(funcname)
  1721.         if var_ay:
  1722.             sig_fn = (sig_fn[0], ';\n'.join((var_ay, sig_fn[1])))
  1724.         return sig_fn
  1726.     def _parse_sig_js(self, jscode):
  1727.         # Examples where `sig` is funcname:
  1728.         # sig=function(a){a=a.split(""); ... ;return a.join("")};
  1729.         # ;c&&(c=sig(decodeURIComponent(c)),a.set(b,encodeURIComponent(c)));return a};
  1730.         # {var l=f,m=h.sp,n=sig(decodeURIComponent(h.s));l.set(m,encodeURIComponent(n))}
  1731.         # sig=function(J){J=J.split(""); ... ;return J.join("")};
  1732.         # ;N&&(N=sig(decodeURIComponent(N)),J.set(R,encodeURIComponent(N)));return J};
  1733.         # {var H=u,k=f.sp,v=sig(decodeURIComponent(f.s));H.set(k,encodeURIComponent(v))}
  1734.         funcname = self._search_regex(
  1735.             (r'\b(?P<var>[\w$]+)&&\((?P=var)=(?P<sig>[\w$]{2,})\(decodeURIComponent\((?P=var)\)\)',
  1736.              r'(?P<sig>[\w$]+)\s*=\s*function\(\s*(?P<arg>[\w$]+)\s*\)\s*{\s*(?P=arg)\s*=\s*(?P=arg)\.split\(\s*""\s*\)\s*;\s*[^}]+;\s*return\s+(?P=arg)\.join\(\s*""\s*\)',
  1737.              r'(?:\b|[^\w$])(?P<sig>[\w$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[\w$]{2}\.[\w$]{2}\(a,\d+\))?',
  1738.              # Old patterns
  1739.              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[\w$]+)\(',
  1740.              r'\b[\w]+\s*&&\s*[\w]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[\w$]+)\(',
  1741.              r'\bm=(?P<sig>[\w$]{2,})\(decodeURIComponent\(h\.s\)\)',
  1742.              # Obsolete patterns
  1743.              r'("|\')signature\1\s*,\s*(?P<sig>[\w$]+)\(',
  1744.              r'\.sig\|\|(?P<sig>[\w$]+)\(',
  1745.              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[\w$]+)\(',
  1746.              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[\w$]+)\(',
  1747.              r'\bc\s*&&\s*[\w]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[\w$]+)\('),
  1748.             jscode, 'Initial JS player signature function name', group='sig')
  1750.         jsi = JSInterpreter(jscode)
  1752.         initial_function = self._extract_sig_fn(jsi, funcname)
  1754.         func = jsi.extract_function_from_code(*initial_function)
  1756.         return lambda s: func([s])
  1758.     def _cached(self, func, *cache_id):
  1759.         def inner(*args, **kwargs):
  1760.             if cache_id not in self._player_cache:
  1761.                 try:
  1762.                     self._player_cache[cache_id] = func(*args, **kwargs)
  1763.                 except ExtractorError as e:
  1764.                     self._player_cache[cache_id] = e
  1765.                 except Exception as e:
  1766.                     self._player_cache[cache_id] = ExtractorError(traceback.format_exc(), cause=e)
  1768.             ret = self._player_cache[cache_id]
  1769.             if isinstance(ret, Exception):
  1770.                 raise ret
  1771.             return ret
  1772.         return inner
  1774.     def _decrypt_signature(self, s, video_id, player_url):
  1775.         """Turn the encrypted s field into a working signature"""
  1776.         extract_sig = self._cached(
  1777.             self._extract_signature_function, 'sig', player_url, self._signature_cache_id(s))
  1778.         func = extract_sig(video_id, player_url, s)
  1779.         self._print_sig_code(func, s)
  1780.         return func(s)
  1782.     # from yt-dlp
  1783.     # See also:
  1784.     # 1. https://github.com/ytdl-org/youtube-dl/issues/29326#issuecomment-894619419
  1785.     # 2. https://code.videolan.org/videolan/vlc/-/blob/4fb284e5af69aa9ac2100ccbdd3b88debec9987f/share/lua/playlist/youtube.lua#L116
  1786.     # 3. https://github.com/ytdl-org/youtube-dl/issues/30097#issuecomment-950157377
  1787.     def _decrypt_nsig(self, n, video_id, player_url):
  1788.         """Turn the encrypted n field into a working signature"""
  1789.         if player_url is None:
  1790.             raise ExtractorError('Cannot decrypt nsig without player_url')
  1792.         try:
  1793.             jsi, player_id, func_code = self._extract_n_function_code(video_id, player_url)
  1794.         except ExtractorError as e:
  1795.             raise ExtractorError('Unable to extract nsig function code', cause=e)
  1796.         if self.get_param('youtube_print_sig_code'):
  1797.             self.to_screen('Extracted nsig function from {0}:\n{1}\n'.format(
  1798.                 player_id, func_code[1]))
  1800.         try:
  1801.             extract_nsig = self._cached(self._extract_n_function_from_code, 'nsig func', player_url)
  1802.             ret = extract_nsig(jsi, func_code)(n)
  1803.         except JSInterpreter.Exception as e:
  1804.             self.report_warning(
  1805.                 '%s (%s %s)' % (
  1806.                     'Unable to decode n-parameter: expect download to be blocked or throttled',
  1807.                     error_to_compat_str(e),
  1808.                     traceback.format_exc()),
  1809.                 video_id=video_id)
  1810.             return
  1812.         self.write_debug('Decrypted nsig {0} => {1}'.format(n, ret))
  1813.         return ret
  1815.     def _extract_n_function_name(self, jscode):
  1816.         func_name, idx = None, None
  1817.         # these special cases are redundant and probably obsolete (2025-04):
  1818.         # they make the tests run ~10% faster without fallback warnings
  1819.         r"""
  1820.         func_name, idx = self._search_regex(
  1821.             # (y=NuD(),Mw(k),q=k.Z[y]||null)&&(q=narray[idx](q),k.set(y,q),k.V||NuD(''))}};
  1822.             # (R="nn"[+J.Z],mW(J),N=J.K[R]||null)&&(N=narray[idx](N),J.set(R,N))}};
  1823.             # or:  (b=String.fromCharCode(110),c=a.get(b))&&c=narray[idx](c)
  1824.             # or:  (b="nn"[+a.D],c=a.get(b))&&(c=narray[idx](c)
  1825.             # or:  (PL(a),b=a.j.n||null)&&(b=narray[idx](b)
  1826.             # or:  (b="nn"[+a.D],vL(a),c=a.j[b]||null)&&(c=narray[idx](c),a.set(b,c),narray.length||nfunc("")
  1827.             # old: (b=a.get("n"))&&(b=narray[idx](b)(?P<c>[a-z])\s*=\s*[a-z]\s*
  1828.             # older: (b=a.get("n"))&&(b=nfunc(b)
  1829.             r'''(?x)
  1830.                 # (expr, ...,
  1831.                 \((?:(?:\s*[\w$]+\s*=)?(?:[\w$"+\.\s(\[]+(?:[)\]]\s*)?),)*
  1832.                   # b=...
  1833.                   (?P<b>[\w$]+)\s*=\s*(?!(?P=b)[^\w$])[\w$]+\s*(?:(?:
  1834.                     \.\s*[\w$]+ |
  1835.                     \[\s*[\w$]+\s*\] |
  1836.                     \.\s*get\s*\(\s*[\w$"]+\s*\)
  1837.                   )\s*){,2}(?:\s*\|\|\s*null(?=\s*\)))?\s*
  1838.                 \)\s*&&\s*\(        # ...)&&(
  1839.                 # b = nfunc, b = narray[idx]
  1840.                 (?P=b)\s*=\s*(?P<nfunc>[\w$]+)\s*
  1841.                     (?:\[\s*(?P<idx>[\w$]+)\s*\]\s*)?
  1842.                     # (...)
  1843.                     \(\s*[\w$]+\s*\)
  1844.             ''', jscode, 'Initial JS player n function name', group=('nfunc', 'idx'),
  1845.             default=(None, None))
  1846.         """
  1848.         if not func_name:
  1849.             # nfunc=function(x){...}|function nfunc(x); ...
  1850.             # ... var y=[nfunc]|y[idx]=nfunc);
  1851.             # obvious REs hang, so use a two-stage tactic
  1852.             for m in re.finditer(r'''(?x)
  1853.                     [\n;]var\s(?:(?:(?!,).)+,|\s)*?(?!\d)[\w$]+(?:\[(?P<idx>\d+)\])?\s*=\s*
  1854.                         (?(idx)|\[\s*)(?P<nfunc>(?!\d)[\w$]+)(?(idx)|\s*\])
  1855.                     \s*?[;\n]
  1856.                     ''', jscode):
  1857.                 func_name = self._search_regex(
  1858.                     r'[;,]\s*(function\s+)?({0})(?(1)|\s*=\s*function)\s*\((?!\d)[\w$]+\)\s*\{1}(?!\s*return\s)'.format(
  1859.                         re.escape(m.group('nfunc')), '{'),
  1860.                     jscode, 'Initial JS player n function name (2)', group=2, default=None)
  1861.                 if func_name:
  1862.                     idx = m.group('idx')
  1863.                     break
  1865.         # thx bashonly: yt-dlp/yt-dlp/pull/10611
  1866.         if not func_name:
  1867.             self.report_warning('Falling back to generic n function search', only_once=True)
  1868.             return self._search_regex(
  1869.                 r'''(?xs)
  1870.                     (?:(?<=[^\w$])|^)       # instead of \b, which ignores $
  1871.                     (?P<name>(?!\d)[a-zA-Z\d_$]+)\s*=\s*function\((?!\d)[a-zA-Z\d_$]+\)
  1872.                     \s*\{(?:(?!};).)+?(?:
  1873.                         ["']enhanced_except_ |
  1874.                         return\s*(?P<q>"|')[a-zA-Z\d-]+_w8_(?P=q)\s*\+\s*[\w$]+
  1875.                     )
  1876.                 ''', jscode, 'Initial JS player n function name', group='name')
  1877.         if not idx:
  1878.             return func_name
  1880.         return self._search_json(
  1881.             r'(?<![\w-])var\s(?:(?:(?!,).)+,|\s)*?{0}\s*='.format(re.escape(func_name)), jscode,
  1882.             'Initial JS player n function list ({0}.{1})'.format(func_name, idx),
  1883.             func_name, contains_pattern=r'\[.+\]', end_pattern='[,;]',
  1884.             transform_source=js_to_json)[int(idx)]
  1886.     def _extract_n_function_code(self, video_id, player_url):
  1887.         player_id = self._extract_player_info(player_url)
  1888.         func_code = self.cache.load('youtube-nsig', player_id, min_ver='2025.04.07')
  1889.         jscode = func_code or self._load_player(video_id, player_url)
  1890.         jsi = JSInterpreter(jscode)
  1892.         if func_code:
  1893.             return jsi, player_id, func_code
  1894.         return self._extract_n_function_code_jsi(video_id, jsi, player_id)
  1896.     def _extract_n_function_code_jsi(self, video_id, jsi, player_id=None):
  1898.         func_name = self._extract_n_function_name(jsi.code)
  1900.         func_code = self._extract_sig_fn(jsi, func_name)
  1902.         if player_id:
  1903.             self.cache.store('youtube-nsig', player_id, func_code)
  1904.         return jsi, player_id, func_code
  1906.     def _extract_n_function_from_code(self, jsi, func_code):
  1907.         func = jsi.extract_function_from_code(*func_code)
  1909.         def extract_nsig(s):
  1910.             try:
  1911.                 ret = func([s], kwargs={'_ytdl_do_not_return': s})
  1912.             except JSInterpreter.Exception:
  1913.                 raise
  1914.             except Exception as e:
  1915.                 raise JSInterpreter.Exception(traceback.format_exc(), cause=e)
  1917.             if ret.startswith('enhanced_except_') or ret.endswith(s):
  1918.                 raise JSInterpreter.Exception('Signature function returned an exception')
  1919.             return ret
  1921.         return extract_nsig
  1923.     def _unthrottle_format_urls(self, video_id, player_url, *formats):
  1925.         def decrypt_nsig(n):
  1926.             return self._cached(self._decrypt_nsig, 'nsig', n, player_url)
  1928.         for fmt in formats:
  1929.             parsed_fmt_url = compat_urllib_parse.urlparse(fmt['url'])
  1930.             n_param = compat_parse_qs(parsed_fmt_url.query).get('n')
  1931.             if not n_param:
  1932.                 continue
  1933.             n_param = n_param[-1]
  1934.             n_response = decrypt_nsig(n_param)(n_param, video_id, player_url)
  1935.             if n_response is None:
  1936.                 # give up if descrambling failed
  1937.                 break
  1938.             fmt['url'] = update_url_query(fmt['url'], {'n': n_response})
  1940.     # from yt-dlp, with tweaks
  1941.     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
  1942.         """
  1943.         Extract signatureTimestamp (sts)
  1944.         Required to tell API what sig/player version is in use.
  1945.         """
  1946.         sts = traverse_obj(ytcfg, 'STS', expected_type=int)
  1947.         if not sts:
  1948.             # Attempt to extract from player
  1949.             if player_url is None:
  1950.                 error_msg = 'Cannot extract signature timestamp without player_url.'
  1951.                 if fatal:
  1952.                     raise ExtractorError(error_msg)
  1953.                 self.report_warning(error_msg)
  1954.                 return
  1955.             code = self._load_player(video_id, player_url, fatal=fatal)
  1956.             sts = int_or_none(self._search_regex(
  1957.                 r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code or '',
  1958.                 'JS player signature timestamp', group='sts', fatal=fatal))
  1959.         return sts
  1961.     def _mark_watched(self, video_id, player_response):
  1962.         playback_url = url_or_none(try_get(
  1963.             player_response,
  1964.             lambda x: x['playbackTracking']['videostatsPlaybackUrl']['baseUrl']))
  1965.         if not playback_url:
  1966.             return
  1968.         # cpn generation algorithm is reverse engineered from base.js.
  1969.         # In fact it works even with dummy cpn.
  1970.         CPN_ALPHABET = string.ascii_letters + string.digits + '-_'
  1971.         cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(16))
  1973.         # more consistent results setting it to right before the end
  1974.         qs = parse_qs(playback_url)
  1975.         video_length = '{0}'.format(float((qs.get('len') or ['1.5'])[0]) - 1)
  1977.         playback_url = update_url_query(
  1978.             playback_url, {
  1979.                 'ver': '2',
  1980.                 'cpn': cpn,
  1981.                 'cmt': video_length,
  1982.                 'el': 'detailpage',  # otherwise defaults to "shorts"
  1983.             })
  1985.         self._download_webpage(
  1986.             playback_url, video_id, 'Marking watched',
  1987.             'Unable to mark watched', fatal=False)
  1989.     @staticmethod
  1990.     def _extract_urls(webpage):
  1991.         # Embedded YouTube player
  1992.         entries = [
  1993.             unescapeHTML(mobj.group('url'))
  1994.             for mobj in re.finditer(r'''(?x)
  1995.             (?:
  1996.                 <iframe[^>]+?src=|
  1997.                 data-video-url=|
  1998.                 <embed[^>]+?src=|
  1999.                 embedSWF\(?:\s*|
  2000.                 <object[^>]+data=|
  2001.                 new\s+SWFObject\(
  2002.             )
  2003.             (["\'])
  2004.                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
  2005.                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
  2006.             \1''', webpage)]
  2008.         # lazyYT YouTube embed
  2009.         entries.extend(list(map(
  2010.             unescapeHTML,
  2011.             re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
  2013.         # Wordpress "YouTube Video Importer" plugin
  2014.         matches = re.findall(r'''(?x)<div[^>]+
  2015.             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
  2016.             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
  2017.         entries.extend(m[-1] for m in matches)
  2019.         return entries
  2021.     @staticmethod
  2022.     def _extract_url(webpage):
  2023.         urls = YoutubeIE._extract_urls(webpage)
  2024.         return urls[0] if urls else None
  2026.     @classmethod
  2027.     def extract_id(cls, url):
  2028.         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
  2029.         if mobj is None:
  2030.             raise ExtractorError('Invalid URL: %s' % url)
  2031.         return mobj.group(2)
  2033.     def _extract_chapters_from_json(self, data, video_id, duration):
  2034.         chapters_list = try_get(
  2035.             data,
  2036.             lambda x: x['playerOverlays']
  2037.                        ['playerOverlayRenderer']
  2038.                        ['decoratedPlayerBarRenderer']
  2039.                        ['decoratedPlayerBarRenderer']
  2040.                        ['playerBar']
  2041.                        ['chapteredPlayerBarRenderer']
  2042.                        ['chapters'],
  2043.             list)
  2044.         if not chapters_list:
  2045.             return
  2047.         def chapter_time(chapter):
  2048.             return float_or_none(
  2049.                 try_get(
  2050.                     chapter,
  2051.                     lambda x: x['chapterRenderer']['timeRangeStartMillis'],
  2052.                     int),
  2053.                 scale=1000)
  2054.         chapters = []
  2055.         for next_num, chapter in enumerate(chapters_list, start=1):
  2056.             start_time = chapter_time(chapter)
  2057.             if start_time is None:
  2058.                 continue
  2059.             end_time = (chapter_time(chapters_list[next_num])
  2060.                         if next_num < len(chapters_list) else duration)
  2061.             if end_time is None:
  2062.                 continue
  2063.             title = try_get(
  2064.                 chapter, lambda x: x['chapterRenderer']['title']['simpleText'],
  2065.                 compat_str)
  2066.             chapters.append({
  2067.                 'start_time': start_time,
  2068.                 'end_time': end_time,
  2069.                 'title': title,
  2070.             })
  2071.         return chapters
  2073.     def _extract_yt_initial_variable(self, webpage, regex, video_id, name):
  2074.         return self._parse_json(self._search_regex(
  2075.             (r'%s\s*%s' % (regex, self._YT_INITIAL_BOUNDARY_RE),
  2076.              regex), webpage, name, default='{}'), video_id, fatal=False)
  2078.     def _real_extract(self, url):
  2079.         url, smuggled_data = unsmuggle_url(url, {})
  2080.         video_id = self._match_id(url)
  2081.         base_url = self.http_scheme() + '//www.youtube.com/'
  2082.         webpage_url = base_url + 'watch?v=' + video_id
  2083.         webpage = self._download_webpage(
  2084.             webpage_url + '&bpctr=9999999999&has_verified=1', video_id, fatal=False)
  2086.         player_response = None
  2087.         player_url = None
  2088.         if webpage:
  2089.             player_response = self._extract_yt_initial_variable(
  2090.                 webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE,
  2091.                 video_id, 'initial player response')
  2092.         is_live = traverse_obj(player_response, ('videoDetails', 'isLive'))
  2094.         if False and not player_response:
  2095.             player_response = self._call_api(
  2096.                 'player', {'videoId': video_id}, video_id)
  2097.         if True or not player_response:
  2098.             origin = 'https://www.youtube.com'
  2099.             pb_context = {'html5Preference': 'HTML5_PREF_WANTS'}
  2101.             player_url = self._extract_player_url(webpage)
  2102.             ytcfg = self._extract_ytcfg(video_id, webpage or '')
  2103.             sts = self._extract_signature_timestamp(video_id, player_url, ytcfg)
  2104.             if sts:
  2105.                 pb_context['signatureTimestamp'] = sts
  2107.             client_names = traverse_obj(self._INNERTUBE_CLIENTS, (
  2108.                 T(dict.items), lambda _, k_v: not k_v[1].get('REQUIRE_PO_TOKEN'),
  2109.                 0))[:1]
  2110.             if 'web' not in client_names:
  2111.                 # webpage links won't download: ignore links and playability
  2112.                 player_response = filter_dict(
  2113.                     player_response or {},
  2114.                     lambda k, _: k not in ('streamingData', 'playabilityStatus'))
  2116.             if is_live and 'ios' not in client_names:
  2117.                 client_names.append('ios')
  2119.             headers = {
  2120.                 'Sec-Fetch-Mode': 'navigate',
  2121.                 'Origin': origin,
  2122.                 'X-Goog-Visitor-Id': self._extract_visitor_data(ytcfg) or '',
  2123.             }
  2124.             auth = self._generate_sapisidhash_header(origin)
  2125.             if auth is not None:
  2126.                 headers['Authorization'] = auth
  2127.                 headers['X-Origin'] = origin
  2129.             for client in traverse_obj(self._INNERTUBE_CLIENTS, (client_names, T(dict))):
  2131.                 query = {
  2132.                     'playbackContext': {
  2133.                         'contentPlaybackContext': pb_context,
  2134.                     },
  2135.                     'contentCheckOk': True,
  2136.                     'racyCheckOk': True,
  2137.                     'context': {
  2138.                         'client': merge_dicts(
  2139.                             traverse_obj(client, ('INNERTUBE_CONTEXT', 'client')), {
  2140.                                 'hl': 'en',
  2141.                                 'timeZone': 'UTC',
  2142.                                 'utcOffsetMinutes': 0,
  2143.                             }),
  2144.                     },
  2145.                     'videoId': video_id,
  2146.                 }
  2148.                 api_headers = merge_dicts(headers, traverse_obj(client, {
  2149.                     'X-YouTube-Client-Name': 'INNERTUBE_CONTEXT_CLIENT_NAME',
  2150.                     'X-YouTube-Client-Version': (
  2151.                         'INNERTUBE_CONTEXT', 'client', 'clientVersion'),
  2152.                     'User-Agent': (
  2153.                         'INNERTUBE_CONTEXT', 'client', 'userAgent'),
  2154.                 }))
  2156.                 api_player_response = self._call_api(
  2157.                     'player', query, video_id, fatal=False, headers=api_headers,
  2158.                     note=join_nonempty(
  2159.                         'Downloading', traverse_obj(query, (
  2160.                             'context', 'client', 'clientName')),
  2161.                         'API JSON', delim=' '))
  2163.                 hls = traverse_obj(
  2164.                     (player_response, api_player_response),
  2165.                     (Ellipsis, 'streamingData', 'hlsManifestUrl', T(url_or_none)))
  2166.                 if len(hls) == 2 and not hls[0] and hls[1]:
  2167.                     player_response['streamingData']['hlsManifestUrl'] = hls[1]
  2168.                 else:
  2169.                     video_details = merge_dicts(*traverse_obj(
  2170.                         (player_response, api_player_response),
  2171.                         (Ellipsis, 'videoDetails', T(dict))))
  2172.                     player_response.update(filter_dict(
  2173.                         api_player_response or {}, cndn=lambda k, _: k != 'captions'))
  2174.                     player_response['videoDetails'] = video_details
  2176.         def is_agegated(playability):
  2177.             if not isinstance(playability, dict):
  2178.                 return
  2180.             if playability.get('desktopLegacyAgeGateReason'):
  2181.                 return True
  2183.             reasons = filter(None, (playability.get(r) for r in ('status', 'reason')))
  2184.             AGE_GATE_REASONS = (
  2185.                 'confirm your age', 'age-restricted', 'inappropriate',  # reason
  2186.                 'age_verification_required', 'age_check_required',  # status
  2187.             )
  2188.             return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
  2190.         def get_playability_status(response):
  2191.             return try_get(response, lambda x: x['playabilityStatus'], dict) or {}
  2193.         playability_status = get_playability_status(player_response)
  2194.         if (is_agegated(playability_status)
  2195.                 and int_or_none(self._downloader.params.get('age_limit'), default=18) >= 18):
  2197.             self.report_age_confirmation()
  2199.             # Thanks: https://github.com/yt-dlp/yt-dlp/pull/3233
  2200.             pb_context = {'html5Preference': 'HTML5_PREF_WANTS'}
  2202.             # Use signatureTimestamp if available
  2203.             # Thanks https://github.com/ytdl-org/youtube-dl/issues/31034#issuecomment-1160718026
  2204.             player_url = self._extract_player_url(webpage)
  2205.             ytcfg = self._extract_ytcfg(video_id, webpage)
  2206.             sts = self._extract_signature_timestamp(video_id, player_url, ytcfg)
  2207.             if sts:
  2208.                 pb_context['signatureTimestamp'] = sts
  2210.             query = {
  2211.                 'playbackContext': {'contentPlaybackContext': pb_context},
  2212.                 'contentCheckOk': True,
  2213.                 'racyCheckOk': True,
  2214.                 'context': {
  2215.                     'client': {'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER', 'clientVersion': '2.0', 'hl': 'en', 'clientScreen': 'EMBED'},
  2216.                     'thirdParty': {'embedUrl': 'https://google.com'},
  2217.                 },
  2218.                 'videoId': video_id,
  2219.             }
  2220.             headers = {
  2221.                 'X-YouTube-Client-Name': '85',
  2222.                 'X-YouTube-Client-Version': '2.0',
  2223.                 'Origin': 'https://www.youtube.com',
  2224.             }
  2226.             video_info = self._call_api('player', query, video_id, fatal=False, headers=headers)
  2227.             age_gate_status = get_playability_status(video_info)
  2228.             if age_gate_status.get('status') == 'OK':
  2229.                 player_response = video_info
  2230.                 playability_status = age_gate_status
  2232.         trailer_video_id = try_get(
  2233.             playability_status,
  2234.             lambda x: x['errorScreen']['playerLegacyDesktopYpcTrailerRenderer']['trailerVideoId'],
  2235.             compat_str)
  2236.         if trailer_video_id:
  2237.             return self.url_result(
  2238.                 trailer_video_id, self.ie_key(), trailer_video_id)
  2240.         def get_text(x):
  2241.             if not x:
  2242.                 return
  2243.             text = x.get('simpleText')
  2244.             if text and isinstance(text, compat_str):
  2245.                 return text
  2246.             runs = x.get('runs')
  2247.             if not isinstance(runs, list):
  2248.                 return
  2249.             return ''.join([r['text'] for r in runs if isinstance(r.get('text'), compat_str)])
  2251.         search_meta = (
  2252.             (lambda x: self._html_search_meta(x, webpage, default=None))
  2253.             if webpage else lambda _: None)
  2255.         video_details = player_response.get('videoDetails') or {}
  2256.         microformat = try_get(
  2257.             player_response,
  2258.             lambda x: x['microformat']['playerMicroformatRenderer'],
  2259.             dict) or {}
  2260.         video_title = video_details.get('title') \
  2261.             or get_text(microformat.get('title')) \
  2262.             or search_meta(['og:title', 'twitter:title', 'title'])
  2263.         video_description = video_details.get('shortDescription')
  2265.         if not smuggled_data.get('force_singlefeed', False):
  2266.             if not self._downloader.params.get('noplaylist'):
  2267.                 multifeed_metadata_list = try_get(
  2268.                     player_response,
  2269.                     lambda x: x['multicamera']['playerLegacyMulticameraRenderer']['metadataList'],
  2270.                     compat_str)
  2271.                 if multifeed_metadata_list:
  2272.                     entries = []
  2273.                     feed_ids = []
  2274.                     for feed in multifeed_metadata_list.split(','):
  2275.                         # Unquote should take place before split on comma (,) since textual
  2276.                         # fields may contain comma as well (see
  2277.                         # https://github.com/ytdl-org/youtube-dl/issues/8536)
  2278.                         feed_data = compat_parse_qs(
  2279.                             compat_urllib_parse_unquote_plus(feed))
  2281.                         def feed_entry(name):
  2282.                             return try_get(
  2283.                                 feed_data, lambda x: x[name][0], compat_str)
  2285.                         feed_id = feed_entry('id')
  2286.                         if not feed_id:
  2287.                             continue
  2288.                         feed_title = feed_entry('title')
  2289.                         title = video_title
  2290.                         if feed_title:
  2291.                             title += ' (%s)' % feed_title
  2292.                         entries.append({
  2293.                             '_type': 'url_transparent',
  2294.                             'ie_key': 'Youtube',
  2295.                             'url': smuggle_url(
  2296.                                 base_url + 'watch?v=' + feed_data['id'][0],
  2297.                                 {'force_singlefeed': True}),
  2298.                             'title': title,
  2299.                         })
  2300.                         feed_ids.append(feed_id)
  2301.                     self.to_screen(
  2302.                         'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
  2303.                         % (', '.join(feed_ids), video_id))
  2304.                     return self.playlist_result(
  2305.                         entries, video_id, video_title, video_description)
  2306.             else:
  2307.                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
  2309.         if not player_url:
  2310.             player_url = self._extract_player_url(webpage)
  2312.         formats = []
  2313.         itags = collections.defaultdict(set)
  2314.         itag_qualities = {}
  2315.         q = qualities(['tiny', 'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'])
  2316.         CHUNK_SIZE = 10 << 20
  2317.         is_live = video_details.get('isLive')
  2319.         streaming_data = player_response.get('streamingData') or {}
  2320.         streaming_formats = streaming_data.get('formats') or []
  2321.         streaming_formats.extend(streaming_data.get('adaptiveFormats') or [])
  2323.         def build_fragments(f):
  2324.             return LazyList({
  2325.                 'url': update_url_query(f['url'], {
  2326.                     'range': '{0}-{1}'.format(range_start, min(range_start + CHUNK_SIZE - 1, f['filesize'])),
  2327.                 })
  2328.             } for range_start in range(0, f['filesize'], CHUNK_SIZE))
  2330.         lower = lambda s: s.lower()
  2332.         for fmt in streaming_formats:
  2333.             if fmt.get('targetDurationSec'):
  2334.                 continue
  2336.             itag = str_or_none(fmt.get('itag'))
  2337.             audio_track = traverse_obj(fmt, ('audioTrack', T(dict))) or {}
  2339.             quality = traverse_obj(fmt, ((
  2340.                 # The 3gp format (17) in android client has a quality of "small",
  2341.                 # but is actually worse than other formats
  2342.                 T(lambda _: 'tiny' if itag == 17 else None),
  2343.                 ('quality', T(lambda q: q if q and q != 'tiny' else None)),
  2344.                 ('audioQuality', T(lower)),
  2345.                 'quality'), T(txt_or_none)), get_all=False)
  2346.             if quality and itag:
  2347.                 itag_qualities[itag] = quality
  2348.             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
  2349.             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
  2350.             # number of fragments that would subsequently be requested with (`&sq=N`)
  2351.             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
  2352.                 continue
  2354.             fmt_url = fmt.get('url')
  2355.             if not fmt_url:
  2356.                 sc = compat_parse_qs(fmt.get('signatureCipher'))
  2357.                 fmt_url = traverse_obj(sc, ('url', -1, T(url_or_none)))
  2358.                 encrypted_sig = traverse_obj(sc, ('s', -1))
  2359.                 if not (fmt_url and encrypted_sig):
  2360.                     continue
  2361.                 player_url = player_url or self._extract_player_url(webpage)
  2362.                 if not player_url:
  2363.                     continue
  2364.                 try:
  2365.                     fmt_url = update_url_query(fmt_url, {
  2366.                         traverse_obj(sc, ('sp', -1)) or 'signature':
  2367.                             [self._decrypt_signature(encrypted_sig, video_id, player_url)],
  2368.                     })
  2369.                 except ExtractorError as e:
  2370.                     self.report_warning('Signature extraction failed: Some formats may be missing',
  2371.                                         video_id=video_id, only_once=True)
  2372.                     self.write_debug(error_to_compat_str(e), only_once=True)
  2373.                     continue
  2375.             language_preference = (
  2376.                 10 if audio_track.get('audioIsDefault')
  2377.                 else -10 if 'descriptive' in (traverse_obj(audio_track, ('displayName', T(lower))) or '')
  2378.                 else -1)
  2379.             name = (
  2380.                 traverse_obj(fmt, ('qualityLabel', T(txt_or_none)))
  2381.                 or quality.replace('audio_quality_', ''))
  2382.             dct = {
  2383.                 'format_id': join_nonempty(itag, fmt.get('isDrc') and 'drc'),
  2384.                 'url': fmt_url,
  2385.                 # Format 22 is likely to be damaged: see https://github.com/yt-dlp/yt-dlp/issues/3372
  2386.                 'source_preference': ((-5 if itag == '22' else -1)
  2387.                                       + (100 if 'Premium' in name else 0)),
  2388.                 'quality': q(quality),
  2389.                 'language': join_nonempty(audio_track.get('id', '').split('.')[0],
  2390.                                           'desc' if language_preference < -1 else '') or None,
  2391.                 'language_preference': language_preference,
  2392.                 # Strictly de-prioritize 3gp formats
  2393.                 'preference': -2 if itag == '17' else None,
  2394.             }
  2395.             if itag:
  2396.                 itags[itag].add(('https', dct.get('language')))
  2397.             self._unthrottle_format_urls(video_id, player_url, dct)
  2398.             dct.update(traverse_obj(fmt, {
  2399.                 'asr': ('audioSampleRate', T(int_or_none)),
  2400.                 'filesize': ('contentLength', T(int_or_none)),
  2401.                 'format_note': ('qualityLabel', T(lambda x: x or quality)),
  2402.                 # for some formats, fps is wrongly returned as 1
  2403.                 'fps': ('fps', T(int_or_none), T(lambda f: f if f > 1 else None)),
  2404.                 'audio_channels': ('audioChannels', T(int_or_none)),
  2405.                 'height': ('height', T(int_or_none)),
  2406.                 'has_drm': ('drmFamilies', T(bool)),
  2407.                 'tbr': (('averageBitrate', 'bitrate'), T(lambda t: float_or_none(t, 1000))),
  2408.                 'width': ('width', T(int_or_none)),
  2409.                 '_duration_ms': ('approxDurationMs', T(int_or_none)),
  2410.             }, get_all=False))
  2411.             mime_mobj = re.match(
  2412.                 r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
  2413.             if mime_mobj:
  2414.                 dct['ext'] = mimetype2ext(mime_mobj.group(1))
  2415.                 dct.update(parse_codecs(mime_mobj.group(2)))
  2416.             single_stream = 'none' in (dct.get(c) for c in ('acodec', 'vcodec'))
  2417.             if single_stream and dct.get('ext'):
  2418.                 dct['container'] = dct['ext'] + '_dash'
  2419.             if single_stream or itag == '17':
  2420.                 # avoid Youtube throttling
  2421.                 dct.update({
  2422.                     'protocol': 'http_dash_segments',
  2423.                     'fragments': build_fragments(dct),
  2424.                 } if dct['filesize'] else {
  2425.                     'downloader_options': {'http_chunk_size': CHUNK_SIZE},  # No longer useful?
  2426.                 })
  2428.             formats.append(dct)
  2430.         def process_manifest_format(f, proto, client_name, itag, all_formats=False):
  2431.             key = (proto, f.get('language'))
  2432.             if not all_formats and key in itags[itag]:
  2433.                 return False
  2434.             itags[itag].add(key)
  2436.             if itag:
  2437.                 f['format_id'] = (
  2438.                     '{0}-{1}'.format(itag, proto)
  2439.                     if all_formats or any(p != proto for p, _ in itags[itag])
  2440.                     else itag)
  2442.             if f.get('source_preference') is None:
  2443.                 f['source_preference'] = -1
  2445.             if itag in ('616', '235'):
  2446.                 f['format_note'] = join_nonempty(f.get('format_note'), 'Premium', delim=' ')
  2447.                 f['source_preference'] += 100
  2449.             f['quality'] = q(traverse_obj(f, (
  2450.                 'format_id', T(lambda s: itag_qualities[s.split('-')[0]])), default=-1))
  2451.             if try_call(lambda: f['fps'] <= 1):
  2452.                 del f['fps']
  2454.             if proto == 'hls' and f.get('has_drm'):
  2455.                 f['has_drm'] = 'maybe'
  2456.                 f['source_preference'] -= 5
  2457.             return True
  2459.         hls_manifest_url = streaming_data.get('hlsManifestUrl')
  2460.         if hls_manifest_url:
  2461.             for f in self._extract_m3u8_formats(
  2462.                     hls_manifest_url, video_id, 'mp4',
  2463.                     entry_protocol='m3u8_native', live=is_live, fatal=False):
  2464.                 if process_manifest_format(
  2465.                         f, 'hls', None, self._search_regex(
  2466.                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
  2467.                     formats.append(f)
  2469.         if self._downloader.params.get('youtube_include_dash_manifest', True):
  2470.             dash_manifest_url = streaming_data.get('dashManifestUrl')
  2471.             if dash_manifest_url:
  2472.                 for f in self._extract_mpd_formats(
  2473.                         dash_manifest_url, video_id, fatal=False):
  2474.                     if process_manifest_format(
  2475.                             f, 'dash', None, f['format_id']):
  2476.                         f['filesize'] = traverse_obj(f, (
  2477.                             ('fragment_base_url', 'url'), T(lambda u: self._search_regex(
  2478.                                 r'/clen/(\d+)', u, 'file size', default=None)),
  2479.                             T(int_or_none)), get_all=False)
  2480.                         formats.append(f)
  2482.         playable_formats = [f for f in formats if not f.get('has_drm')]
  2483.         if formats:
  2484.             if not playable_formats:
  2485.                 # If there are no formats that definitely don't have DRM, all have DRM
  2486.                 self.report_drm(video_id)
  2487.             formats[:] = playable_formats
  2488.         else:
  2489.             if streaming_data.get('licenseInfos'):
  2490.                 raise ExtractorError(
  2491.                     'This video is DRM protected.', expected=True)
  2492.             pemr = try_get(
  2493.                 playability_status,
  2494.                 lambda x: x['errorScreen']['playerErrorMessageRenderer'],
  2495.                 dict) or {}
  2496.             reason = get_text(pemr.get('reason')) or playability_status.get('reason')
  2497.             subreason = pemr.get('subreason')
  2498.             if subreason:
  2499.                 subreason = clean_html(get_text(subreason))
  2500.                 if subreason == 'The uploader has not made this video available in your country.':
  2501.                     countries = microformat.get('availableCountries')
  2502.                     if not countries:
  2503.                         regions_allowed = search_meta('regionsAllowed')
  2504.                         countries = regions_allowed.split(',') if regions_allowed else None
  2505.                     self.raise_geo_restricted(
  2506.                         subreason, countries)
  2507.                 reason += '\n' + subreason
  2508.             if reason:
  2509.                 raise ExtractorError(reason, expected=True)
  2511.         self._sort_formats(formats)
  2513.         keywords = video_details.get('keywords') or []
  2514.         if not keywords and webpage:
  2515.             keywords = [
  2516.                 unescapeHTML(m.group('content'))
  2517.                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
  2518.         for keyword in keywords:
  2519.             if keyword.startswith('yt:stretch='):
  2520.                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
  2521.                 if mobj:
  2522.                     # NB: float is intentional for forcing float division
  2523.                     w, h = (float(v) for v in mobj.groups())
  2524.                     if w > 0 and h > 0:
  2525.                         ratio = w / h
  2526.                         for f in formats:
  2527.                             if f.get('vcodec') != 'none':
  2528.                                 f['stretched_ratio'] = ratio
  2529.                         break
  2531.         thumbnails = []
  2532.         for container in (video_details, microformat):
  2533.             for thumbnail in try_get(
  2534.                     container,
  2535.                     lambda x: x['thumbnail']['thumbnails'], list) or []:
  2536.                 thumbnail_url = url_or_none(thumbnail.get('url'))
  2537.                 if not thumbnail_url:
  2538.                     continue
  2539.                 thumbnails.append({
  2540.                     'height': int_or_none(thumbnail.get('height')),
  2541.                     'url': update_url(thumbnail_url, query=None, fragment=None),
  2542.                     'width': int_or_none(thumbnail.get('width')),
  2543.                 })
  2544.             if thumbnails:
  2545.                 break
  2546.         else:
  2547.             thumbnail = search_meta(['og:image', 'twitter:image'])
  2548.             if thumbnail:
  2549.                 thumbnails = [{'url': thumbnail}]
  2551.         category = microformat.get('category') or search_meta('genre')
  2552.         channel_id = self._extract_channel_id(
  2553.             webpage, videodetails=video_details, metadata=microformat)
  2554.         duration = int_or_none(
  2555.             video_details.get('lengthSeconds')
  2556.             or microformat.get('lengthSeconds')) \
  2557.             or parse_duration(search_meta('duration'))
  2559.         for f in formats:
  2560.             # Some formats may have much smaller duration than others (possibly damaged during encoding)
  2561.             # but avoid false positives with small duration differences.
  2562.             # Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
  2563.             if try_call(lambda x: float(x.pop('_duration_ms')) / duration < 500, args=(f,)):
  2564.                 self.report_warning(
  2565.                     '{0}: Some possibly damaged formats will be deprioritized'.format(video_id), only_once=True)
  2566.                 # Strictly de-prioritize damaged formats
  2567.                 f['preference'] = -10
  2569.         owner_profile_url = self._yt_urljoin(self._extract_author_var(
  2570.             webpage, 'url', videodetails=video_details, metadata=microformat))
  2572.         uploader = self._extract_author_var(
  2573.             webpage, 'name', videodetails=video_details, metadata=microformat)
  2575.         info = {
  2576.             'id': video_id,
  2577.             'title': self._live_title(video_title) if is_live else video_title,
  2578.             'formats': formats,
  2579.             'thumbnails': thumbnails,
  2580.             'description': video_description,
  2581.             'upload_date': unified_strdate(
  2582.                 microformat.get('uploadDate')
  2583.                 or search_meta('uploadDate')),
  2584.             'uploader': uploader,
  2585.             'channel_id': channel_id,
  2586.             'duration': duration,
  2587.             'view_count': int_or_none(
  2588.                 video_details.get('viewCount')
  2589.                 or microformat.get('viewCount')
  2590.                 or search_meta('interactionCount')),
  2591.             'average_rating': float_or_none(video_details.get('averageRating')),
  2592.             'age_limit': 18 if (
  2593.                 microformat.get('isFamilySafe') is False
  2594.                 or search_meta('isFamilyFriendly') == 'false'
  2595.                 or search_meta('og:restrictions:age') == '18+') else 0,
  2596.             'webpage_url': webpage_url,
  2597.             'categories': [category] if category else None,
  2598.             'tags': keywords,
  2599.             'is_live': is_live,
  2600.         }
  2602.         pctr = traverse_obj(
  2603.             (player_response, api_player_response),
  2604.             (Ellipsis, 'captions', 'playerCaptionsTracklistRenderer', T(dict)))
  2605.         if pctr:
  2606.             def process_language(container, base_url, lang_code, query):
  2607.                 lang_subs = []
  2608.                 for fmt in self._SUBTITLE_FORMATS:
  2609.                     query.update({
  2610.                         'fmt': fmt,
  2611.                     })
  2612.                     lang_subs.append({
  2613.                         'ext': fmt,
  2614.                         'url': update_url_query(base_url, query),
  2615.                     })
  2616.                 container[lang_code] = lang_subs
  2618.             def process_subtitles():
  2619.                 subtitles = {}
  2620.                 for caption_track in traverse_obj(pctr, (
  2621.                         Ellipsis, 'captionTracks', lambda _, v: (
  2622.                             v.get('baseUrl') and v.get('languageCode')))):
  2623.                     base_url = self._yt_urljoin(caption_track['baseUrl'])
  2624.                     if not base_url:
  2625.                         continue
  2626.                     lang_code = caption_track['languageCode']
  2627.                     if caption_track.get('kind') != 'asr':
  2628.                         process_language(
  2629.                             subtitles, base_url, lang_code, {})
  2630.                         continue
  2631.                     automatic_captions = {}
  2632.                     process_language(
  2633.                         automatic_captions, base_url, lang_code, {})
  2634.                     for translation_language in traverse_obj(pctr, (
  2635.                             Ellipsis, 'translationLanguages', lambda _, v: v.get('languageCode'))):
  2636.                         translation_language_code = translation_language['languageCode']
  2637.                         process_language(
  2638.                             automatic_captions, base_url, translation_language_code,
  2639.                             {'tlang': translation_language_code})
  2640.                     info['automatic_captions'] = automatic_captions
  2641.                 info['subtitles'] = subtitles
  2643.             process_subtitles()
  2645.         parsed_url = compat_urllib_parse_urlparse(url)
  2646.         for component in (parsed_url.fragment, parsed_url.query):
  2647.             query = compat_parse_qs(component)
  2648.             for k, v in query.items():
  2649.                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
  2650.                     d_k += '_time'
  2651.                     if d_k not in info and k in s_ks:
  2652.                         info[d_k] = parse_duration(query[k][0])
  2654.         if video_description:
  2655.             # Youtube Music Auto-generated description
  2656.             mobj = re.search(r'(?s)(?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?.+\nAuto-generated by YouTube\.\s*$', video_description)
  2657.             if mobj:
  2658.                 release_year = mobj.group('release_year')
  2659.                 release_date = mobj.group('release_date')
  2660.                 if release_date:
  2661.                     release_date = release_date.replace('-', '')
  2662.                     if not release_year:
  2663.                         release_year = release_date[:4]
  2664.                 info.update({
  2665.                     'album': mobj.group('album'.strip()),
  2666.                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
  2667.                     'track': mobj.group('track').strip(),
  2668.                     'release_date': release_date,
  2669.                     'release_year': int_or_none(release_year),
  2670.                 })
  2672.         initial_data = None
  2673.         if webpage:
  2674.             initial_data = self._extract_yt_initial_variable(
  2675.                 webpage, self._YT_INITIAL_DATA_RE, video_id,
  2676.                 'yt initial data')
  2677.         if not initial_data:
  2678.             initial_data = self._call_api(
  2679.                 'next', {'videoId': video_id}, video_id, fatal=False)
  2681.         if initial_data:
  2682.             chapters = self._extract_chapters_from_json(
  2683.                 initial_data, video_id, duration)
  2684.             if not chapters:
  2685.                 for engagment_pannel in (initial_data.get('engagementPanels') or []):
  2686.                     contents = try_get(
  2687.                         engagment_pannel, lambda x: x['engagementPanelSectionListRenderer']['content']['macroMarkersListRenderer']['contents'],
  2688.                         list)
  2689.                     if not contents:
  2690.                         continue
  2692.                     def chapter_time(mmlir):
  2693.                         return parse_duration(
  2694.                             get_text(mmlir.get('timeDescription')))
  2696.                     chapters = []
  2697.                     for next_num, content in enumerate(contents, start=1):
  2698.                         mmlir = content.get('macroMarkersListItemRenderer') or {}
  2699.                         start_time = chapter_time(mmlir)
  2700.                         end_time = chapter_time(try_get(
  2701.                             contents, lambda x: x[next_num]['macroMarkersListItemRenderer'])) \
  2702.                             if next_num < len(contents) else duration
  2703.                         if start_time is None or end_time is None:
  2704.                             continue
  2705.                         chapters.append({
  2706.                             'start_time': start_time,
  2707.                             'end_time': end_time,
  2708.                             'title': get_text(mmlir.get('title')),
  2709.                         })
  2710.                     if chapters:
  2711.                         break
  2712.             if chapters:
  2713.                 info['chapters'] = chapters
  2715.             contents = try_get(
  2716.                 initial_data,
  2717.                 lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'],
  2718.                 list) or []
  2719.             if not info['channel_id']:
  2720.                 channel_id = self._extract_channel_id('', renderers=contents)
  2721.             if not info['uploader']:
  2722.                 info['uploader'] = self._extract_author_var('', 'name', renderers=contents)
  2723.             if not owner_profile_url:
  2724.                 owner_profile_url = self._yt_urljoin(self._extract_author_var('', 'url', renderers=contents))
  2726.             for content in contents:
  2727.                 vpir = content.get('videoPrimaryInfoRenderer')
  2728.                 if vpir:
  2729.                     stl = vpir.get('superTitleLink')
  2730.                     if stl:
  2731.                         stl = get_text(stl)
  2732.                         if try_get(
  2733.                                 vpir,
  2734.                                 lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
  2735.                             info['location'] = stl
  2736.                         else:
  2737.                             # •? doesn't match, but [•]? does; \xa0 = non-breaking space
  2738.                             mobj = re.search(r'([^\xa0\s].*?)[\xa0\s]*S(\d+)[\xa0\s]*[•]?[\xa0\s]*E(\d+)', stl)
  2739.                             if mobj:
  2740.                                 info.update({
  2741.                                     'series': mobj.group(1),
  2742.                                     'season_number': int(mobj.group(2)),
  2743.                                     'episode_number': int(mobj.group(3)),
  2744.                                 })
  2745.                     for tlb in (try_get(
  2746.                             vpir,
  2747.                             lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
  2748.                             list) or []):
  2749.                         tbr = traverse_obj(tlb, ('segmentedLikeDislikeButtonRenderer', 'likeButton', 'toggleButtonRenderer'), 'toggleButtonRenderer') or {}
  2750.                         for getter, regex in [(
  2751.                                 lambda x: x['defaultText']['accessibility']['accessibilityData'],
  2752.                                 r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
  2753.                                     lambda x: x['accessibility'],
  2754.                                     lambda x: x['accessibilityData']['accessibilityData'],
  2755.                                 ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
  2756.                             label = (try_get(tbr, getter, dict) or {}).get('label')
  2757.                             if label:
  2758.                                 mobj = re.match(regex, label)
  2759.                                 if mobj:
  2760.                                     info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
  2761.                                     break
  2762.                     sbr_tooltip = try_get(
  2763.                         vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
  2764.                     if sbr_tooltip:
  2765.                         # however dislike_count was hidden by YT, as if there could ever be dislikable content on YT
  2766.                         like_count, dislike_count = sbr_tooltip.split(' / ')
  2767.                         info.update({
  2768.                             'like_count': str_to_int(like_count),
  2769.                             'dislike_count': str_to_int(dislike_count),
  2770.                         })
  2771.                     else:
  2772.                         info['like_count'] = traverse_obj(vpir, (
  2773.                             'videoActions', 'menuRenderer', 'topLevelButtons', Ellipsis,
  2774.                             'segmentedLikeDislikeButtonViewModel', 'likeButtonViewModel', 'likeButtonViewModel',
  2775.                             'toggleButtonViewModel', 'toggleButtonViewModel', 'defaultButtonViewModel',
  2776.                             'buttonViewModel', (('title', ('accessibilityText', T(lambda s: s.split()), Ellipsis))), T(parse_count)),
  2777.                             get_all=False)
  2779.                 vsir = content.get('videoSecondaryInfoRenderer')
  2780.                 if vsir:
  2781.                     rows = try_get(
  2782.                         vsir,
  2783.                         lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
  2784.                         list) or []
  2785.                     multiple_songs = False
  2786.                     for row in rows:
  2787.                         if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
  2788.                             multiple_songs = True
  2789.                             break
  2790.                     for row in rows:
  2791.                         mrr = row.get('metadataRowRenderer') or {}
  2792.                         mrr_title = mrr.get('title')
  2793.                         if not mrr_title:
  2794.                             continue
  2795.                         mrr_title = get_text(mrr['title'])
  2796.                         mrr_contents_text = get_text(mrr['contents'][0])
  2797.                         if mrr_title == 'License':
  2798.                             info['license'] = mrr_contents_text
  2799.                         elif not multiple_songs:
  2800.                             if mrr_title == 'Album':
  2801.                                 info['album'] = mrr_contents_text
  2802.                             elif mrr_title == 'Artist':
  2803.                                 info['artist'] = mrr_contents_text
  2804.                             elif mrr_title == 'Song':
  2805.                                 info['track'] = mrr_contents_text
  2807.             # this is not extraction but spelunking!
  2808.             carousel_lockups = traverse_obj(
  2809.                 initial_data,
  2810.                 ('engagementPanels', Ellipsis, 'engagementPanelSectionListRenderer',
  2811.                  'content', 'structuredDescriptionContentRenderer', 'items', Ellipsis,
  2812.                  'videoDescriptionMusicSectionRenderer', 'carouselLockups', Ellipsis),
  2813.                 expected_type=dict) or []
  2814.             # try to reproduce logic from metadataRowContainerRenderer above (if it still is)
  2815.             fields = (('ALBUM', 'album'), ('ARTIST', 'artist'), ('SONG', 'track'), ('LICENSES', 'license'))
  2816.             # multiple_songs ?
  2817.             if len(carousel_lockups) > 1:
  2818.                 fields = fields[-1:]
  2819.             for info_row in traverse_obj(
  2820.                     carousel_lockups,
  2821.                     (0, 'carouselLockupRenderer', 'infoRows', Ellipsis, 'infoRowRenderer'),
  2822.                     expected_type=dict):
  2823.                 row_title = traverse_obj(info_row, ('title', 'simpleText'))
  2824.                 row_text = traverse_obj(info_row, 'defaultMetadata', 'expandedMetadata', expected_type=get_text)
  2825.                 if not row_text:
  2826.                     continue
  2827.                 for name, field in fields:
  2828.                     if name == row_title and not info.get(field):
  2829.                         info[field] = row_text
  2831.         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
  2832.             v = info.get(s_k)
  2833.             if v:
  2834.                 info[d_k] = v
  2836.         self.mark_watched(video_id, player_response)
  2838.         return merge_dicts(
  2839.             info, {
  2840.                 'uploader_id': self._extract_uploader_id(owner_profile_url),
  2841.                 'uploader_url': owner_profile_url,
  2842.                 'channel_id': channel_id,
  2843.                 'channel_url': channel_id and self._yt_urljoin('/channel/' + channel_id),
  2844.                 'channel': info['uploader'],
  2845.             })
  2848. class YoutubeTabIE(YoutubeBaseInfoExtractor):
  2849.     IE_DESC = 'YouTube.com tab'
  2850.     _VALID_URL = r'''(?x)
  2851.                     https?://
  2852.                         (?:\w+\.)?
  2853.                         (?:
  2854.                             youtube(?:kids)?\.com|
  2855.                             invidio\.us
  2856.                         )/
  2857.                         (?:
  2858.                             (?:channel|c|user|feed|hashtag)/|
  2859.                             (?:playlist|watch)\?.*?\blist=|
  2860.                             (?!(?:watch|embed|v|e|results)\b)
  2861.                         )
  2862.                         (?P<id>[^/?\#&]+)
  2863.                     '''
  2864.     IE_NAME = 'youtube:tab'
  2866.     _TESTS = [{
  2867.         # Shorts
  2868.         'url': 'https://www.youtube.com/@SuperCooperShorts/shorts',
  2869.         'playlist_mincount': 5,
  2870.         'info_dict': {
  2871.             'description': 'Short clips from Super Cooper Sundays!',
  2872.             'id': 'UCKMA8kHZ8bPYpnMNaUSxfEQ',
  2873.             'title': 'Super Cooper Shorts - Shorts',
  2874.             'uploader': 'Super Cooper Shorts',
  2875.             'uploader_id': '@SuperCooperShorts',
  2876.         },
  2877.     }, {
  2878.         # Channel that does not have a Shorts tab. Test should just download videos on Home tab instead
  2879.         'url': 'https://www.youtube.com/@emergencyawesome/shorts',
  2880.         'info_dict': {
  2881.             'description': 'md5:592c080c06fef4de3c902c4a8eecd850',
  2882.             'id': 'UCDiFRMQWpcp8_KD4vwIVicw',
  2883.             'title': 'Emergency Awesome - Home',
  2884.         },
  2885.         'playlist_mincount': 5,
  2886.         'skip': 'new test page needed to replace `Emergency Awesome - Shorts`',
  2887.     }, {
  2888.         # playlists, multipage
  2889.         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
  2890.         'playlist_mincount': 94,
  2891.         'info_dict': {
  2892.             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
  2893.             'title': r're:Igor Kleiner(?: Ph\.D\.)? - Playlists',
  2894.             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
  2895.             'uploader': 'Igor Kleiner',
  2896.             'uploader_id': '@IgorDataScience',
  2897.         },
  2898.     }, {
  2899.         # playlists, multipage, different order
  2900.         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
  2901.         'playlist_mincount': 94,
  2902.         'info_dict': {
  2903.             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
  2904.             'title': r're:Igor Kleiner(?: Ph\.D\.)? - Playlists',
  2905.             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
  2906.             'uploader': 'Igor Kleiner',
  2907.             'uploader_id': '@IgorDataScience',
  2908.         },
  2909.     }, {
  2910.         # playlists, series
  2911.         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
  2912.         'playlist_mincount': 5,
  2913.         'info_dict': {
  2914.             'id': 'UCYO_jab_esuFRV4b17AJtAw',
  2915.             'title': '3Blue1Brown - Playlists',
  2916.             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
  2917.             'uploader': '3Blue1Brown',
  2918.             'uploader_id': '@3blue1brown',
  2919.         },
  2920.     }, {
  2921.         # playlists, singlepage
  2922.         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
  2923.         'playlist_mincount': 4,
  2924.         'info_dict': {
  2925.             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
  2926.             'title': 'ThirstForScience - Playlists',
  2927.             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
  2928.             'uploader': 'ThirstForScience',
  2929.             'uploader_id': '@ThirstForScience',
  2930.         },
  2931.     }, {
  2932.         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
  2933.         'only_matching': True,
  2934.     }, {
  2935.         # basic, single video playlist
  2936.         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
  2937.         'info_dict': {
  2938.             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
  2939.             'title': 'youtube-dl public playlist',
  2940.             'uploader': 'Sergey M.',
  2941.             'uploader_id': '@sergeym.6173',
  2942.             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
  2943.         },
  2944.         'playlist_count': 1,
  2945.     }, {
  2946.         # empty playlist
  2947.         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
  2948.         'info_dict': {
  2949.             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
  2950.             'title': 'youtube-dl empty playlist',
  2951.             'uploader': 'Sergey M.',
  2952.             'uploader_id': '@sergeym.6173',
  2953.             'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
  2954.         },
  2955.         'playlist_count': 0,
  2956.     }, {
  2957.         # Home tab
  2958.         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
  2959.         'info_dict': {
  2960.             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
  2961.             'title': 'lex will - Home',
  2962.             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
  2963.             'uploader': 'lex will',
  2964.             'uploader_id': '@lexwill718',
  2965.         },
  2966.         'playlist_mincount': 2,
  2967.     }, {
  2968.         # Videos tab
  2969.         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
  2970.         'info_dict': {
  2971.             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
  2972.             'title': 'lex will - Videos',
  2973.             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
  2974.             'uploader': 'lex will',
  2975.             'uploader_id': '@lexwill718',
  2976.         },
  2977.         'playlist_mincount': 975,
  2978.     }, {
  2979.         # Videos tab, sorted by popular
  2980.         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
  2981.         'info_dict': {
  2982.             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
  2983.             'title': 'lex will - Videos',
  2984.             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
  2985.             'uploader': 'lex will',
  2986.             'uploader_id': '@lexwill718',
  2987.         },
  2988.         'playlist_mincount': 199,
  2989.     }, {
  2990.         # Playlists tab
  2991.         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
  2992.         'info_dict': {
  2993.             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
  2994.             'title': 'lex will - Playlists',
  2995.             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
  2996.             'uploader': 'lex will',
  2997.             'uploader_id': '@lexwill718',
  2998.         },
  2999.         'playlist_mincount': 17,
  3000.     }, {
  3001.         # Community tab
  3002.         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
  3003.         'info_dict': {
  3004.             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
  3005.             'title': 'lex will - Community',
  3006.             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
  3007.             'uploader': 'lex will',
  3008.             'uploader_id': '@lexwill718',
  3009.         },
  3010.         'playlist_mincount': 18,
  3011.     }, {
  3012.         # Channels tab
  3013.         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
  3014.         'info_dict': {
  3015.             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
  3016.             'title': r're:lex will - (?:Home|Channels)',
  3017.             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
  3018.             'uploader': 'lex will',
  3019.             'uploader_id': '@lexwill718',
  3020.         },
  3021.         'playlist_mincount': 75,
  3022.     }, {
  3023.         # Releases tab
  3024.         'url': 'https://www.youtube.com/@daftpunk/releases',
  3025.         'info_dict': {
  3026.             'id': 'UC_kRDKYrUlrbtrSiyu5Tflg',
  3027.             'title': 'Daft Punk - Releases',
  3028.             'description': 'Daft Punk (1993 - 2021) - Official YouTube Channel',
  3029.             'uploader_id': '@daftpunk',
  3030.             'uploader': 'Daft Punk',
  3031.         },
  3032.         'playlist_mincount': 36,
  3033.     }, {
  3034.         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
  3035.         'only_matching': True,
  3036.     }, {
  3037.         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
  3038.         'only_matching': True,
  3039.     }, {
  3040.         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
  3041.         'only_matching': True,
  3042.     }, {
  3043.         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
  3044.         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
  3045.         'info_dict': {
  3046.             'title': '29C3: Not my department',
  3047.             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
  3048.             'uploader': 'Christiaan008',
  3049.             'uploader_id': '@ChRiStIaAn008',
  3050.             'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
  3051.         },
  3052.         'playlist_count': 96,
  3053.     }, {
  3054.         'note': 'Large playlist',
  3055.         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
  3056.         'info_dict': {
  3057.             'title': 'Uploads from Cauchemar',
  3058.             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
  3059.             'uploader': 'Cauchemar',
  3060.             'uploader_id': '@Cauchemar89',
  3061.             'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
  3062.         },
  3063.         'playlist_mincount': 1123,
  3064.     }, {
  3065.         # even larger playlist, 8832 videos
  3066.         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
  3067.         'only_matching': True,
  3068.     }, {
  3069.         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
  3070.         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
  3071.         'info_dict': {
  3072.             'title': 'Uploads from Interstellar Movie',
  3073.             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
  3074.             'uploader': 'Interstellar Movie',
  3075.             'uploader_id': '@InterstellarMovie',
  3076.             'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
  3077.         },
  3078.         'playlist_mincount': 21,
  3079.     }, {
  3080.         # https://github.com/ytdl-org/youtube-dl/issues/21844
  3081.         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
  3082.         'info_dict': {
  3083.             'title': 'Data Analysis with Dr Mike Pound',
  3084.             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
  3085.             'uploader': 'Computerphile',
  3086.             'uploader_id': '@Computerphile',
  3087.             'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
  3088.         },
  3089.         'playlist_mincount': 11,
  3090.     }, {
  3091.         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
  3092.         'only_matching': True,
  3093.     }, {
  3094.         # Playlist URL that does not actually serve a playlist
  3095.         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
  3096.         'info_dict': {
  3097.             'id': 'FqZTN594JQw',
  3098.             'ext': 'webm',
  3099.             'title': "Smiley's People 01 detective, Adventure Series, Action",
  3100.             'uploader': 'STREEM',
  3101.             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
  3102.             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
  3103.             'upload_date': '20150526',
  3104.             'license': 'Standard YouTube License',
  3105.             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
  3106.             'categories': ['People & Blogs'],
  3107.             'tags': list,
  3108.             'view_count': int,
  3109.             'like_count': int,
  3110.         },
  3111.         'params': {
  3112.             'skip_download': True,
  3113.         },
  3114.         'skip': 'This video is not available.',
  3115.         'add_ie': [YoutubeIE.ie_key()],
  3116.     }, {
  3117.         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
  3118.         'only_matching': True,
  3119.     }, {
  3120.         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
  3121.         'only_matching': True,
  3122.     }, {
  3123.         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
  3124.         'info_dict': {
  3125.             'id': r're:[\da-zA-Z_-]{8,}',
  3126.             'ext': 'mp4',
  3127.             'title': r're:(?s)[A-Z].{20,}',
  3128.             'uploader': 'Sky News',
  3129.             'uploader_id': '@SkyNews',
  3130.             'uploader_url': r're:https?://(?:www\.)?youtube\.com/@SkyNews',
  3131.             'upload_date': r're:\d{8}',
  3132.             'description': r're:(?s)(?:.*\n)+SUBSCRIBE to our YouTube channel for more videos: http://www\.youtube\.com/skynews *\n.*',
  3133.             'categories': ['News & Politics'],
  3134.             'tags': list,
  3135.             'like_count': int,
  3136.         },
  3137.         'params': {
  3138.             'skip_download': True,
  3139.         },
  3140.     }, {
  3141.         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
  3142.         'info_dict': {
  3143.             'id': 'a48o2S1cPoo',
  3144.             'ext': 'mp4',
  3145.             'title': 'The Young Turks - Live Main Show',
  3146.             'uploader': 'The Young Turks',
  3147.             'uploader_id': 'TheYoungTurks',
  3148.             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
  3149.             'upload_date': '20150715',
  3150.             'license': 'Standard YouTube License',
  3151.             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
  3152.             'categories': ['News & Politics'],
  3153.             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
  3154.             'like_count': int,
  3155.         },
  3156.         'params': {
  3157.             'skip_download': True,
  3158.         },
  3159.         'only_matching': True,
  3160.     }, {
  3161.         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
  3162.         'only_matching': True,
  3163.     }, {
  3164.         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
  3165.         'only_matching': True,
  3166.     }, {
  3167.         'url': 'https://www.youtube.com/feed/trending',
  3168.         'only_matching': True,
  3169.     }, {
  3170.         # needs auth
  3171.         'url': 'https://www.youtube.com/feed/library',
  3172.         'only_matching': True,
  3173.     }, {
  3174.         # needs auth
  3175.         'url': 'https://www.youtube.com/feed/history',
  3176.         'only_matching': True,
  3177.     }, {
  3178.         # needs auth
  3179.         'url': 'https://www.youtube.com/feed/subscriptions',
  3180.         'only_matching': True,
  3181.     }, {
  3182.         # needs auth
  3183.         'url': 'https://www.youtube.com/feed/watch_later',
  3184.         'only_matching': True,
  3185.     }, {
  3186.         # no longer available?
  3187.         'url': 'https://www.youtube.com/feed/recommended',
  3188.         'only_matching': True,
  3189.     }, {
  3190.         # inline playlist with not always working continuations
  3191.         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
  3192.         'only_matching': True,
  3193.     }, {
  3194.         'url': 'https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8',
  3195.         'only_matching': True,
  3196.     }, {
  3197.         'url': 'https://www.youtube.com/course',
  3198.         'only_matching': True,
  3199.     }, {
  3200.         'url': 'https://www.youtube.com/zsecurity',
  3201.         'only_matching': True,
  3202.     }, {
  3203.         'url': 'http://www.youtube.com/NASAgovVideo/videos',
  3204.         'only_matching': True,
  3205.     }, {
  3206.         'url': 'https://www.youtube.com/TheYoungTurks/live',
  3207.         'only_matching': True,
  3208.     }, {
  3209.         'url': 'https://www.youtube.com/hashtag/cctv9',
  3210.         'info_dict': {
  3211.             'id': 'cctv9',
  3212.             'title': '#cctv9',
  3213.         },
  3214.         'playlist_mincount': 350,
  3215.     }, {
  3216.         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
  3217.         'only_matching': True,
  3218.     }, {
  3219.         'note': 'Search tab',
  3220.         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
  3221.         'playlist_mincount': 20,
  3222.         'info_dict': {
  3223.             'id': 'UCYO_jab_esuFRV4b17AJtAw',
  3224.             'title': '3Blue1Brown - Search - linear algebra',
  3225.             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
  3226.             'uploader': '3Blue1Brown',
  3227.             'uploader_id': '@3blue1brown',
  3228.             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
  3229.         },
  3230.     }]
  3232.     @classmethod
  3233.     def suitable(cls, url):
  3234.         return not YoutubeIE.suitable(url) and super(
  3235.             YoutubeTabIE, cls).suitable(url)
  3237.     @staticmethod
  3238.     def _extract_grid_item_renderer(item):
  3239.         assert isinstance(item, dict)
  3240.         for key, renderer in item.items():
  3241.             if not key.startswith('grid') or not key.endswith('Renderer'):
  3242.                 continue
  3243.             if not isinstance(renderer, dict):
  3244.                 continue
  3245.             return renderer
  3247.     @staticmethod
  3248.     def _get_text(r, k):
  3249.         return traverse_obj(
  3250.             r, (k, 'runs', 0, 'text'), (k, 'simpleText'),
  3251.             expected_type=txt_or_none)
  3253.     def _grid_entries(self, grid_renderer):
  3254.         for item in traverse_obj(grid_renderer, ('items', Ellipsis, T(dict))):
  3255.             lockup_view_model = traverse_obj(item, ('lockupViewModel', T(dict)))
  3256.             if lockup_view_model:
  3257.                 entry = self._extract_lockup_view_model(lockup_view_model)
  3258.                 if entry:
  3259.                     yield entry
  3260.                 continue
  3261.             renderer = self._extract_grid_item_renderer(item)
  3262.             if not isinstance(renderer, dict):
  3263.                 continue
  3264.             title = self._get_text(renderer, 'title')
  3265.             # playlist
  3266.             playlist_id = renderer.get('playlistId')
  3267.             if playlist_id:
  3268.                 yield self.url_result(
  3269.                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
  3270.                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
  3271.                     video_title=title)
  3272.                 continue
  3273.             # video
  3274.             video_id = renderer.get('videoId')
  3275.             if video_id:
  3276.                 yield self._extract_video(renderer)
  3277.                 continue
  3278.             # channel
  3279.             channel_id = renderer.get('channelId')
  3280.             if channel_id:
  3281.                 title = self._get_text(renderer, 'title')
  3282.                 yield self.url_result(
  3283.                     'https://www.youtube.com/channel/%s' % channel_id,
  3284.                     ie=YoutubeTabIE.ie_key(), video_title=title)
  3285.                 continue
  3286.             # generic endpoint URL support
  3287.             ep_url = urljoin('https://www.youtube.com/', try_get(
  3288.                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
  3289.                 compat_str))
  3290.             if ep_url:
  3291.                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
  3292.                     if ie.suitable(ep_url):
  3293.                         yield self.url_result(
  3294.                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
  3295.                         break
  3297.     def _shelf_entries_from_content(self, shelf_renderer):
  3298.         content = shelf_renderer.get('content')
  3299.         if not isinstance(content, dict):
  3300.             return
  3301.         renderer = content.get('gridRenderer')
  3302.         if renderer:
  3303.             # TODO: add support for nested playlists so each shelf is processed
  3304.             # as separate playlist
  3305.             # TODO: this includes only first N items
  3306.             for entry in self._grid_entries(renderer):
  3307.                 yield entry
  3308.         renderer = content.get('horizontalListRenderer')
  3309.         if renderer:
  3310.             # TODO
  3311.             pass
  3313.     def _shelf_entries(self, shelf_renderer, skip_channels=False):
  3314.         ep = try_get(
  3315.             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
  3316.             compat_str)
  3317.         shelf_url = urljoin('https://www.youtube.com', ep)
  3318.         if shelf_url:
  3319.             # Skipping links to another channels, note that checking for
  3320.             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
  3321.             # will not work
  3322.             if skip_channels and '/channels?' in shelf_url:
  3323.                 return
  3324.             title = try_get(
  3325.                 shelf_renderer, lambda x: x['title']['runs'][0]['text'], compat_str)
  3326.             yield self.url_result(shelf_url, video_title=title)
  3327.         # Shelf may not contain shelf URL, fallback to extraction from content
  3328.         for entry in self._shelf_entries_from_content(shelf_renderer):
  3329.             yield entry
  3331.     def _playlist_entries(self, video_list_renderer):
  3332.         for content in video_list_renderer['contents']:
  3333.             if not isinstance(content, dict):
  3334.                 continue
  3335.             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
  3336.             if not isinstance(renderer, dict):
  3337.                 continue
  3338.             video_id = renderer.get('videoId')
  3339.             if not video_id:
  3340.                 continue
  3341.             yield self._extract_video(renderer)
  3343.     def _extract_lockup_view_model(self, view_model):
  3344.         content_id = view_model.get('contentId')
  3345.         if not content_id:
  3346.             return
  3347.         content_type = view_model.get('contentType')
  3348.         if content_type not in ('LOCKUP_CONTENT_TYPE_PLAYLIST', 'LOCKUP_CONTENT_TYPE_PODCAST'):
  3349.             self.report_warning(
  3350.                 'Unsupported lockup view model content type "{0}"{1}'.format(content_type, bug_reports_message()), only_once=True)
  3351.             return
  3352.         return merge_dicts(self.url_result(
  3353.             update_url_query('https://www.youtube.com/playlist', {'list': content_id}),
  3354.             ie=YoutubeTabIE.ie_key(), video_id=content_id), {
  3355.                 'title': traverse_obj(view_model, (
  3356.                     'metadata', 'lockupMetadataViewModel', 'title', 'content', T(compat_str))),
  3357.                 'thumbnails': self._extract_thumbnails(view_model, (
  3358.                     'contentImage', 'collectionThumbnailViewModel', 'primaryThumbnail',
  3359.                     'thumbnailViewModel', 'image'), final_key='sources'),
  3360.         })
  3362.     def _extract_shorts_lockup_view_model(self, view_model):
  3363.         content_id = traverse_obj(view_model, (
  3364.             'onTap', 'innertubeCommand', 'reelWatchEndpoint', 'videoId',
  3365.             T(lambda v: v if YoutubeIE.suitable(v) else None)))
  3366.         if not content_id:
  3367.             return
  3368.         return merge_dicts(self.url_result(
  3369.             content_id, ie=YoutubeIE.ie_key(), video_id=content_id), {
  3370.                 'title': traverse_obj(view_model, (
  3371.                     'overlayMetadata', 'primaryText', 'content', T(compat_str))),
  3372.                 'thumbnails': self._extract_thumbnails(
  3373.                     view_model, 'thumbnail', final_key='sources'),
  3374.         })
  3376.     def _video_entry(self, video_renderer):
  3377.         video_id = video_renderer.get('videoId')
  3378.         if video_id:
  3379.             return self._extract_video(video_renderer)
  3381.     def _post_thread_entries(self, post_thread_renderer):
  3382.         post_renderer = try_get(
  3383.             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
  3384.         if not post_renderer:
  3385.             return
  3386.         # video attachment
  3387.         video_renderer = try_get(
  3388.             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict)
  3389.         video_id = None
  3390.         if video_renderer:
  3391.             entry = self._video_entry(video_renderer)
  3392.             if entry:
  3393.                 yield entry
  3394.         # inline video links
  3395.         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
  3396.         for run in runs:
  3397.             if not isinstance(run, dict):
  3398.                 continue
  3399.             ep_url = try_get(
  3400.                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], compat_str)
  3401.             if not ep_url:
  3402.                 continue
  3403.             if not YoutubeIE.suitable(ep_url):
  3404.                 continue
  3405.             ep_video_id = YoutubeIE._match_id(ep_url)
  3406.             if video_id == ep_video_id:
  3407.                 continue
  3408.             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=video_id)
  3410.     def _post_thread_continuation_entries(self, post_thread_continuation):
  3411.         contents = post_thread_continuation.get('contents')
  3412.         if not isinstance(contents, list):
  3413.             return
  3414.         for content in contents:
  3415.             renderer = content.get('backstagePostThreadRenderer')
  3416.             if not isinstance(renderer, dict):
  3417.                 continue
  3418.             for entry in self._post_thread_entries(renderer):
  3419.                 yield entry
  3421.     def _rich_grid_entries(self, contents):
  3422.         for content in traverse_obj(
  3423.                 contents, (Ellipsis, 'richItemRenderer', 'content'),
  3424.                 expected_type=dict):
  3425.             video_renderer = traverse_obj(
  3426.                 content, 'videoRenderer', 'reelItemRenderer',
  3427.                 expected_type=dict)
  3428.             if video_renderer:
  3429.                 entry = self._video_entry(video_renderer)
  3430.                 if entry:
  3431.                     yield entry
  3432.             # shorts item
  3433.             shorts_lockup_view_model = content.get('shortsLockupViewModel')
  3434.             if shorts_lockup_view_model:
  3435.                 entry = self._extract_shorts_lockup_view_model(shorts_lockup_view_model)
  3436.                 if entry:
  3437.                     yield entry
  3438.             # playlist
  3439.             renderer = traverse_obj(
  3440.                 content, 'playlistRenderer', expected_type=dict) or {}
  3441.             title = self._get_text(renderer, 'title')
  3442.             playlist_id = renderer.get('playlistId')
  3443.             if playlist_id:
  3444.                 yield self.url_result(
  3445.                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
  3446.                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
  3447.                     video_title=title)
  3449.     @staticmethod
  3450.     def _build_continuation_query(continuation, ctp=None):
  3451.         query = {
  3452.             'ctoken': continuation,
  3453.             'continuation': continuation,
  3454.         }
  3455.         if ctp:
  3456.             query['itct'] = ctp
  3457.         return query
  3459.     @staticmethod
  3460.     def _extract_next_continuation_data(renderer):
  3461.         next_continuation = try_get(
  3462.             renderer, lambda x: x['continuations'][0]['nextContinuationData'], dict)
  3463.         if not next_continuation:
  3464.             return
  3465.         continuation = next_continuation.get('continuation')
  3466.         if not continuation:
  3467.             return
  3468.         ctp = next_continuation.get('clickTrackingParams')
  3469.         return YoutubeTabIE._build_continuation_query(continuation, ctp)
  3471.     @classmethod
  3472.     def _extract_continuation(cls, renderer):
  3473.         next_continuation = cls._extract_next_continuation_data(renderer)
  3474.         if next_continuation:
  3475.             return next_continuation
  3476.         for command in traverse_obj(renderer, (
  3477.                 ('contents', 'items', 'rows'), Ellipsis, 'continuationItemRenderer',
  3478.                 ('continuationEndpoint', ('button', 'buttonRenderer', 'command')),
  3479.                 (('commandExecutorCommand', 'commands', Ellipsis), None), T(dict))):
  3480.             continuation = traverse_obj(command, ('continuationCommand', 'token', T(compat_str)))
  3481.             if not continuation:
  3482.                 continue
  3483.             ctp = command.get('clickTrackingParams')
  3484.             return cls._build_continuation_query(continuation, ctp)
  3486.     def _entries(self, tab, item_id, webpage):
  3487.         tab_content = try_get(tab, lambda x: x['content'], dict)
  3488.         if not tab_content:
  3489.             return
  3490.         slr_renderer = try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
  3491.         if slr_renderer:
  3492.             is_channels_tab = tab.get('title') == 'Channels'
  3493.             continuation = None
  3494.             slr_contents = try_get(slr_renderer, lambda x: x['contents'], list) or []
  3495.             for slr_content in slr_contents:
  3496.                 if not isinstance(slr_content, dict):
  3497.                     continue
  3498.                 is_renderer = try_get(slr_content, lambda x: x['itemSectionRenderer'], dict)
  3499.                 if not is_renderer:
  3500.                     continue
  3501.                 isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
  3502.                 for isr_content in isr_contents:
  3503.                     if not isinstance(isr_content, dict):
  3504.                         continue
  3505.                     renderer = isr_content.get('playlistVideoListRenderer')
  3506.                     if renderer:
  3507.                         for entry in self._playlist_entries(renderer):
  3508.                             yield entry
  3509.                         continuation = self._extract_continuation(renderer)
  3510.                         continue
  3511.                     renderer = isr_content.get('gridRenderer')
  3512.                     if renderer:
  3513.                         for entry in self._grid_entries(renderer):
  3514.                             yield entry
  3515.                         continuation = self._extract_continuation(renderer)
  3516.                         continue
  3517.                     renderer = isr_content.get('shelfRenderer')
  3518.                     if renderer:
  3519.                         for entry in self._shelf_entries(renderer, not is_channels_tab):
  3520.                             yield entry
  3521.                         continue
  3522.                     renderer = isr_content.get('backstagePostThreadRenderer')
  3523.                     if renderer:
  3524.                         for entry in self._post_thread_entries(renderer):
  3525.                             yield entry
  3526.                         continuation = self._extract_continuation(renderer)
  3527.                         continue
  3528.                     renderer = isr_content.get('videoRenderer')
  3529.                     if renderer:
  3530.                         entry = self._video_entry(renderer)
  3531.                         if entry:
  3532.                             yield entry
  3533.                     renderer = isr_content.get('richGridRenderer')
  3534.                     if renderer:
  3535.                         for from_ in self._rich_grid_entries(
  3536.                                 traverse_obj(renderer, ('contents', Ellipsis, T(dict)))):
  3537.                             yield from_
  3538.                         continuation = self._extract_continuation(renderer)
  3539.                         continue
  3541.                 if not continuation:
  3542.                     continuation = self._extract_continuation(is_renderer)
  3543.             if not continuation:
  3544.                 continuation = self._extract_continuation(slr_renderer)
  3545.         else:
  3546.             rich_grid_renderer = tab_content.get('richGridRenderer')
  3547.             if not rich_grid_renderer:
  3548.                 return
  3549.             for from_ in self._rich_grid_entries(
  3550.                     traverse_obj(rich_grid_renderer, ('contents', Ellipsis, T(dict)))):
  3551.                 yield from_
  3553.             continuation = self._extract_continuation(rich_grid_renderer)
  3555.         ytcfg = self._extract_ytcfg(item_id, webpage)
  3556.         client_version = try_get(
  3557.             ytcfg, lambda x: x['INNERTUBE_CLIENT_VERSION'], compat_str) or '2.20210407.08.00'
  3559.         headers = {
  3560.             'x-youtube-client-name': '1',
  3561.             'x-youtube-client-version': client_version,
  3562.             'content-type': 'application/json',
  3563.         }
  3565.         context = try_get(ytcfg, lambda x: x['INNERTUBE_CONTEXT'], dict) or {
  3566.             'client': {
  3567.                 'clientName': 'WEB',
  3568.                 'clientVersion': client_version,
  3569.             },
  3570.         }
  3571.         visitor_data = try_get(context, lambda x: x['client']['visitorData'], compat_str)
  3573.         identity_token = self._extract_identity_token(ytcfg, webpage)
  3574.         if identity_token:
  3575.             headers['x-youtube-identity-token'] = identity_token
  3577.         data = {
  3578.             'context': context,
  3579.         }
  3581.         for page_num in itertools.count(1):
  3582.             if not continuation:
  3583.                 break
  3584.             if visitor_data:
  3585.                 headers['X-Goog-Visitor-Id'] = visitor_data
  3586.             data['continuation'] = continuation['continuation']
  3587.             data['clickTracking'] = {
  3588.                 'clickTrackingParams': continuation['itct'],
  3589.             }
  3590.             count = 0
  3591.             retries = 3
  3592.             while count <= retries:
  3593.                 try:
  3594.                     # Downloading page may result in intermittent 5xx HTTP error
  3595.                     # that is usually worked around with a retry
  3596.                     response = self._download_json(
  3597.                         'https://www.youtube.com/youtubei/v1/browse',
  3598.                         None, 'Downloading page %d%s' % (page_num, ' (retry #%d)' % count if count else ''),
  3599.                         query={
  3600.                             # 'key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
  3601.                             'prettyPrint': 'false',
  3602.                         },
  3603.                         headers=headers, data=json.dumps(data).encode('utf8'))
  3604.                     break
  3605.                 except ExtractorError as e:
  3606.                     if isinstance(e.cause, compat_HTTPError) and e.cause.code in (500, 503):
  3607.                         count += 1
  3608.                         if count <= retries:
  3609.                             continue
  3610.                     raise
  3611.             if not response:
  3612.                 break
  3614.             visitor_data = try_get(
  3615.                 response, lambda x: x['responseContext']['visitorData'], compat_str) or visitor_data
  3617.             continuation_contents = try_get(
  3618.                 response, lambda x: x['continuationContents'], dict)
  3619.             if continuation_contents:
  3620.                 continuation_renderer = continuation_contents.get('playlistVideoListContinuation')
  3621.                 if continuation_renderer:
  3622.                     for entry in self._playlist_entries(continuation_renderer):
  3623.                         yield entry
  3624.                     continuation = self._extract_continuation(continuation_renderer)
  3625.                     continue
  3626.                 continuation_renderer = continuation_contents.get('gridContinuation')
  3627.                 if continuation_renderer:
  3628.                     for entry in self._grid_entries(continuation_renderer):
  3629.                         yield entry
  3630.                     continuation = self._extract_continuation(continuation_renderer)
  3631.                     continue
  3632.                 continuation_renderer = continuation_contents.get('itemSectionContinuation')
  3633.                 if continuation_renderer:
  3634.                     for entry in self._post_thread_continuation_entries(continuation_renderer):
  3635.                         yield entry
  3636.                     continuation = self._extract_continuation(continuation_renderer)
  3637.                     continue
  3639.             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
  3640.             continuation_items = try_get(
  3641.                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
  3642.             if continuation_items:
  3643.                 continuation_item = continuation_items[0]
  3644.                 if not isinstance(continuation_item, dict):
  3645.                     continue
  3646.                 renderer = self._extract_grid_item_renderer(continuation_item)
  3647.                 if renderer:
  3648.                     grid_renderer = {'items': continuation_items}
  3649.                     for entry in self._grid_entries(grid_renderer):
  3650.                         yield entry
  3651.                     continuation = self._extract_continuation(grid_renderer)
  3652.                     continue
  3653.                 renderer = continuation_item.get('playlistVideoRenderer') or continuation_item.get('itemSectionRenderer')
  3654.                 if renderer:
  3655.                     video_list_renderer = {'contents': continuation_items}
  3656.                     for entry in self._playlist_entries(video_list_renderer):
  3657.                         yield entry
  3658.                     continuation = self._extract_continuation(video_list_renderer)
  3659.                     continue
  3660.                 renderer = continuation_item.get('backstagePostThreadRenderer')
  3661.                 if renderer:
  3662.                     continuation_renderer = {'contents': continuation_items}
  3663.                     for entry in self._post_thread_continuation_entries(continuation_renderer):
  3664.                         yield entry
  3665.                     continuation = self._extract_continuation(continuation_renderer)
  3666.                     continue
  3667.                 renderer = continuation_item.get('richItemRenderer')
  3668.                 if renderer:
  3669.                     for entry in self._rich_grid_entries(continuation_items):
  3670.                         yield entry
  3671.                     continuation = self._extract_continuation({'contents': continuation_items})
  3672.                     continue
  3674.             break
  3676.     @staticmethod
  3677.     def _extract_selected_tab(tabs):
  3678.         for tab in tabs:
  3679.             renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
  3680.             if renderer.get('selected') is True:
  3681.                 return renderer
  3682.         else:
  3683.             raise ExtractorError('Unable to find selected tab')
  3685.     def _extract_uploader(self, metadata, data):
  3686.         uploader = {}
  3687.         renderers = traverse_obj(data,
  3688.                                  ('sidebar', 'playlistSidebarRenderer', 'items'))
  3689.         uploader['channel_id'] = self._extract_channel_id('', metadata=metadata, renderers=renderers)
  3690.         uploader['uploader'] = (
  3691.             self._extract_author_var('', 'name', renderers=renderers)
  3692.             or self._extract_author_var('', 'name', metadata=metadata))
  3693.         uploader['uploader_url'] = self._yt_urljoin(
  3694.             self._extract_author_var('', 'url', metadata=metadata, renderers=renderers))
  3695.         uploader['uploader_id'] = self._extract_uploader_id(uploader['uploader_url'])
  3696.         uploader['channel'] = uploader['uploader']
  3697.         return uploader
  3699.     @classmethod
  3700.     def _extract_alert(cls, data):
  3701.         alerts = []
  3702.         for alert in traverse_obj(data, ('alerts', Ellipsis), expected_type=dict):
  3703.             alert_text = traverse_obj(
  3704.                 alert, (None, lambda x: x['alertRenderer']['text']), get_all=False)
  3705.             if not alert_text:
  3706.                 continue
  3707.             text = cls._get_text(alert_text, 'text')
  3708.             if text:
  3709.                 alerts.append(text)
  3710.         return '\n'.join(alerts)
  3712.     def _extract_from_tabs(self, item_id, webpage, data, tabs):
  3713.         selected_tab = self._extract_selected_tab(tabs)
  3714.         renderer = traverse_obj(data, ('metadata', 'channelMetadataRenderer'),
  3715.                                 expected_type=dict) or {}
  3716.         playlist_id = item_id
  3717.         title = description = None
  3718.         if renderer:
  3719.             channel_title = txt_or_none(renderer.get('title')) or item_id
  3720.             tab_title = txt_or_none(selected_tab.get('title'))
  3721.             title = join_nonempty(
  3722.                 channel_title or item_id, tab_title,
  3723.                 txt_or_none(selected_tab.get('expandedText')),
  3724.                 delim=' - ')
  3725.             description = txt_or_none(renderer.get('description'))
  3726.             playlist_id = txt_or_none(renderer.get('externalId')) or playlist_id
  3727.         else:
  3728.             renderer = traverse_obj(data,
  3729.                                     ('metadata', 'playlistMetadataRenderer'),
  3730.                                     ('header', 'hashtagHeaderRenderer'),
  3731.                                     expected_type=dict) or {}
  3732.             title = traverse_obj(renderer, 'title', ('hashtag', 'simpleText'),
  3733.                                  expected_type=txt_or_none)
  3734.         playlist = self.playlist_result(
  3735.             self._entries(selected_tab, item_id, webpage),
  3736.             playlist_id=playlist_id, playlist_title=title,
  3737.             playlist_description=description)
  3738.         return merge_dicts(playlist, self._extract_uploader(renderer, data))
  3740.     def _extract_from_playlist(self, item_id, url, data, playlist):
  3741.         title = traverse_obj((playlist, data),
  3742.                              (0, 'title'), (1, 'titleText', 'simpleText'),
  3743.                              expected_type=txt_or_none)
  3744.         playlist_id = txt_or_none(playlist.get('playlistId')) or item_id
  3745.         # Inline playlist rendition continuation does not always work
  3746.         # at Youtube side, so delegating regular tab-based playlist URL
  3747.         # processing whenever possible.
  3748.         playlist_url = urljoin(url, traverse_obj(
  3749.             playlist, ('endpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
  3750.             expected_type=url_or_none))
  3751.         if playlist_url and playlist_url != url:
  3752.             return self.url_result(
  3753.                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
  3754.                 video_title=title)
  3755.         return self.playlist_result(
  3756.             self._playlist_entries(playlist), playlist_id=playlist_id,
  3757.             playlist_title=title)
  3759.     def _extract_identity_token(self, ytcfg, webpage):
  3760.         if ytcfg:
  3761.             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], compat_str)
  3762.             if token:
  3763.                 return token
  3764.         return self._search_regex(
  3765.             r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
  3766.             'identity token', default=None)
  3768.     def _real_extract(self, url):
  3769.         item_id = self._match_id(url)
  3770.         url = update_url(url, netloc='www.youtube.com')
  3771.         qs = parse_qs(url)
  3773.         def qs_get(key, default=None):
  3774.             return qs.get(key, [default])[-1]
  3776.         # Go around for /feeds/videos.xml?playlist_id={pl_id}
  3777.         if item_id == 'feeds' and '/feeds/videos.xml?' in url:
  3778.             playlist_id = qs_get('playlist_id')
  3779.             if playlist_id:
  3780.                 return self.url_result(
  3781.                     update_url_query('https://www.youtube.com/playlist', {
  3782.                         'list': playlist_id,
  3783.                     }), ie=self.ie_key(), video_id=playlist_id)
  3785.         # Handle both video/playlist URLs
  3786.         video_id = qs_get('v')
  3787.         playlist_id = qs_get('list')
  3788.         if video_id and playlist_id:
  3789.             if self._downloader.params.get('noplaylist'):
  3790.                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
  3791.                 return self.url_result(video_id, ie=YoutubeIE.ie_key(), video_id=video_id)
  3792.             self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
  3793.         webpage = self._download_webpage(url, item_id)
  3794.         data = self._extract_yt_initial_data(item_id, webpage)
  3795.         tabs = try_get(
  3796.             data, lambda x: x['contents']['twoColumnBrowseResultsRenderer']['tabs'], list)
  3797.         if tabs:
  3798.             return self._extract_from_tabs(item_id, webpage, data, tabs)
  3799.         playlist = try_get(
  3800.             data, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
  3801.         if playlist:
  3802.             return self._extract_from_playlist(item_id, url, data, playlist)
  3803.         # Fallback to video extraction if no playlist alike page is recognized.
  3804.         # First check for the current video then try the v attribute of URL query.
  3805.         video_id = try_get(
  3806.             data, lambda x: x['currentVideoEndpoint']['watchEndpoint']['videoId'],
  3807.             compat_str) or video_id
  3808.         if video_id:
  3809.             return self.url_result(video_id, ie=YoutubeIE.ie_key(), video_id=video_id)
  3810.         # Capture and output alerts
  3811.         alert = self._extract_alert(data)
  3812.         if alert:
  3813.             raise ExtractorError(alert, expected=True)
  3814.         # Failed to recognize
  3815.         raise ExtractorError('Unable to recognize tab page')
  3818. class YoutubePlaylistIE(InfoExtractor):
  3819.     IE_DESC = 'YouTube.com playlists'
  3820.     _VALID_URL = r'''(?x)(?:
  3821.                         (?:https?://)?
  3822.                         (?:\w+\.)?
  3823.                         (?:
  3824.                             (?:
  3825.                                 youtube(?:kids)?\.com|
  3826.                                 invidio\.us
  3827.                             )
  3828.                             /.*?\?.*?\blist=
  3829.                         )?
  3830.                         (?P<id>%(playlist_id)s)
  3831.                      )''' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
  3832.     IE_NAME = 'youtube:playlist'
  3833.     _TESTS = [{
  3834.         'note': 'issue #673',
  3835.         'url': 'PLBB231211A4F62143',
  3836.         'info_dict': {
  3837.             'title': '[OLD]Team Fortress 2 (Class-based LP)',
  3838.             'id': 'PLBB231211A4F62143',
  3839.             'uploader': 'Wickman',
  3840.             'uploader_id': '@WickmanVT',
  3841.             'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
  3842.         },
  3843.         'playlist_mincount': 29,
  3844.     }, {
  3845.         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
  3846.         'info_dict': {
  3847.             'title': 'YDL_safe_search',
  3848.             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
  3849.         },
  3850.         'playlist_count': 2,
  3851.         'skip': 'This playlist is private',
  3852.     }, {
  3853.         'note': 'embedded',
  3854.         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
  3855.         # TODO: full playlist requires _reload_with_unavailable_videos()
  3856.         # 'playlist_count': 4,
  3857.         'playlist_mincount': 1,
  3858.         'info_dict': {
  3859.             'title': 'JODA15',
  3860.             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
  3861.             'uploader': 'milan',
  3862.             'uploader_id': '@milan5503',
  3863.             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
  3864.         },
  3865.     }, {
  3866.         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
  3867.         'playlist_mincount': 455,
  3868.         'info_dict': {
  3869.             'title': '2018 Chinese New Singles (11/6 updated)',
  3870.             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
  3871.             'uploader': 'LBK',
  3872.             'uploader_id': '@music_king',
  3873.             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
  3874.         },
  3875.     }, {
  3876.         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
  3877.         'only_matching': True,
  3878.     }, {
  3879.         # music album playlist
  3880.         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
  3881.         'only_matching': True,
  3882.     }]
  3884.     @classmethod
  3885.     def suitable(cls, url):
  3886.         if YoutubeTabIE.suitable(url):
  3887.             return False
  3888.         if parse_qs(url).get('v', [None])[0]:
  3889.             return False
  3890.         return super(YoutubePlaylistIE, cls).suitable(url)
  3892.     def _real_extract(self, url):
  3893.         playlist_id = self._match_id(url)
  3894.         qs = parse_qs(url)
  3895.         if not qs:
  3896.             qs = {'list': playlist_id}
  3897.         return self.url_result(
  3898.             update_url_query('https://www.youtube.com/playlist', qs),
  3899.             ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
  3902. class YoutubeYtBeIE(InfoExtractor):
  3903.     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
  3904.     _TESTS = [{
  3905.         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
  3906.         'info_dict': {
  3907.             'id': 'yeWKywCrFtk',
  3908.             'ext': 'mp4',
  3909.             'title': 'Small Scale Baler and Braiding Rugs',
  3910.             'uploader': 'Backus-Page House Museum',
  3911.             'uploader_id': '@backuspagemuseum',
  3912.             'uploader_url': r're:https?://(?:www\.)?youtube\.com/@backuspagemuseum',
  3913.             'upload_date': '20161008',
  3914.             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
  3915.             'categories': ['Nonprofits & Activism'],
  3916.             'tags': list,
  3917.             'like_count': int,
  3918.         },
  3919.         'params': {
  3920.             'noplaylist': True,
  3921.             'skip_download': True,
  3922.         },
  3923.     }, {
  3924.         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
  3925.         'only_matching': True,
  3926.     }]
  3928.     def _real_extract(self, url):
  3929.         mobj = re.match(self._VALID_URL, url)
  3930.         video_id = mobj.group('id')
  3931.         playlist_id = mobj.group('playlist_id')
  3932.         return self.url_result(
  3933.             update_url_query('https://www.youtube.com/watch', {
  3934.                 'v': video_id,
  3935.                 'list': playlist_id,
  3936.                 'feature': 'youtu.be',
  3937.             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
  3940. class YoutubeYtUserIE(InfoExtractor):
  3941.     _VALID_URL = r'ytuser:(?P<id>.+)'
  3942.     _TESTS = [{
  3943.         'url': 'ytuser:phihag',
  3944.         'only_matching': True,
  3945.     }]
  3947.     def _real_extract(self, url):
  3948.         user_id = self._match_id(url)
  3949.         return self.url_result(
  3950.             'https://www.youtube.com/user/%s' % user_id,
  3951.             ie=YoutubeTabIE.ie_key(), video_id=user_id)
  3954. class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
  3955.     IE_NAME = 'youtube:favorites'
  3956.     IE_DESC = 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'
  3957.     _VALID_URL = r'https?://(?:www\.)?youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
  3958.     _LOGIN_REQUIRED = True
  3959.     _TESTS = [{
  3960.         'url': ':ytfav',
  3961.         'only_matching': True,
  3962.     }, {
  3963.         'url': ':ytfavorites',
  3964.         'only_matching': True,
  3965.     }]
  3967.     def _real_extract(self, url):
  3968.         return self.url_result(
  3969.             'https://www.youtube.com/playlist?list=LL',
  3970.             ie=YoutubeTabIE.ie_key())
  3973. class YoutubeSearchIE(SearchInfoExtractor, YoutubeBaseInfoExtractor):
  3974.     IE_DESC = 'YouTube.com searches'
  3975.     IE_NAME = 'youtube:search'
  3976.     _SEARCH_KEY = 'ytsearch'
  3977.     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
  3978.     _MAX_RESULTS = float('inf')
  3979.     _TESTS = [{
  3980.         'url': 'ytsearch10:youtube-dl test video',
  3981.         'playlist_count': 10,
  3982.         'info_dict': {
  3983.             'id': 'youtube-dl test video',
  3984.             'title': 'youtube-dl test video',
  3985.         },
  3986.     }]
  3988.     def _get_n_results(self, query, n):
  3989.         """Get a specified number of results for a query"""
  3990.         entries = itertools.islice(self._search_results(query, self._SEARCH_PARAMS), 0, None if n == float('inf') else n)
  3991.         return self.playlist_result(entries, query, query)
  3994. class YoutubeSearchDateIE(YoutubeSearchIE):
  3995.     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
  3996.     _SEARCH_KEY = 'ytsearchdate'
  3997.     IE_DESC = 'YouTube.com searches, newest videos first'
  3998.     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
  3999.     _TESTS = [{
  4000.         'url': 'ytsearchdate10:youtube-dl test video',
  4001.         'playlist_count': 10,
  4002.         'info_dict': {
  4003.             'id': 'youtube-dl test video',
  4004.             'title': 'youtube-dl test video',
  4005.         },
  4006.     }]
  4009. class YoutubeSearchURLIE(YoutubeBaseInfoExtractor):
  4010.     IE_DESC = 'YouTube search URLs with sorting and filter support'
  4011.     IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
  4012.     _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?:[^&]+)(?:[&]|$)'
  4013.     _TESTS = [{
  4014.         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
  4015.         'playlist_mincount': 5,
  4016.         'info_dict': {
  4017.             'id': 'youtube-dl test video',
  4018.             'title': 'youtube-dl test video',
  4019.         },
  4020.         'params': {'playlistend': 5},
  4021.     }, {
  4022.         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
  4023.         'only_matching': True,
  4024.     }]
  4026.     def _real_extract(self, url):
  4027.         qs = parse_qs(url)
  4028.         query = (qs.get('search_query') or qs.get('q'))[-1]
  4029.         params = qs.get('sp', ('',))[-1]
  4030.         return self.playlist_result(self._search_results(query, params), query, query)
  4033. class YoutubeFeedsInfoExtractor(YoutubeTabIE):
  4034.     """
  4035.     Base class for feed extractors
  4037.     Subclasses must define the _FEED_NAME property.
  4038.     """
  4039.     _LOGIN_REQUIRED = True
  4041.     @property
  4042.     def IE_NAME(self):
  4043.         return 'youtube:%s' % self._FEED_NAME
  4045.     def _real_initialize(self):
  4046.         self._login()
  4048.     def _real_extract(self, url):
  4049.         return self.url_result(
  4050.             'https://www.youtube.com/feed/%s' % self._FEED_NAME,
  4051.             ie=YoutubeTabIE.ie_key())
  4054. class YoutubeWatchLaterIE(InfoExtractor):
  4055.     IE_NAME = 'youtube:watchlater'
  4056.     IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
  4057.     _VALID_URL = r':ytwatchlater'
  4058.     _TESTS = [{
  4059.         'url': ':ytwatchlater',
  4060.         'only_matching': True,
  4061.     }]
  4063.     def _real_extract(self, url):
  4064.         return self.url_result(
  4065.             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
  4068. class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
  4069.     IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
  4070.     _VALID_URL = r':ytrec(?:ommended)?'
  4071.     _FEED_NAME = 'recommended'
  4072.     _TESTS = [{
  4073.         'url': ':ytrec',
  4074.         'only_matching': True,
  4075.     }, {
  4076.         'url': ':ytrecommended',
  4077.         'only_matching': True,
  4078.     }]
  4081. class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
  4082.     IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
  4083.     _VALID_URL = r':ytsubs(?:criptions)?'
  4084.     _FEED_NAME = 'subscriptions'
  4085.     _TESTS = [{
  4086.         'url': ':ytsubs',
  4087.         'only_matching': True,
  4088.     }, {
  4089.         'url': ':ytsubscriptions',
  4090.         'only_matching': True,
  4091.     }]
  4094. class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
  4095.     IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
  4096.     _VALID_URL = r':ythistory'
  4097.     _FEED_NAME = 'history'
  4098.     _TESTS = [{
  4099.         'url': ':ythistory',
  4100.         'only_matching': True,
  4101.     }]
  4104. class YoutubeTruncatedURLIE(InfoExtractor):
  4105.     IE_NAME = 'youtube:truncated_url'
  4106.     IE_DESC = False  # Do not list
  4107.     _VALID_URL = r'''(?x)
  4108.         (?:https?://)?
  4109.         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
  4110.         (?:watch\?(?:
  4111.             feature=[a-z_]+|
  4112.             annotation_id=annotation_[^&]+|
  4113.             x-yt-cl=[0-9]+|
  4114.             hl=[^&]*|
  4115.             t=[0-9]+
  4116.         )?
  4117.         |
  4118.             attribution_link\?a=[^&]+
  4119.         )
  4120.         $
  4121.     '''
  4123.     _TESTS = [{
  4124.         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
  4125.         'only_matching': True,
  4126.     }, {
  4127.         'url': 'https://www.youtube.com/watch?',
  4128.         'only_matching': True,
  4129.     }, {
  4130.         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
  4131.         'only_matching': True,
  4132.     }, {
  4133.         'url': 'https://www.youtube.com/watch?feature=foo',
  4134.         'only_matching': True,
  4135.     }, {
  4136.         'url': 'https://www.youtube.com/watch?hl=en-GB',
  4137.         'only_matching': True,
  4138.     }, {
  4139.         'url': 'https://www.youtube.com/watch?t=2372',
  4140.         'only_matching': True,
  4141.     }]
  4143.     def _real_extract(self, url):
  4144.         raise ExtractorError(
  4145.             'Did you forget to quote the URL? Remember that & is a meta '
  4146.             'character in most shells, so you want to put the URL in quotes, '
  4147.             'like  youtube-dl '
  4148.             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
  4149.             ' or simply  youtube-dl BaW_jenozKc  .',
  4150.             expected=True)
  4153. class YoutubeTruncatedIDIE(InfoExtractor):
  4154.     IE_NAME = 'youtube:truncated_id'
  4155.     IE_DESC = False  # Do not list
  4156.     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
  4158.     _TESTS = [{
  4159.         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
  4160.         'only_matching': True,
  4161.     }]
  4163.     def _real_extract(self, url):
  4164.         video_id = self._match_id(url)
  4165.         raise ExtractorError(
  4166.             'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id, url),
  4167.             expected=True)