logo

oasis-root

Compiled tree of Oasis Linux based on own branch at <https://hacktivis.me/git/oasis/> git clone https://anongit.hacktivis.me/git/oasis-root.git

archiveorg.py (47378B)

    

  1. from __future__ import annotations
  3. import json
  4. import re
  5. import urllib.parse
  7. from .common import InfoExtractor
  8. from .youtube import YoutubeBaseInfoExtractor, YoutubeIE
  9. from ..networking import HEADRequest
  10. from ..networking.exceptions import HTTPError
  11. from ..utils import (
  12.     KNOWN_EXTENSIONS,
  13.     ExtractorError,
  14.     bug_reports_message,
  15.     clean_html,
  16.     dict_get,
  17.     extract_attributes,
  18.     get_element_by_id,
  19.     int_or_none,
  20.     join_nonempty,
  21.     js_to_json,
  22.     merge_dicts,
  23.     mimetype2ext,
  24.     orderedSet,
  25.     parse_duration,
  26.     parse_qs,
  27.     str_or_none,
  28.     str_to_int,
  29.     traverse_obj,
  30.     try_get,
  31.     unified_strdate,
  32.     unified_timestamp,
  33.     url_or_none,
  34.     urlhandle_detect_ext,
  35.     variadic,
  36. )
  39. class ArchiveOrgIE(InfoExtractor):
  40.     IE_NAME = 'archive.org'
  41.     IE_DESC = 'archive.org video and audio'
  42.     _VALID_URL = r'https?://(?:www\.)?archive\.org/(?:details|embed)/(?P<id>[^?#]+)(?:[?].*)?$'
  43.     _TESTS = [{
  44.         'url': 'http://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
  45.         'md5': '8af1d4cf447933ed3c7f4871162602db',
  46.         'info_dict': {
  47.             'id': 'XD300-23_68HighlightsAResearchCntAugHumanIntellect',
  48.             'ext': 'ogv',
  49.             'title': '1968 Demo - FJCC Conference Presentation Reel #1',
  50.             'description': 'md5:da45c349df039f1cc8075268eb1b5c25',
  51.             'release_date': '19681210',
  52.             'timestamp': 1268695290,
  53.             'upload_date': '20100315',
  54.             'creators': ['SRI International'],
  55.             'uploader': 'laura@archive.org',
  56.             'thumbnail': r're:https://archive\.org/download/.*\.jpg',
  57.             'display_id': 'XD300-23_68HighlightsAResearchCntAugHumanIntellect.cdr',
  58.             'track': 'XD300-23 68HighlightsAResearchCntAugHumanIntellect',
  60.         },
  61.     }, {
  62.         'url': 'https://archive.org/details/Cops1922',
  63.         'md5': '0869000b4ce265e8ca62738b336b268a',
  64.         'info_dict': {
  65.             'id': 'Cops1922',
  66.             'ext': 'mp4',
  67.             'title': 'Buster Keaton\'s "Cops" (1922)',
  68.             'description': 'md5:cd6f9910c35aedd5fc237dbc3957e2ca',
  69.             'uploader': 'yorkmba99@hotmail.com',
  70.             'timestamp': 1387699629,
  71.             'upload_date': '20131222',
  72.             'display_id': 'Cops-v2.mp4',
  73.             'thumbnail': r're:https://archive\.org/download/.*\.jpg',
  74.             'duration': 1091.96,
  75.         },
  76.     }, {
  77.         'url': 'http://archive.org/embed/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
  78.         'only_matching': True,
  79.     }, {
  80.         'url': 'https://archive.org/details/Election_Ads',
  81.         'md5': 'eec5cddebd4793c6a653b69c3b11f2e6',
  82.         'info_dict': {
  83.             'id': 'Election_Ads/Commercial-JFK1960ElectionAdCampaignJingle.mpg',
  84.             'title': 'Commercial-JFK1960ElectionAdCampaignJingle.mpg',
  85.             'ext': 'mpg',
  86.             'thumbnail': r're:https://archive\.org/download/.*\.jpg',
  87.             'duration': 59.77,
  88.             'display_id': 'Commercial-JFK1960ElectionAdCampaignJingle.mpg',
  89.         },
  90.     }, {
  91.         'url': 'https://archive.org/details/Election_Ads/Commercial-Nixon1960ElectionAdToughonDefense.mpg',
  92.         'md5': 'ea1eed8234e7d4165f38c8c769edef38',
  93.         'info_dict': {
  94.             'id': 'Election_Ads/Commercial-Nixon1960ElectionAdToughonDefense.mpg',
  95.             'title': 'Commercial-Nixon1960ElectionAdToughonDefense.mpg',
  96.             'ext': 'mpg',
  97.             'timestamp': 1205588045,
  98.             'uploader': 'mikedavisstripmaster@yahoo.com',
  99.             'description': '1960 Presidential Campaign Election Commercials John F Kennedy, Richard M Nixon',
  100.             'upload_date': '20080315',
  101.             'display_id': 'Commercial-Nixon1960ElectionAdToughonDefense.mpg',
  102.             'duration': 59.51,
  103.             'license': 'http://creativecommons.org/licenses/publicdomain/',
  104.             'thumbnail': r're:https://archive\.org/download/.*\.jpg',
  105.         },
  106.     }, {
  107.         'url': 'https://archive.org/details/gd1977-05-08.shure57.stevenson.29303.flac16',
  108.         'md5': '7d07ffb42aba6537c28e053efa4b54c9',
  109.         'info_dict': {
  110.             'id': 'gd1977-05-08.shure57.stevenson.29303.flac16/gd1977-05-08d01t01.flac',
  111.             'title': 'Turning',
  112.             'ext': 'flac',
  113.             'track': 'Turning',
  114.             'creators': ['Grateful Dead'],
  115.             'display_id': 'gd1977-05-08d01t01.flac',
  116.             'track_number': 1,
  117.             'album': '1977-05-08 - Barton Hall - Cornell University',
  118.             'duration': 39.8,
  119.         },
  120.     }, {
  121.         'url': 'https://archive.org/details/gd1977-05-08.shure57.stevenson.29303.flac16/gd1977-05-08d01t07.flac',
  122.         'md5': 'a07cd8c6ab4ee1560f8a0021717130f3',
  123.         'info_dict': {
  124.             'id': 'gd1977-05-08.shure57.stevenson.29303.flac16/gd1977-05-08d01t07.flac',
  125.             'title': 'Deal',
  126.             'ext': 'flac',
  127.             'timestamp': 1205895624,
  128.             'uploader': 'mvernon54@yahoo.com',
  129.             'description': 'md5:6c921464414814720c6593810a5c7e3d',
  130.             'upload_date': '20080319',
  131.             'location': 'Barton Hall - Cornell University',
  132.             'duration': 438.68,
  133.             'track': 'Deal',
  134.             'creators': ['Grateful Dead'],
  135.             'album': '1977-05-08 - Barton Hall - Cornell University',
  136.             'release_date': '19770508',
  137.             'display_id': 'gd1977-05-08d01t07.flac',
  138.             'track_number': 7,
  139.         },
  140.     }, {
  141.         # FIXME: give a better error message than just IndexError when all available formats are restricted
  142.         'url': 'https://archive.org/details/lp_the-music-of-russia_various-artists-a-askaryan-alexander-melik',
  143.         'md5': '7cb019baa9b332e82ea7c10403acd180',
  144.         'info_dict': {
  145.             'id': 'lp_the-music-of-russia_various-artists-a-askaryan-alexander-melik/disc1/01.01. Bells Of Rostov.mp3',
  146.             'title': 'Bells Of Rostov',
  147.             'ext': 'mp3',
  148.         },
  149.         'skip': 'restricted',
  150.     }, {
  151.         'url': 'https://archive.org/details/lp_the-music-of-russia_various-artists-a-askaryan-alexander-melik/disc1/02.02.+Song+And+Chorus+In+The+Polovetsian+Camp+From+%22Prince+Igor%22+(Act+2%2C+Scene+1).mp3',
  152.         'md5': '1d0aabe03edca83ca58d9ed3b493a3c3',
  153.         'info_dict': {
  154.             'id': 'lp_the-music-of-russia_various-artists-a-askaryan-alexander-melik/disc1/02.02. Song And Chorus In The Polovetsian Camp From "Prince Igor" (Act 2, Scene 1).mp3',
  155.             'title': 'Song And Chorus In The Polovetsian Camp From "Prince Igor" (Act 2, Scene 1)',
  156.             'ext': 'mp3',
  157.             'timestamp': 1569662587,
  158.             'uploader': 'associate-joygen-odiongan@archive.org',
  159.             'description': 'md5:012b2d668ae753be36896f343d12a236',
  160.             'upload_date': '20190928',
  161.         },
  162.         'skip': 'restricted',
  163.     }, {
  164.         # Original formats are private
  165.         'url': 'https://archive.org/details/irelandthemakingofarepublic',
  166.         'info_dict': {
  167.             'id': 'irelandthemakingofarepublic',
  168.             'title': 'Ireland: The Making of a Republic',
  169.             'upload_date': '20160610',
  170.             'description': 'md5:f70956a156645a658a0dc9513d9e78b7',
  171.             'uploader': 'dimitrios@archive.org',
  172.             'creators': ['British Broadcasting Corporation', 'Time-Life Films'],
  173.             'timestamp': 1465594947,
  174.         },
  175.         'playlist': [
  176.             {
  177.                 'md5': '0b211261b26590d49df968f71b90690d',
  178.                 'info_dict': {
  179.                     'id': 'irelandthemakingofarepublic/irelandthemakingofarepublicreel1_01.mov',
  180.                     'ext': 'mp4',
  181.                     'title': 'irelandthemakingofarepublicreel1_01.mov',
  182.                     'duration': 130.46,
  183.                     'thumbnail': 'https://archive.org/download/irelandthemakingofarepublic/irelandthemakingofarepublic.thumbs/irelandthemakingofarepublicreel1_01_000117.jpg',
  184.                     'display_id': 'irelandthemakingofarepublicreel1_01.mov',
  185.                 },
  186.             }, {
  187.                 'md5': '67335ee3b23a0da930841981c1e79b02',
  188.                 'info_dict': {
  189.                     'id': 'irelandthemakingofarepublic/irelandthemakingofarepublicreel1_02.mov',
  190.                     'ext': 'mp4',
  191.                     'duration': 1395.13,
  192.                     'title': 'irelandthemakingofarepublicreel1_02.mov',
  193.                     'display_id': 'irelandthemakingofarepublicreel1_02.mov',
  194.                     'thumbnail': 'https://archive.org/download/irelandthemakingofarepublic/irelandthemakingofarepublic.thumbs/irelandthemakingofarepublicreel1_02_001374.jpg',
  195.                 },
  196.             }, {
  197.                 'md5': 'e470e86787893603f4a341a16c281eb5',
  198.                 'info_dict': {
  199.                     'id': 'irelandthemakingofarepublic/irelandthemakingofarepublicreel2.mov',
  200.                     'ext': 'mp4',
  201.                     'duration': 1602.67,
  202.                     'title': 'irelandthemakingofarepublicreel2.mov',
  203.                     'thumbnail': 'https://archive.org/download/irelandthemakingofarepublic/irelandthemakingofarepublic.thumbs/irelandthemakingofarepublicreel2_001554.jpg',
  204.                     'display_id': 'irelandthemakingofarepublicreel2.mov',
  205.                 },
  206.             },
  207.         ],
  208.     }, {
  209.         # The reviewbody is None for one of the reviews; just need to extract data without crashing
  210.         'url': 'https://archive.org/details/gd95-04-02.sbd.11622.sbeok.shnf/gd95-04-02d1t04.shn',
  211.         'info_dict': {
  212.             'id': 'gd95-04-02.sbd.11622.sbeok.shnf/gd95-04-02d1t04.shn',
  213.             'ext': 'mp3',
  214.             'title': 'Stuck Inside of Mobile with the Memphis Blues Again',
  215.             'creators': ['Grateful Dead'],
  216.             'duration': 338.31,
  217.             'track': 'Stuck Inside of Mobile with the Memphis Blues Again',
  218.             'description': 'md5:764348a470b986f1217ffd38d6ac7b72',
  219.             'display_id': 'gd95-04-02d1t04.shn',
  220.             'location': 'Pyramid Arena',
  221.             'uploader': 'jon@archive.org',
  222.             'album': '1995-04-02 - Pyramid Arena',
  223.             'upload_date': '20040519',
  224.             'track_number': 4,
  225.             'release_date': '19950402',
  226.             'timestamp': 1084927901,
  227.         },
  228.     }]
  230.     @staticmethod
  231.     def _playlist_data(webpage):
  232.         element = re.findall(r'''(?xs)
  233.             <input
  234.             (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
  235.             \s+class=['"]?js-play8-playlist['"]?
  236.             (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
  237.             \s*/>
  238.         ''', webpage)[0]
  240.         return json.loads(extract_attributes(element)['value'])
  242.     def _real_extract(self, url):
  243.         video_id = urllib.parse.unquote_plus(self._match_id(url))
  244.         identifier, _, entry_id = video_id.partition('/')
  246.         # Archive.org metadata API doesn't clearly demarcate playlist entries
  247.         # or subtitle tracks, so we get them from the embeddable player.
  248.         embed_page = self._download_webpage(f'https://archive.org/embed/{identifier}', identifier)
  249.         playlist = self._playlist_data(embed_page)
  251.         entries = {}
  252.         for p in playlist:
  253.             # If the user specified a playlist entry in the URL, ignore the
  254.             # rest of the playlist.
  255.             if entry_id and p['orig'] != entry_id:
  256.                 continue
  258.             entries[p['orig']] = {
  259.                 'formats': [],
  260.                 'thumbnails': [],
  261.                 'artist': p.get('artist'),
  262.                 'track': p.get('title'),
  263.                 'subtitles': {},
  264.             }
  266.             for track in p.get('tracks', []):
  267.                 if track['kind'] != 'subtitles':
  268.                     continue
  269.                 entries[p['orig']][track['label']] = {
  270.                     'url': 'https://archive.org/' + track['file'].lstrip('/'),
  271.                 }
  273.         metadata = self._download_json('http://archive.org/metadata/' + identifier, identifier)
  274.         m = metadata['metadata']
  275.         identifier = m['identifier']
  277.         info = {
  278.             'id': identifier,
  279.             'title': m['title'],
  280.             'description': clean_html(m.get('description')),
  281.             'uploader': dict_get(m, ['uploader', 'adder']),
  282.             'creators': traverse_obj(m, ('creator', {variadic}, {lambda x: x[0] and list(x)})),
  283.             'license': m.get('licenseurl'),
  284.             'release_date': unified_strdate(m.get('date')),
  285.             'timestamp': unified_timestamp(dict_get(m, ['publicdate', 'addeddate'])),
  286.             'webpage_url': f'https://archive.org/details/{identifier}',
  287.             'location': m.get('venue'),
  288.             'release_year': int_or_none(m.get('year'))}
  290.         for f in metadata['files']:
  291.             if f['name'] in entries:
  292.                 entries[f['name']] = merge_dicts(entries[f['name']], {
  293.                     'id': identifier + '/' + f['name'],
  294.                     'title': f.get('title') or f['name'],
  295.                     'display_id': f['name'],
  296.                     'description': clean_html(f.get('description')),
  297.                     'creators': traverse_obj(f, ('creator', {variadic}, {lambda x: x[0] and list(x)})),
  298.                     'duration': parse_duration(f.get('length')),
  299.                     'track_number': int_or_none(f.get('track')),
  300.                     'album': f.get('album'),
  301.                     'discnumber': int_or_none(f.get('disc')),
  302.                     'release_year': int_or_none(f.get('year'))})
  303.                 entry = entries[f['name']]
  304.             elif traverse_obj(f, 'original', expected_type=str) in entries:
  305.                 entry = entries[f['original']]
  306.             else:
  307.                 continue
  309.             if f.get('format') == 'Thumbnail':
  310.                 entry['thumbnails'].append({
  311.                     'id': f['name'],
  312.                     'url': 'https://archive.org/download/' + identifier + '/' + f['name'],
  313.                     'width': int_or_none(f.get('width')),
  314.                     'height': int_or_none(f.get('width')),
  315.                     'filesize': int_or_none(f.get('size'))})
  317.             _, has_ext, extension = f['name'].rpartition('.')
  318.             if not has_ext:
  319.                 extension = None
  321.             # We don't want to skip private formats if the user has access to them,
  322.             # however without access to an account with such privileges we can't implement/test this.
  323.             # For now to be safe, we will only skip them if there is no user logged in.
  324.             is_logged_in = bool(self._get_cookies('https://archive.org').get('logged-in-sig'))
  325.             if extension in KNOWN_EXTENSIONS and (not f.get('private') or is_logged_in):
  326.                 entry['formats'].append({
  327.                     'url': 'https://archive.org/download/' + identifier + '/' + urllib.parse.quote(f['name']),
  328.                     'format': f.get('format'),
  329.                     'width': int_or_none(f.get('width')),
  330.                     'height': int_or_none(f.get('height')),
  331.                     'filesize': int_or_none(f.get('size')),
  332.                     'protocol': 'https',
  333.                     'source_preference': 0 if f.get('source') == 'original' else -1,
  334.                     'format_note': f.get('source'),
  335.                 })
  337.         for entry in entries.values():
  338.             entry['_format_sort_fields'] = ('source', )
  340.         if len(entries) == 1:
  341.             # If there's only one item, use it as the main info dict
  342.             only_video = next(iter(entries.values()))
  343.             if entry_id:
  344.                 info = merge_dicts(only_video, info)
  345.             else:
  346.                 info = merge_dicts(info, only_video)
  347.         else:
  348.             # Otherwise, we have a playlist.
  349.             info['_type'] = 'playlist'
  350.             info['entries'] = list(entries.values())
  352.         if metadata.get('reviews'):
  353.             info['comments'] = []
  354.             for review in metadata['reviews']:
  355.                 info['comments'].append({
  356.                     'id': review.get('review_id'),
  357.                     'author': review.get('reviewer'),
  358.                     'text': join_nonempty('reviewtitle', 'reviewbody', from_dict=review, delim='\n\n'),
  359.                     'timestamp': unified_timestamp(review.get('createdate')),
  360.                     'parent': 'root'})
  362.         return info
  365. class YoutubeWebArchiveIE(InfoExtractor):
  366.     IE_NAME = 'web.archive:youtube'
  367.     IE_DESC = 'web.archive.org saved youtube videos, "ytarchive:" prefix'
  368.     _VALID_URL = r'''(?x)(?:(?P<prefix>ytarchive:)|
  369.             (?:https?://)?web\.archive\.org/
  370.             (?:web/)?(?:(?P<date>[0-9]{14})?[0-9A-Za-z_*]*/)?  # /web and the version index is optional
  371.             (?:https?(?::|%3[Aa])//)?(?:
  372.                 (?:\w+\.)?youtube\.com(?::(?:80|443))?/watch(?:\.php)?(?:\?|%3[fF])(?:[^\#]+(?:&|%26))?v(?:=|%3[dD])  # Youtube URL
  373.                 |(?:wayback-fakeurl\.archive\.org/yt/)  # Or the internal fake url
  374.             )
  375.         )(?P<id>[0-9A-Za-z_-]{11})
  376.         (?(prefix)
  377.             (?::(?P<date2>[0-9]{14}))?$|
  378.             (?:%26|[#&]|$)
  379.         )'''
  381.     _TESTS = [
  382.         {
  383.             'url': 'https://web.archive.org/web/20150415002341/https://www.youtube.com/watch?v=aYAGB11YrSs',
  384.             'info_dict': {
  385.                 'id': 'aYAGB11YrSs',
  386.                 'ext': 'webm',
  387.                 'title': 'Team Fortress 2 - Sandviches!',
  388.                 'description': 'md5:4984c0f9a07f349fc5d8e82ab7af4eaf',
  389.                 'upload_date': '20110926',
  390.                 'uploader': 'Zeurel',
  391.                 'channel_id': 'UCukCyHaD-bK3in_pKpfH9Eg',
  392.                 'duration': 32,
  393.                 'uploader_id': 'Zeurel',
  394.                 'uploader_url': 'https://www.youtube.com/user/Zeurel',
  395.                 'thumbnail': r're:https?://.*\.(jpg|webp)',
  396.                 'channel_url': 'https://www.youtube.com/channel/UCukCyHaD-bK3in_pKpfH9Eg',
  397.             },
  398.         }, {
  399.             # Internal link
  400.             'url': 'https://web.archive.org/web/2oe/http://wayback-fakeurl.archive.org/yt/97t7Xj_iBv0',
  401.             'info_dict': {
  402.                 'id': '97t7Xj_iBv0',
  403.                 'ext': 'mp4',
  404.                 'title': 'Why Machines That Bend Are Better',
  405.                 'description': 'md5:00404df2c632d16a674ff8df1ecfbb6c',
  406.                 'upload_date': '20190312',
  407.                 'uploader': 'Veritasium',
  408.                 'channel_id': 'UCHnyfMqiRRG1u-2MsSQLbXA',
  409.                 'duration': 771,
  410.                 'uploader_id': '1veritasium',
  411.                 'uploader_url': 'https://www.youtube.com/user/1veritasium',
  412.                 'thumbnail': r're:https?://.*\.(jpg|webp)',
  413.                 'channel_url': 'https://www.youtube.com/channel/UCHnyfMqiRRG1u-2MsSQLbXA',
  414.             },
  415.         }, {
  416.             # Video from 2012, webm format itag 45. Newest capture is deleted video, with an invalid description.
  417.             # Should use the date in the link. Title ends with '- Youtube'. Capture has description in eow-description
  418.             'url': 'https://web.archive.org/web/20120712231619/http://www.youtube.com/watch?v=AkhihxRKcrs&gl=US&hl=en',
  419.             'info_dict': {
  420.                 'id': 'AkhihxRKcrs',
  421.                 'ext': 'webm',
  422.                 'title': 'Limited Run: Mondo\'s Modern Classic 1 of 3 (SDCC 2012)',
  423.                 'upload_date': '20120712',
  424.                 'duration': 398,
  425.                 'description': 'md5:ff4de6a7980cb65d951c2f6966a4f2f3',
  426.                 'uploader_id': 'machinima',
  427.                 'uploader_url': 'https://www.youtube.com/user/machinima',
  428.                 'thumbnail': r're:https?://.*\.(jpg|webp)',
  429.                 'uploader': 'machinima',
  430.             },
  431.         }, {
  432.             # FLV video. Video file URL does not provide itag information
  433.             'url': 'https://web.archive.org/web/20081211103536/http://www.youtube.com/watch?v=jNQXAC9IVRw',
  434.             'info_dict': {
  435.                 'id': 'jNQXAC9IVRw',
  436.                 'ext': 'flv',
  437.                 'title': 'Me at the zoo',
  438.                 'upload_date': '20050423',
  439.                 'channel_id': 'UC4QobU6STFB0P71PMvOGN5A',
  440.                 'duration': 19,
  441.                 'description': 'md5:10436b12e07ac43ff8df65287a56efb4',
  442.                 'uploader_id': 'jawed',
  443.                 'uploader_url': 'https://www.youtube.com/user/jawed',
  444.                 'channel_url': 'https://www.youtube.com/channel/UC4QobU6STFB0P71PMvOGN5A',
  445.                 'thumbnail': r're:https?://.*\.(jpg|webp)',
  446.                 'uploader': 'jawed',
  447.             },
  448.         }, {
  449.             'url': 'https://web.archive.org/web/20110712231407/http://www.youtube.com/watch?v=lTx3G6h2xyA',
  450.             'info_dict': {
  451.                 'id': 'lTx3G6h2xyA',
  452.                 'ext': 'flv',
  453.                 'title': 'Madeon - Pop Culture (live mashup)',
  454.                 'upload_date': '20110711',
  455.                 'uploader': 'Madeon',
  456.                 'channel_id': 'UCqMDNf3Pn5L7pcNkuSEeO3w',
  457.                 'duration': 204,
  458.                 'description': 'md5:f7535343b6eda34a314eff8b85444680',
  459.                 'uploader_id': 'itsmadeon',
  460.                 'uploader_url': 'https://www.youtube.com/user/itsmadeon',
  461.                 'channel_url': 'https://www.youtube.com/channel/UCqMDNf3Pn5L7pcNkuSEeO3w',
  462.                 'thumbnail': r're:https?://.*\.(jpg|webp)',
  463.             },
  464.         }, {
  465.             # First capture is of dead video, second is the oldest from CDX response.
  466.             'url': 'https://web.archive.org/https://www.youtube.com/watch?v=1JYutPM8O6E',
  467.             'info_dict': {
  468.                 'id': '1JYutPM8O6E',
  469.                 'ext': 'mp4',
  470.                 'title': 'Fake Teen Doctor Strikes AGAIN! - Weekly Weird News',
  471.                 'upload_date': '20160218',
  472.                 'channel_id': 'UCdIaNUarhzLSXGoItz7BHVA',
  473.                 'duration': 1235,
  474.                 'description': 'md5:21032bae736421e89c2edf36d1936947',
  475.                 'uploader_id': 'MachinimaETC',
  476.                 'uploader_url': 'https://www.youtube.com/user/MachinimaETC',
  477.                 'channel_url': 'https://www.youtube.com/channel/UCdIaNUarhzLSXGoItz7BHVA',
  478.                 'thumbnail': r're:https?://.*\.(jpg|webp)',
  479.                 'uploader': 'ETC News',
  480.             },
  481.         }, {
  482.             # First capture of dead video, capture date in link links to dead capture.
  483.             'url': 'https://web.archive.org/web/20180803221945/https://www.youtube.com/watch?v=6FPhZJGvf4E',
  484.             'info_dict': {
  485.                 'id': '6FPhZJGvf4E',
  486.                 'ext': 'mp4',
  487.                 'title': 'WTF: Video Games Still Launch BROKEN?! - T.U.G.S.',
  488.                 'upload_date': '20160219',
  489.                 'channel_id': 'UCdIaNUarhzLSXGoItz7BHVA',
  490.                 'duration': 797,
  491.                 'description': 'md5:a1dbf12d9a3bd7cb4c5e33b27d77ffe7',
  492.                 'uploader_id': 'MachinimaETC',
  493.                 'uploader_url': 'https://www.youtube.com/user/MachinimaETC',
  494.                 'channel_url': 'https://www.youtube.com/channel/UCdIaNUarhzLSXGoItz7BHVA',
  495.                 'thumbnail': r're:https?://.*\.(jpg|webp)',
  496.                 'uploader': 'ETC News',
  497.             },
  498.             'expected_warnings': [
  499.                 r'unable to download capture webpage \(it may not be archived\)',
  500.             ],
  501.         }, {   # Very old YouTube page, has - YouTube in title.
  502.             'url': 'http://web.archive.org/web/20070302011044/http://youtube.com/watch?v=-06-KB9XTzg',
  503.             'info_dict': {
  504.                 'id': '-06-KB9XTzg',
  505.                 'ext': 'flv',
  506.                 'title': 'New Coin Hack!! 100% Safe!!',
  507.             },
  508.         }, {
  509.             'url': 'web.archive.org/https://www.youtube.com/watch?v=dWW7qP423y8',
  510.             'info_dict': {
  511.                 'id': 'dWW7qP423y8',
  512.                 'ext': 'mp4',
  513.                 'title': 'It\'s Bootleg AirPods Time.',
  514.                 'upload_date': '20211021',
  515.                 'channel_id': 'UC7Jwj9fkrf1adN4fMmTkpug',
  516.                 'channel_url': 'https://www.youtube.com/channel/UC7Jwj9fkrf1adN4fMmTkpug',
  517.                 'duration': 810,
  518.                 'description': 'md5:7b567f898d8237b256f36c1a07d6d7bc',
  519.                 'thumbnail': r're:https?://.*\.(jpg|webp)',
  520.                 'uploader': 'DankPods',
  521.             },
  522.         }, {
  523.             # player response contains '};' See: https://github.com/ytdl-org/youtube-dl/issues/27093
  524.             'url': 'https://web.archive.org/web/20200827003909if_/http://www.youtube.com/watch?v=6Dh-RL__uN4',
  525.             'info_dict': {
  526.                 'id': '6Dh-RL__uN4',
  527.                 'ext': 'mp4',
  528.                 'title': 'bitch lasagna',
  529.                 'upload_date': '20181005',
  530.                 'channel_id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
  531.                 'channel_url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
  532.                 'duration': 135,
  533.                 'description': 'md5:2dbe4051feeff2dab5f41f82bb6d11d0',
  534.                 'uploader': 'PewDiePie',
  535.                 'uploader_id': 'PewDiePie',
  536.                 'uploader_url': 'https://www.youtube.com/user/PewDiePie',
  537.                 'thumbnail': r're:https?://.*\.(jpg|webp)',
  538.             },
  539.         }, {
  540.             # ~June 2010 Capture. swfconfig
  541.             'url': 'https://web.archive.org/web/0/https://www.youtube.com/watch?v=8XeW5ilk-9Y',
  542.             'info_dict': {
  543.                 'id': '8XeW5ilk-9Y',
  544.                 'ext': 'flv',
  545.                 'title': 'Story of Stuff, The Critique Part 4 of 4',
  546.                 'duration': 541,
  547.                 'description': 'md5:28157da06f2c5e94c97f7f3072509972',
  548.                 'uploader': 'HowTheWorldWorks',
  549.                 'uploader_id': 'HowTheWorldWorks',
  550.                 'thumbnail': r're:https?://.*\.(jpg|webp)',
  551.                 'uploader_url': 'https://www.youtube.com/user/HowTheWorldWorks',
  552.                 'upload_date': '20090520',
  553.             },
  554.         }, {
  555.             # Jan 2011: watch-video-date/eow-date surrounded by whitespace
  556.             'url': 'https://web.archive.org/web/20110126141719/http://www.youtube.com/watch?v=Q_yjX80U7Yc',
  557.             'info_dict': {
  558.                 'id': 'Q_yjX80U7Yc',
  559.                 'ext': 'flv',
  560.                 'title': 'Spray Paint Art by Clay Butler: Purple Fantasy Forest',
  561.                 'uploader_id': 'claybutlermusic',
  562.                 'description': 'md5:4595264559e3d0a0ceb3f011f6334543',
  563.                 'upload_date': '20090803',
  564.                 'uploader': 'claybutlermusic',
  565.                 'thumbnail': r're:https?://.*\.(jpg|webp)',
  566.                 'duration': 132,
  567.                 'uploader_url': 'https://www.youtube.com/user/claybutlermusic',
  568.             },
  569.         }, {
  570.             # ~May 2009 swfArgs. ytcfg is spread out over various vars
  571.             'url': 'https://web.archive.org/web/0/https://www.youtube.com/watch?v=c5uJgG05xUY',
  572.             'info_dict': {
  573.                 'id': 'c5uJgG05xUY',
  574.                 'ext': 'webm',
  575.                 'title': 'Story of Stuff, The Critique Part 1 of 4',
  576.                 'uploader_id': 'HowTheWorldWorks',
  577.                 'uploader': 'HowTheWorldWorks',
  578.                 'uploader_url': 'https://www.youtube.com/user/HowTheWorldWorks',
  579.                 'upload_date': '20090513',
  580.                 'description': 'md5:4ca77d79538064e41e4cc464e93f44f0',
  581.                 'thumbnail': r're:https?://.*\.(jpg|webp)',
  582.                 'duration': 754,
  583.             },
  584.         }, {
  585.             # ~June 2012. Upload date is in another lang so cannot extract.
  586.             'url': 'https://web.archive.org/web/20120607174520/http://www.youtube.com/watch?v=xWTLLl-dQaA',
  587.             'info_dict': {
  588.                 'id': 'xWTLLl-dQaA',
  589.                 'ext': 'mp4',
  590.                 'title': 'Black Nerd eHarmony Video Bio Parody (SPOOF)',
  591.                 'uploader_url': 'https://www.youtube.com/user/BlackNerdComedy',
  592.                 'description': 'md5:e25f0133aaf9e6793fb81c18021d193e',
  593.                 'uploader_id': 'BlackNerdComedy',
  594.                 'uploader': 'BlackNerdComedy',
  595.                 'duration': 182,
  596.                 'thumbnail': r're:https?://.*\.(jpg|webp)',
  597.             },
  598.         }, {
  599.             # ~July 2013
  600.             'url': 'https://web.archive.org/web/*/https://www.youtube.com/watch?v=9eO1aasHyTM',
  601.             'info_dict': {
  602.                 'id': '9eO1aasHyTM',
  603.                 'ext': 'mp4',
  604.                 'title': 'Polar-oid',
  605.                 'description': 'Cameras and bears are dangerous!',
  606.                 'uploader_url': 'https://www.youtube.com/user/punkybird',
  607.                 'uploader_id': 'punkybird',
  608.                 'duration': 202,
  609.                 'channel_id': 'UC62R2cBezNBOqxSerfb1nMQ',
  610.                 'channel_url': 'https://www.youtube.com/channel/UC62R2cBezNBOqxSerfb1nMQ',
  611.                 'upload_date': '20060428',
  612.                 'uploader': 'punkybird',
  613.             },
  614.         }, {
  615.             # April 2020: Player response in player config
  616.             'url': 'https://web.archive.org/web/20200416034815/https://www.youtube.com/watch?v=Cf7vS8jc7dY&gl=US&hl=en',
  617.             'info_dict': {
  618.                 'id': 'Cf7vS8jc7dY',
  619.                 'ext': 'mp4',
  620.                 'title': 'A Dramatic Pool Story (by Jamie Spicer-Lewis) - Game Grumps Animated',
  621.                 'duration': 64,
  622.                 'upload_date': '20200408',
  623.                 'uploader_id': 'GameGrumps',
  624.                 'uploader': 'GameGrumps',
  625.                 'channel_url': 'https://www.youtube.com/channel/UC9CuvdOVfMPvKCiwdGKL3cQ',
  626.                 'channel_id': 'UC9CuvdOVfMPvKCiwdGKL3cQ',
  627.                 'thumbnail': r're:https?://.*\.(jpg|webp)',
  628.                 'description': 'md5:c625bb3c02c4f5fb4205971e468fa341',
  629.                 'uploader_url': 'https://www.youtube.com/user/GameGrumps',
  630.             },
  631.         }, {
  632.             # watch7-user-header with yt-user-info
  633.             'url': 'ytarchive:kbh4T_b4Ixw:20160307085057',
  634.             'info_dict': {
  635.                 'id': 'kbh4T_b4Ixw',
  636.                 'ext': 'mp4',
  637.                 'title': 'Shovel Knight OST - Strike the Earth! Plains of Passage 16 bit SNES style remake / remix',
  638.                 'channel_url': 'https://www.youtube.com/channel/UCnTaGvsHmMy792DWeT6HbGA',
  639.                 'uploader': 'Nelward music',
  640.                 'duration': 213,
  641.                 'description': 'md5:804b4a9ce37b050a5fefdbb23aeba54d',
  642.                 'thumbnail': r're:https?://.*\.(jpg|webp)',
  643.                 'upload_date': '20150503',
  644.                 'channel_id': 'UCnTaGvsHmMy792DWeT6HbGA',
  645.             },
  646.         }, {
  647.             # April 2012
  648.             'url': 'https://web.archive.org/web/0/https://www.youtube.com/watch?v=SOm7mPoPskU',
  649.             'info_dict': {
  650.                 'id': 'SOm7mPoPskU',
  651.                 'ext': 'mp4',
  652.                 'title': 'Boyfriend - Justin Bieber Parody',
  653.                 'uploader_url': 'https://www.youtube.com/user/thecomputernerd01',
  654.                 'uploader': 'thecomputernerd01',
  655.                 'thumbnail': r're:https?://.*\.(jpg|webp)',
  656.                 'description': 'md5:dd7fa635519c2a5b4d566beaecad7491',
  657.                 'duration': 200,
  658.                 'upload_date': '20120407',
  659.                 'uploader_id': 'thecomputernerd01',
  660.             },
  661.         }, {
  662.             'url': 'https://web.archive.org/web/http://www.youtube.com/watch?v=kH-G_aIBlFw',
  663.             'only_matching': True,
  664.         }, {
  665.             'url': 'https://web.archive.org/web/20050214000000_if/http://www.youtube.com/watch?v=0altSZ96U4M',
  666.             'only_matching': True,
  667.         }, {
  668.             # Video not archived, only capture is unavailable video page
  669.             'url': 'https://web.archive.org/web/20210530071008/https://www.youtube.com/watch?v=lHJTf93HL1s&spfreload=10',
  670.             'only_matching': True,
  671.         }, {   # Encoded url
  672.             'url': 'https://web.archive.org/web/20120712231619/http%3A//www.youtube.com/watch%3Fgl%3DUS%26v%3DAkhihxRKcrs%26hl%3Den',
  673.             'only_matching': True,
  674.         }, {
  675.             'url': 'https://web.archive.org/web/20120712231619/http%3A//www.youtube.com/watch%3Fv%3DAkhihxRKcrs%26gl%3DUS%26hl%3Den',
  676.             'only_matching': True,
  677.         }, {
  678.             'url': 'https://web.archive.org/web/20060527081937/http://www.youtube.com:80/watch.php?v=ELTFsLT73fA&amp;search=soccer',
  679.             'only_matching': True,
  680.         }, {
  681.             'url': 'https://web.archive.org/http://www.youtube.com:80/watch?v=-05VVye-ffg',
  682.             'only_matching': True,
  683.         }, {
  684.             'url': 'ytarchive:BaW_jenozKc:20050214000000',
  685.             'only_matching': True,
  686.         }, {
  687.             'url': 'ytarchive:BaW_jenozKc',
  688.             'only_matching': True,
  689.         },
  690.     ]
  691.     _YT_INITIAL_DATA_RE = YoutubeBaseInfoExtractor._YT_INITIAL_DATA_RE
  692.     _YT_INITIAL_PLAYER_RESPONSE_RE = fr'''(?x:
  693.         (?:window\s*\[\s*["\']ytInitialPlayerResponse["\']\s*\]|ytInitialPlayerResponse)\s*=[(\s]*|
  694.         {YoutubeBaseInfoExtractor._YT_INITIAL_PLAYER_RESPONSE_RE}
  695.     )'''
  697.     _YT_DEFAULT_THUMB_SERVERS = ['i.ytimg.com']  # thumbnails most likely archived on these servers
  698.     _YT_ALL_THUMB_SERVERS = orderedSet(
  699.         [*_YT_DEFAULT_THUMB_SERVERS, 'img.youtube.com', *[f'{c}{n or ""}.ytimg.com' for c in ('i', 's') for n in (*range(5), 9)]])
  701.     _WAYBACK_BASE_URL = 'https://web.archive.org/web/%sif_/'
  702.     _OLDEST_CAPTURE_DATE = 20050214000000
  703.     _NEWEST_CAPTURE_DATE = 20500101000000
  705.     def _call_cdx_api(self, item_id, url, filters: list | None = None, collapse: list | None = None, query: dict | None = None, note=None, fatal=False):
  706.         # CDX docs: https://github.com/internetarchive/wayback/blob/master/wayback-cdx-server/README.md
  707.         query = {
  708.             'url': url,
  709.             'output': 'json',
  710.             'fl': 'original,mimetype,length,timestamp',
  711.             'limit': 500,
  712.             'filter': ['statuscode:200'] + (filters or []),
  713.             'collapse': collapse or [],
  714.             **(query or {}),
  715.         }
  716.         res = self._download_json(
  717.             'https://web.archive.org/cdx/search/cdx', item_id,
  718.             note or 'Downloading CDX API JSON', query=query, fatal=fatal)
  719.         if isinstance(res, list) and len(res) >= 2:
  720.             # format response to make it easier to use
  721.             return [dict(zip(res[0], v)) for v in res[1:]]
  722.         elif not isinstance(res, list) or len(res) != 0:
  723.             self.report_warning('Error while parsing CDX API response' + bug_reports_message())
  725.     def _extract_webpage_title(self, webpage):
  726.         page_title = self._html_extract_title(webpage, default='')
  727.         # YouTube video pages appear to always have either 'YouTube -' as prefix or '- YouTube' as suffix.
  728.         return self._html_search_regex(
  729.             r'(?:YouTube\s*-\s*(.*)$)|(?:(.*)\s*-\s*YouTube$)',
  730.             page_title, 'title', default='')
  732.     def _extract_metadata(self, video_id, webpage):
  733.         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None)) if webpage else (lambda x: None))
  734.         player_response = self._search_json(
  735.             self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response',
  736.             video_id, default={})
  737.         initial_data = self._search_json(
  738.             self._YT_INITIAL_DATA_RE, webpage, 'initial data', video_id, default={})
  740.         ytcfg = {}
  741.         for j in re.findall(r'yt\.setConfig\(\s*(?P<json>{\s*(?s:.+?)\s*})\s*\);', webpage):  # ~June 2010
  742.             ytcfg.update(self._parse_json(j, video_id, fatal=False, ignore_extra=True, transform_source=js_to_json, errnote='') or {})
  744.         # XXX: this also may contain a 'ptchn' key
  745.         player_config = (
  746.             self._search_json(
  747.                 r'(?:yt\.playerConfig|ytplayer\.config|swfConfig)\s*=',
  748.                 webpage, 'player config', video_id, default=None)
  749.             or ytcfg.get('PLAYER_CONFIG') or {})
  751.         # XXX: this may also contain a 'creator' key.
  752.         swf_args = self._search_json(r'swfArgs\s*=', webpage, 'swf config', video_id, default={})
  753.         if swf_args and not traverse_obj(player_config, ('args',)):
  754.             player_config['args'] = swf_args
  756.         if not player_response:
  757.             # April 2020
  758.             player_response = self._parse_json(
  759.                 traverse_obj(player_config, ('args', 'player_response')) or '{}', video_id, fatal=False)
  761.         initial_data_video = traverse_obj(
  762.             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'videoPrimaryInfoRenderer'),
  763.             expected_type=dict, get_all=False, default={})
  765.         video_details = traverse_obj(
  766.             player_response, 'videoDetails', expected_type=dict, get_all=False, default={})
  768.         microformats = traverse_obj(
  769.             player_response, ('microformat', 'playerMicroformatRenderer'), expected_type=dict, get_all=False, default={})
  771.         video_title = (
  772.             video_details.get('title')
  773.             or YoutubeBaseInfoExtractor._get_text(microformats, 'title')
  774.             or YoutubeBaseInfoExtractor._get_text(initial_data_video, 'title')
  775.             or traverse_obj(player_config, ('args', 'title'))
  776.             or self._extract_webpage_title(webpage)
  777.             or search_meta(['og:title', 'twitter:title', 'title']))
  779.         def id_from_url(url, type_):
  780.             return self._search_regex(
  781.                 rf'(?:{type_})/([^/#&?]+)', url or '', f'{type_} id', default=None)
  783.         # XXX: would the get_elements_by_... functions be better suited here?
  784.         _CHANNEL_URL_HREF_RE = r'href="[^"]*(?P<url>https?://www\.youtube\.com/(?:user|channel)/[^"]+)"'
  785.         uploader_or_channel_url = self._search_regex(
  786.             [fr'<(?:link\s*itemprop=\"url\"|a\s*id=\"watch-username\").*?\b{_CHANNEL_URL_HREF_RE}>',  # @fd05024
  787.              fr'<div\s*id=\"(?:watch-channel-stats|watch-headline-user-info)\"[^>]*>\s*<a[^>]*\b{_CHANNEL_URL_HREF_RE}'],  # ~ May 2009, ~June 2012
  788.             webpage, 'uploader or channel url', default=None)
  790.         owner_profile_url = url_or_none(microformats.get('ownerProfileUrl'))  # @a6211d2
  792.         # Uploader refers to the /user/ id ONLY
  793.         uploader_id = (
  794.             id_from_url(owner_profile_url, 'user')
  795.             or id_from_url(uploader_or_channel_url, 'user')
  796.             or ytcfg.get('VIDEO_USERNAME'))
  797.         uploader_url = f'https://www.youtube.com/user/{uploader_id}' if uploader_id else None
  799.         # XXX: do we want to differentiate uploader and channel?
  800.         uploader = (
  801.             self._search_regex(
  802.                 [r'<a\s*id="watch-username"[^>]*>\s*<strong>([^<]+)</strong>',  # June 2010
  803.                  r'var\s*watchUsername\s*=\s*\'(.+?)\';',  # ~May 2009
  804.                  r'<div\s*\bid=\"watch-channel-stats"[^>]*>\s*<a[^>]*>\s*(.+?)\s*</a',  # ~May 2009
  805.                  r'<a\s*id="watch-userbanner"[^>]*title="\s*(.+?)\s*"'],  # ~June 2012
  806.                 webpage, 'uploader', default=None)
  807.             or self._html_search_regex(
  808.                 [r'(?s)<div\s*class="yt-user-info".*?<a[^>]*[^>]*>\s*(.*?)\s*</a',  # March 2016
  809.                  r'(?s)<a[^>]*yt-user-name[^>]*>\s*(.*?)\s*</a'],  # july 2013
  810.                 get_element_by_id('watch7-user-header', webpage), 'uploader', default=None)
  811.             or self._html_search_regex(
  812.                 r'<button\s*href="/user/[^>]*>\s*<span[^>]*>\s*(.+?)\s*<',  # April 2012
  813.                 get_element_by_id('watch-headline-user-info', webpage), 'uploader', default=None)
  814.             or traverse_obj(player_config, ('args', 'creator'))
  815.             or video_details.get('author'))
  817.         channel_id = str_or_none(
  818.             video_details.get('channelId')
  819.             or microformats.get('externalChannelId')
  820.             or search_meta('channelId')
  821.             or self._search_regex(
  822.                 r'data-channel-external-id=(["\'])(?P<id>(?:(?!\1).)+)\1',  # @b45a9e6
  823.                 webpage, 'channel id', default=None, group='id')
  824.             or id_from_url(owner_profile_url, 'channel')
  825.             or id_from_url(uploader_or_channel_url, 'channel')
  826.             or traverse_obj(player_config, ('args', 'ucid')))
  828.         channel_url = f'https://www.youtube.com/channel/{channel_id}' if channel_id else None
  829.         duration = int_or_none(
  830.             video_details.get('lengthSeconds')
  831.             or microformats.get('lengthSeconds')
  832.             or traverse_obj(player_config, ('args', ('length_seconds', 'l')), get_all=False)
  833.             or parse_duration(search_meta('duration')))
  834.         description = (
  835.             video_details.get('shortDescription')
  836.             or YoutubeBaseInfoExtractor._get_text(microformats, 'description')
  837.             or clean_html(get_element_by_id('eow-description', webpage))  # @9e6dd23
  838.             or search_meta(['description', 'og:description', 'twitter:description']))
  840.         upload_date = unified_strdate(
  841.             dict_get(microformats, ('uploadDate', 'publishDate'))
  842.             or search_meta(['uploadDate', 'datePublished'])
  843.             or self._search_regex(
  844.                 [r'(?s)id="eow-date.*?>\s*(.*?)\s*</span>',
  845.                  r'(?:id="watch-uploader-info".*?>.*?|["\']simpleText["\']\s*:\s*["\'])(?:Published|Uploaded|Streamed live|Started) on (.+?)[<"\']',  # @7998520
  846.                  r'class\s*=\s*"(?:watch-video-date|watch-video-added post-date)"[^>]*>\s*([^<]+?)\s*<'],  # ~June 2010, ~Jan 2009 (respectively)
  847.                 webpage, 'upload date', default=None))
  849.         return {
  850.             'title': video_title,
  851.             'description': description,
  852.             'upload_date': upload_date,
  853.             'uploader': uploader,
  854.             'channel_id': channel_id,
  855.             'channel_url': channel_url,
  856.             'duration': duration,
  857.             'uploader_url': uploader_url,
  858.             'uploader_id': uploader_id,
  859.         }
  861.     def _extract_thumbnails(self, video_id):
  862.         try_all = 'thumbnails' in self._configuration_arg('check_all')
  863.         thumbnail_base_urls = ['http://{server}/vi{webp}/{video_id}'.format(
  864.             webp='_webp' if ext == 'webp' else '', video_id=video_id, server=server)
  865.             for server in (self._YT_ALL_THUMB_SERVERS if try_all else self._YT_DEFAULT_THUMB_SERVERS) for ext in (('jpg', 'webp') if try_all else ('jpg',))]
  867.         thumbnails = []
  868.         for url in thumbnail_base_urls:
  869.             response = self._call_cdx_api(
  870.                 video_id, url, filters=['mimetype:image/(?:webp|jpeg)'],
  871.                 collapse=['urlkey'], query={'matchType': 'prefix'})
  872.             if not response:
  873.                 continue
  874.             thumbnails.extend(
  875.                 {
  876.                     'url': (self._WAYBACK_BASE_URL % (int_or_none(thumbnail_dict.get('timestamp')) or self._OLDEST_CAPTURE_DATE)) + thumbnail_dict.get('original'),
  877.                     'filesize': int_or_none(thumbnail_dict.get('length')),
  878.                     'preference': int_or_none(thumbnail_dict.get('length')),
  879.                 } for thumbnail_dict in response)
  880.             if not try_all:
  881.                 break
  883.         self._remove_duplicate_formats(thumbnails)
  884.         return thumbnails
  886.     def _get_capture_dates(self, video_id, url_date):
  887.         capture_dates = []
  888.         # Note: CDX API will not find watch pages with extra params in the url.
  889.         response = self._call_cdx_api(
  890.             video_id, f'https://www.youtube.com/watch?v={video_id}',
  891.             filters=['mimetype:text/html'], collapse=['timestamp:6', 'digest'], query={'matchType': 'prefix'}) or []
  892.         all_captures = sorted(int_or_none(r['timestamp']) for r in response if int_or_none(r['timestamp']) is not None)
  894.         # Prefer the new polymer UI captures as we support extracting more metadata from them
  895.         # WBM captures seem to all switch to this layout ~July 2020
  896.         modern_captures = [x for x in all_captures if x >= 20200701000000]
  897.         if modern_captures:
  898.             capture_dates.append(modern_captures[0])
  899.         capture_dates.append(url_date)
  900.         if all_captures:
  901.             capture_dates.append(all_captures[0])
  903.         if 'captures' in self._configuration_arg('check_all'):
  904.             capture_dates.extend(modern_captures + all_captures)
  906.         # Fallbacks if any of the above fail
  907.         capture_dates.extend([self._OLDEST_CAPTURE_DATE, self._NEWEST_CAPTURE_DATE])
  908.         return orderedSet(filter(None, capture_dates))
  910.     def _real_extract(self, url):
  911.         video_id, url_date, url_date_2 = self._match_valid_url(url).group('id', 'date', 'date2')
  912.         url_date = url_date or url_date_2
  914.         urlh = None
  915.         retry_manager = self.RetryManager(fatal=False)
  916.         for retry in retry_manager:
  917.             try:
  918.                 urlh = self._request_webpage(
  919.                     HEADRequest(f'https://web.archive.org/web/2oe_/http://wayback-fakeurl.archive.org/yt/{video_id}'),
  920.                     video_id, note='Fetching archived video file url', expected_status=True)
  921.             except ExtractorError as e:
  922.                 # HTTP Error 404 is expected if the video is not saved.
  923.                 if isinstance(e.cause, HTTPError) and e.cause.status == 404:
  924.                     self.raise_no_formats(
  925.                         'The requested video is not archived, indexed, or there is an issue with web.archive.org (try again later)', expected=True)
  926.                 else:
  927.                     retry.error = e
  929.         if retry_manager.error:
  930.             self.raise_no_formats(retry_manager.error, expected=True, video_id=video_id)
  932.         capture_dates = self._get_capture_dates(video_id, int_or_none(url_date))
  933.         self.write_debug('Captures to try: ' + join_nonempty(*capture_dates, delim=', '))
  934.         info = {'id': video_id}
  935.         for capture in capture_dates:
  936.             webpage = self._download_webpage(
  937.                 (self._WAYBACK_BASE_URL + 'http://www.youtube.com/watch?v=%s') % (capture, video_id),
  938.                 video_id=video_id, fatal=False, errnote='unable to download capture webpage (it may not be archived)',
  939.                 note='Downloading capture webpage')
  940.             current_info = self._extract_metadata(video_id, webpage or '')
  941.             # Try avoid getting deleted video metadata
  942.             if current_info.get('title'):
  943.                 info = merge_dicts(info, current_info)
  944.                 if 'captures' not in self._configuration_arg('check_all'):
  945.                     break
  947.         info['thumbnails'] = self._extract_thumbnails(video_id)
  949.         if urlh:
  950.             url = urllib.parse.unquote(urlh.url)
  951.             video_file_url_qs = parse_qs(url)
  952.             # Attempt to recover any ext & format info from playback url & response headers
  953.             fmt = {'url': url, 'filesize': int_or_none(urlh.headers.get('x-archive-orig-content-length'))}
  954.             itag = try_get(video_file_url_qs, lambda x: x['itag'][0])
  955.             if itag and itag in YoutubeIE._formats:
  956.                 fmt.update(YoutubeIE._formats[itag])
  957.                 fmt.update({'format_id': itag})
  958.             else:
  959.                 mime = try_get(video_file_url_qs, lambda x: x['mime'][0])
  960.                 ext = (mimetype2ext(mime)
  961.                        or urlhandle_detect_ext(urlh)
  962.                        or mimetype2ext(urlh.headers.get('x-archive-guessed-content-type')))
  963.                 fmt.update({'ext': ext})
  964.             info['formats'] = [fmt]
  965.             if not info.get('duration'):
  966.                 info['duration'] = str_to_int(try_get(video_file_url_qs, lambda x: x['dur'][0]))
  968.         if not info.get('title'):
  969.             info['title'] = video_id
  970.         return info