logo

oasis-root

Compiled tree of Oasis Linux based on own branch at <https://hacktivis.me/git/oasis/> git clone https://anongit.hacktivis.me/git/oasis-root.git

nfl.py (16180B)

    

  1. import base64
  2. import json
  3. import re
  4. import time
  5. import uuid
  7. from .anvato import AnvatoIE
  8. from .common import InfoExtractor
  9. from ..utils import (
  10.     ExtractorError,
  11.     clean_html,
  12.     determine_ext,
  13.     get_element_by_class,
  14.     int_or_none,
  15.     make_archive_id,
  16.     url_or_none,
  17.     urlencode_postdata,
  18. )
  19. from ..utils.traversal import traverse_obj
  22. class NFLBaseIE(InfoExtractor):
  23.     _VALID_URL_BASE = r'''(?x)
  24.                     https?://
  25.                         (?P<host>
  26.                             (?:www\.)?
  27.                             (?:
  28.                                 (?:
  29.                                     nfl|
  30.                                     buffalobills|
  31.                                     miamidolphins|
  32.                                     patriots|
  33.                                     newyorkjets|
  34.                                     baltimoreravens|
  35.                                     bengals|
  36.                                     clevelandbrowns|
  37.                                     steelers|
  38.                                     houstontexans|
  39.                                     colts|
  40.                                     jaguars|
  41.                                     (?:titansonline|tennesseetitans)|
  42.                                     denverbroncos|
  43.                                     (?:kc)?chiefs|
  44.                                     raiders|
  45.                                     chargers|
  46.                                     dallascowboys|
  47.                                     giants|
  48.                                     philadelphiaeagles|
  49.                                     (?:redskins|washingtonfootball)|
  50.                                     chicagobears|
  51.                                     detroitlions|
  52.                                     packers|
  53.                                     vikings|
  54.                                     atlantafalcons|
  55.                                     panthers|
  56.                                     neworleanssaints|
  57.                                     buccaneers|
  58.                                     azcardinals|
  59.                                     (?:stlouis|the)rams|
  60.                                     49ers|
  61.                                     seahawks
  62.                                 )\.com|
  63.                                 .+?\.clubs\.nfl\.com
  64.                             )
  65.                         )/
  66.                     '''
  67.     _VIDEO_CONFIG_REGEX = r'<script[^>]+id="[^"]*video-config-[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}[^"]*"[^>]*>\s*({.+});?\s*</script>'
  68.     _ANVATO_PREFIX = 'anvato:GXvEgwyJeWem8KCYXfeoHWknwP48Mboj:'
  70.     _CLIENT_DATA = {
  71.         'clientKey': '4cFUW6DmwJpzT9L7LrG3qRAcABG5s04g',
  72.         'clientSecret': 'CZuvCL49d9OwfGsR',
  73.         'deviceId': str(uuid.uuid4()),
  74.         'deviceInfo': base64.b64encode(json.dumps({
  75.             'model': 'desktop',
  76.             'version': 'Chrome',
  77.             'osName': 'Windows',
  78.             'osVersion': '10.0',
  79.         }, separators=(',', ':')).encode()).decode(),
  80.         'networkType': 'other',
  81.         'peacockUUID': 'undefined',
  82.     }
  83.     _ACCOUNT_INFO = {}
  84.     _API_KEY = '3_Qa8TkWpIB8ESCBT8tY2TukbVKgO5F6BJVc7N1oComdwFzI7H2L9NOWdm11i_BY9f'
  86.     _TOKEN = None
  87.     _TOKEN_EXPIRY = 0
  89.     def _get_account_info(self):
  90.         cookies = self._get_cookies('https://auth-id.nfl.com/')
  91.         login_token = traverse_obj(cookies, (
  92.             (f'glt_{self._API_KEY}', lambda k, _: k.startswith('glt_')), {lambda x: x.value}), get_all=False)
  93.         if not login_token:
  94.             self.raise_login_required()
  95.         if 'ucid' not in cookies:
  96.             raise ExtractorError(
  97.                 'Required cookies for the auth-id.nfl.com domain were not found among passed cookies. '
  98.                 'If using --cookies, these cookies must be exported along with .nfl.com cookies, '
  99.                 'or else try using --cookies-from-browser instead', expected=True)
  101.         account = self._download_json(
  102.             'https://auth-id.nfl.com/accounts.getAccountInfo', None,
  103.             note='Downloading account info', data=urlencode_postdata({
  104.                 'include': 'profile,data',
  105.                 'lang': 'en',
  106.                 'APIKey': self._API_KEY,
  107.                 'sdk': 'js_latest',
  108.                 'login_token': login_token,
  109.                 'authMode': 'cookie',
  110.                 'pageURL': 'https://www.nfl.com/',
  111.                 'sdkBuild': traverse_obj(cookies, (
  112.                     'gig_canary_ver', {lambda x: x.value.partition('-')[0]}), default='15170'),
  113.                 'format': 'json',
  114.             }), headers={'Content-Type': 'application/x-www-form-urlencoded'})
  116.         self._ACCOUNT_INFO = traverse_obj(account, {
  117.             'signatureTimestamp': 'signatureTimestamp',
  118.             'uid': 'UID',
  119.             'uidSignature': 'UIDSignature',
  120.         })
  122.         if len(self._ACCOUNT_INFO) != 3:
  123.             raise ExtractorError('Failed to retrieve account info with provided cookies', expected=True)
  125.     def _get_auth_token(self):
  126.         if self._TOKEN and self._TOKEN_EXPIRY > int(time.time() + 30):
  127.             return
  129.         token = self._download_json(
  130.             'https://api.nfl.com/identity/v3/token%s' % (
  131.                 '/refresh' if self._ACCOUNT_INFO.get('refreshToken') else ''),
  132.             None, headers={'Content-Type': 'application/json'}, note='Downloading access token',
  133.             data=json.dumps({**self._CLIENT_DATA, **self._ACCOUNT_INFO}, separators=(',', ':')).encode())
  135.         self._TOKEN = token['accessToken']
  136.         self._TOKEN_EXPIRY = token['expiresIn']
  137.         self._ACCOUNT_INFO['refreshToken'] = token['refreshToken']
  139.     def _extract_video(self, mcp_id, is_live=False):
  140.         self._get_auth_token()
  141.         data = self._download_json(
  142.             f'https://api.nfl.com/play/v1/asset/{mcp_id}', mcp_id, headers={
  143.                 'Authorization': f'Bearer {self._TOKEN}',
  144.                 'Accept': 'application/json',
  145.                 'Content-Type': 'application/json',
  146.             }, data=json.dumps({'init': True, 'live': is_live}, separators=(',', ':')).encode())
  147.         formats, subtitles = self._extract_m3u8_formats_and_subtitles(
  148.             data['accessUrl'], mcp_id, 'mp4', m3u8_id='hls')
  150.         return {
  151.             'id': mcp_id,
  152.             'formats': formats,
  153.             'subtitles': subtitles,
  154.             'is_live': is_live,
  155.             '_old_archive_ids': [make_archive_id(AnvatoIE, mcp_id)],
  156.             **traverse_obj(data, ('metadata', {
  157.                 'title': ('event', ('def_title', 'friendlyName'), {str}, any),
  158.                 'description': ('event', 'def_description', {str}),
  159.                 'duration': ('event', 'duration', {int_or_none}),
  160.                 'thumbnails': ('thumbnails', ..., 'url', {'url': {url_or_none}}),
  161.             })),
  162.         }
  164.     def _parse_video_config(self, video_config, display_id):
  165.         video_config = self._parse_json(video_config, display_id)
  166.         is_live = traverse_obj(video_config, ('live', {bool})) or False
  167.         item = video_config['playlist'][0]
  168.         if mcp_id := item.get('mcpID'):
  169.             return self._extract_video(mcp_id, is_live=is_live)
  171.         info = {'id': item.get('id') or item['entityId']}
  173.         item_url = item['url']
  174.         ext = determine_ext(item_url)
  175.         if ext == 'm3u8':
  176.             info['formats'] = self._extract_m3u8_formats(item_url, info['id'], 'mp4')
  177.         else:
  178.             info['url'] = item_url
  179.             if item.get('audio') is True:
  180.                 info['vcodec'] = 'none'
  182.         thumbnails = None
  183.         if image_url := traverse_obj(item, 'imageSrc', 'posterImage', expected_type=url_or_none):
  184.             thumbnails = [{
  185.                 'url': image_url,
  186.                 'ext': determine_ext(image_url, 'jpg'),
  187.             }]
  189.         info.update({
  190.             **traverse_obj(item, {
  191.                 'title': ('title', {str}),
  192.                 'description': ('description', {clean_html}),
  193.             }),
  194.             'is_live': is_live,
  195.             'thumbnails': thumbnails,
  196.         })
  197.         return info
  200. class NFLIE(NFLBaseIE):
  201.     IE_NAME = 'nfl.com'
  202.     _VALID_URL = NFLBaseIE._VALID_URL_BASE + r'(?:videos?|listen|audio)/(?P<id>[^/#?&]+)'
  203.     _TESTS = [{
  204.         'url': 'https://www.nfl.com/videos/baker-mayfield-s-game-changing-plays-from-3-td-game-week-14',
  205.         'info_dict': {
  206.             'id': '899441',
  207.             'ext': 'mp4',
  208.             'title': "Baker Mayfield's game-changing plays from 3-TD game Week 14",
  209.             'description': 'md5:85e05a3cc163f8c344340f220521136d',
  210.             'thumbnail': r're:https?://.+\.jpg',
  211.             'duration': 157,
  212.             '_old_archive_ids': ['anvato 899441'],
  213.         },
  214.     }, {
  215.         'url': 'https://www.chiefs.com/listen/patrick-mahomes-travis-kelce-react-to-win-over-dolphins-the-breakdown',
  216.         'md5': '92a517f05bd3eb50fe50244bc621aec8',
  217.         'info_dict': {
  218.             'id': '8b7c3625-a461-4751-8db4-85f536f2bbd0',
  219.             'ext': 'mp3',
  220.             'title': 'Patrick Mahomes, Travis Kelce React to Win Over Dolphins | The Breakdown',
  221.             'description': 'md5:12ada8ee70e6762658c30e223e095075',
  222.             'thumbnail': 'https://static.clubs.nfl.com/image/private/t_editorial_landscape_12_desktop/v1571153441/chiefs/rfljejccnyhhkpkfq855',
  223.         },
  224.     }, {
  225.         'url': 'https://www.buffalobills.com/video/buffalo-bills-military-recognition-week-14',
  226.         'only_matching': True,
  227.     }, {
  228.         'url': 'https://www.raiders.com/audio/instant-reactions-raiders-week-14-loss-to-indianapolis-colts-espn-jason-fitz',
  229.         'only_matching': True,
  230.     }]
  232.     def _real_extract(self, url):
  233.         display_id = self._match_id(url)
  234.         webpage = self._download_webpage(url, display_id)
  235.         return self._parse_video_config(self._search_regex(
  236.             self._VIDEO_CONFIG_REGEX, webpage, 'video config'), display_id)
  239. class NFLArticleIE(NFLBaseIE):
  240.     IE_NAME = 'nfl.com:article'
  241.     _VALID_URL = NFLBaseIE._VALID_URL_BASE + r'news/(?P<id>[^/#?&]+)'
  242.     _TEST = {
  243.         'url': 'https://www.buffalobills.com/news/the-only-thing-we-ve-earned-is-the-noise-bills-coaches-discuss-handling-rising-e',
  244.         'info_dict': {
  245.             'id': 'the-only-thing-we-ve-earned-is-the-noise-bills-coaches-discuss-handling-rising-e',
  246.             'title': "'The only thing we've earned is the noise' | Bills coaches discuss handling rising expectations",
  247.         },
  248.         'playlist_count': 4,
  249.     }
  251.     def _real_extract(self, url):
  252.         display_id = self._match_id(url)
  253.         webpage = self._download_webpage(url, display_id)
  255.         def entries():
  256.             for video_config in re.findall(self._VIDEO_CONFIG_REGEX, webpage):
  257.                 yield self._parse_video_config(video_config, display_id)
  259.         title = clean_html(get_element_by_class(
  260.             'nfl-c-article__title', webpage)) or self._html_search_meta(
  261.             ['og:title', 'twitter:title'], webpage)
  263.         return self.playlist_result(entries(), display_id, title)
  266. class NFLPlusReplayIE(NFLBaseIE):
  267.     IE_NAME = 'nfl.com:plus:replay'
  268.     _VALID_URL = r'https?://(?:www\.)?nfl\.com/plus/games/(?P<slug>[\w-]+)(?:/(?P<id>\d+))?'
  269.     _TESTS = [{
  270.         'url': 'https://www.nfl.com/plus/games/giants-at-vikings-2022-post-1/1572108',
  271.         'info_dict': {
  272.             'id': '1572108',
  273.             'ext': 'mp4',
  274.             'title': 'New York Giants at Minnesota Vikings',
  275.             'description': 'New York Giants play the Minnesota Vikings at U.S. Bank Stadium on January 15, 2023',
  276.             'uploader': 'NFL',
  277.             'upload_date': '20230116',
  278.             'timestamp': 1673864520,
  279.             'duration': 7157,
  280.             'categories': ['Game Highlights'],
  281.             'tags': ['Minnesota Vikings', 'New York Giants', 'Minnesota Vikings vs. New York Giants'],
  282.             'thumbnail': r're:^https?://.*\.jpg',
  283.         },
  284.         'params': {'skip_download': 'm3u8'},
  285.     }, {
  286.         'note': 'Subscription required',
  287.         'url': 'https://www.nfl.com/plus/games/giants-at-vikings-2022-post-1',
  288.         'playlist_count': 4,
  289.         'info_dict': {
  290.             'id': 'giants-at-vikings-2022-post-1',
  291.         },
  292.     }, {
  293.         'note': 'Subscription required',
  294.         'url': 'https://www.nfl.com/plus/games/giants-at-patriots-2011-pre-4',
  295.         'playlist_count': 2,
  296.         'info_dict': {
  297.             'id': 'giants-at-patriots-2011-pre-4',
  298.         },
  299.     }, {
  300.         'note': 'Subscription required',
  301.         'url': 'https://www.nfl.com/plus/games/giants-at-patriots-2011-pre-4',
  302.         'info_dict': {
  303.             'id': '950701',
  304.             'ext': 'mp4',
  305.             'title': 'Giants @ Patriots',
  306.             'description': 'Giants at Patriots on September 01, 2011',
  307.             'uploader': 'NFL',
  308.             'upload_date': '20210724',
  309.             'timestamp': 1627085874,
  310.             'duration': 1532,
  311.             'categories': ['Game Highlights'],
  312.             'tags': ['play-by-play'],
  313.             'thumbnail': r're:^https?://.*\.jpg',
  314.         },
  315.         'params': {
  316.             'skip_download': 'm3u8',
  317.             'extractor_args': {'nflplusreplay': {'type': ['condensed_game']}},
  318.         },
  319.     }]
  321.     _REPLAY_TYPES = {
  322.         'full_game': 'Full Game',
  323.         'full_game_spanish': 'Full Game - Spanish',
  324.         'condensed_game': 'Condensed Game',
  325.         'all_22': 'All-22',
  326.     }
  328.     def _real_initialize(self):
  329.         self._get_account_info()
  331.     def _real_extract(self, url):
  332.         slug, video_id = self._match_valid_url(url).group('slug', 'id')
  333.         requested_types = self._configuration_arg('type', ['all'])
  334.         if 'all' in requested_types:
  335.             requested_types = list(self._REPLAY_TYPES.keys())
  336.         requested_types = traverse_obj(self._REPLAY_TYPES, (None, requested_types))
  338.         if not video_id:
  339.             self._get_auth_token()
  340.             headers = {'Authorization': f'Bearer {self._TOKEN}'}
  341.             game_id = self._download_json(
  342.                 f'https://api.nfl.com/football/v2/games/externalId/slug/{slug}', slug,
  343.                 'Downloading game ID', query={'withExternalIds': 'true'}, headers=headers)['id']
  344.             replays = self._download_json(
  345.                 'https://api.nfl.com/content/v1/videos/replays', slug, 'Downloading replays JSON',
  346.                 query={'gameId': game_id}, headers=headers)
  347.             if len(requested_types) == 1:
  348.                 video_id = traverse_obj(replays, (
  349.                     'items', lambda _, v: v['subType'] == requested_types[0], 'mcpPlaybackId'), get_all=False)
  351.         if video_id:
  352.             return self._extract_video(video_id)
  354.         def entries():
  355.             for replay in traverse_obj(
  356.                 replays, ('items', lambda _, v: v['mcpPlaybackId'] and v['subType'] in requested_types),
  357.             ):
  358.                 yield self._extract_video(replay['mcpPlaybackId'])
  360.         return self.playlist_result(entries(), slug)
  363. class NFLPlusEpisodeIE(NFLBaseIE):
  364.     IE_NAME = 'nfl.com:plus:episode'
  365.     _VALID_URL = r'https?://(?:www\.)?nfl\.com/plus/episodes/(?P<id>[\w-]+)'
  366.     _TESTS = [{
  367.         'note': 'Subscription required',
  368.         'url': 'https://www.nfl.com/plus/episodes/kurt-s-qb-insider-conference-championships',
  369.         'info_dict': {
  370.             'id': '1576832',
  371.             'ext': 'mp4',
  372.             'title': 'Conference Championships',
  373.             'description': 'md5:944f7fab56f7a37430bf8473f5473857',
  374.             'uploader': 'NFL',
  375.             'upload_date': '20230127',
  376.             'timestamp': 1674782760,
  377.             'duration': 730,
  378.             'categories': ['Analysis'],
  379.             'tags': ['Cincinnati Bengals at Kansas City Chiefs (2022-POST-3)'],
  380.             'thumbnail': r're:^https?://.*\.jpg',
  381.         },
  382.         'params': {'skip_download': 'm3u8'},
  383.     }]
  385.     def _real_initialize(self):
  386.         self._get_account_info()
  388.     def _real_extract(self, url):
  389.         slug = self._match_id(url)
  390.         self._get_auth_token()
  391.         video_id = self._download_json(
  392.             f'https://api.nfl.com/content/v1/videos/episodes/{slug}', slug, headers={
  393.                 'Authorization': f'Bearer {self._TOKEN}',
  394.             })['mcpPlaybackId']
  396.         return self._extract_video(video_id)