logo

oasis-root

Compiled tree of Oasis Linux based on own branch at <https://hacktivis.me/git/oasis/> git clone https://anongit.hacktivis.me/git/oasis-root.git

mlb.py (20261B)


  1. import json
  2. import re
  3. import time
  4. import uuid
  5. from .common import InfoExtractor
  6. from ..networking.exceptions import HTTPError
  7. from ..utils import (
  8. ExtractorError,
  9. determine_ext,
  10. int_or_none,
  11. join_nonempty,
  12. jwt_decode_hs256,
  13. parse_duration,
  14. parse_iso8601,
  15. try_get,
  16. url_or_none,
  17. urlencode_postdata,
  18. )
  19. from ..utils.traversal import traverse_obj
  20. class MLBBaseIE(InfoExtractor):
  21. def _real_extract(self, url):
  22. display_id = self._match_id(url)
  23. video = self._download_video_data(display_id)
  24. video_id = video['id']
  25. title = video['title']
  26. feed = self._get_feed(video)
  27. formats = []
  28. for playback in (feed.get('playbacks') or []):
  29. playback_url = playback.get('url')
  30. if not playback_url:
  31. continue
  32. name = playback.get('name')
  33. ext = determine_ext(playback_url)
  34. if ext == 'm3u8':
  35. formats.extend(self._extract_m3u8_formats(
  36. playback_url, video_id, 'mp4',
  37. 'm3u8_native', m3u8_id=name, fatal=False))
  38. else:
  39. f = {
  40. 'format_id': name,
  41. 'url': playback_url,
  42. }
  43. mobj = re.search(r'_(\d+)K_(\d+)X(\d+)', name)
  44. if mobj:
  45. f.update({
  46. 'height': int(mobj.group(3)),
  47. 'tbr': int(mobj.group(1)),
  48. 'width': int(mobj.group(2)),
  49. })
  50. mobj = re.search(r'_(\d+)x(\d+)_(\d+)_(\d+)K\.mp4', playback_url)
  51. if mobj:
  52. f.update({
  53. 'fps': int(mobj.group(3)),
  54. 'height': int(mobj.group(2)),
  55. 'tbr': int(mobj.group(4)),
  56. 'width': int(mobj.group(1)),
  57. })
  58. formats.append(f)
  59. thumbnails = []
  60. for cut in (try_get(feed, lambda x: x['image']['cuts'], list) or []):
  61. src = cut.get('src')
  62. if not src:
  63. continue
  64. thumbnails.append({
  65. 'height': int_or_none(cut.get('height')),
  66. 'url': src,
  67. 'width': int_or_none(cut.get('width')),
  68. })
  69. language = (video.get('language') or 'EN').lower()
  70. return {
  71. 'id': video_id,
  72. 'title': title,
  73. 'formats': formats,
  74. 'description': video.get('description'),
  75. 'duration': parse_duration(feed.get('duration')),
  76. 'thumbnails': thumbnails,
  77. 'timestamp': parse_iso8601(video.get(self._TIMESTAMP_KEY)),
  78. 'subtitles': self._extract_mlb_subtitles(feed, language),
  79. }
  80. class MLBIE(MLBBaseIE):
  81. _VALID_URL = r'''(?x)
  82. https?://
  83. (?:[\da-z_-]+\.)*mlb\.com/
  84. (?:
  85. (?:
  86. (?:[^/]+/)*video/[^/]+/c-|
  87. (?:
  88. shared/video/embed/(?:embed|m-internal-embed)\.html|
  89. (?:[^/]+/)+(?:play|index)\.jsp|
  90. )\?.*?\bcontent_id=
  91. )
  92. (?P<id>\d+)
  93. )
  94. '''
  95. _EMBED_REGEX = [
  96. r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
  97. r'data-video-link=["\'](?P<url>http://m\.mlb\.com/video/[^"\']+)',
  98. ]
  99. _TESTS = [
  100. {
  101. 'url': 'https://www.mlb.com/mariners/video/ackleys-spectacular-catch/c-34698933',
  102. 'md5': '632358dacfceec06bad823b83d21df2d',
  103. 'info_dict': {
  104. 'id': '34698933',
  105. 'ext': 'mp4',
  106. 'title': "Ackley's spectacular catch",
  107. 'description': 'md5:7f5a981eb4f3cbc8daf2aeffa2215bf0',
  108. 'duration': 66,
  109. 'timestamp': 1405995000,
  110. 'upload_date': '20140722',
  111. 'thumbnail': r're:^https?://.*\.jpg$',
  112. },
  113. },
  114. {
  115. 'url': 'https://www.mlb.com/video/stanton-prepares-for-derby/c-34496663',
  116. 'md5': 'bf2619bf9cacc0a564fc35e6aeb9219f',
  117. 'info_dict': {
  118. 'id': '34496663',
  119. 'ext': 'mp4',
  120. 'title': 'Stanton prepares for Derby',
  121. 'description': 'md5:d00ce1e5fd9c9069e9c13ab4faedfa57',
  122. 'duration': 46,
  123. 'timestamp': 1405120200,
  124. 'upload_date': '20140711',
  125. 'thumbnail': r're:^https?://.*\.jpg$',
  126. },
  127. },
  128. {
  129. 'url': 'https://www.mlb.com/video/cespedes-repeats-as-derby-champ/c-34578115',
  130. 'md5': '99bb9176531adc600b90880fb8be9328',
  131. 'info_dict': {
  132. 'id': '34578115',
  133. 'ext': 'mp4',
  134. 'title': 'Cespedes repeats as Derby champ',
  135. 'description': 'md5:08df253ce265d4cf6fb09f581fafad07',
  136. 'duration': 488,
  137. 'timestamp': 1405414336,
  138. 'upload_date': '20140715',
  139. 'thumbnail': r're:^https?://.*\.jpg$',
  140. },
  141. },
  142. {
  143. 'url': 'https://www.mlb.com/video/bautista-on-home-run-derby/c-34577915',
  144. 'md5': 'da8b57a12b060e7663ee1eebd6f330ec',
  145. 'info_dict': {
  146. 'id': '34577915',
  147. 'ext': 'mp4',
  148. 'title': 'Bautista on Home Run Derby',
  149. 'description': 'md5:b80b34031143d0986dddc64a8839f0fb',
  150. 'duration': 52,
  151. 'timestamp': 1405405122,
  152. 'upload_date': '20140715',
  153. 'thumbnail': r're:^https?://.*\.jpg$',
  154. },
  155. },
  156. {
  157. 'url': 'https://www.mlb.com/video/hargrove-homers-off-caldwell/c-1352023483?tid=67793694',
  158. 'only_matching': True,
  159. },
  160. {
  161. 'url': 'http://m.mlb.com/shared/video/embed/embed.html?content_id=35692085&topic_id=6479266&width=400&height=224&property=mlb',
  162. 'only_matching': True,
  163. },
  164. {
  165. 'url': 'http://mlb.mlb.com/shared/video/embed/embed.html?content_id=36599553',
  166. 'only_matching': True,
  167. },
  168. {
  169. 'url': 'http://mlb.mlb.com/es/video/play.jsp?content_id=36599553',
  170. 'only_matching': True,
  171. },
  172. {
  173. 'url': 'https://www.mlb.com/cardinals/video/piscottys-great-sliding-catch/c-51175783',
  174. 'only_matching': True,
  175. },
  176. {
  177. # From http://m.mlb.com/news/article/118550098/blue-jays-kevin-pillar-goes-spidey-up-the-wall-to-rob-tim-beckham-of-a-homer
  178. 'url': 'http://mlb.mlb.com/shared/video/embed/m-internal-embed.html?content_id=75609783&property=mlb&autoplay=true&hashmode=false&siteSection=mlb/multimedia/article_118550098/article_embed&club=mlb',
  179. 'only_matching': True,
  180. },
  181. ]
  182. _TIMESTAMP_KEY = 'date'
  183. @staticmethod
  184. def _get_feed(video):
  185. return video
  186. @staticmethod
  187. def _extract_mlb_subtitles(feed, language):
  188. subtitles = {}
  189. for keyword in (feed.get('keywordsAll') or []):
  190. keyword_type = keyword.get('type')
  191. if keyword_type and keyword_type.startswith('closed_captions_location_'):
  192. cc_location = keyword.get('value')
  193. if cc_location:
  194. subtitles.setdefault(language, []).append({
  195. 'url': cc_location,
  196. })
  197. return subtitles
  198. def _download_video_data(self, display_id):
  199. return self._download_json(
  200. f'http://content.mlb.com/mlb/item/id/v1/{display_id}/details/web-v1.json',
  201. display_id)
  202. class MLBVideoIE(MLBBaseIE):
  203. _VALID_URL = r'https?://(?:www\.)?mlb\.com/(?:[^/]+/)*video/(?P<id>[^/?&#]+)'
  204. _TEST = {
  205. 'url': 'https://www.mlb.com/mariners/video/ackley-s-spectacular-catch-c34698933',
  206. 'md5': '632358dacfceec06bad823b83d21df2d',
  207. 'info_dict': {
  208. 'id': 'c04a8863-f569-42e6-9f87-992393657614',
  209. 'ext': 'mp4',
  210. 'title': "Ackley's spectacular catch",
  211. 'description': 'md5:7f5a981eb4f3cbc8daf2aeffa2215bf0',
  212. 'duration': 66,
  213. 'timestamp': 1405995000,
  214. 'upload_date': '20140722',
  215. 'thumbnail': r're:^https?://.+',
  216. },
  217. }
  218. _TIMESTAMP_KEY = 'timestamp'
  219. @classmethod
  220. def suitable(cls, url):
  221. return False if MLBIE.suitable(url) else super().suitable(url)
  222. @staticmethod
  223. def _get_feed(video):
  224. return video['feeds'][0]
  225. @staticmethod
  226. def _extract_mlb_subtitles(feed, language):
  227. subtitles = {}
  228. for cc_location in (feed.get('closedCaptions') or []):
  229. subtitles.setdefault(language, []).append({
  230. 'url': cc_location,
  231. })
  232. def _download_video_data(self, display_id):
  233. # https://www.mlb.com/data-service/en/videos/[SLUG]
  234. return self._download_json(
  235. 'https://fastball-gateway.mlb.com/graphql',
  236. display_id, query={
  237. 'query': '''{
  238. mediaPlayback(ids: "%s") {
  239. description
  240. feeds(types: CMS) {
  241. closedCaptions
  242. duration
  243. image {
  244. cuts {
  245. width
  246. height
  247. src
  248. }
  249. }
  250. playbacks {
  251. name
  252. url
  253. }
  254. }
  255. id
  256. timestamp
  257. title
  258. }
  259. }''' % display_id, # noqa: UP031
  260. })['data']['mediaPlayback'][0]
  261. class MLBTVIE(InfoExtractor):
  262. _VALID_URL = r'https?://(?:www\.)?mlb\.com/tv/g(?P<id>\d{6})'
  263. _NETRC_MACHINE = 'mlb'
  264. _TESTS = [{
  265. 'url': 'https://www.mlb.com/tv/g661581/vee2eff5f-a7df-4c20-bdb4-7b926fa12638',
  266. 'info_dict': {
  267. 'id': '661581',
  268. 'ext': 'mp4',
  269. 'title': '2022-07-02 - St. Louis Cardinals @ Philadelphia Phillies',
  270. 'release_date': '20220702',
  271. 'release_timestamp': 1656792300,
  272. },
  273. 'params': {'skip_download': 'm3u8'},
  274. }, {
  275. # makeup game: has multiple dates, need to avoid games with 'rescheduleDate'
  276. 'url': 'https://www.mlb.com/tv/g747039/vd22541c4-5a29-45f7-822b-635ec041cf5e',
  277. 'info_dict': {
  278. 'id': '747039',
  279. 'ext': 'mp4',
  280. 'title': '2024-07-29 - Toronto Blue Jays @ Baltimore Orioles',
  281. 'release_date': '20240729',
  282. 'release_timestamp': 1722280200,
  283. },
  284. 'params': {'skip_download': 'm3u8'},
  285. }]
  286. _GRAPHQL_INIT_QUERY = '''\
  287. mutation initSession($device: InitSessionInput!, $clientType: ClientType!, $experience: ExperienceTypeInput) {
  288. initSession(device: $device, clientType: $clientType, experience: $experience) {
  289. deviceId
  290. sessionId
  291. entitlements {
  292. code
  293. }
  294. location {
  295. countryCode
  296. regionName
  297. zipCode
  298. latitude
  299. longitude
  300. }
  301. clientExperience
  302. features
  303. }
  304. }'''
  305. _GRAPHQL_PLAYBACK_QUERY = '''\
  306. mutation initPlaybackSession(
  307. $adCapabilities: [AdExperienceType]
  308. $mediaId: String!
  309. $deviceId: String!
  310. $sessionId: String!
  311. $quality: PlaybackQuality
  312. ) {
  313. initPlaybackSession(
  314. adCapabilities: $adCapabilities
  315. mediaId: $mediaId
  316. deviceId: $deviceId
  317. sessionId: $sessionId
  318. quality: $quality
  319. ) {
  320. playbackSessionId
  321. playback {
  322. url
  323. token
  324. expiration
  325. cdn
  326. }
  327. }
  328. }'''
  329. _APP_VERSION = '7.8.2'
  330. _device_id = None
  331. _session_id = None
  332. _access_token = None
  333. _token_expiry = 0
  334. @property
  335. def _api_headers(self):
  336. if (self._token_expiry - 120) <= time.time():
  337. self.write_debug('Access token has expired; re-logging in')
  338. self._perform_login(*self._get_login_info())
  339. return {'Authorization': f'Bearer {self._access_token}'}
  340. def _real_initialize(self):
  341. if not self._access_token:
  342. self.raise_login_required(
  343. 'All videos are only available to registered users', method='password')
  344. def _set_device_id(self, username):
  345. if not self._device_id:
  346. self._device_id = self.cache.load(
  347. self._NETRC_MACHINE, 'device_ids', default={}).get(username)
  348. if self._device_id:
  349. return
  350. self._device_id = str(uuid.uuid4())
  351. self.cache.store(self._NETRC_MACHINE, 'device_ids', {username: self._device_id})
  352. def _perform_login(self, username, password):
  353. try:
  354. self._access_token = self._download_json(
  355. 'https://ids.mlb.com/oauth2/aus1m088yK07noBfh356/v1/token', None,
  356. 'Logging in', 'Unable to log in', headers={
  357. 'User-Agent': 'okhttp/3.12.1',
  358. 'Content-Type': 'application/x-www-form-urlencoded',
  359. }, data=urlencode_postdata({
  360. 'grant_type': 'password',
  361. 'username': username,
  362. 'password': password,
  363. 'scope': 'openid offline_access',
  364. 'client_id': '0oa3e1nutA1HLzAKG356',
  365. }))['access_token']
  366. except ExtractorError as error:
  367. if isinstance(error.cause, HTTPError) and error.cause.status == 400:
  368. raise ExtractorError('Invalid username or password', expected=True)
  369. raise
  370. self._token_expiry = traverse_obj(self._access_token, ({jwt_decode_hs256}, 'exp', {int})) or 0
  371. self._set_device_id(username)
  372. self._session_id = self._call_api({
  373. 'operationName': 'initSession',
  374. 'query': self._GRAPHQL_INIT_QUERY,
  375. 'variables': {
  376. 'device': {
  377. 'appVersion': self._APP_VERSION,
  378. 'deviceFamily': 'desktop',
  379. 'knownDeviceId': self._device_id,
  380. 'languagePreference': 'ENGLISH',
  381. 'manufacturer': '',
  382. 'model': '',
  383. 'os': '',
  384. 'osVersion': '',
  385. },
  386. 'clientType': 'WEB',
  387. },
  388. }, None, 'session ID')['data']['initSession']['sessionId']
  389. def _call_api(self, data, video_id, description='GraphQL JSON', fatal=True):
  390. return self._download_json(
  391. 'https://media-gateway.mlb.com/graphql', video_id,
  392. f'Downloading {description}', f'Unable to download {description}', fatal=fatal,
  393. headers={
  394. **self._api_headers,
  395. 'Accept': 'application/json',
  396. 'Content-Type': 'application/json',
  397. 'x-client-name': 'WEB',
  398. 'x-client-version': self._APP_VERSION,
  399. }, data=json.dumps(data, separators=(',', ':')).encode())
  400. def _extract_formats_and_subtitles(self, broadcast, video_id):
  401. feed = traverse_obj(broadcast, ('homeAway', {str.title}))
  402. medium = traverse_obj(broadcast, ('type', {str}))
  403. language = traverse_obj(broadcast, ('language', {str.lower}))
  404. format_id = join_nonempty(feed, medium, language)
  405. response = self._call_api({
  406. 'operationName': 'initPlaybackSession',
  407. 'query': self._GRAPHQL_PLAYBACK_QUERY,
  408. 'variables': {
  409. 'adCapabilities': ['GOOGLE_STANDALONE_AD_PODS'],
  410. 'deviceId': self._device_id,
  411. 'mediaId': broadcast['mediaId'],
  412. 'quality': 'PLACEHOLDER',
  413. 'sessionId': self._session_id,
  414. },
  415. }, video_id, f'{format_id} broadcast JSON', fatal=False)
  416. playback = traverse_obj(response, ('data', 'initPlaybackSession', 'playback', {dict}))
  417. m3u8_url = traverse_obj(playback, ('url', {url_or_none}))
  418. token = traverse_obj(playback, ('token', {str}))
  419. if not (m3u8_url and token):
  420. errors = '; '.join(traverse_obj(response, ('errors', ..., 'message', {str})))
  421. if 'not entitled' in errors:
  422. raise ExtractorError(errors, expected=True)
  423. elif errors: # Only warn when 'blacked out' since radio formats are available
  424. self.report_warning(f'API returned errors for {format_id}: {errors}')
  425. else:
  426. self.report_warning(f'No formats available for {format_id} broadcast; skipping')
  427. return [], {}
  428. cdn_headers = {'x-cdn-token': token}
  429. fmts, subs = self._extract_m3u8_formats_and_subtitles(
  430. m3u8_url.replace(f'/{token}/', '/'), video_id, 'mp4',
  431. m3u8_id=format_id, fatal=False, headers=cdn_headers)
  432. for fmt in fmts:
  433. fmt['http_headers'] = cdn_headers
  434. fmt.setdefault('format_note', join_nonempty(feed, medium, delim=' '))
  435. fmt.setdefault('language', language)
  436. if fmt.get('vcodec') == 'none' and fmt['language'] == 'en':
  437. fmt['source_preference'] = 10
  438. return fmts, subs
  439. def _real_extract(self, url):
  440. video_id = self._match_id(url)
  441. data = self._download_json(
  442. 'https://statsapi.mlb.com/api/v1/schedule', video_id, query={
  443. 'gamePk': video_id,
  444. 'hydrate': 'broadcasts(all),statusFlags',
  445. })
  446. metadata = traverse_obj(data, (
  447. 'dates', ..., 'games',
  448. lambda _, v: str(v['gamePk']) == video_id and not v.get('rescheduleDate'), any))
  449. broadcasts = traverse_obj(metadata, (
  450. 'broadcasts', lambda _, v: v['mediaId'] and v['mediaState']['mediaStateCode'] != 'MEDIA_OFF'))
  451. formats, subtitles = [], {}
  452. for broadcast in broadcasts:
  453. fmts, subs = self._extract_formats_and_subtitles(broadcast, video_id)
  454. formats.extend(fmts)
  455. self._merge_subtitles(subs, target=subtitles)
  456. return {
  457. 'id': video_id,
  458. 'title': join_nonempty(
  459. traverse_obj(metadata, ('officialDate', {str})),
  460. traverse_obj(metadata, ('teams', ('away', 'home'), 'team', 'name', {str}, all, {' @ '.join})),
  461. delim=' - '),
  462. 'is_live': traverse_obj(broadcasts, (..., 'mediaState', 'mediaStateCode', {str}, any)) == 'MEDIA_ON',
  463. 'release_timestamp': traverse_obj(metadata, ('gameDate', {parse_iso8601})),
  464. 'formats': formats,
  465. 'subtitles': subtitles,
  466. }
  467. class MLBArticleIE(InfoExtractor):
  468. _VALID_URL = r'https?://www\.mlb\.com/news/(?P<id>[\w-]+)'
  469. _TESTS = [{
  470. 'url': 'https://www.mlb.com/news/manny-machado-robs-guillermo-heredia-reacts',
  471. 'info_dict': {
  472. 'id': '36db7394-343c-4ea3-b8ca-ead2e61bca9a',
  473. 'title': 'Machado\'s grab draws hilarious irate reaction',
  474. 'modified_timestamp': 1675888370,
  475. 'description': 'md5:a19d4eb0487b2cb304e9a176f6b67676',
  476. 'modified_date': '20230208',
  477. },
  478. 'playlist_mincount': 2,
  479. }]
  480. def _real_extract(self, url):
  481. display_id = self._match_id(url)
  482. webpage = self._download_webpage(url, display_id)
  483. apollo_cache_json = self._search_json(r'window\.initState\s*=', webpage, 'window.initState', display_id)['apolloCache']
  484. content_real_info = traverse_obj(
  485. apollo_cache_json, ('ROOT_QUERY', lambda k, _: k.startswith('getArticle')), get_all=False)
  486. return self.playlist_from_matches(
  487. traverse_obj(content_real_info, ('parts', lambda _, v: v['__typename'] == 'Video' or v['type'] == 'video')),
  488. getter=lambda x: f'https://www.mlb.com/video/{x["slug"]}',
  489. ie=MLBVideoIE, playlist_id=content_real_info.get('translationId'),
  490. title=self._html_search_meta('og:title', webpage),
  491. description=content_real_info.get('summary'),
  492. modified_timestamp=parse_iso8601(content_real_info.get('lastUpdatedDate')))