logo

oasis-root

Compiled tree of Oasis Linux based on own branch at <https://hacktivis.me/git/oasis/> git clone https://anongit.hacktivis.me/git/oasis-root.git

cnn.py (13353B)


  1. import json
  2. import re
  3. from .common import InfoExtractor
  4. from ..utils import (
  5. clean_html,
  6. extract_attributes,
  7. int_or_none,
  8. merge_dicts,
  9. parse_duration,
  10. parse_iso8601,
  11. parse_resolution,
  12. try_call,
  13. update_url,
  14. url_or_none,
  15. )
  16. from ..utils.traversal import find_elements, traverse_obj
  17. class CNNIE(InfoExtractor):
  18. _VALID_URL = r'https?://(?:(?:edition|www|money|cnnespanol)\.)?cnn\.com/(?!audio/)(?P<display_id>[^?#]+?)(?:[?#]|$|/index\.html)'
  19. _TESTS = [{
  20. 'url': 'https://www.cnn.com/2024/05/31/sport/video/jadon-sancho-borussia-dortmund-champions-league-exclusive-spt-intl',
  21. 'info_dict': {
  22. 'id': 'med0e97ad0d154f56e29aa96e57192a14226734b6b',
  23. 'display_id': '2024/05/31/sport/video/jadon-sancho-borussia-dortmund-champions-league-exclusive-spt-intl',
  24. 'ext': 'mp4',
  25. 'upload_date': '20240531',
  26. 'description': 'md5:844bcdb0629e1877a7a466c913f4c19c',
  27. 'thumbnail': 'https://media.cnn.com/api/v1/images/stellar/prod/gettyimages-2151936122.jpg?c=original',
  28. 'duration': 373.0,
  29. 'timestamp': 1717148586,
  30. 'title': 'Borussia Dortmund star Jadon Sancho seeks Wembley redemption after 2020 Euros hurt',
  31. 'modified_date': '20240531',
  32. 'modified_timestamp': 1717150140,
  33. },
  34. }, {
  35. 'url': 'https://edition.cnn.com/2024/06/11/politics/video/inmates-vote-jail-nevada-murray-dnt-ac360-digvid',
  36. 'info_dict': {
  37. 'id': 'me522945c4709b299e5cb8657900a7a21ad3b559f9',
  38. 'display_id': '2024/06/11/politics/video/inmates-vote-jail-nevada-murray-dnt-ac360-digvid',
  39. 'ext': 'mp4',
  40. 'description': 'md5:e0120fe5da9ad8259fd707c1cbb64a60',
  41. 'title': 'Here’s how some inmates in closely divided state are now able to vote from jail',
  42. 'timestamp': 1718158269,
  43. 'upload_date': '20240612',
  44. 'thumbnail': 'https://media.cnn.com/api/v1/images/stellar/prod/still-20701554-13565-571-still.jpg?c=original',
  45. 'duration': 202.0,
  46. 'modified_date': '20240612',
  47. 'modified_timestamp': 1718158509,
  48. },
  49. }, {
  50. 'url': 'https://edition.cnn.com/2024/06/11/style/king-charles-portrait-vandalized/index.html',
  51. 'info_dict': {
  52. 'id': 'mef5f52b9e1fe28b1ad192afcbc9206ae984894b68',
  53. 'display_id': '2024/06/11/style/king-charles-portrait-vandalized',
  54. 'ext': 'mp4',
  55. 'thumbnail': 'https://media.cnn.com/api/v1/images/stellar/prod/still-20701257-8846-816-still.jpg?c=original',
  56. 'description': 'md5:19f78338ccec533db0fa8a4511012dae',
  57. 'title': 'Video shows King Charles\' portrait being vandalized by activists',
  58. 'timestamp': 1718113852,
  59. 'upload_date': '20240611',
  60. 'duration': 51.0,
  61. 'modified_timestamp': 1718116193,
  62. 'modified_date': '20240611',
  63. },
  64. }, {
  65. 'url': 'https://edition.cnn.com/videos/media/2022/12/05/robin-meade-final-sign-off-broadcast-hln-mxp-contd-vpx.hln',
  66. 'info_dict': {
  67. 'id': 'mefba13799201b084ea3b1d0f7ca820ae94d4bb5b2',
  68. 'display_id': 'videos/media/2022/12/05/robin-meade-final-sign-off-broadcast-hln-mxp-contd-vpx.hln',
  69. 'ext': 'mp4',
  70. 'thumbnail': 'https://media.cnn.com/api/v1/images/stellar/prod/221205163510-robin-meade-sign-off.jpg?c=original',
  71. 'duration': 158.0,
  72. 'title': 'Robin Meade signs off after HLN\'s last broadcast',
  73. 'description': 'md5:cff3c62d18d2fbc6c5c75cb029b7353b',
  74. 'upload_date': '20221205',
  75. 'timestamp': 1670284296,
  76. 'modified_timestamp': 1670332404,
  77. 'modified_date': '20221206',
  78. },
  79. 'params': {'format': 'direct'},
  80. }, {
  81. 'url': 'https://cnnespanol.cnn.com/video/ataque-misil-israel-beirut-libano-octubre-trax',
  82. 'info_dict': {
  83. 'id': 'me484a43722642aa00627b812fe928f2e99c6e2997',
  84. 'ext': 'mp4',
  85. 'display_id': 'video/ataque-misil-israel-beirut-libano-octubre-trax',
  86. 'timestamp': 1729501452,
  87. 'thumbnail': 'https://media.cnn.com/api/v1/images/stellar/prod/ataqeubeirut-1.jpg?c=original',
  88. 'description': 'md5:256ee7137d161f776cda429654135e52',
  89. 'upload_date': '20241021',
  90. 'duration': 31.0,
  91. 'title': 'VIDEO | Israel lanza un nuevo ataque sobre Beirut',
  92. 'modified_date': '20241021',
  93. 'modified_timestamp': 1729501530,
  94. },
  95. }, {
  96. 'url': 'https://edition.cnn.com/2024/10/16/politics/kamala-harris-fox-news-interview/index.html',
  97. 'info_dict': {
  98. 'id': '2024/10/16/politics/kamala-harris-fox-news-interview',
  99. },
  100. 'playlist_count': 2,
  101. 'playlist': [{
  102. 'md5': '073ffab87b8bef97c9913e71cc18ef9e',
  103. 'info_dict': {
  104. 'id': 'me19d548fdd54df0924087039283128ef473ab397d',
  105. 'ext': 'mp4',
  106. 'title': '\'I\'m not finished\': Harris interview with Fox News gets heated',
  107. 'display_id': 'kamala-harris-fox-news-interview-ebof-digvid',
  108. 'description': 'md5:e7dd3d1a04df916062230b60ca419a0a',
  109. 'thumbnail': 'https://media.cnn.com/api/v1/images/stellar/prod/harris-20241016234916617.jpg?c=original',
  110. 'duration': 173.0,
  111. 'timestamp': 1729122182,
  112. 'upload_date': '20241016',
  113. 'modified_timestamp': 1729194706,
  114. 'modified_date': '20241017',
  115. },
  116. 'params': {'format': 'direct'},
  117. }, {
  118. 'md5': '11604ab4af83b650826753f1ccb8ecff',
  119. 'info_dict': {
  120. 'id': 'med04507d8ca3da827001f63d22af321ec29c7d97b',
  121. 'ext': 'mp4',
  122. 'title': '\'Wise\': Buttigieg on Harris\' handling of interview question about gender transition surgery',
  123. 'display_id': 'pete-buttigieg-harris-fox-newssrc-digvid',
  124. 'description': 'md5:602a8a7e853ed5e574acd3159428c98e',
  125. 'thumbnail': 'https://media.cnn.com/api/v1/images/stellar/prod/buttigieg-20241017040412074.jpg?c=original',
  126. 'duration': 145.0,
  127. 'timestamp': 1729137765,
  128. 'upload_date': '20241017',
  129. 'modified_timestamp': 1729138184,
  130. 'modified_date': '20241017',
  131. },
  132. 'params': {'format': 'direct'},
  133. }],
  134. }]
  135. def _real_extract(self, url):
  136. display_id = self._match_valid_url(url).group('display_id')
  137. webpage = self._download_webpage(url, display_id)
  138. app_id = traverse_obj(
  139. self._search_json(r'window\.env\s*=', webpage, 'window env', display_id, default={}),
  140. ('TOP_AUTH_SERVICE_APP_ID', {str}))
  141. entries = []
  142. for player_data in traverse_obj(webpage, (
  143. {find_elements(tag='div', attr='data-component-name', value='video-player', html=True)},
  144. ..., {extract_attributes}, all, lambda _, v: v['data-media-id'])):
  145. media_id = player_data['data-media-id']
  146. parent_uri = player_data.get('data-video-resource-parent-uri')
  147. formats, subtitles = [], {}
  148. video_data = {}
  149. if parent_uri:
  150. video_data = self._download_json(
  151. 'https://fave.api.cnn.io/v1/video', media_id, fatal=False,
  152. query={
  153. 'id': media_id,
  154. 'stellarUri': parent_uri,
  155. })
  156. for direct_url in traverse_obj(video_data, ('files', ..., 'fileUri', {url_or_none})):
  157. resolution, bitrate = None, None
  158. if mobj := re.search(r'-(?P<res>\d+x\d+)_(?P<tbr>\d+)k\.mp4', direct_url):
  159. resolution, bitrate = mobj.group('res', 'tbr')
  160. formats.append({
  161. 'url': direct_url,
  162. 'format_id': 'direct',
  163. 'quality': 1,
  164. 'tbr': int_or_none(bitrate),
  165. **parse_resolution(resolution),
  166. })
  167. for sub_data in traverse_obj(video_data, (
  168. 'closedCaptions', 'types', lambda _, v: url_or_none(v['track']['url']), 'track')):
  169. subtitles.setdefault(sub_data.get('lang') or 'en', []).append({
  170. 'url': sub_data['url'],
  171. 'name': sub_data.get('label'),
  172. })
  173. if app_id:
  174. media_data = self._download_json(
  175. f'https://medium.ngtv.io/v2/media/{media_id}/desktop', media_id, fatal=False,
  176. query={'appId': app_id})
  177. m3u8_url = traverse_obj(media_data, (
  178. 'media', 'desktop', 'unprotected', 'unencrypted', 'url', {url_or_none}))
  179. if m3u8_url:
  180. fmts, subs = self._extract_m3u8_formats_and_subtitles(
  181. m3u8_url, media_id, 'mp4', m3u8_id='hls', fatal=False)
  182. formats.extend(fmts)
  183. self._merge_subtitles(subs, target=subtitles)
  184. entries.append({
  185. **traverse_obj(player_data, {
  186. 'title': ('data-headline', {clean_html}),
  187. 'description': ('data-description', {clean_html}),
  188. 'duration': ('data-duration', {parse_duration}),
  189. 'timestamp': ('data-publish-date', {parse_iso8601}),
  190. 'thumbnail': (
  191. 'data-poster-image-override', {json.loads}, 'big', 'uri', {url_or_none},
  192. {update_url(query='c=original')}),
  193. 'display_id': 'data-video-slug',
  194. }),
  195. **traverse_obj(video_data, {
  196. 'timestamp': ('dateCreated', 'uts', {int_or_none(scale=1000)}),
  197. 'description': ('description', {clean_html}),
  198. 'title': ('headline', {str}),
  199. 'modified_timestamp': ('lastModified', 'uts', {int_or_none(scale=1000)}),
  200. 'duration': ('trt', {int_or_none}),
  201. }),
  202. 'id': media_id,
  203. 'formats': formats,
  204. 'subtitles': subtitles,
  205. })
  206. if len(entries) == 1:
  207. return {
  208. **entries[0],
  209. 'display_id': display_id,
  210. }
  211. return self.playlist_result(entries, display_id)
  212. class CNNIndonesiaIE(InfoExtractor):
  213. _VALID_URL = r'https?://www\.cnnindonesia\.com/[\w-]+/(?P<upload_date>\d{8})\d+-\d+-(?P<id>\d+)/(?P<display_id>[\w-]+)'
  214. _TESTS = [{
  215. 'url': 'https://www.cnnindonesia.com/ekonomi/20220909212635-89-845885/alasan-harga-bbm-di-indonesia-masih-disubsidi',
  216. 'info_dict': {
  217. 'id': '845885',
  218. 'ext': 'mp4',
  219. 'description': 'md5:e7954bfa6f1749bc9ef0c079a719c347',
  220. 'upload_date': '20220909',
  221. 'title': 'Alasan Harga BBM di Indonesia Masih Disubsidi',
  222. 'timestamp': 1662859088,
  223. 'duration': 120.0,
  224. 'thumbnail': r're:https://akcdn\.detik\.net\.id/visual/2022/09/09/thumbnail-ekopedia-alasan-harga-bbm-disubsidi_169\.jpeg',
  225. 'tags': ['ekopedia', 'subsidi bbm', 'subsidi', 'bbm', 'bbm subsidi', 'harga pertalite naik'],
  226. 'age_limit': 0,
  227. 'release_timestamp': 1662859088,
  228. 'release_date': '20220911',
  229. 'uploader': 'Asfahan Yahsyi',
  230. },
  231. }, {
  232. 'url': 'https://www.cnnindonesia.com/internasional/20220911104341-139-846189/video-momen-charles-disambut-meriah-usai-dilantik-jadi-raja-inggris',
  233. 'info_dict': {
  234. 'id': '846189',
  235. 'ext': 'mp4',
  236. 'upload_date': '20220911',
  237. 'duration': 76.0,
  238. 'timestamp': 1662869995,
  239. 'description': 'md5:ece7b003b3ee7d81c6a5cfede7d5397d',
  240. 'thumbnail': r're:https://akcdn\.detik\.net\.id/visual/2022/09/11/thumbnail-video-1_169\.jpeg',
  241. 'title': 'VIDEO: Momen Charles Disambut Meriah usai Dilantik jadi Raja Inggris',
  242. 'tags': ['raja charles', 'raja charles iii', 'ratu elizabeth', 'ratu elizabeth meninggal dunia', 'raja inggris', 'inggris'],
  243. 'age_limit': 0,
  244. 'release_date': '20220911',
  245. 'uploader': 'REUTERS',
  246. 'release_timestamp': 1662869995,
  247. },
  248. }]
  249. def _real_extract(self, url):
  250. upload_date, video_id, display_id = self._match_valid_url(url).group('upload_date', 'id', 'display_id')
  251. webpage = self._download_webpage(url, display_id)
  252. json_ld_list = list(self._yield_json_ld(webpage, display_id))
  253. json_ld_data = self._json_ld(json_ld_list, display_id)
  254. embed_url = next(
  255. json_ld.get('embedUrl') for json_ld in json_ld_list if json_ld.get('@type') == 'VideoObject')
  256. return merge_dicts(json_ld_data, {
  257. '_type': 'url_transparent',
  258. 'url': embed_url,
  259. 'upload_date': upload_date,
  260. 'tags': try_call(lambda: self._html_search_meta('keywords', webpage).split(', ')),
  261. })