logo

youtube-dl

[mirror] Download/Watch videos from video hostersgit clone https://hacktivis.me/git/mirror/youtube-dl.git

wdr.py (14311B)


  1. # coding: utf-8
  2. from __future__ import unicode_literals
  3. import re
  4. from .common import InfoExtractor
  5. from ..compat import (
  6. compat_str,
  7. compat_urlparse,
  8. )
  9. from ..utils import (
  10. determine_ext,
  11. dict_get,
  12. ExtractorError,
  13. js_to_json,
  14. strip_jsonp,
  15. try_get,
  16. unified_strdate,
  17. update_url_query,
  18. urlhandle_detect_ext,
  19. url_or_none,
  20. )
  21. class WDRIE(InfoExtractor):
  22. __API_URL_TPL = '//deviceids-medp.wdr.de/ondemand/%s/%s'
  23. _VALID_URL = (r'(?:https?:' + __API_URL_TPL) % (r'\d+', r'(?=\d+\.js)|wdr:)(?P<id>\d{6,})')
  24. _GEO_COUNTRIES = ['DE']
  25. _TESTS = [{
  26. 'url': 'http://deviceids-medp.wdr.de/ondemand/155/1557833.js',
  27. 'info_dict': {
  28. 'id': 'mdb-1557833',
  29. 'ext': 'mp4',
  30. 'title': 'Biathlon-Staffel verpasst Podest bei Olympia-Generalprobe',
  31. 'upload_date': '20180112',
  32. },
  33. },
  34. ]
  35. def _asset_url(self, wdr_id):
  36. id_len = max(len(wdr_id), 5)
  37. return ''.join(('https:', self.__API_URL_TPL % (wdr_id[:id_len - 4], wdr_id, ), '.js'))
  38. def _real_extract(self, url):
  39. video_id = self._match_id(url)
  40. if url.startswith('wdr:'):
  41. video_id = url[4:]
  42. url = self._asset_url(video_id)
  43. metadata = self._download_json(
  44. url, video_id, transform_source=strip_jsonp)
  45. is_live = metadata.get('mediaType') == 'live'
  46. tracker_data = metadata['trackerData']
  47. title = tracker_data['trackerClipTitle']
  48. media_resource = metadata['mediaResource']
  49. formats = []
  50. # check if the metadata contains a direct URL to a file
  51. for kind, media in media_resource.items():
  52. if not isinstance(media, dict):
  53. continue
  54. if kind not in ('dflt', 'alt'):
  55. continue
  56. for tag_name, medium_url in media.items():
  57. if tag_name not in ('videoURL', 'audioURL'):
  58. continue
  59. ext = determine_ext(medium_url)
  60. if ext == 'm3u8':
  61. formats.extend(self._extract_m3u8_formats(
  62. medium_url, video_id, 'mp4', 'm3u8_native',
  63. m3u8_id='hls'))
  64. elif ext == 'f4m':
  65. manifest_url = update_url_query(
  66. medium_url, {'hdcore': '3.2.0', 'plugin': 'aasp-3.2.0.77.18'})
  67. formats.extend(self._extract_f4m_formats(
  68. manifest_url, video_id, f4m_id='hds', fatal=False))
  69. elif ext == 'smil':
  70. formats.extend(self._extract_smil_formats(
  71. medium_url, 'stream', fatal=False))
  72. else:
  73. a_format = {
  74. 'url': medium_url
  75. }
  76. if ext == 'unknown_video':
  77. urlh = self._request_webpage(
  78. medium_url, video_id, note='Determining extension')
  79. ext = urlhandle_detect_ext(urlh)
  80. a_format['ext'] = ext
  81. formats.append(a_format)
  82. self._sort_formats(formats)
  83. subtitles = {}
  84. caption_url = media_resource.get('captionURL')
  85. if caption_url:
  86. subtitles['de'] = [{
  87. 'url': caption_url,
  88. 'ext': 'ttml',
  89. }]
  90. captions_hash = media_resource.get('captionsHash')
  91. if isinstance(captions_hash, dict):
  92. for ext, format_url in captions_hash.items():
  93. format_url = url_or_none(format_url)
  94. if not format_url:
  95. continue
  96. subtitles.setdefault('de', []).append({
  97. 'url': format_url,
  98. 'ext': determine_ext(format_url, None) or ext,
  99. })
  100. return {
  101. 'id': tracker_data.get('trackerClipId', video_id),
  102. 'title': self._live_title(title) if is_live else title,
  103. 'alt_title': tracker_data.get('trackerClipSubcategory'),
  104. 'formats': formats,
  105. 'subtitles': subtitles,
  106. 'upload_date': unified_strdate(tracker_data.get('trackerClipAirTime')),
  107. 'is_live': is_live,
  108. }
  109. class WDRPageIE(WDRIE):
  110. _MAUS_REGEX = r'https?://(?:www\.)wdrmaus.de/(?:[^/]+/)*?(?P<maus_id>[^/?#.]+)(?:/?|/index\.php5|\.php5)$'
  111. _PAGE_REGEX = r'/(?:mediathek/)?(?:[^/]+/)*(?P<display_id>[^/]+)\.html'
  112. _VALID_URL = r'https?://(?:www\d?\.)?(?:(?:kinder\.)?wdr\d?|sportschau)\.de' + _PAGE_REGEX + '|' + _MAUS_REGEX
  113. _TESTS = [
  114. {
  115. 'url': 'http://www1.wdr.de/mediathek/video/sendungen/doku-am-freitag/video-geheimnis-aachener-dom-100.html',
  116. # HDS download, MD5 is unstable
  117. 'info_dict': {
  118. 'id': 'mdb-1058683',
  119. 'ext': 'flv',
  120. 'display_id': 'doku-am-freitag/video-geheimnis-aachener-dom-100',
  121. 'title': 'Geheimnis Aachener Dom',
  122. 'alt_title': 'Doku am Freitag',
  123. 'upload_date': '20160304',
  124. 'description': 'md5:87be8ff14d8dfd7a7ee46f0299b52318',
  125. 'is_live': False,
  126. 'subtitles': {'de': [{
  127. 'url': 'http://ondemand-ww.wdr.de/medp/fsk0/105/1058683/1058683_12220974.xml',
  128. 'ext': 'ttml',
  129. }]},
  130. },
  131. 'skip': 'HTTP Error 404: Not Found',
  132. },
  133. {
  134. 'url': 'http://www1.wdr.de/mediathek/audio/wdr3/wdr3-gespraech-am-samstag/audio-schriftstellerin-juli-zeh-100.html',
  135. 'md5': 'f4c1f96d01cf285240f53ea4309663d8',
  136. 'info_dict': {
  137. 'id': 'mdb-1072000',
  138. 'ext': 'mp3',
  139. 'display_id': 'wdr3-gespraech-am-samstag/audio-schriftstellerin-juli-zeh-100',
  140. 'title': 'Schriftstellerin Juli Zeh',
  141. 'alt_title': 'WDR 3 Gespräch am Samstag',
  142. 'upload_date': '20160312',
  143. 'description': 'md5:e127d320bc2b1f149be697ce044a3dd7',
  144. 'is_live': False,
  145. 'subtitles': {}
  146. },
  147. 'skip': 'HTTP Error 404: Not Found',
  148. },
  149. {
  150. 'url': 'http://www1.wdr.de/mediathek/video/live/index.html',
  151. 'info_dict': {
  152. 'id': 'mdb-2296252',
  153. 'ext': 'mp4',
  154. 'title': r're:^WDR Fernsehen im Livestream (?:\(nur in Deutschland erreichbar\) )?[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
  155. 'alt_title': 'WDR Fernsehen Live',
  156. 'upload_date': '20201112',
  157. 'is_live': True,
  158. },
  159. 'params': {
  160. 'skip_download': True, # m3u8 download
  161. },
  162. },
  163. {
  164. 'url': 'http://www1.wdr.de/mediathek/video/sendungen/aktuelle-stunde/aktuelle-stunde-120.html',
  165. 'playlist_mincount': 6,
  166. 'info_dict': {
  167. 'id': 'aktuelle-stunde-120',
  168. },
  169. },
  170. {
  171. 'url': 'http://www.wdrmaus.de/aktuelle-sendung/index.php5',
  172. 'info_dict': {
  173. 'id': 'mdb-2627637',
  174. 'ext': 'mp4',
  175. 'upload_date': 're:^[0-9]{8}$',
  176. 'title': 're:^Die Sendung (?:mit der Maus )?vom [0-9.]{10}$',
  177. },
  178. 'skip': 'The id changes from week to week because of the new episode'
  179. },
  180. {
  181. 'url': 'http://www.wdrmaus.de/filme/sachgeschichten/achterbahn.php5',
  182. 'md5': '803138901f6368ee497b4d195bb164f2',
  183. 'info_dict': {
  184. 'id': 'mdb-186083',
  185. 'ext': 'mp4',
  186. 'upload_date': '20130919',
  187. 'title': 'Sachgeschichte - Achterbahn ',
  188. },
  189. 'skip': 'HTTP Error 404: Not Found',
  190. },
  191. {
  192. 'url': 'http://www1.wdr.de/radio/player/radioplayer116~_layout-popupVersion.html',
  193. # Live stream, MD5 unstable
  194. 'info_dict': {
  195. 'id': 'mdb-869971',
  196. 'ext': 'mp4',
  197. 'title': r're:^COSMO Livestream [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
  198. 'upload_date': '20160101',
  199. },
  200. 'params': {
  201. 'skip_download': True, # m3u8 download
  202. }
  203. },
  204. {
  205. 'url': 'http://www.sportschau.de/handballem2018/handball-nationalmannschaft-em-stolperstein-vorrunde-100.html',
  206. 'info_dict': {
  207. 'id': 'mdb-1556012',
  208. 'ext': 'mp4',
  209. 'title': 'DHB-Vizepräsident Bob Hanning - "Die Weltspitze ist extrem breit"',
  210. 'upload_date': '20180111',
  211. },
  212. 'params': {
  213. 'skip_download': True,
  214. },
  215. 'skip': 'HTTP Error 404: Not Found',
  216. },
  217. {
  218. 'url': 'http://www.sportschau.de/handballem2018/audio-vorschau---die-handball-em-startet-mit-grossem-favoritenfeld-100.html',
  219. 'only_matching': True,
  220. },
  221. {
  222. 'url': 'https://kinder.wdr.de/tv/die-sendung-mit-dem-elefanten/av/video-folge---astronaut-100.html',
  223. 'only_matching': True,
  224. },
  225. ]
  226. def _real_extract(self, url):
  227. mobj = re.match(self._VALID_URL, url)
  228. display_id = dict_get(mobj.groupdict(), ('display_id', 'maus_id'), 'wdrmaus')
  229. webpage = self._download_webpage(url, display_id)
  230. entries = []
  231. # Article with several videos
  232. # for wdr.de the data-extension is in a tag with the class "mediaLink"
  233. # for wdr.de radio players, in a tag with the class "wdrrPlayerPlayBtn"
  234. # for wdrmaus, in a tag with the class "videoButton" (previously a link
  235. # to the page in a multiline "videoLink"-tag)
  236. for mobj in re.finditer(
  237. r'''(?sx)class=
  238. (?:
  239. (["\'])(?:mediaLink|wdrrPlayerPlayBtn|videoButton)\b.*?\1[^>]+|
  240. (["\'])videoLink\b.*?\2[\s]*>\n[^\n]*
  241. )data-extension=(["\'])(?P<data>(?:(?!\3).)+)\3
  242. ''', webpage):
  243. media_link_obj = self._parse_json(
  244. mobj.group('data'), display_id, transform_source=js_to_json,
  245. fatal=False)
  246. if not media_link_obj:
  247. continue
  248. jsonp_url = try_get(
  249. media_link_obj, lambda x: x['mediaObj']['url'], compat_str)
  250. if jsonp_url:
  251. # metadata, or player JS with ['ref'] giving WDR id, or just media, perhaps
  252. clip_id = media_link_obj['mediaObj'].get('ref')
  253. if jsonp_url.endswith('.assetjsonp'):
  254. asset = self._download_json(
  255. jsonp_url, display_id, fatal=False, transform_source=strip_jsonp)
  256. clip_id = try_get(asset, lambda x: x['trackerData']['trackerClipId'], compat_str)
  257. if clip_id:
  258. jsonp_url = self._asset_url(clip_id[4:])
  259. entries.append(self.url_result(jsonp_url, ie=WDRIE.ie_key()))
  260. # Playlist (e.g. https://www1.wdr.de/mediathek/video/sendungen/aktuelle-stunde/aktuelle-stunde-120.html)
  261. if not entries:
  262. entries = [
  263. self.url_result(
  264. compat_urlparse.urljoin(url, mobj.group('href')),
  265. ie=WDRPageIE.ie_key())
  266. for mobj in re.finditer(
  267. r'<a[^>]+\bhref=(["\'])(?P<href>(?:(?!\1).)+)\1[^>]+\bdata-extension=',
  268. webpage) if re.match(self._PAGE_REGEX, mobj.group('href'))
  269. ]
  270. return self.playlist_result(entries, playlist_id=display_id)
  271. class WDRElefantIE(InfoExtractor):
  272. _VALID_URL = r'https?://(?:www\.)wdrmaus\.de/elefantenseite/#(?P<id>.+)'
  273. _TEST = {
  274. 'url': 'http://www.wdrmaus.de/elefantenseite/#elefantenkino_wippe',
  275. # adaptive stream: unstable file MD5
  276. 'info_dict': {
  277. 'title': 'Wippe',
  278. 'id': 'mdb-1198320',
  279. 'ext': 'mp4',
  280. 'age_limit': None,
  281. 'upload_date': '20071003'
  282. },
  283. }
  284. def _real_extract(self, url):
  285. display_id = self._match_id(url)
  286. # Table of Contents seems to always be at this address, so fetch it directly.
  287. # The website fetches configurationJS.php5, which links to tableOfContentsJS.php5.
  288. table_of_contents = self._download_json(
  289. 'https://www.wdrmaus.de/elefantenseite/data/tableOfContentsJS.php5',
  290. display_id)
  291. if display_id not in table_of_contents:
  292. raise ExtractorError(
  293. 'No entry in site\'s table of contents for this URL. '
  294. 'Is the fragment part of the URL (after the #) correct?',
  295. expected=True)
  296. xml_metadata_path = table_of_contents[display_id]['xmlPath']
  297. xml_metadata = self._download_xml(
  298. 'https://www.wdrmaus.de/elefantenseite/' + xml_metadata_path,
  299. display_id)
  300. zmdb_url_element = xml_metadata.find('./movie/zmdb_url')
  301. if zmdb_url_element is None:
  302. raise ExtractorError(
  303. '%s is not a video' % display_id, expected=True)
  304. return self.url_result(zmdb_url_element.text, ie=WDRIE.ie_key())
  305. class WDRMobileIE(InfoExtractor):
  306. _VALID_URL = r'''(?x)
  307. https?://mobile-ondemand\.wdr\.de/
  308. .*?/fsk(?P<age_limit>[0-9]+)
  309. /[0-9]+/[0-9]+/
  310. (?P<id>[0-9]+)_(?P<title>[0-9]+)'''
  311. IE_NAME = 'wdr:mobile'
  312. _WORKING = False # no such domain
  313. _TEST = {
  314. 'url': 'http://mobile-ondemand.wdr.de/CMS2010/mdb/ondemand/weltweit/fsk0/42/421735/421735_4283021.mp4',
  315. 'info_dict': {
  316. 'title': '4283021',
  317. 'id': '421735',
  318. 'ext': 'mp4',
  319. 'age_limit': 0,
  320. },
  321. 'skip': 'Problems with loading data.'
  322. }
  323. def _real_extract(self, url):
  324. mobj = re.match(self._VALID_URL, url)
  325. return {
  326. 'id': mobj.group('id'),
  327. 'title': mobj.group('title'),
  328. 'age_limit': int(mobj.group('age_limit')),
  329. 'url': url,
  330. 'http_headers': {
  331. 'User-Agent': 'mobile',
  332. },
  333. }