logo

youtube-dl

[mirror] Download/Watch videos from video hostersgit clone https://hacktivis.me/git/mirror/youtube-dl.git

ard.py (18910B)

    

  1. # coding: utf-8
  2. from __future__ import unicode_literals
  4. import json
  5. import re
  7. from .common import InfoExtractor
  8. from .generic import GenericIE
  9. from ..utils import (
  10.     determine_ext,
  11.     ExtractorError,
  12.     int_or_none,
  13.     parse_duration,
  14.     qualities,
  15.     str_or_none,
  16.     try_get,
  17.     unified_strdate,
  18.     unified_timestamp,
  19.     update_url_query,
  20.     url_or_none,
  21.     xpath_text,
  22. )
  23. from ..compat import compat_etree_fromstring
  26. class ARDMediathekBaseIE(InfoExtractor):
  27.     _GEO_COUNTRIES = ['DE']
  29.     def _extract_media_info(self, media_info_url, webpage, video_id):
  30.         media_info = self._download_json(
  31.             media_info_url, video_id, 'Downloading media JSON')
  32.         return self._parse_media_info(media_info, video_id, '"fsk"' in webpage)
  34.     def _parse_media_info(self, media_info, video_id, fsk):
  35.         formats = self._extract_formats(media_info, video_id)
  37.         if not formats:
  38.             if fsk:
  39.                 raise ExtractorError(
  40.                     'This video is only available after 20:00', expected=True)
  41.             elif media_info.get('_geoblocked'):
  42.                 self.raise_geo_restricted(
  43.                     'This video is not available due to geoblocking',
  44.                     countries=self._GEO_COUNTRIES)
  46.         self._sort_formats(formats)
  48.         subtitles = {}
  49.         subtitle_url = media_info.get('_subtitleUrl')
  50.         if subtitle_url:
  51.             subtitles['de'] = [{
  52.                 'ext': 'ttml',
  53.                 'url': subtitle_url,
  54.             }]
  56.         return {
  57.             'id': video_id,
  58.             'duration': int_or_none(media_info.get('_duration')),
  59.             'thumbnail': media_info.get('_previewImage'),
  60.             'is_live': media_info.get('_isLive') is True,
  61.             'formats': formats,
  62.             'subtitles': subtitles,
  63.         }
  65.     def _extract_formats(self, media_info, video_id):
  66.         type_ = media_info.get('_type')
  67.         media_array = media_info.get('_mediaArray', [])
  68.         formats = []
  69.         for num, media in enumerate(media_array):
  70.             for stream in media.get('_mediaStreamArray', []):
  71.                 stream_urls = stream.get('_stream')
  72.                 if not stream_urls:
  73.                     continue
  74.                 if not isinstance(stream_urls, list):
  75.                     stream_urls = [stream_urls]
  76.                 quality = stream.get('_quality')
  77.                 server = stream.get('_server')
  78.                 for stream_url in stream_urls:
  79.                     if not url_or_none(stream_url):
  80.                         continue
  81.                     ext = determine_ext(stream_url)
  82.                     if quality != 'auto' and ext in ('f4m', 'm3u8'):
  83.                         continue
  84.                     if ext == 'f4m':
  85.                         formats.extend(self._extract_f4m_formats(
  86.                             update_url_query(stream_url, {
  87.                                 'hdcore': '3.1.1',
  88.                                 'plugin': 'aasp-3.1.1.69.124'
  89.                             }), video_id, f4m_id='hds', fatal=False))
  90.                     elif ext == 'm3u8':
  91.                         formats.extend(self._extract_m3u8_formats(
  92.                             stream_url, video_id, 'mp4', 'm3u8_native',
  93.                             m3u8_id='hls', fatal=False))
  94.                     else:
  95.                         if server and server.startswith('rtmp'):
  96.                             f = {
  97.                                 'url': server,
  98.                                 'play_path': stream_url,
  99.                                 'format_id': 'a%s-rtmp-%s' % (num, quality),
  100.                             }
  101.                         else:
  102.                             f = {
  103.                                 'url': stream_url,
  104.                                 'format_id': 'a%s-%s-%s' % (num, ext, quality)
  105.                             }
  106.                         m = re.search(
  107.                             r'_(?P<width>\d+)x(?P<height>\d+)\.mp4$',
  108.                             stream_url)
  109.                         if m:
  110.                             f.update({
  111.                                 'width': int(m.group('width')),
  112.                                 'height': int(m.group('height')),
  113.                             })
  114.                         if type_ == 'audio':
  115.                             f['vcodec'] = 'none'
  116.                         formats.append(f)
  117.         return formats
  120. class ARDMediathekIE(ARDMediathekBaseIE):
  121.     IE_NAME = 'ARD:mediathek'
  122.     _VALID_URL = r'^https?://(?:(?:(?:www|classic)\.)?ardmediathek\.de|mediathek\.(?:daserste|rbb-online)\.de|one\.ard\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
  124.     _TESTS = [{
  125.         # available till 26.07.2022
  126.         'url': 'http://www.ardmediathek.de/tv/S%C3%9CDLICHT/Was-ist-die-Kunst-der-Zukunft-liebe-Ann/BR-Fernsehen/Video?bcastId=34633636&documentId=44726822',
  127.         'info_dict': {
  128.             'id': '44726822',
  129.             'ext': 'mp4',
  130.             'title': 'Was ist die Kunst der Zukunft, liebe Anna McCarthy?',
  131.             'description': 'md5:4ada28b3e3b5df01647310e41f3a62f5',
  132.             'duration': 1740,
  133.         },
  134.         'params': {
  135.             # m3u8 download
  136.             'skip_download': True,
  137.         }
  138.     }, {
  139.         'url': 'https://one.ard.de/tv/Mord-mit-Aussicht/Mord-mit-Aussicht-6-39-T%C3%B6dliche-Nach/ONE/Video?bcastId=46384294&documentId=55586872',
  140.         'only_matching': True,
  141.     }, {
  142.         # audio
  143.         'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',
  144.         'only_matching': True,
  145.     }, {
  146.         'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
  147.         'only_matching': True,
  148.     }, {
  149.         # audio
  150.         'url': 'http://mediathek.rbb-online.de/radio/Hörspiel/Vor-dem-Fest/kulturradio/Audio?documentId=30796318&topRessort=radio&bcastId=9839158',
  151.         'only_matching': True,
  152.     }, {
  153.         'url': 'https://classic.ardmediathek.de/tv/Panda-Gorilla-Co/Panda-Gorilla-Co-Folge-274/Das-Erste/Video?bcastId=16355486&documentId=58234698',
  154.         'only_matching': True,
  155.     }]
  157.     @classmethod
  158.     def suitable(cls, url):
  159.         return False if ARDBetaMediathekIE.suitable(url) else super(ARDMediathekIE, cls).suitable(url)
  161.     def _real_extract(self, url):
  162.         # determine video id from url
  163.         m = re.match(self._VALID_URL, url)
  165.         document_id = None
  167.         numid = re.search(r'documentId=([0-9]+)', url)
  168.         if numid:
  169.             document_id = video_id = numid.group(1)
  170.         else:
  171.             video_id = m.group('video_id')
  173.         webpage = self._download_webpage(url, video_id)
  175.         ERRORS = (
  176.             ('>Leider liegt eine Störung vor.', 'Video %s is unavailable'),
  177.             ('>Der gewünschte Beitrag ist nicht mehr verfügbar.<',
  178.              'Video %s is no longer available'),
  179.         )
  181.         for pattern, message in ERRORS:
  182.             if pattern in webpage:
  183.                 raise ExtractorError(message % video_id, expected=True)
  185.         if re.search(r'[\?&]rss($|[=&])', url):
  186.             doc = compat_etree_fromstring(webpage.encode('utf-8'))
  187.             if doc.tag == 'rss':
  188.                 return GenericIE()._extract_rss(url, video_id, doc)
  190.         title = self._og_search_title(webpage, default=None) or self._html_search_regex(
  191.             [r'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>',
  192.              r'<meta name="dcterms\.title" content="(.*?)"/>',
  193.              r'<h4 class="headline">(.*?)</h4>',
  194.              r'<title[^>]*>(.*?)</title>'],
  195.             webpage, 'title')
  196.         description = self._og_search_description(webpage, default=None) or self._html_search_meta(
  197.             'dcterms.abstract', webpage, 'description', default=None)
  198.         if description is None:
  199.             description = self._html_search_meta(
  200.                 'description', webpage, 'meta description', default=None)
  201.         if description is None:
  202.             description = self._html_search_regex(
  203.                 r'<p\s+class="teasertext">(.+?)</p>',
  204.                 webpage, 'teaser text', default=None)
  206.         # Thumbnail is sometimes not present.
  207.         # It is in the mobile version, but that seems to use a different URL
  208.         # structure altogether.
  209.         thumbnail = self._og_search_thumbnail(webpage, default=None)
  211.         media_streams = re.findall(r'''(?x)
  212.             mediaCollection\.addMediaStream\([0-9]+,\s*[0-9]+,\s*"[^"]*",\s*
  213.             "([^"]+)"''', webpage)
  215.         if media_streams:
  216.             QUALITIES = qualities(['lo', 'hi', 'hq'])
  217.             formats = []
  218.             for furl in set(media_streams):
  219.                 if furl.endswith('.f4m'):
  220.                     fid = 'f4m'
  221.                 else:
  222.                     fid_m = re.match(r'.*\.([^.]+)\.[^.]+$', furl)
  223.                     fid = fid_m.group(1) if fid_m else None
  224.                 formats.append({
  225.                     'quality': QUALITIES(fid),
  226.                     'format_id': fid,
  227.                     'url': furl,
  228.                 })
  229.             self._sort_formats(formats)
  230.             info = {
  231.                 'formats': formats,
  232.             }
  233.         else:  # request JSON file
  234.             if not document_id:
  235.                 video_id = self._search_regex(
  236.                     r'/play/(?:config|media)/(\d+)', webpage, 'media id')
  237.             info = self._extract_media_info(
  238.                 'http://www.ardmediathek.de/play/media/%s' % video_id,
  239.                 webpage, video_id)
  241.         info.update({
  242.             'id': video_id,
  243.             'title': self._live_title(title) if info.get('is_live') else title,
  244.             'description': description,
  245.             'thumbnail': thumbnail,
  246.         })
  248.         return info
  251. class ARDIE(InfoExtractor):
  252.     _VALID_URL = r'(?P<mainurl>https?://(?:www\.)?daserste\.de/(?:[^/?#&]+/)+(?P<id>[^/?#&]+))\.html'
  253.     _TESTS = [{
  254.         # available till 7.01.2022
  255.         'url': 'https://www.daserste.de/information/talk/maischberger/videos/maischberger-die-woche-video100.html',
  256.         'md5': '867d8aa39eeaf6d76407c5ad1bb0d4c1',
  257.         'info_dict': {
  258.             'id': 'maischberger-die-woche-video100',
  259.             'display_id': 'maischberger-die-woche-video100',
  260.             'ext': 'mp4',
  261.             'duration': 3687.0,
  262.             'title': 'maischberger. die woche vom 7. Januar 2021',
  263.             'upload_date': '20210107',
  264.             'thumbnail': r're:^https?://.*\.jpg$',
  265.         },
  266.     }, {
  267.         'url': 'https://www.daserste.de/information/politik-weltgeschehen/morgenmagazin/videosextern/dominik-kahun-aus-der-nhl-direkt-zur-weltmeisterschaft-100.html',
  268.         'only_matching': True,
  269.     }, {
  270.         'url': 'https://www.daserste.de/information/nachrichten-wetter/tagesthemen/videosextern/tagesthemen-17736.html',
  271.         'only_matching': True,
  272.     }, {
  273.         'url': 'http://www.daserste.de/information/reportage-dokumentation/dokus/videos/die-story-im-ersten-mission-unter-falscher-flagge-100.html',
  274.         'only_matching': True,
  275.     }, {
  276.         'url': 'https://www.daserste.de/unterhaltung/serie/in-aller-freundschaft-die-jungen-aerzte/Drehpause-100.html',
  277.         'only_matching': True,
  278.     }, {
  279.         'url': 'https://www.daserste.de/unterhaltung/film/filmmittwoch-im-ersten/videos/making-ofwendezeit-video-100.html',
  280.         'only_matching': True,
  281.     }]
  283.     def _real_extract(self, url):
  284.         mobj = re.match(self._VALID_URL, url)
  285.         display_id = mobj.group('id')
  287.         player_url = mobj.group('mainurl') + '~playerXml.xml'
  288.         doc = self._download_xml(player_url, display_id)
  289.         video_node = doc.find('./video')
  290.         upload_date = unified_strdate(xpath_text(
  291.             video_node, './broadcastDate'))
  292.         thumbnail = xpath_text(video_node, './/teaserImage//variant/url')
  294.         formats = []
  295.         for a in video_node.findall('.//asset'):
  296.             file_name = xpath_text(a, './fileName', default=None)
  297.             if not file_name:
  298.                 continue
  299.             format_type = a.attrib.get('type')
  300.             format_url = url_or_none(file_name)
  301.             if format_url:
  302.                 ext = determine_ext(file_name)
  303.                 if ext == 'm3u8':
  304.                     formats.extend(self._extract_m3u8_formats(
  305.                         format_url, display_id, 'mp4', entry_protocol='m3u8_native',
  306.                         m3u8_id=format_type or 'hls', fatal=False))
  307.                     continue
  308.                 elif ext == 'f4m':
  309.                     formats.extend(self._extract_f4m_formats(
  310.                         update_url_query(format_url, {'hdcore': '3.7.0'}),
  311.                         display_id, f4m_id=format_type or 'hds', fatal=False))
  312.                     continue
  313.             f = {
  314.                 'format_id': format_type,
  315.                 'width': int_or_none(xpath_text(a, './frameWidth')),
  316.                 'height': int_or_none(xpath_text(a, './frameHeight')),
  317.                 'vbr': int_or_none(xpath_text(a, './bitrateVideo')),
  318.                 'abr': int_or_none(xpath_text(a, './bitrateAudio')),
  319.                 'vcodec': xpath_text(a, './codecVideo'),
  320.                 'tbr': int_or_none(xpath_text(a, './totalBitrate')),
  321.             }
  322.             server_prefix = xpath_text(a, './serverPrefix', default=None)
  323.             if server_prefix:
  324.                 f.update({
  325.                     'url': server_prefix,
  326.                     'playpath': file_name,
  327.                 })
  328.             else:
  329.                 if not format_url:
  330.                     continue
  331.                 f['url'] = format_url
  332.             formats.append(f)
  333.         self._sort_formats(formats)
  335.         _SUB_FORMATS = (
  336.             ('./dataTimedText', 'ttml'),
  337.             ('./dataTimedTextNoOffset', 'ttml'),
  338.             ('./dataTimedTextVtt', 'vtt'),
  339.         )
  341.         subtitles = {}
  342.         for subsel, subext in _SUB_FORMATS:
  343.             for node in video_node.findall(subsel):
  344.                 subtitles.setdefault('de', []).append({
  345.                     'url': node.attrib['url'],
  346.                     'ext': subext,
  347.                 })
  349.         return {
  350.             'id': xpath_text(video_node, './videoId', default=display_id),
  351.             'formats': formats,
  352.             'subtitles': subtitles,
  353.             'display_id': display_id,
  354.             'title': video_node.find('./title').text,
  355.             'duration': parse_duration(video_node.find('./duration').text),
  356.             'upload_date': upload_date,
  357.             'thumbnail': thumbnail,
  358.         }
  361. class ARDBetaMediathekIE(ARDMediathekBaseIE):
  362.     _VALID_URL = r'https://(?:(?:beta|www)\.)?ardmediathek\.de/(?:[^/]+/)?(?:player|live|video)/(?:[^/]+/)*(?P<id>Y3JpZDovL[a-zA-Z0-9]+)'
  363.     _TESTS = [{
  364.         'url': 'https://www.ardmediathek.de/mdr/video/die-robuste-roswita/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy84MWMxN2MzZC0wMjkxLTRmMzUtODk4ZS0wYzhlOWQxODE2NGI/',
  365.         'md5': 'a1dc75a39c61601b980648f7c9f9f71d',
  366.         'info_dict': {
  367.             'display_id': 'die-robuste-roswita',
  368.             'id': '78566716',
  369.             'title': 'Die robuste Roswita',
  370.             'description': r're:^Der Mord.*totgeglaubte Ehefrau Roswita',
  371.             'duration': 5316,
  372.             'thumbnail': 'https://img.ardmediathek.de/standard/00/78/56/67/84/575672121/16x9/960?mandant=ard',
  373.             'timestamp': 1596658200,
  374.             'upload_date': '20200805',
  375.             'ext': 'mp4',
  376.         },
  377.     }, {
  378.         'url': 'https://beta.ardmediathek.de/ard/video/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE',
  379.         'only_matching': True,
  380.     }, {
  381.         'url': 'https://ardmediathek.de/ard/video/saartalk/saartalk-gesellschaftsgift-haltung-gegen-hass/sr-fernsehen/Y3JpZDovL3NyLW9ubGluZS5kZS9TVF84MTY4MA/',
  382.         'only_matching': True,
  383.     }, {
  384.         'url': 'https://www.ardmediathek.de/ard/video/trailer/private-eyes-s01-e01/one/Y3JpZDovL3dkci5kZS9CZWl0cmFnLTE1MTgwYzczLWNiMTEtNGNkMS1iMjUyLTg5MGYzOWQxZmQ1YQ/',
  385.         'only_matching': True,
  386.     }, {
  387.         'url': 'https://www.ardmediathek.de/ard/player/Y3JpZDovL3N3ci5kZS9hZXgvbzEwNzE5MTU/',
  388.         'only_matching': True,
  389.     }, {
  390.         'url': 'https://www.ardmediathek.de/swr/live/Y3JpZDovL3N3ci5kZS8xMzQ4MTA0Mg',
  391.         'only_matching': True,
  392.     }, {
  393.         'url': 'https://www.ardmediathek.de/video/coronavirus-update-ndr-info/astrazeneca-kurz-lockdown-und-pims-syndrom-81/ndr/Y3JpZDovL25kci5kZS84NzE0M2FjNi0wMWEwLTQ5ODEtOTE5NS1mOGZhNzdhOTFmOTI/',
  394.         'only_matching': True,
  395.     }, {
  396.         'url': 'https://www.ardmediathek.de/ard/player/Y3JpZDovL3dkci5kZS9CZWl0cmFnLWQ2NDJjYWEzLTMwZWYtNGI4NS1iMTI2LTU1N2UxYTcxOGIzOQ/tatort-duo-koeln-leipzig-ihr-kinderlein-kommet',
  397.         'only_matching': True,
  398.     }]
  400.     def _real_extract(self, url):
  401.         video_id = self._match_id(url)
  403.         player_page = self._download_json(
  404.             'https://api.ardmediathek.de/public-gateway',
  405.             video_id, data=json.dumps({
  406.                 'query': '''{
  407.   playerPage(client: "ard", clipId: "%s") {
  408.     blockedByFsk
  409.     broadcastedOn
  410.     maturityContentRating
  411.     mediaCollection {
  412.       _duration
  413.       _geoblocked
  414.       _isLive
  415.       _mediaArray {
  416.         _mediaStreamArray {
  417.           _quality
  418.           _server
  419.           _stream
  420.         }
  421.       }
  422.       _previewImage
  423.       _subtitleUrl
  424.       _type
  425.     }
  426.     show {
  427.       title
  428.     }
  429.     synopsis
  430.     title
  431.     tracking {
  432.       atiCustomVars {
  433.         contentId
  434.       }
  435.     }
  436.   }
  437. }''' % video_id,
  438.             }).encode(), headers={
  439.                 'Content-Type': 'application/json'
  440.             })['data']['playerPage']
  441.         title = player_page['title']
  442.         content_id = str_or_none(try_get(
  443.             player_page, lambda x: x['tracking']['atiCustomVars']['contentId']))
  444.         media_collection = player_page.get('mediaCollection') or {}
  445.         if not media_collection and content_id:
  446.             media_collection = self._download_json(
  447.                 'https://www.ardmediathek.de/play/media/' + content_id,
  448.                 content_id, fatal=False) or {}
  449.         info = self._parse_media_info(
  450.             media_collection, content_id or video_id,
  451.             player_page.get('blockedByFsk'))
  452.         age_limit = None
  453.         description = player_page.get('synopsis')
  454.         maturity_content_rating = player_page.get('maturityContentRating')
  455.         if maturity_content_rating:
  456.             age_limit = int_or_none(maturity_content_rating.lstrip('FSK'))
  457.         if not age_limit and description:
  458.             age_limit = int_or_none(self._search_regex(
  459.                 r'\(FSK\s*(\d+)\)\s*$', description, 'age limit', default=None))
  460.         info.update({
  461.             'age_limit': age_limit,
  462.             'title': title,
  463.             'description': description,
  464.             'timestamp': unified_timestamp(player_page.get('broadcastedOn')),
  465.             'series': try_get(player_page, lambda x: x['show']['title']),
  466.         })
  467.         return info