logo

youtube-dl

[mirror] Download/Watch videos from video hostersgit clone https://hacktivis.me/git/mirror/youtube-dl.git

arte.py (12534B)

    

  1. # coding: utf-8
  2. from __future__ import unicode_literals
  4. import re
  6. from .common import InfoExtractor
  7. from ..compat import (
  8.     compat_str,
  9.     compat_urlparse,
  10. )
  11. from ..utils import (
  12.     ExtractorError,
  13.     int_or_none,
  14.     qualities,
  15.     strip_or_none,
  16.     try_get,
  17.     unified_strdate,
  18.     url_or_none,
  19. )
  22. class ArteTVBaseIE(InfoExtractor):
  23.     _ARTE_LANGUAGES = 'fr|de|en|es|it|pl'
  24.     _API_BASE = 'https://api.arte.tv/api/player/v1'
  27. class ArteTVIE(ArteTVBaseIE):
  28.     _VALID_URL = r'''(?x)
  29.                     https?://
  30.                         (?:
  31.                             (?:www\.)?arte\.tv/(?P<lang>%(langs)s)/videos|
  32.                             api\.arte\.tv/api/player/v\d+/config/(?P<lang_2>%(langs)s)
  33.                         )
  34.                         /(?P<id>\d{6}-\d{3}-[AF])
  35.                     ''' % {'langs': ArteTVBaseIE._ARTE_LANGUAGES}
  36.     _TESTS = [{
  37.         'url': 'https://www.arte.tv/en/videos/088501-000-A/mexico-stealing-petrol-to-survive/',
  38.         'info_dict': {
  39.             'id': '088501-000-A',
  40.             'ext': 'mp4',
  41.             'title': 'Mexico: Stealing Petrol to Survive',
  42.             'upload_date': '20190628',
  43.         },
  44.     }, {
  45.         'url': 'https://www.arte.tv/pl/videos/100103-000-A/usa-dyskryminacja-na-porodowce/',
  46.         'only_matching': True,
  47.     }, {
  48.         'url': 'https://api.arte.tv/api/player/v2/config/de/100605-013-A',
  49.         'only_matching': True,
  50.     }]
  52.     def _real_extract(self, url):
  53.         mobj = re.match(self._VALID_URL, url)
  54.         video_id = mobj.group('id')
  55.         lang = mobj.group('lang') or mobj.group('lang_2')
  57.         info = self._download_json(
  58.             '%s/config/%s/%s' % (self._API_BASE, lang, video_id), video_id)
  59.         player_info = info['videoJsonPlayer']
  61.         vsr = try_get(player_info, lambda x: x['VSR'], dict)
  62.         if not vsr:
  63.             error = None
  64.             if try_get(player_info, lambda x: x['custom_msg']['type']) == 'error':
  65.                 error = try_get(
  66.                     player_info, lambda x: x['custom_msg']['msg'], compat_str)
  67.             if not error:
  68.                 error = 'Video %s is not available' % player_info.get('VID') or video_id
  69.             raise ExtractorError(error, expected=True)
  71.         upload_date_str = player_info.get('shootingDate')
  72.         if not upload_date_str:
  73.             upload_date_str = (player_info.get('VRA') or player_info.get('VDA') or '').split(' ')[0]
  75.         title = (player_info.get('VTI') or player_info['VID']).strip()
  76.         subtitle = player_info.get('VSU', '').strip()
  77.         if subtitle:
  78.             title += ' - %s' % subtitle
  80.         qfunc = qualities(['MQ', 'HQ', 'EQ', 'SQ'])
  82.         LANGS = {
  83.             'fr': 'F',
  84.             'de': 'A',
  85.             'en': 'E[ANG]',
  86.             'es': 'E[ESP]',
  87.             'it': 'E[ITA]',
  88.             'pl': 'E[POL]',
  89.         }
  91.         langcode = LANGS.get(lang, lang)
  93.         formats = []
  94.         for format_id, format_dict in vsr.items():
  95.             f = dict(format_dict)
  96.             format_url = url_or_none(f.get('url'))
  97.             streamer = f.get('streamer')
  98.             if not format_url and not streamer:
  99.                 continue
  100.             versionCode = f.get('versionCode')
  101.             l = re.escape(langcode)
  103.             # Language preference from most to least priority
  104.             # Reference: section 6.8 of
  105.             # https://www.arte.tv/sites/en/corporate/files/complete-technical-guidelines-arte-geie-v1-07-1.pdf
  106.             PREFERENCES = (
  107.                 # original version in requested language, without subtitles
  108.                 r'VO{0}$'.format(l),
  109.                 # original version in requested language, with partial subtitles in requested language
  110.                 r'VO{0}-ST{0}$'.format(l),
  111.                 # original version in requested language, with subtitles for the deaf and hard-of-hearing in requested language
  112.                 r'VO{0}-STM{0}$'.format(l),
  113.                 # non-original (dubbed) version in requested language, without subtitles
  114.                 r'V{0}$'.format(l),
  115.                 # non-original (dubbed) version in requested language, with subtitles partial subtitles in requested language
  116.                 r'V{0}-ST{0}$'.format(l),
  117.                 # non-original (dubbed) version in requested language, with subtitles for the deaf and hard-of-hearing in requested language
  118.                 r'V{0}-STM{0}$'.format(l),
  119.                 # original version in requested language, with partial subtitles in different language
  120.                 r'VO{0}-ST(?!{0}).+?$'.format(l),
  121.                 # original version in requested language, with subtitles for the deaf and hard-of-hearing in different language
  122.                 r'VO{0}-STM(?!{0}).+?$'.format(l),
  123.                 # original version in different language, with partial subtitles in requested language
  124.                 r'VO(?:(?!{0}).+?)?-ST{0}$'.format(l),
  125.                 # original version in different language, with subtitles for the deaf and hard-of-hearing in requested language
  126.                 r'VO(?:(?!{0}).+?)?-STM{0}$'.format(l),
  127.                 # original version in different language, without subtitles
  128.                 r'VO(?:(?!{0}))?$'.format(l),
  129.                 # original version in different language, with partial subtitles in different language
  130.                 r'VO(?:(?!{0}).+?)?-ST(?!{0}).+?$'.format(l),
  131.                 # original version in different language, with subtitles for the deaf and hard-of-hearing in different language
  132.                 r'VO(?:(?!{0}).+?)?-STM(?!{0}).+?$'.format(l),
  133.             )
  135.             for pref, p in enumerate(PREFERENCES):
  136.                 if re.match(p, versionCode):
  137.                     lang_pref = len(PREFERENCES) - pref
  138.                     break
  139.             else:
  140.                 lang_pref = -1
  142.             media_type = f.get('mediaType')
  143.             if media_type == 'hls':
  144.                 m3u8_formats = self._extract_m3u8_formats(
  145.                     format_url, video_id, 'mp4', entry_protocol='m3u8_native',
  146.                     m3u8_id=format_id, fatal=False)
  147.                 for m3u8_format in m3u8_formats:
  148.                     m3u8_format['language_preference'] = lang_pref
  149.                 formats.extend(m3u8_formats)
  150.                 continue
  152.             format = {
  153.                 'format_id': format_id,
  154.                 'preference': -10 if f.get('videoFormat') == 'M3U8' else None,
  155.                 'language_preference': lang_pref,
  156.                 'format_note': '%s, %s' % (f.get('versionCode'), f.get('versionLibelle')),
  157.                 'width': int_or_none(f.get('width')),
  158.                 'height': int_or_none(f.get('height')),
  159.                 'tbr': int_or_none(f.get('bitrate')),
  160.                 'quality': qfunc(f.get('quality')),
  161.             }
  163.             if media_type == 'rtmp':
  164.                 format['url'] = f['streamer']
  165.                 format['play_path'] = 'mp4:' + f['url']
  166.                 format['ext'] = 'flv'
  167.             else:
  168.                 format['url'] = f['url']
  170.             formats.append(format)
  172.         self._sort_formats(formats)
  174.         return {
  175.             'id': player_info.get('VID') or video_id,
  176.             'title': title,
  177.             'description': player_info.get('VDE'),
  178.             'upload_date': unified_strdate(upload_date_str),
  179.             'thumbnail': player_info.get('programImage') or player_info.get('VTU', {}).get('IUR'),
  180.             'formats': formats,
  181.         }
  184. class ArteTVEmbedIE(InfoExtractor):
  185.     _VALID_URL = r'https?://(?:www\.)?arte\.tv/player/v\d+/index\.php\?.*?\bjson_url=.+'
  186.     _TESTS = [{
  187.         'url': 'https://www.arte.tv/player/v5/index.php?json_url=https%3A%2F%2Fapi.arte.tv%2Fapi%2Fplayer%2Fv2%2Fconfig%2Fde%2F100605-013-A&lang=de&autoplay=true&mute=0100605-013-A',
  188.         'info_dict': {
  189.             'id': '100605-013-A',
  190.             'ext': 'mp4',
  191.             'title': 'United we Stream November Lockdown Edition #13',
  192.             'description': 'md5:be40b667f45189632b78c1425c7c2ce1',
  193.             'upload_date': '20201116',
  194.         },
  195.     }, {
  196.         'url': 'https://www.arte.tv/player/v3/index.php?json_url=https://api.arte.tv/api/player/v2/config/de/100605-013-A',
  197.         'only_matching': True,
  198.     }]
  200.     @staticmethod
  201.     def _extract_urls(webpage):
  202.         return [url for _, url in re.findall(
  203.             r'<(?:iframe|script)[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?arte\.tv/player/v\d+/index\.php\?.*?\bjson_url=.+?)\1',
  204.             webpage)]
  206.     def _real_extract(self, url):
  207.         qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
  208.         json_url = qs['json_url'][0]
  209.         video_id = ArteTVIE._match_id(json_url)
  210.         return self.url_result(
  211.             json_url, ie=ArteTVIE.ie_key(), video_id=video_id)
  214. class ArteTVPlaylistIE(ArteTVBaseIE):
  215.     _VALID_URL = r'https?://(?:www\.)?arte\.tv/(?P<lang>%s)/videos/(?P<id>RC-\d{6})' % ArteTVBaseIE._ARTE_LANGUAGES
  216.     _TESTS = [{
  217.         'url': 'https://www.arte.tv/en/videos/RC-016954/earn-a-living/',
  218.         'info_dict': {
  219.             'id': 'RC-016954',
  220.             'title': 'Earn a Living',
  221.             'description': 'md5:d322c55011514b3a7241f7fb80d494c2',
  222.         },
  223.         'playlist_mincount': 6,
  224.     }, {
  225.         'url': 'https://www.arte.tv/pl/videos/RC-014123/arte-reportage/',
  226.         'only_matching': True,
  227.     }]
  229.     def _real_extract(self, url):
  230.         lang, playlist_id = re.match(self._VALID_URL, url).groups()
  231.         collection = self._download_json(
  232.             '%s/collectionData/%s/%s?source=videos'
  233.             % (self._API_BASE, lang, playlist_id), playlist_id)
  234.         entries = []
  235.         for video in collection['videos']:
  236.             if not isinstance(video, dict):
  237.                 continue
  238.             video_url = url_or_none(video.get('url')) or url_or_none(video.get('jsonUrl'))
  239.             if not video_url:
  240.                 continue
  241.             video_id = video.get('programId')
  242.             entries.append({
  243.                 '_type': 'url_transparent',
  244.                 'url': video_url,
  245.                 'id': video_id,
  246.                 'title': video.get('title'),
  247.                 'alt_title': video.get('subtitle'),
  248.                 'thumbnail': url_or_none(try_get(video, lambda x: x['mainImage']['url'], compat_str)),
  249.                 'duration': int_or_none(video.get('durationSeconds')),
  250.                 'view_count': int_or_none(video.get('views')),
  251.                 'ie_key': ArteTVIE.ie_key(),
  252.             })
  253.         title = collection.get('title')
  254.         description = collection.get('shortDescription') or collection.get('teaserText')
  255.         return self.playlist_result(entries, playlist_id, title, description)
  258. class ArteTVCategoryIE(ArteTVBaseIE):
  259.     _VALID_URL = r'https?://(?:www\.)?arte\.tv/(?P<lang>%s)/videos/(?P<id>[\w-]+(?:/[\w-]+)*)/?\s*$' % ArteTVBaseIE._ARTE_LANGUAGES
  260.     _TESTS = [{
  261.         'url': 'https://www.arte.tv/en/videos/politics-and-society/',
  262.         'info_dict': {
  263.             'id': 'politics-and-society',
  264.             'title': 'Politics and society',
  265.             'description': 'Investigative documentary series, geopolitical analysis, and international commentary',
  266.         },
  267.         'playlist_mincount': 13,
  268.     },
  269.     ]
  271.     @classmethod
  272.     def suitable(cls, url):
  273.         return (
  274.             not any(ie.suitable(url) for ie in (ArteTVIE, ArteTVPlaylistIE, ))
  275.             and super(ArteTVCategoryIE, cls).suitable(url))
  277.     def _real_extract(self, url):
  278.         lang, playlist_id = re.match(self._VALID_URL, url).groups()
  279.         webpage = self._download_webpage(url, playlist_id)
  281.         items = []
  282.         for video in re.finditer(
  283.                 r'<a\b[^>]*?href\s*=\s*(?P<q>"|\'|\b)(?P<url>https?://www\.arte\.tv/%s/videos/[\w/-]+)(?P=q)' % lang,
  284.                 webpage):
  285.             video = video.group('url')
  286.             if video == url:
  287.                 continue
  288.             if any(ie.suitable(video) for ie in (ArteTVIE, ArteTVPlaylistIE, )):
  289.                 items.append(video)
  291.         if items:
  292.             title = (self._og_search_title(webpage, default=None)
  293.                      or self._html_search_regex(r'<title\b[^>]*>([^<]+)</title>', default=None))
  294.             title = strip_or_none(title.rsplit('|', 1)[0]) or self._generic_title(url)
  296.             result = self.playlist_from_matches(items, playlist_id=playlist_id, playlist_title=title)
  297.             if result:
  298.                 description = self._og_search_description(webpage, default=None)
  299.                 if description:
  300.                     result['description'] = description
  301.                 return result