logo

youtube-dl

[mirror] Download/Watch videos from video hostersgit clone https://hacktivis.me/git/mirror/youtube-dl.git

nexx.py (17106B)

    

  1. # coding: utf-8
  2. from __future__ import unicode_literals
  4. import hashlib
  5. import random
  6. import re
  7. import time
  9. from .common import InfoExtractor
  10. from ..compat import compat_str
  11. from ..utils import (
  12.     ExtractorError,
  13.     int_or_none,
  14.     parse_duration,
  15.     try_get,
  16.     urlencode_postdata,
  17. )
  20. class NexxIE(InfoExtractor):
  21.     _VALID_URL = r'''(?x)
  22.                         (?:
  23.                             https?://api\.nexx(?:\.cloud|cdn\.com)/v3/(?P<domain_id>\d+)/videos/byid/|
  24.                             nexx:(?:(?P<domain_id_s>\d+):)?|
  25.                             https?://arc\.nexx\.cloud/api/video/
  26.                         )
  27.                         (?P<id>\d+)
  28.                     '''
  29.     _TESTS = [{
  30.         # movie
  31.         'url': 'https://api.nexx.cloud/v3/748/videos/byid/128907',
  32.         'md5': '31899fd683de49ad46f4ee67e53e83fe',
  33.         'info_dict': {
  34.             'id': '128907',
  35.             'ext': 'mp4',
  36.             'title': 'Stiftung Warentest',
  37.             'alt_title': 'Wie ein Test abläuft',
  38.             'description': 'md5:d1ddb1ef63de721132abd38639cc2fd2',
  39.             'creator': 'SPIEGEL TV',
  40.             'thumbnail': r're:^https?://.*\.jpg$',
  41.             'duration': 2509,
  42.             'timestamp': 1384264416,
  43.             'upload_date': '20131112',
  44.         },
  45.     }, {
  46.         # episode
  47.         'url': 'https://api.nexx.cloud/v3/741/videos/byid/247858',
  48.         'info_dict': {
  49.             'id': '247858',
  50.             'ext': 'mp4',
  51.             'title': 'Return of the Golden Child (OV)',
  52.             'description': 'md5:5d969537509a92b733de21bae249dc63',
  53.             'release_year': 2017,
  54.             'thumbnail': r're:^https?://.*\.jpg$',
  55.             'duration': 1397,
  56.             'timestamp': 1495033267,
  57.             'upload_date': '20170517',
  58.             'episode_number': 2,
  59.             'season_number': 2,
  60.         },
  61.         'params': {
  62.             'skip_download': True,
  63.         },
  64.         'skip': 'HTTP Error 404: Not Found',
  65.     }, {
  66.         # does not work via arc
  67.         'url': 'nexx:741:1269984',
  68.         'md5': 'c714b5b238b2958dc8d5642addba6886',
  69.         'info_dict': {
  70.             'id': '1269984',
  71.             'ext': 'mp4',
  72.             'title': '1 TAG ohne KLO... wortwörtlich! 😑',
  73.             'alt_title': '1 TAG ohne KLO... wortwörtlich! 😑',
  74.             'thumbnail': r're:^https?://.*\.jpg$',
  75.             'duration': 607,
  76.             'timestamp': 1518614955,
  77.             'upload_date': '20180214',
  78.         },
  79.     }, {
  80.         # free cdn from http://www.spiegel.de/video/eifel-zoo-aufregung-um-ausgebrochene-raubtiere-video-99018031.html
  81.         'url': 'nexx:747:1533779',
  82.         'md5': '6bf6883912b82b7069fb86c2297e9893',
  83.         'info_dict': {
  84.             'id': '1533779',
  85.             'ext': 'mp4',
  86.             'title': 'Aufregung um ausgebrochene Raubtiere',
  87.             'alt_title': 'Eifel-Zoo',
  88.             'description': 'md5:f21375c91c74ad741dcb164c427999d2',
  89.             'thumbnail': r're:^https?://.*\.jpg$',
  90.             'duration': 111,
  91.             'timestamp': 1527874460,
  92.             'upload_date': '20180601',
  93.         },
  94.     }, {
  95.         'url': 'https://api.nexxcdn.com/v3/748/videos/byid/128907',
  96.         'only_matching': True,
  97.     }, {
  98.         'url': 'nexx:748:128907',
  99.         'only_matching': True,
  100.     }, {
  101.         'url': 'nexx:128907',
  102.         'only_matching': True,
  103.     }, {
  104.         'url': 'https://arc.nexx.cloud/api/video/128907.json',
  105.         'only_matching': True,
  106.     }]
  108.     @staticmethod
  109.     def _extract_domain_id(webpage):
  110.         mobj = re.search(
  111.             r'<script\b[^>]+\bsrc=["\'](?:https?:)?//(?:require|arc)\.nexx(?:\.cloud|cdn\.com)/(?:sdk/)?(?P<id>\d+)',
  112.             webpage)
  113.         return mobj.group('id') if mobj else None
  115.     @staticmethod
  116.     def _extract_urls(webpage):
  117.         # Reference:
  118.         # 1. https://nx-s.akamaized.net/files/201510/44.pdf
  120.         entries = []
  122.         # JavaScript Integration
  123.         domain_id = NexxIE._extract_domain_id(webpage)
  124.         if domain_id:
  125.             for video_id in re.findall(
  126.                     r'(?is)onPLAYReady.+?_play\.(?:init|(?:control\.)?addPlayer)\s*\(.+?\s*,\s*["\']?(\d+)',
  127.                     webpage):
  128.                 entries.append(
  129.                     'https://api.nexx.cloud/v3/%s/videos/byid/%s'
  130.                     % (domain_id, video_id))
  132.         # TODO: support more embed formats
  134.         return entries
  136.     @staticmethod
  137.     def _extract_url(webpage):
  138.         return NexxIE._extract_urls(webpage)[0]
  140.     def _handle_error(self, response):
  141.         status = int_or_none(try_get(
  142.             response, lambda x: x['metadata']['status']) or 200)
  143.         if 200 <= status < 300:
  144.             return
  145.         raise ExtractorError(
  146.             '%s said: %s' % (self.IE_NAME, response['metadata']['errorhint']),
  147.             expected=True)
  149.     def _call_api(self, domain_id, path, video_id, data=None, headers={}):
  150.         headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
  151.         result = self._download_json(
  152.             'https://api.nexx.cloud/v3/%s/%s' % (domain_id, path), video_id,
  153.             'Downloading %s JSON' % path, data=urlencode_postdata(data),
  154.             headers=headers)
  155.         self._handle_error(result)
  156.         return result['result']
  158.     def _extract_free_formats(self, video, video_id):
  159.         stream_data = video['streamdata']
  160.         cdn = stream_data['cdnType']
  161.         assert cdn == 'free'
  163.         hash = video['general']['hash']
  165.         ps = compat_str(stream_data['originalDomain'])
  166.         if stream_data['applyFolderHierarchy'] == 1:
  167.             s = ('%04d' % int(video_id))[::-1]
  168.             ps += '/%s/%s' % (s[0:2], s[2:4])
  169.         ps += '/%s/%s_' % (video_id, hash)
  171.         t = 'http://%s' + ps
  172.         fd = stream_data['azureFileDistribution'].split(',')
  173.         cdn_provider = stream_data['cdnProvider']
  175.         def p0(p):
  176.             return '_%s' % p if stream_data['applyAzureStructure'] == 1 else ''
  178.         formats = []
  179.         if cdn_provider == 'ak':
  180.             t += ','
  181.             for i in fd:
  182.                 p = i.split(':')
  183.                 t += p[1] + p0(int(p[0])) + ','
  184.             t += '.mp4.csmil/master.%s'
  185.         elif cdn_provider == 'ce':
  186.             k = t.split('/')
  187.             h = k.pop()
  188.             http_base = t = '/'.join(k)
  189.             http_base = http_base % stream_data['cdnPathHTTP']
  190.             t += '/asset.ism/manifest.%s?dcp_ver=aos4&videostream='
  191.             for i in fd:
  192.                 p = i.split(':')
  193.                 tbr = int(p[0])
  194.                 filename = '%s%s%s.mp4' % (h, p[1], p0(tbr))
  195.                 f = {
  196.                     'url': http_base + '/' + filename,
  197.                     'format_id': '%s-http-%d' % (cdn, tbr),
  198.                     'tbr': tbr,
  199.                 }
  200.                 width_height = p[1].split('x')
  201.                 if len(width_height) == 2:
  202.                     f.update({
  203.                         'width': int_or_none(width_height[0]),
  204.                         'height': int_or_none(width_height[1]),
  205.                     })
  206.                 formats.append(f)
  207.                 a = filename + ':%s' % (tbr * 1000)
  208.                 t += a + ','
  209.             t = t[:-1] + '&audiostream=' + a.split(':')[0]
  210.         else:
  211.             assert False
  213.         if cdn_provider == 'ce':
  214.             formats.extend(self._extract_mpd_formats(
  215.                 t % (stream_data['cdnPathDASH'], 'mpd'), video_id,
  216.                 mpd_id='%s-dash' % cdn, fatal=False))
  217.         formats.extend(self._extract_m3u8_formats(
  218.             t % (stream_data['cdnPathHLS'], 'm3u8'), video_id, 'mp4',
  219.             entry_protocol='m3u8_native', m3u8_id='%s-hls' % cdn, fatal=False))
  221.         return formats
  223.     def _extract_azure_formats(self, video, video_id):
  224.         stream_data = video['streamdata']
  225.         cdn = stream_data['cdnType']
  226.         assert cdn == 'azure'
  228.         azure_locator = stream_data['azureLocator']
  230.         def get_cdn_shield_base(shield_type='', static=False):
  231.             for secure in ('', 's'):
  232.                 cdn_shield = stream_data.get('cdnShield%sHTTP%s' % (shield_type, secure.upper()))
  233.                 if cdn_shield:
  234.                     return 'http%s://%s' % (secure, cdn_shield)
  235.             else:
  236.                 if 'fb' in stream_data['azureAccount']:
  237.                     prefix = 'df' if static else 'f'
  238.                 else:
  239.                     prefix = 'd' if static else 'p'
  240.                 account = int(stream_data['azureAccount'].replace('nexxplayplus', '').replace('nexxplayfb', ''))
  241.                 return 'http://nx-%s%02d.akamaized.net/' % (prefix, account)
  243.         language = video['general'].get('language_raw') or ''
  245.         azure_stream_base = get_cdn_shield_base()
  246.         is_ml = ',' in language
  247.         azure_manifest_url = '%s%s/%s_src%s.ism/Manifest' % (
  248.             azure_stream_base, azure_locator, video_id, ('_manifest' if is_ml else '')) + '%s'
  250.         protection_token = try_get(
  251.             video, lambda x: x['protectiondata']['token'], compat_str)
  252.         if protection_token:
  253.             azure_manifest_url += '?hdnts=%s' % protection_token
  255.         formats = self._extract_m3u8_formats(
  256.             azure_manifest_url % '(format=m3u8-aapl)',
  257.             video_id, 'mp4', 'm3u8_native',
  258.             m3u8_id='%s-hls' % cdn, fatal=False)
  259.         formats.extend(self._extract_mpd_formats(
  260.             azure_manifest_url % '(format=mpd-time-csf)',
  261.             video_id, mpd_id='%s-dash' % cdn, fatal=False))
  262.         formats.extend(self._extract_ism_formats(
  263.             azure_manifest_url % '', video_id, ism_id='%s-mss' % cdn, fatal=False))
  265.         azure_progressive_base = get_cdn_shield_base('Prog', True)
  266.         azure_file_distribution = stream_data.get('azureFileDistribution')
  267.         if azure_file_distribution:
  268.             fds = azure_file_distribution.split(',')
  269.             if fds:
  270.                 for fd in fds:
  271.                     ss = fd.split(':')
  272.                     if len(ss) == 2:
  273.                         tbr = int_or_none(ss[0])
  274.                         if tbr:
  275.                             f = {
  276.                                 'url': '%s%s/%s_src_%s_%d.mp4' % (
  277.                                     azure_progressive_base, azure_locator, video_id, ss[1], tbr),
  278.                                 'format_id': '%s-http-%d' % (cdn, tbr),
  279.                                 'tbr': tbr,
  280.                             }
  281.                             width_height = ss[1].split('x')
  282.                             if len(width_height) == 2:
  283.                                 f.update({
  284.                                     'width': int_or_none(width_height[0]),
  285.                                     'height': int_or_none(width_height[1]),
  286.                                 })
  287.                             formats.append(f)
  289.         return formats
  291.     def _real_extract(self, url):
  292.         mobj = re.match(self._VALID_URL, url)
  293.         domain_id = mobj.group('domain_id') or mobj.group('domain_id_s')
  294.         video_id = mobj.group('id')
  296.         video = None
  298.         def find_video(result):
  299.             if isinstance(result, dict):
  300.                 return result
  301.             elif isinstance(result, list):
  302.                 vid = int(video_id)
  303.                 for v in result:
  304.                     if try_get(v, lambda x: x['general']['ID'], int) == vid:
  305.                         return v
  306.             return None
  308.         response = self._download_json(
  309.             'https://arc.nexx.cloud/api/video/%s.json' % video_id,
  310.             video_id, fatal=False)
  311.         if response and isinstance(response, dict):
  312.             result = response.get('result')
  313.             if result:
  314.                 video = find_video(result)
  316.         # not all videos work via arc, e.g. nexx:741:1269984
  317.         if not video:
  318.             # Reverse engineered from JS code (see getDeviceID function)
  319.             device_id = '%d:%d:%d%d' % (
  320.                 random.randint(1, 4), int(time.time()),
  321.                 random.randint(1e4, 99999), random.randint(1, 9))
  323.             result = self._call_api(domain_id, 'session/init', video_id, data={
  324.                 'nxp_devh': device_id,
  325.                 'nxp_userh': '',
  326.                 'precid': '0',
  327.                 'playlicense': '0',
  328.                 'screenx': '1920',
  329.                 'screeny': '1080',
  330.                 'playerversion': '6.0.00',
  331.                 'gateway': 'html5',
  332.                 'adGateway': '',
  333.                 'explicitlanguage': 'en-US',
  334.                 'addTextTemplates': '1',
  335.                 'addDomainData': '1',
  336.                 'addAdModel': '1',
  337.             }, headers={
  338.                 'X-Request-Enable-Auth-Fallback': '1',
  339.             })
  341.             cid = result['general']['cid']
  343.             # As described in [1] X-Request-Token generation algorithm is
  344.             # as follows:
  345.             #   md5( operation + domain_id + domain_secret )
  346.             # where domain_secret is a static value that will be given by nexx.tv
  347.             # as per [1]. Here is how this "secret" is generated (reversed
  348.             # from _play.api.init function, search for clienttoken). So it's
  349.             # actually not static and not that much of a secret.
  350.             # 1. https://nexxtvstorage.blob.core.windows.net/files/201610/27.pdf
  351.             secret = result['device']['clienttoken'][int(device_id[0]):]
  352.             secret = secret[0:len(secret) - int(device_id[-1])]
  354.             op = 'byid'
  356.             # Reversed from JS code for _play.api.call function (search for
  357.             # X-Request-Token)
  358.             request_token = hashlib.md5(
  359.                 ''.join((op, domain_id, secret)).encode('utf-8')).hexdigest()
  361.             result = self._call_api(
  362.                 domain_id, 'videos/%s/%s' % (op, video_id), video_id, data={
  363.                     'additionalfields': 'language,channel,actors,studio,licenseby,slug,subtitle,teaser,description',
  364.                     'addInteractionOptions': '1',
  365.                     'addStatusDetails': '1',
  366.                     'addStreamDetails': '1',
  367.                     'addCaptions': '1',
  368.                     'addScenes': '1',
  369.                     'addHotSpots': '1',
  370.                     'addBumpers': '1',
  371.                     'captionFormat': 'data',
  372.                 }, headers={
  373.                     'X-Request-CID': cid,
  374.                     'X-Request-Token': request_token,
  375.                 })
  376.             video = find_video(result)
  378.         general = video['general']
  379.         title = general['title']
  381.         cdn = video['streamdata']['cdnType']
  383.         if cdn == 'azure':
  384.             formats = self._extract_azure_formats(video, video_id)
  385.         elif cdn == 'free':
  386.             formats = self._extract_free_formats(video, video_id)
  387.         else:
  388.             # TODO: reverse more cdns
  389.             assert False
  391.         self._sort_formats(formats)
  393.         return {
  394.             'id': video_id,
  395.             'title': title,
  396.             'alt_title': general.get('subtitle'),
  397.             'description': general.get('description'),
  398.             'release_year': int_or_none(general.get('year')),
  399.             'creator': general.get('studio') or general.get('studio_adref'),
  400.             'thumbnail': try_get(
  401.                 video, lambda x: x['imagedata']['thumb'], compat_str),
  402.             'duration': parse_duration(general.get('runtime')),
  403.             'timestamp': int_or_none(general.get('uploaded')),
  404.             'episode_number': int_or_none(try_get(
  405.                 video, lambda x: x['episodedata']['episode'])),
  406.             'season_number': int_or_none(try_get(
  407.                 video, lambda x: x['episodedata']['season'])),
  408.             'formats': formats,
  409.         }
  412. class NexxEmbedIE(InfoExtractor):
  413.     _VALID_URL = r'https?://embed\.nexx(?:\.cloud|cdn\.com)/\d+/(?:video/)?(?P<id>[^/?#&]+)'
  414.     _TESTS = [{
  415.         'url': 'http://embed.nexx.cloud/748/KC1614647Z27Y7T?autoplay=1',
  416.         'md5': '16746bfc28c42049492385c989b26c4a',
  417.         'info_dict': {
  418.             'id': '161464',
  419.             'ext': 'mp4',
  420.             'title': 'Nervenkitzel Achterbahn',
  421.             'alt_title': 'Karussellbauer in Deutschland',
  422.             'description': 'md5:ffe7b1cc59a01f585e0569949aef73cc',
  423.             'creator': 'SPIEGEL TV',
  424.             'thumbnail': r're:^https?://.*\.jpg$',
  425.             'duration': 2761,
  426.             'timestamp': 1394021479,
  427.             'upload_date': '20140305',
  428.         },
  429.         'params': {
  430.             'format': 'bestvideo',
  431.             'skip_download': True,
  432.         },
  433.     }, {
  434.         'url': 'https://embed.nexx.cloud/11888/video/DSRTO7UVOX06S7',
  435.         'only_matching': True,
  436.     }]
  438.     @staticmethod
  439.     def _extract_urls(webpage):
  440.         # Reference:
  441.         # 1. https://nx-s.akamaized.net/files/201510/44.pdf
  443.         # iFrame Embed Integration
  444.         return [mobj.group('url') for mobj in re.finditer(
  445.             r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//embed\.nexx(?:\.cloud|cdn\.com)/\d+/(?:(?!\1).)+)\1',
  446.             webpage)]
  448.     def _real_extract(self, url):
  449.         embed_id = self._match_id(url)
  451.         webpage = self._download_webpage(url, embed_id)
  453.         return self.url_result(NexxIE._extract_url(webpage), ie=NexxIE.ie_key())