logo

youtube-dl

[mirror] Download/Watch videos from video hostersgit clone https://hacktivis.me/git/mirror/youtube-dl.git

arcpublishing.py (7970B)

    

  1. # coding: utf-8
  2. from __future__ import unicode_literals
  4. import re
  6. from .common import InfoExtractor
  7. from ..utils import (
  8.     extract_attributes,
  9.     int_or_none,
  10.     parse_iso8601,
  11.     try_get,
  12. )
  15. class ArcPublishingIE(InfoExtractor):
  16.     _UUID_REGEX = r'[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12}'
  17.     _VALID_URL = r'arcpublishing:(?P<org>[a-z]+):(?P<id>%s)' % _UUID_REGEX
  18.     _TESTS = [{
  19.         # https://www.adn.com/politics/2020/11/02/video-senate-candidates-campaign-in-anchorage-on-eve-of-election-day/
  20.         'url': 'arcpublishing:adn:8c99cb6e-b29c-4bc9-9173-7bf9979225ab',
  21.         'only_matching': True,
  22.     }, {
  23.         # https://www.bostonglobe.com/video/2020/12/30/metro/footage-released-showing-officer-talking-about-striking-protesters-with-car/
  24.         'url': 'arcpublishing:bostonglobe:232b7ae6-7d73-432d-bc0a-85dbf0119ab1',
  25.         'only_matching': True,
  26.     }, {
  27.         # https://www.actionnewsjax.com/video/live-stream/
  28.         'url': 'arcpublishing:cmg:cfb1cf1b-3ab5-4d1b-86c5-a5515d311f2a',
  29.         'only_matching': True,
  30.     }, {
  31.         # https://elcomercio.pe/videos/deportes/deporte-total-futbol-peruano-seleccion-peruana-la-valorizacion-de-los-peruanos-en-el-exterior-tras-un-2020-atipico-nnav-vr-video-noticia/
  32.         'url': 'arcpublishing:elcomercio:27a7e1f8-2ec7-4177-874f-a4feed2885b3',
  33.         'only_matching': True,
  34.     }, {
  35.         # https://www.clickondetroit.com/video/community/2020/05/15/events-surrounding-woodward-dream-cruise-being-canceled/
  36.         'url': 'arcpublishing:gmg:c8793fb2-8d44-4242-881e-2db31da2d9fe',
  37.         'only_matching': True,
  38.     }, {
  39.         # https://www.wabi.tv/video/2020/12/30/trenton-company-making-equipment-pfizer-covid-vaccine/
  40.         'url': 'arcpublishing:gray:0b0ba30e-032a-4598-8810-901d70e6033e',
  41.         'only_matching': True,
  42.     }, {
  43.         # https://www.lateja.cr/el-mundo/video-china-aprueba-con-condiciones-su-primera/dfcbfa57-527f-45ff-a69b-35fe71054143/video/
  44.         'url': 'arcpublishing:gruponacion:dfcbfa57-527f-45ff-a69b-35fe71054143',
  45.         'only_matching': True,
  46.     }, {
  47.         # https://www.fifthdomain.com/video/2018/03/09/is-america-vulnerable-to-a-cyber-attack/
  48.         'url': 'arcpublishing:mco:aa0ca6fe-1127-46d4-b32c-be0d6fdb8055',
  49.         'only_matching': True,
  50.     }, {
  51.         # https://www.vl.no/kultur/2020/12/09/en-melding-fra-en-lytter-endret-julelista-til-lewi-bergrud/
  52.         'url': 'arcpublishing:mentormedier:47a12084-650b-4011-bfd0-3699b6947b2d',
  53.         'only_matching': True,
  54.     }, {
  55.         # https://www.14news.com/2020/12/30/whiskey-theft-caught-camera-henderson-liquor-store/
  56.         'url': 'arcpublishing:raycom:b89f61f8-79fa-4c09-8255-e64237119bf7',
  57.         'only_matching': True,
  58.     }, {
  59.         # https://www.theglobeandmail.com/world/video-ethiopian-woman-who-became-symbol-of-integration-in-italy-killed-on/
  60.         'url': 'arcpublishing:tgam:411b34c1-8701-4036-9831-26964711664b',
  61.         'only_matching': True,
  62.     }, {
  63.         # https://www.pilotonline.com/460f2931-8130-4719-8ea1-ffcb2d7cb685-132.html
  64.         'url': 'arcpublishing:tronc:460f2931-8130-4719-8ea1-ffcb2d7cb685',
  65.         'only_matching': True,
  66.     }]
  67.     _POWA_DEFAULTS = [
  68.         (['cmg', 'prisa'], '%s-config-prod.api.cdn.arcpublishing.com/video'),
  69.         ([
  70.             'adn', 'advancelocal', 'answers', 'bonnier', 'bostonglobe', 'demo',
  71.             'gmg', 'gruponacion', 'infobae', 'mco', 'nzme', 'pmn', 'raycom',
  72.             'spectator', 'tbt', 'tgam', 'tronc', 'wapo', 'wweek',
  73.         ], 'video-api-cdn.%s.arcpublishing.com/api'),
  74.     ]
  76.     @staticmethod
  77.     def _extract_urls(webpage):
  78.         entries = []
  79.         # https://arcpublishing.atlassian.net/wiki/spaces/POWA/overview
  80.         for powa_el in re.findall(r'(<div[^>]+class="[^"]*\bpowa\b[^"]*"[^>]+data-uuid="%s"[^>]*>)' % ArcPublishingIE._UUID_REGEX, webpage):
  81.             powa = extract_attributes(powa_el) or {}
  82.             org = powa.get('data-org')
  83.             uuid = powa.get('data-uuid')
  84.             if org and uuid:
  85.                 entries.append('arcpublishing:%s:%s' % (org, uuid))
  86.         return entries
  88.     def _real_extract(self, url):
  89.         org, uuid = re.match(self._VALID_URL, url).groups()
  90.         for orgs, tmpl in self._POWA_DEFAULTS:
  91.             if org in orgs:
  92.                 base_api_tmpl = tmpl
  93.                 break
  94.         else:
  95.             base_api_tmpl = '%s-prod-cdn.video-api.arcpublishing.com/api'
  96.         if org == 'wapo':
  97.             org = 'washpost'
  98.         video = self._download_json(
  99.             'https://%s/v1/ansvideos/findByUuid' % (base_api_tmpl % org),
  100.             uuid, query={'uuid': uuid})[0]
  101.         title = video['headlines']['basic']
  102.         is_live = video.get('status') == 'live'
  104.         urls = []
  105.         formats = []
  106.         for s in video.get('streams', []):
  107.             s_url = s.get('url')
  108.             if not s_url or s_url in urls:
  109.                 continue
  110.             urls.append(s_url)
  111.             stream_type = s.get('stream_type')
  112.             if stream_type == 'smil':
  113.                 smil_formats = self._extract_smil_formats(
  114.                     s_url, uuid, fatal=False)
  115.                 for f in smil_formats:
  116.                     if f['url'].endswith('/cfx/st'):
  117.                         f['app'] = 'cfx/st'
  118.                         if not f['play_path'].startswith('mp4:'):
  119.                             f['play_path'] = 'mp4:' + f['play_path']
  120.                         if isinstance(f['tbr'], float):
  121.                             f['vbr'] = f['tbr'] * 1000
  122.                             del f['tbr']
  123.                             f['format_id'] = 'rtmp-%d' % f['vbr']
  124.                 formats.extend(smil_formats)
  125.             elif stream_type in ('ts', 'hls'):
  126.                 m3u8_formats = self._extract_m3u8_formats(
  127.                     s_url, uuid, 'mp4', 'm3u8' if is_live else 'm3u8_native',
  128.                     m3u8_id='hls', fatal=False)
  129.                 if all([f.get('acodec') == 'none' for f in m3u8_formats]):
  130.                     continue
  131.                 for f in m3u8_formats:
  132.                     if f.get('acodec') == 'none':
  133.                         f['preference'] = -40
  134.                     elif f.get('vcodec') == 'none':
  135.                         f['preference'] = -50
  136.                     height = f.get('height')
  137.                     if not height:
  138.                         continue
  139.                     vbr = self._search_regex(
  140.                         r'[_x]%d[_-](\d+)' % height, f['url'], 'vbr', default=None)
  141.                     if vbr:
  142.                         f['vbr'] = int(vbr)
  143.                 formats.extend(m3u8_formats)
  144.             else:
  145.                 vbr = int_or_none(s.get('bitrate'))
  146.                 formats.append({
  147.                     'format_id': '%s-%d' % (stream_type, vbr) if vbr else stream_type,
  148.                     'vbr': vbr,
  149.                     'width': int_or_none(s.get('width')),
  150.                     'height': int_or_none(s.get('height')),
  151.                     'filesize': int_or_none(s.get('filesize')),
  152.                     'url': s_url,
  153.                     'preference': -1,
  154.                 })
  155.         self._sort_formats(
  156.             formats, ('preference', 'width', 'height', 'vbr', 'filesize', 'tbr', 'ext', 'format_id'))
  158.         subtitles = {}
  159.         for subtitle in (try_get(video, lambda x: x['subtitles']['urls'], list) or []):
  160.             subtitle_url = subtitle.get('url')
  161.             if subtitle_url:
  162.                 subtitles.setdefault('en', []).append({'url': subtitle_url})
  164.         return {
  165.             'id': uuid,
  166.             'title': self._live_title(title) if is_live else title,
  167.             'thumbnail': try_get(video, lambda x: x['promo_image']['url']),
  168.             'description': try_get(video, lambda x: x['subheadlines']['basic']),
  169.             'formats': formats,
  170.             'duration': int_or_none(video.get('duration'), 100),
  171.             'timestamp': parse_iso8601(video.get('created_date')),
  172.             'subtitles': subtitles,
  173.             'is_live': is_live,
  174.         }