logo

oasis-root

Compiled tree of Oasis Linux based on own branch at <https://hacktivis.me/git/oasis/> git clone https://anongit.hacktivis.me/git/oasis-root.git

brightcove.py (42590B)

    

  1. import base64
  2. import re
  3. import struct
  4. import urllib.parse
  5. import xml.etree.ElementTree
  7. from .adobepass import AdobePassIE
  8. from .common import InfoExtractor
  9. from ..compat import compat_etree_fromstring
  10. from ..networking.exceptions import HTTPError
  11. from ..utils import (
  12.     ExtractorError,
  13.     UnsupportedError,
  14.     clean_html,
  15.     dict_get,
  16.     extract_attributes,
  17.     find_xpath_attr,
  18.     fix_xml_ampersands,
  19.     float_or_none,
  20.     int_or_none,
  21.     join_nonempty,
  22.     js_to_json,
  23.     mimetype2ext,
  24.     parse_iso8601,
  25.     parse_qs,
  26.     smuggle_url,
  27.     str_or_none,
  28.     try_get,
  29.     unescapeHTML,
  30.     unsmuggle_url,
  31.     update_url_query,
  32.     url_or_none,
  33. )
  34. from ..utils.traversal import traverse_obj
  37. class BrightcoveLegacyIE(InfoExtractor):
  38.     IE_NAME = 'brightcove:legacy'
  39.     _VALID_URL = r'(?:https?://.*brightcove\.com/(services|viewer).*?\?|brightcove:)(?P<query>.*)'
  41.     _TESTS = [
  42.         {
  43.             # From http://www.8tv.cat/8aldia/videos/xavier-sala-i-martin-aquesta-tarda-a-8-al-dia/
  44.             'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1654948606001&flashID=myExperience&%40videoPlayer=2371591881001',
  45.             'md5': '5423e113865d26e40624dce2e4b45d95',
  46.             'note': 'Test Brightcove downloads and detection in GenericIE',
  47.             'info_dict': {
  48.                 'id': '2371591881001',
  49.                 'ext': 'mp4',
  50.                 'title': 'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
  51.                 'uploader': '8TV',
  52.                 'description': 'md5:a950cc4285c43e44d763d036710cd9cd',
  53.                 'timestamp': 1368213670,
  54.                 'upload_date': '20130510',
  55.                 'uploader_id': '1589608506001',
  56.             },
  57.             'skip': 'The player has been deactivated by the content owner',
  58.         },
  59.         {
  60.             # From http://medianetwork.oracle.com/video/player/1785452137001
  61.             'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1217746023001&flashID=myPlayer&%40videoPlayer=1785452137001',
  62.             'info_dict': {
  63.                 'id': '1785452137001',
  64.                 'ext': 'flv',
  65.                 'title': 'JVMLS 2012: Arrays 2.0 - Opportunities and Challenges',
  66.                 'description': 'John Rose speaks at the JVM Language Summit, August 1, 2012.',
  67.                 'uploader': 'Oracle',
  68.                 'timestamp': 1344975024,
  69.                 'upload_date': '20120814',
  70.                 'uploader_id': '1460825906',
  71.             },
  72.             'skip': 'video not playable',
  73.         },
  74.         {
  75.             # From http://mashable.com/2013/10/26/thermoelectric-bracelet-lets-you-control-your-body-temperature/
  76.             'url': 'http://c.brightcove.com/services/viewer/federated_f9?&playerID=1265504713001&publisherID=AQ%7E%7E%2CAAABBzUwv1E%7E%2CxP-xFHVUstiMFlNYfvF4G9yFnNaqCw_9&videoID=2750934548001',
  77.             'info_dict': {
  78.                 'id': '2750934548001',
  79.                 'ext': 'mp4',
  80.                 'title': 'This Bracelet Acts as a Personal Thermostat',
  81.                 'description': 'md5:547b78c64f4112766ccf4e151c20b6a0',
  82.                 # 'uploader': 'Mashable',
  83.                 'timestamp': 1382041798,
  84.                 'upload_date': '20131017',
  85.                 'uploader_id': '1130468786001',
  86.             },
  87.         },
  88.         {
  89.             # test that the default referer works
  90.             # from http://national.ballet.ca/interact/video/Lost_in_Motion_II/
  91.             'url': 'http://link.brightcove.com/services/player/bcpid756015033001?bckey=AQ~~,AAAApYJi_Ck~,GxhXCegT1Dp39ilhXuxMJxasUhVNZiil&bctid=2878862109001',
  92.             'info_dict': {
  93.                 'id': '2878862109001',
  94.                 'ext': 'mp4',
  95.                 'title': 'Lost in Motion II',
  96.                 'description': 'md5:363109c02998fee92ec02211bd8000df',
  97.                 'uploader': 'National Ballet of Canada',
  98.             },
  99.             'skip': 'Video gone',
  100.         },
  101.         {
  102.             # test flv videos served by akamaihd.net
  103.             # From http://www.redbull.com/en/bike/stories/1331655643987/replay-uci-dh-world-cup-2014-from-fort-william
  104.             'url': 'http://c.brightcove.com/services/viewer/htmlFederated?%40videoPlayer=ref%3Aevent-stream-356&linkBaseURL=http%3A%2F%2Fwww.redbull.com%2Fen%2Fbike%2Fvideos%2F1331655630249%2Freplay-uci-fort-william-2014-dh&playerKey=AQ%7E%7E%2CAAAApYJ7UqE%7E%2Cxqr_zXk0I-zzNndy8NlHogrCb5QdyZRf&playerID=1398061561001#__youtubedl_smuggle=%7B%22Referer%22%3A+%22http%3A%2F%2Fwww.redbull.com%2Fen%2Fbike%2Fstories%2F1331655643987%2Freplay-uci-dh-world-cup-2014-from-fort-william%22%7D',
  105.             # The md5 checksum changes on each download
  106.             'info_dict': {
  107.                 'id': '3750436379001',
  108.                 'ext': 'flv',
  109.                 'title': 'UCI MTB World Cup 2014: Fort William, UK - Downhill Finals',
  110.                 'uploader': 'RBTV Old (do not use)',
  111.                 'description': 'UCI MTB World Cup 2014: Fort William, UK - Downhill Finals',
  112.                 'timestamp': 1409122195,
  113.                 'upload_date': '20140827',
  114.                 'uploader_id': '710858724001',
  115.             },
  116.             'skip': 'Video gone',
  117.         },
  118.         {
  119.             # playlist with 'videoList'
  120.             # from http://support.brightcove.com/en/video-cloud/docs/playlist-support-single-video-players
  121.             'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=3550052898001&playerKey=AQ%7E%7E%2CAAABmA9XpXk%7E%2C-Kp7jNgisre1fG5OdqpAFUTcs0lP_ZoL',
  122.             'info_dict': {
  123.                 'title': 'Sealife',
  124.                 'id': '3550319591001',
  125.             },
  126.             'playlist_mincount': 7,
  127.             'skip': 'Unsupported URL',
  128.         },
  129.         {
  130.             # playlist with 'playlistTab' (https://github.com/ytdl-org/youtube-dl/issues/9965)
  131.             'url': 'http://c.brightcove.com/services/json/experience/runtime/?command=get_programming_for_experience&playerKey=AQ%7E%7E,AAABXlLMdok%7E,NJ4EoMlZ4rZdx9eU1rkMVd8EaYPBBUlg',
  132.             'info_dict': {
  133.                 'id': '1522758701001',
  134.                 'title': 'Lesson 08',
  135.             },
  136.             'playlist_mincount': 10,
  137.             'skip': 'Unsupported URL',
  138.         },
  139.         {
  140.             # playerID inferred from bcpid
  141.             # from http://www.un.org/chinese/News/story.asp?NewsID=27724
  142.             'url': 'https://link.brightcove.com/services/player/bcpid1722935254001/?bctid=5360463607001&autoStart=false&secureConnections=true&width=650&height=350',
  143.             'only_matching': True,  # Tested in GenericIE
  144.         },
  145.     ]
  147.     _WEBPAGE_TESTS = [{
  148.         # embedded brightcove video
  149.         # it also tests brightcove videos that need to set the 'Referer'
  150.         # in the http requests
  151.         'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
  152.         'info_dict': {
  153.             'id': '2765128793001',
  154.             'ext': 'mp4',
  155.             'title': 'Le cours de bourse : l’analyse technique',
  156.             'description': 'md5:7e9ad046e968cb2d1114004aba466fd9',
  157.             'uploader': 'BFM BUSINESS',
  158.         },
  159.         'params': {
  160.             'skip_download': True,
  161.         },
  162.         'skip': '404 Not Found',
  163.     }, {
  164.         # embedded with itemprop embedURL and video id spelled as `idVideo`
  165.         'url': 'http://bfmbusiness.bfmtv.com/mediaplayer/chroniques/olivier-delamarche/',
  166.         'info_dict': {
  167.             'id': '5255628253001',
  168.             'ext': 'mp4',
  169.             'title': 'md5:37c519b1128915607601e75a87995fc0',
  170.             'description': 'md5:37f7f888b434bb8f8cc8dbd4f7a4cf26',
  171.             'uploader': 'BFM BUSINESS',
  172.             'uploader_id': '876450612001',
  173.             'timestamp': 1482255315,
  174.             'upload_date': '20161220',
  175.         },
  176.         'params': {
  177.             'skip_download': True,
  178.         },
  179.         'skip': 'Redirects, page gone',
  180.     }, {
  181.         # https://github.com/ytdl-org/youtube-dl/issues/2253
  182.         'url': 'http://bcove.me/i6nfkrc3',
  183.         'md5': '0ba9446db037002366bab3b3eb30c88c',
  184.         'info_dict': {
  185.             'id': '3101154703001',
  186.             'ext': 'mp4',
  187.             'title': 'Still no power',
  188.             'uploader': 'thestar.com',
  189.             'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',
  190.         },
  191.         'skip': 'video gone',
  192.     }, {
  193.         # https://github.com/ytdl-org/youtube-dl/issues/3541
  194.         'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
  195.         'info_dict': {
  196.             'id': '3866516442001',
  197.             'ext': 'mp4',
  198.             'title': 'Leer mij vrouwen kennen: Aflevering 1',
  199.             'description': 'Leer mij vrouwen kennen: Aflevering 1',
  200.             'uploader': 'SBS Broadcasting',
  201.         },
  202.         'skip': 'Restricted to Netherlands, 404 Not Found',
  203.         'params': {
  204.             'skip_download': True,  # m3u8 download
  205.         },
  206.     }, {
  207.         # Brightcove video in <iframe>
  208.         'url': 'http://www.un.org/chinese/News/story.asp?NewsID=27724',
  209.         'md5': '36d74ef5e37c8b4a2ce92880d208b968',
  210.         'info_dict': {
  211.             'id': '5360463607001',
  212.             'ext': 'mp4',
  213.             'title': '叙利亚失明儿童在废墟上演唱《心跳》  呼吁获得正常童年生活',
  214.             'description': '联合国儿童基金会中东和北非区域大使、作曲家扎德·迪拉尼（Zade Dirani）在3月15日叙利亚冲突爆发7周年纪念日之际发布了为叙利亚谱写的歌曲《心跳》（HEARTBEAT），为受到六年冲突影响的叙利亚儿童发出强烈呐喊，呼吁世界做出共同努力，使叙利亚儿童重新获得享有正常童年生活的权利。',
  215.             'uploader': 'United Nations',
  216.             'uploader_id': '1362235914001',
  217.             'timestamp': 1489593889,
  218.             'upload_date': '20170315',
  219.         },
  220.         'skip': '404 Not Found',
  221.     }, {
  222.         # Brightcove with UUID in videoPlayer
  223.         'url': 'http://www8.hp.com/cn/zh/home.html',
  224.         'info_dict': {
  225.             'id': '5255815316001',
  226.             'ext': 'mp4',
  227.             'title': 'Sprocket Video - China',
  228.             'description': 'Sprocket Video - China',
  229.             'uploader': 'HP-Video Gallery',
  230.             'timestamp': 1482263210,
  231.             'upload_date': '20161220',
  232.             'uploader_id': '1107601872001',
  233.         },
  234.         'params': {
  235.             'skip_download': True,  # m3u8 download
  236.         },
  237.         'skip': 'video rotates...weekly?',
  238.     }, {
  239.         # Multiple brightcove videos
  240.         # https://github.com/ytdl-org/youtube-dl/issues/2283
  241.         'url': 'http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html',
  242.         'info_dict': {
  243.             'id': 'always-never',
  244.             'title': 'Always / Never - The New Yorker',
  245.         },
  246.         'playlist_count': 3,
  247.         'params': {
  248.             'extract_flat': False,
  249.             'skip_download': True,
  250.         },
  251.         'skip': 'Redirects, page gone',
  252.     }, {
  253.         # BrightcoveInPageEmbed embed
  254.         'url': 'http://www.geekandsundry.com/tabletop-bonus-wils-final-thoughts-on-dread/',
  255.         'info_dict': {
  256.             'id': '4238694884001',
  257.             'ext': 'flv',
  258.             'title': 'Tabletop: Dread, Last Thoughts',
  259.             'description': 'Tabletop: Dread, Last Thoughts',
  260.             'duration': 51690,
  261.         },
  262.         'skip': 'Redirects, page gone',
  263.     }, {
  264.         # Brightcove embed, with no valid 'renditions' but valid 'IOSRenditions'
  265.         # This video can't be played in browsers if Flash disabled and UA set to iPhone, which is actually a false alarm
  266.         'url': 'https://dl.dropboxusercontent.com/u/29092637/interview.html',
  267.         'info_dict': {
  268.             'id': '4785848093001',
  269.             'ext': 'mp4',
  270.             'title': 'The Cardinal Pell Interview',
  271.             'description': 'Sky News Contributor Andrew Bolt interviews George Pell in Rome, following the Cardinal\'s evidence before the Royal Commission into Child Abuse. ',
  272.             'uploader': 'GlobeCast Australia - GlobeStream',
  273.             'uploader_id': '2733773828001',
  274.             'upload_date': '20160304',
  275.             'timestamp': 1457083087,
  276.         },
  277.         'params': {
  278.             # m3u8 downloads
  279.             'skip_download': True,
  280.         },
  281.         'skip': '404 Not Found',
  282.     }, {
  283.         # Brightcove embed with whitespace around attribute names
  284.         'url': 'http://www.stack.com/video/3167554373001/learn-to-hit-open-three-pointers-with-damian-lillard-s-baseline-drift-drill',
  285.         'info_dict': {
  286.             'id': '3167554373001',
  287.             'ext': 'mp4',
  288.             'title': "Learn to Hit Open Three-Pointers With Damian Lillard's Baseline Drift Drill",
  289.             'description': 'md5:57bacb0e0f29349de4972bfda3191713',
  290.             'uploader_id': '1079349493',
  291.             'upload_date': '20140207',
  292.             'timestamp': 1391810548,
  293.         },
  294.         'params': {
  295.             'skip_download': True,
  296.         },
  297.         'skip': '410 Gone',
  298.     }]
  300.     @classmethod
  301.     def _build_brightcove_url(cls, object_str):
  302.         """
  303.         Build a Brightcove url from a xml string containing
  304.         <object class="BrightcoveExperience">{params}</object>
  305.         """
  307.         # Fix up some stupid HTML, see https://github.com/ytdl-org/youtube-dl/issues/1553
  308.         object_str = re.sub(r'(<param(?:\s+[a-zA-Z0-9_]+="[^"]*")*)>',
  309.                             lambda m: m.group(1) + '/>', object_str)
  310.         # Fix up some stupid XML, see https://github.com/ytdl-org/youtube-dl/issues/1608
  311.         object_str = object_str.replace('<--', '<!--')
  312.         # remove namespace to simplify extraction
  313.         object_str = re.sub(r'(<object[^>]*)(xmlns=".*?")', r'\1', object_str)
  314.         object_str = fix_xml_ampersands(object_str)
  316.         try:
  317.             object_doc = compat_etree_fromstring(object_str.encode())
  318.         except xml.etree.ElementTree.ParseError:
  319.             return
  321.         fv_el = find_xpath_attr(object_doc, './param', 'name', 'flashVars')
  322.         if fv_el is not None:
  323.             flashvars = dict(
  324.                 (k, v[0])
  325.                 for k, v in urllib.parse.parse_qs(fv_el.attrib['value']).items())
  326.         else:
  327.             flashvars = {}
  329.         data_url = object_doc.attrib.get('data', '')
  330.         data_url_params = parse_qs(data_url)
  332.         def find_param(name):
  333.             if name in flashvars:
  334.                 return flashvars[name]
  335.             node = find_xpath_attr(object_doc, './param', 'name', name)
  336.             if node is not None:
  337.                 return node.attrib['value']
  338.             return data_url_params.get(name)
  340.         params = {}
  342.         player_id = find_param('playerID') or find_param('playerId')
  343.         if player_id is None:
  344.             raise ExtractorError('Cannot find player ID')
  345.         params['playerID'] = player_id
  347.         player_key = find_param('playerKey')
  348.         # Not all pages define this value
  349.         if player_key is not None:
  350.             params['playerKey'] = player_key
  351.         # These fields hold the id of the video
  352.         video_player = find_param('@videoPlayer') or find_param('videoId') or find_param('videoID') or find_param('@videoList')
  353.         if video_player is not None:
  354.             if isinstance(video_player, list):
  355.                 video_player = video_player[0]
  356.             video_player = video_player.strip()
  357.             # UUID is also possible for videoPlayer (e.g.
  358.             # http://www.popcornflix.com/hoodies-vs-hooligans/7f2d2b87-bbf2-4623-acfb-ea942b4f01dd
  359.             # or http://www8.hp.com/cn/zh/home.html)
  360.             if not (re.match(
  361.                     r'^(?:\d+|[\da-fA-F]{8}-?[\da-fA-F]{4}-?[\da-fA-F]{4}-?[\da-fA-F]{4}-?[\da-fA-F]{12})$',
  362.                     video_player) or video_player.startswith('ref:')):
  363.                 return None
  364.             params['@videoPlayer'] = video_player
  365.         link_base = find_param('linkBaseURL')
  366.         if link_base is not None:
  367.             params['linkBaseURL'] = link_base
  368.         return cls._make_brightcove_url(params)
  370.     @classmethod
  371.     def _build_brightcove_url_from_js(cls, object_js):
  372.         # The layout of JS is as follows:
  373.         # customBC.createVideo = function (width, height, playerID, playerKey, videoPlayer, VideoRandomID) {
  374.         #   // build Brightcove <object /> XML
  375.         # }
  376.         m = re.search(
  377.             r'''(?x)customBC\.createVideo\(
  378.                 .*?                                                  # skipping width and height
  379.                 ["\'](?P<playerID>\d+)["\']\s*,\s*                   # playerID
  380.                 ["\'](?P<playerKey>AQ[^"\']{48})[^"\']*["\']\s*,\s*  # playerKey begins with AQ and is 50 characters
  381.                                                                      # in length, however it's appended to itself
  382.                                                                      # in places, so truncate
  383.                 ["\'](?P<videoID>\d+)["\']                           # @videoPlayer
  384.             ''', object_js)
  385.         if m:
  386.             return cls._make_brightcove_url(m.groupdict())
  388.     @classmethod
  389.     def _make_brightcove_url(cls, params):
  390.         return update_url_query(
  391.             'https://c.brightcove.com/services/viewer/htmlFederated', params)
  393.     @classmethod
  394.     def _extract_brightcove_url(cls, webpage):
  395.         """Try to extract the brightcove url from the webpage, returns None
  396.         if it can't be found
  397.         """
  398.         urls = cls._extract_brightcove_urls(webpage)
  399.         return urls[0] if urls else None
  401.     @classmethod
  402.     def _extract_brightcove_urls(cls, webpage):
  403.         """Return a list of all Brightcove URLs from the webpage """
  405.         url_m = re.search(
  406.             r'''(?x)
  407.                 <meta\s+
  408.                     (?:property|itemprop)=([\'"])(?:og:video|embedURL)\1[^>]+
  409.                     content=([\'"])(?P<url>https?://(?:secure|c)\.brightcove.com/(?:(?!\2).)+)\2
  410.             ''', webpage)
  411.         if url_m:
  412.             url = unescapeHTML(url_m.group('url'))
  413.             # Some sites don't add it, we can't download with this url, for example:
  414.             # http://www.ktvu.com/videos/news/raw-video-caltrain-releases-video-of-man-almost/vCTZdY/
  415.             if 'playerKey' in url or 'videoId' in url or 'idVideo' in url:
  416.                 return [url]
  418.         matches = re.findall(
  419.             r'''(?sx)<object
  420.             (?:
  421.                 [^>]+?class=[\'"][^>]*?BrightcoveExperience.*?[\'"] |
  422.                 [^>]*?>\s*<param\s+name="movie"\s+value="https?://[^/]*brightcove\.com/
  423.             ).+?>\s*</object>''',
  424.             webpage)
  425.         if matches:
  426.             return list(filter(None, [cls._build_brightcove_url(m) for m in matches]))
  428.         matches = re.findall(r'(customBC\.createVideo\(.+?\);)', webpage)
  429.         if matches:
  430.             return list(filter(None, [
  431.                 cls._build_brightcove_url_from_js(custom_bc)
  432.                 for custom_bc in matches]))
  433.         return [src for _, src in re.findall(
  434.             r'<iframe[^>]+src=([\'"])((?:https?:)?//link\.brightcove\.com/services/player/(?!\1).+)\1', webpage)]
  436.     def _extract_from_webpage(self, url, webpage):
  437.         bc_urls = self._extract_brightcove_urls(webpage)
  438.         for bc_url in bc_urls:
  439.             yield self.url_result(smuggle_url(bc_url, {'Referer': url}), BrightcoveLegacyIE)
  441.     def _real_extract(self, url):
  442.         url, smuggled_data = unsmuggle_url(url, {})
  444.         # Change the 'videoId' and others field to '@videoPlayer'
  445.         url = re.sub(r'(?<=[?&])(videoI(d|D)|idVideo|bctid)', '%40videoPlayer', url)
  446.         # Change bckey (used by bcove.me urls) to playerKey
  447.         url = re.sub(r'(?<=[?&])bckey', 'playerKey', url)
  448.         mobj = self._match_valid_url(url)
  449.         query_str = mobj.group('query')
  450.         query = urllib.parse.parse_qs(query_str)
  452.         video_player = query.get('@videoPlayer')
  453.         if video_player:
  454.             # We set the original url as the default 'Referer' header
  455.             referer = query.get('linkBaseURL', [None])[0] or smuggled_data.get('Referer', url)
  456.             video_id = video_player[0]
  457.             if 'playerID' not in query:
  458.                 mobj = re.search(r'/bcpid(\d+)', url)
  459.                 if mobj is not None:
  460.                     query['playerID'] = [mobj.group(1)]
  461.             publisher_id = query.get('publisherId')
  462.             if publisher_id and publisher_id[0].isdigit():
  463.                 publisher_id = publisher_id[0]
  464.             if not publisher_id:
  465.                 player_key = query.get('playerKey')
  466.                 if player_key and ',' in player_key[0]:
  467.                     player_key = player_key[0]
  468.                 else:
  469.                     player_id = query.get('playerID')
  470.                     if player_id and player_id[0].isdigit():
  471.                         headers = {}
  472.                         if referer:
  473.                             headers['Referer'] = referer
  474.                         player_page = self._download_webpage(
  475.                             'https://link.brightcove.com/services/player/bcpid' + player_id[0],
  476.                             video_id, headers=headers, fatal=False)
  477.                         if player_page:
  478.                             player_key = self._search_regex(
  479.                                 r'<param\s+name="playerKey"\s+value="([\w~,-]+)"',
  480.                                 player_page, 'player key', fatal=False)
  481.                 if player_key:
  482.                     enc_pub_id = player_key.split(',')[1].replace('~', '=')
  483.                     publisher_id = struct.unpack('>Q', base64.urlsafe_b64decode(enc_pub_id))[0]
  484.             if publisher_id:
  485.                 brightcove_new_url = f'https://players.brightcove.net/{publisher_id}/default_default/index.html?videoId={video_id}'
  486.                 if referer:
  487.                     brightcove_new_url = smuggle_url(brightcove_new_url, {'referrer': referer})
  488.                 return self.url_result(brightcove_new_url, BrightcoveNewIE.ie_key(), video_id)
  489.         # TODO: figure out if it's possible to extract playlistId from playerKey
  490.         # elif 'playerKey' in query:
  491.         #     player_key = query['playerKey']
  492.         #     return self._get_playlist_info(player_key[0])
  493.         raise UnsupportedError(url)
  496. class BrightcoveNewBaseIE(AdobePassIE):
  497.     def _parse_brightcove_metadata(self, json_data, video_id, headers={}):
  498.         title = json_data['name'].strip()
  500.         formats, subtitles = [], {}
  501.         sources = json_data.get('sources') or []
  502.         for source in sources:
  503.             container = source.get('container')
  504.             ext = mimetype2ext(source.get('type'))
  505.             src = source.get('src')
  506.             if ext == 'm3u8' or container == 'M2TS':
  507.                 if not src:
  508.                     continue
  509.                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
  510.                     src, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
  511.                 subtitles = self._merge_subtitles(subtitles, subs)
  512.             elif ext == 'mpd':
  513.                 if not src:
  514.                     continue
  515.                 fmts, subs = self._extract_mpd_formats_and_subtitles(src, video_id, 'dash', fatal=False)
  516.                 subtitles = self._merge_subtitles(subtitles, subs)
  517.             else:
  518.                 streaming_src = source.get('streaming_src')
  519.                 stream_name, app_name = source.get('stream_name'), source.get('app_name')
  520.                 if not src and not streaming_src and (not stream_name or not app_name):
  521.                     continue
  522.                 tbr = float_or_none(source.get('avg_bitrate'), 1000)
  523.                 height = int_or_none(source.get('height'))
  524.                 width = int_or_none(source.get('width'))
  525.                 f = {
  526.                     'tbr': tbr,
  527.                     'filesize': int_or_none(source.get('size')),
  528.                     'container': container,
  529.                     'ext': ext or container.lower(),
  530.                 }
  531.                 if width == 0 and height == 0:
  532.                     f.update({
  533.                         'vcodec': 'none',
  534.                     })
  535.                 else:
  536.                     f.update({
  537.                         'width': width,
  538.                         'height': height,
  539.                         'vcodec': source.get('codec'),
  540.                     })
  542.                 def build_format_id(kind):
  543.                     return join_nonempty(kind, tbr and f'{int(tbr)}k', height and f'{height}p')
  545.                 if src or streaming_src:
  546.                     f.update({
  547.                         'url': src or streaming_src,
  548.                         'format_id': build_format_id('http' if src else 'http-streaming'),
  549.                         'source_preference': 0 if src else -1,
  550.                     })
  551.                 else:
  552.                     f.update({
  553.                         'url': app_name,
  554.                         'play_path': stream_name,
  555.                         'format_id': build_format_id('rtmp'),
  556.                     })
  557.                 fmts = [f]
  559.             # https://support.brightcove.com/playback-api-video-fields-reference#key_systems_object
  560.             if container == 'WVM' or source.get('key_systems') or ext == 'ism':
  561.                 for f in fmts:
  562.                     f['has_drm'] = True
  563.             formats.extend(fmts)
  565.         if not formats:
  566.             errors = json_data.get('errors')
  567.             if errors:
  568.                 error = errors[0]
  569.                 self.raise_no_formats(
  570.                     error.get('message') or error.get('error_subcode') or error['error_code'], expected=True)
  572.         headers.pop('Authorization', None)  # or else http formats will give error 400
  573.         for f in formats:
  574.             f.setdefault('http_headers', {}).update(headers)
  576.         for text_track in json_data.get('text_tracks', []):
  577.             if text_track.get('kind') != 'captions':
  578.                 continue
  579.             text_track_url = url_or_none(text_track.get('src'))
  580.             if not text_track_url:
  581.                 continue
  582.             lang = (str_or_none(text_track.get('srclang'))
  583.                     or str_or_none(text_track.get('label')) or 'en').lower()
  584.             subtitles.setdefault(lang, []).append({
  585.                 'url': text_track_url,
  586.             })
  588.         is_live = False
  589.         duration = float_or_none(json_data.get('duration'), 1000)
  590.         if duration is not None and duration <= 0:
  591.             is_live = True
  593.         common_res = [(160, 90), (320, 180), (480, 720), (640, 360), (768, 432), (1024, 576), (1280, 720), (1366, 768), (1920, 1080)]
  594.         thumb_base_url = dict_get(json_data, ('poster', 'thumbnail'))
  595.         thumbnails = [{
  596.             'url': re.sub(r'\d+x\d+', f'{w}x{h}', thumb_base_url),
  597.             'width': w,
  598.             'height': h,
  599.         } for w, h in common_res] if thumb_base_url else None
  601.         return {
  602.             'id': video_id,
  603.             'title': title,
  604.             'description': clean_html(json_data.get('description')),
  605.             'thumbnails': thumbnails,
  606.             'duration': duration,
  607.             'timestamp': parse_iso8601(json_data.get('published_at')),
  608.             'uploader_id': json_data.get('account_id'),
  609.             'formats': formats,
  610.             'subtitles': subtitles,
  611.             'tags': json_data.get('tags', []),
  612.             'is_live': is_live,
  613.         }
  616. class BrightcoveNewIE(BrightcoveNewBaseIE):
  617.     IE_NAME = 'brightcove:new'
  618.     _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*(?P<content_type>video|playlist)Id=(?P<video_id>\d+|ref:[^&]+)'
  619.     _TESTS = [{
  620.         'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
  621.         'md5': 'c8100925723840d4b0d243f7025703be',
  622.         'info_dict': {
  623.             'id': '4463358922001',
  624.             'ext': 'mp4',
  625.             'title': 'Meet the man behind Popcorn Time',
  626.             'description': 'md5:eac376a4fe366edc70279bfb681aea16',
  627.             'duration': 165.768,
  628.             'timestamp': 1441391203,
  629.             'upload_date': '20150904',
  630.             'uploader_id': '929656772001',
  631.             'formats': 'mincount:20',
  632.         },
  633.         'skip': '404 Not Found',
  634.     }, {
  635.         # with rtmp streams
  636.         'url': 'http://players.brightcove.net/4036320279001/5d112ed9-283f-485f-a7f9-33f42e8bc042_default/index.html?videoId=4279049078001',
  637.         'info_dict': {
  638.             'id': '4279049078001',
  639.             'ext': 'mp4',
  640.             'title': 'Titansgrave: Chapter 0',
  641.             'description': 'Titansgrave: Chapter 0',
  642.             'duration': 1242.058,
  643.             'timestamp': 1433556729,
  644.             'upload_date': '20150606',
  645.             'uploader_id': '4036320279001',
  646.             'formats': 'mincount:39',
  647.         },
  648.         'params': {
  649.             # m3u8 download
  650.             'skip_download': True,
  651.         },
  652.     }, {
  653.         # playlist stream
  654.         'url': 'https://players.brightcove.net/1752604059001/S13cJdUBz_default/index.html?playlistId=5718313430001',
  655.         'info_dict': {
  656.             'id': '5718313430001',
  657.             'title': 'No Audio Playlist',
  658.         },
  659.         'playlist_count': 7,
  660.         'params': {
  661.             # m3u8 download
  662.             'skip_download': True,
  663.         },
  664.     }, {
  665.         'url': 'http://players.brightcove.net/5690807595001/HyZNerRl7_default/index.html?playlistId=5743160747001',
  666.         'only_matching': True,
  667.     }, {
  668.         # ref: prefixed video id
  669.         'url': 'http://players.brightcove.net/3910869709001/21519b5c-4b3b-4363-accb-bdc8f358f823_default/index.html?videoId=ref:7069442',
  670.         'only_matching': True,
  671.     }, {
  672.         # non numeric ref: prefixed video id
  673.         'url': 'http://players.brightcove.net/710858724001/default_default/index.html?videoId=ref:event-stream-356',
  674.         'only_matching': True,
  675.     }, {
  676.         # unavailable video without message but with error_code
  677.         'url': 'http://players.brightcove.net/1305187701/c832abfb-641b-44eb-9da0-2fe76786505f_default/index.html?videoId=4377407326001',
  678.         'only_matching': True,
  679.     }]
  681.     _WEBPAGE_TESTS = [{
  682.         # brightcove player url embed
  683.         'url': 'https://nbc-2.com/weather/forecast/2022/11/16/forecast-warmest-day-of-the-week/',
  684.         'md5': '2934d5372b354d27083ccf8575dbfee2',
  685.         'info_dict': {
  686.             'id': '6315650313112',
  687.             'title': 'First Alert Forecast: November 15, 2022',
  688.             'ext': 'mp4',
  689.             'tags': ['nbc2', 'forecast'],
  690.             'uploader_id': '6146886170001',
  691.             'thumbnail': r're:^https?://.*\.jpg$',
  692.             'timestamp': 1668574571,
  693.             'duration': 233.375,
  694.             'upload_date': '20221116',
  695.         },
  696.     }, {
  697.         # embedded with video tag only
  698.         'url': 'https://www.gooddishtv.com/tiktok-rapping-chef-mr-pyrex',
  699.         'info_dict': {
  700.             'id': 'tiktok-rapping-chef-mr-pyrex',
  701.             'title': 'TikTok\'s Rapping Chef Makes Jambalaya for the Hosts',
  702.             'thumbnail': r're:^https?://.*\.jpg$',
  703.             'age_limit': 0,
  704.             'description': 'Just in time for Mardi Gras',
  705.         },
  706.         'playlist': [{
  707.             'info_dict': {
  708.                 'id': '6299189544001',
  709.                 'ext': 'mp4',
  710.                 'title': 'TGD_01-032_5',
  711.                 'thumbnail': r're:^https?://.*\.jpg$',
  712.                 'tags': [],
  713.                 'timestamp': 1646078943,
  714.                 'uploader_id': '1569565978001',
  715.                 'upload_date': '20220228',
  716.                 'duration': 217.195,
  717.             },
  718.         }, {
  719.             'info_dict': {
  720.                 'id': '6305565995112',
  721.                 'ext': 'mp4',
  722.                 'title': 'TGD 01-087 (Airs 05.25.22)_Segment 5',
  723.                 'thumbnail': r're:^https?://.*\.jpg$',
  724.                 'tags': [],
  725.                 'timestamp': 1651604591,
  726.                 'uploader_id': '1569565978001',
  727.                 'upload_date': '20220503',
  728.                 'duration': 310.421,
  729.             },
  730.         }],
  731.     }, {
  732.         # Brightcove:new type [2].
  733.         'url': 'http://www.delawaresportszone.com/video-st-thomas-more-earns-first-trip-to-basketball-semis',
  734.         'md5': '2b35148fcf48da41c9fb4591650784f3',
  735.         'info_dict': {
  736.             'id': '5348741021001',
  737.             'ext': 'mp4',
  738.             'upload_date': '20170306',
  739.             'uploader_id': '4191638492001',
  740.             'timestamp': 1488769918,
  741.             'title': 'VIDEO:  St. Thomas More earns first trip to basketball semis',
  742.         },
  743.         'skip': '404 Not Found',
  744.     }, {
  745.         # Alternative brightcove <video> attributes
  746.         'url': 'http://www.programme-tv.net/videos/extraits/81095-guillaume-canet-evoque-les-rumeurs-d-infidelite-de-marion-cotillard-avec-brad-pitt-dans-vivement-dimanche/',
  747.         'info_dict': {
  748.             'id': '81095-guillaume-canet-evoque-les-rumeurs-d-infidelite-de-marion-cotillard-avec-brad-pitt-dans-vivement-dimanche',
  749.             'title': "Guillaume Canet évoque les rumeurs d'infidélité de Marion Cotillard avec Brad Pitt dans Vivement Dimanche, Extraits : toutes les vidéos avec Télé-Loisirs",
  750.         },
  751.         'playlist': [{
  752.             'md5': '732d22ba3d33f2f3fc253c39f8f36523',
  753.             'info_dict': {
  754.                 'id': '5311302538001',
  755.                 'ext': 'mp4',
  756.                 'title': "Guillaume Canet évoque les rumeurs d'infidélité de Marion Cotillard avec Brad Pitt dans Vivement Dimanche",
  757.                 'description': "Guillaume Canet évoque les rumeurs d'infidélité de Marion Cotillard avec Brad Pitt dans Vivement Dimanche (France 2, 5 février 2017)",
  758.                 'timestamp': 1486321708,
  759.                 'upload_date': '20170205',
  760.                 'uploader_id': '800000640001',
  761.             },
  762.             'only_matching': True,
  763.         }],
  764.         'skip': '404 Not Found',
  765.     }, {
  766.         # Brightcove URL in single quotes
  767.         'url': 'http://www.sportsnet.ca/baseball/mlb/sn-presents-russell-martin-world-citizen/',
  768.         'md5': '4ae374f1f8b91c889c4b9203c8c752af',
  769.         'info_dict': {
  770.             'id': '4255764656001',
  771.             'ext': 'mp4',
  772.             'title': 'SN Presents: Russell Martin, World Citizen',
  773.             'description': 'To understand why he was the Toronto Blue Jays’ top off-season priority is to appreciate his background and upbringing in Montreal, where he first developed his baseball skills. Written and narrated by Stephen Brunt.',
  774.             'uploader': 'Rogers Sportsnet',
  775.             'uploader_id': '1704050871',
  776.             'upload_date': '20150525',
  777.             'timestamp': 1432570283,
  778.         },
  779.         'skip': 'Page no longer has URL, now has javascript',
  780.     }]
  782.     @staticmethod
  783.     def _extract_url(ie, webpage):
  784.         urls = BrightcoveNewIE._extract_brightcove_urls(ie, webpage)
  785.         return urls[0] if urls else None
  787.     @staticmethod
  788.     def _extract_brightcove_urls(ie, webpage):
  789.         # Reference:
  790.         # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
  791.         # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#tag
  792.         # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript
  793.         # 4. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/in-page-embed-player-implementation.html
  794.         # 5. https://support.brightcove.com/en/video-cloud/docs/dynamically-assigning-videos-player
  796.         entries = []
  798.         # Look for iframe embeds [1]
  799.         for _, url in re.findall(
  800.                 r'<iframe[^>]+src=(["\'])((?:https?:)?//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
  801.             entries.append(url if url.startswith(('http:', 'https:')) else 'https:' + url)
  803.         # Look for <video> tags [2] and embed_in_page embeds [3]
  804.         # [2] looks like:
  805.         for video, script_tag, account_id, player_id, embed in re.findall(
  806.                 r'''(?isx)
  807.                     (<video(?:-js)?\s+[^>]*\bdata-video-id\s*=\s*['"]?[^>]+>)
  808.                     (?:.*?
  809.                         (<script[^>]+
  810.                             src=["\'](?:https?:)?//players\.brightcove\.net/
  811.                             (\d+)/([^/]+)_([^/]+)/index(?:\.min)?\.js
  812.                         )
  813.                     )?
  814.                 ''', webpage):
  815.             attrs = extract_attributes(video)
  817.             # According to examples from [4] it's unclear whether video id
  818.             # may be optional and what to do when it is
  819.             video_id = attrs.get('data-video-id')
  820.             if not video_id:
  821.                 continue
  823.             account_id = account_id or attrs.get('data-account')
  824.             if not account_id:
  825.                 continue
  827.             player_id = player_id or attrs.get('data-player') or 'default'
  828.             embed = embed or attrs.get('data-embed') or 'default'
  830.             bc_url = f'https://players.brightcove.net/{account_id}/{player_id}_{embed}/index.html?videoId={video_id}'
  832.             # Some brightcove videos may be embedded with video tag only and
  833.             # without script tag or any mentioning of brightcove at all. Such
  834.             # embeds are considered ambiguous since they are matched based only
  835.             # on data-video-id and data-account attributes and in the wild may
  836.             # not be brightcove embeds at all. Let's check reconstructed
  837.             # brightcove URLs in case of such embeds and only process valid
  838.             # ones. By this we ensure there is indeed a brightcove embed.
  839.             if not script_tag and not ie._is_valid_url(
  840.                     bc_url, video_id, 'possible brightcove video'):
  841.                 continue
  843.             entries.append(bc_url)
  845.         return entries
  847.     def _extract_from_webpage(self, url, webpage):
  848.         bc_urls = self._extract_brightcove_urls(self, webpage)
  849.         for bc_url in bc_urls:
  850.             yield self.url_result(smuggle_url(bc_url, {'referrer': url}), BrightcoveNewIE)
  852.     def _real_extract(self, url):
  853.         url, smuggled_data = unsmuggle_url(url, {})
  854.         self._initialize_geo_bypass({
  855.             'countries': smuggled_data.get('geo_countries'),
  856.             'ip_blocks': smuggled_data.get('geo_ip_blocks'),
  857.         })
  859.         account_id, player_id, embed, content_type, video_id = self._match_valid_url(url).groups()
  861.         policy_key_id = f'{account_id}_{player_id}'
  862.         policy_key = self.cache.load('brightcove', policy_key_id)
  863.         policy_key_extracted = False
  864.         store_pk = lambda x: self.cache.store('brightcove', policy_key_id, x)
  866.         def extract_policy_key():
  867.             base_url = f'https://players.brightcove.net/{account_id}/{player_id}_{embed}/'
  868.             config = self._download_json(
  869.                 base_url + 'config.json', video_id, fatal=False) or {}
  870.             policy_key = try_get(
  871.                 config, lambda x: x['video_cloud']['policy_key'])
  872.             if not policy_key:
  873.                 webpage = self._download_webpage(
  874.                     base_url + 'index.min.js', video_id)
  876.                 catalog = self._search_regex(
  877.                     r'catalog\(({.+?})\);', webpage, 'catalog', default=None)
  878.                 if catalog:
  879.                     catalog = self._parse_json(
  880.                         js_to_json(catalog), video_id, fatal=False)
  881.                     if catalog:
  882.                         policy_key = catalog.get('policyKey')
  884.                 if not policy_key:
  885.                     policy_key = self._search_regex(
  886.                         r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1',
  887.                         webpage, 'policy key', group='pk')
  889.             store_pk(policy_key)
  890.             return policy_key
  892.         token = smuggled_data.get('token')
  893.         api_url = f'https://{"edge-auth" if token else "edge"}.api.brightcove.com/playback/v1/accounts/{account_id}/{content_type}s/{video_id}'
  894.         headers = {'Authorization': f'Bearer {token}'} if token else {}
  895.         referrer = smuggled_data.get('referrer')  # XXX: notice the spelling/case of the key
  896.         if referrer:
  897.             headers.update({
  898.                 'Referer': referrer,
  899.                 'Origin': re.search(r'https?://[^/]+', referrer).group(0),
  900.             })
  902.         for _ in range(2):
  903.             if not policy_key:
  904.                 policy_key = extract_policy_key()
  905.                 policy_key_extracted = True
  906.             headers['Accept'] = f'application/json;pk={policy_key}'
  907.             try:
  908.                 json_data = self._download_json(api_url, video_id, headers=headers)
  909.                 break
  910.             except ExtractorError as e:
  911.                 if isinstance(e.cause, HTTPError) and e.cause.status in (401, 403):
  912.                     json_data = self._parse_json(e.cause.response.read().decode(), video_id)[0]
  913.                     message = json_data.get('message') or json_data['error_code']
  914.                     if json_data.get('error_subcode') == 'CLIENT_GEO':
  915.                         self.raise_geo_restricted(msg=message)
  916.                     elif json_data.get('error_code') == 'INVALID_POLICY_KEY' and not policy_key_extracted:
  917.                         policy_key = None
  918.                         store_pk(None)
  919.                         continue
  920.                     raise ExtractorError(message, expected=True)
  921.                 raise
  923.         errors = json_data.get('errors')
  924.         if errors and errors[0].get('error_subcode') == 'TVE_AUTH':
  925.             custom_fields = json_data['custom_fields']
  926.             tve_token = self._extract_mvpd_auth(
  927.                 smuggled_data['source_url'], video_id,
  928.                 custom_fields['bcadobepassrequestorid'],
  929.                 custom_fields['bcadobepassresourceid'])
  930.             json_data = self._download_json(
  931.                 api_url, video_id, headers={
  932.                     'Accept': f'application/json;pk={policy_key}',
  933.                 }, query={
  934.                     'tveToken': tve_token,
  935.                 })
  937.         if content_type == 'playlist':
  938.             return self.playlist_result(
  939.                 (self._parse_brightcove_metadata(vid, vid['id'], headers)
  940.                  for vid in traverse_obj(json_data, ('videos', lambda _, v: v['id']))),
  941.                 json_data.get('id'), json_data.get('name'),
  942.                 json_data.get('description'))
  944.         return self._parse_brightcove_metadata(
  945.             json_data, video_id, headers=headers)