logo

youtube-dl

[mirror] Download/Watch videos from video hostersgit clone https://hacktivis.me/git/mirror/youtube-dl.git

cspan.py (10277B)

    

  1. from __future__ import unicode_literals
  3. import re
  5. from .common import InfoExtractor
  6. from ..utils import (
  7.     determine_ext,
  8.     ExtractorError,
  9.     extract_attributes,
  10.     find_xpath_attr,
  11.     get_element_by_attribute,
  12.     get_element_by_class,
  13.     int_or_none,
  14.     js_to_json,
  15.     merge_dicts,
  16.     parse_iso8601,
  17.     smuggle_url,
  18.     str_to_int,
  19.     unescapeHTML,
  20. )
  21. from .senateisvp import SenateISVPIE
  22. from .ustream import UstreamIE
  25. class CSpanIE(InfoExtractor):
  26.     _VALID_URL = r'https?://(?:www\.)?c-span\.org/video/\?(?P<id>[0-9a-f]+)'
  27.     IE_DESC = 'C-SPAN'
  28.     _TESTS = [{
  29.         'url': 'http://www.c-span.org/video/?313572-1/HolderonV',
  30.         'md5': '94b29a4f131ff03d23471dd6f60b6a1d',
  31.         'info_dict': {
  32.             'id': '315139',
  33.             'title': 'Attorney General Eric Holder on Voting Rights Act Decision',
  34.         },
  35.         'playlist_mincount': 2,
  36.         'skip': 'Regularly fails on travis, for unknown reasons',
  37.     }, {
  38.         'url': 'http://www.c-span.org/video/?c4486943/cspan-international-health-care-models',
  39.         # md5 is unstable
  40.         'info_dict': {
  41.             'id': 'c4486943',
  42.             'ext': 'mp4',
  43.             'title': 'CSPAN - International Health Care Models',
  44.             'description': 'md5:7a985a2d595dba00af3d9c9f0783c967',
  45.         }
  46.     }, {
  47.         'url': 'http://www.c-span.org/video/?318608-1/gm-ignition-switch-recall',
  48.         'info_dict': {
  49.             'id': '342759',
  50.             'title': 'General Motors Ignition Switch Recall',
  51.         },
  52.         'playlist_mincount': 6,
  53.     }, {
  54.         # Video from senate.gov
  55.         'url': 'http://www.c-span.org/video/?104517-1/immigration-reforms-needed-protect-skilled-american-workers',
  56.         'info_dict': {
  57.             'id': 'judiciary031715',
  58.             'ext': 'mp4',
  59.             'title': 'Immigration Reforms Needed to Protect Skilled American Workers',
  60.         },
  61.         'params': {
  62.             'skip_download': True,  # m3u8 downloads
  63.         }
  64.     }, {
  65.         # Ustream embedded video
  66.         'url': 'https://www.c-span.org/video/?114917-1/armed-services',
  67.         'info_dict': {
  68.             'id': '58428542',
  69.             'ext': 'flv',
  70.             'title': 'USHR07 Armed Services Committee',
  71.             'description': 'hsas00-2118-20150204-1000et-07\n\n\nUSHR07 Armed Services Committee',
  72.             'timestamp': 1423060374,
  73.             'upload_date': '20150204',
  74.             'uploader': 'HouseCommittee',
  75.             'uploader_id': '12987475',
  76.         },
  77.     }, {
  78.         # Audio Only
  79.         'url': 'https://www.c-span.org/video/?437336-1/judiciary-antitrust-competition-policy-consumer-rights',
  80.         'only_matching': True,
  81.     }]
  82.     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
  84.     def _real_extract(self, url):
  85.         video_id = self._match_id(url)
  86.         video_type = None
  87.         webpage = self._download_webpage(url, video_id)
  89.         ustream_url = UstreamIE._extract_url(webpage)
  90.         if ustream_url:
  91.             return self.url_result(ustream_url, UstreamIE.ie_key())
  93.         if '&vod' not in url:
  94.             bc = self._search_regex(
  95.                 r"(<[^>]+id='brightcove-player-embed'[^>]+>)",
  96.                 webpage, 'brightcove embed', default=None)
  97.             if bc:
  98.                 bc_attr = extract_attributes(bc)
  99.                 bc_url = self.BRIGHTCOVE_URL_TEMPLATE % (
  100.                     bc_attr.get('data-bcaccountid', '3162030207001'),
  101.                     bc_attr.get('data-noprebcplayerid', 'SyGGpuJy3g'),
  102.                     bc_attr.get('data-newbcplayerid', 'default'),
  103.                     bc_attr['data-bcid'])
  104.                 return self.url_result(smuggle_url(bc_url, {'source_url': url}))
  106.         def add_referer(formats):
  107.             for f in formats:
  108.                 f.setdefault('http_headers', {})['Referer'] = url
  110.         # As of 01.12.2020 this path looks to cover all cases making the rest
  111.         # of the code unnecessary
  112.         jwsetup = self._parse_json(
  113.             self._search_regex(
  114.                 r'(?s)jwsetup\s*=\s*({.+?})\s*;', webpage, 'jwsetup',
  115.                 default='{}'),
  116.             video_id, transform_source=js_to_json, fatal=False)
  117.         if jwsetup:
  118.             info = self._parse_jwplayer_data(
  119.                 jwsetup, video_id, require_title=False, m3u8_id='hls',
  120.                 base_url=url)
  121.             add_referer(info['formats'])
  122.             for subtitles in info['subtitles'].values():
  123.                 for subtitle in subtitles:
  124.                     ext = determine_ext(subtitle['url'])
  125.                     if ext == 'php':
  126.                         ext = 'vtt'
  127.                     subtitle['ext'] = ext
  128.             ld_info = self._search_json_ld(webpage, video_id, default={})
  129.             title = get_element_by_class('video-page-title', webpage) or \
  130.                 self._og_search_title(webpage)
  131.             description = get_element_by_attribute('itemprop', 'description', webpage) or \
  132.                 self._html_search_meta(['og:description', 'description'], webpage)
  133.             return merge_dicts(info, ld_info, {
  134.                 'title': title,
  135.                 'thumbnail': get_element_by_attribute('itemprop', 'thumbnailUrl', webpage),
  136.                 'description': description,
  137.                 'timestamp': parse_iso8601(get_element_by_attribute('itemprop', 'uploadDate', webpage)),
  138.                 'location': get_element_by_attribute('itemprop', 'contentLocation', webpage),
  139.                 'duration': int_or_none(self._search_regex(
  140.                     r'jwsetup\.seclength\s*=\s*(\d+);',
  141.                     webpage, 'duration', fatal=False)),
  142.                 'view_count': str_to_int(self._search_regex(
  143.                     r"<span[^>]+class='views'[^>]*>([\d,]+)\s+Views</span>",
  144.                     webpage, 'views', fatal=False)),
  145.             })
  147.         # Obsolete
  148.         # We first look for clipid, because clipprog always appears before
  149.         patterns = [r'id=\'clip(%s)\'\s*value=\'([0-9]+)\'' % t for t in ('id', 'prog')]
  150.         results = list(filter(None, (re.search(p, webpage) for p in patterns)))
  151.         if results:
  152.             matches = results[0]
  153.             video_type, video_id = matches.groups()
  154.             video_type = 'clip' if video_type == 'id' else 'program'
  155.         else:
  156.             m = re.search(r'data-(?P<type>clip|prog)id=["\'](?P<id>\d+)', webpage)
  157.             if m:
  158.                 video_id = m.group('id')
  159.                 video_type = 'program' if m.group('type') == 'prog' else 'clip'
  160.             else:
  161.                 senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
  162.                 if senate_isvp_url:
  163.                     title = self._og_search_title(webpage)
  164.                     surl = smuggle_url(senate_isvp_url, {'force_title': title})
  165.                     return self.url_result(surl, 'SenateISVP', video_id, title)
  166.                 video_id = self._search_regex(
  167.                     r'jwsetup\.clipprog\s*=\s*(\d+);',
  168.                     webpage, 'jwsetup program id', default=None)
  169.                 if video_id:
  170.                     video_type = 'program'
  171.         if video_type is None or video_id is None:
  172.             error_message = get_element_by_class('VLplayer-error-message', webpage)
  173.             if error_message:
  174.                 raise ExtractorError(error_message)
  175.             raise ExtractorError('unable to find video id and type')
  177.         def get_text_attr(d, attr):
  178.             return d.get(attr, {}).get('#text')
  180.         data = self._download_json(
  181.             'http://www.c-span.org/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id),
  182.             video_id)['video']
  183.         if data['@status'] != 'Success':
  184.             raise ExtractorError('%s said: %s' % (self.IE_NAME, get_text_attr(data, 'error')), expected=True)
  186.         doc = self._download_xml(
  187.             'http://www.c-span.org/common/services/flashXml.php?%sid=%s' % (video_type, video_id),
  188.             video_id)
  190.         description = self._html_search_meta('description', webpage)
  192.         title = find_xpath_attr(doc, './/string', 'name', 'title').text
  193.         thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
  195.         files = data['files']
  196.         capfile = get_text_attr(data, 'capfile')
  198.         entries = []
  199.         for partnum, f in enumerate(files):
  200.             formats = []
  201.             for quality in f.get('qualities', []):
  202.                 formats.append({
  203.                     'format_id': '%s-%sp' % (get_text_attr(quality, 'bitrate'), get_text_attr(quality, 'height')),
  204.                     'url': unescapeHTML(get_text_attr(quality, 'file')),
  205.                     'height': int_or_none(get_text_attr(quality, 'height')),
  206.                     'tbr': int_or_none(get_text_attr(quality, 'bitrate')),
  207.                 })
  208.             if not formats:
  209.                 path = unescapeHTML(get_text_attr(f, 'path'))
  210.                 if not path:
  211.                     continue
  212.                 formats = self._extract_m3u8_formats(
  213.                     path, video_id, 'mp4', entry_protocol='m3u8_native',
  214.                     m3u8_id='hls') if determine_ext(path) == 'm3u8' else [{'url': path, }]
  215.             add_referer(formats)
  216.             self._sort_formats(formats)
  217.             entries.append({
  218.                 'id': '%s_%d' % (video_id, partnum + 1),
  219.                 'title': (
  220.                     title if len(files) == 1 else
  221.                     '%s part %d' % (title, partnum + 1)),
  222.                 'formats': formats,
  223.                 'description': description,
  224.                 'thumbnail': thumbnail,
  225.                 'duration': int_or_none(get_text_attr(f, 'length')),
  226.                 'subtitles': {
  227.                     'en': [{
  228.                         'url': capfile,
  229.                         'ext': determine_ext(capfile, 'dfxp')
  230.                     }],
  231.                 } if capfile else None,
  232.             })
  234.         if len(entries) == 1:
  235.             entry = dict(entries[0])
  236.             entry['id'] = 'c' + video_id if video_type == 'clip' else video_id
  237.             return entry
  238.         else:
  239.             return {
  240.                 '_type': 'playlist',
  241.                 'entries': entries,
  242.                 'title': title,
  243.                 'id': 'c' + video_id if video_type == 'clip' else video_id,
  244.             }