logo

oasis-root

Compiled tree of Oasis Linux based on own branch at <https://hacktivis.me/git/oasis/> git clone https://anongit.hacktivis.me/git/oasis-root.git

wistia.py (16395B)

    

  1. import base64
  2. import re
  3. import urllib.parse
  5. from .common import InfoExtractor
  6. from ..networking import HEADRequest
  7. from ..networking.exceptions import HTTPError
  8. from ..utils import (
  9.     ExtractorError,
  10.     determine_ext,
  11.     filter_dict,
  12.     float_or_none,
  13.     int_or_none,
  14.     parse_qs,
  15.     traverse_obj,
  16.     try_get,
  17.     update_url_query,
  18.     urlhandle_detect_ext,
  19. )
  22. class WistiaBaseIE(InfoExtractor):
  23.     _VALID_ID_REGEX = r'(?P<id>[a-z0-9]{10})'
  24.     _VALID_URL_BASE = r'https?://(?:\w+\.)?wistia\.(?:net|com)/(?:embed/)?'
  25.     _EMBED_BASE_URL = 'http://fast.wistia.net/embed/'
  27.     def _download_embed_config(self, config_type, config_id, referer):
  28.         base_url = self._EMBED_BASE_URL + f'{config_type}/{config_id}'
  29.         video_password = self.get_param('videopassword')
  30.         embed_config = self._download_json(
  31.             base_url + '.json', config_id, headers={
  32.                 'Referer': referer if referer.startswith('http') else base_url,  # Some videos require this.
  33.             }, query=filter_dict({'password': video_password}))
  35.         error = traverse_obj(embed_config, 'error')
  36.         if error:
  37.             raise ExtractorError(
  38.                 f'Error while getting the playlist: {error}', expected=True)
  40.         if traverse_obj(embed_config, (
  41.                 'media', ('embed_options', 'embedOptions'), 'plugin',
  42.                 'passwordProtectedVideo', 'on', any)) == 'true':
  43.             if video_password:
  44.                 raise ExtractorError('Invalid video password', expected=True)
  45.             raise ExtractorError(
  46.                 'This content is password-protected. Use the --video-password option', expected=True)
  48.         return embed_config
  50.     def _get_real_ext(self, url):
  51.         ext = determine_ext(url, default_ext='bin')
  52.         if ext == 'bin':
  53.             urlh = self._request_webpage(
  54.                 HEADRequest(url), None, note='Checking media extension',
  55.                 errnote='HEAD request returned error', fatal=False)
  56.             if urlh:
  57.                 ext = urlhandle_detect_ext(urlh, default='bin')
  58.         return 'mp4' if ext == 'mov' else ext
  60.     def _extract_media(self, embed_config):
  61.         data = embed_config['media']
  62.         video_id = data['hashedId']
  63.         title = data['name']
  65.         formats = []
  66.         thumbnails = []
  67.         for a in data['assets']:
  68.             aurl = a.get('url')
  69.             if not aurl:
  70.                 continue
  71.             astatus = a.get('status')
  72.             atype = a.get('type')
  73.             if (astatus is not None and astatus != 2) or atype in ('preview', 'storyboard'):
  74.                 continue
  75.             elif atype in ('still', 'still_image'):
  76.                 thumbnails.append({
  77.                     'url': aurl.replace('.bin', f'.{self._get_real_ext(aurl)}'),
  78.                     'width': int_or_none(a.get('width')),
  79.                     'height': int_or_none(a.get('height')),
  80.                     'filesize': int_or_none(a.get('size')),
  81.                 })
  82.             else:
  83.                 aext = a.get('ext') or self._get_real_ext(aurl)
  84.                 display_name = a.get('display_name')
  85.                 format_id = atype
  86.                 if atype and atype.endswith('_video') and display_name:
  87.                     format_id = f'{atype[:-6]}-{display_name}'
  88.                 f = {
  89.                     'format_id': format_id,
  90.                     'url': aurl,
  91.                     'tbr': int_or_none(a.get('bitrate')) or None,
  92.                     'quality': 1 if atype == 'original' else None,
  93.                 }
  94.                 if display_name == 'Audio':
  95.                     f.update({
  96.                         'vcodec': 'none',
  97.                     })
  98.                 else:
  99.                     f.update({
  100.                         'width': int_or_none(a.get('width')),
  101.                         'height': int_or_none(a.get('height')),
  102.                         'vcodec': a.get('codec'),
  103.                     })
  104.                 if a.get('container') == 'm3u8' or aext == 'm3u8':
  105.                     ts_f = f.copy()
  106.                     ts_f.update({
  107.                         'ext': 'ts',
  108.                         'format_id': f['format_id'].replace('hls-', 'ts-'),
  109.                         'url': f['url'].replace('.bin', '.ts'),
  110.                     })
  111.                     formats.append(ts_f)
  112.                     f.update({
  113.                         'ext': 'mp4',
  114.                         'protocol': 'm3u8_native',
  115.                     })
  116.                 else:
  117.                     f.update({
  118.                         'container': a.get('container'),
  119.                         'ext': aext,
  120.                         'filesize': int_or_none(a.get('size')),
  121.                     })
  122.                 formats.append(f)
  124.         subtitles = {}
  125.         for caption in data.get('captions', []):
  126.             language = caption.get('language')
  127.             if not language:
  128.                 continue
  129.             subtitles[language] = [{
  130.                 'url': self._EMBED_BASE_URL + 'captions/' + video_id + '.vtt?language=' + language,
  131.             }]
  133.         return {
  134.             'id': video_id,
  135.             'title': title,
  136.             'description': data.get('seoDescription'),
  137.             'formats': formats,
  138.             'thumbnails': thumbnails,
  139.             'duration': float_or_none(data.get('duration')),
  140.             'timestamp': int_or_none(data.get('createdAt')),
  141.             'subtitles': subtitles,
  142.         }
  144.     @classmethod
  145.     def _extract_from_webpage(cls, url, webpage):
  146.         from .teachable import TeachableIE
  148.         if list(TeachableIE._extract_embed_urls(url, webpage)):
  149.             return
  151.         yield from super()._extract_from_webpage(url, webpage)
  153.     @classmethod
  154.     def _extract_wistia_async_embed(cls, webpage):
  155.         # https://wistia.com/support/embed-and-share/video-on-your-website
  156.         # https://wistia.com/support/embed-and-share/channel-embeds
  157.         yield from re.finditer(
  158.             r'''(?sx)
  159.                 <(?:div|section)[^>]+class=([\"'])(?:(?!\1).)*?(?P<type>wistia[a-z_0-9]+)\s*\bwistia_async_(?P<id>[a-z0-9]{10})\b(?:(?!\1).)*?\1
  160.             ''', webpage)
  162.     @classmethod
  163.     def _extract_url_media_id(cls, url):
  164.         mobj = re.search(r'(?:wmediaid|wvideo(?:id)?)]?=(?P<id>[a-z0-9]{10})', urllib.parse.unquote_plus(url))
  165.         if mobj:
  166.             return mobj.group('id')
  169. class WistiaIE(WistiaBaseIE):
  170.     _VALID_URL = rf'(?:wistia:|{WistiaBaseIE._VALID_URL_BASE}(?:iframe|medias)/){WistiaBaseIE._VALID_ID_REGEX}'
  171.     _EMBED_REGEX = [
  172.         r'''(?x)
  173.             <(?:meta[^>]+?content|(?:iframe|script)[^>]+?src)=["\']
  174.             (?P<url>(?:https?:)?//(?:fast\.)?wistia\.(?:net|com)/embed/(?:iframe|medias)/[a-z0-9]{10})
  175.             ''']
  176.     _TESTS = [{
  177.         # with hls video
  178.         'url': 'wistia:807fafadvk',
  179.         'md5': 'daff0f3687a41d9a71b40e0e8c2610fe',
  180.         'info_dict': {
  181.             'id': '807fafadvk',
  182.             'ext': 'mp4',
  183.             'title': 'Drip Brennan Dunn Workshop',
  184.             'description': 'a JV Webinars video',
  185.             'upload_date': '20160518',
  186.             'timestamp': 1463607249,
  187.             'duration': 4987.11,
  188.         },
  189.         'skip': 'video unavailable',
  190.     }, {
  191.         'url': 'wistia:a6ndpko1wg',
  192.         'md5': '10c1ce9c4dde638202513ed17a3767bd',
  193.         'info_dict': {
  194.             'id': 'a6ndpko1wg',
  195.             'ext': 'mp4',
  196.             'title': 'Episode 2: Boxed Water\'s retention is thirsty',
  197.             'upload_date': '20210324',
  198.             'description': 'md5:da5994c2c2d254833b412469d9666b7a',
  199.             'duration': 966.0,
  200.             'timestamp': 1616614369,
  201.             'thumbnail': 'https://embed-ssl.wistia.com/deliveries/53dc60239348dc9b9fba3755173ea4c2.png',
  202.         },
  203.     }, {
  204.         'url': 'wistia:5vd7p4bct5',
  205.         'md5': 'b9676d24bf30945d97060638fbfe77f0',
  206.         'info_dict': {
  207.             'id': '5vd7p4bct5',
  208.             'ext': 'mp4',
  209.             'title': 'md5:eaa9f64c4efd7b5f098b9b6118597679',
  210.             'description': 'md5:a9bea0315f0616aa5df2dc413ddcdd0f',
  211.             'upload_date': '20220915',
  212.             'timestamp': 1663258727,
  213.             'duration': 623.019,
  214.             'thumbnail': r're:https?://embed(?:-ssl)?.wistia.com/.+\.jpg$',
  215.         },
  216.     }, {
  217.         'url': 'wistia:sh7fpupwlt',
  218.         'only_matching': True,
  219.     }, {
  220.         'url': 'http://fast.wistia.net/embed/iframe/sh7fpupwlt',
  221.         'only_matching': True,
  222.     }, {
  223.         'url': 'http://fast.wistia.com/embed/iframe/sh7fpupwlt',
  224.         'only_matching': True,
  225.     }, {
  226.         'url': 'http://fast.wistia.net/embed/medias/sh7fpupwlt.json',
  227.         'only_matching': True,
  228.     }]
  230.     _WEBPAGE_TESTS = [{
  231.         'url': 'https://www.weidert.com/blog/wistia-channels-video-marketing-tool',
  232.         'info_dict': {
  233.             'id': 'cqwukac3z1',
  234.             'ext': 'mp4',
  235.             'title': 'How Wistia Channels Can Help Capture Inbound Value From Your Video Content',
  236.             'duration': 158.125,
  237.             'timestamp': 1618974400,
  238.             'description': 'md5:27abc99a758573560be72600ef95cece',
  239.             'upload_date': '20210421',
  240.             'thumbnail': 'https://embed-ssl.wistia.com/deliveries/6c551820ae950cdee2306d6cbe9ef742.jpg',
  241.         },
  242.     }, {
  243.         'url': 'https://study.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
  244.         'md5': 'b9676d24bf30945d97060638fbfe77f0',
  245.         'info_dict': {
  246.             'id': '5vd7p4bct5',
  247.             'ext': 'mp4',
  248.             'title': 'paywall_north-american-exploration-failed-colonies-of-spain-france-england',
  249.             'upload_date': '20220915',
  250.             'timestamp': 1663258727,
  251.             'duration': 623.019,
  252.             'thumbnail': 'https://embed-ssl.wistia.com/deliveries/83e6ec693e2c05a0ce65809cbaead86a.jpg',
  253.             'description': 'a Paywall Videos video',
  254.         },
  255.     }]
  257.     def _real_extract(self, url):
  258.         video_id = self._match_id(url)
  259.         embed_config = self._download_embed_config('medias', video_id, url)
  260.         return self._extract_media(embed_config)
  262.     @classmethod
  263.     def _extract_embed_urls(cls, url, webpage):
  264.         urls = list(super()._extract_embed_urls(url, webpage))
  265.         for match in cls._extract_wistia_async_embed(webpage):
  266.             if match.group('type') != 'wistia_channel':
  267.                 urls.append('wistia:{}'.format(match.group('id')))
  268.         for match in re.finditer(r'(?:data-wistia-?id=["\']|Wistia\.embed\(["\']|id=["\']wistia_)(?P<id>[a-z0-9]{10})',
  269.                                  webpage):
  270.             urls.append('wistia:{}'.format(match.group('id')))
  271.         if not WistiaChannelIE._extract_embed_urls(url, webpage):  # Fallback
  272.             media_id = cls._extract_url_media_id(url)
  273.             if media_id:
  274.                 urls.append('wistia:{}'.format(match.group('id')))
  275.         return urls
  278. class WistiaPlaylistIE(WistiaBaseIE):
  279.     _VALID_URL = rf'{WistiaBaseIE._VALID_URL_BASE}playlists/{WistiaBaseIE._VALID_ID_REGEX}'
  281.     _TEST = {
  282.         'url': 'https://fast.wistia.net/embed/playlists/aodt9etokc',
  283.         'info_dict': {
  284.             'id': 'aodt9etokc',
  285.         },
  286.         'playlist_count': 3,
  287.     }
  289.     def _real_extract(self, url):
  290.         playlist_id = self._match_id(url)
  291.         playlist = self._download_embed_config('playlists', playlist_id, url)
  293.         entries = []
  294.         for media in (try_get(playlist, lambda x: x[0]['medias']) or []):
  295.             embed_config = media.get('embed_config')
  296.             if not embed_config:
  297.                 continue
  298.             entries.append(self._extract_media(embed_config))
  300.         return self.playlist_result(entries, playlist_id)
  303. class WistiaChannelIE(WistiaBaseIE):
  304.     _VALID_URL = rf'(?:wistiachannel:|{WistiaBaseIE._VALID_URL_BASE}channel/){WistiaBaseIE._VALID_ID_REGEX}'
  306.     _TESTS = [{
  307.         # JSON Embed API returns 403, should fall back to webpage
  308.         'url': 'https://fast.wistia.net/embed/channel/yvyvu7wjbg?wchannelid=yvyvu7wjbg',
  309.         'info_dict': {
  310.             'id': 'yvyvu7wjbg',
  311.             'title': 'Copysmith Tutorials and Education!',
  312.             'description': 'Learn all things Copysmith via short and informative videos!',
  313.         },
  314.         'playlist_mincount': 7,
  315.         'expected_warnings': ['falling back to webpage'],
  316.     }, {
  317.         'url': 'https://fast.wistia.net/embed/channel/3802iirk0l',
  318.         'info_dict': {
  319.             'id': '3802iirk0l',
  320.             'title': 'The Roof',
  321.         },
  322.         'playlist_mincount': 20,
  323.     }, {
  324.         # link to popup video, follow --no-playlist
  325.         'url': 'https://fast.wistia.net/embed/channel/3802iirk0l?wchannelid=3802iirk0l&wmediaid=sp5dqjzw3n',
  326.         'info_dict': {
  327.             'id': 'sp5dqjzw3n',
  328.             'ext': 'mp4',
  329.             'title': 'The Roof S2: The Modern CRO',
  330.             'thumbnail': 'https://embed-ssl.wistia.com/deliveries/dadfa9233eaa505d5e0c85c23ff70741.png',
  331.             'duration': 86.487,
  332.             'description': 'A sales leader on The Roof? Man, they really must be letting anyone up here this season.\n',
  333.             'timestamp': 1619790290,
  334.             'upload_date': '20210430',
  335.         },
  336.         'params': {'noplaylist': True, 'skip_download': True},
  337.     }]
  338.     _WEBPAGE_TESTS = [{
  339.         'url': 'https://www.profitwell.com/recur/boxed-out',
  340.         'info_dict': {
  341.             'id': '6jyvmqz6zs',
  342.             'title': 'Boxed Out',
  343.             'description': 'md5:14a8a93a1dbe236718e6a59f8c8c7bae',
  344.         },
  345.         'playlist_mincount': 30,
  346.     }, {
  347.         # section instead of div
  348.         'url': 'https://360learning.com/studio/onboarding-joei/',
  349.         'info_dict': {
  350.             'id': 'z874k93n2o',
  351.             'title': 'Onboarding Joei.',
  352.             'description': 'Coming to you weekly starting Feb 19th.',
  353.         },
  354.         'playlist_mincount': 20,
  355.     }, {
  356.         'url': 'https://amplitude.com/amplify-sessions?amp%5Bwmediaid%5D=pz0m0l0if3&amp%5Bwvideo%5D=pz0m0l0if3&wchannelid=emyjmwjf79&wmediaid=i8um783bdt',
  357.         'info_dict': {
  358.             'id': 'pz0m0l0if3',
  359.             'title': 'A Framework for Improving Product Team Performance',
  360.             'ext': 'mp4',
  361.             'timestamp': 1653935275,
  362.             'upload_date': '20220530',
  363.             'description': 'Learn how to help your company improve and achieve your product related goals.',
  364.             'duration': 1854.39,
  365.             'thumbnail': 'https://embed-ssl.wistia.com/deliveries/12fd19e56413d9d6f04e2185c16a6f8854e25226.png',
  366.         },
  367.         'params': {'noplaylist': True, 'skip_download': True},
  368.     }]
  370.     def _real_extract(self, url):
  371.         channel_id = self._match_id(url)
  372.         media_id = self._extract_url_media_id(url)
  373.         if not self._yes_playlist(channel_id, media_id, playlist_label='channel'):
  374.             return self.url_result(f'wistia:{media_id}', 'Wistia')
  376.         try:
  377.             data = self._download_embed_config('channel', channel_id, url)
  378.         except (ExtractorError, HTTPError):
  379.             # Some channels give a 403 from the JSON API
  380.             self.report_warning('Failed to download channel data from API, falling back to webpage.')
  381.             webpage = self._download_webpage(f'https://fast.wistia.net/embed/channel/{channel_id}', channel_id)
  382.             data = self._parse_json(
  383.                 self._search_regex(rf'wchanneljsonp-{channel_id}\'\]\s*=[^\"]*\"([A-Za-z0-9=/]*)', webpage, 'jsonp', channel_id),
  384.                 channel_id, transform_source=lambda x: urllib.parse.unquote_plus(base64.b64decode(x).decode('utf-8')))
  386.         # XXX: can there be more than one series?
  387.         series = traverse_obj(data, ('series', 0), default={})
  389.         entries = [
  390.             self.url_result(f'wistia:{video["hashedId"]}', WistiaIE, title=video.get('name'))
  391.             for video in traverse_obj(series, ('sections', ..., 'videos', ...)) or []
  392.             if video.get('hashedId')
  393.         ]
  395.         return self.playlist_result(
  396.             entries, channel_id, playlist_title=series.get('title'), playlist_description=series.get('description'))
  398.     @classmethod
  399.     def _extract_embed_urls(cls, url, webpage):
  400.         yield from super()._extract_embed_urls(url, webpage)
  401.         for match in cls._extract_wistia_async_embed(webpage):
  402.             if match.group('type') == 'wistia_channel':
  403.                 # original url may contain wmediaid query param
  404.                 yield update_url_query(f'wistiachannel:{match.group("id")}', parse_qs(url))