commit: f27224d57b6768569e1aedfaff326605bdb4f049
parent c0071885987b3737d2c586133007c61ab513a477
Author: Remita Amine <remitamine@gmail.com>
Date: Mon, 28 Dec 2020 10:50:29 +0100
[piksel] import format extraction
Diffstat:
2 files changed, 80 insertions(+), 31 deletions(-)
diff --git a/youtube_dl/extractor/nhk.py b/youtube_dl/extractor/nhk.py
@@ -90,7 +90,7 @@ class NhkVodIE(NhkBaseIE):
_TESTS = [{
# video clip
'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999011/',
- 'md5': '256a1be14f48d960a7e61e2532d95ec3',
+ 'md5': '7a90abcfe610ec22a6bfe15bd46b30ca',
'info_dict': {
'id': 'a95j5iza',
'ext': 'mp4',
diff --git a/youtube_dl/extractor/piksel.py b/youtube_dl/extractor/piksel.py
@@ -6,16 +6,33 @@ import re
from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
- ExtractorError,
dict_get,
+ ExtractorError,
int_or_none,
- unescapeHTML,
parse_iso8601,
+ try_get,
+ unescapeHTML,
)
class PikselIE(InfoExtractor):
- _VALID_URL = r'https?://player\.piksel\.com/v/(?:refid/[^/]+/prefid/)?(?P<id>[a-z0-9_]+)'
+ _VALID_URL = r'''(?x)https?://
+ (?:
+ (?:
+ player\.
+ (?:
+ olympusattelecom|
+ vibebyvista
+ )|
+ (?:api|player)\.multicastmedia|
+ (?:api-ovp|player)\.piksel
+ )\.com|
+ (?:
+ mz-edge\.stream\.co|
+ movie-s\.nhk\.or
+ )\.jp|
+ vidego\.baltimorecity\.gov
+ )/v/(?:refid/(?P<refid>[^/]+)/prefid/)?(?P<id>[\w-]+)'''
_TESTS = [
{
'url': 'http://player.piksel.com/v/ums2867l',
@@ -56,46 +73,41 @@ class PikselIE(InfoExtractor):
if mobj:
return mobj.group('url')
+ def _call_api(self, app_token, resource, display_id, query, fatal=True):
+ response = (self._download_json(
+ 'http://player.piksel.com/ws/ws_%s/api/%s/mode/json/apiv/5' % (resource, app_token),
+ display_id, query=query, fatal=fatal) or {}).get('response')
+ failure = try_get(response, lambda x: x['failure']['reason'])
+ if failure:
+ if fatal:
+ raise ExtractorError(failure, expected=True)
+ self.report_warning(failure)
+ return response
+
def _real_extract(self, url):
- display_id = self._match_id(url)
+ ref_id, display_id = re.match(self._VALID_URL, url).groups()
webpage = self._download_webpage(url, display_id)
- video_id = self._search_regex(
- r'data-de-program-uuid=[\'"]([a-z0-9]+)',
- webpage, 'program uuid', default=display_id)
app_token = self._search_regex([
r'clientAPI\s*:\s*"([^"]+)"',
r'data-de-api-key\s*=\s*"([^"]+)"'
], webpage, 'app token')
- response = self._download_json(
- 'http://player.piksel.com/ws/ws_program/api/%s/mode/json/apiv/5' % app_token,
- video_id, query={
- 'v': video_id
- })['response']
- failure = response.get('failure')
- if failure:
- raise ExtractorError(response['failure']['reason'], expected=True)
- video_data = response['WsProgramResponse']['program']['asset']
+ query = {'refid': ref_id, 'prefid': display_id} if ref_id else {'v': display_id}
+ program = self._call_api(
+ app_token, 'program', display_id, query)['WsProgramResponse']['program']
+ video_id = program['uuid']
+ video_data = program['asset']
title = video_data['title']
+ asset_type = dict_get(video_data, ['assetType', 'asset_type'])
formats = []
- m3u8_url = dict_get(video_data, [
- 'm3u8iPadURL',
- 'ipadM3u8Url',
- 'm3u8AndroidURL',
- 'm3u8iPhoneURL',
- 'iphoneM3u8Url'])
- if m3u8_url:
- formats.extend(self._extract_m3u8_formats(
- m3u8_url, video_id, 'mp4', 'm3u8_native',
- m3u8_id='hls', fatal=False))
-
- asset_type = dict_get(video_data, ['assetType', 'asset_type'])
- for asset_file in video_data.get('assetFiles', []):
+ def process_asset_file(asset_file):
+ if not asset_file:
+ return
# TODO: extract rtmp formats
http_url = asset_file.get('http_url')
if not http_url:
- continue
+ return
tbr = None
vbr = int_or_none(asset_file.get('videoBitrate'), 1024)
abr = int_or_none(asset_file.get('audioBitrate'), 1024)
@@ -118,6 +130,43 @@ class PikselIE(InfoExtractor):
'filesize': int_or_none(asset_file.get('filesize')),
'tbr': tbr,
})
+
+ def process_asset_files(asset_files):
+ for asset_file in (asset_files or []):
+ process_asset_file(asset_file)
+
+ process_asset_files(video_data.get('assetFiles'))
+ process_asset_file(video_data.get('referenceFile'))
+ if not formats:
+ asset_id = video_data.get('assetid') or program.get('assetid')
+ if asset_id:
+ process_asset_files(try_get(self._call_api(
+ app_token, 'asset_file', display_id, {
+ 'assetid': asset_id,
+ }, False), lambda x: x['WsAssetFileResponse']['AssetFiles']))
+
+ m3u8_url = dict_get(video_data, [
+ 'm3u8iPadURL',
+ 'ipadM3u8Url',
+ 'm3u8AndroidURL',
+ 'm3u8iPhoneURL',
+ 'iphoneM3u8Url'])
+ if m3u8_url:
+ formats.extend(self._extract_m3u8_formats(
+ m3u8_url, video_id, 'mp4', 'm3u8_native',
+ m3u8_id='hls', fatal=False))
+
+ smil_url = dict_get(video_data, ['httpSmil', 'hdSmil', 'rtmpSmil'])
+ if smil_url:
+ transform_source = None
+ if ref_id == 'nhkworld':
+ # TODO: figure out if this is something to be fixed in urljoin,
+ # _parse_smil_formats or keep it here
+ transform_source = lambda x: x.replace('src="/', 'src="').replace('/media"', '/media/"')
+ formats.extend(self._extract_smil_formats(
+ re.sub(r'/od/[^/]+/', '/od/http/', smil_url), video_id,
+ transform_source=transform_source, fatal=False))
+
self._sort_formats(formats)
subtitles = {}
