logo

oasis-root

Compiled tree of Oasis Linux based on own branch at <https://hacktivis.me/git/oasis/> git clone https://anongit.hacktivis.me/git/oasis-root.git

iqiyi.py (31008B)

    

  1. import hashlib
  2. import itertools
  3. import re
  4. import time
  5. import urllib.parse
  7. from .common import InfoExtractor
  8. from .openload import PhantomJSwrapper
  9. from ..utils import (
  10.     ExtractorError,
  11.     clean_html,
  12.     decode_packed_codes,
  13.     float_or_none,
  14.     format_field,
  15.     get_element_by_attribute,
  16.     get_element_by_id,
  17.     int_or_none,
  18.     js_to_json,
  19.     ohdave_rsa_encrypt,
  20.     parse_age_limit,
  21.     parse_duration,
  22.     parse_iso8601,
  23.     parse_resolution,
  24.     qualities,
  25.     remove_start,
  26.     str_or_none,
  27.     traverse_obj,
  28.     urljoin,
  29. )
  32. def md5_text(text):
  33.     return hashlib.md5(text.encode()).hexdigest()
  36. class IqiyiSDK:
  37.     def __init__(self, target, ip, timestamp):
  38.         self.target = target
  39.         self.ip = ip
  40.         self.timestamp = timestamp
  42.     @staticmethod
  43.     def split_sum(data):
  44.         return str(sum(int(p, 16) for p in data))
  46.     @staticmethod
  47.     def digit_sum(num):
  48.         if isinstance(num, int):
  49.             num = str(num)
  50.         return str(sum(map(int, num)))
  52.     def even_odd(self):
  53.         even = self.digit_sum(str(self.timestamp)[::2])
  54.         odd = self.digit_sum(str(self.timestamp)[1::2])
  55.         return even, odd
  57.     def preprocess(self, chunksize):
  58.         self.target = md5_text(self.target)
  59.         chunks = []
  60.         for i in range(32 // chunksize):
  61.             chunks.append(self.target[chunksize * i:chunksize * (i + 1)])
  62.         if 32 % chunksize:
  63.             chunks.append(self.target[32 - 32 % chunksize:])
  64.         return chunks, list(map(int, self.ip.split('.')))
  66.     def mod(self, modulus):
  67.         chunks, ip = self.preprocess(32)
  68.         self.target = chunks[0] + ''.join(str(p % modulus) for p in ip)
  70.     def split(self, chunksize):
  71.         modulus_map = {
  72.             4: 256,
  73.             5: 10,
  74.             8: 100,
  75.         }
  77.         chunks, ip = self.preprocess(chunksize)
  78.         ret = ''
  79.         for i in range(len(chunks)):
  80.             ip_part = str(ip[i] % modulus_map[chunksize]) if i < 4 else ''
  81.             if chunksize == 8:
  82.                 ret += ip_part + chunks[i]
  83.             else:
  84.                 ret += chunks[i] + ip_part
  85.         self.target = ret
  87.     def handle_input16(self):
  88.         self.target = md5_text(self.target)
  89.         self.target = self.split_sum(self.target[:16]) + self.target + self.split_sum(self.target[16:])
  91.     def handle_input8(self):
  92.         self.target = md5_text(self.target)
  93.         ret = ''
  94.         for i in range(4):
  95.             part = self.target[8 * i:8 * (i + 1)]
  96.             ret += self.split_sum(part) + part
  97.         self.target = ret
  99.     def handleSum(self):
  100.         self.target = md5_text(self.target)
  101.         self.target = self.split_sum(self.target) + self.target
  103.     def date(self, scheme):
  104.         self.target = md5_text(self.target)
  105.         d = time.localtime(self.timestamp)
  106.         strings = {
  107.             'y': str(d.tm_year),
  108.             'm': '%02d' % d.tm_mon,
  109.             'd': '%02d' % d.tm_mday,
  110.         }
  111.         self.target += ''.join(strings[c] for c in scheme)
  113.     def split_time_even_odd(self):
  114.         even, odd = self.even_odd()
  115.         self.target = odd + md5_text(self.target) + even
  117.     def split_time_odd_even(self):
  118.         even, odd = self.even_odd()
  119.         self.target = even + md5_text(self.target) + odd
  121.     def split_ip_time_sum(self):
  122.         chunks, ip = self.preprocess(32)
  123.         self.target = str(sum(ip)) + chunks[0] + self.digit_sum(self.timestamp)
  125.     def split_time_ip_sum(self):
  126.         chunks, ip = self.preprocess(32)
  127.         self.target = self.digit_sum(self.timestamp) + chunks[0] + str(sum(ip))
  130. class IqiyiSDKInterpreter:
  131.     def __init__(self, sdk_code):
  132.         self.sdk_code = sdk_code
  134.     def run(self, target, ip, timestamp):
  135.         self.sdk_code = decode_packed_codes(self.sdk_code)
  137.         functions = re.findall(r'input=([a-zA-Z0-9]+)\(input', self.sdk_code)
  139.         sdk = IqiyiSDK(target, ip, timestamp)
  141.         other_functions = {
  142.             'handleSum': sdk.handleSum,
  143.             'handleInput8': sdk.handle_input8,
  144.             'handleInput16': sdk.handle_input16,
  145.             'splitTimeEvenOdd': sdk.split_time_even_odd,
  146.             'splitTimeOddEven': sdk.split_time_odd_even,
  147.             'splitIpTimeSum': sdk.split_ip_time_sum,
  148.             'splitTimeIpSum': sdk.split_time_ip_sum,
  149.         }
  150.         for function in functions:
  151.             if re.match(r'mod\d+', function):
  152.                 sdk.mod(int(function[3:]))
  153.             elif re.match(r'date[ymd]{3}', function):
  154.                 sdk.date(function[4:])
  155.             elif re.match(r'split\d+', function):
  156.                 sdk.split(int(function[5:]))
  157.             elif function in other_functions:
  158.                 other_functions[function]()
  159.             else:
  160.                 raise ExtractorError(f'Unknown function {function}')
  162.         return sdk.target
  165. class IqiyiIE(InfoExtractor):
  166.     IE_NAME = 'iqiyi'
  167.     IE_DESC = '爱奇艺'
  169.     _VALID_URL = r'https?://(?:(?:[^.]+\.)?iqiyi\.com|www\.pps\.tv)/.+\.html'
  171.     _NETRC_MACHINE = 'iqiyi'
  173.     _TESTS = [{
  174.         'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
  175.         # MD5 checksum differs on my machine and Travis CI
  176.         'info_dict': {
  177.             'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
  178.             'ext': 'mp4',
  179.             'title': '美国德州空中惊现奇异云团 酷似UFO',
  180.         },
  181.     }, {
  182.         'url': 'http://www.iqiyi.com/v_19rrhnnclk.html',
  183.         'md5': 'b7dc800a4004b1b57749d9abae0472da',
  184.         'info_dict': {
  185.             'id': 'e3f585b550a280af23c98b6cb2be19fb',
  186.             'ext': 'mp4',
  187.             # This can be either Simplified Chinese or Traditional Chinese
  188.             'title': r're:^(?:名侦探柯南 国语版：第752集 迫近灰原秘密的黑影 下篇|名偵探柯南 國語版：第752集 迫近灰原秘密的黑影 下篇)$',
  189.         },
  190.         'skip': 'Geo-restricted to China',
  191.     }, {
  192.         'url': 'http://www.iqiyi.com/w_19rt6o8t9p.html',
  193.         'only_matching': True,
  194.     }, {
  195.         'url': 'http://www.iqiyi.com/a_19rrhbc6kt.html',
  196.         'only_matching': True,
  197.     }, {
  198.         'url': 'http://yule.iqiyi.com/pcb.html',
  199.         'info_dict': {
  200.             'id': '4a0af228fddb55ec96398a364248ed7f',
  201.             'ext': 'mp4',
  202.             'title': '第2017-04-21期 女艺人频遭极端粉丝骚扰',
  203.         },
  204.     }, {
  205.         # VIP-only video. The first 2 parts (6 minutes) are available without login
  206.         # MD5 sums omitted as values are different on Travis CI and my machine
  207.         'url': 'http://www.iqiyi.com/v_19rrny4w8w.html',
  208.         'info_dict': {
  209.             'id': 'f3cf468b39dddb30d676f89a91200dc1',
  210.             'ext': 'mp4',
  211.             'title': '泰坦尼克号',
  212.         },
  213.         'skip': 'Geo-restricted to China',
  214.     }, {
  215.         'url': 'http://www.iqiyi.com/a_19rrhb8ce1.html',
  216.         'info_dict': {
  217.             'id': '202918101',
  218.             'title': '灌篮高手 国语版',
  219.         },
  220.         'playlist_count': 101,
  221.     }, {
  222.         'url': 'http://www.pps.tv/w_19rrbav0ph.html',
  223.         'only_matching': True,
  224.     }]
  226.     _FORMATS_MAP = {
  227.         '96': 1,    # 216p, 240p
  228.         '1': 2,     # 336p, 360p
  229.         '2': 3,     # 480p, 504p
  230.         '21': 4,    # 504p
  231.         '4': 5,     # 720p
  232.         '17': 5,    # 720p
  233.         '5': 6,     # 1072p, 1080p
  234.         '18': 7,    # 1080p
  235.     }
  237.     @staticmethod
  238.     def _rsa_fun(data):
  239.         # public key extracted from http://static.iqiyi.com/js/qiyiV2/20160129180840/jobs/i18n/i18nIndex.js
  240.         N = 0xab86b6371b5318aaa1d3c9e612a9f1264f372323c8c0f19875b5fc3b3fd3afcc1e5bec527aa94bfa85bffc157e4245aebda05389a5357b75115ac94f074aefcd
  241.         e = 65537
  243.         return ohdave_rsa_encrypt(data, e, N)
  245.     def _perform_login(self, username, password):
  247.         data = self._download_json(
  248.             'http://kylin.iqiyi.com/get_token', None,
  249.             note='Get token for logging', errnote='Unable to get token for logging')
  250.         sdk = data['sdk']
  251.         timestamp = int(time.time())
  252.         target = (
  253.             f'/apis/reglogin/login.action?lang=zh_TW&area_code=null&email={username}'
  254.             f'&passwd={self._rsa_fun(password.encode())}&agenttype=1&from=undefined&keeplogin=0&piccode=&fromurl=&_pos=1')
  256.         interp = IqiyiSDKInterpreter(sdk)
  257.         sign = interp.run(target, data['ip'], timestamp)
  259.         validation_params = {
  260.             'target': target,
  261.             'server': 'BEA3AA1908656AABCCFF76582C4C6660',
  262.             'token': data['token'],
  263.             'bird_src': 'f8d91d57af224da7893dd397d52d811a',
  264.             'sign': sign,
  265.             'bird_t': timestamp,
  266.         }
  267.         validation_result = self._download_json(
  268.             'http://kylin.iqiyi.com/validate?' + urllib.parse.urlencode(validation_params), None,
  269.             note='Validate credentials', errnote='Unable to validate credentials')
  271.         MSG_MAP = {
  272.             'P00107': 'please login via the web interface and enter the CAPTCHA code',
  273.             'P00117': 'bad username or password',
  274.         }
  276.         code = validation_result['code']
  277.         if code != 'A00000':
  278.             msg = MSG_MAP.get(code)
  279.             if not msg:
  280.                 msg = f'error {code}'
  281.                 if validation_result.get('msg'):
  282.                     msg += ': ' + validation_result['msg']
  283.             self.report_warning('unable to log in: ' + msg)
  284.             return False
  286.         return True
  288.     def get_raw_data(self, tvid, video_id):
  289.         tm = int(time.time() * 1000)
  291.         key = 'd5fb4bd9d50c4be6948c97edd7254b0e'
  292.         sc = md5_text(str(tm) + key + tvid)
  293.         params = {
  294.             'tvid': tvid,
  295.             'vid': video_id,
  296.             'src': '76f90cbd92f94a2e925d83e8ccd22cb7',
  297.             'sc': sc,
  298.             't': tm,
  299.         }
  301.         return self._download_json(
  302.             f'http://cache.m.iqiyi.com/jp/tmts/{tvid}/{video_id}/',
  303.             video_id, transform_source=lambda s: remove_start(s, 'var tvInfoJs='),
  304.             query=params, headers=self.geo_verification_headers())
  306.     def _extract_playlist(self, webpage):
  307.         PAGE_SIZE = 50
  309.         links = re.findall(
  310.             r'<a[^>]+class="site-piclist_pic_link"[^>]+href="(http://www\.iqiyi\.com/.+\.html)"',
  311.             webpage)
  312.         if not links:
  313.             return
  315.         album_id = self._search_regex(
  316.             r'albumId\s*:\s*(\d+),', webpage, 'album ID')
  317.         album_title = self._search_regex(
  318.             r'data-share-title="([^"]+)"', webpage, 'album title', fatal=False)
  320.         entries = list(map(self.url_result, links))
  322.         # Start from 2 because links in the first page are already on webpage
  323.         for page_num in itertools.count(2):
  324.             pagelist_page = self._download_webpage(
  325.                 f'http://cache.video.qiyi.com/jp/avlist/{album_id}/{page_num}/{PAGE_SIZE}/',
  326.                 album_id,
  327.                 note=f'Download playlist page {page_num}',
  328.                 errnote=f'Failed to download playlist page {page_num}')
  329.             pagelist = self._parse_json(
  330.                 remove_start(pagelist_page, 'var tvInfoJs='), album_id)
  331.             vlist = pagelist['data']['vlist']
  332.             for item in vlist:
  333.                 entries.append(self.url_result(item['vurl']))
  334.             if len(vlist) < PAGE_SIZE:
  335.                 break
  337.         return self.playlist_result(entries, album_id, album_title)
  339.     def _real_extract(self, url):
  340.         webpage = self._download_webpage(
  341.             url, 'temp_id', note='download video page')
  343.         # There's no simple way to determine whether an URL is a playlist or not
  344.         # Sometimes there are playlist links in individual videos, so treat it
  345.         # as a single video first
  346.         tvid = self._search_regex(
  347.             r'data-(?:player|shareplattrigger)-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid', default=None)
  348.         if tvid is None:
  349.             playlist_result = self._extract_playlist(webpage)
  350.             if playlist_result:
  351.                 return playlist_result
  352.             raise ExtractorError('Can\'t find any video')
  354.         video_id = self._search_regex(
  355.             r'data-(?:player|shareplattrigger)-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
  357.         formats = []
  358.         for _ in range(5):
  359.             raw_data = self.get_raw_data(tvid, video_id)
  361.             if raw_data['code'] != 'A00000':
  362.                 if raw_data['code'] == 'A00111':
  363.                     self.raise_geo_restricted()
  364.                 raise ExtractorError('Unable to load data. Error code: ' + raw_data['code'])
  366.             data = raw_data['data']
  368.             for stream in data['vidl']:
  369.                 if 'm3utx' not in stream:
  370.                     continue
  371.                 vd = str(stream['vd'])
  372.                 formats.append({
  373.                     'url': stream['m3utx'],
  374.                     'format_id': vd,
  375.                     'ext': 'mp4',
  376.                     'quality': self._FORMATS_MAP.get(vd, -1),
  377.                     'protocol': 'm3u8_native',
  378.                 })
  380.             if formats:
  381.                 break
  383.             self._sleep(5, video_id)
  385.         title = (get_element_by_id('widget-videotitle', webpage)
  386.                  or clean_html(get_element_by_attribute('class', 'mod-play-tit', webpage))
  387.                  or self._html_search_regex(r'<span[^>]+data-videochanged-title="word"[^>]*>([^<]+)</span>', webpage, 'title'))
  389.         return {
  390.             'id': video_id,
  391.             'title': title,
  392.             'formats': formats,
  393.         }
  396. class IqIE(InfoExtractor):
  397.     IE_NAME = 'iq.com'
  398.     IE_DESC = 'International version of iQiyi'
  399.     _VALID_URL = r'https?://(?:www\.)?iq\.com/play/(?:[\w%-]*-)?(?P<id>\w+)'
  400.     _TESTS = [{
  401.         'url': 'https://www.iq.com/play/one-piece-episode-1000-1ma1i6ferf4',
  402.         'md5': '2d7caf6eeca8a32b407094b33b757d39',
  403.         'info_dict': {
  404.             'ext': 'mp4',
  405.             'id': '1ma1i6ferf4',
  406.             'title': '航海王 第1000集',
  407.             'description': 'Subtitle available on Sunday 4PM（GMT+8）.',
  408.             'duration': 1430,
  409.             'timestamp': 1637488203,
  410.             'upload_date': '20211121',
  411.             'episode_number': 1000,
  412.             'episode': 'Episode 1000',
  413.             'series': 'One Piece',
  414.             'age_limit': 13,
  415.             'average_rating': float,
  416.         },
  417.         'params': {
  418.             'format': '500',
  419.         },
  420.         'expected_warnings': ['format is restricted'],
  421.     }, {
  422.         # VIP-restricted video
  423.         'url': 'https://www.iq.com/play/mermaid-in-the-fog-2021-gbdpx13bs4',
  424.         'only_matching': True,
  425.     }]
  426.     _BID_TAGS = {
  427.         '100': '240P',
  428.         '200': '360P',
  429.         '300': '480P',
  430.         '500': '720P',
  431.         '600': '1080P',
  432.         '610': '1080P50',
  433.         '700': '2K',
  434.         '800': '4K',
  435.     }
  436.     _LID_TAGS = {
  437.         '1': 'zh_CN',
  438.         '2': 'zh_TW',
  439.         '3': 'en',
  440.         '4': 'ko',
  441.         '5': 'ja',
  442.         '18': 'th',
  443.         '21': 'my',
  444.         '23': 'vi',
  445.         '24': 'id',
  446.         '26': 'es',
  447.         '27': 'pt',
  448.         '28': 'ar',
  449.     }
  451.     _DASH_JS = '''
  452.         console.log(page.evaluate(function() {
  453.             var tvid = "%(tvid)s"; var vid = "%(vid)s"; var src = "%(src)s";
  454.             var uid = "%(uid)s"; var dfp = "%(dfp)s"; var mode = "%(mode)s"; var lang = "%(lang)s";
  455.             var bid_list = %(bid_list)s; var ut_list = %(ut_list)s; var tm = new Date().getTime();
  456.             var cmd5x_func = %(cmd5x_func)s; var cmd5x_exporter = {}; cmd5x_func({}, cmd5x_exporter, {}); var cmd5x = cmd5x_exporter.cmd5x;
  457.             var authKey = cmd5x(cmd5x('') + tm + '' + tvid);
  458.             var k_uid = Array.apply(null, Array(32)).map(function() {return Math.floor(Math.random() * 15).toString(16)}).join('');
  459.             var dash_paths = {};
  460.             bid_list.forEach(function(bid) {
  461.                 var query = {
  462.                     'tvid': tvid,
  463.                     'bid': bid,
  464.                     'ds': 1,
  465.                     'vid': vid,
  466.                     'src': src,
  467.                     'vt': 0,
  468.                     'rs': 1,
  469.                     'uid': uid,
  470.                     'ori': 'pcw',
  471.                     'ps': 1,
  472.                     'k_uid': k_uid,
  473.                     'pt': 0,
  474.                     'd': 0,
  475.                     's': '',
  476.                     'lid': '',
  477.                     'slid': 0,
  478.                     'cf': '',
  479.                     'ct': '',
  480.                     'authKey': authKey,
  481.                     'k_tag': 1,
  482.                     'ost': 0,
  483.                     'ppt': 0,
  484.                     'dfp': dfp,
  485.                     'prio': JSON.stringify({
  486.                         'ff': 'f4v',
  487.                         'code': 2
  488.                     }),
  489.                     'k_err_retries': 0,
  490.                     'up': '',
  491.                     'su': 2,
  492.                     'applang': lang,
  493.                     'sver': 2,
  494.                     'X-USER-MODE': mode,
  495.                     'qd_v': 2,
  496.                     'tm': tm,
  497.                     'qdy': 'a',
  498.                     'qds': 0,
  499.                     'k_ft1': '143486267424900',
  500.                     'k_ft4': '1572868',
  501.                     'k_ft7': '4',
  502.                     'k_ft5': '1',
  503.                     'bop': JSON.stringify({
  504.                         'version': '10.0',
  505.                         'dfp': dfp
  506.                     }),
  507.                 };
  508.                 var enc_params = [];
  509.                 for (var prop in query) {
  510.                     enc_params.push(encodeURIComponent(prop) + '=' + encodeURIComponent(query[prop]));
  511.                 }
  512.                 ut_list.forEach(function(ut) {
  513.                     enc_params.push('ut=' + ut);
  514.                 })
  515.                 var dash_path = '/dash?' + enc_params.join('&'); dash_path += '&vf=' + cmd5x(dash_path);
  516.                 dash_paths[bid] = dash_path;
  517.             });
  518.             return JSON.stringify(dash_paths);
  519.         }));
  520.         saveAndExit();
  521.     '''
  523.     def _extract_vms_player_js(self, webpage, video_id):
  524.         player_js_cache = self.cache.load('iq', 'player_js')
  525.         if player_js_cache:
  526.             return player_js_cache
  527.         webpack_js_url = self._proto_relative_url(self._search_regex(
  528.             r'<script src="((?:https?:)?//stc\.iqiyipic\.com/_next/static/chunks/webpack-\w+\.js)"', webpage, 'webpack URL'))
  529.         webpack_js = self._download_webpage(webpack_js_url, video_id, note='Downloading webpack JS', errnote='Unable to download webpack JS')
  531.         webpack_map = self._search_json(
  532.             r'["\']\s*\+\s*', webpack_js, 'JS locations', video_id,
  533.             contains_pattern=r'{\s*(?:\d+\s*:\s*["\'][\da-f]+["\']\s*,?\s*)+}',
  534.             end_pattern=r'\[\w+\]\+["\']\.js', transform_source=js_to_json)
  536.         replacement_map = self._search_json(
  537.             r'["\']\s*\+\(\s*', webpack_js, 'replacement map', video_id,
  538.             contains_pattern=r'{\s*(?:\d+\s*:\s*["\'][\w.-]+["\']\s*,?\s*)+}',
  539.             end_pattern=r'\[\w+\]\|\|\w+\)\+["\']\.', transform_source=js_to_json,
  540.             fatal=False) or {}
  542.         for module_index in reversed(webpack_map):
  543.             real_module = replacement_map.get(module_index) or module_index
  544.             module_js = self._download_webpage(
  545.                 f'https://stc.iqiyipic.com/_next/static/chunks/{real_module}.{webpack_map[module_index]}.js',
  546.                 video_id, note=f'Downloading #{module_index} module JS', errnote='Unable to download module JS', fatal=False) or ''
  547.             if 'vms request' in module_js:
  548.                 self.cache.store('iq', 'player_js', module_js)
  549.                 return module_js
  550.         raise ExtractorError('Unable to extract player JS')
  552.     def _extract_cmd5x_function(self, webpage, video_id):
  553.         return self._search_regex(r',\s*(function\s*\([^\)]*\)\s*{\s*var _qda.+_qdc\(\)\s*})\s*,',
  554.                                   self._extract_vms_player_js(webpage, video_id), 'signature function')
  556.     def _update_bid_tags(self, webpage, video_id):
  557.         extracted_bid_tags = self._search_json(
  558.             r'function\s*\([^)]*\)\s*\{\s*"use strict";?\s*var \w\s*=\s*',
  559.             self._extract_vms_player_js(webpage, video_id), 'video tags', video_id,
  560.             contains_pattern=r'{\s*\d+\s*:\s*\{\s*nbid\s*:.+}\s*}',
  561.             end_pattern=r'\s*,\s*\w\s*=\s*\{\s*getNewVd', fatal=False, transform_source=js_to_json)
  562.         if not extracted_bid_tags:
  563.             return
  564.         self._BID_TAGS = {
  565.             bid: traverse_obj(extracted_bid_tags, (bid, 'value'), expected_type=str, default=self._BID_TAGS.get(bid))
  566.             for bid in extracted_bid_tags
  567.         }
  569.     def _get_cookie(self, name, default=None):
  570.         cookie = self._get_cookies('https://iq.com/').get(name)
  571.         return cookie.value if cookie else default
  573.     def _real_extract(self, url):
  574.         video_id = self._match_id(url)
  575.         webpage = self._download_webpage(url, video_id)
  576.         self._update_bid_tags(webpage, video_id)
  578.         next_props = self._search_nextjs_data(webpage, video_id)['props']
  579.         page_data = next_props['initialState']['play']
  580.         video_info = page_data['curVideoInfo']
  582.         uid = traverse_obj(
  583.             self._parse_json(
  584.                 self._get_cookie('I00002', '{}'), video_id, transform_source=urllib.parse.unquote, fatal=False),
  585.             ('data', 'uid'), default=0)
  587.         if uid:
  588.             vip_data = self._download_json(
  589.                 'https://pcw-api.iq.com/api/vtype', video_id, note='Downloading VIP data', errnote='Unable to download VIP data', query={
  590.                     'batch': 1,
  591.                     'platformId': 3,
  592.                     'modeCode': self._get_cookie('mod', 'intl'),
  593.                     'langCode': self._get_cookie('lang', 'en_us'),
  594.                     'deviceId': self._get_cookie('QC005', ''),
  595.                 }, fatal=False)
  596.             ut_list = traverse_obj(vip_data, ('data', 'all_vip', ..., 'vipType'), expected_type=str_or_none)
  597.         else:
  598.             ut_list = ['0']
  600.         # bid 0 as an initial format checker
  601.         dash_paths = self._parse_json(PhantomJSwrapper(self, timeout=120_000).get(
  602.             url, note2='Executing signature code (this may take a couple minutes)',
  603.             html='<!DOCTYPE html>', video_id=video_id, jscode=self._DASH_JS % {
  604.                 'tvid': video_info['tvId'],
  605.                 'vid': video_info['vid'],
  606.                 'src': traverse_obj(next_props, ('initialProps', 'pageProps', 'ptid'),
  607.                                     expected_type=str, default='04022001010011000000'),
  608.                 'uid': uid,
  609.                 'dfp': self._get_cookie('dfp', ''),
  610.                 'mode': self._get_cookie('mod', 'intl'),
  611.                 'lang': self._get_cookie('lang', 'en_us'),
  612.                 'bid_list': '[' + ','.join(['0', *self._BID_TAGS.keys()]) + ']',
  613.                 'ut_list': '[' + ','.join(ut_list) + ']',
  614.                 'cmd5x_func': self._extract_cmd5x_function(webpage, video_id),
  615.             })[1].strip(), video_id)
  617.         formats, subtitles = [], {}
  618.         initial_format_data = self._download_json(
  619.             urljoin('https://cache-video.iq.com', dash_paths['0']), video_id,
  620.             note='Downloading initial video format info', errnote='Unable to download initial video format info')['data']
  622.         preview_time = traverse_obj(
  623.             initial_format_data, ('boss_ts', (None, 'data'), ('previewTime', 'rtime')), expected_type=float_or_none, get_all=False)
  624.         if traverse_obj(initial_format_data, ('boss_ts', 'data', 'prv'), expected_type=int_or_none):
  625.             self.report_warning('This preview video is limited{}'.format(format_field(preview_time, None, ' to %s seconds')))
  627.         # TODO: Extract audio-only formats
  628.         for bid in set(traverse_obj(initial_format_data, ('program', 'video', ..., 'bid'), expected_type=str_or_none)):
  629.             dash_path = dash_paths.get(bid)
  630.             if not dash_path:
  631.                 self.report_warning(f'Unknown format id: {bid}. It is currently not being extracted')
  632.                 continue
  633.             format_data = traverse_obj(self._download_json(
  634.                 urljoin('https://cache-video.iq.com', dash_path), video_id,
  635.                 note=f'Downloading format data for {self._BID_TAGS[bid]}', errnote='Unable to download format data',
  636.                 fatal=False), 'data', expected_type=dict)
  638.             video_format = traverse_obj(format_data, ('program', 'video', lambda _, v: str(v['bid']) == bid),
  639.                                         expected_type=dict, get_all=False) or {}
  640.             extracted_formats = []
  641.             if video_format.get('m3u8Url'):
  642.                 extracted_formats.extend(self._extract_m3u8_formats(
  643.                     urljoin(format_data.get('dm3u8', 'https://cache-m.iq.com/dc/dt/'), video_format['m3u8Url']),
  644.                     'mp4', m3u8_id=bid, fatal=False))
  645.             if video_format.get('mpdUrl'):
  646.                 # TODO: Properly extract mpd hostname
  647.                 extracted_formats.extend(self._extract_mpd_formats(
  648.                     urljoin(format_data.get('dm3u8', 'https://cache-m.iq.com/dc/dt/'), video_format['mpdUrl']),
  649.                     mpd_id=bid, fatal=False))
  650.             if video_format.get('m3u8'):
  651.                 ff = video_format.get('ff', 'ts')
  652.                 if ff == 'ts':
  653.                     m3u8_formats, _ = self._parse_m3u8_formats_and_subtitles(
  654.                         video_format['m3u8'], ext='mp4', m3u8_id=bid, fatal=False)
  655.                     extracted_formats.extend(m3u8_formats)
  656.                 elif ff == 'm4s':
  657.                     mpd_data = traverse_obj(
  658.                         self._parse_json(video_format['m3u8'], video_id, fatal=False), ('payload', ..., 'data'), expected_type=str)
  659.                     if not mpd_data:
  660.                         continue
  661.                     mpd_formats, _ = self._parse_mpd_formats_and_subtitles(
  662.                         mpd_data, bid, format_data.get('dm3u8', 'https://cache-m.iq.com/dc/dt/'))
  663.                     extracted_formats.extend(mpd_formats)
  664.                 else:
  665.                     self.report_warning(f'{ff} formats are currently not supported')
  667.             if not extracted_formats:
  668.                 if video_format.get('s'):
  669.                     self.report_warning(f'{self._BID_TAGS[bid]} format is restricted')
  670.                 else:
  671.                     self.report_warning(f'Unable to extract {self._BID_TAGS[bid]} format')
  672.             for f in extracted_formats:
  673.                 f.update({
  674.                     'quality': qualities(list(self._BID_TAGS.keys()))(bid),
  675.                     'format_note': self._BID_TAGS[bid],
  676.                     **parse_resolution(video_format.get('scrsz')),
  677.                 })
  678.             formats.extend(extracted_formats)
  680.         for sub_format in traverse_obj(initial_format_data, ('program', 'stl', ...), expected_type=dict):
  681.             lang = self._LID_TAGS.get(str_or_none(sub_format.get('lid')), sub_format.get('_name'))
  682.             subtitles.setdefault(lang, []).extend([{
  683.                 'ext': format_ext,
  684.                 'url': urljoin(initial_format_data.get('dstl', 'http://meta.video.iqiyi.com'), sub_format[format_key]),
  685.             } for format_key, format_ext in [('srt', 'srt'), ('webvtt', 'vtt')] if sub_format.get(format_key)])
  687.         extra_metadata = page_data.get('albumInfo') if video_info.get('albumId') and page_data.get('albumInfo') else video_info
  688.         return {
  689.             'id': video_id,
  690.             'title': video_info['name'],
  691.             'formats': formats,
  692.             'subtitles': subtitles,
  693.             'description': video_info.get('mergeDesc'),
  694.             'duration': parse_duration(video_info.get('len')),
  695.             'age_limit': parse_age_limit(video_info.get('rating')),
  696.             'average_rating': traverse_obj(page_data, ('playScoreInfo', 'score'), expected_type=float_or_none),
  697.             'timestamp': parse_iso8601(video_info.get('isoUploadDate')),
  698.             'categories': traverse_obj(extra_metadata, ('videoTagMap', ..., ..., 'name'), expected_type=str),
  699.             'cast': traverse_obj(extra_metadata, ('actorArr', ..., 'name'), expected_type=str),
  700.             'episode_number': int_or_none(video_info.get('order')) or None,
  701.             'series': video_info.get('albumName'),
  702.         }
  705. class IqAlbumIE(InfoExtractor):
  706.     IE_NAME = 'iq.com:album'
  707.     _VALID_URL = r'https?://(?:www\.)?iq\.com/album/(?:[\w%-]*-)?(?P<id>\w+)'
  708.     _TESTS = [{
  709.         'url': 'https://www.iq.com/album/one-piece-1999-1bk9icvr331',
  710.         'info_dict': {
  711.             'id': '1bk9icvr331',
  712.             'title': 'One Piece',
  713.             'description': 'Subtitle available on Sunday 4PM（GMT+8）.',
  714.         },
  715.         'playlist_mincount': 238,
  716.     }, {
  717.         # Movie/single video
  718.         'url': 'https://www.iq.com/album/九龙城寨-2021-22yjnij099k',
  719.         'info_dict': {
  720.             'ext': 'mp4',
  721.             'id': '22yjnij099k',
  722.             'title': '九龙城寨',
  723.             'description': 'md5:8a09f50b8ba0db4dc69bc7c844228044',
  724.             'duration': 5000,
  725.             'timestamp': 1641911371,
  726.             'upload_date': '20220111',
  727.             'series': '九龙城寨',
  728.             'cast': ['Shi Yan Neng', 'Yu Lang', 'Peter  lv', 'Sun Zi Jun', 'Yang Xiao Bo'],
  729.             'age_limit': 13,
  730.             'average_rating': float,
  731.         },
  732.         'expected_warnings': ['format is restricted'],
  733.     }]
  735.     def _entries(self, album_id_num, page_ranges, album_id=None, mode_code='intl', lang_code='en_us'):
  736.         for page_range in page_ranges:
  737.             page = self._download_json(
  738.                 f'https://pcw-api.iq.com/api/episodeListSource/{album_id_num}', album_id,
  739.                 note=f'Downloading video list episodes {page_range.get("msg", "")}',
  740.                 errnote='Unable to download video list', query={
  741.                     'platformId': 3,
  742.                     'modeCode': mode_code,
  743.                     'langCode': lang_code,
  744.                     'endOrder': page_range['to'],
  745.                     'startOrder': page_range['from'],
  746.                 })
  747.             for video in page['data']['epg']:
  748.                 yield self.url_result('https://www.iq.com/play/%s' % (video.get('playLocSuffix') or video['qipuIdStr']),
  749.                                       IqIE.ie_key(), video.get('qipuIdStr'), video.get('name'))
  751.     def _real_extract(self, url):
  752.         album_id = self._match_id(url)
  753.         webpage = self._download_webpage(url, album_id)
  754.         next_data = self._search_nextjs_data(webpage, album_id)
  755.         album_data = next_data['props']['initialState']['album']['videoAlbumInfo']
  757.         if album_data.get('videoType') == 'singleVideo':
  758.             return self.url_result(f'https://www.iq.com/play/{album_id}', IqIE.ie_key())
  759.         return self.playlist_result(
  760.             self._entries(album_data['albumId'], album_data['totalPageRange'], album_id,
  761.                           traverse_obj(next_data, ('props', 'initialProps', 'pageProps', 'modeCode')),
  762.                           traverse_obj(next_data, ('props', 'initialProps', 'pageProps', 'langCode'))),
  763.             album_id, album_data.get('name'), album_data.get('desc'))