logo

youtube-dl

[mirror] Download/Watch videos from video hostersgit clone https://hacktivis.me/git/mirror/youtube-dl.git

external.py (21005B)


  1. from __future__ import unicode_literals
  2. import os
  3. import re
  4. import subprocess
  5. import sys
  6. import tempfile
  7. import time
  8. from .common import FileDownloader
  9. from ..compat import (
  10. compat_setenv,
  11. compat_str,
  12. )
  13. from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
  14. from ..utils import (
  15. cli_option,
  16. cli_valueless_option,
  17. cli_bool_option,
  18. cli_configuration_args,
  19. encodeFilename,
  20. encodeArgument,
  21. handle_youtubedl_headers,
  22. check_executable,
  23. is_outdated_version,
  24. process_communicate_or_kill,
  25. T,
  26. traverse_obj,
  27. )
  28. class ExternalFD(FileDownloader):
  29. def real_download(self, filename, info_dict):
  30. self.report_destination(filename)
  31. tmpfilename = self.temp_name(filename)
  32. self._cookies_tempfile = None
  33. try:
  34. started = time.time()
  35. retval = self._call_downloader(tmpfilename, info_dict)
  36. except KeyboardInterrupt:
  37. if not info_dict.get('is_live'):
  38. raise
  39. # Live stream downloading cancellation should be considered as
  40. # correct and expected termination thus all postprocessing
  41. # should take place
  42. retval = 0
  43. self.to_screen('[%s] Interrupted by user' % self.get_basename())
  44. finally:
  45. if self._cookies_tempfile and os.path.isfile(self._cookies_tempfile):
  46. try:
  47. os.remove(self._cookies_tempfile)
  48. except OSError:
  49. self.report_warning(
  50. 'Unable to delete temporary cookies file "{0}"'.format(self._cookies_tempfile))
  51. if retval == 0:
  52. status = {
  53. 'filename': filename,
  54. 'status': 'finished',
  55. 'elapsed': time.time() - started,
  56. }
  57. if filename != '-':
  58. fsize = os.path.getsize(encodeFilename(tmpfilename))
  59. self.to_screen('\r[%s] Downloaded %s bytes' % (self.get_basename(), fsize))
  60. self.try_rename(tmpfilename, filename)
  61. status.update({
  62. 'downloaded_bytes': fsize,
  63. 'total_bytes': fsize,
  64. })
  65. self._hook_progress(status)
  66. return True
  67. else:
  68. self.to_stderr('\n')
  69. self.report_error('%s exited with code %d' % (
  70. self.get_basename(), retval))
  71. return False
  72. @classmethod
  73. def get_basename(cls):
  74. return cls.__name__[:-2].lower()
  75. @property
  76. def exe(self):
  77. return self.params.get('external_downloader')
  78. @classmethod
  79. def available(cls):
  80. return check_executable(cls.get_basename(), [cls.AVAILABLE_OPT])
  81. @classmethod
  82. def supports(cls, info_dict):
  83. return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
  84. @classmethod
  85. def can_download(cls, info_dict):
  86. return cls.available() and cls.supports(info_dict)
  87. def _option(self, command_option, param):
  88. return cli_option(self.params, command_option, param)
  89. def _bool_option(self, command_option, param, true_value='true', false_value='false', separator=None):
  90. return cli_bool_option(self.params, command_option, param, true_value, false_value, separator)
  91. def _valueless_option(self, command_option, param, expected_value=True):
  92. return cli_valueless_option(self.params, command_option, param, expected_value)
  93. def _configuration_args(self, default=[]):
  94. return cli_configuration_args(self.params, 'external_downloader_args', default)
  95. def _write_cookies(self):
  96. if not self.ydl.cookiejar.filename:
  97. tmp_cookies = tempfile.NamedTemporaryFile(suffix='.cookies', delete=False)
  98. tmp_cookies.close()
  99. self._cookies_tempfile = tmp_cookies.name
  100. self.to_screen('[download] Writing temporary cookies file to "{0}"'.format(self._cookies_tempfile))
  101. # real_download resets _cookies_tempfile; if it's None, save() will write to cookiejar.filename
  102. self.ydl.cookiejar.save(self._cookies_tempfile, ignore_discard=True, ignore_expires=True)
  103. return self.ydl.cookiejar.filename or self._cookies_tempfile
  104. def _call_downloader(self, tmpfilename, info_dict):
  105. """ Either overwrite this or implement _make_cmd """
  106. cmd = [encodeArgument(a) for a in self._make_cmd(tmpfilename, info_dict)]
  107. self._debug_cmd(cmd)
  108. p = subprocess.Popen(
  109. cmd, stderr=subprocess.PIPE)
  110. _, stderr = process_communicate_or_kill(p)
  111. if p.returncode != 0:
  112. self.to_stderr(stderr.decode('utf-8', 'replace'))
  113. return p.returncode
  114. @staticmethod
  115. def _header_items(info_dict):
  116. return traverse_obj(
  117. info_dict, ('http_headers', T(dict.items), Ellipsis))
  118. class CurlFD(ExternalFD):
  119. AVAILABLE_OPT = '-V'
  120. def _make_cmd(self, tmpfilename, info_dict):
  121. cmd = [self.exe, '--location', '-o', tmpfilename, '--compressed']
  122. cookie_header = self.ydl.cookiejar.get_cookie_header(info_dict['url'])
  123. if cookie_header:
  124. cmd += ['--cookie', cookie_header]
  125. for key, val in self._header_items(info_dict):
  126. cmd += ['--header', '%s: %s' % (key, val)]
  127. cmd += self._bool_option('--continue-at', 'continuedl', '-', '0')
  128. cmd += self._valueless_option('--silent', 'noprogress')
  129. cmd += self._valueless_option('--verbose', 'verbose')
  130. cmd += self._option('--limit-rate', 'ratelimit')
  131. retry = self._option('--retry', 'retries')
  132. if len(retry) == 2:
  133. if retry[1] in ('inf', 'infinite'):
  134. retry[1] = '2147483647'
  135. cmd += retry
  136. cmd += self._option('--max-filesize', 'max_filesize')
  137. cmd += self._option('--interface', 'source_address')
  138. cmd += self._option('--proxy', 'proxy')
  139. cmd += self._valueless_option('--insecure', 'nocheckcertificate')
  140. cmd += self._configuration_args()
  141. cmd += ['--', info_dict['url']]
  142. return cmd
  143. def _call_downloader(self, tmpfilename, info_dict):
  144. cmd = [encodeArgument(a) for a in self._make_cmd(tmpfilename, info_dict)]
  145. self._debug_cmd(cmd)
  146. # curl writes the progress to stderr so don't capture it.
  147. p = subprocess.Popen(cmd)
  148. process_communicate_or_kill(p)
  149. return p.returncode
  150. class AxelFD(ExternalFD):
  151. AVAILABLE_OPT = '-V'
  152. def _make_cmd(self, tmpfilename, info_dict):
  153. cmd = [self.exe, '-o', tmpfilename]
  154. for key, val in self._header_items(info_dict):
  155. cmd += ['-H', '%s: %s' % (key, val)]
  156. cookie_header = self.ydl.cookiejar.get_cookie_header(info_dict['url'])
  157. if cookie_header:
  158. cmd += ['-H', 'Cookie: {0}'.format(cookie_header), '--max-redirect=0']
  159. cmd += self._configuration_args()
  160. cmd += ['--', info_dict['url']]
  161. return cmd
  162. class WgetFD(ExternalFD):
  163. AVAILABLE_OPT = '--version'
  164. def _make_cmd(self, tmpfilename, info_dict):
  165. cmd = [self.exe, '-O', tmpfilename, '-nv', '--compression=auto']
  166. if self.ydl.cookiejar.get_cookie_header(info_dict['url']):
  167. cmd += ['--load-cookies', self._write_cookies()]
  168. for key, val in self._header_items(info_dict):
  169. cmd += ['--header', '%s: %s' % (key, val)]
  170. cmd += self._option('--limit-rate', 'ratelimit')
  171. retry = self._option('--tries', 'retries')
  172. if len(retry) == 2:
  173. if retry[1] in ('inf', 'infinite'):
  174. retry[1] = '0'
  175. cmd += retry
  176. cmd += self._option('--bind-address', 'source_address')
  177. proxy = self.params.get('proxy')
  178. if proxy:
  179. for var in ('http_proxy', 'https_proxy'):
  180. cmd += ['--execute', '%s=%s' % (var, proxy)]
  181. cmd += self._valueless_option('--no-check-certificate', 'nocheckcertificate')
  182. cmd += self._configuration_args()
  183. cmd += ['--', info_dict['url']]
  184. return cmd
  185. class Aria2cFD(ExternalFD):
  186. AVAILABLE_OPT = '-v'
  187. @staticmethod
  188. def _aria2c_filename(fn):
  189. return fn if os.path.isabs(fn) else os.path.join('.', fn)
  190. def _make_cmd(self, tmpfilename, info_dict):
  191. cmd = [self.exe, '-c',
  192. '--console-log-level=warn', '--summary-interval=0', '--download-result=hide',
  193. '--http-accept-gzip=true', '--file-allocation=none', '-x16', '-j16', '-s16']
  194. if 'fragments' in info_dict:
  195. cmd += ['--allow-overwrite=true', '--allow-piece-length-change=true']
  196. else:
  197. cmd += ['--min-split-size', '1M']
  198. if self.ydl.cookiejar.get_cookie_header(info_dict['url']):
  199. cmd += ['--load-cookies={0}'.format(self._write_cookies())]
  200. for key, val in self._header_items(info_dict):
  201. cmd += ['--header', '%s: %s' % (key, val)]
  202. cmd += self._configuration_args(['--max-connection-per-server', '4'])
  203. cmd += ['--out', os.path.basename(tmpfilename)]
  204. cmd += self._option('--max-overall-download-limit', 'ratelimit')
  205. cmd += self._option('--interface', 'source_address')
  206. cmd += self._option('--all-proxy', 'proxy')
  207. cmd += self._bool_option('--check-certificate', 'nocheckcertificate', 'false', 'true', '=')
  208. cmd += self._bool_option('--remote-time', 'updatetime', 'true', 'false', '=')
  209. cmd += self._bool_option('--show-console-readout', 'noprogress', 'false', 'true', '=')
  210. cmd += self._configuration_args()
  211. # aria2c strips out spaces from the beginning/end of filenames and paths.
  212. # We work around this issue by adding a "./" to the beginning of the
  213. # filename and relative path, and adding a "/" at the end of the path.
  214. # See: https://github.com/yt-dlp/yt-dlp/issues/276
  215. # https://github.com/ytdl-org/youtube-dl/issues/20312
  216. # https://github.com/aria2/aria2/issues/1373
  217. dn = os.path.dirname(tmpfilename)
  218. if dn:
  219. cmd += ['--dir', self._aria2c_filename(dn) + os.path.sep]
  220. if 'fragments' not in info_dict:
  221. cmd += ['--out', self._aria2c_filename(os.path.basename(tmpfilename))]
  222. cmd += ['--auto-file-renaming=false']
  223. if 'fragments' in info_dict:
  224. cmd += ['--file-allocation=none', '--uri-selector=inorder']
  225. url_list_file = '%s.frag.urls' % (tmpfilename, )
  226. url_list = []
  227. for frag_index, fragment in enumerate(info_dict['fragments']):
  228. fragment_filename = '%s-Frag%d' % (os.path.basename(tmpfilename), frag_index)
  229. url_list.append('%s\n\tout=%s' % (fragment['url'], self._aria2c_filename(fragment_filename)))
  230. stream, _ = self.sanitize_open(url_list_file, 'wb')
  231. stream.write('\n'.join(url_list).encode())
  232. stream.close()
  233. cmd += ['-i', self._aria2c_filename(url_list_file)]
  234. else:
  235. cmd += ['--', info_dict['url']]
  236. return cmd
  237. class Aria2pFD(ExternalFD):
  238. ''' Aria2pFD class
  239. This class support to use aria2p as downloader.
  240. (Aria2p, a command-line tool and Python library to interact with an aria2c daemon process
  241. through JSON-RPC.)
  242. It can help you to get download progress more easily.
  243. To use aria2p as downloader, you need to install aria2c and aria2p, aria2p can download with pip.
  244. Then run aria2c in the background and enable with the --enable-rpc option.
  245. '''
  246. try:
  247. import aria2p
  248. __avail = True
  249. except ImportError:
  250. __avail = False
  251. @classmethod
  252. def available(cls):
  253. return cls.__avail
  254. def _call_downloader(self, tmpfilename, info_dict):
  255. aria2 = self.aria2p.API(
  256. self.aria2p.Client(
  257. host='http://localhost',
  258. port=6800,
  259. secret=''
  260. )
  261. )
  262. options = {
  263. 'min-split-size': '1M',
  264. 'max-connection-per-server': 4,
  265. 'auto-file-renaming': 'false',
  266. }
  267. options['dir'] = os.path.dirname(tmpfilename) or os.path.abspath('.')
  268. options['out'] = os.path.basename(tmpfilename)
  269. if self.ydl.cookiejar.get_cookie_header(info_dict['url']):
  270. options['load-cookies'] = self._write_cookies()
  271. options['header'] = []
  272. for key, val in self._header_items(info_dict):
  273. options['header'].append('{0}: {1}'.format(key, val))
  274. download = aria2.add_uris([info_dict['url']], options)
  275. status = {
  276. 'status': 'downloading',
  277. 'tmpfilename': tmpfilename,
  278. }
  279. started = time.time()
  280. while download.status in ['active', 'waiting']:
  281. download = aria2.get_download(download.gid)
  282. status.update({
  283. 'downloaded_bytes': download.completed_length,
  284. 'total_bytes': download.total_length,
  285. 'elapsed': time.time() - started,
  286. 'eta': download.eta.total_seconds(),
  287. 'speed': download.download_speed,
  288. })
  289. self._hook_progress(status)
  290. time.sleep(.5)
  291. return download.status != 'complete'
  292. class HttpieFD(ExternalFD):
  293. @classmethod
  294. def available(cls):
  295. return check_executable('http', ['--version'])
  296. def _make_cmd(self, tmpfilename, info_dict):
  297. cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
  298. for key, val in self._header_items(info_dict):
  299. cmd += ['%s:%s' % (key, val)]
  300. # httpie 3.1.0+ removes the Cookie header on redirect, so this should be safe for now. [1]
  301. # If we ever need cookie handling for redirects, we can export the cookiejar into a session. [2]
  302. # 1: https://github.com/httpie/httpie/security/advisories/GHSA-9w4w-cpc8-h2fq
  303. # 2: https://httpie.io/docs/cli/sessions
  304. cookie_header = self.ydl.cookiejar.get_cookie_header(info_dict['url'])
  305. if cookie_header:
  306. cmd += ['Cookie:%s' % cookie_header]
  307. return cmd
  308. class FFmpegFD(ExternalFD):
  309. @classmethod
  310. def supports(cls, info_dict):
  311. return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms', 'http_dash_segments')
  312. @classmethod
  313. def available(cls):
  314. return FFmpegPostProcessor().available
  315. def _call_downloader(self, tmpfilename, info_dict):
  316. url = info_dict['url']
  317. ffpp = FFmpegPostProcessor(downloader=self)
  318. if not ffpp.available:
  319. self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
  320. return False
  321. ffpp.check_version()
  322. args = [ffpp.executable, '-y']
  323. for log_level in ('quiet', 'verbose'):
  324. if self.params.get(log_level, False):
  325. args += ['-loglevel', log_level]
  326. break
  327. seekable = info_dict.get('_seekable')
  328. if seekable is not None:
  329. # setting -seekable prevents ffmpeg from guessing if the server
  330. # supports seeking(by adding the header `Range: bytes=0-`), which
  331. # can cause problems in some cases
  332. # https://github.com/ytdl-org/youtube-dl/issues/11800#issuecomment-275037127
  333. # http://trac.ffmpeg.org/ticket/6125#comment:10
  334. args += ['-seekable', '1' if seekable else '0']
  335. args += self._configuration_args()
  336. # start_time = info_dict.get('start_time') or 0
  337. # if start_time:
  338. # args += ['-ss', compat_str(start_time)]
  339. # end_time = info_dict.get('end_time')
  340. # if end_time:
  341. # args += ['-t', compat_str(end_time - start_time)]
  342. cookies = self.ydl.cookiejar.get_cookies_for_url(url)
  343. if cookies:
  344. args.extend(['-cookies', ''.join(
  345. '{0}={1}; path={2}; domain={3};\r\n'.format(
  346. cookie.name, cookie.value, cookie.path, cookie.domain)
  347. for cookie in cookies)])
  348. if info_dict.get('http_headers') and re.match(r'^https?://', url):
  349. # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
  350. # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
  351. headers = handle_youtubedl_headers(info_dict['http_headers'])
  352. args += [
  353. '-headers',
  354. ''.join('%s: %s\r\n' % (key, val) for key, val in headers.items())]
  355. env = None
  356. proxy = self.params.get('proxy')
  357. if proxy:
  358. if not re.match(r'^[\da-zA-Z]+://', proxy):
  359. proxy = 'http://%s' % proxy
  360. if proxy.startswith('socks'):
  361. self.report_warning(
  362. '%s does not support SOCKS proxies. Downloading is likely to fail. '
  363. 'Consider adding --hls-prefer-native to your command.' % self.get_basename())
  364. # Since December 2015 ffmpeg supports -http_proxy option (see
  365. # http://git.videolan.org/?p=ffmpeg.git;a=commit;h=b4eb1f29ebddd60c41a2eb39f5af701e38e0d3fd)
  366. # We could switch to the following code if we are able to detect version properly
  367. # args += ['-http_proxy', proxy]
  368. env = os.environ.copy()
  369. compat_setenv('HTTP_PROXY', proxy, env=env)
  370. compat_setenv('http_proxy', proxy, env=env)
  371. protocol = info_dict.get('protocol')
  372. if protocol == 'rtmp':
  373. player_url = info_dict.get('player_url')
  374. page_url = info_dict.get('page_url')
  375. app = info_dict.get('app')
  376. play_path = info_dict.get('play_path')
  377. tc_url = info_dict.get('tc_url')
  378. flash_version = info_dict.get('flash_version')
  379. live = info_dict.get('rtmp_live', False)
  380. conn = info_dict.get('rtmp_conn')
  381. if player_url is not None:
  382. args += ['-rtmp_swfverify', player_url]
  383. if page_url is not None:
  384. args += ['-rtmp_pageurl', page_url]
  385. if app is not None:
  386. args += ['-rtmp_app', app]
  387. if play_path is not None:
  388. args += ['-rtmp_playpath', play_path]
  389. if tc_url is not None:
  390. args += ['-rtmp_tcurl', tc_url]
  391. if flash_version is not None:
  392. args += ['-rtmp_flashver', flash_version]
  393. if live:
  394. args += ['-rtmp_live', 'live']
  395. if isinstance(conn, list):
  396. for entry in conn:
  397. args += ['-rtmp_conn', entry]
  398. elif isinstance(conn, compat_str):
  399. args += ['-rtmp_conn', conn]
  400. args += ['-i', url, '-c', 'copy']
  401. if self.params.get('test', False):
  402. args += ['-fs', compat_str(self._TEST_FILE_SIZE)]
  403. if protocol in ('m3u8', 'm3u8_native'):
  404. if self.params.get('hls_use_mpegts', False) or tmpfilename == '-':
  405. args += ['-f', 'mpegts']
  406. else:
  407. args += ['-f', 'mp4']
  408. if (ffpp.basename == 'ffmpeg' and is_outdated_version(ffpp._versions['ffmpeg'], '3.2', False)) and (not info_dict.get('acodec') or info_dict['acodec'].split('.')[0] in ('aac', 'mp4a')):
  409. args += ['-bsf:a', 'aac_adtstoasc']
  410. elif protocol == 'rtmp':
  411. args += ['-f', 'flv']
  412. else:
  413. args += ['-f', EXT_TO_OUT_FORMATS.get(info_dict['ext'], info_dict['ext'])]
  414. args = [encodeArgument(opt) for opt in args]
  415. args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
  416. self._debug_cmd(args)
  417. proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env)
  418. try:
  419. retval = proc.wait()
  420. except BaseException as e:
  421. # subprocess.run would send the SIGKILL signal to ffmpeg and the
  422. # mp4 file couldn't be played, but if we ask ffmpeg to quit it
  423. # produces a file that is playable (this is mostly useful for live
  424. # streams). Note that Windows is not affected and produces playable
  425. # files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
  426. if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32':
  427. process_communicate_or_kill(proc, b'q')
  428. else:
  429. proc.kill()
  430. proc.wait()
  431. raise
  432. return retval
  433. class AVconvFD(FFmpegFD):
  434. pass
  435. _BY_NAME = dict(
  436. (klass.get_basename(), klass)
  437. for name, klass in globals().items()
  438. if name.endswith('FD') and name != 'ExternalFD'
  439. )
  440. def list_external_downloaders():
  441. return sorted(_BY_NAME.keys())
  442. def get_external_downloader(external_downloader):
  443. """ Given the name of the executable, see whether we support the given
  444. downloader . """
  445. # Drop .exe extension on Windows
  446. bn = os.path.splitext(os.path.basename(external_downloader))[0]
  447. return _BY_NAME[bn]