commit: 33db85c571304bbd6863e3407ad8d08764c9e53b
parent f33923cba7670ea2e82f233c1f88210eb41f7c3b
Author: teddy171 <teddy171@qq.com>
Date: Fri, 10 Feb 2023 04:19:27 +0800
[feat]: Add support to external downloader aria2p (#31500)
* feat: add class Aria2pFD
* feat: create call_downloader function
* feat: a colorful download interface to aria2pFD
* feat: change value name
* Apply suggestions from code review
Co-authored-by: dirkf <fieldhouse@gmx.net>
* Typo in suggestion
* fix: remove unused value
* fix: add not function to return value(0 is normal); add total_seconds to download.eta(timedelta object); add waiting status when hook progress
* fix: remove unuse method ..utils.format_bytes
* fix: be up to flake8
* fix: be up to flake8
* Apply suggestions from code review
* [feat] test external downloader aria2p
* [feat] test external downloader aria2p
* [fix] test_external_downloader.py
* Apply suggestions from code review
Co-authored-by: dirkf <fieldhouse@gmx.net>
* Apply suggestions from code review
Co-authored-by: dirkf <fieldhouse@gmx.net>
* Update test/test_external_downloader.py
Co-authored-by: dirkf <fieldhouse@gmx.net>
* Update test/test_external_downloader.py
Co-authored-by: dirkf <fieldhouse@gmx.net>
* Update youtube_dl/downloader/external.py
Co-authored-by: dirkf <fieldhouse@gmx.net>
* refactoring code and fix bugs
* Apply suggestions from code review
* Rename test_external_downloader.py to test_downloader_external.py
---------
Co-authored-by: dirkf <fieldhouse@gmx.net>
Diffstat:
5 files changed, 193 insertions(+), 24 deletions(-)
diff --git a/test/helper.py b/test/helper.py
@@ -89,6 +89,17 @@ class FakeYDL(YoutubeDL):
self.report_warning = types.MethodType(report_warning, self)
+class FakeLogger(object):
+ def debug(self, msg):
+ pass
+
+ def warning(self, msg):
+ pass
+
+ def error(self, msg):
+ pass
+
+
def gettestcases(include_onlymatching=False):
for ie in youtube_dl.extractor.gen_extractors():
for tc in ie.get_testcases(include_onlymatching):
diff --git a/test/test_downloader_external.py b/test/test_downloader_external.py
@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+# coding: utf-8
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import re
+import sys
+import subprocess
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import (
+ FakeLogger,
+ http_server_port,
+ try_rm,
+)
+from youtube_dl import YoutubeDL
+from youtube_dl.compat import compat_http_server
+from youtube_dl.utils import encodeFilename
+from youtube_dl.downloader.external import Aria2pFD
+import threading
+
+TEST_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+TEST_SIZE = 10 * 1024
+
+
+class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
+ def log_message(self, format, *args):
+ pass
+
+ def send_content_range(self, total=None):
+ range_header = self.headers.get('Range')
+ start = end = None
+ if range_header:
+ mobj = re.match(r'bytes=(\d+)-(\d+)', range_header)
+ if mobj:
+ start, end = (int(mobj.group(i)) for i in (1, 2))
+ valid_range = start is not None and end is not None
+ if valid_range:
+ content_range = 'bytes %d-%d' % (start, end)
+ if total:
+ content_range += '/%d' % total
+ self.send_header('Content-Range', content_range)
+ return (end - start + 1) if valid_range else total
+
+ def serve(self, range=True, content_length=True):
+ self.send_response(200)
+ self.send_header('Content-Type', 'video/mp4')
+ size = TEST_SIZE
+ if range:
+ size = self.send_content_range(TEST_SIZE)
+ if content_length:
+ self.send_header('Content-Length', size)
+ self.end_headers()
+ self.wfile.write(b'#' * size)
+
+ def do_GET(self):
+ if self.path == '/regular':
+ self.serve()
+ elif self.path == '/no-content-length':
+ self.serve(content_length=False)
+ elif self.path == '/no-range':
+ self.serve(range=False)
+ elif self.path == '/no-range-no-content-length':
+ self.serve(range=False, content_length=False)
+ else:
+ assert False, 'unrecognised server path'
+
+
+@unittest.skipUnless(Aria2pFD.available(), 'aria2p module not found')
+class TestAria2pFD(unittest.TestCase):
+ def setUp(self):
+ self.httpd = compat_http_server.HTTPServer(
+ ('127.0.0.1', 0), HTTPTestRequestHandler)
+ self.port = http_server_port(self.httpd)
+ self.server_thread = threading.Thread(target=self.httpd.serve_forever)
+ self.server_thread.daemon = True
+ self.server_thread.start()
+
+ def download(self, params, ep):
+ with subprocess.Popen(
+ ['aria2c', '--enable-rpc'],
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL
+ ) as process:
+ if not process.poll():
+ filename = 'testfile.mp4'
+ params['logger'] = FakeLogger()
+ params['outtmpl'] = filename
+ ydl = YoutubeDL(params)
+ try_rm(encodeFilename(filename))
+ self.assertEqual(ydl.download(['http://127.0.0.1:%d/%s' % (self.port, ep)]), 0)
+ self.assertEqual(os.path.getsize(encodeFilename(filename)), TEST_SIZE)
+ try_rm(encodeFilename(filename))
+ process.kill()
+
+ def download_all(self, params):
+ for ep in ('regular', 'no-content-length', 'no-range', 'no-range-no-content-length'):
+ self.download(params, ep)
+
+ def test_regular(self):
+ self.download_all({'external_downloader': 'aria2p'})
+
+ def test_chunked(self):
+ self.download_all({
+ 'external_downloader': 'aria2p',
+ 'http_chunk_size': 1000,
+ })
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/test/test_downloader_http.py b/test/test_downloader_http.py
@@ -9,7 +9,11 @@ import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-from test.helper import http_server_port, try_rm
+from test.helper import (
+ FakeLogger,
+ http_server_port,
+ try_rm,
+)
from youtube_dl import YoutubeDL
from youtube_dl.compat import compat_http_server
from youtube_dl.downloader.http import HttpFD
@@ -66,17 +70,6 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
assert False
-class FakeLogger(object):
- def debug(self, msg):
- pass
-
- def warning(self, msg):
- pass
-
- def error(self, msg):
- pass
-
-
class TestHttpFD(unittest.TestCase):
def setUp(self):
self.httpd = compat_http_server.HTTPServer(
diff --git a/test/test_http.py b/test/test_http.py
@@ -8,7 +8,10 @@ import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-from test.helper import http_server_port
+from test.helper import (
+ FakeLogger,
+ http_server_port,
+)
from youtube_dl import YoutubeDL
from youtube_dl.compat import compat_http_server, compat_urllib_request
import ssl
@@ -52,17 +55,6 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
assert False
-class FakeLogger(object):
- def debug(self, msg):
- pass
-
- def warning(self, msg):
- pass
-
- def error(self, msg):
- pass
-
-
class TestHTTP(unittest.TestCase):
def setUp(self):
self.httpd = compat_http_server.HTTPServer(
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
@@ -200,6 +200,64 @@ class Aria2cFD(ExternalFD):
return cmd
+class Aria2pFD(ExternalFD):
+ ''' Aria2pFD class
+ This class support to use aria2p as downloader.
+ (Aria2p, a command-line tool and Python library to interact with an aria2c daemon process
+ through JSON-RPC.)
+ It can help you to get download progress more easily.
+ To use aria2p as downloader, you need to install aria2c and aria2p, aria2p can download with pip.
+ Then run aria2c in the background and enable with the --enable-rpc option.
+ '''
+ try:
+ import aria2p
+ __avail = True
+ except ImportError:
+ __avail = False
+
+ @classmethod
+ def available(cls):
+ return cls.__avail
+
+ def _call_downloader(self, tmpfilename, info_dict):
+ aria2 = self.aria2p.API(
+ self.aria2p.Client(
+ host='http://localhost',
+ port=6800,
+ secret=''
+ )
+ )
+
+ options = {
+ 'min-split-size': '1M',
+ 'max-connection-per-server': 4,
+ 'auto-file-renaming': 'false',
+ }
+ options['dir'] = os.path.dirname(tmpfilename) or os.path.abspath('.')
+ options['out'] = os.path.basename(tmpfilename)
+ options['header'] = []
+ for key, val in info_dict['http_headers'].items():
+ options['header'].append('{0}: {1}'.format(key, val))
+ download = aria2.add_uris([info_dict['url']], options)
+ status = {
+ 'status': 'downloading',
+ 'tmpfilename': tmpfilename,
+ }
+ started = time.time()
+ while download.status in ['active', 'waiting']:
+ download = aria2.get_download(download.gid)
+ status.update({
+ 'downloaded_bytes': download.completed_length,
+ 'total_bytes': download.total_length,
+ 'elapsed': time.time() - started,
+ 'eta': download.eta.total_seconds(),
+ 'speed': download.download_speed,
+ })
+ self._hook_progress(status)
+ time.sleep(.5)
+ return download.status != 'complete'
+
+
class HttpieFD(ExternalFD):
@classmethod
def available(cls):
