commit: 7bce2ad441b874e7a1cf8cc81059c5601d832697
parent ca71e56c481c6d5ce69b4756f8f8c0aff97d79b5
Author: dirkf <fieldhouse@gmx.net>
Date: Thu, 20 Jul 2023 18:49:48 +0100
[build] Fix various Jython CI and test issues
Diffstat:
4 files changed, 45 insertions(+), 19 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -111,10 +111,12 @@ jobs:
run-tests-ext: bat
# jython
- os: ubuntu-20.04
+ python-version: 2.7
python-impl: jython
ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }}
run-tests-ext: sh
- os: ubuntu-20.04
+ python-version: 2.7
python-impl: jython
ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download' || 'nodownload' }}
run-tests-ext: sh
@@ -163,22 +165,22 @@ jobs:
'print(sys.path)' \
| ${expected} -
#-------- Python 3.12 -
- - name: Set up Python 3.12 environment
- if: ${{ matrix.python-version == '3.12' }}
+ - name: Set up CPython 3.12 environment
+ if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }}
shell: bash
run: |
PYENV_ROOT=$HOME/.local/share/pyenv
echo "PYENV_ROOT=${PYENV_ROOT}" >> "$GITHUB_ENV"
- name: Cache Python 3.12
id: cache312
- if: ${{ matrix.python-version == '3.12' }}
+ if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }}
uses: actions/cache@v3
with:
key: python-3.12
path: |
${{ env.PYENV_ROOT }}
- name: Build and set up Python 3.12
- if: ${{ matrix.python-version == '3.12' && ! steps.cache312.outputs.cache-hit }}
+ if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' && ! steps.cache312.outputs.cache-hit }}
# dl and build locally
shell: bash
run: |
@@ -192,7 +194,7 @@ jobs:
git clone "https://github.com/pyenv/pyenv.git" "$PYENV_ROOT"
pyenv install 3.12.0b4
- name: Locate Python 3.12
- if: ${{ matrix.python-version == '3.12' }}
+ if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }}
shell: bash
run: |
PYTHONHOME="${{ env.PYENV_ROOT }}/versions/3.12.0b4"
@@ -200,7 +202,7 @@ jobs:
echo "PATH=${PYTHONHOME}/bin:$PATH" >> "$GITHUB_ENV"
#-------- Python 2.7 --
- name: Set up Python 2.7
- if: ${{ matrix.python-version == '2.7' }}
+ if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.7' }}
# install 2.7
shell: bash
run: |
@@ -208,7 +210,7 @@ jobs:
echo "PYTHONHOME=/usr" >> "$GITHUB_ENV"
#-------- Python 2.6 --
- name: Set up Python 2.6 environment
- if: ${{ matrix.python-version == '2.6' }}
+ if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' }}
shell: bash
run: |
openssl_name=openssl-1.0.2u
@@ -228,7 +230,7 @@ jobs:
${{ env.openssl_dir }}
${{ env.PYENV_ROOT }}
- name: Build and set up Python 2.6
- if: ${{ matrix.python-version == '2.6' && ! steps.cache26.outputs.cache-hit }}
+ if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' && ! steps.cache26.outputs.cache-hit }}
# dl and build locally
shell: bash
run: |
@@ -266,7 +268,7 @@ jobs:
export LD_LIBRARY_PATH="$openssl_lib"
pyenv install 2.6.9
- name: Locate Python 2.6
- if: ${{ matrix.python-version == '2.6' }}
+ if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' }}
shell: bash
run: |
PYTHONHOME="${{ env.PYENV_ROOT }}/versions/2.6.9"
@@ -288,7 +290,7 @@ jobs:
echo "PIP=pip" >> "$GITHUB_ENV"
- name: Cache Jython
id: cachejy
- if: ${{ matrix.python-impl == 'jython' }}
+ if: ${{ matrix.python-impl == 'jython' && matrix.python-version == '2.7' }}
uses: actions/cache@v3
with:
# 2.7.3 now available, may solve SNI issue
@@ -296,7 +298,7 @@ jobs:
path: |
${{ env.JYTHON_ROOT }}
- name: Install Jython
- if: ${{ matrix.python-impl == 'jython' && ! steps.cachejy.outputs.cache-hit }}
+ if: ${{ matrix.python-impl == 'jython' && matrix.python-version == '2.7' && ! steps.cachejy.outputs.cache-hit }}
shell: bash
run: |
JYTHON_ROOT="${{ env.JYTHON_ROOT }}"
@@ -309,6 +311,11 @@ jobs:
run: |
JYTHON_ROOT="${{ env.JYTHON_ROOT }}"
echo "${JYTHON_ROOT}/bin" >> $GITHUB_PATH
+ - name: Install supporting Python 2.7 if possible
+ if: ${{ steps.cachejy.outputs.cache-hit }}
+ shell: bash
+ run: |
+ sudo apt-get install -y python2.7 || true
#-------- pip ---------
- name: Set up supported Python ${{ matrix.python-version }} pip
if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.7' }}
@@ -391,6 +398,11 @@ jobs:
if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }}
shell: bash
run: |
+ # set PYTHON_VER
+ PYTHON_VER=${{ matrix.python-version }}
+ [ "${PYTHON_VER#*-}" != "$PYTHON_VER" ] || PYTHON_VER="${{ matrix.python-impl }}-${PYTHON_VER}"
+ echo "PYTHON_VER=$PYTHON_VER" >> "$GITHUB_ENV"
+ echo "PYTHON_IMPL=${{ matrix.python-impl }}" >> "$GITHUB_ENV"
# define a test to validate the Python version used by nosetests
printf '%s\n' \
'from __future__ import unicode_literals' \
@@ -405,7 +417,7 @@ jobs:
' def test_python_ver(self):' \
' self.assertEqual(["%d" % v for v in sys.version_info[:2]], self.ver[-1].split(".")[:2])' \
' self.assertTrue(sys.version.startswith(self.ver[-1]))' \
- ' self.assertIn(self.ver[0], sys.version.lower())' \
+ ' self.assertIn(self.ver[0], ",".join((sys.version, platform.python_implementation())).lower())' \
' def test_python_impl(self):' \
' self.assertIn(platform.python_implementation().lower(), (os.environ["PYTHON_IMPL"], self.ver[0]))' \
> test/test_python.py
@@ -415,8 +427,6 @@ jobs:
continue-on-error: ${{ matrix.ytdl-test-set == 'download' || matrix.python-impl == 'jython' }}
env:
YTDL_TEST_SET: ${{ matrix.ytdl-test-set }}
- PYTHON_VER: ${{ matrix.python-version }}
- PYTHON_IMPL: ${{ matrix.python-impl }}
run: |
./devscripts/run_tests.${{ matrix.run-tests-ext }}
flake8:
diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py
@@ -118,3 +118,14 @@ module_src = '\n'.join(module_contents) + '\n'
with io.open(lazy_extractors_filename, 'wt', encoding='utf-8') as f:
f.write(module_src)
+
+# work around JVM byte code module limit in Jython
+if sys.platform.startswith('java') and sys.version_info[:2] == (2, 7):
+ import subprocess
+ from youtube_dl.compat import compat_subprocess_get_DEVNULL
+ # if Python 2.7 is available, use it to compile the module for Jython
+ try:
+ # if Python 2.7 is available, use it to compile the module for Jython
+ subprocess.check_call(['python2.7', '-m', 'py_compile', lazy_extractors_filename], stdout=compat_subprocess_get_DEVNULL())
+ except Exception:
+ pass
diff --git a/test/test_http.py b/test/test_http.py
@@ -45,6 +45,7 @@ from youtube_dl.utils import (
)
from test.helper import (
+ expectedFailureIf,
FakeYDL,
FakeLogger,
http_server_port,
@@ -243,6 +244,11 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
class TestHTTP(unittest.TestCase):
+ # when does it make sense to check the SSL certificate?
+ _check_cert = (
+ sys.version_info >= (3, 2)
+ or (sys.version_info[0] == 2 and sys.version_info[1:] >= (7, 19)))
+
def setUp(self):
# HTTP server
self.http_httpd = compat_http_server.HTTPServer(
@@ -307,10 +313,7 @@ class TestHTTP(unittest.TestCase):
else self.https_port if scheme == 'https'
else self.http_port, path)
- @unittest.skipUnless(
- sys.version_info >= (3, 2)
- or (sys.version_info[0] == 2 and sys.version_info[1:] >= (7, 9)),
- 'No support for certificate check in SSL')
+ @unittest.skipUnless(_check_cert, 'No support for certificate check in SSL')
def test_nocheckcertificate(self):
with FakeYDL({'logger': FakeLogger()}) as ydl:
with self.assertRaises(compat_urllib_error.URLError):
@@ -376,6 +379,8 @@ class TestHTTP(unittest.TestCase):
with self.assertRaises(compat_urllib_HTTPError):
do_req(code, 'GET')
+ # Jython 2.7.1 times out for some reason
+ @expectedFailureIf(sys.platform.startswith('java') and sys.version_info < (2, 7, 2))
def test_content_type(self):
# https://github.com/yt-dlp/yt-dlp/commit/379a4f161d4ad3e40932dcf5aca6e6fb9715ab28
with FakeYDL({'nocheckcertificate': True}) as ydl:
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
@@ -131,7 +131,7 @@ if sys.version_info[0] == 2 or sys.version_info < (3, 3):
def load(self, rawdata):
must_have_value = 0
if not isinstance(rawdata, dict):
- if sys.version_info[:2] != (2, 7):
+ if sys.version_info[:2] != (2, 7) or sys.platform.startswith('java'):
# attribute must have value for parsing
rawdata, must_have_value = re.subn(
r'(?i)(;\s*)(secure|httponly)(\s*(?:;|$))', r'\1\2=\2\3', rawdata)
