commit: 2500300c2a5986ace34390aa473a8bd51f83622c
parent 58fc5bde47215d9e7c60647dd21202a254b3b066
Author: dirkf <fieldhouse@gmx.net>
Date: Thu, 29 Jun 2023 15:27:12 +0100
[workflows/ci.yml] Restore test support for Py 3.2
Diffstat:
6 files changed, 328 insertions(+), 30 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,73 +1,349 @@
name: CI
-on: [push, pull_request]
+
+env:
+ # add 3.10+ after patching nose (https://github.com/nose-devs/nose/issues/1099)
+ # or switching to fork of https://github.com/mdmintz/pynose
+ all-cpython-versions: 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9
+ main-cpython-versions: 2.7, 3.2, 3.5, 3.9
+ pypy-versions: pypy-2.7, pypy-3.6, pypy-3.7
+ cpython-versions: all
+ # test-set: both
+ test-set: core
+
+on:
+ push:
+ pull_request:
+ workflow_dispatch:
+ inputs:
+ cpython-versions:
+ type: choice
+ description: CPython versions (main = 2.7, 3.2, 3.5, 3.9)
+ options:
+ - all
+ - main
+ required: true
+ default: main
+ test-set:
+ type: choice
+ description: core, download
+ options:
+ - both
+ - core
+ - download
+ required: true
+ default: core
+
+permissions:
+ contents: read
+
jobs:
+ select:
+ name: Select tests from inputs
+ runs-on: ubuntu-latest
+ outputs:
+ cpython-versions: ${{ steps.run.outputs.cpython-versions }}
+ test-set: ${{ steps.run.outputs.test-set }}
+ own-pip-versions: ${{ steps.run.outputs.own-pip-versions }}
+ steps:
+ - id: run
+ run: |
+ # Make a JSON Array from comma/space-separated string (no extra escaping)
+ json_list() { \
+ ret=""; IFS="${IFS},"; set -- $*; \
+ for a in "$@"; do \
+ ret=$(printf '%s"%s"' "${ret}${ret:+, }" "$a"); \
+ done; \
+ printf '[%s]' "$ret"; }
+ tests="${{ inputs.test-set || env.test-set }}"
+ [ $tests = both ] && tests="core download"
+ printf 'test-set=%s\n' "$(json_list $tests)" >> "$GITHUB_OUTPUT"
+ versions="${{ inputs.cpython-versions || env.cpython-versions }}"
+ if [ "$versions" = all ]; then \
+ versions="${{ env.all-cpython-versions }}"; else \
+ versions="${{ env.main-cpython-versions }}"; \
+ fi
+ printf 'cpython-versions=%s\n' \
+ "$(json_list ${versions}${versions:+, }${{ env.pypy-versions }})" >> "$GITHUB_OUTPUT"
+ # versions with a special get-pip.py in a per-version subdirectory
+ printf 'own-pip-versions=%s\n' \
+ "$(json_list 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6)" >> "$GITHUB_OUTPUT"
+
tests:
- name: Tests
+ name: Run tests
+ needs: select
+ permissions:
+ contents: read
+ packages: write
runs-on: ${{ matrix.os }}
strategy:
fail-fast: true
matrix:
os: [ubuntu-20.04]
- python-version: [2.6, 2.7, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, pypy-2.7, pypy-3.6, pypy-3.7]
+ # outside steps, use github.env...., not env....
+ python-version: ${{ fromJSON(needs.select.outputs.cpython-versions) }}
python-impl: [cpython]
- ytdl-test-set: [core, download]
+ ytdl-test-set: ${{ fromJSON(needs.select.outputs.test-set) }}
run-tests-ext: [sh]
include:
- # python 3.2 is only available on windows via setup-python
- os: windows-2019
python-version: 3.2
python-impl: cpython
- ytdl-test-set: core
+ ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }}
run-tests-ext: bat
- os: windows-2019
python-version: 3.2
python-impl: cpython
- ytdl-test-set: download
+ ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download' || 'nodownload' }}
run-tests-ext: bat
# jython
- os: ubuntu-20.04
python-impl: jython
- ytdl-test-set: core
+ ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }}
run-tests-ext: sh
- os: ubuntu-20.04
python-impl: jython
- ytdl-test-set: download
+ ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download' || 'nodownload' }}
run-tests-ext: sh
steps:
- - uses: actions/checkout@v3
+ - name: Checkout
+ uses: actions/checkout@v3
+ #-------- Python 3 -----
- name: Set up supported Python ${{ matrix.python-version }}
+ id: setup-python
+ if: ${{ matrix.python-impl == 'cpython' && matrix.python-version != '2.6' && matrix.python-version != '2.7'}}
# wrap broken actions/setup-python@v4
uses: ytdl-org/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
cache-build: true
allow-build: info
+ - name: Locate supported Python ${{ matrix.python-version }}
+ if: ${{ env.pythonLocation }}
+ shell: bash
+ run: |
+ echo "PYTHONHOME=${pythonLocation}" >> "$GITHUB_ENV"
+ export expected="${{ steps.setup-python.outputs.python-path }}"
+ dirname() { printf '%s\n' \
+ 'import os, sys' \
+ 'print(os.path.dirname(sys.argv[1]))' \
+ | ${expected} - "$1"; }
+ expd="$(dirname "$expected")"
+ export python="$(command -v python)"
+ [ "$expd" = "$(dirname "$python")" ] || echo "PATH=$expd:${PATH}" >> "$GITHUB_ENV"
+ [ -x "$python" ] || printf '%s\n' \
+ 'import os' \
+ 'exp = os.environ["expected"]' \
+ 'python = os.environ["python"]' \
+ 'exps = os.path.split(exp)' \
+ 'if python and (os.path.dirname(python) == exp[0]):' \
+ ' exit(0)' \
+ 'exps[1] = "python" + os.path.splitext(exps[1])[1]' \
+ 'python = os.path.join(*exps)' \
+ 'try:' \
+ ' os.symlink(exp, python)' \
+ 'except AttributeError:' \
+ ' os.rename(exp, python)' \
+ | ${expected} -
+ printf '%s\n' \
+ 'import sys' \
+ 'print(sys.path)' \
+ | ${expected} -
+ #-------- Python 2.7 --
+ - name: Set up Python 2.7
+ if: ${{ matrix.python-version == '2.7' }}
+ # install 2.7
+ run: |
+ sudo apt-get install -y python2 python-is-python2
+ echo "PYTHONHOME=/usr" >> "$GITHUB_ENV"
+ #-------- Python 2.6 --
+ - name: Set up Python 2.6 environment
+ if: ${{ matrix.python-version == '2.6' }}
+ run: |
+ openssl_name=openssl-1.0.2u
+ echo "openssl_name=${openssl_name}" >> "$GITHUB_ENV"
+ openssl_dir=$HOME/.local/opt/$openssl_name
+ echo "openssl_dir=${openssl_dir}" >> "$GITHUB_ENV"
+ PYENV_ROOT=$HOME/.local/share/pyenv
+ echo "PYENV_ROOT=${PYENV_ROOT}" >> "$GITHUB_ENV"
+ sudo apt-get install -y openssl ca-certificates
+ - name: Cache Python 2.6
+ id: cache26
+ if: ${{ matrix.python-version == '2.6' }}
+ uses: actions/cache@v3
+ with:
+ key: python-2.6.9
+ path: |
+ ${{ env.openssl_dir }}
+ ${{ env.PYENV_ROOT }}
+ - name: Build and set up Python 2.6
+ if: ${{ matrix.python-version == '2.6' && ! steps.cache26.outputs.cache-hit }}
+ # dl and build locally
+ run: |
+ # Install build environment
+ sudo apt-get install -y build-essential llvm libssl-dev tk-dev \
+ libncursesw5-dev libreadline-dev libsqlite3-dev \
+ libffi-dev xz-utils zlib1g-dev libbz2-dev liblzma-dev
+ # Download and install OpenSSL 1.0.2, back in time
+ openssl_name=${{ env.openssl_name }}
+ openssl_targz=${openssl_name}.tar.gz
+ openssl_dir=${{ env.openssl_dir }}
+ openssl_inc=$openssl_dir/include
+ openssl_lib=$openssl_dir/lib
+ openssl_ssl=$openssl_dir/ssl
+ curl -L "https://www.openssl.org/source/$openssl_targz" -o $openssl_targz
+ tar -xf $openssl_targz
+ ( cd $openssl_name; \
+ ./config --prefix=$openssl_dir --openssldir=${openssl_dir}/ssl \
+ --libdir=lib -Wl,-rpath=${openssl_dir}/lib shared zlib-dynamic && \
+ make && \
+ make install )
+ rm -rf $openssl_name
+ rmdir $openssl_ssl/certs && ln -s /etc/ssl/certs $openssl_ssl/certs
+
+ # Download PyEnv from its GitHub repository.
+ export PYENV_ROOT=${{ env.PYENV_ROOT }}
+ export PATH=$PYENV_ROOT/bin:$PATH
+ git clone https://github.com/pyenv/pyenv.git $PYENV_ROOT
+ eval "$(pyenv init --path)"
+
+ # Prevent pyenv build trying (and failing) to update pip
+ export GET_PIP=get-pip-2.6.py
+ echo 'import sys; sys.exit(0)' > ${GET_PIP}
+ GET_PIP=$(realpath $GET_PIP)
+
+ # Build and install Python
+ export CFLAGS="-I$openssl_inc"
+ export LDFLAGS="-L$openssl_lib"
+ export LD_LIBRARY_PATH="$openssl_lib"
+ pyenv install 2.6.9
+ echo "PYTHONHOME=${PYENV_ROOT}" >> "$GITHUB_ENV"
+ echo "PATH=$PYENV_ROOT/bin:$PATH" >> "$GITHUB_ENV"
+ - name: Set up cached Python 2.6
+ if: ${{ steps.cache26.outputs.cache-hit }}
+ run: |
+ export PYENV_ROOT
+ export PATH=$PYENV_ROOT/bin:$PATH
+ eval "$(pyenv init --path)"
+ pyenv local 2.6.9
+ echo "PYTHONHOME=${PYENV_ROOT}" >> "$GITHUB_ENV"
+ echo "PATH=$PYENV_ROOT/bin:$PATH" >> "$GITHUB_ENV"
+ #-------- Jython ------
- name: Set up Java 8
if: ${{ matrix.python-impl == 'jython' }}
uses: actions/setup-java@v2
with:
java-version: 8
distribution: 'zulu'
- - name: Install Jython
+ - name: Setup Jython environment
if: ${{ matrix.python-impl == 'jython' }}
run: |
- wget https://repo1.maven.org/maven2/org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar -O jython-installer.jar
- java -jar jython-installer.jar -s -d "$HOME/jython"
- echo "$HOME/jython/bin" >> $GITHUB_PATH
- - name: Install nose
- if: ${{ matrix.python-impl != 'jython' }}
- run: pip install nose
+ echo "JYTHON_ROOT=${HOME}/jython" >> "$GITHUB_ENV"
+ - name: Cache Jython
+ id: cachejy
+ if: ${{ matrix.python-impl == 'jython' }}
+ uses: actions/cache@v3
+ with:
+ # 2.7.3 now available, may solve SNI issue
+ key: jython-2.7.1
+ path: |
+ ${{ env.JYTHON_ROOT }}
+ - name: Install Jython
+ if: ${{ matrix.python-impl == 'jython' && ! steps.cachejy.outputs.cache-hit }}
+ run: |
+ JYTHON_ROOT="${{ env.JYTHON_ROOT }}"
+ curl -L "https://repo1.maven.org/maven2/org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar" -o jython-installer.jar
+ java -jar jython-installer.jar -s -d "${JYTHON_ROOT}"
+ echo "${JYTHON_ROOT}/bin" >> $GITHUB_PATH
+ - name: Set up cached Jython
+ if: ${{ steps.cachejy.outputs.cache-hit }}
+ run: |
+ JYTHON_ROOT="${{ env.JYTHON_ROOT }}"
+ echo "${JYTHON_ROOT}/bin" >> $GITHUB_PATH
+ #-------- pip ---------
+ - name: Set up supported Python ${{ matrix.python-version }} pip
+ if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.6' || matrix.python-version == '2.7' }}
+ # This step may run in either Linux or Windows
+ shell: bash
+ run: |
+ echo "$PATH"
+ echo "$PYTHONHOME"
+ # curl is available on both Windows and Linux, -L follows redirects, -O gets name
+ python -m ensurepip || python -m pip --version || { \
+ get_pip="${{ contains(needs.select.outputs.own-pip-versions, matrix.python-version) && format('{0}/', matrix.python-version) || '' }}"; \
+ curl -L -O "https://bootstrap.pypa.io/pip/${get_pip}get-pip.py"; \
+ python get-pip.py; }
+ - name: Set up other Python ${{ matrix.python-version }} pip
+ if: ${{ matrix.python-version == '3.2' && steps.setup-python.outputs.python-path }}
+ shell: bash
+ run: |
+ # https://files.pythonhosted.org/packages/8a/e9/8468cd68b582b06ef554be0b96b59f59779627131aad48f8a5bce4b13450/wheel-0.29.0-py2.py3-none-any.whl
+ # https://files.pythonhosted.org/packages/06/4b/86a670fd21f7849adb092e40883c48dcd0d66b8a878fc8d63b7f0ea04213/setuptools-29.0.1-py2.py3-none-any.whl
+ python -m pip --version || { \
+ curl -L -O "https://bootstrap.pypa.io/pip/3.2/get-pip.py"; \
+ curl -L -O "https://files.pythonhosted.org/packages/b2/d0/cd115fe345dd6f07ec1c780020a7dfe74966fceeb171e0f20d1d4905b0b7/pip-7.1.2-py2.py3-none-any.whl"; \
+ python -v get-pip.py --no-setuptools --no-wheel pip-7.1.2-py2.py3-none-any.whl; }
+
+ #-------- nose --------
+ - name: Install nose for Python ${{ matrix.python-version }}
+ if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.6' || matrix.python-version == '2.7' }}
+ shell: bash
+ run: |
+ echo "$PATH"
+ echo "$PYTHONHOME"
+ python --version
+ python -m pip --version
+ python -m pip nose --version || python -m pip install nose
+ - name: Install nose for other Python ${{ matrix.python-version }}
+ if: ${{ matrix.python-version == '3.2' && steps.setup-python.outputs.python-path }}
+ shell: bash
+ run: |
+ python -m pip nose --version || { \
+ curl -L -O "https://files.pythonhosted.org/packages/15/d8/dd071918c040f50fa1cf80da16423af51ff8ce4a0f2399b7bf8de45ac3d9/nose-1.3.7-py3-none-any.whl"; \
+ python --version; \
+ printf '%s\n' \
+ 'import sys' \
+ 'print(sys.path)' \
+ | python -; \
+ python -m pip --version; \
+ python -m pip install nose-1.3.7-py3-none-any.whl; }
- name: Install nose (Jython)
if: ${{ matrix.python-impl == 'jython' }}
- # Working around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb)
+ # Work around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb)
run: |
- wget https://files.pythonhosted.org/packages/99/4f/13fb671119e65c4dce97c60e67d3fd9e6f7f809f2b307e2611f4701205cb/nose-1.3.7-py2-none-any.whl
- pip install nose-1.3.7-py2-none-any.whl
+ pip nose --version || { \
+ curl -L -O "https://files.pythonhosted.org/packages/99/4f/13fb671119e65c4dce97c60e67d3fd9e6f7f809f2b307e2611f4701205cb/nose-1.3.7-py2-none-any.whl"; \
+ pip --version; \
+ pip install nose-1.3.7-py2-none-any.whl; }
+ - name: Set up nosetest test
+ if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }}
+ shell: bash
+ run: |
+ # define a test to validate the Python version used by nosetests
+ printf '%s\n' \
+ 'from __future__ import unicode_literals' \
+ 'import sys, os, platform, unittest' \
+ 'class TestPython(unittest.TestCase):' \
+ ' def setUp(self):' \
+ ' self.ver = os.environ["PYTHON_VER"].split("-")' \
+ ' def test_python_ver(self):' \
+ ' self.assertEqual(sys.version[:3], self.ver[-1])' \
+ ' self.assertTrue(sys.version.startswith(self.ver[-1]))' \
+ ' self.assertIn(self.ver[0], sys.version.lower())' \
+ ' def test_python_impl(self):' \
+ ' self.assertIn(platform.python_implementation().lower(), (os.environ["PYTHON_IMPL"], self.ver[0]))' \
+ > test/test_python.py
+ #-------- TESTS -------
- name: Run tests
+ if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }}
continue-on-error: ${{ matrix.ytdl-test-set == 'download' || matrix.python-impl == 'jython' }}
env:
YTDL_TEST_SET: ${{ matrix.ytdl-test-set }}
- run: ./devscripts/run_tests.${{ matrix.run-tests-ext }}
+ PYTHON_VER: ${{ matrix.python-version }}
+ PYTHON_IMPL: ${{ matrix.python-impl }}
+
+ run: |
+ ./devscripts/run_tests.${{ matrix.run-tests-ext }}
+
flake8:
name: Linter
runs-on: ubuntu-latest
@@ -81,3 +357,4 @@ jobs:
run: pip install flake8
- name: Run flake8
run: flake8 .
+
diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py
@@ -6,6 +6,10 @@ import os
from os.path import dirname as dirn
import sys
+from youtube_dl.compat import compat_register_utf8
+
+compat_register_utf8()
+
print('WARNING: Lazy loading extractors is an experimental feature that may not always work', file=sys.stderr)
sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
diff --git a/test/test_execution.py b/test/test_execution.py
@@ -10,10 +10,13 @@ import os
import subprocess
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from youtube_dl.compat import compat_register_utf8
+
from youtube_dl.utils import encodeArgument
rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+compat_register_utf8()
try:
_DEV_NULL = subprocess.DEVNULL
@@ -25,13 +28,14 @@ class TestExecution(unittest.TestCase):
def test_import(self):
subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
+ @unittest.skipIf(sys.version_info < (2, 7), 'Python 2.6 doesn\'t support package execution')
def test_module_exec(self):
- if sys.version_info >= (2, 7): # Python 2.6 doesn't support package execution
- subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+ subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)
def test_main_exec(self):
subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+ @unittest.skipIf(sys.version_info < (2, 7), 'Python 2.6 doesn\'t support package execution')
def test_cmdline_umlauts(self):
p = subprocess.Popen(
[sys.executable, 'youtube_dl/__main__.py', encodeArgument('รค'), '--version'],
diff --git a/test/test_unicode_literals.py b/test/test_unicode_literals.py
@@ -15,6 +15,7 @@ IGNORED_FILES = [
'setup.py', # http://bugs.python.org/issue13943
'conf.py',
'buildserver.py',
+ 'get-pip.py',
]
IGNORED_DIRS = [
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
@@ -5,7 +5,6 @@ from __future__ import unicode_literals
__license__ = 'Public Domain'
-import codecs
import io
import os
import random
@@ -17,6 +16,7 @@ from .options import (
)
from .compat import (
compat_getpass,
+ compat_register_utf8,
compat_shlex_split,
workaround_optparse_bug9161,
)
@@ -46,10 +46,8 @@ from .YoutubeDL import YoutubeDL
def _real_main(argv=None):
- # Compatibility fixes for Windows
- if sys.platform == 'win32':
- # https://github.com/ytdl-org/youtube-dl/issues/820
- codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
+ # Compatibility fix for Windows
+ compat_register_utf8()
workaround_optparse_bug9161()
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
@@ -31,13 +31,17 @@ try:
compat_str, compat_basestring, compat_chr = (
unicode, basestring, unichr
)
- from .casefold import casefold as compat_casefold
-
except NameError:
compat_str, compat_basestring, compat_chr = (
str, str, chr
)
+
+# casefold
+try:
+ compat_str.casefold
compat_casefold = lambda s: s.casefold()
+except AttributeError:
+ from .casefold import casefold as compat_casefold
try:
import collections.abc as compat_collections_abc
@@ -3137,6 +3141,15 @@ else:
compat_open = open
+# compat_register_utf8
+def compat_register_utf8():
+ if sys.platform == 'win32':
+ # https://github.com/ytdl-org/youtube-dl/issues/820
+ from codecs import register, lookup
+ register(
+ lambda name: lookup('utf-8') if name == 'cp65001' else None)
+
+
legacy = [
'compat_HTMLParseError',
'compat_HTMLParser',
@@ -3203,6 +3216,7 @@ __all__ = [
'compat_print',
'compat_re_Match',
'compat_re_Pattern',
+ 'compat_register_utf8',
'compat_setenv',
'compat_shlex_quote',
'compat_shlex_split',
