commit: df1c9e9aba6b6f38228ee8ed63d6a797cc094c97 parent 88ea21dfc94ac6ed63f64dccf403e94d1d901665 Author: fosslinux <fosslinux@aussies.space> Date: Fri, 27 Dec 2024 16:04:03 +1100 Support mirrors withinDiffstat:
39 files changed, 159 insertions(+), 83 deletions(-)diff --git a/DEVEL.md b/DEVEL.md@@ -84,3 +84,31 @@ libraries that are compiled and installed. - Patches are licensed under the license of the project which they are patching. - All files (excluding files within submodules) must comply with REUSE v3.0. + +## `sources` file + +The format of the sources file for a HTTP source file is: + +``` +<url> <checksum> [filename] +``` + +For a Git repository snapshot: + +``` +<git url> <URL to HTTP snapshot> <checksum> [filename] +``` + +The format of a git url must always be `git://<path to repository>~<reference>`. +`reference` is, for instance, a commit, tag or branch. Always use +Git protocol path, except for GitHub, in which `git://` is automagically +changed to `https://`. + +The URL to HTTP snapshot may be `_` (a single underscore) if no HTTP snapshot +exists. In this case, the filename is compulsory. The checksum is of the Git +snapshot, generated with `git archive`. See `mirror.sh` for more detailed rules. + +Some specific helpful things: +- prefer `.tar.gz` +- For GitHub, all snpashots are generated with long commit IDs, so use long + commit IDs.diff --git a/download-distfiles.sh b/download-distfiles.sh@@ -9,10 +9,19 @@ download_source() { local distfiles="${1}" local url="${2}" - local checksum="${3}" - local fname="${4}" + shift 2 + if [[ "${url}" == git://* ]]; then + url="${1}" + shift + fi + local checksum="${1}" + local fname="${2}" # Default to basename of url if not given fname="${fname:-$(basename "${url}")}" + if [ "${fname}" = "_" ]; then + echo "ERROR: ${url} must have a filename specified" + exit 1 + fi local dest_path="${distfiles}/${fname}" if ! [ -e "${dest_path}" ]; thendiff --git a/lib/generator.py b/lib/generator.py@@ -10,9 +10,11 @@ This file contains all code required to generate the boot image for live-bootstr import hashlib import os +import random import shutil import tarfile import traceback + import requests # pylint: disable=too-many-instance-attributes @@ -24,11 +26,13 @@ class Generator(): git_dir = os.path.join(os.path.dirname(os.path.join(__file__)), '..') distfiles_dir = os.path.join(git_dir, 'distfiles') - def __init__(self, arch, external_sources, early_preseed, repo_path): + # pylint: disable=too-many-arguments,too-many-positional-arguments + def __init__(self, arch, external_sources, early_preseed, repo_path, mirrors): self.arch = arch self.early_preseed = early_preseed self.external_sources = external_sources self.repo_path = repo_path + self.mirrors = mirrors self.source_manifest = self.get_source_manifest(not self.external_sources) self.early_source_manifest = self.get_source_manifest(True) self.target_dir = None @@ -278,8 +282,7 @@ actual: {readable_hash}\n\ When in doubt, try deleting the file in question -- it will be downloaded again when running \ this script the next time") - @staticmethod - def download_file(url, directory, file_name, silent=False): + def download_file(self, url, directory, file_name, silent=False): """ Download a single source archive. """ @@ -297,14 +300,33 @@ this script the next time") if not os.path.isfile(abs_file_name): if not silent: print(f"Downloading: {file_name}") - response = requests.get(url, allow_redirects=True, stream=True, - headers=headers, timeout=20) - if response.status_code == 200: - with open(abs_file_name, 'wb') as target_file: - target_file.write(response.raw.read()) + + def do_download(source): + response = requests.get(source, allow_redirects=True, stream=True, + headers=headers, timeout=20) + if response.status_code == 200: + with open(abs_file_name, 'wb') as target_file: + target_file.write(response.raw.read()) + return True + print(f"Download failed from {option}: {response.status_code} {response.reason}") + return False + + done = False + if self.mirrors: + options = [f"{x}/{file_name}" for x in self.mirrors] else: - raise requests.HTTPError("Download failed: HTTP " + - str(response.status_code) + " " + response.reason) + options = [] + random.shuffle(options) + for option in options: + if do_download(option): + done = True + break + + if not done: + if url == "_" or not do_download(url): + raise requests.RequestException(f"Unable to download {url} from ", + "any mirror or original") + return abs_file_name def get_packages(self): @@ -344,6 +366,9 @@ this script the next time") for source in sources.readlines(): source = source.strip().split(" ") + if source[0].startswith("git://"): + source = source[1:] + if len(source) > 2: file_name = source[2] else:diff --git a/rootfs.py b/rootfs.py@@ -50,6 +50,12 @@ def create_configuration_file(args): config.write("KERNEL_BOOTSTRAP=False\n") config.write(f"BUILD_KERNELS={args.update_checksums or args.build_kernels}\n") config.write(f"CONFIGURATOR={args.configurator}\n") + if not args.external_sources: + if args.mirrors: + config.write(f"MIRRORS=\"{" ".join(args.mirrors)}\"\n") + config.write(f"MIRRORS_LEN={len(args.mirrors)}\n") + else: + config.write("MIRRORS_LEN=0\n") # pylint: disable=too-many-statements,too-many-branches def main(): @@ -94,6 +100,9 @@ def main(): parser.add_argument("--configurator", help="Run the interactive configurator", action="store_true") + parser.add_argument("-m", "--mirrors", + help="Mirrors to download distfiles from", + nargs='+') parser.add_argument("-r", "--repo", help="Path to prebuilt binary packages", nargs=None) parser.add_argument("--early-preseed", @@ -187,7 +196,8 @@ def main(): generator = Generator(arch=args.arch, external_sources=args.external_sources, repo_path=args.repo, - early_preseed=args.early_preseed) + early_preseed=args.early_preseed, + mirrors=args.mirrors) bootstrap(args, generator, target, args.target_size)diff --git a/steps/autogen-5.18.16/sources b/steps/autogen-5.18.16/sources@@ -1,6 +1,4 @@ -https://github.com/schierlm/gnu-autogen-bootstrapping/archive/refs/tags/autogen-5.18.16-v1.0.1.tar.gz 953ba180b18acff188a0a8700770c7cf2fc97e1683c7b9699a5a748b542ccdd5 +git://github.com/schierlm/gnu-autogen-bootstrapping~autogen-5.18.16-v1.0.1 https://github.com/schierlm/gnu-autogen-bootstrapping/archive/refs/tags/autogen-5.18.16-v1.0.1.tar.gz 953ba180b18acff188a0a8700770c7cf2fc97e1683c7b9699a5a748b542ccdd5 https://mirrors.kernel.org/gnu/autogen/rel5.18.16/autogen-5.18.16.tar.xz f8a13466b48faa3ba99fe17a069e71c9ab006d9b1cfabe699f8c60a47d5bb49a -https://git.savannah.gnu.org/cgit/autogen.git/snapshot/autogen-5.18.16.tar.gz 0c04ab2f7ce13c4a1c06c4abc7dfe75312aad89b8b0a1068e5e563787eb56632 -https://files.bootstrapping.world/autogen-5.18.16.tar.gz 0c04ab2f7ce13c4a1c06c4abc7dfe75312aad89b8b0a1068e5e563787eb56632 -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-8f4538a5.tar.gz e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab -https://files.bootstrapping.world/gnulib-8f4538a5.tar.gz e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab +git://git.savannah.gnu.org/autogen.git~v5.18.16 https://git.savannah.gnu.org/cgit/autogen.git/snapshot/autogen-5.18.16.tar.gz 0c04ab2f7ce13c4a1c06c4abc7dfe75312aad89b8b0a1068e5e563787eb56632 +git://git.savannah.gnu.org/gnulib.git~8f4538a5 _ e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab gnulib-8f4538a5.tar.gzdiff --git a/steps/bash-2.05b/pass1.kaem b/steps/bash-2.05b/pass1.kaem@@ -16,8 +16,8 @@ mkdir build src cd build # Extract -cp ${DISTFILES}/${pkg}.tar.bz2 ../src/ -bzip2 -d -f ../src/${pkg}.tar.bz2 +cp ${DISTFILES}/${pkg}.tar.gz ../src/ +gzip -d -f ../src/${pkg}.tar.gz tar xf ../src/${pkg}.tar rm -r ../src/ cd ${pkg}diff --git a/steps/bash-2.05b/sources b/steps/bash-2.05b/sources@@ -1 +1 @@ -https://src.fedoraproject.org/repo/pkgs/bash/bash-2.05b.tar.bz2/f3e5428ed52a4f536f571a945d5de95d/bash-2.05b.tar.bz2 1ce4e5b47a6354531389f0adefb54dee2823227bf6e1e59a31c0e9317a330822 +https://mirrors.kernel.org/gnu/bash/bash-2.05b.tar.gz ba03d412998cc54bd0b0f2d6c32100967d3137098affdc2d32e6e7c11b163fe4diff --git a/steps/bc-1.07.1/sources b/steps/bc-1.07.1/sources@@ -1 +1 @@ -https://mirrors.kernel.org/slackware/slackware64-15.0/source/ap/bc/bc-1.07.1.tar.xz 95396f88fc719a1bf8bd463809119526fef44a42ab9eb708335c2cb79bc801c6 +https://mirrors.kernel.org/gnu/bc/bc-1.07.1.tar.gz 62adfca89b0a1c0164c2cdca59ca210c1d44c3ffc46daf9931cf4942664cb02adiff --git a/steps/bison-2.3/sources b/steps/bison-2.3/sources@@ -1,3 +1,2 @@ http://mirrors.kernel.org/gnu/bison/bison-2.3.tar.bz2 b10d7e9e354be72aee4e4911cf19dd27b5c527d4e7200857365b5fcdeea0dffb -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-b28236b.tar.gz 0190f28cb155fedd22bf8558c3e8705eed9eacfb7ae29e7508d025a68eb90899 -https://files.bootstrapping.world/gnulib-b28236b.tar.gz 0190f28cb155fedd22bf8558c3e8705eed9eacfb7ae29e7508d025a68eb90899 +git://git.savannah.gnu.org/gnulib.git~b28236b _ 0190f28cb155fedd22bf8558c3e8705eed9eacfb7ae29e7508d025a68eb90899 gnulib-b28236b.tar.gzdiff --git a/steps/bison-3.4.2/sources b/steps/bison-3.4.2/sources@@ -1,3 +1,2 @@ http://mirrors.kernel.org/gnu/bison/bison-3.4.2.tar.xz 27d05534699735dc69e86add5b808d6cb35900ad3fd63fa82e3eb644336abfa0 -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-672663a.tar.gz 8cced51f89a950472473856f86e88f5daf97a2347756125ccdc8ee907deec570 -https://files.bootstrapping.world/gnulib-672663a.tar.gz 8cced51f89a950472473856f86e88f5daf97a2347756125ccdc8ee907deec570 +git://git.savannah.gnu.org/gnulib.git~672663a _ 8cced51f89a950472473856f86e88f5daf97a2347756125ccdc8ee907deec570 gnulib-672663a.tar.gzdiff --git a/steps/bzip2-1.0.8/pass1.kaem b/steps/bzip2-1.0.8/pass1.kaem@@ -15,8 +15,8 @@ mkdir build src cd build # Extract -cp ${DISTFILES}/${pkg}.tar.xz ../src/ -unxz --file ../src/${pkg}.tar.xz --output ../src/${pkg}.tar +cp ${DISTFILES}/${pkg}.tar.gz ../src/ +gzip -d -f ../src/${pkg}.tar.gz tar xf ../src/${pkg}.tar rm -r ../src cd ${pkg}diff --git a/steps/bzip2-1.0.8/sources b/steps/bzip2-1.0.8/sources@@ -1 +1 @@ -https://mirrors.kernel.org/slackware/slackware-14.0/patches/source/bzip2/bzip2-1.0.8.tar.xz 47fd74b2ff83effad0ddf62074e6fad1f6b4a77a96e121ab421c20a216371a1f +https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269diff --git a/steps/coreutils-9.4/sources b/steps/coreutils-9.4/sources@@ -1,7 +1,5 @@ -http://git.savannah.gnu.org/cgit/coreutils.git/snapshot/coreutils-9.4.tar.xz 8fb56810310253300b3d6f84e68dc97eb2d74e1f4f78e05776831d9d82e4f2d7 -https://files.bootstrapping.world/coreutils-9.4.tar.xz 8fb56810310253300b3d6f84e68dc97eb2d74e1f4f78e05776831d9d82e4f2d7 -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-bb5bb43.tar.gz b8aa1ac1b18c67f081486069e6a7a5564f20431c2313a94c20a46dcfb904be2a -https://files.bootstrapping.world/gnulib-bb5bb43.tar.gz b8aa1ac1b18c67f081486069e6a7a5564f20431c2313a94c20a46dcfb904be2a +git://git.savannah.gnu.org/coreutils.git~v9.4 http://git.savannah.gnu.org/cgit/coreutils.git/snapshot/coreutils-9.4.tar.xz 8fb56810310253300b3d6f84e68dc97eb2d74e1f4f78e05776831d9d82e4f2d7 +git://git.savannah.gnu.org/gnulib.git~bb5bb43 _ b8aa1ac1b18c67f081486069e6a7a5564f20431c2313a94c20a46dcfb904be2a gnulib-bb5bb43.tar.gz http://ftp.unicode.org/Public/15.0.0/ucd/UnicodeData.txt 806e9aed65037197f1ec85e12be6e8cd870fc5608b4de0fffd990f689f376a73 UnicodeData-15.0.0.txt http://ftp.unicode.org/Public/15.0.0/ucd/PropList.txt e05c0a2811d113dae4abd832884199a3ea8d187ee1b872d8240a788a96540bfd PropList-15.0.0.txt http://ftp.unicode.org/Public/15.0.0/ucd/DerivedCoreProperties.txt d367290bc0867e6b484c68370530bdd1a08b6b32404601b8c7accaf83e05628d DerivedCoreProperties-15.0.0.txtdiff --git a/steps/diffutils-3.10/sources b/steps/diffutils-3.10/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/diffutils/diffutils-3.10.tar.xz 90e5e93cc724e4ebe12ede80df1634063c7a855692685919bfe60b556c9bd09e -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-5d2fe24.tar.gz 72e7bb2d1d75e63d1c46d33b8dd22e8eb60afdba4af3e7251151b5c2a6f00bfb -https://files.bootstrapping.world/gnulib-5d2fe24.tar.gz 72e7bb2d1d75e63d1c46d33b8dd22e8eb60afdba4af3e7251151b5c2a6f00bfb +git://git.savannah.gnu.org/gnulib.git~5d2fe24 _ 72e7bb2d1d75e63d1c46d33b8dd22e8eb60afdba4af3e7251151b5c2a6f00bfb gnulib-5d2fe24.tar.gzdiff --git a/steps/ed-1.4/sources b/steps/ed-1.4/sources@@ -1 +1 @@ -https://mirrors.kernel.org/slackware/slackware-13.37/source/a/ed/ed-1.4.tar.xz 102c80e6da527c6b8eebd5195cd05fc71808d60735d73e8bb503a5e294475007 +https://mirrors.kernel.org/gnu/ed/ed-1.4.tar.gz db36da85ee1a9d8bafb4b041bd4c8c11becba0c43ec446353b67045de1634fdadiff --git a/steps/findutils-4.2.33/pass1.sh b/steps/findutils-4.2.33/pass1.sh@@ -3,8 +3,6 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -EXTRA_DISTFILES="gnulib-8e128e.tar.gz" - src_prepare() { . ../../import-gnulib.shdiff --git a/steps/findutils-4.2.33/sources b/steps/findutils-4.2.33/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/findutils/findutils-4.2.33.tar.gz 813cd9405aceec5cfecbe96400d01e90ddad7b512d3034487176ce5258ab0f78 -https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-8e128e.tar.gz 0cfbf866bc39c31f25fa0e56af1e56c5e5c92fc1e5d51242ebafef7ea211f3d5 -https://files.bootstrapping.world/gnulib-8e128e.tar.gz 0cfbf866bc39c31f25fa0e56af1e56c5e5c92fc1e5d51242ebafef7ea211f3d5 +git://git.savannah.gnu.org/gnulib.git~8e128e _ 0cfbf866bc39c31f25fa0e56af1e56c5e5c92fc1e5d51242ebafef7ea211f3d5 gnulib-8e128e.tar.gzdiff --git a/steps/gcc-10.4.0/sources b/steps/gcc-10.4.0/sources@@ -1 +1 @@ -http://ftp.gnu.org/gnu/gcc/gcc-10.4.0/gcc-10.4.0.tar.xz c9297d5bcd7cb43f3dfc2fed5389e948c9312fd962ef6a4ce455cff963ebe4f1 +http://mirrors.kernel.org/gnu/gcc/gcc-10.4.0/gcc-10.4.0.tar.xz c9297d5bcd7cb43f3dfc2fed5389e948c9312fd962ef6a4ce455cff963ebe4f1diff --git a/steps/gcc-13.1.0/sources b/steps/gcc-13.1.0/sources@@ -1 +1 @@ -https://ftp.gnu.org/gnu/gcc/gcc-13.1.0/gcc-13.1.0.tar.xz 61d684f0aa5e76ac6585ad8898a2427aade8979ed5e7f85492286c4dfc13ee86 +https://mirrors.kernel.org/gnu/gcc/gcc-13.1.0/gcc-13.1.0.tar.xz 61d684f0aa5e76ac6585ad8898a2427aade8979ed5e7f85492286c4dfc13ee86diff --git a/steps/gettext-0.21/sources b/steps/gettext-0.21/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/gettext/gettext-0.21.tar.xz d20fcbb537e02dcf1383197ba05bd0734ef7bf5db06bdb241eb69b7d16b73192 -https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-7daa86f.tar.gz 2d911c2f2ed97b347d6d360b742abdc98aa626d4f8f847ee682c7cde12e90871 -https://files.bootstrapping.world/gnulib-7daa86f.tar.gz 2d911c2f2ed97b347d6d360b742abdc98aa626d4f8f847ee682c7cde12e90871 +git://git.savannah.gnu.org/gnulib.git~7daa86f _ 2d911c2f2ed97b347d6d360b742abdc98aa626d4f8f847ee682c7cde12e90871 gnulib-7daa86f.tar.gzdiff --git a/steps/grep-3.7/pass1.sh b/steps/grep-3.7/pass1.sh@@ -2,9 +2,6 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -urls="https://mirrors.kernel.org/gnu/grep/grep-3.7.tar.xz - http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-8f4538a5.tar.gz" - src_prepare() { rm configure find . -name 'Makefile.in' -deletediff --git a/steps/grep-3.7/sources b/steps/grep-3.7/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/grep/grep-3.7.tar.xz 5c10da312460aec721984d5d83246d24520ec438dd48d7ab5a05dbc0d6d6823c -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-8f4538a5.tar.gz e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab -https://files.bootstrapping.world/gnulib-8f4538a5.tar.gz e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab +git://git.savannah.gnu.org/gnulib.git~8f4538a5 _ e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab gnulib-8f4538a5.tar.gzdiff --git a/steps/grub-2.06/sources b/steps/grub-2.06/sources@@ -1,6 +1,5 @@ https://mirrors.kernel.org/gnu/grub/grub-2.06.tar.xz b79ea44af91b93d17cd3fe80bdae6ed43770678a9a5ae192ccea803ebb657ee1 -https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-d271f86.tar.gz 31d69d3d251e39135b5194ddc6f897910d344059f7494d96a739aecbf7ac2b66 -https://files.bootstrapping.world/gnulib-d271f86.tar.gz 31d69d3d251e39135b5194ddc6f897910d344059f7494d96a739aecbf7ac2b66 +git://git.savannah.gnu.org/gnulib.git~d271f86 _ 31d69d3d251e39135b5194ddc6f897910d344059f7494d96a739aecbf7ac2b66 gnulib-d271f86.tar.gz http://ftp.unicode.org/Public/9.0.0/ucd/UnicodeData.txt 68dfc414d28257b9b5d6ddbb8b466c768c00ebdf6cbf7784364a9b6cad55ee8f UnicodeData-9.0.0.txt http://ftp.unicode.org/Public/9.0.0/ucd/PropList.txt f413ea8dbd3858de72f3148b47dd0586019761357d1481e3b65f3a025bc27f82 PropList-9.0.0.txt http://ftp.unicode.org/Public/9.0.0/ucd/DerivedCoreProperties.txt 6662c7e30b572df5d948c092692f52bcc79ab36d49a063a73d6435042db6fb3b DerivedCoreProperties-9.0.0.txtdiff --git a/steps/guile-3.0.9/sources b/steps/guile-3.0.9/sources@@ -1,7 +1,5 @@ https://mirrors.kernel.org/gnu/guile/guile-3.0.7.tar.xz f57d86c70620271bfceb7a9be0c81744a033f08adc7ceba832c9917ab3e691b7 https://mirrors.kernel.org/gnu/guile/guile-3.0.9.tar.xz 1a2625ac72b2366e95792f3fe758fd2df775b4044a90a4a9787326e66c0d750d -https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-901694b9.tar.gz f9aad85de1f41d57c9368d304020ffbf354a5e56db1297f022c3d12181134e56 -https://files.bootstrapping.world/gnulib-901694b9.tar.gz f9aad85de1f41d57c9368d304020ffbf354a5e56db1297f022c3d12181134e56 -https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-356a414e.tar.gz fc9973f1a9243fdc4b98d33d7704f3c71bfdc4c2ef96899b8f28cade7290a714 -https://files.bootstrapping.world/gnulib-356a414e.tar.gz fc9973f1a9243fdc4b98d33d7704f3c71bfdc4c2ef96899b8f28cade7290a714 -https://github.com/schierlm/guile-psyntax-bootstrapping/archive/refs/tags/guile-3.0.7.tar.gz 14cda9c416506dfadf60c14fc623ff01ef99b87564a78d0a29c5d17143c97609 +git://git.savannah.gnu.org/gnulib.git~901694b9 _ f9aad85de1f41d57c9368d304020ffbf354a5e56db1297f022c3d12181134e56 gnulib-901694b9.tar.gz +git://git.savannah.gnu.org/gnulib.git~356a414e _ fc9973f1a9243fdc4b98d33d7704f3c71bfdc4c2ef96899b8f28cade7290a714 gnulib-356a414e.tar.gz +git://github.com/schierlm/guile-psyntax-bootstrapping~guile-3.0.7 https://github.com/schierlm/guile-psyntax-bootstrapping/archive/refs/tags/guile-3.0.7.tar.gz 14cda9c416506dfadf60c14fc623ff01ef99b87564a78d0a29c5d17143c97609diff --git a/steps/gzip-1.13/sources b/steps/gzip-1.13/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/gzip/gzip-1.13.tar.xz 7454eb6935db17c6655576c2e1b0fabefd38b4d0936e0f87f48cd062ce91a057 -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-5651802.tar.gz 56f1221eb682c3502ee097f583f44673570753cb452346ad4806d94560c3fac9 -https://files.bootstrapping.world/gnulib-5651802.tar.gz 56f1221eb682c3502ee097f583f44673570753cb452346ad4806d94560c3fac9 +git://git.savannah.gnu.org/gnulib.git~5651802 _ 56f1221eb682c3502ee097f583f44673570753cb452346ad4806d94560c3fac9 gnulib-5651802.tar.gzdiff --git a/steps/helpers.sh b/steps/helpers.sh@@ -21,8 +21,6 @@ _get_files() { prefix="${1}" fs= if [ -n "$(ls 2>/dev/null)" ]; then - # This can be removed once Debian 12 is stable - # shellcheck disable=SC2035 fs=$(echo *) fi if [ -n "$(ls .[0-z]* 2>/dev/null)" ]; then @@ -258,18 +256,48 @@ build() { unset extract } +# An inventive way to randomise with what we know we always have +randomize() { + if command -v shuf >/dev/null 2>&1; then + # shellcheck disable=SC2086 + shuf -e ${1} | tr '\n' ' ' + else + mkdir -p /tmp/random + for item in ${1}; do + touch --date=@${RANDOM} /tmp/random/"${item//\//~*~*}" + done + # cannot rely on find existing + # shellcheck disable=SC2012 + ls -1 -t /tmp/random | sed 's:~\*~\*:/:g' | tr '\n' ' ' + rm -r /tmp/random + fi +} + download_source_line() { - url="${1}" + if [[ "${1}" == git://* ]]; then + shift + fi + upstream_url="${1}" checksum="${2}" fname="${3}" # Default to basename of url if not given - fname="${fname:-$(basename "${url}")}" + fname="${fname:-$(basename "${upstream_url}")}" if ! [ -e "${fname}" ]; then - curl --fail --retry 5 --location "${url}" --output "${fname}" || true + for mirror in $(randomize "${MIRRORS}"); do + mirror_url="${mirror}/${fname}" + echo "${mirror_url}" + curl --fail --retry 3 --location "${mirror_url}" --output "${fname}" || true && break + done + if ! [ -e "${fname}" ] && [ "${upstream_url}" != "_" ]; then + curl --fail --retry 3 --location "${upstream_url}" --output "${fname}" || true + fi fi } check_source_line() { + if [[ "${1}" == git://* ]]; then + shift + fi url="${1}" checksum="${2}" fname="${3}" @@ -304,6 +332,9 @@ default_src_get() { # Intelligently extracts a file based upon its filetype. extract_file() { + if [[ "${1}" == git://* ]]; then + shift + fi f="${3:-$(basename "${1}")}" # shellcheck disable=SC2154 case "${noextract}" indiff --git a/steps/improve/setup_repo.sh b/steps/improve/setup_repo.sh@@ -6,4 +6,4 @@ # mkdir -p /external/repo -tar -cf - --exclude='/external/repo/*' --exclude='/external/repo-preseeded/*' --exclude='/external/distfiles/*' --exclude='/dev/*' --exclude='/proc/*' --exclude='/sys/*' --exclude='/tmp/*' / | bzip2 --best > /external/repo/base.tar.bz2 +#tar -cf - --exclude='/external/repo/*' --exclude='/external/repo-preseeded/*' --exclude='/external/distfiles/*' --exclude='/dev/*' --exclude='/proc/*' --exclude='/sys/*' --exclude='/tmp/*' / | bzip2 --best > /external/repo/base.tar.bz2diff --git a/steps/kexec-tools-2.0.22/sources b/steps/kexec-tools-2.0.22/sources@@ -1 +1 @@ -https://github.com/horms/kexec-tools/archive/refs/tags/v2.0.22.tar.gz af618de7848142f204b57811f703de3ae7aa3f5bc5d52226db35800fa8fc4dff +git://github.com/horms/kexec-tools~v2.0.22 https://github.com/horms/kexec-tools/archive/refs/tags/v2.0.22.tar.gz af618de7848142f204b57811f703de3ae7aa3f5bc5d52226db35800fa8fc4dffdiff --git a/steps/libtool-2.4.7/sources b/steps/libtool-2.4.7/sources@@ -1,3 +1,2 @@ http://mirrors.kernel.org/gnu/libtool/libtool-2.4.7.tar.xz 4f7f217f057ce655ff22559ad221a0fd8ef84ad1fc5fcb6990cecc333aa1635d -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-a521820.tar.gz 719b399fe09a8f6ca14ba8c4a9a60ce9f93f4892effb50961ef3d8cd1a33ff65 -https://files.bootstrapping.world/gnulib-a521820.tar.gz 719b399fe09a8f6ca14ba8c4a9a60ce9f93f4892effb50961ef3d8cd1a33ff65 +git://git.savannah.gnu.org/gnulib.git~a521820 _ 719b399fe09a8f6ca14ba8c4a9a60ce9f93f4892effb50961ef3d8cd1a33ff65 gnulib-a521820.tar.gzdiff --git a/steps/libunistring-0.9.10/sources b/steps/libunistring-0.9.10/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/libunistring/libunistring-0.9.10.tar.xz eb8fb2c3e4b6e2d336608377050892b54c3c983b646c561836550863003c05d7 -https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-52a06cb3.tar.gz 009989b81c0bebc5f6550636ed653fbcb237dafc2af5c706f3522087ca571e4d -https://files.bootstrapping.world/gnulib-52a06cb3.tar.gz 009989b81c0bebc5f6550636ed653fbcb237dafc2af5c706f3522087ca571e4d +git://git.savannah.gnu.org/gnulib.git~52a06cb3 _ 009989b81c0bebc5f6550636ed653fbcb237dafc2af5c706f3522087ca571e4d gnulib-52a06cb3.tar.gzdiff --git a/steps/m4-1.4.19/sources b/steps/m4-1.4.19/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/m4/m4-1.4.19.tar.xz 63aede5c6d33b6d9b13511cd0be2cac046f2e70fd0a07aa9573a04a82783af96 -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-3639c57.tar.gz 97dfbad67832641bc7f73437617b78abeafb9946723f19cf4c2ceecfc65fa48d -https://files.bootstrapping.world/gnulib-3639c57.tar.gz 97dfbad67832641bc7f73437617b78abeafb9946723f19cf4c2ceecfc65fa48d +git://git.savannah.gnu.org/gnulib.git~3639c57 _ 97dfbad67832641bc7f73437617b78abeafb9946723f19cf4c2ceecfc65fa48d gnulib-3639c57.tar.gzdiff --git a/steps/openssl-3.0.13/sources b/steps/openssl-3.0.13/sources@@ -1 +1 @@ -http://snapshot.debian.org/archive/debian/20240325T091329Z/pool/main/o/openssl/openssl_3.0.13.orig.tar.gz 88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313 +https://github.com/openssl/openssl/releases/download/openssl-3.0.13/openssl-3.0.13.tar.gz 88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313diff --git a/steps/patch-2.7.6/sources b/steps/patch-2.7.6/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/patch/patch-2.7.6.tar.xz ac610bda97abe0d9f6b7c963255a11dcb196c25e337c61f94e4778d632f1d8fd -https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-e017871.tar.gz a285dc300c3d9c25cc06e38827ef40f6073ec3b9b0fcb5bba433f943be92d8d4 -https://files.bootstrapping.world/gnulib-e017871.tar.gz a285dc300c3d9c25cc06e38827ef40f6073ec3b9b0fcb5bba433f943be92d8d4 +git://git.savannah.gnu.org/gnulib.git~e017871 _ a285dc300c3d9c25cc06e38827ef40f6073ec3b9b0fcb5bba433f943be92d8d4 gnulib-e017871.tar.gzdiff --git a/steps/pkg-config-0.29.2/sources b/steps/pkg-config-0.29.2/sources@@ -1 +1 @@ -http://distfiles.macports.org/pkgconfig/pkg-config-0.29.2.tar.gz 6fc69c01688c9458a57eb9a1664c9aba372ccda420a02bf4429fe610e7e7d591 +https://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz 6fc69c01688c9458a57eb9a1664c9aba372ccda420a02bf4429fe610e7e7d591diff --git a/steps/python-3.11.1/sources b/steps/python-3.11.1/sources@@ -22,4 +22,3 @@ http://ftp.unicode.org/Public/14.0.0/ucd/Unihan.zip 2ae4519b2b82cd4d15379c17e57b http://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP437.TXT 6bad4dabcdf5940227c7d81fab130dcb18a77850b5d79de28b5dc4e047b0aaac https://www.ietf.org/rfc/rfc3454.txt eb722fa698fb7e8823b835d9fd263e4cdb8f1c7b0d234edf7f0e3bd2ccbb2c79 https://github.com/openssl/openssl/releases/download/openssl-3.0.13/openssl-3.0.13.tar.gz 88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313 -https://github.com/openssl/openssl/releases/download/openssl-3.0.13/openssl-3.0.13.tar.gz 88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313diff --git a/steps/sed-4.8/sources b/steps/sed-4.8/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/sed/sed-4.8.tar.xz f79b0cfea71b37a8eeec8490db6c5f7ae7719c35587f21edb0617f370eeff633 -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-d279bc.tar.gz 12cfa21abf618a274017d6b18e95fc6582519d7c08e2403e5c5772ccdd5b85f4 -https://files.bootstrapping.world/gnulib-d279bc.tar.gz 12cfa21abf618a274017d6b18e95fc6582519d7c08e2403e5c5772ccdd5b85f4 +git://git.savannah.gnu.org/gnulib.git~d279bc _ 12cfa21abf618a274017d6b18e95fc6582519d7c08e2403e5c5772ccdd5b85f4 gnulib-d279bc.tar.gzdiff --git a/steps/tar-1.34/sources b/steps/tar-1.34/sources@@ -1,3 +1,2 @@ http://mirrors.kernel.org/gnu/tar/tar-1.34.tar.xz 63bebd26879c5e1eea4352f0d03c991f966aeb3ddeb3c7445c902568d5411d28 -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-30820c.tar.gz df807e694deea2dcba0c43af318394f3e3fcd52658c3b71b61dad0ce0c0cfb77 -https://files.bootstrapping.world/gnulib-30820c.tar.gz df807e694deea2dcba0c43af318394f3e3fcd52658c3b71b61dad0ce0c0cfb77 +git://git.savannah.gnu.org/gnulib.git~30820c _ df807e694deea2dcba0c43af318394f3e3fcd52658c3b71b61dad0ce0c0cfb77 gnulib-30820c.tar.gzdiff --git a/steps/texinfo-6.7/sources b/steps/texinfo-6.7/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/texinfo/texinfo-6.7.tar.xz 988403c1542d15ad044600b909997ba3079b10e03224c61188117f3676b02caa -https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-b81ec69.tar.gz 1aeea67b7b3883ebcf2b90bc01f4182d7de073a052dabd3749f20c5aa4ad3e27 -https://files.bootstrapping.world/gnulib-b81ec69.tar.gz 1aeea67b7b3883ebcf2b90bc01f4182d7de073a052dabd3749f20c5aa4ad3e27 +git://git.savannah.gnu.org/gnulib.git~b81ec69 _ 1aeea67b7b3883ebcf2b90bc01f4182d7de073a052dabd3749f20c5aa4ad3e27 gnulib-b81ec69.tar.gzdiff --git a/steps/xz-5.4.1/sources b/steps/xz-5.4.1/sources@@ -1 +1 @@ -http://ixpeering.dl.sourceforge.net/project/lzmautils/xz-5.4.1.tar.bz2 dd172acb53867a68012f94c17389401b2f274a1aa5ae8f84cbfb8b7e383ea8d3 +https://github.com/tukaani-project/xz/releases/download/v5.4.1/xz-5.4.1.tar.bz2 dd172acb53867a68012f94c17389401b2f274a1aa5ae8f84cbfb8b7e383ea8d3