commit: b934782b5f4d578ae3acd32ba488b8108c83bd53 parent 3896c1dfc9001fcbfaf1e80a27bd0cd43205106a Author: Samuel Tyler <fosslinux@aussies.space> Date: Mon, 3 Feb 2025 10:13:46 +1100 Merge pull request #493 from fosslinux/mirrors MirrorsDiffstat:
62 files changed, 580 insertions(+), 338 deletions(-)diff --git a/.github/workflows/bwrap.yml b/.github/workflows/bwrap.yml@@ -32,20 +32,24 @@ jobs: uses: actions/cache/restore@v4 with: path: | - distfiles + mirror + mirror-state key: cache-${{ hashFiles('steps/*/sources') }} + restore-keys: | + cache- - name: Get sources if: steps.cache.outputs.cache-hit != 'true' - run: ./download-distfiles.sh + run: mkdir -p mirror mirror-state && ./mirror.sh mirror mirror-state - name: Cache sources if: steps.cache.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: path: | - distfiles + mirror + mirror-state key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass1 + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass1 --mirror file://${PWD}/mirror - name: Archive created packages if: failure() # archive failed builds progress uses: actions/upload-artifact@v4 @@ -87,18 +91,12 @@ jobs: uses: actions/cache/restore@v4 with: path: | - distfiles - key: cache-${{ hashFiles('steps/*/sources') }} - - name: Get sources - if: steps.cache.outputs.cache-hit != 'true' - run: ./download-distfiles.sh - - name: Cache sources - if: steps.cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: | - distfiles + mirror + mirror-state key: cache-${{ hashFiles('steps/*/sources') }} + fail-on-cache-miss: true + - name: Copy distfiles + run: ./download-distfiles.sh file:///${PWD}/mirror - name: Run bootstrap run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass2 - name: Archive created packages @@ -142,20 +140,14 @@ jobs: uses: actions/cache/restore@v4 with: path: | - distfiles - key: cache-${{ hashFiles('steps/*/sources') }} - - name: Get sources - if: steps.cache.outputs.cache-hit != 'true' - run: ./download-distfiles.sh - - name: Cache sources - if: steps.cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: | - distfiles + mirror + mirror-state key: cache-${{ hashFiles('steps/*/sources') }} + fail-on-cache-miss: true + - name: Copy distfiles + run: ./download-distfiles.sh file:///${PWD}/mirror - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3 + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3 --mirror file://${PWD}/mirror - name: Archive created packages if: always() # archive both failed and successful builds uses: actions/upload-artifact@v4diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml@@ -21,7 +21,7 @@ jobs: - name: Install pylint run: sudo pip3 install pylint - name: pylint - run: pylint rootfs.py lib/utils.py lib/generator.py lib/target.py check_substitutes.py --disable=duplicate-code + run: pylint rootfs.py source_manifest.py lib/*.py --disable=duplicate-code shellcheck: name: Lint shell files @@ -30,7 +30,7 @@ jobs: - name: Checkout repo uses: actions/checkout@v3 - name: shellcheck - run: shellcheck steps/helpers.sh download-distfiles.sh + run: shellcheck steps/helpers.sh download-distfiles.sh mirror.sh reuse: name: Lint reuse information @@ -42,12 +42,3 @@ jobs: run: sudo pip3 install reuse - name: reuse run: reuse lint - - substitutes: - name: Check validity of substituted tarballs - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@v3 - - name: Check validity of substitutes - run: ./check_substitutes.pydiff --git a/DEVEL.md b/DEVEL.md@@ -84,3 +84,43 @@ libraries that are compiled and installed. - Patches are licensed under the license of the project which they are patching. - All files (excluding files within submodules) must comply with REUSE v3.0. + +## `sources` file + +The format of the sources file for a HTTP source file is: + +``` +<url> <checksum> [filename] +``` + +For a Git repository snapshot: + +``` +<git url> <URL to HTTP snapshot> <checksum> [filename] +``` + +The format of a git url must always be `git://<path to repository>~<reference>`. +`reference` is, for instance, a commit, tag or branch. Always use +Git protocol path, except for GitHub, in which `git://` is automagically +changed to `https://`. + +The URL to HTTP snapshot may be `_` (a single underscore) if no HTTP snapshot +exists. In this case, the filename is compulsory. The checksum is of the Git +snapshot, generated with `git archive`. See `mirror.sh` for more detailed rules. + +Some specific helpful things: +- prefer `.tar.gz` +- For GitHub, all snapshots are generated with long commit IDs, so use long + commit IDs. + +## Running a mirror + +There are 3 things you need to run a mirror; + +1. A tool to perform the mirroring (e.g. `mirror.sh` in this repository); we + encourage diversity in this area. +2. A server on which to host the mirror; this should have HTTP support, and + optionally HTTPS. It is of greatly diminished utility if it does not support + plain HTTP. +3. Automatic updates. This should involve updating the Git repository and + updating the mirror as required.diff --git a/LICENSES/BSD-3-Clause-flex.txt b/LICENSES/BSD-3-Clause-flex.txt@@ -0,0 +1,42 @@ +Flex carries the copyright used for BSD software, slightly modified +because it originated at the Lawrence Berkeley (not Livermore!) Laboratory, +which operates under a contract with the Department of Energy: + +Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007 The Flex Project. + +Copyright (c) 1990, 1997 The Regents of the University of California. +All rights reserved. + +This code is derived from software contributed to Berkeley by +Vern Paxson. + +The United States Government has rights in this work pursuant +to contract no. DE-AC03-76SF00098 between the United States +Department of Energy and the University of California. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +Neither the name of the University nor the names of its contributors +may be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. + +This basically says "do whatever you please with this software except +remove this notice or take advantage of the University's (or the flex +authors') name". + +Note that the "flex.skl" scanner skeleton carries no copyright notice. +You are free to do whatever you please with scanners generated using flex; +for them, you are not even bound by the above copyright.diff --git a/check_substitutes.py b/check_substitutes.py@@ -1,162 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: GPL-3.0-or-later -# -# SPDX-FileCopyrightText: 2024 fosslinux <fosslinux@aussies.space> - -"""Check that substituted files are the same.""" -import bz2 -import filecmp -import gzip -import itertools -import lzma -import shutil -import tarfile -import tempfile -import sys -import os - -from lib.generator import Generator - -# Get a temporary directory to work in -working = tempfile.mkdtemp() - -# Colour constants -# pylint: disable=too-few-public-methods -class Colors(): - """ANSI Color Codes""" - GREY = "\033[90m" - RED = "\033[91m" - GREEN = "\033[92m" - ORANGE = "\033[91m\033[93m" - YELLOW = "\033[93m" - END = "\033[0m" - -def traverse_path(base_root): - """Takes a path and returns a set of all directories and files in that path.""" - all_dirs = set() - all_files = set() - for root, directories, files in os.walk(base_root, topdown=True): - for d in directories: - all_dirs.add(os.path.join(root, d).lstrip(base_root)) - for f in files: - all_files.add(os.path.join(root, f).lstrip(base_root)) - return (all_dirs, all_files) - -class Distfile(): - """Represents one distfile and operations performed on it.""" - def __init__(self, i, url): - self.i = i - self.url = url - self.out_file = f"{i}-{os.path.basename(url)}" - self.filepath = "" - - def download(self): - """Downloads the distfile.""" - Generator.download_file(self.url, working, self.out_file, silent=True) - self.filepath = os.path.join(working, self.out_file) - - def decompress(self): - """Decompresses the distfile.""" - compression = self.out_file.rsplit('.', maxsplit=1)[-1] - decompress_func = { - "gz": gzip.open, - "tgz": gzip.open, - "bz2": bz2.open, - "xz": lzma.open, - "lzma": lzma.open - } - if compression not in decompress_func: - # No decompression needed - return - # Remove the compression extension - new_path = '.'.join(self.filepath.split('.')[:-1]) - # tgz -> .tar - if compression == "tgz": - new_path = f"{new_path}.tar" - # Move the decompressed binary stream to a new file - with decompress_func[compression](self.filepath, 'rb') as fin: - with open(new_path, 'wb') as fout: - shutil.copyfileobj(fin, fout) - self.filepath = new_path - - def extract(self): - """Extracts the distfile.""" - # Sanity check - if not tarfile.is_tarfile(self.filepath): - return - out_dir = os.path.join(working, f"{self.i}") - os.mkdir(out_dir) - with tarfile.open(self.filepath, 'r') as f: - f.extractall(path=out_dir) - self.filepath = out_dir - - # It makes more sense here to label them d1 and d2 rather than have one be self. - # pylint: disable=no-self-argument - def compare(d1, d2): - """Compares the distfile to another distfile.""" - if not os.path.isdir(d1.filepath): - # Compare files - return filecmp.cmp(d1.filepath, d2.filepath, shallow=False) - if not os.path.isdir(d2.filepath): - # Then, d2 is a file and d1 is a directory - return False - # Otherwise it's two directories - dirnames1, filenames1 = traverse_path(d1.filepath) - dirnames2, filenames2 = traverse_path(d2.filepath) - if dirnames1 != dirnames2: - return False - if filenames1 != filenames2: - return False - return filecmp.cmpfiles(d1.filepath, d2.filepath, filenames1, shallow=False) - -def check(*args): - """Check if a list of distfiles are equivalent.""" - notequiv = [] - # Find all pairs that are not equivalent - for pair in itertools.combinations(args, 2): - if pair[0].compare(pair[1]): - print(f"{Colors.GREY}DEBUG: {pair[0].url} is equivalent to {pair[1].url}{Colors.END}") - else: - notequiv.append(pair) - - # Decompress all, and check again - for d in {y for x in notequiv for y in x}: - d.decompress() - for pair in notequiv.copy(): - if pair[0].compare(pair[1]): - # pylint: disable=line-too-long - print(f"{Colors.YELLOW}NOTE: {pair[0].url} is equivalent to {pair[1].url} when decompressed{Colors.END}") - notequiv.remove(pair) - - # Extract all, and check again - for d in {y for x in notequiv for y in x}: - d.extract() - has_error = False - for pair in notequiv: - if pair[0].compare(pair[1]): - # pylint: disable=line-too-long - print(f"{Colors.ORANGE}WARN: {pair[0].url} is equivalent to {pair[1].url} when extracted{Colors.END}") - else: - has_error = True - # pylint: disable=line-too-long - print(f"{Colors.RED}ERROR: {pair[0].url} is not equivalent to {pair[1].url}!{Colors.END}") - - return has_error - -def main(): - """Main function.""" - has_error = False - with open("substitutes", 'r', encoding="utf-8") as f: - for line in f.readlines(): - urls = line.strip().split(' ') - distfiles = [] - for i, url in enumerate(urls): - distfiles.append(Distfile(i, url)) - for distfile in distfiles: - distfile.download() - if check(*distfiles): - has_error = True - sys.exit(has_error) - -if __name__ == "__main__": - main()diff --git a/download-distfiles.sh b/download-distfiles.sh@@ -4,35 +4,58 @@ # # SPDX-License-Identifier: GPL-3.0-or-later +# Optional arguments: Mirrors + download_source() { - local distfiles="${1}" - local url="${2}" - local checksum="${3}" - local fname="${4}" + local distfiles=${1} + local url=${2} + shift 2 + if [[ ${url} == git://* ]]; then + url=${1} + shift + fi + local checksum=${1} + local fname=${2} # Default to basename of url if not given - fname="${fname:-$(basename "${url}")}" + fname=${fname:-$(basename "${url}")} + if [ "${fname}" = "_" ]; then + echo "ERROR: ${url} must have a filename specified" + exit 1 + fi - local dest_path="${distfiles}/${fname}" + local dest_path=${distfiles}/${fname} if ! [ -e "${dest_path}" ]; then echo "Downloading ${fname}" + if [ "${mirrors_len}" -ne 0 ]; then + local mirror_ix=$((RANDOM % mirrors_len)) + url=${mirrors[${mirror_ix}]}/${fname} + fi curl --fail --location "${url}" --output "${dest_path}" || true fi } check_source() { - local distfiles="${1}" - local url="${2}" - local checksum="${3}" - local fname="${4}" + local distfiles=${1} + local url=${2} + shift 2 + if [[ ${url} == git://* ]]; then + url=${1} + shift + fi + local checksum=${1} + local fname=${2} # Default to basename of url if not given - fname="${fname:-$(basename "${url}")}" + fname=${fname:-$(basename "${url}")} - local dest_path="${distfiles}/${fname}" + local dest_path=${distfiles}/${fname} echo "${checksum} ${dest_path}" | sha256sum -c } set -e +mirrors=( "$@" ) +mirrors_len=$# + cd "$(dirname "$(readlink -f "$0")")" mkdir -p distfilesdiff --git a/lib/generator.py b/lib/generator.py@@ -10,9 +10,11 @@ This file contains all code required to generate the boot image for live-bootstr import hashlib import os +import random import shutil import tarfile import traceback + import requests # pylint: disable=too-many-instance-attributes @@ -24,11 +26,13 @@ class Generator(): git_dir = os.path.join(os.path.dirname(os.path.join(__file__)), '..') distfiles_dir = os.path.join(git_dir, 'distfiles') - def __init__(self, arch, external_sources, early_preseed, repo_path): + # pylint: disable=too-many-arguments,too-many-positional-arguments + def __init__(self, arch, external_sources, early_preseed, repo_path, mirrors): self.arch = arch self.early_preseed = early_preseed self.external_sources = external_sources self.repo_path = repo_path + self.mirrors = mirrors self.source_manifest = self.get_source_manifest(not self.external_sources) self.early_source_manifest = self.get_source_manifest(True) self.target_dir = None @@ -278,8 +282,7 @@ actual: {readable_hash}\n\ When in doubt, try deleting the file in question -- it will be downloaded again when running \ this script the next time") - @staticmethod - def download_file(url, directory, file_name, silent=False): + def download_file(self, url, directory, file_name, silent=False): """ Download a single source archive. """ @@ -297,14 +300,33 @@ this script the next time") if not os.path.isfile(abs_file_name): if not silent: print(f"Downloading: {file_name}") - response = requests.get(url, allow_redirects=True, stream=True, - headers=headers, timeout=20) - if response.status_code == 200: - with open(abs_file_name, 'wb') as target_file: - target_file.write(response.raw.read()) + + def do_download(source): + response = requests.get(source, allow_redirects=True, stream=True, + headers=headers, timeout=20) + if response.status_code == 200: + with open(abs_file_name, 'wb') as target_file: + target_file.write(response.raw.read()) + return True + print(f"Download failed from {option}: {response.status_code} {response.reason}") + return False + + done = False + if self.mirrors: + options = [f"{x}/{file_name}" for x in self.mirrors] else: - raise requests.HTTPError("Download failed: HTTP " + - str(response.status_code) + " " + response.reason) + options = [] + random.shuffle(options) + for option in options: + if do_download(option): + done = True + break + + if not done: + if url == "_" or not do_download(url): + raise requests.RequestException(f"Unable to download {url} from ", + "any mirror or original") + return abs_file_name def get_packages(self): @@ -344,6 +366,9 @@ this script the next time") for source in sources.readlines(): source = source.strip().split(" ") + if source[0].startswith("git://"): + source = source[1:] + if len(source) > 2: file_name = source[2] else:diff --git a/lib/simple_mirror.py b/lib/simple_mirror.py@@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +""" +This creates a simple "mirror" from a directory +""" +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2025 fosslinux <fosslinux@aussies.space> + +import http.server +import socketserver + +class SimpleMirror(socketserver.TCPServer): + """Simple HTTP mirror from a directory""" + def __init__(self, directory: str): + self.directory = directory + super().__init__(("localhost", 0), self._handler) + + @property + def port(self): + """Port the HTTP server of the mirror is running on""" + return self.server_address[1] + + def _handler(self, *args, **kwargs): + return http.server.SimpleHTTPRequestHandler(*args, directory=self.directory, **kwargs)diff --git a/lib/utils.py b/lib/utils.py@@ -12,7 +12,7 @@ import shutil import subprocess import sys -def run(*args, **kwargs): +def run(*args, cleanup=None, **kwargs): """A small wrapper around subprocess.run""" arguments = [str(arg) for arg in args if arg is not None] @@ -23,6 +23,8 @@ def run(*args, **kwargs): return subprocess.run(arguments, check=True, **kwargs) except subprocess.CalledProcessError: print("Bootstrapping failed") + if cleanup: + cleanup() sys.exit(1) def run_as_root(*args, **kwargs):diff --git a/mirror.sh b/mirror.sh@@ -0,0 +1,186 @@ +#!/bin/sh +# This script is intentionally written in POSIX sh to be cross-platform. +# +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2025 fosslinux <fosslinux@aussies.space> + +set -e + +# Check arguments, etc +if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then + echo "Usage: $0 <destination> [state]" + exit 1 +fi + +dest=$1 +if [ ! -d "${dest}" ]; then + echo "${dest} must be a directory" + exit 1 +fi + +if [ ! -w "${dest}" ]; then + echo "you must be able to write to ${dest}" + exit 1 +fi +dest=$(realpath "${dest}") + +state=$2 +if [ "${state}" = "" ]; then + state="${PWD}/mirrorstate" +fi +state="$(realpath "${state}")" + +####### + +# Download a HTTP file +download_file() { + url=${1} + out=${2} + # Download the file + continue_arg="" + if [ -e "${out}" ]; then + continue_arg="-C -" + fi + # shellcheck disable=SC2086 + curl -L ${continue_arg} -o "${out}" "${url}" +} + +# Check if a git reference exists in a given repository +git_ref_exists() { + repo=${1} + ref=${2} + # change this to git show-ref once it is sufficiently not-new + ( cd "${repo}" || exit && git cat-file -t "${ref}" >/dev/null 2>&1 ) + return $? +} + +checksum_file() { + sha256sum "${1}" | cut -d' ' -f1 +} + +do_file() { + uri=${1} + echo "${uri}" + + if echo "${uri}" | grep -qE "^https?://"; then + # HTTP file + checksum=${2} + filename=${3} + if [ "${filename}" = "" ]; then + filename=$(basename "${uri}") + fi + + # Check if the file is already downloaded & the checksum is the same + dest_file=${dest}/${filename} + if [ -e "${dest_file}" ]; then + existing_checksum=$(checksum_file "${dest_file}") + if [ "${checksum}" = "${existing_checksum}" ]; then + # There is nothing we need to do here + return + fi + fi + + # Attempt up to 3 times + retries=3 + matching=no + while [ "${retries}" -gt 0 ]; do + download_file "${uri}" "${dest}/${filename}" + my_checksum=$(checksum_file "${dest_file}") + if [ "${checksum}" = "${my_checksum}" ]; then + matching="yes" + break + fi + retries=$((retries - 1)) + if [ "${retries}" -gt 0 ]; then + echo "${uri}: checksum did not match, trying again" + rm "${dest}/${filename}" + fi + sleep 1 + done + + if [ "${matching}" = "no" ]; then + echo "${uri}: checksums do not match" + exit 1 + fi + elif echo "${uri}" | grep -qE "^git://"; then + # Creating a tarball from a git repository + + # Very unfortunately, different sites have different rules. + uri_path=${uri#git://} + # GitHub does not have git:// protocol support + if echo "${uri}" | grep -Eq "^git://github.com"; then + uri=https://${uri_path} + fi + repo=${uri%~*} + outdir=${state}/git/${repo#*://} + reference=${uri##*~} + + http_src=${2} + checksum=${3} + tarball=${4:-$(basename "${http_src}")} + if [ "${tarball}" = "_" ]; then + echo "${uri}: ERROR! Must have tarball name if no http source." + exit 1 + fi + tarball=${dest}/${tarball} + + # Check if tarball already generated + matches checksum + checksum=${3} + if [ -e "${tarball}" ]; then + existing_checksum=$(checksum_file "${tarball}") + if [ "${existing_checksum}" = "${checksum}" ]; then + return + fi + rm "${tarball}" + fi + + # Clone the repository, or update it if needed + if [ ! -e "${outdir}" ]; then + mkdir -p "$(dirname "${outdir}")" + git clone "${repo}" "${outdir}" + elif ! git_ref_exists "${outdir}" "${reference}" ; then + ( + cd "${outdir}" || exit + git pull + ) + fi + + # Sanity check: the reference we want exists + if ! git_ref_exists "${outdir}" "${reference}"; then + echo "${reference} not found in ${repo} (${outdir})" + exit 1 + fi + + # Generate the prefix for the tarball + prefix_ref=${reference} + # All git repositories we already use remove "v"s from the beginning + # of branch/tag names in the tarball prefix + if echo "${reference}" | grep -Eq "^v[0-9]"; then + prefix_ref=$(echo "${reference}" | sed "s/^v//") + fi + prefix=$(basename "${repo}" | sed "s/.git$//")-${prefix_ref} + + ( + cd "${outdir}" || exit + git config tar.tar.gz.command gzip + # -T1 avoids non-determinism due to threading + # This may not be correct for forges other than Savannah + git config tar.tar.xz.command "xz -T1" + git archive "${reference}" -o "${tarball}" --prefix "${prefix}/" + ) + + my_checksum=$(sha256sum "${tarball}" | cut -d' ' -f1) + if [ "${my_checksum}" != "${checksum}" ]; then + echo "${uri}: generated tarball does not match checksum" + exit 1 + fi + fi +} + +for src in steps/*/sources; do + while read -r line; do + # shellcheck disable=SC2086 + do_file ${line} + uri=$(echo "${line}" | cut -d' ' -f1) + done < "${src}" +donediff --git a/rootfs.py b/rootfs.py@@ -16,10 +16,13 @@ you can run bootstap inside chroot. import argparse import os +import signal +import threading -from lib.utils import run, run_as_root -from lib.target import Target from lib.generator import Generator, stage0_arch_map +from lib.simple_mirror import SimpleMirror +from lib.target import Target +from lib.utils import run, run_as_root def create_configuration_file(args): """ @@ -50,6 +53,12 @@ def create_configuration_file(args): config.write("KERNEL_BOOTSTRAP=False\n") config.write(f"BUILD_KERNELS={args.update_checksums or args.build_kernels}\n") config.write(f"CONFIGURATOR={args.configurator}\n") + if not args.external_sources: + if args.mirrors: + config.write(f"MIRRORS=\"{" ".join(args.mirrors)}\"\n") + config.write(f"MIRRORS_LEN={len(args.mirrors)}\n") + else: + config.write("MIRRORS_LEN=0\n") # pylint: disable=too-many-statements,too-many-branches def main(): @@ -94,6 +103,9 @@ def main(): parser.add_argument("--configurator", help="Run the interactive configurator", action="store_true") + parser.add_argument("-m", "--mirrors", + help="Mirrors to download distfiles from", + nargs='+') parser.add_argument("-r", "--repo", help="Path to prebuilt binary packages", nargs=None) parser.add_argument("--early-preseed", @@ -168,6 +180,19 @@ def main(): # Set constant umask os.umask(0o022) + # file:// mirrors + mirror_servers = [] + if args.mirrors: + for i, mirror in enumerate(args.mirrors): + if mirror.startswith("file://"): + path = mirror.removeprefix("file://") + if not path.startswith("/"): + raise ValueError("A file:// mirror must be an absolute path.") + + server = SimpleMirror(path) + args.mirrors[i] = f"http://127.0.0.1:{server.port}" + mirror_servers.append(server) + # bootstrap.cfg try: os.remove(os.path.join('steps', 'bootstrap.cfg')) @@ -184,14 +209,25 @@ def main(): if args.tmpfs: target.tmpfs(size=args.tmpfs_size) + for server in mirror_servers: + thread = threading.Thread(target=server.serve_forever) + thread.start() + + def cleanup(*_): + for server in mirror_servers: + server.shutdown() + signal.signal(signal.SIGINT, cleanup) + generator = Generator(arch=args.arch, external_sources=args.external_sources, repo_path=args.repo, - early_preseed=args.early_preseed) + early_preseed=args.early_preseed, + mirrors=args.mirrors) - bootstrap(args, generator, target, args.target_size) + bootstrap(args, generator, target, args.target_size, cleanup) + cleanup() -def bootstrap(args, generator, target, size): +def bootstrap(args, generator, target, size, cleanup): """Kick off bootstrap process.""" print(f"Bootstrapping {args.arch}", flush=True) if args.chroot: @@ -206,7 +242,8 @@ print(shutil.which('chroot')) arch = stage0_arch_map.get(args.arch, args.arch) init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') - run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.target_dir, init) + run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.target_dir, init, + cleanup=cleanup) elif args.bwrap: init = '/init' @@ -235,7 +272,8 @@ print(shutil.which('chroot')) '--proc', '/proc', '--bind', '/sys', '/sys', '--tmpfs', '/tmp', - init) + init, + cleanup=cleanup) elif args.bare_metal: if args.kernel: @@ -293,7 +331,7 @@ print(shutil.which('chroot')) ] if not args.interactive: arg_list += ['-no-reboot', '-nographic'] - run(args.qemu_cmd, *arg_list) + run(args.qemu_cmd, *arg_list, cleanup=cleanup) if __name__ == "__main__": main()diff --git a/source_manifest.py b/source_manifest.py@@ -7,8 +7,6 @@ for the bootstrapping process. # SPDX-License-Identifier: GPL-3.0-or-later # SPDX-FileCopyrightText: 2023 Dor Askayo <dor.askayo@gmail.com> -import argparse - from lib.generator import Generator def main():diff --git a/steps/SHA256SUMS.pkgs b/steps/SHA256SUMS.pkgs@@ -40,13 +40,13 @@ db8757b5fe77f56073caa034bfe9b3af6076d7299db966c4348e06f01a34415f curl-8.5.0_1.t 9534d834b018343b8335d0ef0b5faff3ee18e4ecbecf43a2a80deb972283f5d6 dhcpcd-10.0.1_0.tar.bz2 e6372051f90c40d1f4a5cca2f8930187b363dfc916bcb973c72bca6c0aba3cd5 diffutils-2.7_0.tar.bz2 41e3129b974391fe4ace69914fdef9b0fe72c6f5d87d495750fb89bdd999fa01 diffutils-3.10_0.tar.bz2 -bda6afcd3a390a34a57443269a4857ccc470129f7a557981778c145fd841cbd1 dist-3.5-236_0.tar.bz2 +283a66575b4d7c835f6ebabb38cfb5d7d62032f832ad949f067a85b8a378c36c dist-3.5-236_0.tar.bz2 26c0835a8b9be115082a2866e906363947528169c7274bd3b82752df77e99f11 e2fsprogs-1.45.7_0.tar.bz2 9c6f7e156f299f1d0948537e28f00d40e727a3debae3cce158e9bce827136214 ed-1.4_0.tar.bz2 0061d240f9bb2bb534db7003ddcc4eae9981954ad72e8f41ff1d76dc449ebd37 file-5.44_0.tar.bz2 eec07e0d710ed3dd8d0ba6a1ec6ca9f1fa3a611bbb225195435679175317bf90 findutils-4.2.33_0.tar.bz2 80798628ccb38521f42e1ca1abe4f0244e4b283e6f662cd354ab70e61a9b4675 flex-2.5.11_0.tar.bz2 -b3a8d59229289ca5a1de6a24c9c828d2847144f6662e0d5b7f41e7ff6e57abaa flex-2.5.33_0.tar.bz2 +62b4ba8e73daf1f4ac4fc1f90e99394e40d7c03dce9fed8539dbc01d024c62e8 flex-2.5.33_0.tar.bz2 e10060b55e2378a065cf8590cae17400c8befc6b3bb0cbbb7093f1b9f46ec4cc flex-2.6.4_0.tar.bz2 5619cf9a893902491a9d8a8f8da4d39efb3606d7d0a72cb4aaa9d1ff798b5495 gawk-3.0.4_0.tar.bz2 62e90abe883f6ccd679adf42d00c2c18a9809bbab599411c81b6c4ac4d446452 gawk-5.3.0_0.tar.bz2 @@ -97,7 +97,7 @@ a18c4b2e5de2bfe5bb3ee9d360484fcfebad3df042f1859d4aa333dd60f55e56 opendoas-6.8.2 287b0a8c0de14a2817e68ab023bfc502a0880d8c34386162351d5ffaf942da70 perl-5.000_0.tar.bz2 9617799cdc9d4be39d31af54b9bcae47941d68416e9cc9c92cbc2ed08785dbc1 perl-5.003_0.tar.bz2 74d64a8af080022432fa94dba449090419d25b103d247710dc0b6102a4ad86a6 perl-5.10.1_0.tar.bz2 -d381e910532061ac97e48ff09906bddbcb0956b37fc47d8c666f56a56b50c589 perl-5.32.1_0.tar.bz2 +5321a3a3bae7d689b085b415f4f445ebf19b801514340879a2b4c1bf97f05533 perl-5.32.1_0.tar.bz2 d63d2fda752759778ae2d0dd08e6c023a81afbfc63ce11db860fc6858ae26c7f perl-5.6.2_0.tar.bz2 c39ab70bce0bd09f013109ad7370085e90d77ef2b55abcfdc922d7832524c9cf perl5.004-05_0.tar.bz2 2ac211ad3cf966dd89434c175970e10699c363b66228abc1642766aeafe2766a perl5.005-03_0.tar.bz2diff --git a/steps/autogen-5.18.16/sources b/steps/autogen-5.18.16/sources@@ -1,6 +1,4 @@ -https://github.com/schierlm/gnu-autogen-bootstrapping/archive/refs/tags/autogen-5.18.16-v1.0.1.tar.gz 953ba180b18acff188a0a8700770c7cf2fc97e1683c7b9699a5a748b542ccdd5 +git://github.com/schierlm/gnu-autogen-bootstrapping~autogen-5.18.16-v1.0.1 https://github.com/schierlm/gnu-autogen-bootstrapping/archive/refs/tags/autogen-5.18.16-v1.0.1.tar.gz 953ba180b18acff188a0a8700770c7cf2fc97e1683c7b9699a5a748b542ccdd5 https://mirrors.kernel.org/gnu/autogen/rel5.18.16/autogen-5.18.16.tar.xz f8a13466b48faa3ba99fe17a069e71c9ab006d9b1cfabe699f8c60a47d5bb49a -https://git.savannah.gnu.org/cgit/autogen.git/snapshot/autogen-5.18.16.tar.gz 0c04ab2f7ce13c4a1c06c4abc7dfe75312aad89b8b0a1068e5e563787eb56632 -https://files.bootstrapping.world/autogen-5.18.16.tar.gz 0c04ab2f7ce13c4a1c06c4abc7dfe75312aad89b8b0a1068e5e563787eb56632 -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-8f4538a5.tar.gz e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab -https://files.bootstrapping.world/gnulib-8f4538a5.tar.gz e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab +git://git.savannah.gnu.org/autogen.git~v5.18.16 https://git.savannah.gnu.org/cgit/autogen.git/snapshot/autogen-5.18.16.tar.gz 0c04ab2f7ce13c4a1c06c4abc7dfe75312aad89b8b0a1068e5e563787eb56632 +git://git.savannah.gnu.org/gnulib.git~8f4538a5 _ e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab gnulib-8f4538a5.tar.gzdiff --git a/steps/bash-2.05b/pass1.kaem b/steps/bash-2.05b/pass1.kaem@@ -16,8 +16,8 @@ mkdir build src cd build # Extract -cp ${DISTFILES}/${pkg}.tar.bz2 ../src/ -bzip2 -d -f ../src/${pkg}.tar.bz2 +cp ${DISTFILES}/${pkg}.tar.gz ../src/ +gzip -d -f ../src/${pkg}.tar.gz tar xf ../src/${pkg}.tar rm -r ../src/ cd ${pkg}diff --git a/steps/bash-2.05b/sources b/steps/bash-2.05b/sources@@ -1 +1 @@ -https://src.fedoraproject.org/repo/pkgs/bash/bash-2.05b.tar.bz2/f3e5428ed52a4f536f571a945d5de95d/bash-2.05b.tar.bz2 1ce4e5b47a6354531389f0adefb54dee2823227bf6e1e59a31c0e9317a330822 +https://mirrors.kernel.org/gnu/bash/bash-2.05b.tar.gz ba03d412998cc54bd0b0f2d6c32100967d3137098affdc2d32e6e7c11b163fe4diff --git a/steps/bc-1.07.1/sources b/steps/bc-1.07.1/sources@@ -1 +1 @@ -https://mirrors.kernel.org/slackware/slackware64-15.0/source/ap/bc/bc-1.07.1.tar.xz 95396f88fc719a1bf8bd463809119526fef44a42ab9eb708335c2cb79bc801c6 +https://mirrors.kernel.org/gnu/bc/bc-1.07.1.tar.gz 62adfca89b0a1c0164c2cdca59ca210c1d44c3ffc46daf9931cf4942664cb02adiff --git a/steps/bison-2.3/sources b/steps/bison-2.3/sources@@ -1,3 +1,2 @@ http://mirrors.kernel.org/gnu/bison/bison-2.3.tar.bz2 b10d7e9e354be72aee4e4911cf19dd27b5c527d4e7200857365b5fcdeea0dffb -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-b28236b.tar.gz 0190f28cb155fedd22bf8558c3e8705eed9eacfb7ae29e7508d025a68eb90899 -https://files.bootstrapping.world/gnulib-b28236b.tar.gz 0190f28cb155fedd22bf8558c3e8705eed9eacfb7ae29e7508d025a68eb90899 +git://git.savannah.gnu.org/gnulib.git~b28236b _ 0190f28cb155fedd22bf8558c3e8705eed9eacfb7ae29e7508d025a68eb90899 gnulib-b28236b.tar.gzdiff --git a/steps/bison-3.4.2/sources b/steps/bison-3.4.2/sources@@ -1,3 +1,2 @@ http://mirrors.kernel.org/gnu/bison/bison-3.4.2.tar.xz 27d05534699735dc69e86add5b808d6cb35900ad3fd63fa82e3eb644336abfa0 -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-672663a.tar.gz 8cced51f89a950472473856f86e88f5daf97a2347756125ccdc8ee907deec570 -https://files.bootstrapping.world/gnulib-672663a.tar.gz 8cced51f89a950472473856f86e88f5daf97a2347756125ccdc8ee907deec570 +git://git.savannah.gnu.org/gnulib.git~672663a _ 8cced51f89a950472473856f86e88f5daf97a2347756125ccdc8ee907deec570 gnulib-672663a.tar.gzdiff --git a/steps/bzip2-1.0.8/pass1.kaem b/steps/bzip2-1.0.8/pass1.kaem@@ -15,8 +15,8 @@ mkdir build src cd build # Extract -cp ${DISTFILES}/${pkg}.tar.xz ../src/ -unxz --file ../src/${pkg}.tar.xz --output ../src/${pkg}.tar +cp ${DISTFILES}/${pkg}.tar.gz ../src/ +gzip -d -f ../src/${pkg}.tar.gz tar xf ../src/${pkg}.tar rm -r ../src cd ${pkg}diff --git a/steps/bzip2-1.0.8/sources b/steps/bzip2-1.0.8/sources@@ -1 +1 @@ -https://mirrors.kernel.org/slackware/slackware-14.0/patches/source/bzip2/bzip2-1.0.8.tar.xz 47fd74b2ff83effad0ddf62074e6fad1f6b4a77a96e121ab421c20a216371a1f +https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269diff --git a/steps/coreutils-9.4/sources b/steps/coreutils-9.4/sources@@ -1,7 +1,5 @@ -http://git.savannah.gnu.org/cgit/coreutils.git/snapshot/coreutils-9.4.tar.xz 8fb56810310253300b3d6f84e68dc97eb2d74e1f4f78e05776831d9d82e4f2d7 -https://files.bootstrapping.world/coreutils-9.4.tar.xz 8fb56810310253300b3d6f84e68dc97eb2d74e1f4f78e05776831d9d82e4f2d7 -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-bb5bb43.tar.gz b8aa1ac1b18c67f081486069e6a7a5564f20431c2313a94c20a46dcfb904be2a -https://files.bootstrapping.world/gnulib-bb5bb43.tar.gz b8aa1ac1b18c67f081486069e6a7a5564f20431c2313a94c20a46dcfb904be2a +git://git.savannah.gnu.org/coreutils.git~v9.4 http://git.savannah.gnu.org/cgit/coreutils.git/snapshot/coreutils-9.4.tar.xz 8fb56810310253300b3d6f84e68dc97eb2d74e1f4f78e05776831d9d82e4f2d7 +git://git.savannah.gnu.org/gnulib.git~bb5bb43 _ b8aa1ac1b18c67f081486069e6a7a5564f20431c2313a94c20a46dcfb904be2a gnulib-bb5bb43.tar.gz http://ftp.unicode.org/Public/15.0.0/ucd/UnicodeData.txt 806e9aed65037197f1ec85e12be6e8cd870fc5608b4de0fffd990f689f376a73 UnicodeData-15.0.0.txt http://ftp.unicode.org/Public/15.0.0/ucd/PropList.txt e05c0a2811d113dae4abd832884199a3ea8d187ee1b872d8240a788a96540bfd PropList-15.0.0.txt http://ftp.unicode.org/Public/15.0.0/ucd/DerivedCoreProperties.txt d367290bc0867e6b484c68370530bdd1a08b6b32404601b8c7accaf83e05628d DerivedCoreProperties-15.0.0.txtdiff --git a/steps/diffutils-3.10/sources b/steps/diffutils-3.10/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/diffutils/diffutils-3.10.tar.xz 90e5e93cc724e4ebe12ede80df1634063c7a855692685919bfe60b556c9bd09e -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-5d2fe24.tar.gz 72e7bb2d1d75e63d1c46d33b8dd22e8eb60afdba4af3e7251151b5c2a6f00bfb -https://files.bootstrapping.world/gnulib-5d2fe24.tar.gz 72e7bb2d1d75e63d1c46d33b8dd22e8eb60afdba4af3e7251151b5c2a6f00bfb +git://git.savannah.gnu.org/gnulib.git~5d2fe24 _ 72e7bb2d1d75e63d1c46d33b8dd22e8eb60afdba4af3e7251151b5c2a6f00bfb gnulib-5d2fe24.tar.gzdiff --git a/steps/dist-3.5-236/patches/env.patch b/steps/dist-3.5-236/patches/env.patch@@ -4,8 +4,8 @@ SPDX-FileCopyrightText: 2021 fosslinux <fosslinux@aussies.space> SPDX-License-Identifier: Artistic-1.0 ---- dist-3.5-236/bin/perload 2021-04-21 19:01:35.803767498 +1000 -+++ dist-3.5-236/bin/perload 2021-04-21 19:01:47.959850217 +1000 +--- dist-2cec35331a912b165e2dd135d22de81f34bbc83f/bin/perload 2021-04-21 19:01:35.803767498 +1000 ++++ dist-2cec35331a912b165e2dd135d22de81f34bbc83f/bin/perload 2021-04-21 19:01:47.959850217 +1000 @@ -450,8 +450,6 @@ :# This perl program uses dynamic loading [generated by perload] :#diff --git a/steps/dist-3.5-236/sources b/steps/dist-3.5-236/sources@@ -1 +1 @@ -http://deb.debian.org/debian/pool/main/d/dist/dist_3.5-236.orig.tar.gz 05fa4f6ea9f05adf8f577699cb3f5b88b20dfce86b0d0cebbfb072fe5933d38f +git://github.com/rmanfredi/dist~2cec35331a912b165e2dd135d22de81f34bbc83f https://github.com/rmanfredi/dist/archive/2cec35331a912b165e2dd135d22de81f34bbc83f.tar.gz 6d2c9e953de2c136c77c9b6485fbc61e8291a2a70689f2a07c79d9381bf9dbcbdiff --git a/steps/ed-1.4/sources b/steps/ed-1.4/sources@@ -1 +1 @@ -https://mirrors.kernel.org/slackware/slackware-13.37/source/a/ed/ed-1.4.tar.xz 102c80e6da527c6b8eebd5195cd05fc71808d60735d73e8bb503a5e294475007 +https://mirrors.kernel.org/gnu/ed/ed-1.4.tar.gz db36da85ee1a9d8bafb4b041bd4c8c11becba0c43ec446353b67045de1634fdadiff --git a/steps/findutils-4.2.33/pass1.sh b/steps/findutils-4.2.33/pass1.sh@@ -3,8 +3,6 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -EXTRA_DISTFILES="gnulib-8e128e.tar.gz" - src_prepare() { . ../../import-gnulib.shdiff --git a/steps/findutils-4.2.33/sources b/steps/findutils-4.2.33/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/findutils/findutils-4.2.33.tar.gz 813cd9405aceec5cfecbe96400d01e90ddad7b512d3034487176ce5258ab0f78 -https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-8e128e.tar.gz 0cfbf866bc39c31f25fa0e56af1e56c5e5c92fc1e5d51242ebafef7ea211f3d5 -https://files.bootstrapping.world/gnulib-8e128e.tar.gz 0cfbf866bc39c31f25fa0e56af1e56c5e5c92fc1e5d51242ebafef7ea211f3d5 +git://git.savannah.gnu.org/gnulib.git~8e128e _ 0cfbf866bc39c31f25fa0e56af1e56c5e5c92fc1e5d51242ebafef7ea211f3d5 gnulib-8e128e.tar.gzdiff --git a/steps/flex-2.5.11/pass1.sh b/steps/flex-2.5.11/pass1.sh@@ -8,7 +8,6 @@ src_prepare() { default touch config.h - rm parse.c parse.h scan.c skel.c } src_compile() {diff --git a/steps/flex-2.5.11/patches/scan_l.patch b/steps/flex-2.5.11/patches/scan_l.patch@@ -8,8 +8,8 @@ Comments are unsupported by our flex. diff --git scan.l scan.l index 18d0de8..c251a5e 100644 ---- flex-2.5.11/scan.l -+++ flex-2.5.11/scan.l +--- flex-d160f0247ba1611aa59d28f027d6292ba24abb50/scan.l ++++ flex-d160f0247ba1611aa59d28f027d6292ba24abb50/scan.l @@ -335,8 +335,8 @@ LEXOPT [aceknopr]diff --git a/steps/flex-2.5.11/patches/yyin.patch b/steps/flex-2.5.11/patches/yyin.patch@@ -9,8 +9,8 @@ acidentally re-declare it. diff --git flexdef.h flexdef.h index 3eb710a..94ef024 100644 ---- flex-2.5.11/flexdef.h -+++ flex-2.5.11/flexdef.h +--- flex-d160f0247ba1611aa59d28f027d6292ba24abb50/flexdef.h ++++ flex-d160f0247ba1611aa59d28f027d6292ba24abb50/flexdef.h @@ -421,7 +421,7 @@ extern int yymore_really_used, reject_really_used; */ @@ -31,8 +31,8 @@ index 3eb710a..94ef024 100644 extern char *action_array; extern int action_size; extern int defs1_offset, prolog_offset, action_offset, action_index; ---- flex-2.5.11/scan.l -+++ flex-2.5.11/scan.l +--- flex-d160f0247ba1611aa59d28f027d6292ba24abb50/scan.l ++++ flex-d160f0247ba1611aa59d28f027d6292ba24abb50/scan.l @@ -32,6 +32,7 @@ /* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR */ /* PURPOSE. */diff --git a/steps/flex-2.5.11/sources b/steps/flex-2.5.11/sources@@ -1 +1 @@ -http://ftp-tel.sjtu.edu.cn/sites/ftp.openpkg.org/sources/DST/flex/flex-2.5.11.tar.gz bc79b890f35ca38d66ff89a6e3758226131e51ccbd10ef78d5ff150b7bd73689 +git://github.com/westes/flex~d160f0247ba1611aa59d28f027d6292ba24abb50 https://github.com/westes/flex/archive/d160f0247ba1611aa59d28f027d6292ba24abb50.tar.gz 68aa10c473b6010ffad680cada09fc4eec6b3cc6e415cc2339e5fc2385ccc142diff --git a/steps/flex-2.5.33/pass1.sh b/steps/flex-2.5.33/pass1.sh@@ -6,12 +6,6 @@ src_prepare() { default AUTOPOINT=true AUTOMAKE=automake-1.15 ACLOCAL=aclocal-1.15 autoreconf-2.69 -fi - - # Remove pregenerated files - rm parse.c parse.h scan.c - - # Remove pregenerated .info - rm doc/flex.info } src_configure() {diff --git a/steps/flex-2.5.33/patches/disable-unavailables.patch b/steps/flex-2.5.33/patches/disable-unavailables.patch@@ -0,0 +1,50 @@ +Disable things we can't regenerate for varied reasons. + +SPDX-FileCopyrightText: 2025 fosslinux <fosslinux@aussies.space> + +SPDX-License-Identifier: BSD-3-Clause-flex + +diff --color -ru configure.in +--- flex-e6f147b7a5f2ec2dc862dc9d30b3734b9555a1ea/configure.in 2024-12-28 11:29:45.497750429 +1100 ++++ flex-e6f147b7a5f2ec2dc862dc9d30b3734b9555a1ea/configure.in 2024-12-28 12:37:00.190457545 +1100 +@@ -31,9 +31,6 @@ + + dnl checks for programs + +-AM_GNU_GETTEXT([external]) +-AM_GNU_GETTEXT_VERSION(0.12) +- + AC_PROG_YACC + AM_PROG_LEX + AC_PROG_CC +@@ -97,12 +94,10 @@ + flex.spec + Makefile + doc/Makefile +-website/Makefile + examples/Makefile + examples/fastwc/Makefile + examples/manual/Makefile + m4/Makefile +-po/Makefile.in + tools/Makefile + tests/Makefile + tests/TEMPLATE/Makefile +diff --color -ru Makefile.am +--- flex-e6f147b7a5f2ec2dc862dc9d30b3734b9555a1ea/Makefile.am 2024-12-28 11:29:45.496750429 +1100 ++++ flex-e6f147b7a5f2ec2dc862dc9d30b3734b9555a1ea/Makefile.am 2024-12-28 13:32:54.945214018 +1100 +@@ -135,13 +135,12 @@ + doc \ + m4 \ + examples \ +- po \ + tools \ + tests + + localedir = $(datadir)/locale + AM_CPPFLAGS = -DLOCALEDIR=\"$(localedir)\" -I@includedir@ -I$(top_srcdir)/intl +-LIBS = @LIBINTL@ @LIBS@ ++LIBS = @LIBS@ + + skel.c: flex.skl mkskel.sh flexint.h tables_shared.h + sed 's/m4_/m4postproc_/g; s/m4preproc_/m4_/g' $(srcdir)/flex.skl | $(m4) -P -DFLEX_MAJOR_VERSION=`echo $(VERSION)|cut -f 1 -d .` -DFLEX_MINOR_VERSION=`echo $(VERSION)|cut -f 2 -d .` -DFLEX_SUBMINOR_VERSION=`echo $(VERSION)|cut -f 3 -d .` | sed 's/m4postproc_/m4_/g' | $(SHELL) $(srcdir)/mkskel.sh >skel.cdiff --git a/steps/flex-2.5.33/sources b/steps/flex-2.5.33/sources@@ -1 +1 @@ -http://ftp-tel.sjtu.edu.cn/sites/ftp.openpkg.org/sources/DST/flex/flex-2.5.33.tar.gz c40385e142989c91989413f3c5a31282b2ffdca16b69cd3ecfde537b8a474921 +git://github.com/westes/flex~e6f147b7a5f2ec2dc862dc9d30b3734b9555a1ea https://github.com/westes/flex/archive/e6f147b7a5f2ec2dc862dc9d30b3734b9555a1ea.tar.gz baec69069ff58b7cdbe0103ffc16f29d4857428c29efcdf685c574d8300fd838diff --git a/steps/gcc-10.4.0/sources b/steps/gcc-10.4.0/sources@@ -1 +1 @@ -http://ftp.gnu.org/gnu/gcc/gcc-10.4.0/gcc-10.4.0.tar.xz c9297d5bcd7cb43f3dfc2fed5389e948c9312fd962ef6a4ce455cff963ebe4f1 +http://mirrors.kernel.org/gnu/gcc/gcc-10.4.0/gcc-10.4.0.tar.xz c9297d5bcd7cb43f3dfc2fed5389e948c9312fd962ef6a4ce455cff963ebe4f1diff --git a/steps/gcc-13.1.0/sources b/steps/gcc-13.1.0/sources@@ -1 +1 @@ -https://ftp.gnu.org/gnu/gcc/gcc-13.1.0/gcc-13.1.0.tar.xz 61d684f0aa5e76ac6585ad8898a2427aade8979ed5e7f85492286c4dfc13ee86 +https://mirrors.kernel.org/gnu/gcc/gcc-13.1.0/gcc-13.1.0.tar.xz 61d684f0aa5e76ac6585ad8898a2427aade8979ed5e7f85492286c4dfc13ee86diff --git a/steps/gettext-0.21/sources b/steps/gettext-0.21/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/gettext/gettext-0.21.tar.xz d20fcbb537e02dcf1383197ba05bd0734ef7bf5db06bdb241eb69b7d16b73192 -https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-7daa86f.tar.gz 2d911c2f2ed97b347d6d360b742abdc98aa626d4f8f847ee682c7cde12e90871 -https://files.bootstrapping.world/gnulib-7daa86f.tar.gz 2d911c2f2ed97b347d6d360b742abdc98aa626d4f8f847ee682c7cde12e90871 +git://git.savannah.gnu.org/gnulib.git~7daa86f _ 2d911c2f2ed97b347d6d360b742abdc98aa626d4f8f847ee682c7cde12e90871 gnulib-7daa86f.tar.gzdiff --git a/steps/grep-3.7/pass1.sh b/steps/grep-3.7/pass1.sh@@ -2,9 +2,6 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -urls="https://mirrors.kernel.org/gnu/grep/grep-3.7.tar.xz - http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-8f4538a5.tar.gz" - src_prepare() { rm configure find . -name 'Makefile.in' -deletediff --git a/steps/grep-3.7/sources b/steps/grep-3.7/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/grep/grep-3.7.tar.xz 5c10da312460aec721984d5d83246d24520ec438dd48d7ab5a05dbc0d6d6823c -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-8f4538a5.tar.gz e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab -https://files.bootstrapping.world/gnulib-8f4538a5.tar.gz e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab +git://git.savannah.gnu.org/gnulib.git~8f4538a5 _ e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab gnulib-8f4538a5.tar.gzdiff --git a/steps/grub-2.06/sources b/steps/grub-2.06/sources@@ -1,6 +1,5 @@ https://mirrors.kernel.org/gnu/grub/grub-2.06.tar.xz b79ea44af91b93d17cd3fe80bdae6ed43770678a9a5ae192ccea803ebb657ee1 -https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-d271f86.tar.gz 31d69d3d251e39135b5194ddc6f897910d344059f7494d96a739aecbf7ac2b66 -https://files.bootstrapping.world/gnulib-d271f86.tar.gz 31d69d3d251e39135b5194ddc6f897910d344059f7494d96a739aecbf7ac2b66 +git://git.savannah.gnu.org/gnulib.git~d271f86 _ 31d69d3d251e39135b5194ddc6f897910d344059f7494d96a739aecbf7ac2b66 gnulib-d271f86.tar.gz http://ftp.unicode.org/Public/9.0.0/ucd/UnicodeData.txt 68dfc414d28257b9b5d6ddbb8b466c768c00ebdf6cbf7784364a9b6cad55ee8f UnicodeData-9.0.0.txt http://ftp.unicode.org/Public/9.0.0/ucd/PropList.txt f413ea8dbd3858de72f3148b47dd0586019761357d1481e3b65f3a025bc27f82 PropList-9.0.0.txt http://ftp.unicode.org/Public/9.0.0/ucd/DerivedCoreProperties.txt 6662c7e30b572df5d948c092692f52bcc79ab36d49a063a73d6435042db6fb3b DerivedCoreProperties-9.0.0.txtdiff --git a/steps/guile-3.0.9/sources b/steps/guile-3.0.9/sources@@ -1,7 +1,5 @@ https://mirrors.kernel.org/gnu/guile/guile-3.0.7.tar.xz f57d86c70620271bfceb7a9be0c81744a033f08adc7ceba832c9917ab3e691b7 https://mirrors.kernel.org/gnu/guile/guile-3.0.9.tar.xz 1a2625ac72b2366e95792f3fe758fd2df775b4044a90a4a9787326e66c0d750d -https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-901694b9.tar.gz f9aad85de1f41d57c9368d304020ffbf354a5e56db1297f022c3d12181134e56 -https://files.bootstrapping.world/gnulib-901694b9.tar.gz f9aad85de1f41d57c9368d304020ffbf354a5e56db1297f022c3d12181134e56 -https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-356a414e.tar.gz fc9973f1a9243fdc4b98d33d7704f3c71bfdc4c2ef96899b8f28cade7290a714 -https://files.bootstrapping.world/gnulib-356a414e.tar.gz fc9973f1a9243fdc4b98d33d7704f3c71bfdc4c2ef96899b8f28cade7290a714 -https://github.com/schierlm/guile-psyntax-bootstrapping/archive/refs/tags/guile-3.0.7.tar.gz 14cda9c416506dfadf60c14fc623ff01ef99b87564a78d0a29c5d17143c97609 +git://git.savannah.gnu.org/gnulib.git~901694b9 _ f9aad85de1f41d57c9368d304020ffbf354a5e56db1297f022c3d12181134e56 gnulib-901694b9.tar.gz +git://git.savannah.gnu.org/gnulib.git~356a414e _ fc9973f1a9243fdc4b98d33d7704f3c71bfdc4c2ef96899b8f28cade7290a714 gnulib-356a414e.tar.gz +git://github.com/schierlm/guile-psyntax-bootstrapping~guile-3.0.7 https://github.com/schierlm/guile-psyntax-bootstrapping/archive/refs/tags/guile-3.0.7.tar.gz 14cda9c416506dfadf60c14fc623ff01ef99b87564a78d0a29c5d17143c97609diff --git a/steps/gzip-1.13/sources b/steps/gzip-1.13/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/gzip/gzip-1.13.tar.xz 7454eb6935db17c6655576c2e1b0fabefd38b4d0936e0f87f48cd062ce91a057 -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-5651802.tar.gz 56f1221eb682c3502ee097f583f44673570753cb452346ad4806d94560c3fac9 -https://files.bootstrapping.world/gnulib-5651802.tar.gz 56f1221eb682c3502ee097f583f44673570753cb452346ad4806d94560c3fac9 +git://git.savannah.gnu.org/gnulib.git~5651802 _ 56f1221eb682c3502ee097f583f44673570753cb452346ad4806d94560c3fac9 gnulib-5651802.tar.gzdiff --git a/steps/helpers.sh b/steps/helpers.sh@@ -21,8 +21,6 @@ _get_files() { prefix="${1}" fs= if [ -n "$(ls 2>/dev/null)" ]; then - # This can be removed once Debian 12 is stable - # shellcheck disable=SC2035 fs=$(echo *) fi if [ -n "$(ls .[0-z]* 2>/dev/null)" ]; then @@ -258,18 +256,48 @@ build() { unset extract } +# An inventive way to randomise with what we know we always have +randomize() { + if command -v shuf >/dev/null 2>&1; then + # shellcheck disable=SC2086 + shuf -e ${1} | tr '\n' ' ' + else + mkdir -p /tmp/random + for item in ${1}; do + touch --date=@${RANDOM} /tmp/random/"${item//\//~*~*}" + done + # cannot rely on find existing + # shellcheck disable=SC2012 + ls -1 -t /tmp/random | sed 's:~\*~\*:/:g' | tr '\n' ' ' + rm -r /tmp/random + fi +} + download_source_line() { - url="${1}" + if [[ "${1}" == git://* ]]; then + shift + fi + upstream_url="${1}" checksum="${2}" fname="${3}" # Default to basename of url if not given - fname="${fname:-$(basename "${url}")}" + fname="${fname:-$(basename "${upstream_url}")}" if ! [ -e "${fname}" ]; then - curl --fail --retry 5 --location "${url}" --output "${fname}" || true + for mirror in $(randomize "${MIRRORS}"); do + mirror_url="${mirror}/${fname}" + echo "${mirror_url}" + curl --fail --retry 3 --location "${mirror_url}" --output "${fname}" || true && break + done + if ! [ -e "${fname}" ] && [ "${upstream_url}" != "_" ]; then + curl --fail --retry 3 --location "${upstream_url}" --output "${fname}" || true + fi fi } check_source_line() { + if [[ "${1}" == git://* ]]; then + shift + fi url="${1}" checksum="${2}" fname="${3}" @@ -304,6 +332,9 @@ default_src_get() { # Intelligently extracts a file based upon its filetype. extract_file() { + if [[ "${1}" == git://* ]]; then + shift + fi f="${3:-$(basename "${1}")}" # shellcheck disable=SC2154 case "${noextract}" indiff --git a/steps/improve/setup_repo.sh b/steps/improve/setup_repo.sh@@ -6,4 +6,4 @@ # mkdir -p /external/repo -tar -cf - --exclude='/external/repo/*' --exclude='/external/repo-preseeded/*' --exclude='/external/distfiles/*' --exclude='/dev/*' --exclude='/proc/*' --exclude='/sys/*' --exclude='/tmp/*' / | bzip2 --best > /external/repo/base.tar.bz2 +#tar -cf - --exclude='/external/repo/*' --exclude='/external/repo-preseeded/*' --exclude='/external/distfiles/*' --exclude='/dev/*' --exclude='/proc/*' --exclude='/sys/*' --exclude='/tmp/*' / | bzip2 --best > /external/repo/base.tar.bz2diff --git a/steps/kexec-tools-2.0.22/sources b/steps/kexec-tools-2.0.22/sources@@ -1 +1 @@ -https://github.com/horms/kexec-tools/archive/refs/tags/v2.0.22.tar.gz af618de7848142f204b57811f703de3ae7aa3f5bc5d52226db35800fa8fc4dff +git://github.com/horms/kexec-tools~v2.0.22 https://github.com/horms/kexec-tools/archive/refs/tags/v2.0.22.tar.gz af618de7848142f204b57811f703de3ae7aa3f5bc5d52226db35800fa8fc4dffdiff --git a/steps/libtool-2.4.7/sources b/steps/libtool-2.4.7/sources@@ -1,3 +1,2 @@ http://mirrors.kernel.org/gnu/libtool/libtool-2.4.7.tar.xz 4f7f217f057ce655ff22559ad221a0fd8ef84ad1fc5fcb6990cecc333aa1635d -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-a521820.tar.gz 719b399fe09a8f6ca14ba8c4a9a60ce9f93f4892effb50961ef3d8cd1a33ff65 -https://files.bootstrapping.world/gnulib-a521820.tar.gz 719b399fe09a8f6ca14ba8c4a9a60ce9f93f4892effb50961ef3d8cd1a33ff65 +git://git.savannah.gnu.org/gnulib.git~a521820 _ 719b399fe09a8f6ca14ba8c4a9a60ce9f93f4892effb50961ef3d8cd1a33ff65 gnulib-a521820.tar.gzdiff --git a/steps/libunistring-0.9.10/sources b/steps/libunistring-0.9.10/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/libunistring/libunistring-0.9.10.tar.xz eb8fb2c3e4b6e2d336608377050892b54c3c983b646c561836550863003c05d7 -https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-52a06cb3.tar.gz 009989b81c0bebc5f6550636ed653fbcb237dafc2af5c706f3522087ca571e4d -https://files.bootstrapping.world/gnulib-52a06cb3.tar.gz 009989b81c0bebc5f6550636ed653fbcb237dafc2af5c706f3522087ca571e4d +git://git.savannah.gnu.org/gnulib.git~52a06cb3 _ 009989b81c0bebc5f6550636ed653fbcb237dafc2af5c706f3522087ca571e4d gnulib-52a06cb3.tar.gzdiff --git a/steps/m4-1.4.19/sources b/steps/m4-1.4.19/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/m4/m4-1.4.19.tar.xz 63aede5c6d33b6d9b13511cd0be2cac046f2e70fd0a07aa9573a04a82783af96 -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-3639c57.tar.gz 97dfbad67832641bc7f73437617b78abeafb9946723f19cf4c2ceecfc65fa48d -https://files.bootstrapping.world/gnulib-3639c57.tar.gz 97dfbad67832641bc7f73437617b78abeafb9946723f19cf4c2ceecfc65fa48d +git://git.savannah.gnu.org/gnulib.git~3639c57 _ 97dfbad67832641bc7f73437617b78abeafb9946723f19cf4c2ceecfc65fa48d gnulib-3639c57.tar.gzdiff --git a/steps/openssl-3.0.13/sources b/steps/openssl-3.0.13/sources@@ -1 +1 @@ -http://snapshot.debian.org/archive/debian/20240325T091329Z/pool/main/o/openssl/openssl_3.0.13.orig.tar.gz 88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313 +https://github.com/openssl/openssl/releases/download/openssl-3.0.13/openssl-3.0.13.tar.gz 88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313diff --git a/steps/patch-2.7.6/sources b/steps/patch-2.7.6/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/patch/patch-2.7.6.tar.xz ac610bda97abe0d9f6b7c963255a11dcb196c25e337c61f94e4778d632f1d8fd -https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-e017871.tar.gz a285dc300c3d9c25cc06e38827ef40f6073ec3b9b0fcb5bba433f943be92d8d4 -https://files.bootstrapping.world/gnulib-e017871.tar.gz a285dc300c3d9c25cc06e38827ef40f6073ec3b9b0fcb5bba433f943be92d8d4 +git://git.savannah.gnu.org/gnulib.git~e017871 _ a285dc300c3d9c25cc06e38827ef40f6073ec3b9b0fcb5bba433f943be92d8d4 gnulib-e017871.tar.gzdiff --git a/steps/perl-5.32.1/pass1.sh b/steps/perl-5.32.1/pass1.sh@@ -24,8 +24,8 @@ src_prepare() { # Regenerate configure + config_h.SH rm -f Configure config_h.SH - ln -s ../metaconfig-5.32.1\~rc1/.package . - ln -s ../metaconfig-5.32.1\~rc1/U . + ln -s ../metaconfig-5.32.1/.package . + ln -s ../metaconfig-5.32.1/U . metaconfig -m }diff --git a/steps/perl-5.32.1/patches/reproducibility.patch b/steps/perl-5.32.1/patches/reproducibility.patch@@ -7,8 +7,8 @@ generate with correct values for live-bootstrap. NOTE: this patch CANNOT be applied to a non-live-bootstrap environment. ---- metaconfig-5.32.1~rc1/U/threads/archname.U 2022-02-27 21:30:03.155396204 +1100 -+++ metaconfig-5.32.1~rc1/U/threads/archname.U 2022-02-27 21:30:49.392396204 +1100 +--- metaconfig-5.32.1/U/threads/archname.U 2022-02-27 21:30:03.155396204 +1100 ++++ metaconfig-5.32.1/U/threads/archname.U 2022-02-27 21:30:49.392396204 +1100 @@ -79,7 +79,7 @@ ?X: Very GCCian. *) archname=`echo $targetarch|sed 's,^[^-]*-,,'` ;; @@ -18,8 +18,8 @@ NOTE: this patch CANNOT be applied to a non-live-bootstrap environment. case "$archname" in '') dflt="$tarch";; *) dflt="$archname";; ---- metaconfig-5.32.1~rc1/U/modified/Oldconfig.U 2022-02-27 21:31:26.911396204 +1100 -+++ metaconfig-5.32.1~rc1/U/modified/Oldconfig.U 2022-02-27 21:32:31.846396204 +1100 +--- metaconfig-5.32.1/U/modified/Oldconfig.U 2022-02-27 21:31:26.911396204 +1100 ++++ metaconfig-5.32.1/U/modified/Oldconfig.U 2022-02-27 21:32:31.846396204 +1100 @@ -117,16 +117,13 @@ ?X: on some machines to avoid the error message when uname is not found; e.g. ?X: old SUN-OS 3.2 would not execute hostname in (uname -a || hostname). Sigh!diff --git a/steps/perl-5.32.1/patches/reproducibility2.patch b/steps/perl-5.32.1/patches/reproducibility2.patch@@ -7,8 +7,8 @@ generate with correct values for live-bootstrap. NOTE: this patch CANNOT be applied to a non-live-bootstrap environment. ---- perl-5.32.1/../metaconfig-5.32.1~rc1/dist/U/archname.U 2022-02-26 10:51:45.343097807 +1100 -+++ perl-5.32.1/../metaconfig-5.32.1~rc1/dist/U/archname.U 2022-02-26 10:51:51.742527859 +1100 +--- metaconfig-5.32.1/dist/U/archname.U 2022-02-26 10:51:45.343097807 +1100 ++++ metaconfig-5.32.1/dist/U/archname.U 2022-02-26 10:51:51.742527859 +1100 @@ -72,5 +72,5 @@ rp='What is your architecture name' . ./myread @@ -16,8 +16,8 @@ NOTE: this patch CANNOT be applied to a non-live-bootstrap environment. -myarchname="$tarch" +myarchname="i386" ---- perl-5.32.1/../metaconfig-5.32.1~rc1/dist/U/Oldconfig.U 2022-02-27 10:55:04.890396204 +1100 -+++ perl-5.32.1/../metaconfig-5.32.1~rc1/dist/U/Oldconfig.U 2022-02-27 11:00:31.324396204 +1100 +--- metaconfig-5.32.1/dist/U/Oldconfig.U 2022-02-27 10:55:04.890396204 +1100 ++++ metaconfig-5.32.1/dist/U/Oldconfig.U 2022-02-27 11:00:31.324396204 +1100 @@ -109,16 +109,13 @@ ?LINT:extern hostarch libswanted libs ?LINT:change hostarch libswanted libsdiff --git a/steps/perl-5.32.1/sources b/steps/perl-5.32.1/sources@@ -1,2 +1,2 @@ http://www.cpan.org/src/5.0/perl-5.32.1.tar.xz 57cc47c735c8300a8ce2fa0643507b44c4ae59012bfdad0121313db639e02309 -http://deb.debian.org/debian/pool/main/p/perl/perl_5.32.1.orig-regen-configure.tar.gz 1d179b41283f12ad83f9758430f6ddc49bdf20db5c396aeae7e51ebb4e4afd29 +git://github.com/Perl/metaconfig~5.32.1 https://github.com/Perl/metaconfig/archive/refs/tags/5.32.1.tar.gz 23abd0d49995229426775c7b1a973e0722faf99bc52e5030d188f0f37973e841diff --git a/steps/pkg-config-0.29.2/sources b/steps/pkg-config-0.29.2/sources@@ -1 +1 @@ -http://distfiles.macports.org/pkgconfig/pkg-config-0.29.2.tar.gz 6fc69c01688c9458a57eb9a1664c9aba372ccda420a02bf4429fe610e7e7d591 +https://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz 6fc69c01688c9458a57eb9a1664c9aba372ccda420a02bf4429fe610e7e7d591diff --git a/steps/python-3.11.1/sources b/steps/python-3.11.1/sources@@ -22,4 +22,3 @@ http://ftp.unicode.org/Public/14.0.0/ucd/Unihan.zip 2ae4519b2b82cd4d15379c17e57b http://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP437.TXT 6bad4dabcdf5940227c7d81fab130dcb18a77850b5d79de28b5dc4e047b0aaac https://www.ietf.org/rfc/rfc3454.txt eb722fa698fb7e8823b835d9fd263e4cdb8f1c7b0d234edf7f0e3bd2ccbb2c79 https://github.com/openssl/openssl/releases/download/openssl-3.0.13/openssl-3.0.13.tar.gz 88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313 -https://github.com/openssl/openssl/releases/download/openssl-3.0.13/openssl-3.0.13.tar.gz 88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313diff --git a/steps/sed-4.8/sources b/steps/sed-4.8/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/sed/sed-4.8.tar.xz f79b0cfea71b37a8eeec8490db6c5f7ae7719c35587f21edb0617f370eeff633 -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-d279bc.tar.gz 12cfa21abf618a274017d6b18e95fc6582519d7c08e2403e5c5772ccdd5b85f4 -https://files.bootstrapping.world/gnulib-d279bc.tar.gz 12cfa21abf618a274017d6b18e95fc6582519d7c08e2403e5c5772ccdd5b85f4 +git://git.savannah.gnu.org/gnulib.git~d279bc _ 12cfa21abf618a274017d6b18e95fc6582519d7c08e2403e5c5772ccdd5b85f4 gnulib-d279bc.tar.gzdiff --git a/steps/tar-1.34/sources b/steps/tar-1.34/sources@@ -1,3 +1,2 @@ http://mirrors.kernel.org/gnu/tar/tar-1.34.tar.xz 63bebd26879c5e1eea4352f0d03c991f966aeb3ddeb3c7445c902568d5411d28 -http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-30820c.tar.gz df807e694deea2dcba0c43af318394f3e3fcd52658c3b71b61dad0ce0c0cfb77 -https://files.bootstrapping.world/gnulib-30820c.tar.gz df807e694deea2dcba0c43af318394f3e3fcd52658c3b71b61dad0ce0c0cfb77 +git://git.savannah.gnu.org/gnulib.git~30820c _ df807e694deea2dcba0c43af318394f3e3fcd52658c3b71b61dad0ce0c0cfb77 gnulib-30820c.tar.gzdiff --git a/steps/texinfo-6.7/sources b/steps/texinfo-6.7/sources@@ -1,3 +1,2 @@ https://mirrors.kernel.org/gnu/texinfo/texinfo-6.7.tar.xz 988403c1542d15ad044600b909997ba3079b10e03224c61188117f3676b02caa -https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-b81ec69.tar.gz 1aeea67b7b3883ebcf2b90bc01f4182d7de073a052dabd3749f20c5aa4ad3e27 -https://files.bootstrapping.world/gnulib-b81ec69.tar.gz 1aeea67b7b3883ebcf2b90bc01f4182d7de073a052dabd3749f20c5aa4ad3e27 +git://git.savannah.gnu.org/gnulib.git~b81ec69 _ 1aeea67b7b3883ebcf2b90bc01f4182d7de073a052dabd3749f20c5aa4ad3e27 gnulib-b81ec69.tar.gzdiff --git a/steps/xz-5.4.1/sources b/steps/xz-5.4.1/sources@@ -1 +1 @@ -http://ixpeering.dl.sourceforge.net/project/lzmautils/xz-5.4.1.tar.bz2 dd172acb53867a68012f94c17389401b2f274a1aa5ae8f84cbfb8b7e383ea8d3 +https://github.com/tukaani-project/xz/releases/download/v5.4.1/xz-5.4.1.tar.bz2 dd172acb53867a68012f94c17389401b2f274a1aa5ae8f84cbfb8b7e383ea8d3diff --git a/substitutes b/substitutes@@ -1,3 +0,0 @@ -https://mirrors.kernel.org/gnu/bash/bash-2.05b.tar.gz https://src.fedoraproject.org/repo/pkgs/bash/bash-2.05b.tar.bz2/f3e5428ed52a4f536f571a945d5de95d/bash-2.05b.tar.bz2 -https://mirrors.kernel.org/gnu/bc/bc-1.07.1.tar.gz https://mirrors.kernel.org/slackware/slackware64-15.0/source/ap/bc/bc-1.07.1.tar.xz -https://mirrors.kernel.org/gnu/ed/ed-1.4.tar.gz https://mirrors.kernel.org/slackware/slackware-13.37/source/a/ed/ed-1.4.tar.xz