commit: 0ce50a6393c1ee08985da2124d76595a9cbbdd1b
parent 6ea1b8e29515e4bfc5696f2b0e1bb59051b9028b
Author: fosslinux <fosslinux@aussies.space>
Date: Sun, 22 May 2022 10:03:21 +1000
Add options for;
- disk to be created (blank disk given to live-bootstrap) (default)
- disk to already exist but sources downloaded within live-bootstrap
- sources to be downloaded outside live-bootstrap (non-blank disk given
to live-bootstrap)
Also migrate sysb to use sys_transfer in QEMU mode also.
Note that this means copy_sysc is now irrelevant. sysc is *always*
sourced from sysa.
Diffstat:
9 files changed, 219 insertions(+), 51 deletions(-)
diff --git a/lib/utils.py b/lib/utils.py
@@ -32,10 +32,11 @@ def create_disk(image, disk_type, fs_type, size):
loop_dev = run('sudo', 'losetup', '-f', capture_output=True).stdout.decode().strip()
run('sudo', 'losetup', '-P', loop_dev, image)
# Create the partition
- run('sudo', 'parted', '--script', image, 'mklabel', disk_type, 'mkpart',
- 'primary', 'ext4', '0%', '100%')
- run('sudo', 'partprobe', loop_dev)
- run('sudo', 'mkfs.' + fs_type, loop_dev + "p1")
+ if disk_type != "none":
+ run('sudo', 'parted', '--script', image, 'mklabel', disk_type, 'mkpart',
+ 'primary', 'ext4', '0%', '100%')
+ run('sudo', 'partprobe', loop_dev)
+ run('sudo', 'mkfs.' + fs_type, loop_dev + "p1")
return loop_dev
def mount(source, target, fs_type, options='', **kwargs):
diff --git a/rootfs.py b/rootfs.py
@@ -33,7 +33,10 @@ def create_configuration_file(args):
config.write("FORCE_TIMESTAMPS=" + str(args.force_timestamps) + "\n")
config.write("CHROOT=" + str(args.chroot or args.bwrap) + "\n")
config.write("UPDATE_CHECKSUMS=" + str(args.update_checksums) + "\n")
- config.write("DISK=sda1\n")
+ if args.external_sources:
+ config.write("DISK=sda1\n")
+ else:
+ config.write("DISK=sda\n")
def main():
"""
@@ -57,6 +60,8 @@ def main():
parser.add_argument("--update-checksums",
help="Update checksum files.",
action="store_true")
+ parser.add_argument("--external-sources",
+ help="Download sources externally from live-bootstrap.")
parser.add_argument("--no-create-config",
help="Do not automatically create config file",
action="store_true")
@@ -116,7 +121,7 @@ def main():
pass
system_c = SysC(arch=args.arch, preserve_tmp=args.preserve,
- tmpdir=args.tmpdir)
+ tmpdir=args.tmpdir, external_sources=args.external_sources)
system_b = SysB(arch=args.arch, preserve_tmp=args.preserve)
system_a = SysA(arch=args.arch, preserve_tmp=args.preserve,
tmpdir=args.tmpdir,
@@ -138,7 +143,6 @@ print(shutil.which('chroot'))
system_c.prepare(mount_tmpfs=True,
create_disk_image=False)
system_a.prepare(mount_tmpfs=True,
- copy_sysc=True,
create_initramfs=False,
repo_path=args.repo)
@@ -151,7 +155,6 @@ print(shutil.which('chroot'))
system_c.prepare(mount_tmpfs=False,
create_disk_image=False)
system_a.prepare(mount_tmpfs=False,
- copy_sysc=True,
create_initramfs=False,
repo_path=args.repo)
@@ -187,7 +190,6 @@ print(shutil.which('chroot'))
system_c.prepare(mount_tmpfs=True,
create_disk_image=True)
system_a.prepare(mount_tmpfs=True,
- copy_sysc=False,
create_initramfs=True,
repo_path=args.repo)
@@ -211,7 +213,6 @@ print(shutil.which('chroot'))
system_c.prepare(mount_tmpfs=True,
create_disk_image=True)
system_a.prepare(mount_tmpfs=True,
- copy_sysc=False,
create_initramfs=True,
repo_path=args.repo)
@@ -223,7 +224,6 @@ print(shutil.which('chroot'))
system_c.prepare(mount_tmpfs=True,
create_disk_image=True)
system_a.prepare(mount_tmpfs=True,
- copy_sysc=False,
create_initramfs=True,
repo_path=args.repo)
@@ -232,6 +232,7 @@ print(shutil.which('chroot'))
'-m', str(args.qemu_ram) + 'M',
'-no-reboot',
'-hda', system_c.dev_name,
+ '-nic', 'user,ipv6=off,model=e1000',
'-kernel', args.kernel,
'-initrd', system_a.initramfs_path,
'-nographic',
diff --git a/sysa.py b/sysa.py
@@ -18,7 +18,7 @@ class SysA(SysGeneral):
Class responsible for preparing sources for System A.
"""
# pylint: disable=too-many-instance-attributes,too-many-arguments
- def __init__(self, arch, preserve_tmp, tmpdir, sysb_dir, sysc_tmp):
+ def __init__(self, arch, preserve_tmp, tmpdir, sysb_dir, sysc_dir):
self.git_dir = os.path.dirname(os.path.join(__file__))
self.arch = arch
self.preserve_tmp = preserve_tmp
@@ -32,9 +32,9 @@ class SysA(SysGeneral):
self.base_dir = self.sysa_dir
self.cache_dir = os.path.join(self.sys_dir, 'distfiles')
self.sysb_dir = sysb_dir
- self.sysc_tmp = sysc_tmp
+ self.sysc_dir = sysc_dir
- def prepare(self, mount_tmpfs, copy_sysc, create_initramfs, repo_path):
+ def prepare(self, mount_tmpfs, create_initramfs, repo_path):
"""
Prepare directory structure for System A.
We create an empty tmp directory, unpack stage0-posix.
@@ -51,8 +51,7 @@ class SysA(SysGeneral):
# sysb must be added to sysa as it is another initramfs stage
self.sysb()
- if copy_sysc:
- self.sysc()
+ self.sysc()
if repo_path:
repo_dir = os.path.join(self.tmp_dir, 'usr', 'src', 'repo-preseeded')
@@ -66,17 +65,17 @@ class SysA(SysGeneral):
self.get_packages()
shutil.copytree(self.sys_dir, os.path.join(self.tmp_dir, 'sysa'),
- shutil.ignore_patterns('tmp'))
+ ignore=shutil.ignore_patterns('tmp'))
def sysb(self):
"""Copy in sysb files for sysb."""
shutil.copytree(self.sysb_dir, os.path.join(self.tmp_dir, 'sysb'),
- shutil.ignore_patterns('tmp'))
+ ignore=shutil.ignore_patterns('tmp'))
def sysc(self):
"""Copy in sysc files for sysc."""
- shutil.copytree(self.sysc_tmp, os.path.join(self.tmp_dir, 'sysc'),
- shutil.ignore_patterns('tmp'))
+ shutil.copytree(self.sysc_dir, os.path.join(self.tmp_dir, 'sysc'),
+ ignore=shutil.ignore_patterns('tmp', 'distfiles'))
def stage0_posix(self):
"""Copy in all of the stage0-posix"""
diff --git a/sysa/helpers.sh b/sysa/helpers.sh
@@ -412,6 +412,7 @@ populate_device_nodes() {
sys_transfer() {
local dest=$1
+ local sys_sources=$2
mkdir -p "${dest}/${PREFIX}/bin" "${dest}/${PREFIX}/src"
@@ -419,11 +420,13 @@ sys_transfer() {
cp "${PREFIX}/bin/bash" "${PREFIX}/bin/tar" "${PREFIX}/bin/bzip2" "${dest}${PREFIX}/bin/"
# Transfer misc files
- cp "${SOURCES}/helpers.sh" "${SOURCES}/SHA256SUMS.pkgs" "${SOURCES}/bootstrap.cfg" "${dest}/"
+ cp "${SOURCES}/helpers.sh" "${SOURCES}/SHA256SUMS.pkgs" "${SOURCES}/bootstrap.cfg" "${dest}/${PREFIX}/src"
- cp -r "${PREFIX}/src/" "${dest}${PREFIX}/"
+ cp -r "${sys_sources}/*" "${dest}/${PREFIX}/src"
+ cp -f "${sys_sources}/init" "${dest}/"
+ cp -r "${PREFIX}/src/repo" "${dest}/${PREFIX}/src"
- shift
+ shift 2
# Copy additional binaries
set -- "${@/#/${PREFIX}/bin/}"
cp "$@" "${dest}${PREFIX}/bin/"
diff --git a/sysa/run.sh b/sysa/run.sh
@@ -22,16 +22,11 @@ export SRCDIR="${srcdir}"
create_sysb() {
# Copy everything in
echo "Creating sysb rootfs"
- mkdir -p "/sysb${PREFIX}"
- for d in bin include lib libexec share src; do
- # Minimise RAM (storage) use - use hard links
- cp -rl "${PREFIX}/${d}" "/sysb${PREFIX}/${d}"
- done
- cp "${SOURCES}/helpers.sh" "${SOURCES}/SHA256SUMS.pkgs" "${SOURCES}/bootstrap.cfg" "/sysb/${SRCDIR}"
- populate_device_nodes /sysb
+ sys_transfer /sysb_image /sysb
+ cp -rl /sysc /sysb/sysc_src
echo "Creating sysb initramfs"
gen_initramfs_list.sh -o "${PREFIX}/boot/initramfs-sysb.cpio.gz" /sysb
- rm -rf /sysb # Cleanup
+ rm -rf /sysb /sysb_image # Cleanup
}
go_sysb() {
@@ -256,6 +251,6 @@ if [ "${CHROOT}" = False ]; then
fi
# In chroot mode transition directly into System C.
-SYSC=/sysc
-sys_transfer "${SYSC}" gzip patch
+SYSC=/sysc_image
+sys_transfer "${SYSC}" /sysc gzip patch
exec chroot "${SYSC}" /init
diff --git a/sysb/init b/sysb/init
@@ -7,7 +7,31 @@
set -e
+. /usr/src/helpers.sh
+
+export PATH=/usr/bin
+export PREFIX=/usr
+export SOURCES=/usr/src
+
+echo
+echo "Installing packages into sysb"
+
+install_tar() {
+ echo "${1}: installing package"
+ src_apply_tar "$@"
+}
+
+# Install needed packages.
+install_tar coreutils-5.0 0
+install_tar sed-4.0.9 0
+
+install_tar bzip2-1.0.8 0
+install_tar coreutils-6.10 0
+install_tar e2fsprogs-1.45.7 0
+install_tar grep-2.4 0
+install_tar kexec-tools-2.0.22 0
+install_tar util-linux-2.19.1 0
+
# Begin sysb bootstrapping process
-mv ./run.sh /usr/src
cd /usr/src
./run.sh
diff --git a/sysb/run.sh b/sysb/run.sh
@@ -34,10 +34,15 @@ create_hdx
ask_disk() {
echo
echo "What disk would you like to use for live-bootstrap?"
- echo "(live-bootstrap assumes you have pre-prepared the disk)."
- echo "Please provide in format sdxx (as you would find under /dev)."
+ echo "This disk may have pre-prepared sources on it."
+ echo "If there is no partition we will make one".
+ echo "Please provide in format sdxx (as you would find under /dev),"
+ echo "or sdx if it is a blank disk. An ext4 partition is expected on"
+ echo "existing disks."
echo "You can type 'list' to get a list of disks to help you figure"
echo "out which is the right disk."
+ echo "NO WARRANTY IS PROVIDED FOR BUGGY BEHAVIOUR, INCLUDING THAT"
+ echo "REGARDING DISKS & DATA."
echo
read -r DISK
@@ -57,6 +62,15 @@ if [ -z "${DISK}" ] || ! [ -e "/dev/${DISK}" ]; then
echo "DISK=${DISK}" >> /usr/src/bootstrap.cfg
fi
+if [ -z "$(fdisk -l "/dev/${DISK}" | grep -E "${DISK}p?[0-9]")" ]; then
+ echo "Creating partition table and partition"
+ echo ";" | sfdisk "/dev/${DISK}"
+ mkfs.ext4 "/dev/${DISK}1"
+ DISK="${DISK}1"
+fi
+
+echo "export DISK=${DISK}" >> /usr/src/bootstrap.cfg
+
PREFIX=/usr
SOURCES="${PREFIX}/src"
SYSC=/sysc
@@ -68,7 +82,7 @@ mount -t ext4 "/dev/${DISK}" /sysc
# Copy over appropriate data
echo "Copying data into sysc"
-sys_transfer "${SYSC}" gzip patch
+sys_transfer "${SYSC}" /sysc_src gzip patch
sync
# switch_root into sysc 1. for simplicity 2. to avoid kexecing again
diff --git a/sysc.py b/sysc.py
@@ -21,10 +21,11 @@ class SysC(SysGeneral):
dev_name = None
# pylint: disable=too-many-instance-attributes
- def __init__(self, arch, preserve_tmp, tmpdir):
+ def __init__(self, arch, preserve_tmp, tmpdir, external_sources):
self.git_dir = os.path.dirname(os.path.join(__file__))
self.arch = arch
self.preserve_tmp = preserve_tmp
+ self.external_sources = external_sources
self.sys_dir = os.path.join(self.git_dir, 'sysc')
self.cache_dir = os.path.join(self.sys_dir, 'distfiles')
@@ -55,18 +56,154 @@ class SysC(SysGeneral):
if create_disk_image:
# Create + mount a disk for QEMU to use
disk_path = os.path.join(self.tmp_dir, 'disk.img')
- self.dev_name = create_disk(disk_path, "msdos", "ext4", '8G')
- rootfs_dir = os.path.join(self.tmp_dir, 'mnt')
- os.mkdir(rootfs_dir)
- mount(self.dev_name + "p1", rootfs_dir, 'ext4')
+ if self.external_sources:
+ self.dev_name = create_disk(disk_path, "msdos", "ext4", '8G')
+ self.rootfs_dir = os.path.join(self.tmp_dir, 'mnt')
+ os.mkdir(rootfs_dir)
+ mount(self.dev_name + "p1", self.rootfs_dir, 'ext4')
+ else:
+ self.dev_name = create_disk(disk_path, "none", "ext4", '8G')
# Use chown to allow executing user to access it
run('sudo', 'chown', getpass.getuser(), self.dev_name)
- run('sudo', 'chown', getpass.getuser(), rootfs_dir)
+ if self.external_sources:
+ run('sudo', 'chown', getpass.getuser(), rootfs_dir)
else:
rootfs_dir = self.tmp_dir
- copytree(self.sys_dir, rootfs_dir, ignore=shutil.ignore_patterns("tmp"))
+ if self.external_sources:
+ self.get_packages()
# Unmount tmp/mnt if it was mounted
if create_disk_image:
umount(rootfs_dir)
+
+ # pylint: disable=line-too-long,too-many-statements
+ def get_packages(self):
+ """Prepare remaining sources"""
+ # bash 5.1
+ self.get_file("https://mirrors.kernel.org/gnu/bash/bash-5.1.tar.gz")
+
+ # xz 5.0.5
+ self.get_file("https://ixpeering.dl.sourceforge.net/project/lzmautils/xz-5.0.5.tar.bz2")
+
+ # automake 1.11.2
+ self.get_file("https://mirrors.kernel.org/gnu/automake/automake-1.11.2.tar.bz2")
+
+ # autoconf 2.69
+ self.get_file("https://mirrors.kernel.org/gnu/autoconf/autoconf-2.69.tar.xz")
+
+ # automake 1.15.1
+ self.get_file("https://mirrors.kernel.org/gnu/automake/automake-1.15.1.tar.xz")
+
+ # tar 1.34
+ self.get_file(["https://mirrors.kernel.org/gnu/tar/tar-1.34.tar.xz",
+ "https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-30820c.tar.gz"])
+
+ # coreutils 8.32
+ self.get_file(["https://git.savannah.gnu.org/cgit/coreutils.git/snapshot/coreutils-8.32.tar.gz",
+ "https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-d279bc.tar.gz"])
+
+ # pkg-config 0.29.2
+ self.get_file("http://gentoo.osuosl.org/distfiles/pkg-config-0.29.2.tar.gz")
+
+ # make 4.2.1
+ self.get_file("https://ftp.gnu.org/gnu/make/make-4.2.1.tar.gz")
+
+ # gmp 6.2.1
+ self.get_file("https://mirrors.kernel.org/gnu/gmp/gmp-6.2.1.tar.xz")
+
+ # autoconf archive 2021.02.19
+ self.get_file("https://mirrors.kernel.org/gnu/autoconf-archive/autoconf-archive-2021.02.19.tar.xz")
+
+ # mpfr 4.1.0
+ self.get_file("https://mirrors.kernel.org/gnu/mpfr/mpfr-4.1.0.tar.xz")
+
+ # mpc 1.2.1
+ self.get_file("https://mirrors.kernel.org/gnu/mpc/mpc-1.2.1.tar.gz")
+
+ # flex 2.5.33
+ self.get_file("http://download.nust.na/pub2/openpkg1/sources/DST/flex/flex-2.5.33.tar.gz")
+
+ # bison 2.3
+ self.get_file(["https://mirrors.kernel.org/gnu/bison/bison-2.3.tar.bz2",
+ "https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-b28236b.tar.gz"])
+
+ # bison 3.4.2
+ self.get_file(["https://mirrors.kernel.org/gnu/bison/bison-3.4.2.tar.xz",
+ "https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-672663a.tar.gz"])
+
+ # perl 5.10.5
+ self.get_file("https://www.cpan.org/src/5.0/perl-5.10.1.tar.bz2")
+
+ # dist 3.5-236
+ # Debian's version is used because upstream is not to be found (dead?)
+ self.get_file("http://deb.debian.org/debian/pool/main/d/dist/dist_3.5-236.orig.tar.gz",
+ output="dist-3.5-236.tar.gz")
+
+ # perl 5.32.1
+ self.get_file(["https://www.cpan.org/src/5.0/perl-5.32.1.tar.xz",
+ "http://deb.debian.org/debian/pool/main/p/perl/perl_5.32.1.orig-regen-configure.tar.gz"])
+
+ # libarchive-3.5.2
+ self.get_file("https://libarchive.org/downloads/libarchive-3.5.2.tar.xz")
+
+ # openssl-1.1.1l
+ self.get_file("https://artfiles.org/openssl.org/source/old/1.1.1/openssl-1.1.1l.tar.gz")
+
+ # curl 7.83.0
+ self.get_file("https://master.dl.sourceforge.net/project/curl.mirror/curl-7_83_0/curl-7.83.0.tar.xz?viasf=1",
+ output="curl-7.83.0.tar.xz")
+
+ # ca-certificates-3.78
+ self.get_file("https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_78_RTM/src/nss-3.78.tar.gz",
+ output="ca-certificates-3.78.tar.gz")
+
+ # xbps 0.59.1
+ self.get_file("https://github.com/void-linux/xbps/archive/refs/tags/0.59.1.tar.gz",
+ output="xbps-0.59.1.tar.gz")
+
+ # autoconf 2.71
+ self.get_file("https://mirrors.kernel.org/gnu/autoconf/autoconf-2.71.tar.xz")
+
+ # automake 1.16.3
+ self.get_file("https://mirrors.kernel.org/gnu/automake/automake-1.16.3.tar.xz")
+
+ # patch 2.7.6
+ self.get_file(["https://mirrors.kernel.org/gnu/patch/patch-2.7.6.tar.xz",
+ "https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-e017871.tar.gz"])
+
+ # gettext 0.21
+ self.get_file(["https://mirrors.kernel.org/gnu/gettext/gettext-0.21.tar.xz",
+ "https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-7daa86f.tar.gz"])
+
+ # texinfo 6.7
+ self.get_file(["https://mirrors.kernel.org/gnu/texinfo/texinfo-6.7.tar.xz",
+ "https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-b81ec69.tar.gz"])
+
+ # zlib 1.2.12
+ self.get_file("https://www.zlib.net/zlib-1.2.12.tar.xz")
+
+ # gcc 4.7.4
+ self.get_file("https://mirrors.kernel.org/gnu/gcc/gcc-4.7.4/gcc-4.7.4.tar.bz2")
+
+ # gperf 3.1
+ self.get_file("https://mirrors.kernel.org/gnu/gperf/gperf-3.1.tar.gz")
+
+ # libunistring 0.9.10
+ self.get_file(["https://mirrors.kernel.org/gnu/libunistring/libunistring-0.9.10.tar.xz",
+ "https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-52a06cb3.tar.gz"])
+
+ # libffi 3.3
+ self.get_file("https://github.com/libffi/libffi/releases/download/v3.3/libffi-3.3.tar.gz")
+
+ # libatomic_ops 7.6.10
+ self.get_file("https://github.com/ivmai/libatomic_ops/releases/download/v7.6.10/libatomic_ops-7.6.10.tar.gz")
+
+ # boehm-gc 8.0.4
+ self.get_file("https://www.hboehm.info/gc/gc_source/gc-8.0.4.tar.gz")
+
+ # guile 3.0.7
+ self.get_file(["https://mirrors.kernel.org/gnu/guile/guile-3.0.7.tar.xz",
+ "https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-901694b9.tar.gz",
+ "https://github.com/schierlm/guile-psyntax-bootstrapping/archive/refs/tags/guile-3.0.7.tar.gz"],
+ output=["guile-3.0.7.tar.xz", "gnulib-901694b9.tar.gz", "guile-psyntax-bootstrapping.tar.gz"])
diff --git a/sysc/init b/sysc/init
@@ -7,7 +7,7 @@
set -e
# shellcheck source=sysa/helpers.sh
-. helpers.sh
+. /usr/src/helpers.sh
export PATH=/usr/bin:/usr/sbin
export PREFIX=/usr
@@ -54,12 +54,6 @@ install_tar util-linux-2.19.1 0
# Fix invocation of bash from perl
ln -s /usr/bin/bash /usr/bin/sh
-# Prepare sysc directory structure
-mkdir -p /usr/src
-mv run*.sh /usr/src
-mv helpers.sh SHA256SUMS.pkgs bootstrap.cfg /usr/src
-mv /*-* /usr/src # build scripts
-
# Check tarballs
cd "${DISTFILES}"
sha256sum -c /SHA256SUMS.sources
