commit: 51fe3f999179985284ab5abde180c7762d1f4d68
parent ca0e701ee8e0687856775428fde3c702bac03dd4
Author: fosslinux <fosslinux@aussies.space>
Date: Wed, 27 Dec 2023 22:59:23 +0000
Merge pull request #368 from Googulator/always-preserve
Rename tmpdir to target and always preserve it, dropping --preserve
Diffstat:
6 files changed, 159 insertions(+), 171 deletions(-)
diff --git a/.github/workflows/bwrap.yml b/.github/workflows/bwrap.yml
@@ -43,15 +43,15 @@ jobs:
distfiles
key: cache-${{ hashFiles('steps/*/sources') }}
- name: Run bootstrap
- run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass1
+ run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass1
- name: Archive created packages
if: failure() # archive failed builds progress
uses: actions/upload-artifact@v3
with:
name: packages
- path: tmp/external/repo/**
+ path: target/external/repo/**
- name: Tar pass1 image
- run: tar -cf pass1_image.tar tmp
+ run: tar -cf pass1_image.tar target
- name: Archive pass1_image
uses: actions/upload-artifact@v3
with:
@@ -96,15 +96,15 @@ jobs:
distfiles
key: cache-${{ hashFiles('steps/*/sources') }}
- name: Run bootstrap
- run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass2
+ run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass2
- name: Archive created packages
if: failure() # archive failed builds progress
uses: actions/upload-artifact@v3
with:
name: internal_packages_pass2
- path: tmp/external/repo/**
+ path: target/external/repo/**
- name: Tar pass2 image
- run: tar -cf pass2_image.tar tmp
+ run: tar -cf pass2_image.tar target
- name: Archive pass2_image
uses: actions/upload-artifact@v3
with:
@@ -149,10 +149,10 @@ jobs:
distfiles
key: cache-${{ hashFiles('steps/*/sources') }}
- name: Run bootstrap
- run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass3
+ run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3
- name: Archive created packages
if: always() # archive both failed and successful builds
uses: actions/upload-artifact@v3
with:
name: packages
- path: tmp/external/repo/**
+ path: target/external/repo/**
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -21,7 +21,7 @@ jobs:
- name: Install pylint
run: sudo pip3 install pylint
- name: pylint
- run: pylint rootfs.py lib/utils.py lib/generator.py lib/tmpdir.py --disable=duplicate-code
+ run: pylint rootfs.py lib/utils.py lib/generator.py lib/target.py --disable=duplicate-code
shellcheck:
name: Lint shell files
diff --git a/lib/generator.py b/lib/generator.py
@@ -29,26 +29,26 @@ class Generator():
self.external_sources = external_sources
self.repo_path = repo_path
self.source_manifest = self.get_source_manifest(not self.external_sources)
- self.tmp_dir = None
+ self.target_dir = None
self.external_dir = None
- def reuse(self, tmpdir):
+ def reuse(self, target):
"""
Reuse a previously prepared bwrap environment for further stages.
"""
- self.tmp_dir = tmpdir.path
- self.external_dir = os.path.join(self.tmp_dir, 'external')
+ self.target_dir = target.path
+ self.external_dir = os.path.join(self.target_dir, 'external')
self.distfiles()
- def prepare(self, tmpdir, using_kernel=False, kernel_bootstrap=False, target_size=0):
+ def prepare(self, target, using_kernel=False, kernel_bootstrap=False, target_size=0):
"""
Prepare basic media of live-bootstrap.
/steps -- contains steps to be built
/ -- contains seed to allow steps to be built, containing custom
scripts and stage0-posix
"""
- self.tmp_dir = tmpdir.path
- self.external_dir = os.path.join(self.tmp_dir, 'external')
+ self.target_dir = target.path
+ self.external_dir = os.path.join(self.target_dir, 'external')
# We use ext3 here; ext4 actually has a variety of extensions that
# have been added with varying levels of recency
@@ -59,33 +59,33 @@ class Generator():
# argument matrix ... or we could just use ext3 instead which
# is effectively universally the same
if kernel_bootstrap:
- init_path = os.path.join(self.tmp_dir, 'init')
+ init_path = os.path.join(self.target_dir, 'init')
os.mkdir(init_path)
- self.tmp_dir = init_path
+ self.target_dir = init_path
if self.repo_path or self.external_sources:
- tmpdir.add_disk("external", filesystem="ext3")
- tmpdir.mount_disk("external", "external")
+ target.add_disk("external", filesystem="ext3")
+ target.mount_disk("external", "external")
else:
- self.external_dir = os.path.join(self.tmp_dir, 'external')
+ self.external_dir = os.path.join(self.target_dir, 'external')
elif using_kernel:
- self.tmp_dir = os.path.join(self.tmp_dir, 'disk')
- tmpdir.add_disk("disk",
+ self.target_dir = os.path.join(self.target_dir, 'disk')
+ target.add_disk("disk",
filesystem="ext3",
size=(target_size + "M") if target_size else "16G",
bootable=True)
- tmpdir.mount_disk("disk", "disk")
- self.external_dir = os.path.join(self.tmp_dir, 'external')
+ target.mount_disk("disk", "disk")
+ self.external_dir = os.path.join(self.target_dir, 'external')
os.makedirs(self.external_dir, exist_ok=True)
if self.early_preseed:
# Extract tar containing preseed
with tarfile.open(self.early_preseed, "r") as seed:
- seed.extractall(self.tmp_dir)
+ seed.extractall(self.target_dir)
shutil.copy2(os.path.join(self.git_dir, 'seed', 'preseeded.kaem'),
- os.path.join(self.tmp_dir, 'kaem.x86'))
+ os.path.join(self.target_dir, 'kaem.x86'))
else:
self.stage0_posix()
self.seed()
@@ -101,25 +101,25 @@ class Generator():
shutil.copytree(self.repo_path, repo_dir)
if kernel_bootstrap:
- self.create_builder_hex0_disk_image(self.tmp_dir + '.img', target_size)
+ self.create_builder_hex0_disk_image(self.target_dir + '.img', target_size)
if kernel_bootstrap and (self.external_sources or self.repo_path):
- tmpdir.umount_disk('external')
+ target.umount_disk('external')
elif using_kernel:
- tmpdir.umount_disk('disk')
+ target.umount_disk('disk')
def steps(self):
"""Copy in steps."""
self.get_packages()
- shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.tmp_dir, 'steps'))
+ shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.target_dir, 'steps'))
def stage0_posix(self):
"""Copy in all of the stage0-posix"""
stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix')
for entry in os.listdir(stage0_posix_base_dir):
orig = os.path.join(stage0_posix_base_dir, entry)
- target = os.path.join(self.tmp_dir, entry)
+ target = os.path.join(self.target_dir, entry)
if os.path.isfile(orig):
shutil.copy2(orig, target)
else:
@@ -128,14 +128,14 @@ class Generator():
arch = stage0_arch_map.get(self.arch, self.arch)
kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix', 'bootstrap-seeds',
'POSIX', arch, 'kaem-optional-seed')
- shutil.copy2(kaem_optional_seed, os.path.join(self.tmp_dir, 'init'))
+ shutil.copy2(kaem_optional_seed, os.path.join(self.target_dir, 'init'))
def seed(self):
"""Copy in extra seed files"""
seed_dir = os.path.join(self.git_dir, 'seed')
for entry in os.listdir(seed_dir):
if os.path.isfile(os.path.join(seed_dir, entry)):
- shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.tmp_dir, entry))
+ shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.target_dir, entry))
@staticmethod
def add_fiwix_files(file_list_path, dirpath):
@@ -149,14 +149,14 @@ class Generator():
def create_fiwix_file_list(self):
"""Create a list of files to populate Fiwix file system"""
- file_list_path = os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1',
+ file_list_path = os.path.join(self.target_dir, 'steps', 'lwext4-1.0.0-lb1',
'files', 'fiwix-file-list.txt')
- shutil.copyfile(os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1',
+ shutil.copyfile(os.path.join(self.target_dir, 'steps', 'lwext4-1.0.0-lb1',
'files', 'early-artifacts-needed-after-fiwix.txt'),
file_list_path)
save_cwd = os.getcwd()
- os.chdir(self.tmp_dir)
+ os.chdir(self.target_dir)
self.add_fiwix_files(file_list_path, 'steps')
self.add_fiwix_files(file_list_path, 'distfiles')
os.chdir(save_cwd)
@@ -170,7 +170,7 @@ class Generator():
shutil.copy2(os.path.join(self.distfiles_dir, file),
os.path.join(out, file))
- early_distfile_dir = os.path.join(self.tmp_dir, 'external', 'distfiles')
+ early_distfile_dir = os.path.join(self.target_dir, 'external', 'distfiles')
main_distfile_dir = os.path.join(self.external_dir, 'distfiles')
if early_distfile_dir != main_distfile_dir:
@@ -217,7 +217,7 @@ class Generator():
"""Append srcfs file system to disk image"""
save_cwd = os.getcwd()
- os.chdir(self.tmp_dir)
+ os.chdir(self.target_dir)
self.output_tree(image_file, '.')
# Add commands to kick off stage0-posix
diff --git a/lib/target.py b/lib/target.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: 2023 fosslinux <fosslinux@aussies.space>
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""
+Contains a class that represents a target directory
+"""
+
+import enum
+import getpass
+import os
+
+from lib.utils import mount, umount, create_disk, run_as_root
+
+class TargetType(enum.Enum):
+ """Different types of target dirs we can have"""
+ NONE = 0
+ TMPFS = 1
+
+class Target:
+ """
+ Represents a target directory
+ """
+
+ _disks = {}
+ _disk_filesystems = {}
+ _mountpoints = {}
+
+ def __init__(self, path="target"):
+ self.path = os.path.abspath(path)
+ self._type = TargetType.NONE
+
+ if not os.path.exists(self.path):
+ os.mkdir(self.path)
+
+ def __del__(self):
+ for path in self._mountpoints:
+ print(f"Unmounting {path}")
+ umount(path)
+
+ for disk in self._disks.values():
+ print(f"Detaching {disk}")
+ run_as_root("losetup", "-d", disk)
+
+ def tmpfs(self, size="8G"):
+ """Mount a tmpfs"""
+ print(f"Mounting tmpfs on {self.path}")
+ mount("tmpfs", self.path, "tmpfs", f"size={size}")
+ self._type = TargetType.TMPFS
+
+ # pylint: disable=too-many-arguments
+ def add_disk(self,
+ name,
+ size="16G",
+ filesystem="ext4",
+ tabletype="msdos",
+ bootable=False,
+ mkfs_args=None):
+ """Add a disk"""
+ disk_path = os.path.join(self.path, f"{name}.img")
+ self._disks[name] = create_disk(disk_path,
+ tabletype,
+ filesystem,
+ size,
+ bootable,
+ mkfs_args)
+ self._disk_filesystems[name] = filesystem
+ # Allow executing user to access it
+ run_as_root("chown", getpass.getuser(), self._disks[name])
+
+ def mount_disk(self, name, mountpoint=None):
+ """Mount the disk"""
+ if mountpoint is None:
+ mountpoint = f"{name}_mnt"
+ mountpoint = os.path.join(self.path, mountpoint)
+ os.mkdir(mountpoint)
+ mount(self._disks[name] + "p1", mountpoint, self._disk_filesystems[name])
+ # Allow executing user to access it
+ run_as_root("chown", getpass.getuser(), mountpoint)
+ self._mountpoints[name] = mountpoint
+ return mountpoint
+
+ def umount_disk(self, name):
+ """Unmount a disk"""
+ umount(self._mountpoints[name])
+ del self._mountpoints[name]
+
+ def get_disk(self, name):
+ """Get the path to a device of a disk"""
+ return self._disks.get(name)
diff --git a/lib/tmpdir.py b/lib/tmpdir.py
@@ -1,101 +0,0 @@
-#!/usr/bin/env python3
-
-# SPDX-FileCopyrightText: 2023 fosslinux <fosslinux@aussies.space>
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-"""
-Contains a class that represents a tmpdir
-"""
-
-import enum
-import getpass
-import os
-import shutil
-
-from lib.utils import mount, umount, create_disk, run_as_root
-
-class TmpType(enum.Enum):
- """Different types of tmpdirs we can have"""
- NONE = 0
- TMPFS = 1
-
-class Tmpdir:
- """
- Represents a tmpdir
- """
-
- _disks = {}
- _disk_filesystems = {}
- _mountpoints = {}
-
- def __init__(self, preserve, path="tmp"):
- self.path = os.path.abspath(path)
- self.preserve = preserve
- self._type = TmpType.NONE
-
- if not os.path.exists(self.path):
- os.mkdir(self.path)
-
- def __del__(self):
- for path in self._mountpoints:
- print(f"Unmounting {path}")
- umount(path)
-
- if not self.preserve:
- for disk in self._disks.values():
- print(f"Detaching {disk}")
- run_as_root("losetup", "-d", disk)
-
- if self._type == TmpType.TMPFS:
- print(f"Unmounting tmpdir from {self.path}")
- umount(self.path)
-
- print(f"Removing {self.path}")
- shutil.rmtree(self.path, ignore_errors=True)
-
- def tmpfs(self, size="8G"):
- """Mount a tmpfs"""
- print(f"Mounting tmpfs on {self.path}")
- mount("tmpfs", self.path, "tmpfs", f"size={size}")
- self._type = TmpType.TMPFS
-
- # pylint: disable=too-many-arguments
- def add_disk(self,
- name,
- size="16G",
- filesystem="ext4",
- tabletype="msdos",
- bootable=False,
- mkfs_args=None):
- """Add a disk"""
- disk_path = os.path.join(self.path, f"{name}.img")
- self._disks[name] = create_disk(disk_path,
- tabletype,
- filesystem,
- size,
- bootable,
- mkfs_args)
- self._disk_filesystems[name] = filesystem
- # Allow executing user to access it
- run_as_root("chown", getpass.getuser(), self._disks[name])
-
- def mount_disk(self, name, mountpoint=None):
- """Mount the disk"""
- if mountpoint is None:
- mountpoint = f"{name}_mnt"
- mountpoint = os.path.join(self.path, mountpoint)
- os.mkdir(mountpoint)
- mount(self._disks[name] + "p1", mountpoint, self._disk_filesystems[name])
- # Allow executing user to access it
- run_as_root("chown", getpass.getuser(), mountpoint)
- self._mountpoints[name] = mountpoint
- return mountpoint
-
- def umount_disk(self, name):
- """Unmount a disk"""
- umount(self._mountpoints[name])
- del self._mountpoints[name]
-
- def get_disk(self, name):
- """Get the path to a device of a disk"""
- return self._disks.get(name)
diff --git a/rootfs.py b/rootfs.py
@@ -17,7 +17,7 @@ import argparse
import os
from lib.utils import run, run_as_root
-from lib.tmpdir import Tmpdir
+from lib.target import Target
from lib.generator import Generator, stage0_arch_map
def create_configuration_file(args):
@@ -58,11 +58,9 @@ def main():
action="store_true")
parser.add_argument("-bw", "--bwrap", help="Run inside a bwrap sandbox",
action="store_true")
- parser.add_argument("-p", "--preserve", help="Do not remove temporary dir",
- action="store_true")
- parser.add_argument("-t", "--tmpdir", help="Temporary directory",
- default="tmp")
- parser.add_argument("--tmpfs", help="Use a tmpfs on tmpdir",
+ parser.add_argument("-t", "--target", help="Target directory",
+ default="target")
+ parser.add_argument("--tmpfs", help="Use a tmpfs on target",
action="store_true")
parser.add_argument("--tmpfs-size", help="Size of the tmpfs",
default="8G")
@@ -128,7 +126,7 @@ def main():
if args.arch != "x86":
print("Only x86 is supported at the moment, other arches are for development only.")
- # Tmp validation
+ # Tmpfs validation
if args.bwrap and args.tmpfs:
raise ValueError("tmpfs cannot be used with bwrap.")
@@ -156,19 +154,19 @@ def main():
with open(os.path.join('steps', 'bootstrap.cfg'), 'a', encoding='UTF-8'):
pass
- # tmpdir
- tmpdir = Tmpdir(path=args.tmpdir, preserve=args.preserve)
+ # target
+ target = Target(path=args.target)
if args.tmpfs:
- tmpdir.tmpfs(size=args.tmpfs_size)
+ target.tmpfs(size=args.tmpfs_size)
generator = Generator(arch=args.arch,
external_sources=args.external_sources,
repo_path=args.repo,
early_preseed=args.early_preseed)
- bootstrap(args, generator, tmpdir, args.target_size)
+ bootstrap(args, generator, target, args.target_size)
-def bootstrap(args, generator, tmpdir, size):
+def bootstrap(args, generator, target, size):
"""Kick off bootstrap process."""
print(f"Bootstrapping {args.arch}")
if args.chroot:
@@ -179,28 +177,28 @@ print(shutil.which('chroot'))
chroot_binary = run_as_root('python3', '-c', find_chroot,
capture_output=True).stdout.decode().strip()
- generator.prepare(tmpdir, using_kernel=False)
+ generator.prepare(target, using_kernel=False)
arch = stage0_arch_map.get(args.arch, args.arch)
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
- run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.tmp_dir, init)
+ run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.target_dir, init)
elif args.bwrap:
init = '/init'
if not args.internal_ci or args.internal_ci == "pass1":
- generator.prepare(tmpdir, using_kernel=False)
+ generator.prepare(target, using_kernel=False)
arch = stage0_arch_map.get(args.arch, args.arch)
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
else:
- generator.reuse(tmpdir)
+ generator.reuse(target)
run('env', '-i', 'bwrap', '--unshare-user',
'--uid', '0',
'--gid', '0',
'--unshare-net' if args.external_sources else None,
'--setenv', 'PATH', '/usr/bin',
- '--bind', generator.tmp_dir, '/',
+ '--bind', generator.target_dir, '/',
'--dir', '/dev',
'--dev-bind', '/dev/null', '/dev/null',
'--dev-bind', '/dev/zero', '/dev/zero',
@@ -216,44 +214,44 @@ print(shutil.which('chroot'))
elif args.bare_metal:
if args.kernel:
- generator.prepare(tmpdir, using_kernel=True, target_size=size)
- image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir))
+ generator.prepare(target, using_kernel=True, target_size=size)
+ path = os.path.join(args.target, os.path.relpath(generator.target_dir, args.target))
print("Please:")
- print(f" 1. Take {image_path}/initramfs and your kernel, boot using this.")
- print(f" 2. Take {image_path}/disk.img and put this on a writable storage medium.")
+ print(f" 1. Take {path}/initramfs and your kernel, boot using this.")
+ print(f" 2. Take {path}/disk.img and put this on a writable storage medium.")
else:
- generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size)
- image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir))
+ generator.prepare(target, kernel_bootstrap=True, target_size=size)
+ path = os.path.join(args.target, os.path.relpath(generator.target_dir, args.target))
print("Please:")
- print(f" 1. Take {image_path}.img and write it to a boot drive and then boot it.")
+ print(f" 1. Take {path}.img and write it to a boot drive and then boot it.")
else:
if args.kernel:
- generator.prepare(tmpdir, using_kernel=True, target_size=size)
+ generator.prepare(target, using_kernel=True, target_size=size)
run(args.qemu_cmd,
'-enable-kvm',
'-m', str(args.qemu_ram) + 'M',
'-smp', str(args.cores),
'-no-reboot',
- '-drive', 'file=' + tmpdir.get_disk("disk") + ',format=raw',
- '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
+ '-drive', 'file=' + target.get_disk("disk") + ',format=raw',
+ '-drive', 'file=' + target.get_disk("external") + ',format=raw',
'-nic', 'user,ipv6=off,model=e1000',
'-kernel', args.kernel,
'-nographic',
'-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw')
else:
- generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size)
+ generator.prepare(target, kernel_bootstrap=True, target_size=size)
arg_list = [
'-enable-kvm',
'-m', str(args.qemu_ram) + 'M',
'-smp', str(args.cores),
'-no-reboot',
- '-drive', 'file=' + generator.tmp_dir + '.img' + ',format=raw'
+ '-drive', 'file=' + generator.target_dir + '.img' + ',format=raw'
]
- if tmpdir.get_disk("external") is not None:
+ if target.get_disk("external") is not None:
arg_list += [
- '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
+ '-drive', 'file=' + target.get_disk("external") + ',format=raw',
]
arg_list += [
'-machine', 'kernel-irqchip=split',
