commit: 51b0bf840516988d9c920449f9bbe5ba3ce09149
parent 4f9f56f006f15b49a4c17c81784becef8cefedc2
Author: fosslinux <fosslinux@aussies.space>
Date: Sat, 28 Jan 2023 11:11:32 +1100
Rework tmpdir & associated a bit.
- Split out tmpdir logic into a separate entity & add the appropriate
arguments and checks.
- sysb can be removed since there is now no associated logic.
- Move disk/etc logic into tmpdir.py.
Diffstat:
| M | .cirrus.yml | 2 | +- |
| M | lib/sysgeneral.py | 33 | ++------------------------------- |
| A | lib/tmpdir.py | 99 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| M | lib/utils.py | 2 | +- |
| M | rootfs.py | 118 | +++++++++++++++++++++++++++++-------------------------------------------------- |
| M | sysa.py | 33 | +++++++++++---------------------- |
| D | sysb.py | 21 | --------------------- |
| M | sysc.py | 57 | +++++++++++++-------------------------------------------- |
8 files changed, 170 insertions(+), 195 deletions(-)
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -13,7 +13,7 @@ pylint_task:
- apt-get -y clean
- pip3 install pylint
check_script:
- - pylint rootfs.py sysa.py sysb.py sysc.py lib/utils.py lib/sysgeneral.py --disable=duplicate-code
+ - pylint rootfs.py sysa.py sysc.py lib/utils.py lib/sysgeneral.py lib/tmpdir.py --disable=duplicate-code
shell_lint_task:
container:
diff --git a/lib/sysgeneral.py b/lib/sysgeneral.py
@@ -4,20 +4,17 @@ This file contains a few functions to be shared by all Sys* classes
"""
# SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
-# SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
+# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
# SPDX-License-Identifier: GPL-3.0-or-later
import os
-import shutil
import hashlib
import glob
import subprocess
import requests
-from lib.utils import mount, umount
-
class SysGeneral:
"""
A class from which all Sys* class are extended.
@@ -25,38 +22,12 @@ class SysGeneral:
"""
# All of these are variables defined in the individual Sys* classes
- preserve_tmp = None
- tmp_dir = None
cache_dir = None
base_dir = None
git_dir = None
sys_dir = None
initramfs_path = None
- mounted_tmpfs = False
-
- def __del__(self):
- if not self.preserve_tmp:
- self.remove_tmp()
-
- def remove_tmp(self):
- """Remove the tmp directory"""
- if self.tmp_dir is None:
- return
-
- if self.mounted_tmpfs:
- print(f"Unmounting tmpfs from {self.tmp_dir}")
- umount(self.tmp_dir)
-
- print(f"Removing {self.tmp_dir}")
- shutil.rmtree(self.tmp_dir, ignore_errors=True)
-
- def mount_tmpfs(self):
- """Mount the tmpfs for this sysx"""
- if not os.path.isdir(self.tmp_dir):
- os.mkdir(self.tmp_dir)
- print(f"Mounting tmpfs on {self.tmp_dir}")
- mount('tmpfs', self.tmp_dir, 'tmpfs', 'size=8G')
- self.mounted_tmpfs = True
+ tmp_dir = None
def check_file(self, file_name, expected_hash):
"""Check hash of downloaded source file."""
diff --git a/lib/tmpdir.py b/lib/tmpdir.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: 2023 fosslinux <fosslinux@aussies.space>
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""
+Contains a class that represents a tmpdir
+"""
+
+import enum
+import getpass
+import os
+import shutil
+
+from lib.utils import mount, umount, create_disk, run
+
+class TmpType(enum.Enum):
+ """Different types of tmpdirs we can have"""
+ NONE = 0
+ TMPFS = 1
+
+class Tmpdir:
+ """
+ Represents a tmpdir
+ """
+
+ _syses = {}
+ _disks = {}
+ _disk_filesystems = {}
+ _mountpoints = {}
+
+ def __init__(self, preserve, path="tmp"):
+ self.path = os.path.abspath(path)
+ self.preserve = preserve
+ self._type = TmpType.NONE
+
+ if not os.path.exists(self.path):
+ os.mkdir(self.path)
+
+ def __del__(self):
+ for path in self._mountpoints:
+ print(f"Unmounting {path}")
+ umount(path)
+
+ if not self.preserve:
+ for disk in self._disks.values():
+ print(f"Detaching {disk}")
+ run("sudo", "losetup", "-d", disk)
+
+ if self._type == TmpType.TMPFS:
+ print(f"Unmounting tmpdir from {self.path}")
+ umount(self.path)
+
+ print(f"Removing {self.path}")
+ shutil.rmtree(self.path, ignore_errors=True)
+
+ def tmpfs(self, size="8G"):
+ """Mount a tmpfs"""
+ print(f"Mounting tmpfs on {self.path}")
+ mount("tmpfs", self.path, "tmpfs", f"size={size}")
+ self._type = TmpType.TMPFS
+
+ def add_sys(self, name, subdir=None):
+ """Create a subdirectory and register a sys"""
+ if subdir is None:
+ subdir = name
+ sys_path = os.path.join(self.path, name)
+ if not os.path.exists(sys_path):
+ os.mkdir(sys_path)
+ return sys_path
+
+ def add_disk(self, name, size="8G", filesystem="ext4"):
+ """Add a disk"""
+ disk_path = os.path.join(self.path, f"{name}.img")
+ self._disks[name] = create_disk(disk_path, "msdos", filesystem, size)
+ self._disk_filesystems[name] = filesystem
+ # Allow executing user to access it
+ run("sudo", "chown", getpass.getuser(), self._disks[name])
+
+ def mount_disk(self, name, mountpoint=None):
+ """Mount the disk"""
+ if mountpoint is None:
+ mountpoint = f"{name}_mnt"
+ mountpoint = os.path.join(self.path, mountpoint)
+ os.mkdir(mountpoint)
+ mount(self._disks[name] + "p1", mountpoint, self._disk_filesystems[name])
+ # Allow executing user to access it
+ run("sudo", "chown", getpass.getuser(), mountpoint)
+ self._mountpoints[name] = mountpoint
+ return mountpoint
+
+ def umount_disk(self, name):
+ """Unmount a disk"""
+ umount(self._mountpoints[name])
+ del self._mountpoints[name]
+
+ def get_disk(self, name):
+ """Get the path to a device of a disk"""
+ return self._disks[name]
diff --git a/lib/utils.py b/lib/utils.py
@@ -5,7 +5,7 @@ This file contains a few self-contained helper functions
# SPDX-License-Identifier: GPL-3.0-or-later
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
-# SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
+# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
import os
import shutil
diff --git a/rootfs.py b/rootfs.py
@@ -11,17 +11,16 @@ you can run bootstap inside chroot.
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
# SPDX-FileCopyrightText: 2021 Bastian Bittorf <bb@npl.de>
# SPDX-FileCopyrightText: 2021 Melg Eight <public.melg8@gmail.com>
-# SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
+# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
import argparse
import os
-import shutil
from sysa import SysA
-from sysb import SysB
from sysc import SysC
from lib.utils import run
from lib.sysgeneral import stage0_arch_map
+from lib.tmpdir import Tmpdir
def create_configuration_file(args):
"""
@@ -54,7 +53,12 @@ def main():
action="store_true")
parser.add_argument("-p", "--preserve", help="Do not remove temporary dir",
action="store_true")
- parser.add_argument("-t", "--tmpdir", help="Temporary directory")
+ parser.add_argument("-t", "--tmpdir", help="Temporary directory",
+ default="tmp")
+ parser.add_argument("--tmpfs", help="Use a tmpfs on tmpdir",
+ action="store_true")
+ parser.add_argument("--tmpfs-size", help="Size of the tmpfs",
+ default="8G")
parser.add_argument("--force-timestamps",
help="Force all files timestamps to be 0 unix time",
action="store_true")
@@ -82,13 +86,12 @@ def main():
parser.add_argument("-qk", "--kernel", help="Kernel to use (default is ./kernel)",
default="kernel")
- parser.add_argument("-m", "--minikernel", help="Use minikernel",
- action="store_true")
parser.add_argument("-b", "--bare-metal", help="Build images for bare metal",
action="store_true")
args = parser.parse_args()
+ # Mode validation
def check_types():
count = 0
if args.qemu:
@@ -97,24 +100,28 @@ def main():
count += 1
if args.bwrap:
count += 1
- if args.minikernel:
- count += 1
if args.bare_metal:
count += 1
return count
if check_types() > 1:
- raise ValueError("No more than one of qemu, chroot, bwrap, minikernel, bare metal "
+ raise ValueError("No more than one of qemu, chroot, bwrap, bare metal"
"may be used.")
if check_types() == 0:
- raise ValueError("One of qemu, chroot, bwrap, minikernel or bare metal must be selected.")
-
- if args.bare_metal:
- args.no_create_config = True
+ raise ValueError("One of qemu, chroot, bwrap, or bare metal must be selected.")
+ # Arch validation
if args.arch != "x86":
raise ValueError("Only x86 is supported at the moment.")
+ # Tmp validation
+ if args.bwrap and args.tmpfs:
+ raise ValueError("tmpfs cannot be used writh bwrap.")
+
+ # bootstrap.cfg
+ if args.bare_metal:
+ args.no_create_config = True
+
try:
os.remove(os.path.join('sysa', 'bootstrap.cfg'))
except FileNotFoundError:
@@ -125,20 +132,21 @@ def main():
with open(os.path.join('sysa', 'bootstrap.cfg'), 'a', encoding='UTF-8'):
pass
- system_c = SysC(arch=args.arch, preserve_tmp=args.preserve,
- tmpdir=args.tmpdir, external_sources=args.external_sources)
- system_b = SysB(arch=args.arch, preserve_tmp=args.preserve)
- system_a = SysA(arch=args.arch, preserve_tmp=args.preserve,
- early_preseed=args.early_preseed, tmpdir=args.tmpdir,
- external_sources=args.external_sources,
- sysb_dir=system_b.sys_dir, sysc_dir=system_c.sys_dir)
+ # tmpdir
+ tmpdir = Tmpdir(path=args.tmpdir, preserve=args.preserve)
+ if args.tmpfs:
+ tmpdir.tmpfs(size=args.tmpfs_size)
- if args.tmpdir is not None:
- os.makedirs(args.tmpdir, exist_ok=True)
+ # sys
+ system_c = SysC(arch=args.arch, tmpdir=tmpdir,
+ external_sources=args.external_sources)
+ system_a = SysA(arch=args.arch, early_preseed=args.early_preseed,
+ tmpdir=tmpdir, external_sources=args.external_sources,
+ repo_path=args.repo)
- bootstrap(args, system_a, system_b, system_c)
+ bootstrap(args, system_a, system_c, tmpdir)
-def bootstrap(args, system_a, system_b, system_c):
+def bootstrap(args, system_a, system_c, tmpdir):
"""Kick off bootstrap process."""
print(f"Bootstrapping {args.arch} -- SysA")
if args.chroot:
@@ -149,25 +157,17 @@ print(shutil.which('chroot'))
chroot_binary = run('sudo', 'python3', '-c', find_chroot,
capture_output=True).stdout.decode().strip()
- system_c.prepare(mount_tmpfs=True,
- create_disk_image=False)
- system_a.prepare(mount_tmpfs=True,
- create_initramfs=False,
- repo_path=args.repo)
+ system_c.prepare(create_disk_image=False)
+ system_a.prepare(create_initramfs=False)
- # sysa
arch = stage0_arch_map.get(args.arch, args.arch)
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
run('sudo', 'env', '-i', 'PATH=/bin', chroot_binary, system_a.tmp_dir, init)
elif args.bwrap:
- system_c.prepare(mount_tmpfs=False,
- create_disk_image=False)
- system_a.prepare(mount_tmpfs=False,
- create_initramfs=False,
- repo_path=args.repo)
+ system_c.prepare(create_disk_image=False)
+ system_a.prepare(create_initramfs=False)
- # sysa
arch = stage0_arch_map.get(args.arch, args.arch)
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
run('bwrap', '--unshare-user',
@@ -192,55 +192,23 @@ print(shutil.which('chroot'))
'--tmpfs', '/sysc_image/tmp',
init)
- elif args.minikernel:
- if os.path.isdir('kritis-linux'):
- shutil.rmtree('kritis-linux')
-
- system_c.prepare(mount_tmpfs=True,
- create_disk_image=True)
- system_a.prepare(mount_tmpfs=True,
- create_initramfs=True,
- repo_path=args.repo)
-
- run('git', 'clone',
- '--depth', '1', '--branch', 'v0.7',
- 'https://github.com/bittorf/kritis-linux.git')
- run('kritis-linux/ci_helper.sh',
- '--private',
- '--multi', '1',
- '--repeat', '1',
- '--arch', args.arch,
- '--qemucpu', '486',
- '--kernel', '3.18.140',
- '--features', 'kflock,highrestimers',
- # Hack to add -hda /dev/blah
- '--ramsize', str(args.qemu_ram) + 'M -hda ' + system_b.dev_name,
- '--initrd', system_a.initramfs_path,
- '--log', '/tmp/bootstrap.log')
-
elif args.bare_metal:
- system_c.prepare(mount_tmpfs=True,
- create_disk_image=True)
- system_a.prepare(mount_tmpfs=True,
- create_initramfs=True,
- repo_path=args.repo)
+ system_c.prepare(create_disk_image=True)
+ system_a.prepare(create_initramfs=True)
print("Please:")
- print(" 1. Take sysa/tmp/initramfs and your kernel, boot using this.")
- print(" 2. Take sysc/tmp/disk.img and put this on a writable storage medium.")
+ print(" 1. Take tmp/sysa/initramfs and your kernel, boot using this.")
+ print(" 2. Take tmp/sysc/disk.img and put this on a writable storage medium.")
else:
- system_c.prepare(mount_tmpfs=True,
- create_disk_image=True)
- system_a.prepare(mount_tmpfs=True,
- create_initramfs=True,
- repo_path=args.repo)
+ system_c.prepare(create_disk_image=True)
+ system_a.prepare(create_initramfs=True)
run(args.qemu_cmd,
'-enable-kvm',
'-m', str(args.qemu_ram) + 'M',
'-no-reboot',
- '-hda', system_c.dev_name,
+ '-hda', tmpdir.get_disk("sysc"),
'-nic', 'user,ipv6=off,model=e1000',
'-kernel', args.kernel,
'-initrd', system_a.initramfs_path,
diff --git a/sysa.py b/sysa.py
@@ -4,7 +4,7 @@
# SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
# SPDX-FileCopyrightText: 2021 Melg Eight <public.melg8@gmail.com>
-# SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
+# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
import os
from distutils.dir_util import copy_tree
@@ -22,37 +22,26 @@ class SysA(SysGeneral):
git_dir = os.path.dirname(os.path.join(__file__))
sys_dir = os.path.join(git_dir, 'sysa')
+ sysb_dir = os.path.join(git_dir, 'sysb')
+ sysc_dir = os.path.join(git_dir, 'sysc')
cache_dir = os.path.join(sys_dir, 'distfiles')
# pylint: disable=too-many-arguments
- def __init__(self, arch, preserve_tmp, external_sources,
- early_preseed, tmpdir, sysb_dir, sysc_dir):
+ def __init__(self, tmpdir, arch, external_sources,
+ early_preseed, repo_path):
self.arch = arch
- self.preserve_tmp = preserve_tmp
self.early_preseed = early_preseed
-
- if tmpdir is None:
- self.tmp_dir = os.path.join(self.git_dir, 'tmp')
- else:
- self.tmp_dir = os.path.join(tmpdir, 'sysa')
- self.sysa_dir = os.path.join(self.tmp_dir, 'sysa')
- self.base_dir = self.sysa_dir
-
- self.sysb_dir = sysb_dir
- self.sysc_dir = sysc_dir
self.external_sources = external_sources
+ self.repo_path = repo_path
- def prepare(self, mount_tmpfs, create_initramfs, repo_path):
+ self.tmp_dir = tmpdir.add_sys("sysa")
+
+ def prepare(self, create_initramfs):
"""
Prepare directory structure for System A.
We create an empty tmp directory, unpack stage0-posix.
Rest of the files are unpacked into more structured directory /sysa
"""
- if mount_tmpfs:
- self.mount_tmpfs()
- else:
- os.mkdir(self.tmp_dir)
-
if self.early_preseed:
# Extract tar containing preseed
with tarfile.open(self.early_preseed, "r") as seed:
@@ -69,9 +58,9 @@ class SysA(SysGeneral):
self.sysc(create_initramfs)
- if repo_path:
+ if self.repo_path:
repo_dir = os.path.join(self.tmp_dir, 'usr', 'src', 'repo-preseeded')
- shutil.copytree(repo_path, repo_dir)
+ shutil.copytree(self.repo_path, repo_dir)
if create_initramfs:
self.make_initramfs()
diff --git a/sysb.py b/sysb.py
@@ -1,21 +0,0 @@
-#!/usr/bin/env python3
-"""System B"""
-# SPDX-License-Identifier: GPL-3.0-or-later
-# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
-# SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
-
-import os
-
-from lib.sysgeneral import SysGeneral
-
-class SysB(SysGeneral):
- """
- Class responsible for preparing sources for System B.
- """
-
- git_dir = os.path.dirname(os.path.join(__file__))
- sys_dir = os.path.join(git_dir, 'sysb')
-
- def __init__(self, arch, preserve_tmp):
- self.arch = arch
- self.preserve_tmp = preserve_tmp
diff --git a/sysc.py b/sysc.py
@@ -2,13 +2,12 @@
"""System C"""
# SPDX-License-Identifier: GPL-3.0-or-later
# SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
-# SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
+# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
import os
-import getpass
-from lib.utils import mount, umount, create_disk, run, copytree
+from lib.utils import copytree
from lib.sysgeneral import SysGeneral
# pylint: disable=consider-using-with
@@ -21,60 +20,30 @@ class SysC(SysGeneral):
git_dir = os.path.dirname(os.path.join(__file__))
sys_dir = os.path.join(git_dir, 'sysc')
cache_dir = os.path.join(sys_dir, 'distfiles')
- dev_name = None
- def __init__(self, arch, preserve_tmp, tmpdir, external_sources):
+ def __init__(self, tmpdir, arch, external_sources):
self.arch = arch
- self.preserve_tmp = preserve_tmp
self.external_sources = external_sources
+ self._tmpdir = tmpdir
- if tmpdir is None:
- self.tmp_dir = os.path.join(self.sys_dir, 'tmp')
- else:
- self.tmp_dir = os.path.join(tmpdir, 'sysc')
+ self.tmp_dir = tmpdir.add_sys("sysc")
- def __del__(self):
- if not self.preserve_tmp:
- if self.dev_name is not None:
- print(f"Detaching {self.dev_name}")
- run('sudo', 'losetup', '-d', self.dev_name)
-
- super().__del__()
-
- def prepare(self, mount_tmpfs, create_disk_image):
+ def prepare(self, create_disk_image):
"""
Prepare directory structure for System C.
"""
- if mount_tmpfs:
- self.mount_tmpfs()
- else:
- os.mkdir(self.tmp_dir)
-
- rootfs_dir = None
-
if create_disk_image:
- # Create + mount a disk for QEMU to use
- disk_path = os.path.join(self.tmp_dir, 'disk.img')
- if self.external_sources:
- self.dev_name = create_disk(disk_path, "msdos", "ext4", '8G')
- rootfs_dir = os.path.join(self.tmp_dir, 'mnt')
- os.mkdir(rootfs_dir)
- mount(self.dev_name + "p1", rootfs_dir, 'ext4')
- else:
- self.dev_name = create_disk(disk_path, "none", "ext4", '8G')
- # Use chown to allow executing user to access it
- run('sudo', 'chown', getpass.getuser(), self.dev_name)
- if self.external_sources:
- run('sudo', 'chown', getpass.getuser(), rootfs_dir)
- else:
- rootfs_dir = self.tmp_dir
+ self._tmpdir.add_disk("sysc")
if self.external_sources:
+ if create_disk_image:
+ rootfs_dir = self._tmpdir.mount_disk("sysc")
+ else:
+ rootfs_dir = self.tmp_dir
source_manifest = self.get_source_manifest()
self.get_packages(source_manifest)
copytree(self.cache_dir, os.path.join(rootfs_dir, "distfiles"))
- # Unmount tmp/mnt if it was mounted
- if create_disk_image and self.external_sources:
- umount(rootfs_dir)
+ if create_disk_image:
+ self._tmpdir.umount_disk("sysc")
