commit: 05c13dd64ede6a8f1f83d7a16b304d07e1379b51
parent 6ed2e09f3a533a1efdd807a77a7c00a07bf984f1
Author: fosslinux <fosslinux@aussies.space>
Date: Wed, 8 Nov 2023 11:30:20 +1100
Modify rootfs.py infrastructure to support the new layout
Diffstat:
8 files changed, 374 insertions(+), 410 deletions(-)
diff --git a/lib/generator.py b/lib/generator.py
@@ -0,0 +1,336 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
+# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
+# SPDX-FileCopyrightText: 2021 Melg Eight <public.melg8@gmail.com>
+# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
+
+import glob
+import hashlib
+import os
+import shutil
+import subprocess
+import tarfile
+import requests
+
+class Generator():
+ """
+ Class responsible for generating the basic media to be consumed.
+ """
+
+ git_dir = os.path.join(os.path.dirname(os.path.join(__file__)), '..')
+ distfiles_dir = os.path.join(git_dir, 'distfiles')
+
+ # pylint: disable=too-many-arguments
+ def __init__(self, tmpdir, arch, external_sources,
+ early_preseed, repo_path):
+ self.arch = arch
+ self.early_preseed = early_preseed
+ self.external_sources = external_sources
+ self.repo_path = repo_path
+ self.tmpdir = tmpdir
+ self.tmp_dir = tmpdir.path
+
+ def prepare(self, using_kernel=False, kernel_bootstrap=False):
+ """
+ Prepare basic media of live-bootstrap.
+ /steps -- contains steps to be built
+ / -- contains seed to allow steps to be built, containing custom
+ scripts and stage0-posix
+ """
+ self.external_dir = os.path.join(self.tmp_dir, 'external')
+ # We use ext3 here; ext4 actually has a variety of extensions that
+ # have been added with varying levels of recency
+ # Linux 4.9.10 does not support a bunch of them
+ # Attempting to disable extensions that a particular e2fsprogs
+ # is *unaware* of causes the filesystem creation to fail
+ # We could hypothetically detect e2fsprogs version and create an
+ # argument matrix ... or we could just use ext3 instead which
+ # is effectively universally the same
+ if kernel_bootstrap:
+ init_path = os.path.join(self.tmp_dir, 'init')
+
+ os.mkdir(init_path)
+ self.tmp_dir = init_path
+
+ if self.repo_path or self.external_sources:
+ self.tmpdir.add_disk("external", filesystem="ext3")
+ self.tmpdir.mount_disk("external", "external")
+ else:
+ self.tmpdir.add_disk("external", tabletype="none")
+ elif using_kernel:
+ self.tmp_dir = os.path.join(self.tmp_dir, 'disk')
+ self.tmpdir.add_disk("disk", filesystem="ext3")
+ self.tmpdir.mount_disk("disk", "disk")
+ self.external_dir = os.path.join(self.tmp_dir, 'external')
+
+ os.makedirs(self.external_dir, exist_ok=True)
+
+ if self.early_preseed:
+ # Extract tar containing preseed
+ with tarfile.open(self.early_preseed, "r") as seed:
+ seed.extractall(self.tmp_dir)
+ shutil.copy2(os.path.join(self.git_dir, 'seed', 'preseeded.kaem'),
+ os.path.join(self.tmp_dir, 'kaem.x86'))
+ else:
+ self.stage0_posix()
+ self.seed()
+
+ self.steps()
+
+ self.distfiles()
+
+ self.create_fiwix_file_list()
+
+ if self.repo_path:
+ repo_dir = os.path.join(self.external_dir, 'repo-preseeded')
+ shutil.copytree(self.repo_path, repo_dir)
+
+ if kernel_bootstrap:
+ self.create_builder_hex0_disk_image(os.path.join(self.tmp_dir, 'disk.img'))
+
+ if kernel_bootstrap and (self.external_sources or self.repo_path):
+ self.tmpdir.umount_disk('external')
+ elif using_kernel:
+ self.tmpdir.umount_disk('disk')
+
+ def steps(self):
+ """Copy in steps."""
+ source_manifest = self.get_source_manifest()
+ self.get_packages(source_manifest)
+
+ shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.tmp_dir, 'steps'))
+
+ def stage0_posix(self):
+ """Copy in all of the stage0-posix"""
+ stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix')
+ for f in os.listdir(stage0_posix_base_dir):
+ orig = os.path.join(stage0_posix_base_dir, f)
+ to = os.path.join(self.tmp_dir, f)
+ if os.path.isfile(orig):
+ shutil.copy2(orig, to)
+ else:
+ shutil.copytree(orig, to)
+
+ arch = stage0_arch_map.get(self.arch, self.arch)
+ kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix', 'bootstrap-seeds',
+ 'POSIX', arch, 'kaem-optional-seed')
+ shutil.copy2(kaem_optional_seed, os.path.join(self.tmp_dir, 'init'))
+
+ def seed(self):
+ """Copy in extra seed files"""
+ seed_dir = os.path.join(self.git_dir, 'seed')
+ for f in os.listdir(seed_dir):
+ if os.path.isfile(os.path.join(seed_dir, f)):
+ shutil.copy2(os.path.join(seed_dir, f), os.path.join(self.tmp_dir, f))
+
+ def add_fiwix_files(self, file_list_path, dirpath):
+ """Add files to the list to populate Fiwix file system"""
+ for root, _, filepaths in os.walk(dirpath):
+ if 'stage0-posix' in root:
+ continue
+ with open(file_list_path, 'a', encoding="utf-8") as file_list:
+ for filepath in filepaths:
+ file_list.write(f"/{os.path.join(root, filepath)}\n")
+
+ def create_fiwix_file_list(self):
+ """Create a list of files to populate Fiwix file system"""
+ file_list_path = os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1',
+ 'files', 'fiwix-file-list.txt')
+ shutil.copyfile(os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1',
+ 'files', 'early-artifacts-needed-after-fiwix.txt'),
+ file_list_path)
+
+ save_cwd = os.getcwd()
+ os.chdir(self.tmp_dir)
+ self.add_fiwix_files(file_list_path, 'steps')
+ self.add_fiwix_files(file_list_path, 'distfiles')
+ os.chdir(save_cwd)
+
+ def distfiles(self):
+ """Copy in distfiles"""
+ def copy_no_network_distfiles(out):
+ # Note that no network == no disk for kernel bootstrap mode
+ with open(os.path.join(self.git_dir, 'steps', 'pre-network-sources'), 'r') as source_list:
+ for file in source_list.readlines():
+ file = file.strip()
+ shutil.copy2(os.path.join(self.distfiles_dir, file),
+ os.path.join(out, file))
+
+ early_distfile_dir = os.path.join(self.tmp_dir, 'external', 'distfiles')
+ main_distfile_dir = os.path.join(self.external_dir, 'distfiles')
+
+ if early_distfile_dir != main_distfile_dir:
+ os.makedirs(early_distfile_dir)
+ copy_no_network_distfiles(early_distfile_dir)
+
+ if self.external_sources:
+ os.mkdir(main_distfile_dir)
+ shutil.copytree(self.distfiles_dir, main_distfile_dir)
+ else:
+ os.mkdir(main_distfile_dir)
+ copy_no_network_distfiles(main_distfile_dir)
+
+ @staticmethod
+ def output_dir(srcfs_file, dirpath):
+ """Add a directory to srcfs file system"""
+ srcline = f"src 0 {dirpath}\n"
+ srcfs_file.write(srcline.encode())
+
+ @staticmethod
+ def output_file(srcfs_file, filepath):
+ """Add a file to srcfs file system"""
+ srcline = f"src {os.path.getsize(filepath)} {filepath}\n"
+ srcfs_file.write(srcline.encode())
+ with open(filepath, 'rb') as srcfile:
+ srcfs_file.write(srcfile.read())
+
+ def output_tree(self, srcfs_file, treepath):
+ """Add a tree of files to srcfs file system"""
+ self.output_dir(srcfs_file, treepath)
+ for root, dirs, files in os.walk(treepath):
+ if ".git" in root:
+ continue
+ for dirpath in dirs:
+ if ".git" in dirpath:
+ continue
+ self.output_dir(srcfs_file, os.path.join(root, dirpath))
+
+ for filepath in files:
+ if ".git" in filepath:
+ continue
+ self.output_file(srcfs_file, os.path.join(root, filepath))
+
+ def append_srcfs(self, image_file):
+ """Append srcfs file system to disk image"""
+ save_cwd = os.getcwd()
+
+ os.chdir(self.tmp_dir)
+ self.output_tree(image_file, '.')
+
+ # Add commands to kick off stage0-posix
+ cmd = ' '.join(['hex0',
+ './bootstrap-seeds/POSIX/x86/hex0_x86.hex0',
+ './bootstrap-seeds/POSIX/x86/hex0-seed\n'])
+ image_file.write(cmd.encode())
+ cmd = ' '.join(['hex0',
+ './bootstrap-seeds/POSIX/x86/kaem-minimal.hex0',
+ './bootstrap-seeds/POSIX/x86/kaem-optional-seed\n'])
+ image_file.write(cmd.encode())
+ cmd = ' '.join(['./bootstrap-seeds/POSIX/x86/kaem-optional-seed', './kaem.x86\n'])
+ image_file.write(cmd.encode())
+
+ os.chdir(save_cwd)
+
+ def create_builder_hex0_disk_image(self, image_file_name):
+ """Create builder-hex0 disk image"""
+ shutil.copyfile(os.path.join('seed', 'stage0-posix', 'bootstrap-seeds',
+ 'NATIVE', 'x86', 'builder-hex0-x86-stage1.img'),
+ image_file_name)
+
+ with open(image_file_name, 'ab') as image_file:
+ # Append stage2 hex0 source
+ with open(os.path.join('kernel-bootstrap', 'builder-hex0-x86-stage2.hex0'),
+ encoding="utf-8") as infile:
+ image_file.write(infile.read().encode())
+ # Pad to next sector
+ current_size = os.stat(image_file_name).st_size
+ while current_size % 512 != 0:
+ image_file.write(b'\0')
+ current_size += 1
+ self.append_srcfs(image_file)
+
+ current_size = os.stat(image_file_name).st_size
+
+ megabyte = 1024 * 1024
+ # fill file with zeros up to next megabyte
+ extra = current_size % megabyte
+ round_up = megabyte - extra
+ with open(image_file_name, 'ab') as image_file:
+ image_file.write(b'\0' * round_up)
+ current_size += round_up
+
+ # fill file with zeros up to desired size, one megabyte at a time
+ with open(image_file_name, 'ab') as image_file:
+ while current_size < 16384 * megabyte:
+ image_file.write(b'\0' * megabyte)
+ current_size += megabyte
+
+ def check_file(self, file_name, expected_hash):
+ """Check hash of downloaded source file."""
+ with open(file_name, "rb") as downloaded_file:
+ downloaded_content = downloaded_file.read() # read entire file as bytes
+ readable_hash = hashlib.sha256(downloaded_content).hexdigest()
+ if expected_hash == readable_hash:
+ return
+ raise ValueError(f"Checksum mismatch for file {os.path.basename(file_name)}:\n\
+expected: {expected_hash}\n\
+actual: {readable_hash}\n\
+When in doubt, try deleting the file in question -- it will be downloaded again when running \
+this script the next time")
+
+ def download_file(self, url, directory, file_name):
+ """
+ Download a single source archive.
+ """
+ abs_file_name = os.path.join(directory, file_name)
+
+ # Create a directory for downloaded file
+ if not os.path.isdir(directory):
+ os.mkdir(directory)
+
+ # Actually download the file
+ headers = {
+ "Accept-Encoding": "identity"
+ }
+ if not os.path.isfile(abs_file_name):
+ print(f"Downloading: {file_name}")
+ response = requests.get(url, allow_redirects=True, stream=True,
+ headers=headers, timeout=20)
+ if response.status_code == 200:
+ with open(abs_file_name, 'wb') as target_file:
+ target_file.write(response.raw.read())
+ else:
+ raise requests.HTTPError("Download failed.")
+ return abs_file_name
+
+ def get_packages(self, source_manifest):
+ """Prepare remaining sources"""
+ for line in source_manifest.split("\n"):
+ line = line.strip().split(" ")
+
+ path = self.download_file(line[2], line[1], line[3])
+ self.check_file(path, line[0])
+
+ @classmethod
+ def get_source_manifest(cls):
+ """
+ Generate a source manifest for the system.
+ """
+ manifest_lines = []
+ directory = os.path.relpath(cls.distfiles_dir, cls.git_dir)
+
+ # Find all source files
+ steps_dir = os.path.join(cls.git_dir, 'steps')
+ for file in os.listdir(steps_dir):
+ if os.path.isdir(os.path.join(steps_dir, file)):
+ sourcef = os.path.join(steps_dir, file, "sources")
+ if os.path.exists(sourcef):
+ # Read sources from the source file
+ with open(sourcef, "r", encoding="utf_8") as sources:
+ for line in sources.readlines():
+ line = line.strip().split(" ")
+
+ if len(line) > 2:
+ file_name = line[2]
+ else:
+ # Automatically determine file name based on URL.
+ file_name = os.path.basename(line[0])
+
+ manifest_lines.append(f"{line[1]} {directory} {line[0]} {file_name}")
+
+ return "\n".join(manifest_lines)
+
+stage0_arch_map = {
+ "amd64": "AMD64",
+}
diff --git a/lib/sysgeneral.py b/lib/sysgeneral.py
@@ -1,132 +0,0 @@
-#!/usr/bin/env python3
-"""
-This file contains a few functions to be shared by all Sys* classes
-"""
-
-# SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
-# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
-# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-import os
-import hashlib
-import glob
-import subprocess
-
-import requests
-
-class SysGeneral:
- """
- A class from which all Sys* class are extended.
- Contains functions used in all Sys*
- """
-
- # All of these are variables defined in the individual Sys* classes
- cache_dir = None
- base_dir = None
- git_dir = None
- sys_dir = None
- initramfs_path = None
- tmp_dir = None
-
- def check_file(self, file_name, expected_hash):
- """Check hash of downloaded source file."""
- with open(file_name, "rb") as downloaded_file:
- downloaded_content = downloaded_file.read() # read entire file as bytes
- readable_hash = hashlib.sha256(downloaded_content).hexdigest()
- if expected_hash == readable_hash:
- return
- raise ValueError(f"Checksum mismatch for file {os.path.basename(file_name)}:\n\
-expected: {expected_hash}\n\
-actual: {readable_hash}\n\
-When in doubt, try deleting the file in question -- it will be downloaded again when running \
-this script the next time")
-
- def download_file(self, url, directory, file_name):
- """
- Download a single source archive.
- """
- abs_file_name = os.path.join(directory, file_name)
-
- # Create a directory for downloaded file
- if not os.path.isdir(directory):
- os.mkdir(directory)
-
- # Actually download the file
- headers = {
- "Accept-Encoding": "identity"
- }
- if not os.path.isfile(abs_file_name):
- print(f"Downloading: {file_name}")
- response = requests.get(url, allow_redirects=True, stream=True,
- headers=headers, timeout=20)
- if response.status_code == 200:
- with open(abs_file_name, 'wb') as target_file:
- target_file.write(response.raw.read())
- else:
- raise requests.HTTPError("Download failed.")
- return abs_file_name
-
- def get_packages(self, source_manifest):
- """Prepare remaining sources"""
- for line in source_manifest.split("\n"):
- line = line.strip().split(" ")
-
- path = self.download_file(line[2], line[1], line[3])
- self.check_file(path, line[0])
-
- @classmethod
- def get_source_manifest(cls):
- """
- Generate a source manifest for the system.
- """
- manifest_lines = []
- directory = os.path.relpath(cls.cache_dir, cls.git_dir)
-
- # Find all source files
- for file in os.listdir(cls.sys_dir):
- if os.path.isdir(os.path.join(cls.sys_dir, file)):
- sourcef = os.path.join(cls.sys_dir, file, "sources")
- if os.path.exists(sourcef):
- # Read sources from the source file
- with open(sourcef, "r", encoding="utf_8") as sources:
- for line in sources.readlines():
- line = line.strip().split(" ")
-
- if len(line) > 2:
- file_name = line[2]
- else:
- # Automatically determine file name based on URL.
- file_name = os.path.basename(line[0])
-
- manifest_lines.append(f"{line[1]} {directory} {line[0]} {file_name}")
-
- return "\n".join(manifest_lines)
-
- def make_initramfs(self):
- """Package binary bootstrap seeds and sources into initramfs."""
- self.initramfs_path = os.path.join(self.tmp_dir, 'initramfs')
-
- # Create a list of files to go within the initramfs
- file_list = glob.glob(os.path.join(self.tmp_dir, '**'), recursive=True)
-
- # Use built-in removeprefix once we can use Python 3.9
- def remove_prefix(text, prefix):
- if text.startswith(prefix):
- return text[len(prefix):]
- return text # or whatever
-
- file_list = [remove_prefix(f, self.tmp_dir + os.sep) for f in file_list]
-
- # Create the initramfs
- with open(self.initramfs_path, "w", encoding="utf_8") as initramfs:
- # pylint: disable=consider-using-with
- cpio = subprocess.Popen(
- ["cpio", "--format", "newc", "--create",
- "--directory", self.tmp_dir],
- stdin=subprocess.PIPE, stdout=initramfs)
- cpio.communicate(input='\n'.join(file_list).encode())
-
-stage0_arch_map = {
- "amd64": "AMD64",
-}
diff --git a/lib/tmpdir.py b/lib/tmpdir.py
@@ -24,7 +24,6 @@ class Tmpdir:
Represents a tmpdir
"""
- _syses = {}
_disks = {}
_disk_filesystems = {}
_mountpoints = {}
@@ -60,19 +59,10 @@ class Tmpdir:
mount("tmpfs", self.path, "tmpfs", f"size={size}")
self._type = TmpType.TMPFS
- def add_sys(self, name, subdir=None):
- """Create a subdirectory and register a sys"""
- if subdir is None:
- subdir = name
- sys_path = os.path.join(self.path, name)
- if not os.path.exists(sys_path):
- os.mkdir(sys_path)
- return sys_path
-
- def add_disk(self, name, size="16G", filesystem="ext4"):
+ def add_disk(self, name, size="16G", filesystem="ext4", tabletype="msdos", mkfs_args=[]):
"""Add a disk"""
disk_path = os.path.join(self.path, f"{name}.img")
- self._disks[name] = create_disk(disk_path, "msdos", filesystem, size)
+ self._disks[name] = create_disk(disk_path, tabletype, filesystem, size, mkfs_args=mkfs_args)
self._disk_filesystems[name] = filesystem
# Allow executing user to access it
run_as_root("chown", getpass.getuser(), self._disks[name])
diff --git a/lib/utils.py b/lib/utils.py
@@ -31,7 +31,7 @@ def run_as_root(*args, **kwargs):
return run("sudo", *args, **kwargs)
return run(*args, **kwargs)
-def create_disk(image, disk_type, fs_type, size):
+def create_disk(image, disk_type, fs_type, size, mkfs_args=[]):
"""Create a disk image, with a filesystem on it"""
run('truncate', '-s', size, image)
# First find the device we will use, then actually use it
@@ -40,9 +40,9 @@ def create_disk(image, disk_type, fs_type, size):
# Create the partition
if disk_type != "none":
run_as_root('parted', '--script', image, 'mklabel', disk_type, 'mkpart',
- 'primary', 'ext4', '0%', '100%')
+ 'primary', fs_type, '0%', '100%')
run_as_root('partprobe', loop_dev)
- run_as_root('mkfs.' + fs_type, loop_dev + "p1")
+ run_as_root('mkfs.' + fs_type, loop_dev + "p1", *mkfs_args)
return loop_dev
def mount(source, target, fs_type, options='', **kwargs):
diff --git a/rootfs.py b/rootfs.py
@@ -17,18 +17,16 @@ import argparse
import os
import shutil
-from sysa import SysA
-from sysc import SysC
from lib.utils import run, run_as_root
-from lib.sysgeneral import stage0_arch_map
from lib.tmpdir import Tmpdir
+from lib.generator import Generator, stage0_arch_map
def create_configuration_file(args):
"""
Creates bootstrap.cfg file which would contain options used to
customize bootstrap.
"""
- config_path = os.path.join('sysa', 'bootstrap.cfg')
+ config_path = os.path.join('steps', 'bootstrap.cfg')
with open(config_path, "w", encoding="utf_8") as config:
config.write(f"FORCE_TIMESTAMPS={args.force_timestamps}\n")
config.write(f"CHROOT={args.chroot or args.bwrap}\n")
@@ -38,7 +36,10 @@ def create_configuration_file(args):
config.write(f"INTERNAL_CI={args.internal_ci}\n")
config.write(f"BARE_METAL={args.bare_metal}\n")
if (args.bare_metal or args.qemu) and not args.kernel:
- config.write("DISK=sda\n")
+ if args.repo or args.external_sources:
+ config.write("DISK=sdb1\n")
+ else:
+ config.write("DISK=sdb\n")
config.write("KERNEL_BOOTSTRAP=True\n")
else:
config.write("DISK=sda1\n")
@@ -49,7 +50,7 @@ def create_configuration_file(args):
def main():
"""
A few command line arguments to customize bootstrap.
- This function also creates SysA object which prepares directory
+ This function also creates object which prepares directory
structure with bootstrap seeds and all sources.
"""
parser = argparse.ArgumentParser()
@@ -151,16 +152,15 @@ def main():
if args.tmpfs:
tmpdir.tmpfs(size=args.tmpfs_size)
- # sys
- system_c = SysC(arch=args.arch, tmpdir=tmpdir,
- external_sources=args.external_sources)
- system_a = SysA(arch=args.arch, early_preseed=args.early_preseed,
- tmpdir=tmpdir, external_sources=args.external_sources,
- repo_path=args.repo)
+ generator = Generator(tmpdir=tmpdir,
+ arch=args.arch,
+ external_sources=args.external_sources,
+ repo_path=args.repo,
+ early_preseed=args.early_preseed)
- bootstrap(args, system_a, system_c, tmpdir)
+ bootstrap(args, generator, tmpdir)
-def bootstrap(args, system_a, system_c, tmpdir):
+def bootstrap(args, generator, tmpdir):
"""Kick off bootstrap process."""
print(f"Bootstrapping {args.arch} -- SysA")
if args.chroot:
@@ -171,17 +171,15 @@ print(shutil.which('chroot'))
chroot_binary = run_as_root('python3', '-c', find_chroot,
capture_output=True).stdout.decode().strip()
- system_c.prepare(create_disk_image=False)
- system_a.prepare(create_initramfs=False)
+ generator.prepare(using_kernel=False)
arch = stage0_arch_map.get(args.arch, args.arch)
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
- run_as_root('env', '-i', 'PATH=/bin', chroot_binary, system_a.tmp_dir, init)
+ run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.tmp_dir, init)
elif args.bwrap:
if not args.internal_ci or args.internal_ci == "pass1":
- system_c.prepare(create_disk_image=False)
- system_a.prepare(create_initramfs=False)
+ generator.prepare(using_kernel=False)
arch = stage0_arch_map.get(args.arch, args.arch)
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
@@ -191,7 +189,7 @@ print(shutil.which('chroot'))
'--unshare-net',
'--clearenv',
'--setenv', 'PATH', '/usr/bin',
- '--bind', system_a.tmp_dir, '/',
+ '--bind', generator.tmp_dir, '/',
'--dir', '/dev',
'--dev-bind', '/dev/null', '/dev/null',
'--dev-bind', '/dev/zero', '/dev/zero',
@@ -210,7 +208,7 @@ print(shutil.which('chroot'))
'--unshare-net' if args.external_sources else None,
'--clearenv',
'--setenv', 'PATH', '/usr/bin',
- '--bind', system_a.tmp_dir + "/sysc_image", '/',
+ '--bind', generator.tmp_dir + "/sysc_image", '/',
'--dir', '/dev',
'--dev-bind', '/dev/null', '/dev/null',
'--dev-bind', '/dev/zero', '/dev/zero',
@@ -226,40 +224,39 @@ print(shutil.which('chroot'))
elif args.bare_metal:
if args.kernel:
- system_c.prepare(create_disk_image=True)
- system_a.prepare(create_initramfs=True)
+ generator.prepare(using_kernel=True)
print("Please:")
- print(" 1. Take tmp/sysa/initramfs and your kernel, boot using this.")
- print(" 2. Take tmp/sysc/disk.img and put this on a writable storage medium.")
+ print(" 1. Take tmp/initramfs and your kernel, boot using this.")
+ print(" 2. Take tmp/disk.img and put this on a writable storage medium.")
else:
- system_a.prepare(create_initramfs=True, kernel_bootstrap=True)
+ generator.prepare(kernel_bootstrap=True)
print("Please:")
- print(" 1. Take tmp/sysa/sysa.img and write it to a boot drive and then boot it.")
+ print(" 1. Take tmp/disk.img and write it to a boot drive and then boot it.")
else:
if args.kernel:
- system_c.prepare(create_disk_image=True)
- system_a.prepare(create_initramfs=True)
+ generator.prepare(using_kernel=True)
run(args.qemu_cmd,
'-enable-kvm',
'-m', str(args.qemu_ram) + 'M',
'-smp', str(args.cores),
'-no-reboot',
- '-hda', tmpdir.get_disk("sysc"),
+ '-drive', 'file=' + tmpdir.get_disk("disk") + ',format=raw',
+ '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
'-nic', 'user,ipv6=off,model=e1000',
'-kernel', args.kernel,
- '-initrd', system_a.initramfs_path,
'-nographic',
- '-append', 'console=ttyS0')
+ '-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw')
else:
- system_a.prepare(create_initramfs=True, kernel_bootstrap=True)
+ generator.prepare(kernel_bootstrap=True)
run(args.qemu_cmd,
'-enable-kvm',
'-m', "4G",
'-smp', str(args.cores),
'-no-reboot',
- '-drive', 'file=' + os.path.join(system_a.tmp_dir, 'sysa.img') + ',format=raw',
+ '-drive', 'file=' + os.path.join(generator.tmp_dir, 'disk.img') + ',format=raw',
+ '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
'-machine', 'kernel-irqchip=split',
'-nic', 'user,ipv6=off,model=e1000',
'-nographic')
diff --git a/steps/env b/steps/env
@@ -1,4 +1,4 @@
-DISTFILES=/distfiles
+DISTFILES=/external/distfiles
PREFIX=/usr
BINDIR=${PREFIX}/bin
LIBDIR=${PREFIX}/lib/mes
diff --git a/steps/lwext4-1.0.0-lb1/files/make_fiwix_initrd.c b/steps/lwext4-1.0.0-lb1/files/make_fiwix_initrd.c
@@ -202,7 +202,7 @@ int main(int argc, char **argv)
char zeros[BLOCK_SIZE];
unsigned int next_file_address;
-
+
next_file_address = *((unsigned int *) 0x7F8D);
printf("Starting fiwix.ext2 at addr 0x%08x\n", next_file_address);
diff --git a/sysa.py b/sysa.py
@@ -1,227 +0,0 @@
-#!/usr/bin/env python3
-"""System A"""
-# SPDX-License-Identifier: GPL-3.0-or-later
-# SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
-# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
-# SPDX-FileCopyrightText: 2021 Melg Eight <public.melg8@gmail.com>
-# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
-
-import os
-# pylint: disable=deprecated-module
-from distutils.dir_util import copy_tree
-import shutil
-import tarfile
-
-from lib.sysgeneral import SysGeneral, stage0_arch_map
-
-# pylint: disable=consider-using-with
-# pylint: disable=too-many-instance-attributes
-class SysA(SysGeneral):
- """
- Class responsible for preparing sources for System A.
- """
-
- git_dir = os.path.dirname(os.path.join(__file__))
- sys_dir = os.path.join(git_dir, 'sysa')
- sysb_dir = os.path.join(git_dir, 'sysb')
- sysc_dir = os.path.join(git_dir, 'sysc')
- cache_dir = os.path.join(sys_dir, 'distfiles')
-
- # pylint: disable=too-many-arguments
- def __init__(self, tmpdir, arch, external_sources,
- early_preseed, repo_path):
- self.arch = arch
- self.early_preseed = early_preseed
- self.external_sources = external_sources
- self.repo_path = repo_path
-
- self.tmp_dir = tmpdir.add_sys("sysa")
-
- def prepare(self, create_initramfs, kernel_bootstrap=False):
- """
- Prepare directory structure for System A.
- We create an empty tmp directory, unpack stage0-posix.
- Rest of the files are unpacked into more structured directory /sysa
- """
- if self.early_preseed:
- # Extract tar containing preseed
- with tarfile.open(self.early_preseed, "r") as seed:
- seed.extractall(self.tmp_dir)
- shutil.copy2(os.path.join(self.sys_dir, 'base-preseeded.kaem'),
- os.path.join(self.tmp_dir, 'kaem.x86'))
- else:
- self.stage0_posix()
-
- self.sysa()
-
- # sysb must be added to sysa as it is another initramfs stage
- self.sysb()
-
- self.sysc(create_initramfs)
-
- if kernel_bootstrap:
- self.create_fiwix_file_list()
- self.create_builder_hex0_disk_image(os.path.join(self.tmp_dir, 'sysa.img'))
- return
-
- if self.repo_path:
- repo_dir = os.path.join(self.tmp_dir, 'usr', 'src', 'repo-preseeded')
- shutil.copytree(self.repo_path, repo_dir)
-
- if create_initramfs:
- self.make_initramfs()
-
- def sysa(self):
- """Copy in sysa files for sysa."""
- source_manifest = self.get_source_manifest()
- self.get_packages(source_manifest)
-
- shutil.copytree(self.sys_dir, os.path.join(self.tmp_dir, 'sysa'),
- ignore=shutil.ignore_patterns('tmp'))
-
- def sysb(self):
- """Copy in sysb files for sysb."""
- shutil.copytree(self.sysb_dir, os.path.join(self.tmp_dir, 'sysb'),
- ignore=shutil.ignore_patterns('tmp'))
-
- def sysc(self, create_initramfs):
- """Copy in sysc files for sysc."""
- if create_initramfs or not self.external_sources:
- ignore = shutil.ignore_patterns('tmp', 'distfiles')
- else:
- ignore = shutil.ignore_patterns('tmp')
- shutil.copytree(self.sysc_dir, os.path.join(self.tmp_dir, 'sysc'),
- ignore=ignore)
-
- def stage0_posix(self):
- """Copy in all of the stage0-posix"""
- stage0_posix_base_dir = os.path.join(self.sys_dir, 'stage0-posix', 'src')
- copy_tree(stage0_posix_base_dir, self.tmp_dir)
-
- arch = stage0_arch_map.get(self.arch, self.arch)
- kaem_optional_seed = os.path.join(self.sys_dir, 'stage0-posix', 'src', 'bootstrap-seeds',
- 'POSIX', arch, 'kaem-optional-seed')
- shutil.copy2(kaem_optional_seed, os.path.join(self.tmp_dir, 'init'))
-
- # stage0-posix hook to continue running live-bootstrap
- shutil.copy2(os.path.join(self.sys_dir, 'after.kaem'),
- os.path.join(self.tmp_dir, 'after.kaem'))
-
- def add_fiwix_files(self, file_list_path, dirpath):
- """Add files to the list to populate Fiwix file system"""
- for root, _, filepaths in os.walk(dirpath):
- if 'stage0-posix' in root:
- continue
- if root == os.path.join('sysc', 'distfiles'):
- continue
- with open(file_list_path, 'a', encoding="utf-8") as file_list:
- for filepath in filepaths:
- file_list.write(f"/{os.path.join(root, filepath)}\n")
-
- def create_fiwix_file_list(self):
- """Create a list of files to populate Fiwix file system"""
- file_list_path = os.path.join(self.tmp_dir, 'sysa', 'lwext4-1.0.0-lb1',
- 'files', 'fiwix-file-list.txt')
- shutil.copyfile(os.path.join(self.tmp_dir, 'sysa', 'lwext4-1.0.0-lb1',
- 'files', 'early-artifacts-needed-after-fiwix.txt'),
- file_list_path)
-
- save_cwd = os.getcwd()
- self.add_fiwix_files(file_list_path, 'sysa')
- self.add_fiwix_files(file_list_path, 'sysb')
- self.add_fiwix_files(file_list_path, 'sysc')
- os.chdir(save_cwd)
-
- @staticmethod
- def output_dir(srcfs_file, dirpath):
- """Add a directory to srcfs file system"""
- srcline = f"src 0 {dirpath}\n"
- srcfs_file.write(srcline.encode())
-
- @staticmethod
- def output_file(srcfs_file, filepath):
- """Add a file to srcfs file system"""
- srcline = f"src {os.path.getsize(filepath)} {filepath}\n"
- srcfs_file.write(srcline.encode())
- with open(filepath, 'rb') as srcfile:
- srcfs_file.write(srcfile.read())
-
- def output_tree(self, srcfs_file, treepath):
- """Add a tree of files to srcfs file system"""
- self.output_dir(srcfs_file, treepath)
- for root, dirs, files in os.walk(treepath):
- if ".git" in root:
- continue
- for dirpath in dirs:
- if ".git" in dirpath:
- continue
- self.output_dir(srcfs_file, os.path.join(root, dirpath))
-
- for filepath in files:
- if ".git" in filepath:
- continue
- self.output_file(srcfs_file, os.path.join(root, filepath))
-
- def append_srcfs(self, image_file):
- """Append srcfs file system to sysa disk image"""
- save_cwd = os.getcwd()
-
- os.chdir(os.path.join(self.tmp_dir, 'sysa', 'stage0-posix', 'src'))
- self.output_tree(image_file, '.')
-
- os.chdir(self.tmp_dir)
- shutil.move(os.path.join('sysa', 'stage0-posix'), '.')
- self.output_tree(image_file, 'sysa')
- self.output_tree(image_file, 'sysb')
- self.output_tree(image_file, 'sysc')
- shutil.move('stage0-posix', 'sysa')
- shutil.copyfile(os.path.join('sysa', 'after.kaem'), 'after.kaem')
- self.output_file(image_file, 'after.kaem')
-
- # Add commands to kick off stage0-posix
- cmd = ' '.join(['hex0',
- './bootstrap-seeds/POSIX/x86/hex0_x86.hex0',
- './bootstrap-seeds/POSIX/x86/hex0-seed\n'])
- image_file.write(cmd.encode())
- cmd = ' '.join(['hex0',
- './bootstrap-seeds/POSIX/x86/kaem-minimal.hex0',
- './bootstrap-seeds/POSIX/x86/kaem-optional-seed\n'])
- image_file.write(cmd.encode())
- cmd = ' '.join(['./bootstrap-seeds/POSIX/x86/kaem-optional-seed', './kaem.x86\n'])
- image_file.write(cmd.encode())
-
- os.chdir(save_cwd)
-
- def create_builder_hex0_disk_image(self, image_file_name):
- """Create builder-hex0 disk image"""
- shutil.copyfile(os.path.join('sysa', 'stage0-posix', 'src', 'bootstrap-seeds',
- 'NATIVE', 'x86', 'builder-hex0-x86-stage1.img'),
- image_file_name)
-
- with open(image_file_name, 'ab') as image_file:
- # Append stage2 hex0 source
- with open(os.path.join('kernel-bootstrap', 'builder-hex0-x86-stage2.hex0'),
- encoding="utf-8") as infile:
- image_file.write(infile.read().encode())
- # Pad to next sector
- current_size = os.stat(image_file_name).st_size
- while current_size % 512 != 0:
- image_file.write(b'\0')
- current_size += 1
- self.append_srcfs(image_file)
-
- current_size = os.stat(image_file_name).st_size
-
- megabyte = 1024 * 1024
- # fill file with zeros up to next megabyte
- extra = current_size % megabyte
- round_up = megabyte - extra
- with open(image_file_name, 'ab') as image_file:
- image_file.write(b'\0' * round_up)
- current_size += round_up
-
- # fill file with zeros up to desired size, one megabyte at a time
- with open(image_file_name, 'ab') as image_file:
- while current_size < 16384 * megabyte:
- image_file.write(b'\0' * megabyte)
- current_size += megabyte
