logo

live-bootstrap

Mirror of <https://github.com/fosslinux/live-bootstrap>

generator.py (14863B)

    

  1. #!/usr/bin/env python3
  2. """
  3. This file contains all code required to generate the boot image for live-bootstrap
  4. """
  5. # SPDX-License-Identifier: GPL-3.0-or-later
  6. # SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
  7. # SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
  8. # SPDX-FileCopyrightText: 2021 Melg Eight <public.melg8@gmail.com>
  9. # SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
  11. import hashlib
  12. import os
  13. import shutil
  14. import tarfile
  15. import traceback
  16. import requests
  18. # pylint: disable=too-many-instance-attributes
  19. class Generator():
  20.     """
  21.     Class responsible for generating the basic media to be consumed.
  22.     """
  24.     git_dir = os.path.join(os.path.dirname(os.path.join(__file__)), '..')
  25.     distfiles_dir = os.path.join(git_dir, 'distfiles')
  27.     def __init__(self, arch, external_sources, early_preseed, repo_path):
  28.         self.arch = arch
  29.         self.early_preseed = early_preseed
  30.         self.external_sources = external_sources
  31.         self.repo_path = repo_path
  32.         self.source_manifest = self.get_source_manifest(not self.external_sources)
  33.         self.early_source_manifest = self.get_source_manifest(True)
  34.         self.target_dir = None
  35.         self.external_dir = None
  37.     def reuse(self, target):
  38.         """
  39.         Reuse a previously prepared bwrap environment for further stages.
  40.         """
  41.         self.target_dir = target.path
  42.         self.external_dir = os.path.join(self.target_dir, 'external')
  43.         self.distfiles()
  45.     def prepare(self, target, using_kernel=False, kernel_bootstrap=False, target_size=0):
  46.         """
  47.         Prepare basic media of live-bootstrap.
  48.         /steps -- contains steps to be built
  49.         / -- contains seed to allow steps to be built, containing custom
  50.              scripts and stage0-posix
  51.         """
  52.         self.target_dir = target.path
  53.         self.external_dir = os.path.join(self.target_dir, 'external')
  55.         # We use ext3 here; ext4 actually has a variety of extensions that
  56.         # have been added with varying levels of recency
  57.         # Linux 4.9.10 does not support a bunch of them
  58.         # Attempting to disable extensions that a particular e2fsprogs
  59.         # is *unaware* of causes the filesystem creation to fail
  60.         # We could hypothetically detect e2fsprogs version and create an
  61.         # argument matrix ... or we could just use ext3 instead which
  62.         # is effectively universally the same
  63.         if kernel_bootstrap:
  64.             self.target_dir = os.path.join(self.target_dir, 'init')
  65.             os.mkdir(self.target_dir)
  67.             if not self.repo_path and not self.external_sources:
  68.                 self.external_dir = os.path.join(self.target_dir, 'external')
  69.         elif using_kernel:
  70.             self.target_dir = os.path.join(self.target_dir, 'disk')
  71.             self.external_dir = os.path.join(self.target_dir, 'external')
  73.         if self.early_preseed:
  74.             # Extract tar containing preseed
  75.             with tarfile.open(self.early_preseed, "r") as seed:
  76.                 seed.extractall(self.target_dir)
  77.             if os.path.exists(os.path.join(self.target_dir, 'steps')):
  78.                 shutil.rmtree(os.path.join(self.target_dir, 'steps'))
  79.             if os.path.exists(self.external_dir):
  80.                 shutil.rmtree(self.external_dir)
  81.             shutil.copy2(os.path.join(self.git_dir, 'seed', 'preseeded.kaem'),
  82.                          os.path.join(self.target_dir, 'kaem.x86'))
  83.         else:
  84.             self.stage0_posix(kernel_bootstrap)
  85.             self.seed()
  87.         os.makedirs(self.external_dir)
  89.         self.steps()
  91.         self.distfiles()
  93.         if self.repo_path:
  94.             repo_dir = os.path.join(self.external_dir, 'repo-preseeded')
  95.             shutil.copytree(self.repo_path, repo_dir)
  97.         if kernel_bootstrap:
  98.             self.create_builder_hex0_disk_image(self.target_dir + '.img', target_size)
  100.             if self.repo_path or self.external_sources:
  101.                 mkfs_args = ['-d', os.path.join(target.path, 'external')]
  102.                 target.add_disk("external", filesystem="ext3", mkfs_args=mkfs_args)
  103.         elif using_kernel:
  104.             mkfs_args = ['-F', '-d', os.path.join(target.path, 'disk')]
  105.             target.add_disk("disk",
  106.                             filesystem="ext3",
  107.                             size=(str(target_size) + "M") if target_size else "16G",
  108.                             bootable=True,
  109.                             mkfs_args=mkfs_args)
  111.     def steps(self):
  112.         """Copy in steps."""
  113.         self.get_packages()
  115.         shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.target_dir, 'steps'))
  117.     def stage0_posix(self, kernel_bootstrap=False):
  118.         """Copy in all of the stage0-posix"""
  119.         stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix')
  120.         for entry in os.listdir(stage0_posix_base_dir):
  121.             if kernel_bootstrap and entry == 'bootstrap-seeds':
  122.                 continue
  123.             orig = os.path.join(stage0_posix_base_dir, entry)
  124.             target = os.path.join(self.target_dir, entry)
  125.             if os.path.isfile(orig):
  126.                 shutil.copy2(orig, target)
  127.             else:
  128.                 shutil.copytree(orig, target)
  130.         if not kernel_bootstrap:
  131.             arch = stage0_arch_map.get(self.arch, self.arch)
  132.             kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix',
  133.                                               'bootstrap-seeds', 'POSIX', arch,
  134.                                               'kaem-optional-seed')
  135.             shutil.copy2(kaem_optional_seed, os.path.join(self.target_dir, 'init'))
  137.     def seed(self):
  138.         """Copy in extra seed files"""
  139.         seed_dir = os.path.join(self.git_dir, 'seed')
  140.         for entry in os.listdir(seed_dir):
  141.             if os.path.isfile(os.path.join(seed_dir, entry)):
  142.                 shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.target_dir, entry))
  144.     def distfiles(self):
  145.         """Copy in distfiles"""
  146.         def copy_no_network_distfiles(out, early):
  147.             # Note that "no disk" implies "no network" for kernel bootstrap mode
  148.             manifest = self.early_source_manifest if early else self.source_manifest
  149.             for file in manifest:
  150.                 file = file[3].strip()
  151.                 shutil.copy2(os.path.join(self.distfiles_dir, file),
  152.                              os.path.join(out, file))
  154.         early_distfile_dir = os.path.join(self.target_dir, 'external', 'distfiles')
  155.         main_distfile_dir = os.path.join(self.external_dir, 'distfiles')
  157.         if early_distfile_dir != main_distfile_dir:
  158.             os.makedirs(early_distfile_dir, exist_ok=True)
  159.             copy_no_network_distfiles(early_distfile_dir, True)
  161.         if self.external_sources:
  162.             shutil.copytree(self.distfiles_dir, main_distfile_dir, dirs_exist_ok=True)
  163.         else:
  164.             os.mkdir(main_distfile_dir)
  165.             copy_no_network_distfiles(main_distfile_dir, False)
  167.     @staticmethod
  168.     def output_dir(srcfs_file, dirpath):
  169.         """Add a directory to srcfs file system"""
  170.         srcline = f"src 0 {dirpath}\n"
  171.         srcfs_file.write(srcline.encode())
  173.     @staticmethod
  174.     def output_file(srcfs_file, filepath):
  175.         """Add a file to srcfs file system"""
  176.         srcline = f"src {os.path.getsize(filepath)} {filepath}\n"
  177.         srcfs_file.write(srcline.encode())
  178.         with open(filepath, 'rb') as srcfile:
  179.             srcfs_file.write(srcfile.read())
  181.     def output_tree(self, srcfs_file, treepath):
  182.         """Add a tree of files to srcfs file system"""
  183.         self.output_dir(srcfs_file, treepath)
  184.         for root, dirs, files in os.walk(treepath):
  185.             if ".git" in root:
  186.                 continue
  187.             for dirpath in dirs:
  188.                 if ".git" in dirpath:
  189.                     continue
  190.                 self.output_dir(srcfs_file, os.path.join(root, dirpath))
  192.             for filepath in files:
  193.                 if ".git" in filepath:
  194.                     continue
  195.                 self.output_file(srcfs_file, os.path.join(root, filepath))
  197.     def append_srcfs(self, image_file):
  198.         """Append srcfs file system to disk image"""
  199.         save_cwd = os.getcwd()
  201.         os.chdir(self.target_dir)
  202.         self.output_tree(image_file, '.')
  204.         # Add commands to kick off stage0-posix
  205.         cmd = ' '.join(['src',
  206.                         '0',
  207.                         '/bootstrap-seeds\n'])
  208.         image_file.write(cmd.encode())
  209.         cmd = ' '.join(['src',
  210.                         '0',
  211.                         '/bootstrap-seeds/POSIX\n'])
  212.         image_file.write(cmd.encode())
  213.         cmd = ' '.join(['src',
  214.                         '0',
  215.                         '/bootstrap-seeds/POSIX/x86\n'])
  216.         image_file.write(cmd.encode())
  217.         cmd = ' '.join(['hex0',
  218.                         '/x86/hex0_x86.hex0',
  219.                         '/bootstrap-seeds/POSIX/x86/hex0-seed\n'])
  220.         image_file.write(cmd.encode())
  221.         cmd = ' '.join(['hex0',
  222.                         '/x86/kaem-minimal.hex0',
  223.                         '/bootstrap-seeds/POSIX/x86/kaem-optional-seed\n'])
  224.         image_file.write(cmd.encode())
  225.         cmd = ' '.join(['hex0',
  226.                         '/x86/kaem-minimal.hex0',
  227.                         '/init\n'])
  228.         image_file.write(cmd.encode())
  229.         cmd = ' '.join(['/bootstrap-seeds/POSIX/x86/kaem-optional-seed', '/kaem.x86\n'])
  230.         image_file.write(cmd.encode())
  231.         os.chdir(save_cwd)
  233.     def create_builder_hex0_disk_image(self, image_file_name, size):
  234.         """Create builder-hex0 disk image"""
  235.         with open(image_file_name, 'ab') as image_file:
  236.             # Compile and write stage1 binary seed
  237.             with open(os.path.join('builder-hex0', 'builder-hex0-x86-stage1.hex0'),
  238.                       encoding="utf-8") as infile:
  239.                 for line in infile:
  240.                     image_file.write(bytes.fromhex(line.split('#')[0].split(';')[0].strip()))
  241.             # Append stage2 hex0 source
  242.             with open(os.path.join('builder-hex0', 'builder-hex0-x86-stage2.hex0'),
  243.                       encoding="utf-8") as infile:
  244.                 image_file.write(infile.read().encode())
  245.             # Pad to next sector
  246.             current_size = os.stat(image_file_name).st_size
  247.             while current_size % 512 != 0:
  248.                 image_file.write(b'\0')
  249.                 current_size += 1
  250.             self.append_srcfs(image_file)
  252.         current_size = os.stat(image_file_name).st_size
  254.         megabyte = 1024 * 1024
  255.         # fill file with zeros up to next megabyte
  256.         extra = current_size % megabyte
  257.         round_up = megabyte - extra
  258.         with open(image_file_name, 'ab') as image_file:
  259.             image_file.write(b'\0' * round_up)
  260.         current_size += round_up
  262.         # extend file up to desired size
  263.         if current_size < size * megabyte:
  264.             with open(image_file_name, 'ab') as image_file:
  265.                 image_file.truncate(size * megabyte)
  267.     @staticmethod
  268.     def check_file(file_name, expected_hash):
  269.         """Check hash of downloaded source file."""
  270.         with open(file_name, "rb") as downloaded_file:
  271.             downloaded_content = downloaded_file.read() # read entire file as bytes
  272.         readable_hash = hashlib.sha256(downloaded_content).hexdigest()
  273.         if expected_hash == readable_hash:
  274.             return
  275.         raise ValueError(f"Checksum mismatch for file {os.path.basename(file_name)}:\n\
  276. expected: {expected_hash}\n\
  277. actual:   {readable_hash}\n\
  278. When in doubt, try deleting the file in question -- it will be downloaded again when running \
  279. this script the next time")
  281.     @staticmethod
  282.     def download_file(url, directory, file_name, silent=False):
  283.         """
  284.         Download a single source archive.
  285.         """
  286.         abs_file_name = os.path.join(directory, file_name)
  288.         # Create a directory for downloaded file
  289.         if not os.path.isdir(directory):
  290.             os.mkdir(directory)
  292.         # Actually download the file
  293.         headers = {
  294.                 "Accept-Encoding": "identity",
  295.                 "User-Agent": "curl/7.88.1"
  296.         }
  297.         if not os.path.isfile(abs_file_name):
  298.             if not silent:
  299.                 print(f"Downloading: {file_name}")
  300.             response = requests.get(url, allow_redirects=True, stream=True,
  301.                     headers=headers, timeout=20)
  302.             if response.status_code == 200:
  303.                 with open(abs_file_name, 'wb') as target_file:
  304.                     target_file.write(response.raw.read())
  305.             else:
  306.                 raise requests.HTTPError("Download failed: HTTP " +
  307.                     str(response.status_code) + " " + response.reason)
  308.         return abs_file_name
  310.     def get_packages(self):
  311.         """Prepare remaining sources"""
  312.         for line in self.source_manifest:
  313.             try:
  314.                 path = self.download_file(line[2], line[1], line[3])
  315.             except requests.HTTPError:
  316.                 print(traceback.format_exc())
  317.         for line in self.source_manifest:
  318.             path = os.path.join(line[1], line[3])
  319.             self.check_file(path, line[0])
  321.     @classmethod
  322.     def get_source_manifest(cls, pre_network=False):
  323.         """
  324.         Generate a source manifest for the system.
  325.         """
  326.         entries = []
  327.         directory = os.path.relpath(cls.distfiles_dir, cls.git_dir)
  329.         # Find all source files
  330.         steps_dir = os.path.join(cls.git_dir, 'steps')
  331.         with open(os.path.join(steps_dir, 'manifest'), 'r', encoding="utf_8") as file:
  332.             for line in file:
  333.                 if pre_network and line.strip().startswith("improve: ") and "network" in line:
  334.                     break
  336.                 if not line.strip().startswith("build: "):
  337.                     continue
  339.                 step = line.split(" ")[1].split("#")[0].strip()
  340.                 sourcef = os.path.join(steps_dir, step, "sources")
  341.                 if os.path.exists(sourcef):
  342.                     # Read sources from the source file
  343.                     with open(sourcef, "r", encoding="utf_8") as sources:
  344.                         for source in sources.readlines():
  345.                             source = source.strip().split(" ")
  347.                             if len(source) > 2:
  348.                                 file_name = source[2]
  349.                             else:
  350.                                 # Automatically determine file name based on URL.
  351.                                 file_name = os.path.basename(source[0])
  353.                             entry = (source[1], directory, source[0], file_name)
  354.                             if entry not in entries:
  355.                                 entries.append(entry)
  357.         return entries
  359. stage0_arch_map = {
  360.     "amd64": "AMD64",
  361. }