logo

live-bootstrap

Mirror of <https://github.com/fosslinux/live-bootstrap>

helpers.sh (16681B)

    

  1. #!/bin/bash -e
  3. # SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
  4. # SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
  5. # SPDX-FileCopyrightText: 2021 Paul Dersey <pdersey@gmail.com>
  6. # SPDX-FileCopyrightText: 2021 Melg Eight <public.melg8@gmail.com>
  7. #
  8. # SPDX-License-Identifier: GPL-3.0-or-later
  10. # Set constant umask
  11. umask 022
  13. # Get a list of files
  14. get_files() {
  15.     echo "."
  16.     _get_files "${1}"
  17. }
  19. _get_files() {
  20.     local prefix
  21.     prefix="${1}"
  22.     fs=
  23.     if [ -n "$(ls 2>/dev/null)" ]; then
  24.         # This can be removed once Debian 12 is stable
  25.         # shellcheck disable=SC2035
  26.         fs=$(echo *)
  27.     fi
  28.     if [ -n "$(ls .[0-z]* 2>/dev/null)" ]; then
  29.         fs="${fs} $(echo .[0-z]*)"
  30.     fi
  31.     for f in ${fs}; do
  32.         # Archive symlinks to directories as symlinks
  33.         echo "${prefix}/${f}"
  34.         if [ -d "./${f}" ] && ! [ -h "./${f}" ]; then
  35.             cd "./${f}"
  36.             _get_files "${prefix}/${f}"
  37.             cd ..
  38.         fi
  39.     done
  40. }
  42. # Reset all timestamps to unix time 0
  43. reset_timestamp() {
  44.     if command -v find >/dev/null 2>&1; then
  45.         # find does not error out on exec error
  46.         find . -print0 | xargs -0 touch -h -t 197001010000.00
  47.     else
  48.         # A rudimentary find implementation that does the trick
  49.         fs=
  50.         if [ -n "$(ls 2>/dev/null)" ]; then
  51.             fs=$(echo ./*)
  52.         fi
  53.         if [ -n "$(ls .[0-z]* 2>/dev/null)" ]; then
  54.             fs="${fs} $(echo .[0-z]*)"
  55.         fi
  56.         for f in ${fs}; do
  57.             touch -h -t 197001010000.00 "./${f}"
  58.             if [ -d "./${f}" ]; then
  59.                 cd "./${f}"
  60.                 reset_timestamp
  61.                 cd ..
  62.             fi
  63.         done
  64.     fi
  65. }
  67. # Fake grep
  68. _grep() {
  69.     local text="${1}"
  70.     local fname="${2}"
  71.     if command -v grep >/dev/null 2>&1; then
  72.         grep "${text}" "${fname}"
  73.     else
  74.         # shellcheck disable=SC2162
  75.         while read line; do
  76.             case "${line}" in *"${text}"*)
  77.                 echo "${line}" ;;
  78.             esac
  79.         done < "${fname}"
  80.     fi
  81. }
  83. get_revision() {
  84.     local pkg=$1
  85.     local oldpwd="${PWD}"
  86.     cd "/external/repo"
  87.     # Get revision (n time this package has been built)
  88.     revision=$( (ls -1 "${pkg}"* 2>/dev/null || true) | wc -l | sed 's/ *//g')
  89.     cd "${oldpwd}"
  90. }
  92. # Installs binary packages from an earlier run
  93. # This is useful to speed up development cycle
  94. bin_preseed() {
  95.     if [ -d "/external/repo-preseeded" ]; then
  96.         get_revision "${pkg}"
  97.         cd "/external/repo-preseeded"
  98.         test -e "${pkg}_${revision}.tar.bz2" || return 1
  99.         if [ "${UPDATE_CHECKSUMS}" = "True" ] || src_checksum "${pkg}" $((revision)); then
  100.             echo "${pkg}: installing prebuilt package."
  101.             mv "${pkg}_${revision}"* /external/repo || return 1
  102.             cd "/external/repo"
  103.             rm -f /tmp/filelist.txt
  104.             src_apply "${pkg}" $((revision))
  105.             cd "${SRCDIR}"
  106.             return 0
  107.         fi
  108.     fi
  109.     return 1
  110. }
  112. # Removes either an existing package or file
  113. uninstall() {
  114.     local in_fs in_pkg symlinks
  115.     while [ $# -gt 0 ]; do
  116.         removing="$1"
  117.         case "${removing}" in
  118.             /*)
  119.                 # Removing a file
  120.                 echo "removing file: ${removing}."
  121.                 rm -f "${removing}"
  122.                 ;;
  123.             *)
  124.                 echo "${removing}: uninstalling."
  125.                 local oldpwd="${PWD}"
  126.                 mkdir -p "/tmp/removing"
  127.                 cd "/tmp/removing"
  128.                 get_revision "${removing}"
  129.                 local filename="/external/repo/${removing}_$((revision-1)).tar.bz2"
  130.                 # Initial bzip2 built against meslibc has broken pipes
  131.                 bzip2 -dc "${filename}" | tar -xf -
  132.                 # reverse to have files before directories
  133.                 if command -v find >/dev/null 2>&1; then
  134.                     find . | sort -r > ../filelist
  135.                 else
  136.                     get_files . | tac > ../filelist
  137.                 fi
  138.                 # shellcheck disable=SC2162
  139.                 while read file; do
  140.                     if [ -d "${file}" ]; then
  141.                         if [ -z "$(ls -A "/${file}")" ]; then
  142.                             rmdir "/${file}"
  143.                         fi
  144.                     else
  145.                         # in some cases we might be uninstalling a file that has already been overwritten
  146.                         # in this case we don't want to remove it
  147.                         in_fs="$(sha256sum "${file}" 2>/dev/null | cut -d' ' -f1)"
  148.                         in_pkg="$(sha256sum "/${file}" 2>/dev/null | cut -d' ' -f1)"
  149.                         if [ "${in_fs}" = "${in_pkg}" ]; then
  150.                             rm -f "/${file}"
  151.                         fi
  152.                         if [ -h "${file}" ]; then
  153.                             symlinks="${symlinks} ${file}"
  154.                         fi
  155.                     fi
  156.                 done < ../filelist
  157.                 rm -f ../filelist
  158.                 for link in ${symlinks}; do
  159.                     if [ ! -e "/${link}" ]; then
  160.                         rm -f "/${link}"
  161.                     fi
  162.                 done
  163.                 cd "${oldpwd}"
  164.                 rm -rf "/tmp/removing"
  165.                 ;;
  166.         esac
  167.         shift
  168.     done
  169. }
  171. # Common build steps
  172. # Build function provides a few common stages with default implementation
  173. # that can be overridden on per package basis in the build script.
  174. # build takes two arguments:
  175. # 1) name-version of the package
  176. # 2) optionally specify build script. Default is pass$((revision+1)).sh
  177. # 3) optionally specify directory to cd into
  178. build() {
  179.     pkg=$1
  180.     get_revision "${pkg}"
  181.     script_name=${2:-pass$((revision+1)).sh}
  182.     dirname=${3:-${pkg}}
  184.     # shellcheck disable=SC2015
  185.     bin_preseed && return || true # Normal build if preseed fails
  187.     cd "${SRCDIR}/${pkg}" || (echo "Cannot cd into ${pkg}!"; kill $$)
  188.     echo "${pkg}: beginning build using script ${script_name}"
  189.     base_dir="${PWD}"
  190.     if [ -e "${base_dir}/patches-$(basename "${script_name}" .sh)" ]; then
  191.         patch_dir="${base_dir}/patches-$(basename "${script_name}" .sh)"
  192.     else
  193.         patch_dir="${base_dir}/patches"
  194.     fi
  195.     mk_dir="${base_dir}/mk"
  196.     files_dir="${base_dir}/files"
  198.     rm -rf "build"
  199.     mkdir "build"
  200.     cd "build"
  202.     build_script="${base_dir}/${script_name}"
  203.     if test -e "${build_script}"; then
  204.         # shellcheck source=/dev/null
  205.         . "${build_script}"
  206.     fi
  208.     echo "${pkg}: getting sources."
  209.     build_stage=src_get
  210.     call $build_stage
  212.     echo "${pkg}: unpacking source."
  213.     build_stage=src_unpack
  214.     call $build_stage
  215.     unset EXTRA_DISTFILES
  217.     cd "${dirname}" || (echo "Cannot cd into build/${dirname}!"; kill $$)
  219.     echo "${pkg}: preparing source."
  220.     build_stage=src_prepare
  221.     call $build_stage
  223.     echo "${pkg}: configuring source."
  224.     build_stage=src_configure
  225.     call $build_stage
  227.     echo "${pkg}: compiling source."
  228.     build_stage=src_compile
  229.     call $build_stage
  231.     echo "${pkg}: install to fakeroot."
  232.     mkdir -p "${DESTDIR}"
  233.     build_stage=src_install
  234.     call $build_stage
  236.     echo "${pkg}: postprocess binaries."
  237.     build_stage=src_postprocess
  238.     call $build_stage
  240.     echo "${pkg}: creating package."
  241.     cd "${DESTDIR}"
  242.     src_pkg
  244.     src_checksum "${pkg}" "${revision}"
  246.     echo "${pkg}: cleaning up."
  247.     rm -rf "${SRCDIR}/${pkg}/build"
  248.     rm -rf "${DESTDIR}"
  250.     echo "${pkg}: installing package."
  251.     src_apply "${pkg}" "${revision}"
  253.     echo "${pkg}: build successful"
  255.     cd "${SRCDIR}"
  257.     unset -f src_get src_unpack src_prepare src_configure src_compile src_install src_postprocess
  258.     unset extract
  259. }
  261. download_source_line() {
  262.     url="${1}"
  263.     checksum="${2}"
  264.     fname="${3}"
  265.     # Default to basename of url if not given
  266.     fname="${fname:-$(basename "${url}")}"
  267.     if ! [ -e "${fname}" ]; then
  268.         curl --fail --retry 5 --location "${url}" --output "${fname}" || true
  269.     fi
  270. }
  272. check_source_line() {
  273.     url="${1}"
  274.     checksum="${2}"
  275.     fname="${3}"
  276.     # Default to basename of url if not given
  277.     fname="${fname:-$(basename "${url}")}"
  278.     if ! [ -e "${fname}" ]; then
  279.         false
  280.     fi
  281.     echo "${checksum}  ${fname}" > "${fname}.sum"
  282.     sha256sum -c "${fname}.sum"
  283.     rm "${fname}.sum"
  284. }
  286. # Default get function that downloads source tarballs.
  287. default_src_get() {
  288.     # shellcheck disable=SC2153
  289.     cd "${DISTFILES}"
  290.     # shellcheck disable=SC2162
  291.     while read line; do
  292.         # This is intentional - we want to split out ${line} into separate arguments.
  293.         # shellcheck disable=SC2086
  294.         download_source_line ${line}
  295.     done < "${base_dir}/sources"
  296.     # shellcheck disable=SC2162
  297.     while read line; do
  298.         # This is intentional - we want to split out ${line} into separate arguments.
  299.         # shellcheck disable=SC2086
  300.         check_source_line ${line}
  301.     done < "${base_dir}/sources"
  302.     cd -
  303. }
  305. # Intelligently extracts a file based upon its filetype.
  306. extract_file() {
  307.     f="${3:-$(basename "${1}")}"
  308.     # shellcheck disable=SC2154
  309.     case "${noextract}" in
  310.         *${f}*)
  311.             cp "${DISTFILES}/${f}" .
  312.             ;;
  313.         *)
  314.             case "${f}" in
  315.                 *.tar* | *.tgz)
  316.                     # shellcheck disable=SC2153
  317.                     if test -e "${PREFIX}/libexec/rmt"; then
  318.                         # Again, we want to split out into words.
  319.                         # shellcheck disable=SC2086
  320.                         tar --no-same-owner -xf "${DISTFILES}/${f}" ${extract}
  321.                     else
  322.                         # shellcheck disable=SC2086
  323.                         case "${f}" in
  324.                         *.tar.gz) tar -xzf "${DISTFILES}/${f}" ${extract} ;;
  325.                         *.tar.bz2)
  326.                             # Initial bzip2 built against meslibc has broken pipes
  327.                             bzip2 -dc "${DISTFILES}/${f}" | tar -xf - ${extract} ;;
  328.                         *.tar.xz | *.tar.lzma)
  329.                             if test -e "${PREFIX}/bin/xz"; then
  330.                                 tar -xf "${DISTFILES}/${f}" --use-compress-program=xz ${extract}
  331.                             else
  332.                                 unxz --file "${DISTFILES}/${f}" | tar -xf - ${extract}
  333.                             fi
  334.                             ;;
  335.                         esac
  336.                     fi
  337.                     ;;
  338.                 *)
  339.                     cp "${DISTFILES}/${f}" .
  340.                     ;;
  341.             esac
  342.             ;;
  343.     esac
  344. }
  346. # Default unpacking function that unpacks all sources.
  347. default_src_unpack() {
  348.     # Handle the first one differently
  349.     first_line=$(head -n 1 ../sources)
  350.     # Again, we want to split out into words.
  351.     # shellcheck disable=SC2086
  352.     extract_file ${first_line}
  353.     # This assumes there is only one directory in the tarball
  354.     # Get the dirname "smartly"
  355.     if ! [ -e "${dirname}" ]; then
  356.         for i in *; do
  357.             if [ -d "${i}" ]; then
  358.                 dirname="${i}"
  359.                 break
  360.             fi
  361.         done
  362.     fi
  363.     # shellcheck disable=SC2162
  364.     tail -n +2 ../sources | while read line; do
  365.         # shellcheck disable=SC2086
  366.         extract_file ${line}
  367.     done
  368. }
  370. # Default function to prepare source code.
  371. # It applies all patches from patch_dir (at the moment only -p0 patches are supported).
  372. # Patches are applied from the parent directory.
  373. # Then it copies our custom makefile and any other custom files from files directory.
  374. default_src_prepare() {
  375.     if test -d "${patch_dir}"; then
  376.         if ls "${patch_dir}"/*.patch >/dev/null 2>&1; then
  377.             for p in "${patch_dir}"/*.patch; do
  378.                 echo "Applying patch: ${p}"
  379.                 patch -d.. -Np0 < "${p}"
  380.             done
  381.         fi
  382.     fi
  384.     makefile="${mk_dir}/main.mk"
  385.     if test -e "${makefile}"; then
  386.         cp "${makefile}" Makefile
  387.     fi
  389.     if test -d "${files_dir}"; then
  390.         cp --no-preserve=mode "${files_dir}"/* "${PWD}/"
  391.     fi
  392. }
  394. # Default function for configuring source.
  395. default_src_configure() {
  396.     :
  397. }
  399. # Default function for compiling source. It simply runs make without any parameters.
  400. default_src_compile() {
  401.     make "${MAKEJOBS}" -f Makefile PREFIX="${PREFIX}"
  402. }
  404. # Default installing function. PREFIX should be set by run.sh script.
  405. # Note that upstream makefiles might ignore PREFIX and have to be configured in configure stage.
  406. default_src_install() {
  407.     make -f Makefile install PREFIX="${PREFIX}" DESTDIR="${DESTDIR}"
  408. }
  410. # Default function for postprocessing binaries.
  411. default_src_postprocess() {
  412.     if (command -v find && command -v file && command -v strip) >/dev/null 2>&1; then
  413.         # Logic largely taken from void linux 06-strip-and-debug-pkgs.sh
  414.         # shellcheck disable=SC2162
  415.         find "${DESTDIR}" -type f | while read f; do
  416.             case "$(file -bi "${f}")" in
  417.                 application/x-executable*) strip "${f}" ;;
  418.                 application/x-sharedlib*|application/x-pie-executable*)
  419.                     machine_set="$(file -b "${f}")"
  420.                     case "${machine_set}" in
  421.                         *no\ machine*) ;; # don't strip ELF container-only
  422.                         *) strip --strip-unneeded "${f}" ;;
  423.                     esac
  424.                     ;;
  425.                 application/x-archive*) strip --strip-debug "${f}" ;;
  426.             esac
  427.         done
  428.     fi
  429. }
  431. src_pkg() {
  432.     touch -t 197001010000.00 .
  433.     reset_timestamp
  435.     local tar_basename="${pkg}_${revision}.tar"
  436.     local dest_tar="/external/repo/${tar_basename}"
  437.     local filelist=/tmp/filelist.txt
  439.     cd /external/repo
  440.     # If grep is unavailable, then tar --sort is unavailable.
  441.     # So this does not need a command -v grep.
  442.     if tar --help | grep ' \-\-sort' >/dev/null 2>&1; then
  443.         tar -C "${DESTDIR}" --sort=name --hard-dereference \
  444.             --numeric-owner --owner=0 --group=0 --mode=go=rX,u+rw -cf "${dest_tar}" .
  445.     else
  446.         local olddir
  447.         olddir=$PWD
  448.         cd "${DESTDIR}"
  449.         local null
  450.         if command -v find >/dev/null 2>&1 && command -v sort >/dev/null 2>&1; then
  451.             find . -print0 | LC_ALL=C sort -z > "${filelist}"
  452.             null="--null"
  453.         elif command -v sort >/dev/null 2>&1; then
  454.             get_files .  | LC_ALL=C sort > "${filelist}"
  455.         else
  456.             get_files . > ${filelist}
  457.         fi
  458.         tar --no-recursion ${null} --files-from "${filelist}" \
  459.                 --numeric-owner --owner=0 --group=0 --mode=go=rX,u+rw -cf "${dest_tar}"
  460.         rm -f "$filelist"
  461.         cd "$olddir"
  462.     fi
  463.     touch -t 197001010000.00 "${tar_basename}"
  464.     bzip2 --best "${tar_basename}"
  465. }
  467. src_checksum() {
  468.     local pkg=$1 revision=$2
  469.     local rval=0
  470.     if ! [ "$UPDATE_CHECKSUMS" = True ] ; then
  471.         # We avoid using pipes as that is not supported by initial sha256sum from mescc-tools-extra
  472.         local checksum_file=/tmp/checksum
  473.         _grep "${pkg}_${revision}" "${SRCDIR}/SHA256SUMS.pkgs" > "${checksum_file}" || true
  474.         # Check there is something in checksum_file
  475.         if ! [ -s "${checksum_file}" ]; then
  476.             echo "${pkg}: no checksum stored!"
  477.             false
  478.         fi
  479.         echo "${pkg}: checksumming created package."
  480.         sha256sum -c "${checksum_file}" || rval=$?
  481.         rm "${checksum_file}"
  482.     fi
  483.     return "${rval}"
  484. }
  486. src_apply() {
  487.     local pkg="${1}" revision="${2}"
  488.     local TAR_PREFIX BZIP2_PREFIX
  490.     # Make sure we have at least one copy of tar
  491.     if [[ "${pkg}" == tar-* ]]; then
  492.         mkdir -p /tmp
  493.         cp "${PREFIX}/bin/tar" "/tmp/tar"
  494.         TAR_PREFIX="/tmp/"
  495.     fi
  497.     # Bash does not like to be overwritten
  498.     if [[ "${pkg}" == bash-* ]]; then
  499.         rm "${PREFIX}/bin/bash"
  500.     fi
  502.     # Overwriting files is mega busted, so do it manually
  503.     # shellcheck disable=SC2162
  504.     if [ -e /tmp/filelist.txt ]; then
  505.         while IFS= read -d $'\0' file; do
  506.             rm -f "/${file}" >/dev/null 2>&1 || true
  507.         done < /tmp/filelist.txt
  508.     fi
  510.     # Bzip2 does not like to be overwritten
  511.     if [[ "${pkg}" == bzip2-* ]]; then
  512.         mkdir -p /tmp
  513.         mv "${PREFIX}/bin/bzip2" "/tmp/bzip2"
  514.         BZIP2_PREFIX="/tmp/"
  515.     fi
  516.     "${BZIP2_PREFIX}bzip2" -dc "/external/repo/${pkg}_${revision}.tar.bz2" | \
  517.         "${TAR_PREFIX}tar" -C / -xpf -
  518.     rm -f "/tmp/bzip2" "/tmp/tar"
  519. }
  521. # Check if bash function exists
  522. fn_exists() {
  523.     test "$(type -t "$1")" == 'function'
  524. }
  526. # Call package specific function or default implementation.
  527. call() {
  528.     if fn_exists "$1"; then
  529.         $1
  530.     else
  531.         default_"${1}"
  532.     fi
  533. }
  535. # Call default build stage function
  536. default() {
  537.     "default_${build_stage}"
  538. }