logo

live-bootstrap

Mirror of <https://github.com/fosslinux/live-bootstrap>

helpers.sh (17639B)

    

  1. #!/bin/bash -e
  3. # SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
  4. # SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
  5. # SPDX-FileCopyrightText: 2021 Paul Dersey <pdersey@gmail.com>
  6. # SPDX-FileCopyrightText: 2021 Melg Eight <public.melg8@gmail.com>
  7. #
  8. # SPDX-License-Identifier: GPL-3.0-or-later
  10. # Set constant umask
  11. umask 022
  13. # Get a list of files
  14. get_files() {
  15.     echo "."
  16.     _get_files "${1}"
  17. }
  19. _get_files() {
  20.     local prefix
  21.     prefix="${1}"
  22.     fs=
  23.     if [ -n "$(ls 2>/dev/null)" ]; then
  24.         fs=$(echo *)
  25.     fi
  26.     if [ -n "$(ls .[0-z]* 2>/dev/null)" ]; then
  27.         fs="${fs} $(echo .[0-z]*)"
  28.     fi
  29.     for f in ${fs}; do
  30.         # Archive symlinks to directories as symlinks
  31.         echo "${prefix}/${f}"
  32.         if [ -d "./${f}" ] && ! [ -h "./${f}" ]; then
  33.             cd "./${f}"
  34.             _get_files "${prefix}/${f}"
  35.             cd ..
  36.         fi
  37.     done
  38. }
  40. # Reset all timestamps to unix time 0
  41. reset_timestamp() {
  42.     if command -v find >/dev/null 2>&1; then
  43.         # find does not error out on exec error
  44.         find . -print0 | xargs -0 touch -h -t 197001010000.00
  45.     else
  46.         # A rudimentary find implementation that does the trick
  47.         fs=
  48.         if [ -n "$(ls 2>/dev/null)" ]; then
  49.             fs=$(echo ./*)
  50.         fi
  51.         if [ -n "$(ls .[0-z]* 2>/dev/null)" ]; then
  52.             fs="${fs} $(echo .[0-z]*)"
  53.         fi
  54.         for f in ${fs}; do
  55.             touch -h -t 197001010000.00 "./${f}"
  56.             if [ -d "./${f}" ]; then
  57.                 cd "./${f}"
  58.                 reset_timestamp
  59.                 cd ..
  60.             fi
  61.         done
  62.     fi
  63. }
  65. # Fake grep
  66. _grep() {
  67.     local text="${1}"
  68.     local fname="${2}"
  69.     if command -v grep >/dev/null 2>&1; then
  70.         grep "${text}" "${fname}"
  71.     else
  72.         # shellcheck disable=SC2162
  73.         while read line; do
  74.             case "${line}" in *"${text}"*)
  75.                 echo "${line}" ;;
  76.             esac
  77.         done < "${fname}"
  78.     fi
  79. }
  81. get_revision() {
  82.     local pkg=$1
  83.     local oldpwd="${PWD}"
  84.     cd "/external/repo"
  85.     # Get revision (n time this package has been built)
  86.     revision=$( (ls -1 "${pkg}"* 2>/dev/null || true) | wc -l | sed 's/ *//g')
  87.     cd "${oldpwd}"
  88. }
  90. # Installs binary packages from an earlier run
  91. # This is useful to speed up development cycle
  92. bin_preseed() {
  93.     if [ -d "/external/repo-preseeded" ]; then
  94.         get_revision "${pkg}"
  95.         cd "/external/repo-preseeded"
  96.         test -e "${pkg}_${revision}.tar.bz2" || return 1
  97.         if [ "${UPDATE_CHECKSUMS}" = "True" ] || src_checksum "${pkg}" $((revision)); then
  98.             echo "${pkg}: installing prebuilt package."
  99.             mv "${pkg}_${revision}"* /external/repo || return 1
  100.             cd "/external/repo"
  101.             rm -f /tmp/filelist.txt
  102.             src_apply "${pkg}" $((revision))
  103.             cd "${SRCDIR}"
  104.             return 0
  105.         fi
  106.     fi
  107.     return 1
  108. }
  110. # Removes either an existing package or file
  111. uninstall() {
  112.     local in_fs in_pkg symlinks
  113.     while [ $# -gt 0 ]; do
  114.         removing="$1"
  115.         case "${removing}" in
  116.             /*)
  117.                 # Removing a file
  118.                 echo "removing file: ${removing}."
  119.                 rm -f "${removing}"
  120.                 ;;
  121.             *)
  122.                 echo "${removing}: uninstalling."
  123.                 local oldpwd="${PWD}"
  124.                 mkdir -p "/tmp/removing"
  125.                 cd "/tmp/removing"
  126.                 get_revision "${removing}"
  127.                 local filename="/external/repo/${removing}_$((revision-1)).tar.bz2"
  128.                 # Initial bzip2 built against meslibc has broken pipes
  129.                 bzip2 -dc "${filename}" | tar -xf -
  130.                 # reverse to have files before directories
  131.                 if command -v find >/dev/null 2>&1; then
  132.                     find . | sort -r > ../filelist
  133.                 else
  134.                     get_files . | tac > ../filelist
  135.                 fi
  136.                 # shellcheck disable=SC2162
  137.                 while read file; do
  138.                     if [ -d "${file}" ]; then
  139.                         if [ -z "$(ls -A "/${file}")" ]; then
  140.                             rmdir "/${file}"
  141.                         fi
  142.                     elif [ -h "${file}" ]; then
  143.                         symlinks="${symlinks} ${file}"
  144.                     else
  145.                         # in some cases we might be uninstalling a file that has already been overwritten
  146.                         # in this case we don't want to remove it
  147.                         in_fs="$(sha256sum "${file}" 2>/dev/null | cut -d' ' -f1)"
  148.                         in_pkg="$(sha256sum "/${file}" 2>/dev/null | cut -d' ' -f1)"
  149.                         if [ "${in_fs}" = "${in_pkg}" ]; then
  150.                             rm -f "/${file}"
  151.                         fi
  152.                     fi
  153.                 done < ../filelist
  154.                 rm -f ../filelist
  155.                 for link in ${symlinks}; do
  156.                     if [ ! -e "/${link}" ]; then
  157.                         rm -f "/${link}"
  158.                     fi
  159.                 done
  160.                 cd "${oldpwd}"
  161.                 rm -rf "/tmp/removing"
  162.                 ;;
  163.         esac
  164.         shift
  165.     done
  166. }
  168. # Common build steps
  169. # Build function provides a few common stages with default implementation
  170. # that can be overridden on per package basis in the build script.
  171. # build takes two arguments:
  172. # 1) name-version of the package
  173. # 2) optionally specify build script. Default is pass$((revision+1)).sh
  174. # 3) optionally specify directory to cd into
  175. build() {
  176.     pkg=$1
  177.     get_revision "${pkg}"
  178.     script_name=${2:-pass$((revision+1)).sh}
  179.     dirname=${3:-${pkg}}
  181.     # shellcheck disable=SC2015
  182.     bin_preseed && return || true # Normal build if preseed fails
  184.     cd "${SRCDIR}/${pkg}" || (echo "Cannot cd into ${pkg}!"; kill $$)
  185.     echo "${pkg}: beginning build using script ${script_name}"
  186.     base_dir="${PWD}"
  187.     if [ -e "${base_dir}/patches-$(basename "${script_name}" .sh)" ]; then
  188.         patch_dir="${base_dir}/patches-$(basename "${script_name}" .sh)"
  189.     else
  190.         patch_dir="${base_dir}/patches"
  191.     fi
  192.     mk_dir="${base_dir}/mk"
  193.     files_dir="${base_dir}/files"
  195.     rm -rf "build"
  196.     mkdir "build"
  197.     cd "build"
  199.     build_script="${base_dir}/${script_name}"
  200.     if test -e "${build_script}"; then
  201.         # shellcheck source=/dev/null
  202.         . "${build_script}"
  203.     fi
  205.     echo "${pkg}: getting sources."
  206.     build_stage=src_get
  207.     call $build_stage
  209.     echo "${pkg}: unpacking source."
  210.     build_stage=src_unpack
  211.     call $build_stage
  212.     unset EXTRA_DISTFILES
  214.     cd "${dirname}" || (echo "Cannot cd into build/${dirname}!"; kill $$)
  216.     echo "${pkg}: preparing source."
  217.     build_stage=src_prepare
  218.     call $build_stage
  220.     echo "${pkg}: configuring source."
  221.     build_stage=src_configure
  222.     call $build_stage
  224.     echo "${pkg}: compiling source."
  225.     build_stage=src_compile
  226.     call $build_stage
  228.     echo "${pkg}: install to fakeroot."
  229.     mkdir -p "${DESTDIR}"
  230.     build_stage=src_install
  231.     call $build_stage
  233.     echo "${pkg}: postprocess binaries."
  234.     build_stage=src_postprocess
  235.     call $build_stage
  237.     echo "${pkg}: creating package."
  238.     cd "${DESTDIR}"
  239.     src_pkg
  241.     src_checksum "${pkg}" "${revision}"
  243.     echo "${pkg}: cleaning up."
  244.     rm -rf "${SRCDIR}/${pkg}/build"
  245.     rm -rf "${DESTDIR}"
  247.     echo "${pkg}: installing package."
  248.     src_apply "${pkg}" "${revision}"
  250.     echo "${pkg}: build successful"
  252.     cd "${SRCDIR}"
  254.     unset -f src_get src_unpack src_prepare src_configure src_compile src_install src_postprocess
  255.     unset extract
  256. }
  258. # An inventive way to randomise with what we know we always have
  259. randomize() {
  260.     if command -v shuf >/dev/null 2>&1; then
  261.         # shellcheck disable=SC2086
  262.         shuf -e ${1} | tr '\n' ' '
  263.     else
  264.         mkdir -p /tmp/random
  265.         for item in ${1}; do
  266.             touch --date=@${RANDOM} /tmp/random/"${item//\//~*~*}"
  267.         done
  268.         # cannot rely on find existing
  269.         # shellcheck disable=SC2012
  270.         ls -1 -t /tmp/random | sed 's:~\*~\*:/:g' | tr '\n' ' '
  271.         rm -r /tmp/random
  272.     fi
  273. }
  275. download_source_line() {
  276.     if [[ "${1}" == git://* ]]; then
  277.         shift
  278.     fi
  279.     upstream_url="${1}"
  280.     checksum="${2}"
  281.     fname="${3}"
  282.     # Default to basename of url if not given
  283.     fname="${fname:-$(basename "${upstream_url}")}"
  284.     if ! [ -e "${fname}" ]; then
  285.         for mirror in $(randomize "${MIRRORS}"); do
  286.             mirror_url="${mirror}/${fname}"
  287.             echo "${mirror_url}"
  288.             curl --fail --retry 3 --location "${mirror_url}" --output "${fname}" || true && break
  289.         done
  290.         if ! [ -e "${fname}" ] && [ "${upstream_url}" != "_" ]; then
  291.             curl --fail --retry 3 --location "${upstream_url}" --output "${fname}" || true
  292.         fi
  293.     fi
  294. }
  296. check_source_line() {
  297.     if [[ "${1}" == git://* ]]; then
  298.         shift
  299.     fi
  300.     url="${1}"
  301.     checksum="${2}"
  302.     fname="${3}"
  303.     # Default to basename of url if not given
  304.     fname="${fname:-$(basename "${url}")}"
  305.     if ! [ -e "${fname}" ]; then
  306.         echo "${fname} does not exist!"
  307.         false
  308.     fi
  309.     echo "${checksum}  ${fname}" > "${fname}.sum"
  310.     sha256sum -c "${fname}.sum"
  311.     rm "${fname}.sum"
  312. }
  314. # Default get function that downloads source tarballs.
  315. default_src_get() {
  316.     # shellcheck disable=SC2153
  317.     cd "${DISTFILES}"
  318.     # shellcheck disable=SC2162
  319.     while read line; do
  320.         # This is intentional - we want to split out ${line} into separate arguments.
  321.         # shellcheck disable=SC2086
  322.         download_source_line ${line}
  323.     done < "${base_dir}/sources"
  324.     # shellcheck disable=SC2162
  325.     while read line; do
  326.         # This is intentional - we want to split out ${line} into separate arguments.
  327.         # shellcheck disable=SC2086
  328.         check_source_line ${line}
  329.     done < "${base_dir}/sources"
  330.     cd -
  331. }
  333. # Intelligently extracts a file based upon its filetype.
  334. extract_file() {
  335.     if [[ "${1}" == git://* ]]; then
  336.         shift
  337.     fi
  338.     f="${3:-$(basename "${1}")}"
  339.     # shellcheck disable=SC2154
  340.     case "${noextract}" in
  341.         *${f}*)
  342.             cp "${DISTFILES}/${f}" .
  343.             ;;
  344.         *)
  345.             case "${f}" in
  346.                 *.tar* | *.tgz)
  347.                     # shellcheck disable=SC2153
  348.                     if test -e "${PREFIX}/libexec/rmt"; then
  349.                         # Again, we want to split out into words.
  350.                         # shellcheck disable=SC2086
  351.                         tar --no-same-owner -xf "${DISTFILES}/${f}" ${extract}
  352.                     else
  353.                         # shellcheck disable=SC2086
  354.                         case "${f}" in
  355.                         *.tar.gz) tar -xzf "${DISTFILES}/${f}" ${extract} ;;
  356.                         *.tar.bz2)
  357.                             # Initial bzip2 built against meslibc has broken pipes
  358.                             bzip2 -dc "${DISTFILES}/${f}" | tar -xf - ${extract} ;;
  359.                         *.tar.xz | *.tar.lzma)
  360.                             if test -e "${PREFIX}/bin/xz"; then
  361.                                 tar -xf "${DISTFILES}/${f}" --use-compress-program=xz ${extract}
  362.                             else
  363.                                 unxz --file "${DISTFILES}/${f}" | tar -xf - ${extract}
  364.                             fi
  365.                             ;;
  366.                         esac
  367.                     fi
  368.                     ;;
  369.                 *)
  370.                     cp "${DISTFILES}/${f}" .
  371.                     ;;
  372.             esac
  373.             ;;
  374.     esac
  375. }
  377. # Default unpacking function that unpacks all sources.
  378. default_src_unpack() {
  379.     # Handle the first one differently
  380.     first_line=$(head -n 1 ../sources)
  381.     # Again, we want to split out into words.
  382.     # shellcheck disable=SC2086
  383.     extract_file ${first_line}
  384.     # This assumes there is only one directory in the tarball
  385.     # Get the dirname "smartly"
  386.     if ! [ -e "${dirname}" ]; then
  387.         for i in *; do
  388.             if [ -d "${i}" ]; then
  389.                 dirname="${i}"
  390.                 break
  391.             fi
  392.         done
  393.     fi
  394.     # shellcheck disable=SC2162
  395.     tail -n +2 ../sources | while read line; do
  396.         # shellcheck disable=SC2086
  397.         extract_file ${line}
  398.     done
  399. }
  401. # Default function to prepare source code.
  402. # It applies all patches from patch_dir (at the moment only -p0 patches are supported).
  403. # Patches are applied from the parent directory.
  404. # Then it copies our custom makefile and any other custom files from files directory.
  405. default_src_prepare() {
  406.     if test -d "${patch_dir}"; then
  407.         if ls "${patch_dir}"/*.patch >/dev/null 2>&1; then
  408.             for p in "${patch_dir}"/*.patch; do
  409.                 echo "Applying patch: ${p}"
  410.                 patch -d.. -Np0 < "${p}"
  411.             done
  412.         fi
  413.     fi
  415.     makefile="${mk_dir}/main.mk"
  416.     if test -e "${makefile}"; then
  417.         cp "${makefile}" Makefile
  418.     fi
  420.     if test -d "${files_dir}"; then
  421.         cp --no-preserve=mode "${files_dir}"/* "${PWD}/"
  422.     fi
  423. }
  425. # Default function for configuring source.
  426. default_src_configure() {
  427.     :
  428. }
  430. # Default function for compiling source. It simply runs make without any parameters.
  431. default_src_compile() {
  432.     make "${MAKEJOBS}" -f Makefile PREFIX="${PREFIX}"
  433. }
  435. # Default installing function. PREFIX should be set by run.sh script.
  436. # Note that upstream makefiles might ignore PREFIX and have to be configured in configure stage.
  437. default_src_install() {
  438.     make -f Makefile install PREFIX="${PREFIX}" DESTDIR="${DESTDIR}"
  439. }
  441. # Default function for postprocessing binaries.
  442. default_src_postprocess() {
  443.     if (command -v find && command -v file && command -v strip) >/dev/null 2>&1; then
  444.         # Logic largely taken from void linux 06-strip-and-debug-pkgs.sh
  445.         # shellcheck disable=SC2162
  446.         find "${DESTDIR}" -type f | while read f; do
  447.             case "$(file -bi "${f}")" in
  448.                 application/x-executable*) strip "${f}" ;;
  449.                 application/x-sharedlib*|application/x-pie-executable*)
  450.                     machine_set="$(file -b "${f}")"
  451.                     case "${machine_set}" in
  452.                         *no\ machine*) ;; # don't strip ELF container-only
  453.                         *) strip --strip-unneeded "${f}" ;;
  454.                     esac
  455.                     ;;
  456.                 application/x-archive*) strip --strip-debug "${f}" ;;
  457.             esac
  458.         done
  459.     fi
  460. }
  462. src_pkg() {
  463.     touch -t 197001010000.00 .
  464.     reset_timestamp
  466.     local tar_basename="${pkg}_${revision}.tar"
  467.     local dest_tar="/external/repo/${tar_basename}"
  468.     local filelist=/tmp/filelist.txt
  470.     cd /external/repo
  471.     # If grep is unavailable, then tar --sort is unavailable.
  472.     # So this does not need a command -v grep.
  473.     if tar --help | grep ' \-\-sort' >/dev/null 2>&1; then
  474.         tar -C "${DESTDIR}" --sort=name --hard-dereference \
  475.             --numeric-owner --owner=0 --group=0 --mode=go=rX,u+rw -cf "${dest_tar}" .
  476.     else
  477.         local olddir
  478.         olddir=$PWD
  479.         cd "${DESTDIR}"
  480.         local null
  481.         if command -v find >/dev/null 2>&1 && command -v sort >/dev/null 2>&1; then
  482.             find . -print0 | LC_ALL=C sort -z > "${filelist}"
  483.             null="--null"
  484.         elif command -v sort >/dev/null 2>&1; then
  485.             get_files .  | LC_ALL=C sort > "${filelist}"
  486.         else
  487.             get_files . > ${filelist}
  488.         fi
  489.         tar --no-recursion ${null} --files-from "${filelist}" \
  490.                 --numeric-owner --owner=0 --group=0 --mode=go=rX,u+rw -cf "${dest_tar}"
  491.         rm -f "$filelist"
  492.         cd "$olddir"
  493.     fi
  494.     touch -t 197001010000.00 "${tar_basename}"
  495.     bzip2 --best "${tar_basename}"
  496. }
  498. src_checksum() {
  499.     local pkg=$1 revision=$2
  500.     local rval=0
  501.     if ! [ "$UPDATE_CHECKSUMS" = True ] ; then
  502.         # We avoid using pipes as that is not supported by initial sha256sum from mescc-tools-extra
  503.         local checksum_file=/tmp/checksum
  504.         _grep "${pkg}_${revision}" "${SRCDIR}/SHA256SUMS.pkgs" > "${checksum_file}" || true
  505.         # Check there is something in checksum_file
  506.         if ! [ -s "${checksum_file}" ]; then
  507.             echo "${pkg}: no checksum stored!"
  508.             false
  509.         fi
  510.         echo "${pkg}: checksumming created package."
  511.         sha256sum -c "${checksum_file}" || rval=$?
  512.         rm "${checksum_file}"
  513.     fi
  514.     return "${rval}"
  515. }
  517. src_apply() {
  518.     local pkg="${1}" revision="${2}"
  519.     local TAR_PREFIX BZIP2_PREFIX
  521.     # Make sure we have at least one copy of tar
  522.     if [[ "${pkg}" == tar-* ]]; then
  523.         mkdir -p /tmp
  524.         cp "${PREFIX}/bin/tar" "/tmp/tar"
  525.         TAR_PREFIX="/tmp/"
  526.     fi
  528.     # Bash does not like to be overwritten
  529.     if [[ "${pkg}" == bash-* ]]; then
  530.         rm "${PREFIX}/bin/bash"
  531.     fi
  533.     # Overwriting files is mega busted, so do it manually
  534.     # shellcheck disable=SC2162
  535.     if [ -e /tmp/filelist.txt ]; then
  536.         while IFS= read -d $'\0' file; do
  537.             rm -f "/${file}" >/dev/null 2>&1 || true
  538.         done < /tmp/filelist.txt
  539.     fi
  541.     # Bzip2 does not like to be overwritten
  542.     if [[ "${pkg}" == bzip2-* ]]; then
  543.         mkdir -p /tmp
  544.         mv "${PREFIX}/bin/bzip2" "/tmp/bzip2"
  545.         BZIP2_PREFIX="/tmp/"
  546.     fi
  547.     "${BZIP2_PREFIX}bzip2" -dc "/external/repo/${pkg}_${revision}.tar.bz2" | \
  548.         "${TAR_PREFIX}tar" -C / -xpf -
  549.     rm -f "/tmp/bzip2" "/tmp/tar"
  550. }
  552. # Check if bash function exists
  553. fn_exists() {
  554.     test "$(type -t "$1")" == 'function'
  555. }
  557. # Call package specific function or default implementation.
  558. call() {
  559.     if fn_exists "$1"; then
  560.         $1
  561.     else
  562.         default_"${1}"
  563.     fi
  564. }
  566. # Call default build stage function
  567. default() {
  568.     "default_${build_stage}"
  569. }