From 57f6f2ba8020633a8bc35e9ff5facbbab75fe6d6 Mon Sep 17 00:00:00 2001
From: Matthew Lear <matthew.lear@raspberrypi.com>
Date: Tue, 4 Nov 2025 12:36:36 +0000
Subject: [PATCH 1/4] build: New util for verified file fetch and cache support

vfetch added to perform metadata-backed file get+verify operations,
storing the asset in a cache directory where it can be retrieved
transparently on subsequent invocations.
Add new global IG top level cache dir keyed off workroot.
---
 bin/vfetch                     | 159 +++++++++++++++++++++++++++++++++
 layer/base/sys-build-base.yaml |  22 +++--
 2 files changed, 173 insertions(+), 8 deletions(-)
 create mode 100755 bin/vfetch

diff --git a/bin/vfetch b/bin/vfetch
new file mode 100755
index 00000000..442a026b
--- /dev/null
+++ b/bin/vfetch
@@ -0,0 +1,159 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# File fetch with verification and cache support
+# Usage:
+#   [IGconf_sys_cachedir=/path/to/cache] <script> <metadata.file> </path/to/output.file>
+#
+# Metadata - mirrors supported, use consistent name:
+#   <name> <algo:hex> <url or local path>
+#   algo: sha256|sha256sum|sha512|sha512sum
+
+
+msg() {
+   echo -e "$*"
+}
+
+err (){
+   >&2 msg "Error: $*"
+}
+
+die (){
+   [[ -n "$*" ]] && err "$*"
+   exit 1
+}
+
+
+meta="${1:-}"; dest="${2:-}"
+[[ -n "$meta" && -n "$dest" ]] || die "Expected args: <metadata.file> </path/to/output.file>"
+
+
+# Enforce that dest is a file path, not a directory
+if [[ -d "$dest" || "$dest" == */ ]]; then
+   die "Output path must be a file, not a directory"
+fi
+
+
+# Default cache location
+cache_dir="${IGconf_sys_cachedir:-/tmp}"
+mkdir -p -- "$cache_dir"
+
+
+# Output path is always an explicit file path
+out_path="$dest"
+out_dir="$(dirname -- "$out_path")"
+mkdir -p -- "$out_dir"
+
+
+# digest <algo> <file> -> hex
+digest()
+{
+   case "$1" in
+      sha256|sha256sum) sha256sum "$2" | sed 's/[[:space:]].*$//' ;;
+      sha512|sha512sum) sha512sum "$2" | sed 's/[[:space:]].*$//' ;;
+      *) die "Unsupported algo: $1" ;;
+   esac
+}
+
+
+# atomic file copy <src> <dst>
+atomic_install_copy()
+{
+   local src="$1" dst="$2" dstdir tmp
+   dstdir="$(dirname -- "$dst")"
+   tmp="$(mktemp -p "$dstdir" ".fetch.install.XXXXXX")"
+   cp -f -- "$src" "$tmp"
+   mv -f -- "$tmp" "$dst"
+}
+
+
+# Two-pass approach:
+# pass 1 tries cache against any candidate digest
+# pass 2 downloads per candidate
+target_name=""
+
+
+# Pass 1: determine target, verify cache against all candidate digests
+cache_path=""
+
+while IFS= read -r line || [[ -n "$line" ]]; do
+   [[ "$line" =~ ^[[:space:]]*$ ]] && continue
+   [[ "$line" =~ ^[[:space:]]*# ]] && continue
+
+   set -- $line
+   name="${1:-}"; algo_hex="${2:-}"; src="${3:-}"
+   [[ -n "$name" && -n "$algo_hex" && -n "$src" ]] || continue
+
+   if [[ -z "$target_name" ]]; then
+      target_name="$name"
+      cache_path="${cache_dir}/${target_name}"
+   fi
+   [[ "$name" == "$target_name" ]] || continue
+
+   algo="${algo_hex%%:*}"
+   expected="${algo_hex#*:}"
+   expected="${expected,,}"
+   if [[ "$algo" == "$expected" || -z "$expected" ]]; then
+      die "Malformed metadata for $target_name: expected '<algo>:<hex>'"
+   fi
+
+   if [[ -f "$cache_path" ]]; then
+      got="$(digest "$algo" "$cache_path")"
+      if [[ "${got,,}" == "$expected" ]]; then
+         atomic_install_copy "$cache_path" "$out_path"
+         msg "Using verified cache: $cache_path -> $out_path"
+         exit 0
+      fi
+   fi
+done < "${meta}"
+
+[[ -n "${target_name}" ]] || die "No metadata entries in $meta"
+
+
+# Pass 2: download each candidate and verify with its own expected digest
+while IFS= read -r line || [[ -n "${line}" ]]; do
+   [[ "${line}" =~ ^[[:space:]]*$ ]] && continue
+   [[ "${line}" =~ ^[[:space:]]*# ]] && continue
+
+   set -- $line
+   name="${1:-}"; algo_hex="${2:-}"; src="${3:-}"
+   [[ -n "$name" && -n "$algo_hex" && -n "$src" ]] || continue
+   [[ "$name" == "$target_name" ]] || continue
+
+   algo="${algo_hex%%:*}"
+   expected="${algo_hex#*:}"
+   expected="${expected,,}"
+   if [[ "$algo" == "$expected" || -z "$expected" ]]; then
+      die "Malformed metadata for $target_name: expected '<algo>:<hex>'"
+   fi
+
+   tmp_cache="$(mktemp -p "$cache_dir" ".fetch.${target_name}.XXXXXX")"
+   if [[ "$src" == http://* || "$src" == https://* ]]; then
+      if ! curl --fail --location --silent --show-error --retry 3 --retry-delay 2 \
+         --connect-timeout 30 --output "$tmp_cache" "$src"; then
+         rm -f -- "$tmp_cache"
+         continue
+      fi
+   else
+      if ! cp -f -- "$src" "$tmp_cache"; then
+         rm -f -- "$tmp_cache"
+         continue
+      fi
+   fi
+
+   got="$(digest "$algo" "$tmp_cache")"
+   if [[ "${got,,}" != "$expected" ]]; then
+      err "Checksum mismatch for ${target_name} from ${src}"
+      rm -f -- "${tmp_cache}"
+      continue
+   fi
+
+   atomic_install_copy "$tmp_cache" "$out_path"
+   mv -f -- "$tmp_cache" "$cache_path"
+   msg "Fetched: $src -> $out_path cached at $cache_path"
+   exit 0
+done < "${meta}"
+
+
+die "Failed: no verified source succeeded for ${target_name}"
diff --git a/layer/base/sys-build-base.yaml b/layer/base/sys-build-base.yaml
index bbbbc4c4..0d83f08e 100644
--- a/layer/base/sys-build-base.yaml
+++ b/layer/base/sys-build-base.yaml
@@ -39,6 +39,20 @@
 # X-Env-Var-apt_keydir-Valid: string-or-empty
 # X-Env-Var-apt_keydir-Set: y
 #
+# X-Env-Var-workroot: $(realpath --canonicalize-missing ./work)
+# X-Env-Var-workroot-Desc: The root work directory. Depending on the system(s)
+#  being built, this directory can amount to a substantial amount of consumed
+#  disk space.
+# X-Env-Var-workroot-Required: n
+# X-Env-Var-workroot-Valid: string
+# X-Env-Var-workroot-Set: y
+#
+# X-Env-Var-cachedir: ${IGconf_sys_workroot}/cache
+# X-Env-Var-cachedir-Desc: Global cache root directory for all operations.
+# X-Env-Var-cachedir-Required: n
+# X-Env-Var-cachedir-Valid: string
+# X-Env-Var-cachedir-Set: y
+#
 # X-Env-Var-apt_cachedir:
 # X-Env-Var-apt_cachedir-Desc: Cache directory for APT operations.
 #  Use this to specify the root directory APT uses when downloading packages.
@@ -51,13 +65,5 @@
 # X-Env-Var-apt_cachedir-Valid: string-or-empty
 # X-Env-Var-apt_cachedir-Set: force
 #
-# X-Env-Var-workroot: $(realpath --canonicalize-missing ./work)
-# X-Env-Var-workroot-Desc: The root work directory. Depending on the system(s)
-#  being built, this directory can amount to a substantial amount of consumed
-#  disk space.
-# X-Env-Var-workroot-Required: n
-# X-Env-Var-workroot-Valid: string
-# X-Env-Var-workroot-Set: y
-#
 # METAEND
 ---

From 3b7a014be95df4a4c18b1ac48661c7622e70fc96 Mon Sep 17 00:00:00 2001
From: Matthew Lear <matthew.lear@raspberrypi.com>
Date: Wed, 5 Nov 2025 15:37:55 +0000
Subject: [PATCH 2/4] core: Improve handling of build host pkgs pt1

To decouple from reliance on distro-provided packages for the build
machine, introduce a new lightweight build framework for building
packages from source. rpi-image-gen can use this scheme at bootstrap
to ensure it always has required versions of mission critical
programs. Build bdebstrap and genimage using this scheme, installing
them to a private sysroot where they can be used for all builds.

The trade off is a short on-off delay on the initial rpi-image-gen
invocation plus additional distro deps. Longer term, distro dependencies
will be reduced signifficantly by introducing the ability to run builds
in a container.

Fixes: #113
---
 README.adoc                                   |   7 +-
 depends                                       |  14 +-
 docs/layer/sys-build-base.html                |  42 +++-
 docs/provisioning/index.adoc                  |   2 +-
 docs/provisioning/index.html                  |   2 +-
 layer/base/sys-build-base.yaml                |   6 +
 layer/sbom/gen.sh                             |   2 +-
 lib/tools.sh                                  |  47 ++++
 package/Makefile                              |  39 ++++
 package/bdebstrap/Makefile                    |   4 +
 package/bdebstrap/bdebstrap-0.7.0.meta        |   1 +
 .../patches/0.7.0/001-disable-pandoc.patch    |  52 +++++
 .../0.7.0/002-data-files-installation.patch   |  11 +
 package/genimage/Makefile                     |   5 +
 package/genimage/genimage-19.meta             |   1 +
 package/libconfuse/Makefile                   |   4 +
 package/libconfuse/libconfuse-3.3.meta        |   1 +
 package/shared/depgen.py                      | 164 ++++++++++++++
 package/shared/pkg-env.mk                     |  13 ++
 package/shared/pkg.mk                         | 207 ++++++++++++++++++
 rpi-image-gen                                 |  22 +-
 21 files changed, 626 insertions(+), 20 deletions(-)
 create mode 100644 lib/tools.sh
 create mode 100644 package/Makefile
 create mode 100644 package/bdebstrap/Makefile
 create mode 100644 package/bdebstrap/bdebstrap-0.7.0.meta
 create mode 100644 package/bdebstrap/patches/0.7.0/001-disable-pandoc.patch
 create mode 100644 package/bdebstrap/patches/0.7.0/002-data-files-installation.patch
 create mode 100644 package/genimage/Makefile
 create mode 100644 package/genimage/genimage-19.meta
 create mode 100644 package/libconfuse/Makefile
 create mode 100644 package/libconfuse/libconfuse-3.3.meta
 create mode 100755 package/shared/depgen.py
 create mode 100644 package/shared/pkg-env.mk
 create mode 100644 package/shared/pkg.mk

diff --git a/README.adoc b/README.adoc
index 5509e2c8..83da031b 100644
--- a/README.adoc
+++ b/README.adoc
@@ -70,11 +70,11 @@ sudo ./install_deps.sh
 
 See the `depends` file for the complete list of required packages.
 
-=== Important
+rpi-image-gen builds its own copies of critical host tools so it always has current, fully featured versions and supports architectures the workstation distribution might not package. Installing and running them from a private sysroot keeps the host system untouched. The trade-off is a small one-time build delay, plus additional native dependencies that most developer workstations already carry. The resulting tools live in a reusable sysroot that stays isolated from distribution packages and can be reused for every build.
 
-rpi-image-gen has been developed on Raspberry Pi OS which, at the time of writing, is Debian Bookworm arm64. It heavily favours Debian-based systems and will run on non-arm64 platforms (such as x86_64) via QEMU emulation or inside container environments. However, there is currently no formal support for these non-native environments.
+=== Important
 
-If utilising sparse images in your workflow, e.g. with *rpi-sb-provisioner* (https://github.com/raspberrypi/rpi-sb-provisioner), please refer to the link:docs/provisioning/index.adoc[provisioning documentation] for important advice.
+rpi-image-gen is developed on Raspberry Pi OS and supports builds on Debian Bookworm and Trixie arm64 systems. It expects a Debian-based host and will run on non-arm64 platforms (such as x86_64) via QEMU emulation or in container environments, although those non-native setups aren’t formally supported.
 
 == Documentation
 
@@ -98,6 +98,7 @@ See the `examples/` directory for tips and help
 * `layer/` - Layer library
 * `layer-hooks/` - Common hooks used by layer library
 * `lib/` - Execution helpers and macros, eg CLI handling, reusable constructs
+* `package/` - Build recipes and framework for tools
 * `scripts/` - Dedicated functional hooks, eg for bdebstrap
 * `site/` - Core Python engine classes
 * `templates/` - Templating assets, eg doc generation
diff --git a/depends b/depends
index 29c21eee..c1234023 100644
--- a/depends
+++ b/depends
@@ -5,10 +5,8 @@ mke2fs:e2fsprogs
 grep
 rsync
 curl
-genimage
 mtools
 mmdebstrap
-bdebstrap
 podman
 zstd
 pv
@@ -22,6 +20,18 @@ fdisk
 python3-yaml
 python3-debian
 python3-jsonschema
+# pkg buildsys
+python3-pip
+dpkg-dev
+make
+build-essential
+autoconf
+automake
+libtool
+autopoint
+flex
+gettext
+pkg-config
 # doc gen only
 # python3-markdown
 # asciidoctor
diff --git a/docs/layer/sys-build-base.html b/docs/layer/sys-build-base.html
index 62741629..4b5a99d2 100644
--- a/docs/layer/sys-build-base.html
+++ b/docs/layer/sys-build-base.html
@@ -233,6 +233,40 @@ <h2>Configuration Variables</h2>
                     </td>
                 </tr>
                 
+                <tr>
+                    <td><code>IGconf_sys_workroot</code></td>
+                    <td>The root work directory. Depending on the system(s)
+ being built, this directory can amount to a substantial amount of consumed
+ disk space.</td>
+                    <td>
+                       
+                           
+                           <code class="long-default">$(realpath --canonicalize-missing ./work)</code>
+                           
+                       
+                    </td>
+                    <td>Non-empty string value</td>
+                    <td>
+                        <a href="variable-validation.html#set-policies" class="badge policy-immediate" title="Click for policy and validation help">immediate</a>
+                    </td>
+                </tr>
+                
+                <tr>
+                    <td><code>IGconf_sys_cachedir</code></td>
+                    <td>Global cache root directory for all operations.</td>
+                    <td>
+                       
+                           
+                           <code class="long-default">${IGconf_sys_workroot}/cache</code>
+                           
+                       
+                    </td>
+                    <td>Non-empty string value</td>
+                    <td>
+                        <a href="variable-validation.html#set-policies" class="badge policy-immediate" title="Click for policy and validation help">immediate</a>
+                    </td>
+                </tr>
+                
                 <tr>
                     <td><code>IGconf_sys_apt_cachedir</code></td>
                     <td>Cache directory for APT operations.
@@ -254,14 +288,12 @@ <h2>Configuration Variables</h2>
                 </tr>
                 
                 <tr>
-                    <td><code>IGconf_sys_workroot</code></td>
-                    <td>The root work directory. Depending on the system(s)
- being built, this directory can amount to a substantial amount of consumed
- disk space.</td>
+                    <td><code>IGconf_sys_buildroot</code></td>
+                    <td>Global build root directory for source builds.</td>
                     <td>
                        
                            
-                           <code class="long-default">$(realpath --canonicalize-missing ./work)</code>
+                           <code class="long-default">${IGconf_sys_workroot}/build</code>
                            
                        
                     </td>
diff --git a/docs/provisioning/index.adoc b/docs/provisioning/index.adoc
index 5be323ca..10d7589a 100644
--- a/docs/provisioning/index.adoc
+++ b/docs/provisioning/index.adoc
@@ -46,7 +46,7 @@ IDP requires fastboot to be installed in the client. On Debian systems, this can
 
 [WARNING]
 ====
-Older versions of `genimage` can be problematic when creating sparse images. For example, `genimage` v16 currently shipping in Debian Bookworm does not create usable sparse images of _vfat_ filesystems. Raspberry Pi provisioning tools such as `rpi-sb-provisioner` (https://github.com/raspberrypi/rpi-sb-provisioner) rely on sparse image format, so it is recommended to use the most up-to-date version of `genimage` as possible (e.g. manual build and install to `/usr/local/bin/genimage`).
+Raspberry Pi provisioning tools such as `rpi-sb-provisioner` (https://github.com/raspberrypi/rpi-sb-provisioner) rely on sparse images, so rpi-image-gen ships its own up-to-date `genimage` to avoid compatibility problems seen with older distribution versions.
 ====
 
 
diff --git a/docs/provisioning/index.html b/docs/provisioning/index.html
index 57b983b2..7436ca75 100644
--- a/docs/provisioning/index.html
+++ b/docs/provisioning/index.html
@@ -235,7 +235,7 @@ <h3 id="_dependencies"><a class="anchor" href="#_dependencies"></a><a class="lin
 </td>
 <td class="content">
 <div class="paragraph">
-<p>Older versions of <code>genimage</code> can be problematic when creating sparse images. For example, <code>genimage</code> v16 currently shipping in Debian Bookworm does not create usable sparse images of <em>vfat</em> filesystems. Raspberry Pi provisioning tools such as <code>rpi-sb-provisioner</code> (<a href="https://github.com/raspberrypi/rpi-sb-provisioner" class="bare">https://github.com/raspberrypi/rpi-sb-provisioner</a>) rely on sparse image format, so it is recommended to use the most up-to-date version of <code>genimage</code> as possible (e.g. manual build and install to <code>/usr/local/bin/genimage</code>).</p>
+<p>Raspberry Pi provisioning tools such as <code>rpi-sb-provisioner</code> (<a href="https://github.com/raspberrypi/rpi-sb-provisioner" class="bare">https://github.com/raspberrypi/rpi-sb-provisioner</a>) rely on sparse images, so rpi-image-gen ships its own up-to-date <code>genimage</code> to avoid compatibility problems seen with older distribution versions.</p>
 </div>
 </td>
 </tr>
diff --git a/layer/base/sys-build-base.yaml b/layer/base/sys-build-base.yaml
index 0d83f08e..73cbb7bb 100644
--- a/layer/base/sys-build-base.yaml
+++ b/layer/base/sys-build-base.yaml
@@ -65,5 +65,11 @@
 # X-Env-Var-apt_cachedir-Valid: string-or-empty
 # X-Env-Var-apt_cachedir-Set: force
 #
+# X-Env-Var-buildroot: ${IGconf_sys_workroot}/build
+# X-Env-Var-buildroot-Desc: Global build root directory for source builds.
+# X-Env-Var-buildroot-Required: n
+# X-Env-Var-buildroot-Valid: string
+# X-Env-Var-buildroot-Set: y
+#
 # METAEND
 ---
diff --git a/layer/sbom/gen.sh b/layer/sbom/gen.sh
index 3eabb15f..bff7af51 100755
--- a/layer/sbom/gen.sh
+++ b/layer/sbom/gen.sh
@@ -17,7 +17,7 @@ SYFT_VER=v1.32.0
 # If host has syft, use it
 if ! hash syft 2>/dev/null; then
    curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh \
-      | sh -s -- -b "${IGconf_sys_workroot}"/host/bin "${SYFT_VER}"
+      | sh -s -- -b "${IGconf_sys_workroot}/${DEB_BUILD_GNU_TYPE}/usr/bin" "${SYFT_VER}"
 fi
 
 SYFT=$(syft --version 2>/dev/null) || die "syft is unusable"
diff --git a/lib/tools.sh b/lib/tools.sh
new file mode 100644
index 00000000..ca86a839
--- /dev/null
+++ b/lib/tools.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+bootstrap_build_tools() {
+   : "${ctx[FINALENV]?missing ctx[FINALENV]}"
+   : "${ctx[EXEC_PATH]?missing ctx[EXEC_PATH]}"
+   : "${ctx[PYTHON_PATH]?missing ctx[PYTHON_PATH]}"
+   : "${IGconf_sys_workroot?missing IGconf_sys_workroot}
+   : "${DEB_BUILD_GNU_TYPE?missing DEB_BUILD_GNU_TYPE}
+
+   # Mission critical
+   local tools=(bdebstrap)
+
+   if value=$(get_var IGconf_image_provider "${ctx[FINALENV]}") \
+      && [[ $value == genimage ]]; then
+         tools+=(genimage)
+   fi
+
+   # Build and install tools to this location
+   local destdir=${IGconf_sys_workroot}/${DEB_BUILD_GNU_TYPE}
+   local prefix=/usr
+
+   runenv "${ctx[FINALENV]}" \
+      make -s -j$(nproc) -C ${IGTOP}/package "${tools[@]}" \
+      PKG_DESTDIR=$destdir PKG_PREFIX=$prefix
+
+   # Fixup paths to ensure installed tools run transparently. There are many
+   # caveats with this approach, and it risks env leakage. Leveraging a
+   # containerised build vehicle approach is preferred.
+   local paths=(
+      "${destdir}${prefix}/local/bin"\
+      "${destdir}${prefix}/bin"\
+      "${ctx[EXEC_PATH]}"
+      )
+   ctx[EXEC_PATH]="$(IFS=:; echo "${paths[*]}")"
+   PATH="${ctx[EXEC_PATH]}"
+   export PATH
+
+   local pyver="$(python3 -c 'import sysconfig; print("python"+sysconfig.get_python_version())')"
+   paths=(\
+      "${destdir}${prefix}/local/lib/${pyver}/dist-packages"\
+      "${destdir}${prefix}/lib/${pyver}/dist-packages"\
+      "${ctx[PYTHON_PATH]}"
+   )
+   ctx[PYTHON_PATH]="$(IFS=:; echo "${paths[*]}")"
+   PYTHONPATH="${ctx[PYTHON_PATH]}"
+   export PYTHONPATH
+}
diff --git a/package/Makefile b/package/Makefile
new file mode 100644
index 00000000..9969c057
--- /dev/null
+++ b/package/Makefile
@@ -0,0 +1,39 @@
+$(if $(IGconf_sys_buildroot),, $(error IGconf_sys_buildroot is not defined))
+
+PKG_ROOT := $(abspath $(CURDIR))
+PKG_ENV  := $(abspath $(PKG_ROOT)/shared/pkg-env.mk)
+DEPGEN   := $(abspath $(PKG_ROOT)/shared/depgen.py)
+
+PACKAGE_MAKEFILES := $(wildcard $(PKG_ROOT)/*/Makefile)
+KNOWN_PKGS := $(sort $(patsubst $(PKG_ROOT)/%/Makefile,%,$(PACKAGE_MAKEFILES)))
+
+ifeq ($(strip $(MAKECMDGOALS)),)
+  $(error No packages specified)
+endif
+
+REQUESTED_PKGS := $(filter $(KNOWN_PKGS),$(MAKECMDGOALS))
+PACKAGE_GOALS := $(sort $(REQUESTED_PKGS))
+
+DEP_DIR  := $(IGconf_sys_buildroot)/pkgdeps
+DEPFILES := $(PACKAGE_GOALS:%=$(DEP_DIR)/%.deps.mk)
+
+ALL_PACKAGE_STAMPS :=
+
+.PHONY: all clean $(KNOWN_PKGS)
+
+-include $(DEPFILES)
+
+all: $(ALL_PACKAGE_STAMPS)
+
+$(DEP_DIR)/%.deps.mk: $(PKG_ROOT)/%/Makefile $(PKG_ENV) $(DEPGEN)
+	@mkdir -p "$(DEP_DIR)"
+	@python3 "$(DEPGEN)" \
+		--package "$*" \
+		--makefile "$<" \
+		--pkg-dir "$(PKG_ROOT)/$*" \
+		--pkg-env "$(PKG_ENV)" \
+		--out "$@"
+
+clean-%:
+	@rm -f "$(DEP_DIR)/$*.deps.mk"
+	@rm -rf "$(IGconf_sys_buildroot)/$*"
diff --git a/package/bdebstrap/Makefile b/package/bdebstrap/Makefile
new file mode 100644
index 00000000..0982a057
--- /dev/null
+++ b/package/bdebstrap/Makefile
@@ -0,0 +1,4 @@
+PKG_NAME := bdebstrap
+PKG_VER := 0.7.0
+PKG_BUILD_SCHEME := pysrc
+include ../shared/pkg.mk
diff --git a/package/bdebstrap/bdebstrap-0.7.0.meta b/package/bdebstrap/bdebstrap-0.7.0.meta
new file mode 100644
index 00000000..e6be6f1b
--- /dev/null
+++ b/package/bdebstrap/bdebstrap-0.7.0.meta
@@ -0,0 +1 @@
+bdebstrap-0.7.0.tar.gz sha256:c8675f637af16a4471c8f61627895d21e56ef5d4bd13fa2a18cd01513395a009 https://github.com/bdrung/bdebstrap/archive/refs/tags/v0.7.0.tar.gz
diff --git a/package/bdebstrap/patches/0.7.0/001-disable-pandoc.patch b/package/bdebstrap/patches/0.7.0/001-disable-pandoc.patch
new file mode 100644
index 00000000..515a4280
--- /dev/null
+++ b/package/bdebstrap/patches/0.7.0/001-disable-pandoc.patch
@@ -0,0 +1,52 @@
+diff --git a/setup.py b/setup.py
+index cb96612..4370eb5 100755
+--- a/setup.py
++++ b/setup.py
+@@ -33,6 +33,15 @@ HOOKS = ["hooks/disable-units", "hooks/enable-units"]
+ MAN_PAGES = ["bdebstrap.1"]
+ 
+ 
++STUB_MAN = """\
++.TH {name} 1 "" "bdebstrap" "User Commands"
++.SH NAME
++{name} \\- stub manpage
++.SH DESCRIPTION
++This man page was generated during build because pandoc was not available.
++See the project documentation for details.
++"""
++
+ class DocCommand(Command):
+     """A custom command to build the documentation using pandoc."""
+ 
+@@ -47,12 +56,27 @@ class DocCommand(Command):
+ 
+     # pylint: disable-next=no-self-use
+     def run(self) -> None:
+-        """Run pandoc."""
+         logger = logging.getLogger(__name__)
++
++        def have_pandoc() -> bool:
++            try:
++                subprocess.run(["pandoc", "-v"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False)
++                return True
++            except FileNotFoundError:
++                return False
++
++        use_pandoc = have_pandoc()
++
+         for man_page in MAN_PAGES:
+-            command = ["pandoc", "-s", "-t", "man", man_page + ".md", "-o", man_page]
+-            logger.info("running command: %s", " ".join(command))
+-            subprocess.check_call(command)
++            stem = man_page.rsplit(".", 1)[0]
++            md = man_page + ".md"
++            if use_pandoc and os.path.exists(md):
++                cmd = ["pandoc", "-s", "-t", "man", md, "-o", man_page]
++                logger.info("running command: %s", " ".join(cmd))
++                subprocess.check_call(cmd)
++            else:
++                with open(man_page, "w", encoding="utf-8") as f:
++                    f.write(STUB_MAN.format(name=stem))
+ 
+ 
+ # pylint: disable-next=too-few-public-methods
diff --git a/package/bdebstrap/patches/0.7.0/002-data-files-installation.patch b/package/bdebstrap/patches/0.7.0/002-data-files-installation.patch
new file mode 100644
index 00000000..eb89f387
--- /dev/null
+++ b/package/bdebstrap/patches/0.7.0/002-data-files-installation.patch
@@ -0,0 +1,11 @@
+diff --git a/setup.py b/setup.py
+index 4370eb5..8ab7092 100755
+--- a/setup.py
++++ b/setup.py
+@@ -143,5 +143,5 @@ if __name__ == "__main__":
+         install_requires=["ruamel.yaml"],
+         scripts=["bdebstrap"],
+         py_modules=[],
+-        data_files=[("/usr/share/man/man1", MAN_PAGES), ("/usr/share/bdebstrap/hooks", HOOKS)],
++        data_files=[("share/man/man1", MAN_PAGES), ("share/bdebstrap/hooks", HOOKS)],
+     )
diff --git a/package/genimage/Makefile b/package/genimage/Makefile
new file mode 100644
index 00000000..60582e90
--- /dev/null
+++ b/package/genimage/Makefile
@@ -0,0 +1,5 @@
+PKG_NAME := genimage
+PKG_VER := 19
+PKG_BUILD_SCHEME := autotools
+PKG_DEPS := libconfuse
+include ../shared/pkg.mk
diff --git a/package/genimage/genimage-19.meta b/package/genimage/genimage-19.meta
new file mode 100644
index 00000000..d3a5db74
--- /dev/null
+++ b/package/genimage/genimage-19.meta
@@ -0,0 +1 @@
+genimage-19.tar.gz sha256:f620b1cd0299c12a17f523d29cdad44075d1c87c47020bb63a98bf51901d8db5 https://github.com/pengutronix/genimage/archive/refs/tags/v19.tar.gz
diff --git a/package/libconfuse/Makefile b/package/libconfuse/Makefile
new file mode 100644
index 00000000..f025ff0b
--- /dev/null
+++ b/package/libconfuse/Makefile
@@ -0,0 +1,4 @@
+PKG_NAME := libconfuse
+PKG_VER := 3.3
+PKG_BUILD_SCHEME := autotools
+include ../shared/pkg.mk
diff --git a/package/libconfuse/libconfuse-3.3.meta b/package/libconfuse/libconfuse-3.3.meta
new file mode 100644
index 00000000..211b0c2d
--- /dev/null
+++ b/package/libconfuse/libconfuse-3.3.meta
@@ -0,0 +1 @@
+libconfuse-3.3.tar.gz sha256:cb90c06f2dbec971792af576d5b9a382fb3c4ca2b1deea55ea262b403f4e641e https://github.com/libconfuse/libconfuse/archive/refs/tags/v3.3.tar.gz
diff --git a/package/shared/depgen.py b/package/shared/depgen.py
new file mode 100755
index 00000000..ab8494f7
--- /dev/null
+++ b/package/shared/depgen.py
@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+
+# Generate a Make fragment to describe package's dependencies.
+# This alows Make to build the dependency graph for a package.
+
+from __future__ import annotations
+
+import argparse
+import pathlib
+import re
+
+
+ASSIGN_RE = re.compile(r"^\s*PKG_(NAME|VER|DEPS)\s*(?::=|\+=|=)\s*(.*)$")
+RESET_VARS = (
+    "PKG_NAME",
+    "PKG_VER",
+    "PKG_DEPS",
+    "PKG_WORK_DIR",
+    "PKG_SOURCE_STAMP",
+    "PKG_PATCH_STAMP",
+    "PKG_BUILD_STAMP",
+    "PKG_INSTALL_STAMP",
+)
+
+
+def sanitise(name: str) -> str:
+    return re.sub(r"[^A-Za-z0-9_]+", "_", name)
+
+
+def parse_makefile(path: pathlib.Path) -> tuple[str | None, str | None, list[str]]:
+    text = path.read_text(encoding="utf-8")
+    buffer = ""
+    name: str | None = None
+    version: str | None = None
+    deps: list[str] = []
+
+    for raw in text.splitlines():
+        stripped = raw.split("#", 1)[0].rstrip()
+        if not stripped and not buffer:
+            continue
+
+        if stripped.endswith("\\"):
+            buffer += stripped[:-1] + " "
+            continue
+
+        if buffer:
+            stripped = buffer + stripped
+            buffer = ""
+
+        match = ASSIGN_RE.match(stripped)
+        if not match:
+            continue
+
+        key, rhs = match.groups()
+        rhs = rhs.strip()
+        if not rhs:
+            continue
+
+        if key == "DEPS":
+            deps.extend(rhs.split())
+        elif key == "NAME" and name is None:
+            name = rhs.split()[0]
+        elif key == "VER" and version is None:
+            version = rhs.split()[0]
+
+    ordered: list[str] = []
+    seen: set[str] = set()
+    for dep in deps:
+        if dep not in seen:
+            seen.add(dep)
+            ordered.append(dep)
+
+    return name, version, ordered
+
+
+def render(
+    *,
+    pkg_identifier: str,
+    pkg_name: str | None,
+    pkg_version: str | None,
+    deps: list[str],
+    pkg_dir: pathlib.Path,
+    pkg_env: pathlib.Path,
+) -> str:
+    actual_name = pkg_name or pkg_identifier
+    pkg_id = sanitise(actual_name)
+    dep_ids = [(dep, sanitise(dep)) for dep in deps]
+
+    lines: list[str] = []
+    lines.append("# Autogenerated")
+    lines.append("")
+    lines.append(f"PKG_NAME := {actual_name}")
+    if pkg_version:
+        lines.append(f"PKG_VER := {pkg_version}")
+    if deps:
+        lines.append(f"PKG_DEPS := {' '.join(deps)}")
+    lines.append(f"include {pkg_env}")
+    lines.append("")
+
+    lines.append(f"PKG_DIR_{pkg_id} := {pkg_dir}")
+    lines.append(f"PKG_STAMP_{pkg_id} := $(PKG_INSTALL_STAMP)")
+    if pkg_version:
+        lines.append(f"PKG_VER_{pkg_id} := $(PKG_VER)")
+    lines.append(f"PKG_DEPS_{pkg_id} := {' '.join(deps) if deps else ''}")
+    lines.append("")
+
+    if deps:
+        for dep in deps:
+            lines.append(f"-include $(DEP_DIR)/{dep}.deps.mk")
+        lines.append("")
+
+    lines.append(f"ALL_PACKAGE_STAMPS += $(PKG_STAMP_{pkg_id})")
+    lines.append("")
+
+    lines.append(f".PHONY: {actual_name}")
+    lines.append(f"{actual_name}: $(PKG_STAMP_{pkg_id})")
+    lines.append("")
+
+    if deps:
+        prereqs = " ".join(f"$(PKG_STAMP_{d_id})" for _, d_id in dep_ids)
+        lines.append(f"$(PKG_STAMP_{pkg_id}): {prereqs}")
+    else:
+        lines.append(f"$(PKG_STAMP_{pkg_id}):")
+    if pkg_version:
+        lines.append(f"\t@echo \"==> Build {actual_name} {pkg_version}\"")
+    else:
+        sys.stderr.write("no version\n")
+        sys.exit(1)
+    lines.append(f"\t@$(MAKE) -C $(PKG_DIR_{pkg_id}) --no-print-directory")
+    lines.append("")
+
+    for var in RESET_VARS:
+        lines.append(f"undefine {var}")
+    lines.append("")
+
+    return "\n".join(lines)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Generate a package depfile")
+    parser.add_argument("--package", required=True)
+    parser.add_argument("--makefile", required=True, type=pathlib.Path)
+    parser.add_argument("--pkg-dir", required=True, type=pathlib.Path)
+    parser.add_argument("--pkg-env", required=True, type=pathlib.Path)
+    parser.add_argument("--out", required=True, type=pathlib.Path)
+    args = parser.parse_args()
+
+    name, version, deps = parse_makefile(args.makefile)
+
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    content = render(
+        pkg_identifier=args.package,
+        pkg_name=name,
+        pkg_version=version,
+        deps=deps,
+        pkg_dir=args.pkg_dir.resolve(),
+        pkg_env=args.pkg_env.resolve(),
+    )
+    args.out.write_text(content + "\n", encoding="utf-8")
+
+
+if __name__ == "__main__":
+    main()
+
diff --git a/package/shared/pkg-env.mk b/package/shared/pkg-env.mk
new file mode 100644
index 00000000..0ce91879
--- /dev/null
+++ b/package/shared/pkg-env.mk
@@ -0,0 +1,13 @@
+$(if $(PKG_NAME),, $(error PKG_NAME must be specified))
+$(if $(PKG_VER),,  $(error PKG_VER must be specified))
+
+$(if $(IGconf_sys_buildroot),,$(error IGconf_sys_buildroot is not defined))
+
+PKG_DEPS ?=
+PKG_CTX ?=
+
+PKG_WORK_DIR := $(abspath $(IGconf_sys_buildroot)/$(PKG_CTX)/$(PKG_NAME))
+PKG_SOURCE_STAMP := $(PKG_WORK_DIR)/$(PKG_NAME)-$(PKG_VER).source
+PKG_PATCH_STAMP := $(PKG_WORK_DIR)/$(PKG_NAME)-$(PKG_VER).patch
+PKG_BUILD_STAMP := $(PKG_WORK_DIR)/$(PKG_NAME)-$(PKG_VER).build
+PKG_INSTALL_STAMP := $(PKG_WORK_DIR)/$(PKG_NAME)-$(PKG_VER).install
diff --git a/package/shared/pkg.mk b/package/shared/pkg.mk
new file mode 100644
index 00000000..3922168c
--- /dev/null
+++ b/package/shared/pkg.mk
@@ -0,0 +1,207 @@
+PKG_SHARED_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
+include $(PKG_SHARED_DIR)/pkg-env.mk
+
+DEB_BUILD_GNU_TYPE ?= $(shell dpkg-architecture -qDEB_BUILD_GNU_TYPE)
+DEB_HOST_GNU_TYPE  ?= $(shell dpkg-architecture -qDEB_HOST_GNU_TYPE)
+
+# Tools are built for build arch by default
+PKG_BUILD_GNU_TYPE := $(or $(strip $(PKG_BUILD_GNU_TYPE)),$(DEB_BUILD_GNU_TYPE))
+PKG_HOST_GNU_TYPE  := $(or $(strip $(PKG_HOST_GNU_TYPE)),$(DEB_BUILD_GNU_TYPE))
+
+PKG_DESTDIR         ?= $(abspath $(IGconf_sys_buildroot)/staging/$(PKG_HOST_GNU_TYPE))
+PKG_RUNTIME_DESTDIR ?= $(abspath $(IGconf_sys_buildroot)/host/$(PKG_HOST_GNU_TYPE))
+PKG_PREFIX          ?= /usr
+PKG_CACHE_ROOT      := $(abspath $(if $(strip $(IGconf_sys_cachedir)),$(IGconf_sys_cachedir),/tmp))
+
+PKG_INSTALL_PATH := $(abspath $(PKG_DESTDIR)/$(PKG_PREFIX))
+PKG_RUNTIME_PATH := $(abspath $(PKG_RUNTIME_DESTDIR)/$(PKG_PREFIX))
+
+THIS_PKG ?= $(CURDIR)
+
+PKG_META ?= $(PKG_NAME)-$(PKG_VER).meta
+PKG_META_PATH := $(abspath $(THIS_PKG)/$(PKG_META))
+ifeq (,$(wildcard $(PKG_META_PATH)))
+$(error $(PKG_META_PATH) not found)
+endif
+
+PKG_META_PATH := $(abspath $(CURDIR)/$(PKG_META))
+PKG_SOURCE_DIR ?= $(PKG_NAME)-$(PKG_VER)
+PKG_ARCHIVE ?= $(PKG_NAME)-$(PKG_VER).tar.gz
+PKG_SUBDIR ?= .
+
+PKG_ENVIRONMENT    ?=
+PKG_BUILD_SCHEME   ?=
+PKG_PATCH_DIR      ?= patches/$(PKG_VER)
+PKG_PATCHES        := $(sort $(wildcard $(abspath $(THIS_PKG)/$(PKG_PATCH_DIR))/*.patch))
+PKG_PATCH_STRIP    ?= 1
+PKG_UNPACK_STRIP   ?= 1
+PKG_CONFIGURE_OPTS ?=
+PKG_LIB_SHARED     ?= lib$(patsubst lib%,%,$(PKG_NAME))*.so*
+
+PKG_CPPFLAGS ?= -I$(PKG_INSTALL_PATH)/include
+PKG_CFLAGS   ?= -I$(PKG_INSTALL_PATH)/include
+PKG_CXXFLAGS ?= -I$(PKG_INSTALL_PATH)/include
+
+# We want relocatable execution, eg for container chroot add-ons
+# Can split for build/device if needed
+PKG_RPATH_REL := $$ORIGIN/../lib:$$ORIGIN/../lib/$(PKG_HOST_GNU_TYPE)
+PKG_LDFLAGS  := -L$(PKG_INSTALL_PATH)/lib -L$(PKG_INSTALL_PATH)/lib/$(PKG_HOST_GNU_TYPE) \
+	-Wl,-rpath,$(PKG_RPATH_REL) \
+	-Wl,-rpath-link,$(PKG_INSTALL_PATH)/lib \
+	-Wl,-rpath-link,$(PKG_INSTALL_PATH)/lib/$(PKG_HOST_GNU_TYPE)
+
+PKG_ARCHIVE_PATH := $(PKG_WORK_DIR)/$(PKG_ARCHIVE)
+PKG_SOURCE_ROOT := $(PKG_WORK_DIR)/$(PKG_SOURCE_DIR)
+PKG_SOURCE_PATH := $(PKG_SOURCE_ROOT)/$(PKG_SUBDIR)
+PKG_BUILD_PATH := $(PKG_SOURCE_ROOT)-build
+PKG_BUILD_LOG := $(PKG_WORK_DIR)/$(PKG_NAME)-$(PKG_VER).log
+
+
+.DEFAULT_GOAL := $(PKG_INSTALL_STAMP)
+
+
+# Command runner with logging
+define run
+	printf 'target:%s\n' "$@" >> $(PKG_BUILD_LOG)
+	printf 'run:%s\n' "$(1)" >> $(PKG_BUILD_LOG)
+	if $(1) >> $(PKG_BUILD_LOG) 2>&1 ; then \
+		true; \
+	else \
+		_err=$$?; \
+		cat "$(PKG_BUILD_LOG)"; \
+		exit $$_err; \
+	fi
+endef
+
+
+# Get source, unpack, patch
+$(PKG_WORK_DIR) $(PKG_BUILD_PATH) $(PKG_DESTDIR) $(PKG_RUNTIME_DESTDIR):
+	@mkdir -p $@
+
+$(PKG_BUILD_LOG):
+	@mkdir -p $(@D)
+	@: > $@
+
+$(PKG_ARCHIVE_PATH): | $(PKG_META_PATH) $(PKG_WORK_DIR) $(PKG_BUILD_LOG)
+	@$(call run,vfetch $(PKG_META_PATH) $@)
+
+$(PKG_SOURCE_ROOT): $(PKG_ARCHIVE_PATH) $(PKG_PATCHES)
+	@test -d $@ && rm -rf $@ ; true
+	@mkdir -p $@
+	@tar -xpf $< --strip-components $(PKG_UNPACK_STRIP) -C $@
+
+ifeq ($(strip $(PKG_PATCHES)),)
+$(PKG_PATCH_STAMP): | $(PKG_SOURCE_ROOT)
+	@touch $@
+else
+$(PKG_PATCH_STAMP): $(PKG_PATCHES) | $(PKG_SOURCE_ROOT)
+	@$(call run,for patch in $(PKG_PATCHES); do echo $$patch && \
+		patch -d $(firstword $|) -p$(PKG_PATCH_STRIP) < "$$patch"; done)
+	@touch $@
+endif
+
+$(PKG_SOURCE_STAMP): $(PKG_PATCH_STAMP)
+	@touch $@
+
+
+source: $(PKG_SOURCE_STAMP)
+
+
+# Build and Install
+
+ifeq ($(PKG_BUILD_SCHEME),pysrc)
+
+# https://github.com/pypa/pip/issues/10978
+define installwheel
+	$(call run,env $(PKG_ENVIRONMENT) \
+		python3 -m pip install \
+			--find-links $(PIP_WHEELS) \
+			--root=$(1) \
+			--prefix="$(PKG_PREFIX)" \
+			--ignore-installed \
+			$(2) \
+			$(PKG_NAME)==$(PKG_VER))
+endef
+
+PIP_CACHE  := $(PKG_CACHE_ROOT)/pip-cache
+PIP_WHEELS := $(PKG_CACHE_ROOT)/pip-wheels
+
+PKG_ENVIRONMENT += PIP_CACHE_DIR=$(PIP_CACHE)
+PKG_ENVIRONMENT += PIP_PREFER_BINARY=1
+
+$(PKG_CACHE_ROOT) $(PIP_WHEELS):
+	@mkdir -p $@
+
+$(PKG_BUILD_STAMP): $(PKG_SOURCE_STAMP) | $(PKG_CACHE) $(PIP_WHEELS)
+	@$(call run,env $(PKG_ENVIRONMENT) \
+		python3 -m pip wheel --wheel-dir $(PIP_WHEELS) $(PKG_SOURCE_PATH))
+	@touch $@
+
+$(PKG_INSTALL_STAMP): $(PKG_BUILD_STAMP) | $(PKG_DESTDIR) $(PKG_RUNTIME_DESTDIR)
+	@$(foreach d,$|,$(call installwheel,$(d));)
+	@rm -rf \
+		$(PKG_RUNTIME_PATH)/lib/python*/dist-packages/*.dist-info \
+		$(PKG_RUNTIME_PATH)/lib/python*/dist-packages/*.data \
+		$(PKG_RUNTIME_PATH)/lib/python*/dist-packages/test* \
+		$(PKG_RUNTIME_PATH)/lib/python*/dist-packages/__pycache__ \
+		$(PKG_RUNTIME_PATH)/share/doc \
+		$(PKG_RUNTIME_PATH)/share/man
+	@touch $@
+
+else ifeq ($(PKG_BUILD_SCHEME),autotools)
+
+PKG_ENVIRONMENT += PKG_CONFIG_SYSROOT_DIR=$(PKG_DESTDIR)
+PKG_ENVIRONMENT += PKG_CONFIG_PATH=$(PKG_INSTALL_PATH)/lib/pkgconfig:$(PKG_INSTALL_PATH)/share/pkgconfig
+PKG_ENVIRONMENT += PKG_CONFIG_LIBDIR=$(PKG_INSTALL_PATH)/lib/pkgconfig:$(PKG_INSTALL_PATH)/lib/$(PKG_HOST_GNU_TYPE)/pkgconfig
+PKG_ENVIRONMENT += PKG_CONFIG=pkg-config
+PKG_ENVIRONMENT += CPPFLAGS="$(PKG_CPPFLAGS)"
+PKG_ENVIRONMENT += CFLAGS="$(PKG_CFLAGS)"
+PKG_ENVIRONMENT += CXXFLAGS="$(PKG_CXXFLAGS)"
+PKG_ENVIRONMENT += LDFLAGS="$(PKG_LDFLAGS)"
+PKG_ENVIRONMENT += MAKEINFO=true
+
+$(PKG_SOURCE_PATH)/configure: $(PKG_SOURCE_STAMP)
+	@$(call run,(cd $(@D) && \
+		if test -f ./configure; then \
+			:; \
+		elif test -x ./autogen.sh; then \
+			NOCONFIGURE=1 ./autogen.sh; \
+		else \
+			autoreconf -fi; \
+		fi))
+	@test -s $@
+
+$(PKG_BUILD_PATH)/Makefile: $(PKG_SOURCE_PATH)/configure | $(PKG_BUILD_PATH) $(PKG_DESTDIR)
+	@$(call run,(cd $(PKG_BUILD_PATH) && \
+		env $(PKG_ENVIRONMENT) \
+		$(PKG_SOURCE_PATH)/configure \
+		--build=$(PKG_BUILD_GNU_TYPE) \
+		--host=$(PKG_HOST_GNU_TYPE) \
+		--with-sysroot=$(PKG_DESTDIR) \
+		--prefix=$(PKG_PREFIX) \
+		--sysconfdir=/etc \
+		$(PKG_CONFIGURE_OPTS)))
+	@test -s $@
+
+$(PKG_BUILD_STAMP): $(PKG_BUILD_PATH)/Makefile
+	@$(call run,env $(PKG_ENVIRONMENT) \
+		$(MAKE) -C $(PKG_BUILD_PATH))
+	@touch $@
+
+$(PKG_INSTALL_STAMP): $(PKG_BUILD_STAMP) | $(PKG_RUNTIME_DESTDIR) $(PKG_DESTDIR)
+	@$(foreach d,$|,$(call run,env $(PKG_ENVIRONMENT) \
+		$(MAKE) -C $(PKG_BUILD_PATH) install DESTDIR=$(d));)
+	@rm -rf \
+		$(PKG_RUNTIME_PATH)/lib/*.a \
+		$(PKG_RUNTIME_PATH)/lib/*.la \
+		$(PKG_RUNTIME_PATH)/include \
+		$(PKG_RUNTIME_PATH)/lib/pkgconfig \
+		$(PKG_RUNTIME_PATH)/share/doc \
+		$(PKG_RUNTIME_PATH)/share/man
+	@touch $@
+else
+$(error Unknown build scheme)
+endif
+
+.PHONY: install
+install: $(PKG_INSTALL_STAMP)
diff --git a/rpi-image-gen b/rpi-image-gen
index 7edaf6b9..be991846 100755
--- a/rpi-image-gen
+++ b/rpi-image-gen
@@ -26,6 +26,7 @@ readonly IGTOP=$(rootpath)
 source "$IGTOP/lib/cli.sh"
 source "$IGTOP/lib/common.sh"
 source "$IGTOP/lib/dependencies.sh"
+source "$IGTOP/lib/tools.sh"
 
 
 # Global context
@@ -48,6 +49,7 @@ path_refresh()
 {
    # Defaults
    ctx[EXEC_PATH]="$IGTOP/bin:$PATH"
+   ctx[PYTHON_PATH]="${PYTHONPATH:-}"
    ctx[LAYER_PATH]="$IGTOP/device:$IGTOP/image:$IGTOP/layer"
    ctx[CONFIG_PATH]="$IGTOP/config"
 
@@ -130,6 +132,8 @@ parameter_assembly()
    if dpkg-architecture >/dev/null 2>&1; then
       DEB_BUILD_ARCH="${DEB_BUILD_ARCH:-$(dpkg-architecture -qDEB_BUILD_ARCH)}"
       DEB_HOST_ARCH="${DEB_HOST_ARCH:-$(dpkg-architecture -a${ARCH:-arm64} -qDEB_HOST_ARCH)}"
+      DEB_BUILD_GNU_TYPE="${DEB_BUILD_GNU_TYPE:-$(dpkg-architecture -a"$DEB_BUILD_ARCH" -qDEB_BUILD_GNU_TYPE)}"
+      DEB_HOST_GNU_TYPE="${DEB_HOST_GNU_TYPE:-$(dpkg-architecture -a"$DEB_HOST_ARCH" -qDEB_HOST_GNU_TYPE)}"
 
       if [ "$DEB_BUILD_ARCH" = "$DEB_HOST_ARCH" ]; then
          TOOLCHAIN_MODE="native"
@@ -139,9 +143,11 @@ parameter_assembly()
 
       printf 'DEB_BUILD_ARCH="%s"\n' "$DEB_BUILD_ARCH" >> "${ctx[IGENVF]}"
       printf 'DEB_HOST_ARCH="%s"\n' "$DEB_HOST_ARCH" >> "${ctx[IGENVF]}"
+      printf 'DEB_BUILD_GNU_TYPE="%s"\n' "$DEB_BUILD_GNU_TYPE" >> "${ctx[IGENVF]}"
+      printf 'DEB_HOST_GNU_TYPE="%s"\n' "$DEB_HOST_GNU_TYPE" >> "${ctx[IGENVF]}"
       printf 'TOOLCHAIN_MODE="%s"\n' "$TOOLCHAIN_MODE" >> "${ctx[IGENVF]}"
    else
-      warn "dpkg-architecture not found: source build support unavailable"
+      die "dpkg-architecture not found: source build support unavailable"
    fi
 
    # Config sanity checks
@@ -232,6 +238,7 @@ collect_layers()
 ###############################################################################
 # Stage 3: Filesystem build preparation
 #   Create output directories
+#   Build mission critical tools
 #   Initialise bdebstrap environment - apt, proxy, cache, etc
 #   Assemble compatible layers
 ###############################################################################
@@ -253,7 +260,9 @@ prepare_build_config()
          *prebuild|*postbuild|*preimage|*postimage|\
          IGconf_sbom_hook|\
          IGconf_deploy_dir|\
-         IGconf_deploy_hook)
+         IGconf_deploy_hook|\
+         IGconf_sys_workroot|\
+         IGconf_sys_cachedir)
             declare -g "$key"="$val"
             ;;
       esac
@@ -263,13 +272,12 @@ prepare_build_config()
 
    # Create output dirs
    mkdir -p "$IGconf_target_dir" \
-            "$IGconf_sys_workroot/host/bin"
+      "$IGconf_sys_workroot" \
+      "$IGconf_sys_cachedir"
    [[ -n "${IGconf_image_outputdir:-}" ]] && mkdir -p "$IGconf_image_outputdir"
 
-   # Host programs can be installed here
-   ctx[EXEC_PATH]="${IGconf_sys_workroot}/host/bin:${ctx[EXEC_PATH]}"
-   PATH=${ctx[EXEC_PATH]}
-   export PATH
+   # Build required host tools
+   bootstrap_build_tools
 
    # Create initial bdebstrap env
    _bdebstrap=( --force

From e856b76fbab48ebf1d08754ba13b085d2bedda1e Mon Sep 17 00:00:00 2001
From: Matthew Lear <matthew.lear@raspberrypi.com>
Date: Thu, 13 Nov 2025 13:29:46 +0000
Subject: [PATCH 3/4] engine: cfg: Add env key support

Extend the config parser to support env keys. Variables defined under
the env key will be present in the build environment.
---
 docs/config/index.adoc |  8 +++++-
 docs/config/index.html |  8 +++++-
 site/config_loader.py  | 61 +++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 74 insertions(+), 3 deletions(-)

diff --git a/docs/config/index.adoc b/docs/config/index.adoc
index 6961875a..93c8f6ca 100644
--- a/docs/config/index.adoc
+++ b/docs/config/index.adoc
@@ -15,7 +15,7 @@ rpi-image-gen provides a flexible, hierarchical approach to managing build varia
 
 === Variable Naming Convention
 
-All configuration variables in rpi-image-gen use the `IGconf_` prefix followed by a structured naming pattern:
+Excluding environment keys, configuration variables in rpi-image-gen use the `IGconf_` prefix followed by a structured naming pattern:
 
 [source]
 ----
@@ -101,6 +101,9 @@ YAML provides the most flexible configuration format with native support for hie
 include:
   file: base.yaml
 
+env:
+  MYVAR: UNCHANGED
+
 device:
   class: pi5
   storage_type: sd
@@ -120,6 +123,9 @@ INI format provides traditional section-based configuration:
 ----
 !include base.cfg
 
+[env]
+MYVAR = UNCHANGED
+
 [device]
 class = pi5
 storage_type = sd
diff --git a/docs/config/index.html b/docs/config/index.html
index ab11ab1d..108f41c1 100644
--- a/docs/config/index.html
+++ b/docs/config/index.html
@@ -234,7 +234,7 @@ <h2 id="_core_architecture"><a class="anchor" href="#_core_architecture"></a><a
 <div class="sect2">
 <h3 id="_variable_naming_convention"><a class="anchor" href="#_variable_naming_convention"></a><a class="link" href="#_variable_naming_convention">Variable Naming Convention</a></h3>
 <div class="paragraph">
-<p>All configuration variables in rpi-image-gen use the <code>IGconf_</code> prefix followed by a structured naming pattern:</p>
+<p>Excluding environment keys, configuration variables in rpi-image-gen use the <code>IGconf_</code> prefix followed by a structured naming pattern:</p>
 </div>
 <div class="listingblock">
 <div class="content">
@@ -357,6 +357,9 @@ <h3 id="_yaml_configuration_format"><a class="anchor" href="#_yaml_configuration
 <pre class="highlight"><code class="language-yaml" data-lang="yaml">include:
   file: base.yaml
 
+env:
+  MYVAR: UNCHANGED
+
 device:
   class: pi5
   storage_type: sd
@@ -378,6 +381,9 @@ <h3 id="_ini_configuration_format"><a class="anchor" href="#_ini_configuration_f
 <div class="content">
 <pre class="highlight"><code class="language-ini" data-lang="ini">!include base.cfg
 
+[env]
+MYVAR = UNCHANGED
+
 [device]
 class = pi5
 storage_type = sd
diff --git a/site/config_loader.py b/site/config_loader.py
index 8375c8b8..bf670c9e 100644
--- a/site/config_loader.py
+++ b/site/config_loader.py
@@ -162,6 +162,7 @@ def _load_ini_recursive(path: Path, visited: set, cfg: configparser.ConfigParser
 
         # Disable interpolation
         self.config = configparser.ConfigParser(interpolation=None)
+        self.config.optionxform = str
         _load_ini_recursive(Path(self.cfg_path).resolve(), set(), self.config)
 
         for section in self.config.sections():
@@ -306,11 +307,19 @@ def _get_value(self, env_key: str, cfg_value: str) -> str:
     def load_section(self, section: str):
         if section not in self.data:
             raise ValueError(f"Section [{section}] not found in {self.cfg_path}")
+        if section.lower() == "env":
+            self._load_env_section()
+            return
         for key, value in self.data[section].items():
             self._set_env_if_unset(self._env_key(section, key), self._expand(value))
 
     def load_all(self):
+        # Load [env] section first so that downstream config can rely on it
+        if "env" in self.data:
+            self.load_section("env")
         for section in self.data.keys():
+            if section == "env":
+                continue
             self.load_section(section)
 
         # Process override variables that don't correspond to config file entries
@@ -323,7 +332,11 @@ def _load_override_only_variables(self):
 
         # Build set of all environment keys that were processed from config file
         processed_env_keys = set()
+        if "env" in self.data:
+            processed_env_keys.update(self.data["env"].keys())
         for section_name, section_data in self.data.items():
+            if section_name == "env":
+                continue
             for key in section_data.keys():
                 env_key = self._env_key(section_name, key)
                 processed_env_keys.add(env_key)
@@ -364,9 +377,15 @@ def write_file(self, file_path: str, section: Optional[str] = None):
 
             # First, process all keys from the config file
             sections = [section] if section else list(self.data.keys())
+            if section is None and "env" in self.data:
+                self._write_env_section(f, processed_env_keys, None)
             for sect in sections:
                 if sect not in self.data:
                     raise ValueError(f"Section [{sect}] not found in {self.cfg_path}")
+                if sect == "env":
+                    if section is not None:
+                        self._write_env_section(f, processed_env_keys, section)
+                    continue
                 for key, value in self.data[sect].items():
                     env_key = self._env_key(sect, key)
                     processed_env_keys.add(env_key)
@@ -398,7 +417,7 @@ def _write_override_only_var(self, file_handle, override_key: str, section_filte
         else:
             # If we can't parse it and we're filtering by section, skip it
             # (assume it doesn't belong to the filtered section)
-            if section_filter is not None:
+            if section_filter is not None and section_filter != "env":
                 return
 
         file_handle.write(f'{override_key}="{effective_value}"\n')
@@ -409,6 +428,46 @@ def _write_override_only_var(self, file_handle, override_key: str, section_filte
         else:
             print(f"OVR {override_key}={effective_value}")
 
+    def _load_env_section(self):
+        for key, value in self.data.get("env", {}).items():
+            existing = os.environ.get(key)
+
+            if existing is not None and key not in self.overrides:
+                print(f"ENV {key}={existing} (preserved)")
+                continue
+
+            if key in self.overrides:
+                final_value = self._expand(self.overrides[key])
+                label = "OVR"
+            else:
+                final_value = self._expand(value)
+                label = "CFG"
+
+            os.environ[key] = final_value
+            print(f"{label} {key}={final_value}")
+
+    def _write_env_section(self, file_handle, processed_env_keys: set, section_filter: Optional[str]):
+        if section_filter is not None and section_filter != "env":
+            return
+        for key, value in self.data.get("env", {}).items():
+            processed_env_keys.add(key)
+            if key in os.environ:
+                effective_value = os.environ[key]
+                source = "env"
+            elif key in self.overrides:
+                effective_value = self._expand(self.overrides[key])
+                source = "override"
+            else:
+                effective_value = self._expand(value)
+                source = "config"
+            file_handle.write(f'{key}="{effective_value}"\n')
+            if source == "env":
+                print(f"ENV {key}={effective_value}")
+            elif source == "override":
+                print(f"OVR {key}={effective_value}")
+            else:
+                print(f"CFG {key}={effective_value}")
+
 
 def ConfigLoader_register_parser(subparsers):
     parser = subparsers.add_parser("config", help="Config utilities (.ini/.yaml)")

From c57f6b91221f7920e425553e7b19aceb535d1795 Mon Sep 17 00:00:00 2001
From: Matthew Lear <matthew.lear@raspberrypi.com>
Date: Thu, 13 Nov 2025 13:53:37 +0000
Subject: [PATCH 4/4] core: Categorise host build dependencies

Re-work the dependency installation mech to support different
categories of host dependencies and validate if a program installed as a
dependency can be located post-install.
Applying categories to dependencies should be beneficial when introducing
containerised execution support.
Write bootstrap information to a new output dir declared by the sys base
layer.

Fixes #106
---
 depends                        |  74 +++++++------
 docs/layer/sys-build-base.html |  18 ++++
 layer/base/sys-build-base.yaml |   8 ++
 lib/dependencies.sh            | 186 ++++++++++++++++++++++-----------
 rpi-image-gen                  |  16 ++-
 5 files changed, 200 insertions(+), 102 deletions(-)

diff --git a/depends b/depends
index c1234023..a220ae15 100644
--- a/depends
+++ b/depends
@@ -1,38 +1,44 @@
-realpath:coreutils
-zip
-mkdosfs:dosfstools
-mke2fs:e2fsprogs
-grep
-rsync
-curl
-mtools
-mmdebstrap
-podman
-zstd
-pv
-newuidmap:uidmap
-python-is-python3
-dbus-user-session
-btrfs-progs
-dctrl-tools
-uuidgen:uuid-runtime
-fdisk
-python3-yaml
-python3-debian
-python3-jsonschema
+all:realpath:coreutils
+all:bash
+all:mmdebstrap
+all:podman
+all:newuidmap:uidmap
+all::python-is-python3
+all::dbus-user-session
+
+bootstrap::python3-yaml
+bootstrap::python3-debian
+
+# image build essential
+build:zip
+build:mkdosfs:dosfstools
+build:mke2fs:e2fsprogs
+build:grep
+build:rsync
+build:curl
+build:mtools
+build:zstd
+build:pv
+build:mkfs.btrfs:btrfs-progs
+build::dctrl-tools
+build:uuidgen:uuid-runtime
+build:fdisk
+build::python3-jsonschema
+build::dpkg-dev
+
 # pkg buildsys
-python3-pip
-dpkg-dev
-make
-build-essential
-autoconf
-automake
-libtool
-autopoint
-flex
-gettext
-pkg-config
-# doc gen only
+build::python3-pip
+build:make
+build::build-essential
+build:autoconf
+build:automake
+build::libtool
+build:autopoint
+build:flex
+build:gettext
+build:pkg-config
+
+# maintainer only (docgen)
 # python3-markdown
 # asciidoctor
 # python3-jinja2
diff --git a/docs/layer/sys-build-base.html b/docs/layer/sys-build-base.html
index 4b5a99d2..14af1af4 100644
--- a/docs/layer/sys-build-base.html
+++ b/docs/layer/sys-build-base.html
@@ -303,6 +303,24 @@ <h2>Configuration Variables</h2>
                     </td>
                 </tr>
                 
+                <tr>
+                    <td><code>IGconf_sys_bootstrapdir</code></td>
+                    <td>Directory for bootstrap artefacts (e.g. env, layer
+ order, metadata) used to resume the build in a containerised (or later)
+ invocation.</td>
+                    <td>
+                       
+                           
+                           <code class="long-default">${IGconf_sys_workroot}/bootstrap</code>
+                           
+                       
+                    </td>
+                    <td>Non-empty string value</td>
+                    <td>
+                        <a href="variable-validation.html#set-policies" class="badge policy-immediate" title="Click for policy and validation help">immediate</a>
+                    </td>
+                </tr>
+                
             </tbody>
         </table>
         
diff --git a/layer/base/sys-build-base.yaml b/layer/base/sys-build-base.yaml
index 73cbb7bb..144f9094 100644
--- a/layer/base/sys-build-base.yaml
+++ b/layer/base/sys-build-base.yaml
@@ -71,5 +71,13 @@
 # X-Env-Var-buildroot-Valid: string
 # X-Env-Var-buildroot-Set: y
 #
+# X-Env-Var-bootstrapdir: ${IGconf_sys_workroot}/bootstrap
+# X-Env-Var-bootstrapdir-Desc: Directory for bootstrap artefacts (e.g. env, layer
+#  order, metadata) used to resume the build in a containerised (or later)
+#  invocation.
+# X-Env-Var-bootstrapdir-Required: n
+# X-Env-Var-bootstrapdir-Valid: string
+# X-Env-Var-bootstrapdir-Set: y
+#
 # METAEND
 ---
diff --git a/lib/dependencies.sh b/lib/dependencies.sh
index c4991299..3371a811 100644
--- a/lib/dependencies.sh
+++ b/lib/dependencies.sh
@@ -1,78 +1,138 @@
 #!/bin/bash
 
-# dependencies_check
-# $@	Dependency files to check
+# Options:
+#   --category <name>   Category selector
+#   --install           Automatically install missing packages via apt
+#   <files...>          Dependency manifest files to process
 #
-# Each dependency is in the form of a tool to test for, optionally followed by
-# a : and the name of a package if the package on a Debian-ish system is not
-# named for the tool (i.e., qemu-user-static).
+# A manifest declares dependencies in the form of a triplet:
+#  category:program:package
+# The package field is optional and used if the package on a Debian-ish system
+# is not named for the program (i.e., qemu-user-static).
 dependencies_check()
 {
-	local depfile deps missing=() op=$1
-
-	if [[ "$op" == install ]] ; then
-		shift 1
-	fi
-
-	for depfile in "$@"; do
-		if [[ -e "$depfile" ]]; then
-			mapfile -t dep_lines < <(grep -v '^[[:space:]]*#' "$depfile" | grep -v '^[[:space:]]*$')
-			deps="${dep_lines[*]}"
-		fi
-		for dep in $deps; do
-			if ! hash "${dep%:*}" 2>/dev/null; then
-				if ! dpkg -s "${dep#*:}" > /dev/null 2>&1; then
-					missing+=("${dep#*:}")
-				fi
-			fi
-		done
-	done
-
-	if [[ "${missing[*]}" ]]; then
-		echo "Required dependencies not installed"
-		echo
-		echo "This can be resolved on Debian systems by installing:"
-		echo "${missing[@]}"
-		echo
-		echo "Script install_deps.sh can be used for this purpose."
-		echo
-
-		if [[ "$op" == install ]] ; then
-			apt install -y "${missing[@]}"
-		else
-			exit 1
-		fi
-	fi
+   local category=all install_mode=0
+   local -a depfiles=()
+   local -a missing=()
+
+   while [[ $# -gt 0 ]]; do
+      case "$1" in
+         --category)
+            category=$2
+            shift 2
+            ;;
+         --install)
+            install_mode=1
+            shift
+            ;;
+         *)
+            depfiles+=("$1")
+            shift
+            ;;
+      esac
+   done
+
+   (( ${#depfiles[@]} )) || return 0
+
+   for depfile in "${depfiles[@]}"; do
+      [[ -r "$depfile" ]] || continue
+      while IFS= read -r line || [[ -n $line ]]; do
+         [[ $line =~ ^[[:space:]]*(#|$) ]] && continue
+
+         IFS=':' read -r dep_category tool pkg _ <<<"$line"
+         [[ -n $dep_category ]] || continue
+
+         [[ -n $pkg ]] || pkg=$tool
+         [[ -n $tool ]] || tool=""
+
+         if [[ $category != all && $dep_category != all && $dep_category != "$category" ]]; then
+            continue
+         fi
+
+         if [[ -z $tool ]]; then
+            # Package-only dependency
+            dpkg -s "$pkg" >/dev/null 2>&1 || missing+=("$pkg")
+            continue
+         fi
+
+         if command -v "$tool" >/dev/null 2>&1; then
+            continue  # installed and available
+         fi
+
+         if dpkg -s "$pkg" >/dev/null 2>&1; then
+            echo "Dependency $tool ($pkg) is installed, but not found"
+            exit 1  # installed but unavailable
+         fi
+
+         missing+=("$pkg") # not installed yet
+      done < "$depfile"
+   done
+
+   if [[ ${#missing[@]} -gt 0 ]]; then
+      local -a unique=()
+      declare -A seen=()
+      for pkg in "${missing[@]}"; do
+         [[ -n ${seen[$pkg]:-} ]] && continue
+         seen[$pkg]=1
+         unique+=("$pkg")
+      done
+
+      echo "Required dependencies (${category}) not installed"
+      echo
+      echo "This can be resolved on Debian systems by installing:"
+      echo "${unique[@]}"
+      echo
+      echo "Script install_deps.sh can be used for this purpose."
+      echo
+
+      if (( install_mode )); then
+         apt install -y "${unique[@]}"
+      else
+         exit 1
+      fi
+   fi
 
     # If we're building on a native arm platform, we don't need to check for
     # binfmt_misc or require it to be loaded.
 
-	binfmt_misc_required=1
-
-	case $(uname -m) in
-		aarch64)
-			binfmt_misc_required=0
-			;;
-		arm*)
-			binfmt_misc_required=0
-			;;
-	esac
-
-	if [[ "${binfmt_misc_required}" == "1" ]]; then
-		if ! grep -q "/proc/sys/fs/binfmt_misc" /proc/mounts; then
-			echo "Module binfmt_misc not loaded in host"
-			echo "Please run:"
-			echo "  sudo modprobe binfmt_misc"
-			exit 1
-		fi
-	fi
+   binfmt_misc_required=1
+
+   case $(uname -m) in
+      aarch64)
+         binfmt_misc_required=0
+         ;;
+      arm*)
+         binfmt_misc_required=0
+         ;;
+   esac
+
+   if [[ "${binfmt_misc_required}" == "1" ]]; then
+      if ! grep -q "/proc/sys/fs/binfmt_misc" /proc/mounts; then
+         echo "Module binfmt_misc not loaded in host"
+         echo "Please run:"
+         echo "  sudo modprobe binfmt_misc"
+         exit 1
+      fi
+   fi
 }
 
 
 dependencies_install()
 {
-	if [ "$(id -u)" != "0" ]; then
-		>&2 echo "Please run as root to install dependencies."; exit 1
-	fi
-	dependencies_check install "$@"
+   if [ "$(id -u)" != "0" ]; then
+      >&2 echo "Please run as root to install dependencies."; exit 1
+   fi
+   local category=all
+   while [[ $# -gt 0 ]]; do
+      case "$1" in
+         --category)
+            category=$2
+            shift 2
+            ;;
+         *)
+            break
+            ;;
+      esac
+   done
+   dependencies_check --install --category "$category" "$@"
 }
diff --git a/rpi-image-gen b/rpi-image-gen
index be991846..d8116e01 100755
--- a/rpi-image-gen
+++ b/rpi-image-gen
@@ -231,6 +231,12 @@ collect_layers()
 
    ctx[LAYER_ORDER]="$layer_order"
    ctx[FINALENV]="$finalenv"
+
+   # Write bootstrap information
+   local bdir=$(get_var IGconf_sys_bootstrapdir "${ctx[FINALENV]}") || die
+   mkdir -p "$bdir"
+   cp "${ctx[FINALENV]}"   "${bdir}/config.env"
+   cp "${ctx[LAYER_ORDER]}" "${bdir}/layer.order"
 }
 
 
@@ -551,7 +557,7 @@ main() {
          cli_parse_build ctx "$@"
          shift $?
          path_refresh
-         dependencies_check "$IGTOP/depends" || exit 1
+         dependencies_check --category all "$IGTOP/depends" || exit 1
          run_stage parameter_assembly "$@"
          run_stage collect_layers
          run_stage prepare_build_config
@@ -564,7 +570,7 @@ main() {
          cli_parse_clean ctx "$@"
          shift $?
          path_refresh
-         dependencies_check "$IGTOP/depends" || exit 1
+         dependencies_check --category all "$IGTOP/depends" || exit 1
          run_stage parameter_assembly "$@"
          run_stage collect_layers
          run_stage clean_worktree
@@ -574,21 +580,21 @@ main() {
          cli_parse_layer ctx "$@"
          shift $?
          path_refresh
-         dependencies_check "$IGTOP/depends" || exit 1
+         dependencies_check --category all "$IGTOP/depends" || exit 1
          exec "$IGTOP/bin/ig" layer --path "${ctx[LAYER_PATH]}" "$@"
          ;;
 
       metadata)
          cli_parse_metadata ctx "$@"
          shift $?
-         dependencies_check "$IGTOP/depends" || exit 1
+         dependencies_check --category all "$IGTOP/depends" || exit 1
          exec "$IGTOP/bin/ig" metadata "$@"
          ;;
 
       config)
          cli_parse_config ctx "$@"
          shift $?
-         dependencies_check "$IGTOP/depends" || exit 1
+         dependencies_check --category all "$IGTOP/depends" || exit 1
          exec "$IGTOP/bin/ig" config "$@"
          ;;