Harden OS/installer/CI: bootc observability, kernel-level taskd loopback, supply-chain pinning

.github/workflows/ci.yml

@@ -21,11 +21,14 @@ jobs:
     name: Format, lint, and test
     runs-on: ubuntu-latest
     steps:
+      # Pinning policy (matches os-e2e.yml): first-party actions/* are pinned to
+      # major tags; third-party actions are pinned to a full commit SHA with a
+      # trailing version comment.
       - uses: actions/checkout@v4
-      - uses: dtolnay/rust-toolchain@1.85.0
+      - uses: dtolnay/rust-toolchain@8641a17e25bf5b40c118d48fe0f81e8655731839 # 1.85.0
         with:
           components: clippy,rustfmt
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
       - run: cargo fmt --all -- --check
       - run: cargo clippy --workspace --all-targets --locked -- -D warnings
       - run: cargo test --workspace --locked
@@ -44,8 +47,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
-      - uses: dtolnay/rust-toolchain@1.85.0
-      - uses: Swatinem/rust-cache@v2
+      - uses: dtolnay/rust-toolchain@8641a17e25bf5b40c118d48fe0f81e8655731839 # 1.85.0
+      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
       # The quality job already runs the full test suite on ubuntu; skip the
       # duplicate run and keep only the platform-specific probe there.
       - run: cargo test --workspace --locked

os/Containerfile

@@ -1,14 +1,20 @@
 # syntax=docker/dockerfile:1.7
 
+# Supply-chain: base images are pinned by @sha256 digest so a retagged or
+# compromised upstream tag cannot silently change the build inputs. The digest
+# is authoritative; the human-readable tag is kept for context and must be
+# updated together with the digest (resolve via `skopeo inspect` /
+# `podman manifest inspect`). See security-review.md finding #4.
 ARG RUST_VERSION=1.85
-FROM docker.io/library/rust:${RUST_VERSION}-bookworm AS rust-builder
+FROM docker.io/library/rust:${RUST_VERSION}-bookworm@sha256:e51d0265072d2d9d5d320f6a44dde6b9ef13653b035098febd68cce8fa7c0bc4 AS rust-builder
 
 WORKDIR /src
 COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
 COPY crates ./crates
 RUN cargo build --release --locked --bin andromeda --bin andromeda-taskd
 
-FROM quay.io/fedora/fedora-bootc:44 AS payload
+# Pinned by digest; tag :44 kept for context (see rust-builder note above).
+FROM quay.io/fedora/fedora-bootc:44@sha256:69944835958dda565f8d613587535a3d90e3256cf0a0f0e29bca5ca3d2360d06 AS payload
 
 ARG PLATFORM_VARIANT=pc_x86_64
 ARG PLATFORM_ARCHITECTURE=x86_64

os/files/usr/lib/systemd/system/andromeda-taskd.service

@@ -11,6 +11,15 @@ Environment=ANDROMEDA_STATE_DIR=/var/lib/andromeda-taskd/state
 ExecStart=/usr/bin/andromeda-taskd
 Restart=on-failure
 RestartSec=2s
+# Kernel-level loopback enforcement (cgroup/BPF on packets): taskd can only ever
+# serve loopback regardless of ANDROMEDA_LISTEN, so a misconfigured non-loopback
+# bind cannot expose the unauthenticated API to the network. This is orthogonal
+# to RestrictAddressFamilies below (which filters socket() families, not routes)
+# and does not conflict with it. See security-review.md finding #2.
+IPAddressAllow=localhost
+IPAddressDeny=any
+LockPersonality=yes
+MemoryDenyWriteExecute=yes
 NoNewPrivileges=yes
 PrivateDevices=yes
 PrivateTmp=yes
@@ -21,11 +30,14 @@ ProtectHostname=yes
 ProtectKernelLogs=yes
 ProtectKernelModules=yes
 ProtectKernelTunables=yes
+ProtectProc=invisible
 ProtectSystem=strict
 RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
 RestrictNamespaces=yes
 RestrictRealtime=yes
+RestrictSUIDSGID=yes
 SystemCallArchitectures=native
+SystemCallFilter=@system-service
 UMask=0077
 
 [Install]

os/installer/andromeda-ci.ks

@@ -15,6 +15,11 @@ reqpart --add-boot
 part / --fstype=ext4 --size=8192 --grow --label=andromeda-root
 
 bootloader --timeout=1 --append="console=tty0 console=ttyS0,115200n8 andromeda.ci=1"
+# SECURITY: --target-imgref is a mutable, unsigned tag. Before trusting remote
+# `bootc switch`/upgrade from this ref, a bootc signing policy
+# (sigstore/containers-policy.json pinning a trusted signing key) is a REQUIRED
+# follow-up; without it a compromised registry or retagged image is booted on
+# the next update. See security-review.md finding #4.
 bootc --source-imgref=containers-storage:localhost/andromeda:v1 --target-imgref=ghcr.io/oratis/andromeda:edge --stateroot=andromeda
 shutdown
 

os/installer/collect-anaconda-diagnostics.sh

@@ -15,19 +15,39 @@ diagnostic_logs=(
     /tmp/andromeda-uefi-fallback.log
 )
 
+# The bootc payload runs under Anaconda 44's Payloads DBus module; its stderr
+# lands in that module's journal rather than reliably in program.log. Capture it
+# first-class so a "bootc ... exited with status 1" is root-causable.
+readonly PAYLOADS_UNIT_GLOB='org.fedoraproject.Anaconda.Modules.Payloads*'
+
 emit_file() {
     local diagnostic_file="$1"
+    # max_lines <= 0 emits the file in full. program.log is emitted in full
+    # because the failing bootc payload phase can be well beyond the last lines;
+    # the CI serial sink is a host file, so full emission is cheap.
+    local max_lines="${2:-2000}"
 
     [[ -f "${diagnostic_file}" ]] || return 0
     printf '\n===== %s =====\n' "${diagnostic_file}" >"${SERIAL_DEVICE}"
-    tail -n 2000 "${diagnostic_file}" >"${SERIAL_DEVICE}"
+    if (( max_lines <= 0 )); then
+        cat "${diagnostic_file}" >"${SERIAL_DEVICE}"
+    else
+        tail -n "${max_lines}" "${diagnostic_file}" >"${SERIAL_DEVICE}"
+    fi
 }
 
 if [[ -c "${SERIAL_DEVICE}" ]]; then
     printf 'ANDROMEDA_INSTALLER_DIAGNOSTICS_START\n' >"${SERIAL_DEVICE}"
     for diagnostic_log in "${diagnostic_logs[@]}"; do
-        emit_file "${diagnostic_log}"
+        if [[ "${diagnostic_log}" == /tmp/program.log ]]; then
+            emit_file "${diagnostic_log}" 0
+        else
+            emit_file "${diagnostic_log}"
+        fi
     done
+    printf '\n===== journalctl payloads =====\n' >"${SERIAL_DEVICE}"
+    journalctl -u "${PAYLOADS_UNIT_GLOB}" --boot --no-pager \
+        >"${SERIAL_DEVICE}" 2>&1 || true
     printf '\n===== journalctl =====\n' >"${SERIAL_DEVICE}"
     journalctl --boot --no-pager --lines=2000 >"${SERIAL_DEVICE}" 2>&1 || true
     printf 'ANDROMEDA_INSTALLER_DIAGNOSTICS_END\n' >"${SERIAL_DEVICE}"
@@ -42,6 +62,8 @@ if mountpoint --quiet "${ESP_MOUNT}"; then
     done
     journalctl --boot --no-pager \
         >"${ESP_DIAGNOSTICS}/journal.log" 2>&1 || true
+    journalctl -u "${PAYLOADS_UNIT_GLOB}" --boot --no-pager \
+        >"${ESP_DIAGNOSTICS}/anaconda-payloads-journal.log" 2>&1 || true
     lsblk --output NAME,SIZE,TYPE,FSTYPE,LABEL,PARTLABEL,PARTTYPE,UUID \
         >"${ESP_DIAGNOSTICS}/lsblk.txt" 2>&1 || true
     findmnt --real \

os/installer/interactive-defaults.ks

@@ -1,10 +1,21 @@
+# SECURITY: --target-imgref is a mutable, unsigned tag. Before trusting remote
+# `bootc switch`/upgrade from this ref, a bootc signing policy
+# (sigstore/containers-policy.json pinning a trusted signing key) is a REQUIRED
+# follow-up; without it a compromised registry or retagged image is booted on
+# the next update. See security-review.md finding #4.
 bootc --source-imgref=containers-storage:localhost/andromeda:v1 --target-imgref=ghcr.io/oratis/andromeda:edge
 selinux --enforcing
 
 %pre --erroronfail --log=/tmp/andromeda-installer-preflight.log
 /usr/libexec/andromeda-installer-preflight interactive
 %end
 
+# Leave disk-side evidence on the ESP when an interactive install fails, so a
+# human can pull the drive and read the anaconda/bootc diagnostics afterwards.
+%onerror --log=/tmp/andromeda-installer-onerror.log
+/usr/libexec/andromeda-collect-anaconda-diagnostics
+%end
+
 %post --nochroot --erroronfail --log=/tmp/andromeda-uefi-fallback.log
 /usr/libexec/andromeda-install-uefi-fallback /mnt/sysimage /mnt/sysroot interactive
 %end

os/scripts/test-containerfile-layer-budget.sh

@@ -5,6 +5,11 @@ REPOSITORY_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 readonly REPOSITORY_ROOT
 readonly CONTAINERFILE="${REPOSITORY_ROOT}/os/Containerfile"
 readonly MINIMUM_PAYLOAD_DNF_LAYERS=20
+readonly OUTPUT_DIR="${1:-${ANDROMEDA_OUTPUT_DIR:-${REPOSITORY_ROOT}/output}}"
+# Upper bound for any single OCI layer bootc stages one-at-a-time into the
+# installer's small /var/tmp overlay. Default 4 GiB; override for a build whose
+# base layers are known-larger via ANDROMEDA_MAX_PAYLOAD_LAYER_BYTES.
+readonly MAXIMUM_PAYLOAD_LAYER_BYTES="${ANDROMEDA_MAX_PAYLOAD_LAYER_BYTES:-4294967296}"
 
 # The Anaconda live environment has a small writable overlay. bootc stages one
 # OCI blob at a time in /var/tmp while importing the payload, so combining the
@@ -26,5 +31,52 @@ if (( payload_dnf_layers < MINIMUM_PAYLOAD_DNF_LAYERS )); then
     exit 1
 fi
 
+# The DNF-layer-count floor above is a proxy for "no single over-consolidated
+# blob". It cannot catch "20 small layers + 1 huge layer", which would still
+# overflow the installer /var/tmp overlay while bootc stages that one blob (the
+# real cause of the recent bootc install failure). When build-iso.sh has already
+# produced andromeda-v1-history.json, directly assert the largest single layer
+# stays under budget. In the unit-test CI context no image has been built, so
+# the size data is unavailable and this check is skipped with a log line rather
+# than failing.
+history_json="${ANDROMEDA_LAYER_HISTORY_JSON:-${OUTPUT_DIR}/andromeda-v1-history.json}"
+if [[ -f "${history_json}" ]]; then
+    max_layer_bytes="$(
+        python3 - "${history_json}" <<'PY'
+import json
+import sys
+
+with open(sys.argv[1], encoding="utf-8") as handle:
+    layers = json.load(handle)
+
+
+def layer_size(layer):
+    value = layer.get("size", layer.get("Size", 0))
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return 0
+
+
+print(max((layer_size(layer) for layer in layers), default=0))
+PY
+    )"
+    if ! [[ "${max_layer_bytes}" =~ ^[0-9]+$ ]]; then
+        printf 'Could not parse layer sizes from %s.\n' "${history_json}" >&2
+        exit 1
+    fi
+    if (( max_layer_bytes > MAXIMUM_PAYLOAD_LAYER_BYTES )); then
+        printf 'Largest image layer is %d bytes; the verified installer overlay budget is %d bytes.\n' \
+            "${max_layer_bytes}" "${MAXIMUM_PAYLOAD_LAYER_BYTES}" >&2
+        printf 'A single oversized layer can overflow the installer /var/tmp overlay during bootc staging even when the target disk has ample free space.\n' >&2
+        exit 1
+    fi
+    printf 'ANDROMEDA_CONTAINERFILE_LAYER_SIZE_OK max_layer_bytes=%d budget_bytes=%d\n' \
+        "${max_layer_bytes}" "${MAXIMUM_PAYLOAD_LAYER_BYTES}"
+else
+    printf 'ANDROMEDA_CONTAINERFILE_LAYER_SIZE_SKIP reason=no-history-json path=%s\n' \
+        "${history_json}"
+fi
+
 printf 'ANDROMEDA_CONTAINERFILE_LAYER_BUDGET_OK payload_dnf_layers=%d\n' \
     "${payload_dnf_layers}"

os/scripts/test-hardware-matrix.sh

@@ -34,11 +34,21 @@ profile_timeout_seconds=600
 if [[ -c /dev/kvm ]]; then
     accel=kvm
     cpu=host
+elif [[ "${ANDROMEDA_ALLOW_TCG:-0}" != "1" ]]; then
+    # Mirror test-gcp-nested.sh:41's `test -c /dev/kvm` precheck. Full-system TCG
+    # emulation is ~10x slower; the TCG per-profile budget (3600s x 3 profiles)
+    # can exceed the os-e2e job `timeout-minutes: 150`, turning a missing
+    # /dev/kvm into confusing timeouts rather than a clear failure. Fail fast;
+    # set ANDROMEDA_ALLOW_TCG=1 to force the slow TCG path for local debugging.
+    printf 'test-hardware-matrix.sh: /dev/kvm is unavailable.\n' >&2
+    printf 'Full-system TCG emulation is ~10x slower and its per-profile budget can exceed the os-e2e job timeout (150m).\n' >&2
+    printf 'Set ANDROMEDA_ALLOW_TCG=1 to force the slow TCG path for local debugging.\n' >&2
+    exit 1
 else
     # Full-system TCG emulation is roughly an order of magnitude slower than
     # KVM; the 600 s per-profile budget would always time out.
     profile_timeout_seconds=3600
-    printf 'WARNING: /dev/kvm is unavailable; falling back to TCG emulation with a %s s per-profile timeout. Expect a very slow run.\n' \
+    printf 'WARNING: /dev/kvm is unavailable; ANDROMEDA_ALLOW_TCG=1 is set, falling back to TCG emulation with a %s s per-profile timeout. Expect a very slow run.\n' \
         "${profile_timeout_seconds}" >&2
 fi
 readonly profile_timeout_seconds