From d81c395fce20dda1df5c85f13e3b614b08533c3c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 11 Sep 2025 13:57:46 -0400 Subject: [PATCH 1/2] ci: Unify more of hack/ and tests/ A key thing for me is that the `Justfile` should be a one-stop shop for development of the project. It can't have everything but it should answer the basic questions of "how do I build and test this project". This aligns the recently added tmt-on-GHA flow a *bit* more closely with some of that. Biggest is to use the `just build-integration-test-image` as the canonical way to build a container image with our testing stuff in it; which uses our main Dockerfile Other cleanups: - Change test script to move into tests/tmt/ as a workaround for https://github.com/teemtee/tmt/pull/3037#issuecomment-3259585271 - Change the qemu logic to use SMBIOS credentials so we don't have to carry around both a disk image and a SSH key - Change qemu to use `-snapshot` so we can reuse disks - Change the scripts to accept data via argv[1] and not environment - Drop the hardcoded testing directory and use `target/` as a generic build artifact dir Signed-off-by: Colin Walters --- .github/workflows/ci.yml | 11 +- .github/workflows/integration.yml | 85 ++++--- .packit.yaml | 30 +-- Justfile | 8 +- Makefile | 3 - crates/tests-integration/src/install.rs | 4 + crates/xtask/src/xtask.rs | 78 ------- hack/Containerfile | 15 +- hack/Containerfile.drop-lbis | 3 + .../share/containers/systemd/curl-base.image | 2 + .../share/containers/systemd/curl.container | 3 + .../systemd/jboss-webserver-5.image | 6 + .../usr/share/containers/systemd/podman.image | 2 + hack/packages.txt | 5 + hack/provision-derived.sh | 41 +++- tests/build.sh | 172 +++++--------- tests/run-tmt.sh | 26 +++ tests/test.sh | 70 ------ tmt/plans/bootc-integration.fmf | 46 ---- tmt/plans/integration.fmf | 23 +- tmt/tests/bootc-install-provision.fmf | 3 - tmt/tests/bootc-install-provision.sh | 215 ------------------ tmt/tests/booted/README.md | 3 +- 23 files changed, 239 insertions(+), 615 deletions(-) create mode 100644 hack/Containerfile.drop-lbis create mode 100644 hack/lbi/usr/share/containers/systemd/curl-base.image create mode 100644 hack/lbi/usr/share/containers/systemd/curl.container create mode 100644 hack/lbi/usr/share/containers/systemd/jboss-webserver-5.image create mode 100644 hack/lbi/usr/share/containers/systemd/podman.image create mode 100644 hack/packages.txt create mode 100755 tests/run-tmt.sh delete mode 100755 tests/test.sh delete mode 100644 tmt/plans/bootc-integration.fmf delete mode 100644 tmt/tests/bootc-install-provision.fmf delete mode 100755 tmt/tests/bootc-install-provision.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f3ece383..48598a036 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,9 @@ jobs: run: sudo apt update && sudo apt install just - uses: actions/checkout@v4 - name: Build and run container integration tests - run: sudo just run-container-integration run-container-external-tests + run: | + sudo just build + sudo just run-container-integration run-container-external-tests container-continuous: if: ${{ !contains(github.event.pull_request.labels.*.name, 'control/skip-ci') }} runs-on: ubuntu-24.04 @@ -105,7 +107,8 @@ jobs: set -xeu # Build images to test; TODO investigate doing single container builds # via GHA and pushing to a temporary registry to share among workflows? - sudo just build-integration-test-image + sudo just build + sudo just build-install-test-image sudo podman build -t localhost/bootc-fsverity -f ci/Containerfile.install-fsverity # TODO move into a container, and then have this tool run other containers @@ -120,9 +123,9 @@ jobs: sudo podman run --privileged --pid=host -v /:/run/host -v $(pwd):/src:ro -v /var/tmp:/var/tmp \ -v /run/dbus:/run/dbus -v /run/systemd:/run/systemd localhost/bootc /src/crates/ostree-ext/ci/priv-integration.sh # Nondestructive but privileged tests - sudo bootc-integration-tests host-privileged localhost/bootc-integration + sudo bootc-integration-tests host-privileged localhost/bootc-integration-install # Install tests - sudo bootc-integration-tests install-alongside localhost/bootc-integration + sudo bootc-integration-tests install-alongside localhost/bootc-integration-install # system-reinstall-bootc tests cargo build --release -p system-reinstall-bootc diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b0d4b8a54..aee202013 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,84 +1,81 @@ -name: bootc integration test +# This workflow builds a container across a matrix of OSes, +# generates a disk image from that, and runs integration tests +# using tmt + libvirt (using nested virt support in the default GHA runners). +name: Build+TMT on: pull_request: - branches: [main] + branches: [main] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: build: strategy: + fail-fast: false matrix: - test_os: [fedora-41, fedora-42, fedora-43, centos-9] - test_runner: [ubuntu-latest, ubuntu-24.04-arm] + test_os: [fedora-42, fedora-43, centos-9, centos-10] - runs-on: ${{ matrix.test_runner }} + runs-on: ubuntu-24.04 steps: - - name: Install podman for heredoc support + - name: Install dependencies run: | set -eux echo 'deb [trusted=yes] https://ftp.debian.org/debian/ testing main' | sudo tee /etc/apt/sources.list.d/testing.list sudo apt update - sudo apt install -y crun/testing podman/testing + sudo apt install -y crun/testing podman/testing just qemu-utils - uses: actions/checkout@v4 - - name: Build bootc and bootc image - env: - TEST_OS: ${{ matrix.test_os }} - run: sudo -E TEST_OS=$TEST_OS tests/build.sh + - name: Set architecture variable + id: set_arch + run: echo "ARCH=$(arch)" >> $GITHUB_ENV - - name: Grant sudo user permission to archive files + - name: Build container and disk image run: | - sudo chmod 0755 /tmp/tmp-bootc-build/id_rsa - - - name: Archive bootc disk image - disk.raw - if: matrix.test_runner == 'ubuntu-latest' - uses: actions/upload-artifact@v4 - with: - name: PR-${{ github.event.number }}-${{ matrix.test_os }}-disk - path: /tmp/tmp-bootc-build/disk.raw - retention-days: 1 + sudo tests/build.sh ${{ matrix.test_os }} - - name: Archive SSH private key - id_rsa - if: matrix.test_runner == 'ubuntu-latest' + - name: Archive disk image uses: actions/upload-artifact@v4 with: - name: PR-${{ github.event.number }}-${{ matrix.test_os }}-id_rsa - path: /tmp/tmp-bootc-build/id_rsa + name: PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ env.ARCH }}-disk + path: target/bootc-integration-test.qcow2 retention-days: 1 test: needs: build strategy: + fail-fast: false matrix: - test_os: [fedora-41, fedora-42, fedora-43, centos-9] - tmt_plan: [test-01-readonly, test-20-local-upgrade, test-21-logically-bound-switch, test-22-logically-bound-install, test-23-install-outside-container, test-24-local-upgrade-reboot] + test_os: [fedora-42, fedora-43, centos-9, centos-10] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Install dependence + - name: Set architecture variable + id: set_arch + run: echo "ARCH=$(arch)" >> $GITHUB_ENV + + - name: Install deps run: | sudo apt-get update - sudo apt install -y qemu-kvm qemu-system - pip install --user tmt + # see https://tmt.readthedocs.io/en/stable/overview.html#install + sudo apt install -y libkrb5-dev pkg-config libvirt-dev genisoimage qemu-kvm qemu-utils libvirt-daemon-system + pip install --user "tmt[provision-virtual]" - name: Create folder to save disk image - run: mkdir -p /tmp/tmp-bootc-build + run: mkdir -p target - name: Download disk.raw uses: actions/download-artifact@v4 with: - name: PR-${{ github.event.number }}-${{ matrix.test_os }}-disk - path: /tmp/tmp-bootc-build - - - name: Download id_rsa - uses: actions/download-artifact@v4 - with: - name: PR-${{ github.event.number }}-${{ matrix.test_os }}-id_rsa - path: /tmp/tmp-bootc-build + name: PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ env.ARCH }}-disk + path: target - name: Enable KVM group perms run: | @@ -87,14 +84,16 @@ jobs: sudo udevadm trigger --name-match=kvm ls -l /dev/kvm + - name: Workaround https://github.com/teemtee/testcloud/issues/18 + run: sudo rm -f /usr/bin/chcon && sudo ln -sr /usr/bin/true /usr/bin/chcon + - name: Run test - env: - TMT_PLAN_NAME: ${{ matrix.tmt_plan }} - run: chmod 600 /tmp/tmp-bootc-build/id_rsa && tests/test.sh + run: | + tests/run-tmt.sh - name: Archive TMT logs if: always() uses: actions/upload-artifact@v4 with: - name: tmt-log-PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ matrix.tmt_plan }} + name: tmt-log-PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ env.ARCH }}-${{ matrix.tmt_plan }} path: /var/tmp/tmt diff --git a/.packit.yaml b/.packit.yaml index 678360e1d..1747d4282 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -59,24 +59,26 @@ jobs: owner: rhcontainerbot project: bootc enable_net: true + # TODO notifications: failure_comment: message: "bootc Copr build failed for {commit_sha}. @admin check logs {logs_url} and packit dashboard {packit_dashboard_url}" - - job: tests - trigger: pull_request - targets: - - centos-stream-9-x86_64 - - centos-stream-9-aarch64 - - centos-stream-10-x86_64 - - centos-stream-10-aarch64 - - fedora-42-x86_64 - - fedora-42-aarch64 - - fedora-rawhide-x86_64 - - fedora-rawhide-aarch64 - tmt_plan: /integration - skip_build: true - identifier: integration-test + # TODO: Readd some tmt tests that install the built RPM and e.g. test out system-reinstall-bootc + # - job: tests + # trigger: pull_request + # targets: + # - centos-stream-9-x86_64 + # - centos-stream-9-aarch64 + # - centos-stream-10-x86_64 + # - centos-stream-10-aarch64 + # - fedora-42-x86_64 + # - fedora-42-aarch64 + # - fedora-rawhide-x86_64 + # - fedora-rawhide-aarch64 + # tmt_plan: /integration + # skip_build: true + # identifier: integration-test - job: propose_downstream trigger: release diff --git a/Justfile b/Justfile index 133de4d1f..86804929b 100644 --- a/Justfile +++ b/Justfile @@ -3,8 +3,14 @@ build *ARGS: podman build --jobs=4 -t localhost/bootc {{ARGS}} . # This container image has additional testing content and utilities -build-integration-test-image *ARGS: build +build-integration-test-image *ARGS: podman build --jobs=4 -t localhost/bootc-integration -f hack/Containerfile {{ARGS}} . + # Keep these in sync with what's used in hack/lbi + podman pull -q --retry 5 --retry-delay 5s quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.access.redhat.com/ubi9/podman:latest + +# Only used by ci.yml right now +build-install-test-image: build-integration-test-image + cd hack && podman build -t localhost/bootc-integration-install -f Containerfile.drop-lbis # Run container integration tests run-container-integration: build-integration-test-image diff --git a/Makefile b/Makefile index 03708b0a7..c70f7896c 100644 --- a/Makefile +++ b/Makefile @@ -96,9 +96,6 @@ bin-archive: all test-bin-archive: all $(MAKE) install-all DESTDIR=tmp-install && $(TAR_REPRODUCIBLE) --zstd -C tmp-install -cf target/bootc.tar.zst . && rm tmp-install -rf -test-tmt: - cargo xtask test-tmt - test: tests/build.sh && tests/test.sh diff --git a/crates/tests-integration/src/install.rs b/crates/tests-integration/src/install.rs index 172fea17b..aeef12c8d 100644 --- a/crates/tests-integration/src/install.rs +++ b/crates/tests-integration/src/install.rs @@ -26,6 +26,10 @@ pub(crate) fn delete_ostree(sh: &Shell) -> Result<(), anyhow::Error> { if !Path::new("/ostree/").exists() { return Ok(()); } + // TODO: This shouldn't be leaking out of installs + cmd!(sh, "sudo umount -Rl /ostree/bootc/storage/overlay") + .ignore_status() + .run()?; cmd!(sh, "sudo /bin/sh -c 'rm -rf /ostree/'").run()?; Ok(()) } diff --git a/crates/xtask/src/xtask.rs b/crates/xtask/src/xtask.rs index ec42cfeee..d19c54d8d 100644 --- a/crates/xtask/src/xtask.rs +++ b/crates/xtask/src/xtask.rs @@ -13,11 +13,6 @@ use xshell::{cmd, Shell}; mod man; const NAME: &str = "bootc"; -const TEST_IMAGES: &[&str] = &[ - "quay.io/curl/curl-base:latest", - "quay.io/curl/curl:latest", - "registry.access.redhat.com/ubi9/podman:latest", -]; const TAR_REPRODUCIBLE_OPTS: &[&str] = &[ "--sort=name", "--owner=0", @@ -43,7 +38,6 @@ const TASKS: &[(&str, fn(&Shell) -> Result<()>)] = &[ ("package", package), ("package-srpm", package_srpm), ("spec", spec), - ("test-tmt", test_tmt), ]; fn try_main() -> Result<()> { @@ -100,78 +94,6 @@ fn gitrev(sh: &Shell) -> Result { } } -#[context("test-integration")] -fn all_plan_files(sh: &Shell) -> Result> { - // We need to split most of our tests into separate plans because tmt doesn't - // support automatic isolation. (xref) - let mut all_plan_files = - sh.read_dir("plans")? - .into_iter() - .try_fold(Vec::new(), |mut acc, ent| -> Result<_> { - let path = Utf8PathBuf::try_from(ent)?; - let Some(ext) = path.extension() else { - return Ok(acc); - }; - if ext != "fmf" { - return Ok(acc); - } - let stem = path.file_stem().expect("file stem"); - let Some((prefix, suffix)) = stem.split_once('-') else { - return Ok(acc); - }; - if prefix != "test" { - return Ok(acc); - } - let Some((priority, _)) = suffix.split_once('-') else { - anyhow::bail!("Invalid test {path}"); - }; - let priority: u32 = priority - .parse() - .with_context(|| format!("Parsing {path}"))?; - acc.push((priority, stem.to_string())); - Ok(acc) - })?; - all_plan_files.sort_by_key(|v| v.0); - println!("Discovered plans: {all_plan_files:?}"); - Ok(all_plan_files) -} - -#[context("test-integration")] -fn test_tmt(sh: &Shell) -> Result<()> { - let mut tests = all_plan_files(sh)?; - if let Ok(name) = std::env::var("TMT_TEST") { - tests.retain(|x| x.1.as_str() == name); - if tests.is_empty() { - anyhow::bail!("Failed to match test: {name}"); - } - } - - // pull some small images that are used for LBI installation tests - cmd!(sh, "podman pull {TEST_IMAGES...}").run()?; - - for (_prio, name) in tests { - // cc https://pagure.io/testcloud/pull-request/174 - cmd!(sh, "rm -vf /var/tmp/tmt/testcloud/images/disk.qcow2").run()?; - let verbose_enabled = std::env::var("TMT_VERBOSE") - .ok() - .and_then(|s| s.parse::().ok()) - .unwrap_or(0); - - let verbose = if verbose_enabled == 1 { - Some("-vvvvv".to_string()) - } else { - None - }; - - if let Err(e) = cmd!(sh, "tmt {verbose...} run plans -n {name}").run() { - // tmt annoyingly does not output errors by default - let _ = cmd!(sh, "tmt run -l report -vvv").run(); - return Err(e.into()); - } - } - Ok(()) -} - /// Return a string formatted version of the git commit timestamp, up to the minute /// but not second because, well, we're not going to build more than once a second. #[context("Finding git timestamp")] diff --git a/hack/Containerfile b/hack/Containerfile index fa7e23a68..6ac333557 100644 --- a/hack/Containerfile +++ b/hack/Containerfile @@ -1,4 +1,7 @@ -# This injects some extra testing stuff into our image +# Build a container image that has extra testing stuff in it, such +# as nushell, some preset logically bound images, etc. This expects +# to create an image derived FROM localhost/bootc which was created +# by the Dockerfile at top. FROM scratch as context # We only need this stuff in the initial context @@ -11,7 +14,15 @@ ARG variant= # And this layer has additional stuff for testing, such as nushell etc. RUN --mount=type=bind,from=context,target=/run/context <> /usr/lib/bootc/kargs.d/20-console.toml +kargs = ["console=ttyS0,115200n8"] +KARGEOF +# And cloud-init stuff +ln -s ../cloud-init.target /usr/lib/systemd/system/default.target.wants + # Stock extra cleaning of logs and caches in general (mostly dnf) rm /var/log/* /var/cache /var/lib/{dnf,rpm-state,rhsm} -rf # And clean root's homedir rm /var/roothome/.config -rf +cat >/usr/lib/tmpfiles.d/bootc-cloud-init.conf <<'EOF' +d /var/lib/cloud 0755 root root - - +EOF # Fast track tmpfiles.d content from the base image, xref # https://gitlab.com/fedora/bootc/base-images/-/merge_requests/92 @@ -65,3 +80,23 @@ if ! grep -q -r sudo /usr/lib/sysusers.d; then g sudo 16 EOF fi + +# dhcpcd +if rpm -q dhcpcd &>/dev/null; then +if ! grep -q -r dhcpcd /usr/lib/sysusers.d; then + cat >/usr/lib/sysusers.d/bootc-dhcpcd-workaround.conf <<'EOF' +u dhcpcd - 'Minimalistic DHCP client' /var/lib/dhcpcd +EOF +fi +cat >/usr/lib/tmpfiles.d/bootc-dhcpd.conf <<'EOF' +d /var/lib/dhcpcd 0755 root dhcpcd - - +EOF + rm -rf /var/lib/dhcpcd +fi +# dhclient +if test -d /var/lib/dhclient; then + cat >/usr/lib/tmpfiles.d/bootc-dhclient.conf <<'EOF' +d /var/lib/dhclient 0755 root root - - +EOF + rm -rf /var/lib/dhclient +fi diff --git a/tests/build.sh b/tests/build.sh index dfbb6c5a1..642d0601d 100755 --- a/tests/build.sh +++ b/tests/build.sh @@ -4,116 +4,66 @@ set -exuo pipefail # This script basically builds bootc from source using the provided base image, # then runs the target tests. -mkdir -p /tmp/tmp-bootc-build -BOOTC_TEMPDIR="/tmp/tmp-bootc-build" - -# Get OS info from TEST_OS env -OS_ID=$(echo "$TEST_OS" | cut -d '-' -f 1) -OS_VERSION_ID=$(echo "$TEST_OS" | cut -d '-' -f 2) - -# Base image -case "$OS_ID" in - "centos") - TIER1_IMAGE_URL="quay.io/centos-bootc/centos-bootc:stream${OS_VERSION_ID}" +# If provided should be of the form fedora-42 or centos-10 +target=${1:-} + +bcvk=$(which bcvk 2>/dev/null || true) +if test -z "${bcvk}" && test "$(id -u)" != 0; then + echo "This script currently requires full root"; exit 1 +fi + +build_args=() +if test -n "${target:-}"; then + shift + # Get OS info from TEST_OS env + OS_ID=$(echo "$target" | cut -d '-' -f 1) + OS_VERSION_ID=$(echo "$target" | cut -d '-' -f 2) + + # Base image + case "$OS_ID" in + "centos") + BASE="quay.io/centos-bootc/centos-bootc:stream${OS_VERSION_ID}" ;; - "fedora") - TIER1_IMAGE_URL="quay.io/fedora/fedora-bootc:${OS_VERSION_ID}" + "fedora") + BASE="quay.io/fedora/fedora-bootc:${OS_VERSION_ID}" ;; -esac - -CONTAINERFILE="${BOOTC_TEMPDIR}/Containerfile" -tee "$CONTAINERFILE" > /dev/null << CONTAINERFILEOF -FROM $TIER1_IMAGE_URL as build - -WORKDIR /code - -RUN <> /usr/lib/bootc/kargs.d/20-console.toml -kargs = ["console=ttyS0,115200n8"] -KARGEOF - -# For test-22-logically-bound-install -cp -a /code/tmt/tests/lbi/usr/. /usr -ln -s /usr/share/containers/systemd/curl.container /usr/lib/bootc/bound-images.d/curl.container -ln -s /usr/share/containers/systemd/curl-base.image /usr/lib/bootc/bound-images.d/curl-base.image -ln -s /usr/share/containers/systemd/podman.image /usr/lib/bootc/bound-images.d/podman.image - -# Install rsync which is required by tmt -dnf -y install cloud-init rsync -dnf -y clean all - -rm -rf /var/cache /var/lib/dnf -EORUN -CONTAINERFILEOF - -LOCAL_IMAGE="localhost/bootc:test" -podman build \ - --retry 5 \ - --retry-delay 5s \ - -v "$(pwd)":/code:z \ - -t "$LOCAL_IMAGE" \ - -f "$CONTAINERFILE" \ - "$BOOTC_TEMPDIR" - -SSH_KEY=${BOOTC_TEMPDIR}/id_rsa -ssh-keygen -f "${SSH_KEY}" -N "" -q -t rsa-sha2-256 -b 2048 - -truncate -s 10G "${BOOTC_TEMPDIR}/disk.raw" - -# For test-22-logically-bound-install -podman pull --retry 5 --retry-delay 5s quay.io/curl/curl:latest -podman pull --retry 5 --retry-delay 5s quay.io/curl/curl-base:latest -podman pull --retry 5 --retry-delay 5s registry.access.redhat.com/ubi9/podman:latest - -podman run \ - --rm \ - --privileged \ - --pid=host \ - --security-opt label=type:unconfined_t \ - -v /var/lib/containers:/var/lib/containers \ - -v /dev:/dev \ - -v "$BOOTC_TEMPDIR":/output \ - "$LOCAL_IMAGE" \ - bootc install to-disk \ - --filesystem "xfs" \ - --root-ssh-authorized-keys "/output/id_rsa.pub" \ - --karg=console=ttyS0,115200n8 \ - --generic-image \ - --via-loopback \ - /output/disk.raw + *) echo "Unknown OS: ${OS_ID}" 1>&2; exit 1 + ;; + esac + build_args+=("--build-arg=base=$BASE") +fi + +just build ${build_args[@]} +just build-integration-test-image + +# Host builds will have this already, but we use it as a general dumping space +# for output artifacts +mkdir -p target + +SIZE=10G +DISK=target/bootc-integration-test.qcow2 +rm -vf "${DISK}" +# testcloud barfs on .raw +if test -n "${bcvk}"; then + bcvk to-disk --format=qcow2 --disk-size "${SIZE}" localhost/bootc-integration "${DISK}" +else + TMPDISK=target/bootc-integration-test.raw + truncate -s "${SIZE}" "${TMPDISK}" + podman run \ + --rm \ + --privileged \ + --pid=host \ + --security-opt label=type:unconfined_t \ + -v /var/lib/containers:/var/lib/containers \ + -v /dev:/dev \ + -v $(pwd)/target:/target \ + localhost/bootc-integration \ + bootc install to-disk \ + --filesystem "xfs" \ + --karg=console=ttyS0,115200n8 \ + --generic-image \ + --via-loopback \ + /target/$(basename ${TMPDISK}) + qemu-img convert -f raw -O qcow2 ${TMPDISK} ${DISK} + rm -f "${TMPDISK}" +fi diff --git a/tests/run-tmt.sh b/tests/run-tmt.sh new file mode 100755 index 000000000..8b89c02d1 --- /dev/null +++ b/tests/run-tmt.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -exuo pipefail + +# You must have invoked test/build.sh before running this. +# This is basically a wrapper for tmt which sets up context +# (to point to our disk image) and works around bugs in +# tmt and testcloud. +# Use e.g. `./tests/run-tmt.sh plan --name test-21-logically-bound-switch` +# to run an individual test. + +# Ensure we're in the topdir canonically +cd $(git rev-parse --show-toplevel) + +DISK=$(pwd)/target/bootc-integration-test.qcow2 +test -f "${DISK}" + +# Move the tmt bits to a subdirectory to work around https://github.com/teemtee/tmt/issues/4062 +mkdir -p target/tmt-workdir +rsync -a --delete --force .fmf tmt target/tmt-workdir/ + +# Hack around https://github.com/teemtee/testcloud/issues/17 +rm -vrf /var/tmp/tmt/testcloud/images/bootc-integration-test.qcow2 + +cd target/tmt-workdir +# TMT will rsync tmt-* scripts to TMT_SCRIPTS_DIR=/var/lib/tmt/scripts +exec tmt --context "test_disk_image=${DISK}" run --all -e TMT_SCRIPTS_DIR=/var/lib/tmt/scripts "$@" diff --git a/tests/test.sh b/tests/test.sh deleted file mode 100755 index 71fad8a45..000000000 --- a/tests/test.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -set -exuo pipefail - -# This script runs disk image with qemu-system and run tmt against this vm. - -BOOTC_TEMPDIR="/tmp/tmp-bootc-build" -SSH_OPTIONS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5) -SSH_KEY=${BOOTC_TEMPDIR}/id_rsa - -ARCH=$(uname -m) -case "$ARCH" in -"aarch64") - qemu-system-aarch64 \ - -name bootc-vm \ - -enable-kvm \ - -machine virt \ - -cpu host \ - -m 2G \ - -bios /usr/share/AAVMF/AAVMF_CODE.fd \ - -drive file="${BOOTC_TEMPDIR}/disk.raw",if=virtio,format=raw \ - -net nic,model=virtio \ - -net user,hostfwd=tcp::2222-:22 \ - -display none \ - -daemonize - ;; -"x86_64") - qemu-system-x86_64 \ - -name bootc-vm \ - -enable-kvm \ - -cpu host \ - -m 2G \ - -drive file="${BOOTC_TEMPDIR}/disk.raw",if=virtio,format=raw \ - -net nic,model=virtio \ - -net user,hostfwd=tcp::2222-:22 \ - -display none \ - -daemonize - ;; -*) - echo "Only support x86_64 and aarch64" >&2 - exit 1 - ;; -esac - -wait_for_ssh_up() { - SSH_STATUS=$(ssh "${SSH_OPTIONS[@]}" -i "$SSH_KEY" -p 2222 root@"${1}" '/bin/bash -c "echo -n READY"') - if [[ $SSH_STATUS == READY ]]; then - echo 1 - else - echo 0 - fi -} - -for _ in $(seq 0 30); do - RESULT=$(wait_for_ssh_up "localhost") - if [[ $RESULT == 1 ]]; then - echo "SSH is ready now! 🥳" - break - fi - sleep 10 -done - -# Make sure VM is ready for testing -ssh "${SSH_OPTIONS[@]}" \ - -i "$SSH_KEY" \ - -p 2222 \ - root@localhost \ - "bootc status" - -# TMT will rsync tmt-* scripts to TMT_SCRIPTS_DIR=/var/lib/tmt/scripts -tmt run --all --verbose -e TMT_SCRIPTS_DIR=/var/lib/tmt/scripts provision --how connect --guest localhost --port 2222 --user root --key "$SSH_KEY" plan --name "/tmt/plans/bootc-integration/${TMT_PLAN_NAME}" diff --git a/tmt/plans/bootc-integration.fmf b/tmt/plans/bootc-integration.fmf deleted file mode 100644 index 76bd84732..000000000 --- a/tmt/plans/bootc-integration.fmf +++ /dev/null @@ -1,46 +0,0 @@ -execute: - how: tmt - -/test-01-readonly: - summary: Execute booted readonly/nondestructive tests - discover: - how: fmf - test: - - /tmt/tests/test-01-readonly - -/test-20-local-upgrade: - summary: Execute local upgrade tests - discover: - how: fmf - test: - - /tmt/tests/test-20-local-upgrade - -/test-21-logically-bound-switch: - summary: Execute logically bound images tests for switching images - discover: - how: fmf - test: - - /tmt/tests/test-21-logically-bound-switch - -/test-22-logically-bound-install: - summary: Execute logically bound images tests for switching images - environment+: - LBI: enabled - discover: - how: fmf - test: - - /tmt/tests/test-22-logically-bound-install - -/test-23-install-outside-container: - summary: Execute tests for installing outside of a container - discover: - how: fmf - test: - - /tmt/tests/test-23-install-outside-container - -/test-24-local-upgrade-reboot: - summary: Execute local upgrade tests with automated reboot - discover: - how: fmf - test: - - /tmt/tests/test-24-local-upgrade-reboot diff --git a/tmt/plans/integration.fmf b/tmt/plans/integration.fmf index a02912c79..34ad9416e 100644 --- a/tmt/plans/integration.fmf +++ b/tmt/plans/integration.fmf @@ -1,24 +1,15 @@ -# Please change the image when you run this plan locally with tmt run -# tmt run -vvvvv plan -n /integration/test-01-readonly -# local image: file:///home/foobar/image.qcow2 provision: how: virtual - image: fedora-rawhide -prepare: - - how: install - package: - - podman - - skopeo - - jq + # Build via `./tests/build.sh` + image: $@{test_disk_image} execute: how: tmt -/test-01-readonly: +/readonly-tests: summary: Execute booted readonly/nondestructive tests discover: how: fmf test: - - /tmt/tests/bootc-install-provision - /tmt/tests/test-01-readonly /test-20-local-upgrade: @@ -26,7 +17,6 @@ execute: discover: how: fmf test: - - /tmt/tests/bootc-install-provision - /tmt/tests/test-20-local-upgrade /test-21-logically-bound-switch: @@ -34,17 +24,13 @@ execute: discover: how: fmf test: - - /tmt/tests/bootc-install-provision - /tmt/tests/test-21-logically-bound-switch /test-22-logically-bound-install: summary: Execute logically bound images tests for switching images - environment+: - LBI: enabled discover: how: fmf test: - - /tmt/tests/bootc-install-provision - /tmt/tests/test-22-logically-bound-install /test-23-install-outside-container: @@ -52,7 +38,6 @@ execute: discover: how: fmf test: - - /tmt/tests/bootc-install-provision - /tmt/tests/test-23-install-outside-container /test-24-local-upgrade-reboot: @@ -60,7 +45,6 @@ execute: discover: how: fmf test: - - /tmt/tests/bootc-install-provision - /tmt/tests/test-24-local-upgrade-reboot /test-25-soft-reboot: @@ -68,5 +52,4 @@ execute: discover: how: fmf test: - - /tmt/tests/bootc-install-provision - /tmt/tests/test-25-soft-reboot diff --git a/tmt/tests/bootc-install-provision.fmf b/tmt/tests/bootc-install-provision.fmf deleted file mode 100644 index d55e5c71a..000000000 --- a/tmt/tests/bootc-install-provision.fmf +++ /dev/null @@ -1,3 +0,0 @@ -summary: Run bootc install to-existing-root to deploy bootc image -test: ./bootc-install-provision.sh -duration: 30m diff --git a/tmt/tests/bootc-install-provision.sh b/tmt/tests/bootc-install-provision.sh deleted file mode 100755 index ca8f82dce..000000000 --- a/tmt/tests/bootc-install-provision.sh +++ /dev/null @@ -1,215 +0,0 @@ -#!/bin/bash -set -exuo pipefail - -# This script basically builds bootc from source using the provided base image, -# then runs the target tests. We need to do this because at the moment -# packit/tmt/testing-farm effectively only support RPMs, not container images. -# https://issues.redhat.com/browse/TFT-2751 - -BOOTC_TEMPDIR=$(mktemp -d) -trap 'rm -rf -- "$BOOTC_TEMPDIR"' EXIT - -# LBI only enabled for test-22-logically-bound-install -LBI="${LBI:-disabled}" - -# Get OS info -source /etc/os-release -case "$ID" in - "centos") - TIER1_IMAGE_URL="${TIER1_IMAGE_URL:-quay.io/centos-bootc/centos-bootc:stream${VERSION_ID}}" - ;; - "fedora") - TIER1_IMAGE_URL="${TIER1_IMAGE_URL:-quay.io/fedora/fedora-bootc:${VERSION_ID}}" - ;; -esac - -if [ "$TMT_REBOOT_COUNT" -eq 0 ]; then - # Let's move to bootc root folder - cd ../.. - # Fedora CI: https://github.com/fedora-ci/dist-git-pipeline/blob/master/Jenkinsfile#L145 - # OSCI: https://gitlab.cee.redhat.com/osci-pipelines/dist-git-pipeline/-/blob/master/Jenkinsfile?ref_type=heads#L93 - if [[ -v KOJI_TASK_ID ]] || [[ -v CI_KOJI_TASK_ID ]]; then - # Just left those ls commands here to ring the bell for me when something changed - echo "$TMT_SOURCE_DIR" - ls -al "$TMT_SOURCE_DIR" - ls -al "$TMT_SOURCE_DIR/SRPMS" - ls -al /etc/yum.repos.d - cat /etc/yum.repos.d/test-artifacts.repo - ls -al /var/share/test-artifacts - fi - - # TMT needs this key - cp -r /root/.ssh "$BOOTC_TEMPDIR" - - # Running on Testing Farm - if [[ -d "/var/ARTIFACTS" ]]; then - cp -r /var/ARTIFACTS "$BOOTC_TEMPDIR" - # Running on local machine with tmt run - else - cp -r /var/tmp/tmt "$BOOTC_TEMPDIR" - fi - - # Some rhts-*, rstrnt-* and tmt-* commands are in /usr/local/bin - cp -r /usr/local/bin "$BOOTC_TEMPDIR" - - # Check image building folder content - ls -al "$BOOTC_TEMPDIR" - - CONTAINERFILE=${BOOTC_TEMPDIR}/Containerfile - - COMMON_CONTAINERFILE="${BOOTC_TEMPDIR}/common_containerfile" - tee "$COMMON_CONTAINERFILE" > /dev/null << COMMONEOF -RUN <> /usr/lib/bootc/kargs.d/20-console.toml -kargs = ["console=ttyS0,115200n8"] -KARGEOF - -# cloud-init and rsync are required by TMT -dnf -y install cloud-init rsync -ln -s ../cloud-init.target /usr/lib/systemd/system/default.target.wants -dnf -y clean all - -rm -rf /var/cache /var/lib/dnf -EORUN - -# Some rhts-*, rstrnt-* and tmt-* commands are in /usr/local/bin -COPY bin /usr/local/bin - -# In Testing Farm, all ssh things should be reserved for ssh command run after reboot -COPY .ssh /var/roothome/.ssh -COMMONEOF - - if [[ -v KOJI_TASK_ID ]] || [[ -v CI_KOJI_TASK_ID ]]; then - FEDORA_CI_CONTAINERFILE="${BOOTC_TEMPDIR}/fedora_ci_containerfile" - tee "$FEDORA_CI_CONTAINERFILE" > /dev/null << FEDORACIEOF -FROM $TIER1_IMAGE_URL - -RUN dnf -y upgrade /rpms/*.rpm -FEDORACIEOF - cat >"$CONTAINERFILE" < /dev/null <"$CONTAINERFILE" <> "$CONTAINERFILE" - else - # In local machine, TMT work dir /var/tmp/tmt should be reserved - echo "COPY tmt /var/tmp/tmt" >> "$CONTAINERFILE" - fi - - # For test-22-logically-bound-install - if [[ "$LBI" == "enabled" ]]; then - echo "RUN cp -a /code/tmt/tests/lbi/usr/. /usr && ln -s /usr/share/containers/systemd/curl.container /usr/lib/bootc/bound-images.d/curl.container && ln -s /usr/share/containers/systemd/curl-base.image /usr/lib/bootc/bound-images.d/curl-base.image && ln -s /usr/share/containers/systemd/podman.image /usr/lib/bootc/bound-images.d/podman.image" >> "$CONTAINERFILE" - podman pull --retry 5 --retry-delay 5s quay.io/curl/curl:latest - podman pull --retry 5 --retry-delay 5s quay.io/curl/curl-base:latest - podman pull --retry 5 --retry-delay 5s registry.access.redhat.com/ubi9/podman:latest - fi - - cat "$CONTAINERFILE" - # Retry here to avoid quay.io "502 Bad Gateway" - # bind mount bootc source code folder for bootc binary building and run test provision - # bind mount /var/share/test-artifacts for bootc RPM package installation in Fedora CI and OSCI - if [[ -v KOJI_TASK_ID ]] || [[ -v CI_KOJI_TASK_ID ]]; then - podman build \ - --retry 5 \ - --retry-delay 5s \ - --tls-verify=false \ - -v /var/share/test-artifacts:/rpms:z \ - -v "$(pwd)":/code:z \ - -t localhost/bootc:tmt \ - -f "$CONTAINERFILE" \ - "$BOOTC_TEMPDIR" - else - podman build \ - --retry 5 \ - --retry-delay 5s \ - --tls-verify=false \ - -v "$(pwd)":/code:z \ - -t localhost/bootc:tmt \ - -f "$CONTAINERFILE" \ - "$BOOTC_TEMPDIR" - fi - - podman images - podman run \ - --rm \ - --tls-verify=false \ - --privileged \ - --pid=host \ - -v /:/target \ - -v /dev:/dev \ - -v /var/lib/containers:/var/lib/containers \ - -v /root/.ssh:/output \ - --security-opt label=type:unconfined_t \ - "localhost/bootc:tmt" \ - bootc install to-existing-root --target-transport containers-storage --acknowledge-destructive - - # Reboot - tmt-reboot -elif [ "$TMT_REBOOT_COUNT" -eq 1 ]; then - # Some simple and fast checkings - bootc status - echo "$PATH" - printenv - if [[ -d "/var/ARTIFACTS" ]]; then - ls -al /var/ARTIFACTS - else - ls -al /var/tmp/tmt - fi - ls -al /usr/local/bin - echo "Bootc system on TMT/TF runner" - - exit 0 -fi diff --git a/tmt/tests/booted/README.md b/tmt/tests/booted/README.md index 7c138d27b..359bb3962 100644 --- a/tmt/tests/booted/README.md +++ b/tmt/tests/booted/README.md @@ -1,4 +1,3 @@ # Booted tests -These are intended to run via tmt; use e.g. -`make test-tmt`. +These are intended to run via tmt. From 6c415c50ad073623a473368281b60ad23b9ec910 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 18 Sep 2025 08:05:34 -0400 Subject: [PATCH 2/2] tests: Bump timeout for expect This should reduce the flake rate. Signed-off-by: Colin Walters --- crates/tests-integration/src/system_reinstall.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/tests-integration/src/system_reinstall.rs b/crates/tests-integration/src/system_reinstall.rs index 70bb3edc6..f6ed44a5d 100644 --- a/crates/tests-integration/src/system_reinstall.rs +++ b/crates/tests-integration/src/system_reinstall.rs @@ -6,11 +6,13 @@ use rustix::fs::statfs; use std::{ fs::{self}, path::Path, + time::Duration, }; use crate::install; -const TIMEOUT: u64 = 60000; +/// A generous timeout since some CI runs may be slower +const TIMEOUT: Duration = std::time::Duration::from_secs(5 * 60); fn get_deployment_dir() -> Result { let base_path = Path::new("/ostree/deploy/default/deploy"); @@ -58,7 +60,7 @@ pub(crate) fn run(image: &str, testargs: libtest_mimic::Arguments) -> Result<()> let mut p: PtySession = rexpect::spawn( format!("/usr/bin/system-reinstall-bootc {image}").as_str(), - Some(TIMEOUT), + Some(TIMEOUT.as_millis().try_into().unwrap()), )?; // Basic flow stdout verification @@ -132,7 +134,7 @@ pub(crate) fn run(image: &str, testargs: libtest_mimic::Arguments) -> Result<()> // Run system-reinstall-bootc let mut p: PtySession = rexpect::spawn( format!("/usr/bin/system-reinstall-bootc {image}").as_str(), - Some(TIMEOUT), + Some(TIMEOUT.as_millis().try_into().unwrap()), )?; p.exp_regex("Found only one user ([^:]+) with ([\\d]+) SSH authorized keys.")?;