ci: Unify more of hack/ and tests/

.github/workflows/ci.yml

@@ -56,7 +56,9 @@ jobs:
         run: sudo apt update && sudo apt install just
       - uses: actions/checkout@v4
       - name: Build and run container integration tests
-        run: sudo just run-container-integration run-container-external-tests
+        run: |
+          sudo just build
+          sudo just run-container-integration run-container-external-tests
   container-continuous:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'control/skip-ci') }}
     runs-on: ubuntu-24.04
@@ -105,7 +107,8 @@ jobs:
           set -xeu
           # Build images to test; TODO investigate doing single container builds
           # via GHA and pushing to a temporary registry to share among workflows?
-          sudo just build-integration-test-image
+          sudo just build
+          sudo just build-install-test-image
           sudo podman build -t localhost/bootc-fsverity -f ci/Containerfile.install-fsverity
 
           # TODO move into a container, and then have this tool run other containers
@@ -120,9 +123,9 @@ jobs:
           sudo podman run --privileged --pid=host -v /:/run/host -v $(pwd):/src:ro -v /var/tmp:/var/tmp \
             -v /run/dbus:/run/dbus -v /run/systemd:/run/systemd localhost/bootc /src/crates/ostree-ext/ci/priv-integration.sh
           # Nondestructive but privileged tests
-          sudo bootc-integration-tests host-privileged localhost/bootc-integration
+          sudo bootc-integration-tests host-privileged localhost/bootc-integration-install
           # Install tests
-          sudo bootc-integration-tests install-alongside localhost/bootc-integration
+          sudo bootc-integration-tests install-alongside localhost/bootc-integration-install
 
           # system-reinstall-bootc tests
           cargo build --release -p system-reinstall-bootc

.github/workflows/integration.yml

@@ -1,84 +1,81 @@
-name: bootc integration test
+# This workflow builds a container across a matrix of OSes,
+# generates a disk image from that, and runs integration tests
+# using tmt + libvirt (using nested virt support in the default GHA runners).
+name: Build+TMT
 on:
   pull_request:
-    branches: [main]
+    branches: [main]  
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   build:
     strategy:
+      fail-fast: false
       matrix:
-        test_os: [fedora-41, fedora-42, fedora-43, centos-9]
-        test_runner: [ubuntu-latest, ubuntu-24.04-arm]
+        test_os: [fedora-42, fedora-43, centos-9, centos-10]
 
-    runs-on: ${{ matrix.test_runner }}
+    runs-on: ubuntu-24.04
 
     steps:
-      - name: Install podman for heredoc support
+      - name: Install dependencies
         run: |
           set -eux
           echo 'deb [trusted=yes] https://ftp.debian.org/debian/ testing main' | sudo tee /etc/apt/sources.list.d/testing.list
           sudo apt update
-          sudo apt install -y crun/testing podman/testing
+          sudo apt install -y crun/testing podman/testing just qemu-utils
 
       - uses: actions/checkout@v4
 
-      - name: Build bootc and bootc image
-        env:
-          TEST_OS: ${{ matrix.test_os }}
-        run: sudo -E TEST_OS=$TEST_OS tests/build.sh
+      - name: Set architecture variable
+        id: set_arch
+        run: echo "ARCH=$(arch)" >> $GITHUB_ENV
 
-      - name: Grant sudo user permission to archive files
+      - name: Build container and disk image
         run: |
-          sudo chmod 0755 /tmp/tmp-bootc-build/id_rsa
-
-      - name: Archive bootc disk image - disk.raw
-        if: matrix.test_runner == 'ubuntu-latest'
-        uses: actions/upload-artifact@v4
-        with:
-          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-disk
-          path: /tmp/tmp-bootc-build/disk.raw
-          retention-days: 1
+          sudo tests/build.sh ${{ matrix.test_os }}
 
-      - name: Archive SSH private key - id_rsa
-        if: matrix.test_runner == 'ubuntu-latest'
+      - name: Archive disk image
         uses: actions/upload-artifact@v4
         with:
-          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-id_rsa
-          path: /tmp/tmp-bootc-build/id_rsa
+          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ env.ARCH }}-disk
+          path: target/bootc-integration-test.qcow2
           retention-days: 1
 
   test:
     needs: build
     strategy:
+      fail-fast: false
       matrix:
-        test_os: [fedora-41, fedora-42, fedora-43, centos-9]
-        tmt_plan: [test-01-readonly, test-20-local-upgrade, test-21-logically-bound-switch, test-22-logically-bound-install, test-23-install-outside-container, test-24-local-upgrade-reboot]
+        test_os: [fedora-42, fedora-43, centos-9, centos-10]
 
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install dependence
+      - name: Set architecture variable
+        id: set_arch
+        run: echo "ARCH=$(arch)" >> $GITHUB_ENV
+
+      - name: Install deps
         run: |
           sudo apt-get update
-          sudo apt install -y qemu-kvm qemu-system
-          pip install --user tmt
+          # see https://tmt.readthedocs.io/en/stable/overview.html#install
+          sudo apt install -y libkrb5-dev pkg-config libvirt-dev genisoimage qemu-kvm qemu-utils libvirt-daemon-system
+          pip install --user "tmt[provision-virtual]"
 
       - name: Create folder to save disk image
-        run: mkdir -p /tmp/tmp-bootc-build
+        run: mkdir -p target
 
       - name: Download disk.raw
         uses: actions/download-artifact@v4
         with:
-          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-disk
-          path: /tmp/tmp-bootc-build
-
-      - name: Download id_rsa
-        uses: actions/download-artifact@v4
-        with:
-          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-id_rsa
-          path: /tmp/tmp-bootc-build
+          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ env.ARCH }}-disk
+          path: target
 
       - name: Enable KVM group perms
         run: |
@@ -87,14 +84,16 @@ jobs:
           sudo udevadm trigger --name-match=kvm
           ls -l /dev/kvm
 
+      - name: Workaround https://github.com/teemtee/testcloud/issues/18
+        run: sudo rm -f /usr/bin/chcon && sudo ln -sr /usr/bin/true /usr/bin/chcon
+
       - name: Run test
-        env:
-          TMT_PLAN_NAME: ${{ matrix.tmt_plan }}
-        run: chmod 600 /tmp/tmp-bootc-build/id_rsa && tests/test.sh
+        run: |
+          tests/run-tmt.sh
 
       - name: Archive TMT logs
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: tmt-log-PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ matrix.tmt_plan }}
+          name: tmt-log-PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ env.ARCH }}-${{ matrix.tmt_plan }}
           path: /var/tmp/tmt

.packit.yaml

@@ -59,24 +59,26 @@ jobs:
     owner: rhcontainerbot
     project: bootc
     enable_net: true
+    # TODO
     notifications:
       failure_comment:
         message: "bootc Copr build failed for {commit_sha}. @admin check logs {logs_url} and packit dashboard {packit_dashboard_url}"
 
-  - job: tests
-    trigger: pull_request
-    targets:
-      - centos-stream-9-x86_64
-      - centos-stream-9-aarch64
-      - centos-stream-10-x86_64
-      - centos-stream-10-aarch64
-      - fedora-42-x86_64
-      - fedora-42-aarch64
-      - fedora-rawhide-x86_64
-      - fedora-rawhide-aarch64
-    tmt_plan: /integration
-    skip_build: true
-    identifier: integration-test
+  # TODO: Readd some tmt tests that install the built RPM and e.g. test out system-reinstall-bootc
+  # - job: tests
+  #   trigger: pull_request
+  #   targets:
+  #     - centos-stream-9-x86_64
+  #     - centos-stream-9-aarch64
+  #     - centos-stream-10-x86_64
+  #     - centos-stream-10-aarch64
+  #     - fedora-42-x86_64
+  #     - fedora-42-aarch64
+  #     - fedora-rawhide-x86_64
+  #     - fedora-rawhide-aarch64
+  #   tmt_plan: /integration
+  #   skip_build: true
+  #   identifier: integration-test
 
   - job: propose_downstream
     trigger: release

Justfile

@@ -3,8 +3,14 @@ build *ARGS:
     podman build --jobs=4 -t localhost/bootc {{ARGS}} .
 
 # This container image has additional testing content and utilities
-build-integration-test-image *ARGS: build
+build-integration-test-image *ARGS:
     podman build --jobs=4 -t localhost/bootc-integration -f hack/Containerfile {{ARGS}} .
+    # Keep these in sync with what's used in hack/lbi
+    podman pull -q --retry 5 --retry-delay 5s quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.access.redhat.com/ubi9/podman:latest
+
+# Only used by ci.yml right now
+build-install-test-image: build-integration-test-image
+    cd hack && podman build -t localhost/bootc-integration-install -f Containerfile.drop-lbis
 
 # Run container integration tests
 run-container-integration: build-integration-test-image

Makefile

@@ -96,9 +96,6 @@ bin-archive: all
 test-bin-archive: all
 	$(MAKE) install-all DESTDIR=tmp-install && $(TAR_REPRODUCIBLE) --zstd -C tmp-install -cf target/bootc.tar.zst . && rm tmp-install -rf
 
-test-tmt:
-	cargo xtask test-tmt
-
 test:
 	tests/build.sh && tests/test.sh
 

crates/tests-integration/src/install.rs

@@ -26,6 +26,10 @@ pub(crate) fn delete_ostree(sh: &Shell) -> Result<(), anyhow::Error> {
     if !Path::new("/ostree/").exists() {
         return Ok(());
     }
+    // TODO: This shouldn't be leaking out of installs
+    cmd!(sh, "sudo umount -Rl /ostree/bootc/storage/overlay")
+        .ignore_status()
+        .run()?;
     cmd!(sh, "sudo /bin/sh -c 'rm -rf /ostree/'").run()?;
     Ok(())
 }

crates/tests-integration/src/system_reinstall.rs

@@ -6,11 +6,13 @@ use rustix::fs::statfs;
 use std::{
     fs::{self},
     path::Path,
+    time::Duration,
 };
 
 use crate::install;
 
-const TIMEOUT: u64 = 60000;
+/// A generous timeout since some CI runs may be slower
+const TIMEOUT: Duration = std::time::Duration::from_secs(5 * 60);
 
 fn get_deployment_dir() -> Result<std::path::PathBuf> {
     let base_path = Path::new("/ostree/deploy/default/deploy");
@@ -58,7 +60,7 @@ pub(crate) fn run(image: &str, testargs: libtest_mimic::Arguments) -> Result<()>
 
             let mut p: PtySession = rexpect::spawn(
                 format!("/usr/bin/system-reinstall-bootc {image}").as_str(),
-                Some(TIMEOUT),
+                Some(TIMEOUT.as_millis().try_into().unwrap()),
             )?;
 
             // Basic flow stdout verification
@@ -132,7 +134,7 @@ pub(crate) fn run(image: &str, testargs: libtest_mimic::Arguments) -> Result<()>
             // Run system-reinstall-bootc
             let mut p: PtySession = rexpect::spawn(
                 format!("/usr/bin/system-reinstall-bootc {image}").as_str(),
-                Some(TIMEOUT),
+                Some(TIMEOUT.as_millis().try_into().unwrap()),
             )?;
 
             p.exp_regex("Found only one user ([^:]+) with ([\\d]+) SSH authorized keys.")?;

crates/xtask/src/xtask.rs

@@ -13,11 +13,6 @@ use xshell::{cmd, Shell};
 mod man;
 
 const NAME: &str = "bootc";
-const TEST_IMAGES: &[&str] = &[
-    "quay.io/curl/curl-base:latest",
-    "quay.io/curl/curl:latest",
-    "registry.access.redhat.com/ubi9/podman:latest",
-];
 const TAR_REPRODUCIBLE_OPTS: &[&str] = &[
     "--sort=name",
     "--owner=0",
@@ -43,7 +38,6 @@ const TASKS: &[(&str, fn(&Shell) -> Result<()>)] = &[
     ("package", package),
     ("package-srpm", package_srpm),
     ("spec", spec),
-    ("test-tmt", test_tmt),
 ];
 
 fn try_main() -> Result<()> {
@@ -100,78 +94,6 @@ fn gitrev(sh: &Shell) -> Result<String> {
     }
 }
 
-#[context("test-integration")]
-fn all_plan_files(sh: &Shell) -> Result<Vec<(u32, String)>> {
-    // We need to split most of our tests into separate plans because tmt doesn't
-    // support automatic isolation. (xref)
-    let mut all_plan_files =
-        sh.read_dir("plans")?
-            .into_iter()
-            .try_fold(Vec::new(), |mut acc, ent| -> Result<_> {
-                let path = Utf8PathBuf::try_from(ent)?;
-                let Some(ext) = path.extension() else {
-                    return Ok(acc);
-                };
-                if ext != "fmf" {
-                    return Ok(acc);
-                }
-                let stem = path.file_stem().expect("file stem");
-                let Some((prefix, suffix)) = stem.split_once('-') else {
-                    return Ok(acc);
-                };
-                if prefix != "test" {
-                    return Ok(acc);
-                }
-                let Some((priority, _)) = suffix.split_once('-') else {
-                    anyhow::bail!("Invalid test {path}");
-                };
-                let priority: u32 = priority
-                    .parse()
-                    .with_context(|| format!("Parsing {path}"))?;
-                acc.push((priority, stem.to_string()));
-                Ok(acc)
-            })?;
-    all_plan_files.sort_by_key(|v| v.0);
-    println!("Discovered plans: {all_plan_files:?}");
-    Ok(all_plan_files)
-}
-
-#[context("test-integration")]
-fn test_tmt(sh: &Shell) -> Result<()> {
-    let mut tests = all_plan_files(sh)?;
-    if let Ok(name) = std::env::var("TMT_TEST") {
-        tests.retain(|x| x.1.as_str() == name);
-        if tests.is_empty() {
-            anyhow::bail!("Failed to match test: {name}");
-        }
-    }
-
-    // pull some small images that are used for LBI installation tests
-    cmd!(sh, "podman pull {TEST_IMAGES...}").run()?;
-
-    for (_prio, name) in tests {
-        // cc https://pagure.io/testcloud/pull-request/174
-        cmd!(sh, "rm -vf /var/tmp/tmt/testcloud/images/disk.qcow2").run()?;
-        let verbose_enabled = std::env::var("TMT_VERBOSE")
-            .ok()
-            .and_then(|s| s.parse::<u32>().ok())
-            .unwrap_or(0);
-
-        let verbose = if verbose_enabled == 1 {
-            Some("-vvvvv".to_string())
-        } else {
-            None
-        };
-
-        if let Err(e) = cmd!(sh, "tmt {verbose...} run plans -n {name}").run() {
-            // tmt annoyingly does not output errors by default
-            let _ = cmd!(sh, "tmt run -l report -vvv").run();
-            return Err(e.into());
-        }
-    }
-    Ok(())
-}
-
 /// Return a string formatted version of the git commit timestamp, up to the minute
 /// but not second because, well, we're not going to build more than once a second.
 #[context("Finding git timestamp")]