More cfs testing2

.github/workflows/ci.yml

@@ -207,9 +207,9 @@ jobs:
       - name: Unit and container integration tests
         run: just test-container
 
-      - name: Run readonly TMT tests
-        # TODO: expand to more tests
-        run: just test-tmt readonly
+      - name: Run TMT tests
+        # Note that this one only runs a subset of tests right now
+        run: just test-composefs
 
       - name: Archive TMT logs
         if: always()

Justfile

@@ -11,6 +11,13 @@
 
 # --------------------------------------------------------------------
 
+# This image is just the base image plus our updated bootc binary
+base_img := "localhost/bootc"
+# Derives from the above and adds nushell, cloudinit etc.
+integration_img := base_img + "-integration"
+# Has a synthetic upgrade
+integration_upgrade_img := integration_img + "-upgrade"
+
 # ostree: The default
 # composefs-sealeduki-sdboot: A system with a sealed composefs using systemd-boot
 variant := env("BOOTC_variant", "ostree")
@@ -33,8 +40,8 @@ buildargs := "--build-arg=base=" + base + " --build-arg=variant=" + variant
 # Note commonly you might want to override the base image via e.g.
 # `just build --build-arg=base=quay.io/fedora/fedora-bootc:42`
 build:
-    podman build {{base_buildargs}} -t localhost/bootc-bin {{buildargs}} .
-    ./tests/build-sealed {{variant}} localhost/bootc-bin localhost/bootc
+    podman build {{base_buildargs}} -t {{base_img}}-bin {{buildargs}} .
+    ./tests/build-sealed {{variant}} {{base_img}}-bin {{base_img}}
 
 # Build a sealed image from current sources.
 build-sealed:
@@ -66,27 +73,27 @@ package: _packagecontainer
 
 # This container image has additional testing content and utilities
 build-integration-test-image: build
-    cd hack && podman build {{base_buildargs}} -t localhost/bootc-integration-bin -f Containerfile .
-    ./tests/build-sealed {{variant}} localhost/bootc-integration-bin localhost/bootc-integration
+    cd hack && podman build {{base_buildargs}} -t {{integration_img}}-bin -f Containerfile .
+    ./tests/build-sealed {{variant}} {{integration_img}}-bin {{integration_img}}
     # Keep these in sync with what's used in hack/lbi
     podman pull -q --retry 5 --retry-delay 5s quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.access.redhat.com/ubi9/podman:latest
 
-# Build+test composefs; compat alias
+# Build+test using the `composefs-sealeduki-sdboot` variant.
 test-composefs:
     # These first two are currently a distinct test suite from tmt that directly
     # runs an integration test binary in the base image via bcvk
     just variant=composefs-sealeduki-sdboot build
-    cargo run --release -p tests-integration -- composefs-bcvk localhost/bootc
-    # We're trying to move more testing to tmt, so 
-    just variant=composefs-sealeduki-sdboot test-tmt readonly
+    cargo run --release -p tests-integration -- composefs-bcvk {{base_img}}
+    # We're trying to move more testing to tmt
+    just variant=composefs-sealeduki-sdboot test-tmt readonly local-upgrade-reboot
 
 # Only used by ci.yml right now
 build-install-test-image: build-integration-test-image
-    cd hack && podman build {{base_buildargs}} -t localhost/bootc-integration-install -f Containerfile.drop-lbis
+    cd hack && podman build {{base_buildargs}} -t {{integration_img}}-install -f Containerfile.drop-lbis
 
 # These tests accept the container image as input, and may spawn it.
 run-container-external-tests:
-   ./tests/container/run localhost/bootc
+   ./tests/container/run {{base_img}}
 
 # We build the unit tests into a container image
 build-units:
@@ -101,8 +108,18 @@ validate:
 #
 # To run an individual test, pass it as an argument like:
 # `just test-tmt readonly`
-test-tmt *ARGS: build-integration-test-image
-    cargo xtask run-tmt --env=BOOTC_variant={{variant}} localhost/bootc-integration {{ARGS}}
+test-tmt *ARGS: build-integration-test-image _build-upgrade-image
+    @just test-tmt-nobuild {{ARGS}}
+
+# Generate a local synthetic upgrade
+_build-upgrade-image:
+    cat tmt/tests/Dockerfile.upgrade | podman build -t {{integration_upgrade_img}}-bin --from={{integration_img}}-bin -
+    ./tests/build-sealed {{variant}} {{integration_upgrade_img}}-bin {{integration_upgrade_img}}
+
+# Assume the localhost/bootc-integration image is up to date, and just run tests.
+# Useful for iterating on tests quickly.
+test-tmt-nobuild *ARGS:
+    cargo xtask run-tmt --env=BOOTC_variant={{variant}} --env=BOOTC_upgrade_image={{integration_upgrade_img}} {{integration_img}} {{ARGS}}
 
 # Cleanup all test VMs created by tmt tests
 tmt-vm-cleanup:
@@ -112,7 +129,7 @@ tmt-vm-cleanup:
 test-container: build-units build-integration-test-image
     podman run --rm --read-only localhost/bootc-units /usr/bin/bootc-units
     # Pass these through for cross-checking
-    podman run --rm --env=BOOTC_variant={{variant}} --env=BOOTC_base={{base}} localhost/bootc-integration bootc-integration-tests container
+    podman run --rm --env=BOOTC_variant={{variant}} --env=BOOTC_base={{base}} {{integration_img}} bootc-integration-tests container
 
 # Remove all container images built (locally) via this Justfile, by matching a label
 clean-local-images:

Makefile

@@ -30,15 +30,16 @@ prefix ?= /usr
 # the code is really tiny.
 # (Note we should also make installation of the units conditional on the rhsm feature)
 CARGO_FEATURES ?= $(shell . /usr/lib/os-release; if echo "$$ID_LIKE" |grep -qF rhel; then echo rhsm; fi)
+CARGO_PROFILE ?= release
 
 all: bin manpages
 
 bin:
-	cargo build --release --features "$(CARGO_FEATURES)"
+	cargo build --profile $(CARGO_PROFILE) --features "$(CARGO_FEATURES)"
 
 .PHONY: manpages
 manpages:
-	cargo run --package xtask -- manpages
+	cargo run --profile $(CARGO_PROFILE) --package xtask -- manpages
 
 STORAGE_RELATIVE_PATH ?= $(shell realpath -m -s --relative-to="$(prefix)/lib/bootc/storage" /sysroot/ostree/bootc/storage)
 install:

crates/xtask/src/xtask.rs

@@ -491,18 +491,18 @@ fn generate_random_suffix() -> String {
         .collect()
 }
 
-/// Sanitize a plan name for use in a VM name
+/// Sanitize a name for use in a VM name
 /// Replaces non-alphanumeric characters (except - and _) with dashes
-/// Returns "plan" if the result would be empty
-fn sanitize_plan_name(plan: &str) -> String {
-    let sanitized = plan
+/// Returns "tmt" if the result would be empty
+fn sanitize_name(name: &str) -> String {
+    let sanitized = name
         .replace('/', "-")
         .replace(|c: char| !c.is_alphanumeric() && c != '-' && c != '_', "-")
         .trim_matches('-')
         .to_string();
 
     if sanitized.is_empty() {
-        "plan".to_string()
+        "tmt".to_string()
     } else {
         sanitized
     }
@@ -524,6 +524,7 @@ const COMMON_INST_ARGS: &[&str] = &[
     // TODO: Pass down the Secure Boot keys for tests if present
     "--firmware=uefi-insecure",
     "--label=bootc.test=1",
+    "--bind-storage-ro",
 ];
 
 /// Run TMT tests using bcvk for VM management
@@ -560,38 +561,38 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> {
     // Change to workdir for running tmt commands
     let _dir = sh.push_dir(workdir);
 
-    // Get the list of plans
-    println!("Discovering test plans...");
-    let plans_output = cmd!(sh, "tmt plan ls")
+    // Get the list of tests
+    println!("Discovering tests...");
+    let tests_output = cmd!(sh, "tmt test ls")
         .read()
         .context("Getting list of test plans")?;
 
-    let mut plans: Vec<&str> = plans_output
+    let mut tests: Vec<&str> = tests_output
         .lines()
         .map(|line| line.trim())
         .filter(|line| !line.is_empty() && line.starts_with("/"))
         .collect();
 
-    // Filter plans based on user arguments
+    // Filter tests based on user arguments
     if !filter_args.is_empty() {
-        let original_count = plans.len();
-        plans.retain(|plan| filter_args.iter().any(|arg| plan.contains(arg.as_str())));
-        if plans.len() < original_count {
+        let original_count = tests.len();
+        tests.retain(|t| filter_args.iter().any(|arg| t.contains(arg.as_str())));
+        if tests.len() < original_count {
             println!(
-                "Filtered from {} to {} plan(s) based on arguments: {:?}",
+                "Filtered from {} to {} test(s) based on arguments: {:?}",
                 original_count,
-                plans.len(),
+                tests.len(),
                 filter_args
             );
         }
     }
 
-    if plans.is_empty() {
-        println!("No test plans found");
+    if tests.is_empty() {
+        println!("No tests found");
         return Ok(());
     }
 
-    println!("Found {} test plan(s): {:?}", plans.len(), plans);
+    println!("Found {} test(s): {:?}", tests.len(), tests);
 
     // Generate a random suffix for VM names
     let random_suffix = generate_random_suffix();
@@ -600,18 +601,17 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> {
     let mut all_passed = true;
     let mut test_results = Vec::new();
 
-    // Run each plan in its own VM
-    for plan in plans {
-        let plan_name = sanitize_plan_name(plan);
-        let vm_name = format!("bootc-tmt-{}-{}", random_suffix, plan_name);
+    // Run each test in its own VM
+    for test in tests {
+        let test_name = sanitize_name(test);
+        let vm_name = format!("bootc-tmt-{}-{}", random_suffix, test_name);
 
         println!("\n========================================");
-        println!("Running plan: {}", plan);
+        println!("Running test: {}", test);
         println!("VM name: {}", vm_name);
         println!("========================================\n");
 
         // Launch VM with bcvk
-
         let launch_result = cmd!(
             sh,
             "bcvk libvirt run --name {vm_name} --detach {COMMON_INST_ARGS...} {image}"
@@ -620,9 +620,9 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> {
         .context("Launching VM with bcvk");
 
         if let Err(e) = launch_result {
-            eprintln!("Failed to launch VM for plan {}: {:#}", plan, e);
+            eprintln!("Failed to launch VM for test {test}: {e:#}");
             all_passed = false;
-            test_results.push((plan.to_string(), false));
+            test_results.push((test.to_string(), false));
             continue;
         }
 
@@ -645,10 +645,10 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> {
         let (ssh_port, ssh_key) = match vm_info {
             Ok((port, key)) => (port, key),
             Err(e) => {
-                eprintln!("Failed to get VM info for plan {}: {:#}", plan, e);
+                eprintln!("Failed to get VM info for plan {test}: {e:#}");
                 cleanup_vm();
                 all_passed = false;
-                test_results.push((plan.to_string(), false));
+                test_results.push((test.to_string(), false));
                 continue;
             }
         };
@@ -661,10 +661,10 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> {
         let key_file = match key_file {
             Ok(f) => f,
             Err(e) => {
-                eprintln!("Failed to create SSH key file for plan {}: {:#}", plan, e);
+                eprintln!("Failed to create SSH key file for plan {test}: {e:#}");
                 cleanup_vm();
                 all_passed = false;
-                test_results.push((plan.to_string(), false));
+                test_results.push((test.to_string(), false));
                 continue;
             }
         };
@@ -675,19 +675,19 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> {
         let key_path = match key_path {
             Ok(p) => p,
             Err(e) => {
-                eprintln!("Failed to convert key path for plan {}: {:#}", plan, e);
+                eprintln!("Failed to convert key path for test {test}: {e:#}");
                 cleanup_vm();
                 all_passed = false;
-                test_results.push((plan.to_string(), false));
+                test_results.push((test.to_string(), false));
                 continue;
             }
         };
 
         if let Err(e) = std::fs::write(&key_path, ssh_key) {
-            eprintln!("Failed to write SSH key for plan {}: {:#}", plan, e);
+            eprintln!("Failed to write SSH key for test {test}: {e:#}");
             cleanup_vm();
             all_passed = false;
-            test_results.push((plan.to_string(), false));
+            test_results.push((test.to_string(), false));
             continue;
         }
 
@@ -696,38 +696,42 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> {
             use std::os::unix::fs::PermissionsExt;
             let perms = std::fs::Permissions::from_mode(0o600);
             if let Err(e) = std::fs::set_permissions(&key_path, perms) {
-                eprintln!("Failed to set key permissions for plan {}: {:#}", plan, e);
+                eprintln!("Failed to set key permissions for test {test}: {e:#}");
                 cleanup_vm();
                 all_passed = false;
-                test_results.push((plan.to_string(), false));
+                test_results.push((test.to_string(), false));
                 continue;
             }
         }
 
         // Verify SSH connectivity
         println!("Verifying SSH connectivity...");
         if let Err(e) = verify_ssh_connectivity(sh, ssh_port, &key_path) {
-            eprintln!("SSH verification failed for plan {}: {:#}", plan, e);
+            eprintln!("SSH verification failed for test {test}: {e:#}");
             cleanup_vm();
             all_passed = false;
-            test_results.push((plan.to_string(), false));
+            test_results.push((test.to_string(), false));
             continue;
         }
 
         println!("SSH connectivity verified");
 
         let ssh_port_str = ssh_port.to_string();
 
-        // Run tmt for this specific plan using connect provisioner
-        println!("Running tmt tests for plan {}...", plan);
+        // Run tmt for this specific test using connect provisioner
+        println!("Running tmt test {}...", test);
 
-        // Run tmt for this specific plan
-        // Note: provision must come before plan for connect to work properly
+        // Run tmt for this specific test
+        // Note: provision must come before tests for connect to work properly
         let context = context.clone();
         let how = ["--how=connect", "--guest=localhost", "--user=root"];
+        let env = ["TMT_SCRIPTS_DIR=/var/lib/tmt/scripts", "BCVK_EXPORT=1"]
+            .into_iter()
+            .chain(args.env.iter().map(|v| v.as_str()))
+            .flat_map(|v| ["--environment", v]);
         let test_result = cmd!(
             sh,
-            "tmt {context...} run --all -e TMT_SCRIPTS_DIR=/var/lib/tmt/scripts provision {how...} --port {ssh_port_str} --key {key_path} plan --name {plan}"
+            "tmt {context...} run --all {env...} provision {how...} --port {ssh_port_str} --key {key_path} tests --name {test}"
         )
         .run();
 
@@ -736,13 +740,13 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> {
 
         match test_result {
             Ok(_) => {
-                println!("Plan {} completed successfully", plan);
-                test_results.push((plan.to_string(), true));
+                println!("Test {} completed successfully", test);
+                test_results.push((test.to_string(), true));
             }
             Err(e) => {
-                eprintln!("Plan {} failed: {:#}", plan, e);
+                eprintln!("Test {} failed: {:#}", test, e);
                 all_passed = false;
-                test_results.push((plan.to_string(), false));
+                test_results.push((test.to_string(), false));
             }
         }
 
@@ -775,14 +779,15 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> {
     println!("\n========================================");
     println!("Test Summary");
     println!("========================================");
-    for (plan, passed) in &test_results {
+    for (test_name, passed) in &test_results {
         let status = if *passed { "PASSED" } else { "FAILED" };
-        println!("{}: {}", plan, status);
+        println!("{}: {}", test_name, status);
     }
     println!("========================================\n");
 
     if !all_passed {
-        anyhow::bail!("Some test plans failed");
+        cmd!(sh, "tmt run -l report -vvv").ignore_status().run()?;
+        anyhow::bail!("Some tests failed");
     }
 
     Ok(())
@@ -813,7 +818,7 @@ fn tmt_provision(sh: &Shell, args: &TmtProvisionArgs) -> Result<()> {
     // Use ds=iid-datasource-none to disable cloud-init for faster boot
     cmd!(
         sh,
-        "bcvk libvirt run --name {vm_name} --detach {COMMON_INST_ARGS...} {image}"
+        "bcvk libvirt run --bind-storage-ro --name {vm_name} --detach {COMMON_INST_ARGS...} {image}"
     )
     .run()
     .context("Launching VM with bcvk")?;