From aa1c2aee8d01fb87296617dfa90821d9a81a83e4 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 24 Nov 2025 09:33:24 -0500 Subject: [PATCH 1/7] ci: use latest git bcvk To fix SELinux issues. Signed-off-by: Colin Walters --- .github/actions/bootc-ubuntu-setup/action.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/actions/bootc-ubuntu-setup/action.yml b/.github/actions/bootc-ubuntu-setup/action.yml index 4d7cf0d0a..8e0178b71 100644 --- a/.github/actions/bootc-ubuntu-setup/action.yml +++ b/.github/actions/bootc-ubuntu-setup/action.yml @@ -75,6 +75,8 @@ runs: - name: Install libvirt and virtualization stack if: ${{ inputs.libvirt == 'true' }} shell: bash + env: + GH_TOKEN: ${{ github.token }} run: | set -xeuo pipefail export BCVK_VERSION=0.6.0 @@ -83,9 +85,9 @@ runs: echo LIBVIRT_DEFAULT_URI=qemu:///session >> $GITHUB_ENV td=$(mktemp -d) cd $td - # Install bcvk + # Install bcvk from PR 159 + gh run download 19640807620 --name bcvk-binary --repo bootc-dev/bcvk target=bcvk-$(arch)-unknown-linux-gnu - /bin/time -f '%E %C' curl -LO https://github.com/bootc-dev/bcvk/releases/download/v${BCVK_VERSION}/${target}.tar.gz tar xzf ${target}.tar.gz sudo install -T ${target} /usr/bin/bcvk cd - From 25f0c5dad8f8a4a12034f2e40a0d233ea6327499 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 19 Nov 2025 15:37:36 -0500 Subject: [PATCH 2/7] tmt: Generate integration.fmf from test code We need to run most of our tests in a separate provisioned machine, which means it needs an individual plan. And then we need a test for that plan. And then we need the *actual test code*. This "triplication" is a huge annoying pain. TMT is soooo complicated, yet as far as I can tell it doesn't offer us any tools to solve this. So we'll do it here, cut over to generating the TMT stuff from metadata defined in the test file. Hence adding a test is just: - Write a new tests/booted/foo.nu - `cargo xtask update-generated` Signed-off-by: Colin Walters --- Cargo.lock | 1 + crates/xtask/Cargo.toml | 1 + crates/xtask/src/tmt.rs | 356 ++++++++++++++++++ crates/xtask/src/xtask.rs | 4 + docs/src/host-v1.schema.json | 7 + docs/src/man/bootc-switch.8.md | 2 +- docs/src/man/bootc.8.md | 1 - tmt/plans/integration.fmf | 85 ++--- tmt/tests/booted/test-01-readonly.nu | 19 + tmt/tests/booted/test-26-examples-build.sh | 24 ++ .../booted/test-custom-selinux-policy.nu | 9 + tmt/tests/booted/test-factory-reset.nu | 5 + .../booted/test-image-pushpull-upgrade.nu | 5 + tmt/tests/booted/test-image-upgrade-reboot.nu | 5 + .../booted/test-install-outside-container.nu | 5 + .../booted/test-logically-bound-install.nu | 5 + .../booted/test-logically-bound-switch.nu | 5 + tmt/tests/booted/test-soft-reboot.nu | 5 + tmt/tests/booted/test-usroverlay.nu | 5 + tmt/tests/test-01-readonly.fmf | 3 - tmt/tests/test-20-local-upgrade.fmf | 3 - tmt/tests/test-21-logically-bound-switch.fmf | 3 - tmt/tests/test-22-logically-bound-install.fmf | 3 - .../test-23-install-outside-container.fmf | 3 - tmt/tests/test-23-usroverlay.fmf | 3 - tmt/tests/test-24-local-upgrade-reboot.fmf | 3 - tmt/tests/test-25-soft-reboot.fmf | 3 - tmt/tests/test-26-examples-build.fmf | 18 - tmt/tests/test-27-custom-selinux-policy.fmf | 3 - tmt/tests/test-28-factory-reset.fmf | 3 - tmt/tests/tests.fmf | 66 ++++ 31 files changed, 567 insertions(+), 96 deletions(-) create mode 100644 crates/xtask/src/tmt.rs create mode 100644 tmt/tests/booted/test-01-readonly.nu create mode 100755 tmt/tests/booted/test-26-examples-build.sh delete mode 100644 tmt/tests/test-01-readonly.fmf delete mode 100644 tmt/tests/test-20-local-upgrade.fmf delete mode 100644 tmt/tests/test-21-logically-bound-switch.fmf delete mode 100644 tmt/tests/test-22-logically-bound-install.fmf delete mode 100644 tmt/tests/test-23-install-outside-container.fmf delete mode 100644 tmt/tests/test-23-usroverlay.fmf delete mode 100644 tmt/tests/test-24-local-upgrade-reboot.fmf delete mode 100644 tmt/tests/test-25-soft-reboot.fmf delete mode 100644 tmt/tests/test-26-examples-build.fmf delete mode 100644 tmt/tests/test-27-custom-selinux-policy.fmf delete mode 100644 tmt/tests/test-28-factory-reset.fmf create mode 100644 tmt/tests/tests.fmf diff --git a/Cargo.lock b/Cargo.lock index aba4f1e92..0789cb98a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3309,6 +3309,7 @@ dependencies = [ "rand 0.9.2", "serde", "serde_json", + "serde_yaml", "tar", "tempfile", "toml", diff --git a/crates/xtask/Cargo.toml b/crates/xtask/Cargo.toml index 28aced786..73ec3a486 100644 --- a/crates/xtask/Cargo.toml +++ b/crates/xtask/Cargo.toml @@ -29,6 +29,7 @@ xshell = { workspace = true } # Crate-specific dependencies mandown = "1.1.0" rand = "0.9" +serde_yaml = "0.9" tar = "0.4" [lints] diff --git a/crates/xtask/src/tmt.rs b/crates/xtask/src/tmt.rs new file mode 100644 index 000000000..26d931c9e --- /dev/null +++ b/crates/xtask/src/tmt.rs @@ -0,0 +1,356 @@ +use anyhow::{Context, Result}; +use camino::Utf8Path; +use fn_error_context::context; + +// Generation markers for integration.fmf +const PLAN_MARKER_BEGIN: &str = "# BEGIN GENERATED PLANS\n"; +const PLAN_MARKER_END: &str = "# END GENERATED PLANS\n"; + +/// Parse tmt metadata from a test file +/// Looks for: +/// # number: N +/// # tmt: +/// # +fn parse_tmt_metadata(content: &str) -> Result> { + let mut number = None; + let mut in_tmt_block = false; + let mut yaml_lines = Vec::new(); + + for line in content.lines().take(50) { + let trimmed = line.trim(); + + // Look for "# number: N" line + if let Some(rest) = trimmed.strip_prefix("# number:") { + number = Some(rest.trim().parse::() + .context("Failed to parse number field")?); + continue; + } + + if trimmed == "# tmt:" { + in_tmt_block = true; + continue; + } else if in_tmt_block { + // Stop if we hit a line that doesn't start with #, or is just "#" + if !trimmed.starts_with('#') || trimmed == "#" { + break; + } + // Remove the leading # and preserve indentation + if let Some(yaml_line) = line.strip_prefix('#') { + yaml_lines.push(yaml_line); + } + } + } + + let Some(number) = number else { + return Ok(None); + }; + + let yaml_content = yaml_lines.join("\n"); + let extra: serde_yaml::Value = if yaml_content.trim().is_empty() { + serde_yaml::Value::Mapping(serde_yaml::Mapping::new()) + } else { + serde_yaml::from_str(&yaml_content) + .with_context(|| format!("Failed to parse tmt metadata YAML:\n{}", yaml_content))? + }; + + Ok(Some(TmtMetadata { number, extra })) +} + +#[derive(Debug, serde::Deserialize)] +#[serde(rename_all = "kebab-case")] +struct TmtMetadata { + /// Test number for ordering and naming + number: u32, + /// All other fmf attributes (summary, duration, adjust, require, etc.) + /// Note: summary and duration are typically required by fmf + #[serde(flatten)] + extra: serde_yaml::Value, +} + +#[derive(Debug)] +struct TestDef { + number: u32, + name: String, + test_command: String, + /// All fmf attributes to pass through (summary, duration, adjust, etc.) + extra: serde_yaml::Value, +} + +/// Generate tmt/plans/integration.fmf from test definitions +#[context("Updating TMT integration.fmf")] +pub(crate) fn update_integration() -> Result<()> { + // Define tests in order + let mut tests = vec![]; + + // Scan for test-*.nu and test-*.sh files in tmt/tests/booted/ + let booted_dir = Utf8Path::new("tmt/tests/booted"); + + for entry in std::fs::read_dir(booted_dir)? { + let entry = entry?; + let path = entry.path(); + let Some(filename) = path.file_name().and_then(|n| n.to_str()) else { + continue; + }; + + // Extract stem (filename without "test-" prefix and extension) + let Some(stem) = filename + .strip_prefix("test-") + .and_then(|s| s.strip_suffix(".nu").or_else(|| s.strip_suffix(".sh"))) + else { + continue; + }; + + let content = std::fs::read_to_string(&path) + .with_context(|| format!("Reading {}", filename))?; + + let metadata = parse_tmt_metadata(&content) + .with_context(|| format!("Parsing tmt metadata from {}", filename))? + .with_context(|| format!("Missing tmt metadata in {}", filename))?; + + // Remove number prefix if present (e.g., "01-readonly" -> "readonly", "26-examples-build" -> "examples-build") + let display_name = stem + .split_once('-') + .and_then(|(prefix, suffix)| { + if prefix.chars().all(|c| c.is_ascii_digit()) { + Some(suffix.to_string()) + } else { + None + } + }) + .unwrap_or_else(|| stem.to_string()); + + // Derive relative path from booted_dir + let relative_path = path + .strip_prefix("tmt/tests/") + .with_context(|| format!("Failed to get relative path for {}", filename))?; + + // Determine test command based on file extension + let extension = if filename.ends_with(".nu") { + "nu" + } else if filename.ends_with(".sh") { + "bash" + } else { + anyhow::bail!("Unsupported test file extension: {}", filename); + }; + + let test_command = format!("{} {}", extension, relative_path.display()); + + tests.push(TestDef { + number: metadata.number, + name: display_name, + test_command, + extra: metadata.extra, + }); + } + + // Sort tests by number + tests.sort_by_key(|t| t.number); + + // Generate single tests.fmf file + let tests_dir = Utf8Path::new("tmt/tests"); + let tests_fmf_path = tests_dir.join("tests.fmf"); + let mut tests_content = String::new(); + + // Add generated code marker + tests_content.push_str("# THIS IS GENERATED CODE - DO NOT EDIT\n"); + tests_content.push_str("# Generated by: cargo xtask tmt\n"); + tests_content.push_str("\n"); + + for test in &tests { + tests_content.push_str(&format!("/test-{:02}-{}:\n", test.number, test.name)); + + // Serialize all fmf attributes from metadata (summary, duration, adjust, etc.) + if let serde_yaml::Value::Mapping(map) = &test.extra { + if !map.is_empty() { + let extra_yaml = serde_yaml::to_string(&test.extra) + .context("Serializing extra metadata")?; + for line in extra_yaml.lines() { + if !line.trim().is_empty() { + tests_content.push_str(&format!(" {}\n", line)); + } + } + } + } + + // Add the test command (derived from file type, not in metadata) + if test.test_command.contains('\n') { + tests_content.push_str(" test: |\n"); + for line in test.test_command.lines() { + tests_content.push_str(&format!(" {}\n", line)); + } + } else { + tests_content.push_str(&format!(" test: {}\n", test.test_command)); + } + + tests_content.push_str("\n"); + } + + // Only write if content changed + let needs_update = match std::fs::read_to_string(&tests_fmf_path) { + Ok(existing) => existing != tests_content, + Err(_) => true, + }; + + if needs_update { + std::fs::write(&tests_fmf_path, tests_content) + .context("Writing tests.fmf")?; + println!("Generated {}", tests_fmf_path); + } else { + println!("Unchanged: {}", tests_fmf_path); + } + + // Generate plans section (at root level, no indentation) + let mut plans_section = String::new(); + for test in &tests { + plans_section.push_str(&format!("/plan-{:02}-{}:\n", test.number, test.name)); + + // Extract summary from extra metadata + if let serde_yaml::Value::Mapping(map) = &test.extra { + if let Some(summary) = map.get(&serde_yaml::Value::String("summary".to_string())) { + if let Some(summary_str) = summary.as_str() { + plans_section.push_str(&format!(" summary: {}\n", summary_str)); + } + } + } + + plans_section.push_str(" discover:\n"); + plans_section.push_str(" how: fmf\n"); + plans_section.push_str(" test:\n"); + plans_section.push_str(&format!(" - /tmt/tests/tests/test-{:02}-{}\n", test.number, test.name)); + + // Extract and serialize adjust section if present + if let serde_yaml::Value::Mapping(map) = &test.extra { + if let Some(adjust) = map.get(&serde_yaml::Value::String("adjust".to_string())) { + let adjust_yaml = serde_yaml::to_string(adjust) + .context("Serializing adjust metadata")?; + plans_section.push_str(" adjust:\n"); + for line in adjust_yaml.lines() { + if !line.trim().is_empty() { + plans_section.push_str(&format!(" {}\n", line)); + } + } + } + } + + plans_section.push_str("\n"); + } + + // Update integration.fmf with generated plans + let output_path = Utf8Path::new("tmt/plans/integration.fmf"); + let existing_content = std::fs::read_to_string(output_path) + .context("Reading integration.fmf")?; + + // Replace plans section + let (before_plans, rest) = existing_content.split_once(PLAN_MARKER_BEGIN) + .context("Missing # BEGIN GENERATED PLANS marker in integration.fmf")?; + let (_old_plans, after_plans) = rest.split_once(PLAN_MARKER_END) + .context("Missing # END GENERATED PLANS marker in integration.fmf")?; + + let new_content = format!( + "{}{}{}{}{}", + before_plans, PLAN_MARKER_BEGIN, plans_section, PLAN_MARKER_END, after_plans + ); + + // Only write if content changed + let needs_update = match std::fs::read_to_string(output_path) { + Ok(existing) => existing != new_content, + Err(_) => true, + }; + + if needs_update { + std::fs::write(output_path, new_content)?; + println!("Generated {}", output_path); + } else { + println!("Unchanged: {}", output_path); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_tmt_metadata_basic() { + let content = r#"# number: 1 +# tmt: +# summary: Execute booted readonly/nondestructive tests +# duration: 30m +# +# Run all readonly tests in sequence +use tap.nu +"#; + + let metadata = parse_tmt_metadata(content).unwrap().unwrap(); + assert_eq!(metadata.number, 1); + + // Verify extra fields are captured + let extra = metadata.extra.as_mapping().unwrap(); + assert_eq!( + extra.get(&serde_yaml::Value::String("summary".to_string())), + Some(&serde_yaml::Value::String("Execute booted readonly/nondestructive tests".to_string())) + ); + assert_eq!( + extra.get(&serde_yaml::Value::String("duration".to_string())), + Some(&serde_yaml::Value::String("30m".to_string())) + ); + } + + #[test] + fn test_parse_tmt_metadata_with_adjust() { + let content = r#"# number: 27 +# tmt: +# summary: Execute custom selinux policy test +# duration: 30m +# adjust: +# - when: running_env != image_mode +# enabled: false +# because: these tests require features only available in image mode +# +use std assert +"#; + + let metadata = parse_tmt_metadata(content).unwrap().unwrap(); + assert_eq!(metadata.number, 27); + + // Verify adjust section is in extra + let extra = metadata.extra.as_mapping().unwrap(); + assert!(extra.contains_key(&serde_yaml::Value::String("adjust".to_string()))); + } + + #[test] + fn test_parse_tmt_metadata_no_metadata() { + let content = r#"# Just a comment +use std assert +"#; + + let result = parse_tmt_metadata(content).unwrap(); + assert!(result.is_none()); + } + + #[test] + fn test_parse_tmt_metadata_shell_script() { + let content = r#"# number: 26 +# tmt: +# summary: Test bootc examples build scripts +# duration: 45m +# adjust: +# - when: running_env != image_mode +# enabled: false +# +#!/bin/bash +set -eux +"#; + + let metadata = parse_tmt_metadata(content).unwrap().unwrap(); + assert_eq!(metadata.number, 26); + + let extra = metadata.extra.as_mapping().unwrap(); + assert_eq!( + extra.get(&serde_yaml::Value::String("duration".to_string())), + Some(&serde_yaml::Value::String("45m".to_string())) + ); + assert!(extra.contains_key(&serde_yaml::Value::String("adjust".to_string()))); + } +} diff --git a/crates/xtask/src/xtask.rs b/crates/xtask/src/xtask.rs index bbe56ce6e..69ff612a5 100644 --- a/crates/xtask/src/xtask.rs +++ b/crates/xtask/src/xtask.rs @@ -16,6 +16,7 @@ use rand::Rng; use xshell::{cmd, Shell}; mod man; +mod tmt; const NAME: &str = "bootc"; const TAR_REPRODUCIBLE_OPTS: &[&str] = &[ @@ -399,6 +400,9 @@ fn update_generated(sh: &Shell) -> Result<()> { // Update JSON schemas update_json_schemas(sh)?; + // Update TMT integration.fmf + tmt::update_integration()?; + Ok(()) } diff --git a/docs/src/host-v1.schema.json b/docs/src/host-v1.schema.json index 21e65e634..e0bdf2880 100644 --- a/docs/src/host-v1.schema.json +++ b/docs/src/host-v1.schema.json @@ -122,6 +122,13 @@ "description": "A bootable entry", "type": "object", "properties": { + "bootDigest": { + "description": "The sha256sum of vmlinuz + initrd\nOnly `Some` for Type1 boot entries", + "type": [ + "string", + "null" + ] + }, "bootType": { "description": "Whether this deployment is to be booted via Type1 (vmlinuz + initrd) or Type2 (UKI) entry", "$ref": "#/$defs/BootType" diff --git a/docs/src/man/bootc-switch.8.md b/docs/src/man/bootc-switch.8.md index df647971d..116f98553 100644 --- a/docs/src/man/bootc-switch.8.md +++ b/docs/src/man/bootc-switch.8.md @@ -61,7 +61,7 @@ Soft reboot allows faster system restart by avoiding full hardware reboot when p **--transport**=*TRANSPORT* - The transport; e.g. oci, oci-archive, containers-storage. Defaults to `registry` + The transport; e.g. registry, oci, oci-archive, docker-daemon, containers-storage. Defaults to `registry` Default: registry diff --git a/docs/src/man/bootc.8.md b/docs/src/man/bootc.8.md index 3c22aacb4..dd9ee7d18 100644 --- a/docs/src/man/bootc.8.md +++ b/docs/src/man/bootc.8.md @@ -34,7 +34,6 @@ pulled and `bootc upgrade`. | **bootc install** | Install the running container to a target | | **bootc container** | Operations which can be executed as part of a container build | | **bootc composefs-finalize-staged** | | -| **bootc config-diff** | Diff current /etc configuration versus default | diff --git a/tmt/plans/integration.fmf b/tmt/plans/integration.fmf index 41b22deee..317ec1b81 100644 --- a/tmt/plans/integration.fmf +++ b/tmt/plans/integration.fmf @@ -1,3 +1,4 @@ +# Common settings for all plans provision: how: virtual image: $@{test_disk_image} @@ -35,98 +36,90 @@ prepare: execute: how: tmt -/readonly-tests: +# BEGIN GENERATED PLANS +/plan-01-readonly: summary: Execute booted readonly/nondestructive tests discover: how: fmf test: - - /tmt/tests/test-01-readonly + - /tmt/tests/tests/test-01-readonly -/test-20-local-upgrade: +/plan-20-image-pushpull-upgrade: summary: Execute local upgrade tests discover: how: fmf test: - - /tmt/tests/test-20-local-upgrade + - /tmt/tests/tests/test-20-image-pushpull-upgrade -/test-21-logically-bound-switch: +/plan-21-logically-bound-switch: summary: Execute logically bound images tests for switching images discover: how: fmf test: - - /tmt/tests/test-21-logically-bound-switch + - /tmt/tests/tests/test-21-logically-bound-switch -/test-22-logically-bound-install: - summary: Execute logically bound images tests for switching images +/plan-22-logically-bound-install: + summary: Execute logically bound images tests for installing image discover: how: fmf test: - - /tmt/tests/test-22-logically-bound-install + - /tmt/tests/tests/test-22-logically-bound-install -/test-23-install-outside-container: +/plan-23-install-outside-container: summary: Execute tests for installing outside of a container discover: how: fmf test: - - /tmt/tests/test-23-install-outside-container + - /tmt/tests/tests/test-23-install-outside-container -/test-24-local-upgrade-reboot: - summary: Execute local upgrade tests with automated reboot +/plan-23-usroverlay: + summary: Execute tests for bootc usrover discover: how: fmf test: - - /tmt/tests/test-24-local-upgrade-reboot + - /tmt/tests/tests/test-23-usroverlay -/test-25-soft-reboot: - summary: Soft reboot support +/plan-24-image-upgrade-reboot: + summary: Execute local upgrade tests discover: how: fmf test: - - /tmt/tests/test-25-soft-reboot + - /tmt/tests/tests/test-24-image-upgrade-reboot -/test-26-examples-build: - summary: Test bootc examples build scripts +/plan-25-soft-reboot: + summary: Execute soft reboot test discover: how: fmf test: - - /tmt/tests/test-26-examples-build - adjust: - - when: running_env != image_mode - enabled: false - because: packit tests use RPM bootc and does not install /usr/lib/bootc/initramfs-setup + - /tmt/tests/tests/test-25-soft-reboot -/test-27-custom-selinux-policy: - summary: Execute restorecon test on system with custom selinux policy +/plan-26-examples-build: + summary: Test bootc examples build scripts discover: how: fmf test: - - /tmt/tests/test-27-custom-selinux-policy + - /tmt/tests/tests/test-26-examples-build adjust: - - when: running_env != image_mode - enabled: false - because: tmt-reboot does not work with systemd reboot in testing farm environment (see bug-soft-reboot.md) + - when: running_env != image_mode + enabled: false + because: packit tests use RPM bootc and does not install /usr/lib/bootc/initramfs-setup -/test-23-usroverlay: - summary: Usroverlay +/plan-27-custom-selinux-policy: + summary: Execute custom selinux policy test discover: how: fmf test: - - /tmt/tests/test-23-usroverlay + - /tmt/tests/tests/test-27-custom-selinux-policy + adjust: + - when: running_env != image_mode + enabled: false + because: these tests require features only available in image mode -/test-28-factory-reset: - summary: Factory reset +/plan-28-factory-reset: + summary: Execute factory reset tests discover: how: fmf test: - - /tmt/tests/test-28-factory-reset + - /tmt/tests/tests/test-28-factory-reset -/test-29-soft-reboot-selinux-policy: - summary: Test soft reboot with SELinux policy changes - discover: - how: fmf - test: - - /tmt/tests/test-29-soft-reboot-selinux-policy - adjust: - - when: running_env != image_mode - enabled: false - because: tmt-reboot does not work with systemd reboot in testing farm environment (see bug-soft-reboot.md) +# END GENERATED PLANS diff --git a/tmt/tests/booted/test-01-readonly.nu b/tmt/tests/booted/test-01-readonly.nu new file mode 100644 index 000000000..12e1315d4 --- /dev/null +++ b/tmt/tests/booted/test-01-readonly.nu @@ -0,0 +1,19 @@ +# number: 1 +# tmt: +# summary: Execute booted readonly/nondestructive tests +# duration: 30m +# +# Run all readonly tests in sequence +use tap.nu + +tap begin "readonly tests" + +# Get all readonly test files and run them in order +let tests = (ls booted/readonly/*-test-*.nu | get name | sort) + +for test_file in $tests { + print $"Running ($test_file)..." + nu $test_file +} + +tap ok \ No newline at end of file diff --git a/tmt/tests/booted/test-26-examples-build.sh b/tmt/tests/booted/test-26-examples-build.sh new file mode 100755 index 000000000..5895419d8 --- /dev/null +++ b/tmt/tests/booted/test-26-examples-build.sh @@ -0,0 +1,24 @@ +# number: 26 +# tmt: +# summary: Test bootc examples build scripts +# duration: 45m +# adjust: +# - when: running_env != image_mode +# enabled: false +# because: packit tests use RPM bootc and does not install /usr/lib/bootc/initramfs-setup +# +#!/bin/bash +set -eux + +# Test bootc-bls example +echo "Testing bootc-bls example..." +cd examples/bootc-bls +./build + +# Test bootc-uki example +echo "Testing bootc-uki example..." +cd ../bootc-uki +./build.base +./build.final + +echo "All example builds completed successfully" diff --git a/tmt/tests/booted/test-custom-selinux-policy.nu b/tmt/tests/booted/test-custom-selinux-policy.nu index b484a1292..be39e177d 100644 --- a/tmt/tests/booted/test-custom-selinux-policy.nu +++ b/tmt/tests/booted/test-custom-selinux-policy.nu @@ -1,3 +1,12 @@ +# number: 27 +# tmt: +# summary: Execute custom selinux policy test +# duration: 30m +# adjust: +# - when: running_env != image_mode +# enabled: false +# because: these tests require features only available in image mode +# # Verify that correct labels are applied after a deployment use std assert use tap.nu diff --git a/tmt/tests/booted/test-factory-reset.nu b/tmt/tests/booted/test-factory-reset.nu index a89b10027..0841eec82 100644 --- a/tmt/tests/booted/test-factory-reset.nu +++ b/tmt/tests/booted/test-factory-reset.nu @@ -1,3 +1,8 @@ +# number: 28 +# tmt: +# summary: Execute factory reset tests +# duration: 30m +# use std assert use tap.nu use bootc_testlib.nu diff --git a/tmt/tests/booted/test-image-pushpull-upgrade.nu b/tmt/tests/booted/test-image-pushpull-upgrade.nu index ff1c5abe4..5367dcc8e 100644 --- a/tmt/tests/booted/test-image-pushpull-upgrade.nu +++ b/tmt/tests/booted/test-image-pushpull-upgrade.nu @@ -1,3 +1,8 @@ +# number: 20 +# tmt: +# summary: Execute local upgrade tests +# duration: 30m +# # This test does: # bootc image copy-to-storage # podman build diff --git a/tmt/tests/booted/test-image-upgrade-reboot.nu b/tmt/tests/booted/test-image-upgrade-reboot.nu index b2ad5bc60..a636b6303 100644 --- a/tmt/tests/booted/test-image-upgrade-reboot.nu +++ b/tmt/tests/booted/test-image-upgrade-reboot.nu @@ -1,3 +1,8 @@ +# number: 24 +# tmt: +# summary: Execute local upgrade tests +# duration: 30m +# # This test does: # bootc image copy-to-storage # podman build diff --git a/tmt/tests/booted/test-install-outside-container.nu b/tmt/tests/booted/test-install-outside-container.nu index 5341e8c42..a44420f79 100644 --- a/tmt/tests/booted/test-install-outside-container.nu +++ b/tmt/tests/booted/test-install-outside-container.nu @@ -1,3 +1,8 @@ +# number: 23 +# tmt: +# summary: Execute tests for installing outside of a container +# duration: 30m +# use std assert use tap.nu diff --git a/tmt/tests/booted/test-logically-bound-install.nu b/tmt/tests/booted/test-logically-bound-install.nu index 6f1891804..29e42e8ca 100644 --- a/tmt/tests/booted/test-logically-bound-install.nu +++ b/tmt/tests/booted/test-logically-bound-install.nu @@ -1,3 +1,8 @@ +# number: 22 +# tmt: +# summary: Execute logically bound images tests for installing image +# duration: 30m +# use std assert use tap.nu diff --git a/tmt/tests/booted/test-logically-bound-switch.nu b/tmt/tests/booted/test-logically-bound-switch.nu index f1846a7aa..09dc1d999 100644 --- a/tmt/tests/booted/test-logically-bound-switch.nu +++ b/tmt/tests/booted/test-logically-bound-switch.nu @@ -1,3 +1,8 @@ +# number: 21 +# tmt: +# summary: Execute logically bound images tests for switching images +# duration: 30m +# # This test does: # bootc image switch bootc-bound-image # diff --git a/tmt/tests/booted/test-soft-reboot.nu b/tmt/tests/booted/test-soft-reboot.nu index e131dd712..dd3374e13 100644 --- a/tmt/tests/booted/test-soft-reboot.nu +++ b/tmt/tests/booted/test-soft-reboot.nu @@ -1,3 +1,8 @@ +# number: 25 +# tmt: +# summary: Execute soft reboot test +# duration: 30m +# # Verify that soft reboot works (on by default) use std assert use tap.nu diff --git a/tmt/tests/booted/test-usroverlay.nu b/tmt/tests/booted/test-usroverlay.nu index 6fb93346a..ca68b239e 100644 --- a/tmt/tests/booted/test-usroverlay.nu +++ b/tmt/tests/booted/test-usroverlay.nu @@ -1,3 +1,8 @@ +# number: 23 +# tmt: +# summary: Execute tests for bootc usrover +# duration: 30m +# # Verify that bootc usroverlay works use std assert use tap.nu diff --git a/tmt/tests/test-01-readonly.fmf b/tmt/tests/test-01-readonly.fmf deleted file mode 100644 index 7789daad7..000000000 --- a/tmt/tests/test-01-readonly.fmf +++ /dev/null @@ -1,3 +0,0 @@ -summary: Execute booted readonly/nondestructive tests -test: ls booted/readonly/*-test-*.nu | sort -n | while read t; do nu $t; done -duration: 30m diff --git a/tmt/tests/test-20-local-upgrade.fmf b/tmt/tests/test-20-local-upgrade.fmf deleted file mode 100644 index 431722448..000000000 --- a/tmt/tests/test-20-local-upgrade.fmf +++ /dev/null @@ -1,3 +0,0 @@ -summary: Execute local upgrade tests -test: nu booted/test-image-pushpull-upgrade.nu -duration: 30m diff --git a/tmt/tests/test-21-logically-bound-switch.fmf b/tmt/tests/test-21-logically-bound-switch.fmf deleted file mode 100644 index ad7b0a1f4..000000000 --- a/tmt/tests/test-21-logically-bound-switch.fmf +++ /dev/null @@ -1,3 +0,0 @@ -summary: Execute logically bound images tests for switching images -test: nu booted/test-logically-bound-switch.nu -duration: 30m diff --git a/tmt/tests/test-22-logically-bound-install.fmf b/tmt/tests/test-22-logically-bound-install.fmf deleted file mode 100644 index 0152c17bc..000000000 --- a/tmt/tests/test-22-logically-bound-install.fmf +++ /dev/null @@ -1,3 +0,0 @@ -summary: Execute logically bound images tests for installing image -test: nu booted/test-logically-bound-install.nu -duration: 30m diff --git a/tmt/tests/test-23-install-outside-container.fmf b/tmt/tests/test-23-install-outside-container.fmf deleted file mode 100644 index a48017859..000000000 --- a/tmt/tests/test-23-install-outside-container.fmf +++ /dev/null @@ -1,3 +0,0 @@ -summary: Execute tests for installing outside of a container -test: nu booted/test-install-outside-container.nu -duration: 30m diff --git a/tmt/tests/test-23-usroverlay.fmf b/tmt/tests/test-23-usroverlay.fmf deleted file mode 100644 index fe273d36c..000000000 --- a/tmt/tests/test-23-usroverlay.fmf +++ /dev/null @@ -1,3 +0,0 @@ -summary: Execute tests for bootc usrover -test: nu booted/test-usroverlay.nu -duration: 30m diff --git a/tmt/tests/test-24-local-upgrade-reboot.fmf b/tmt/tests/test-24-local-upgrade-reboot.fmf deleted file mode 100644 index 59ded687d..000000000 --- a/tmt/tests/test-24-local-upgrade-reboot.fmf +++ /dev/null @@ -1,3 +0,0 @@ -summary: Execute local upgrade tests -test: nu booted/test-image-upgrade-reboot.nu -duration: 30m diff --git a/tmt/tests/test-25-soft-reboot.fmf b/tmt/tests/test-25-soft-reboot.fmf deleted file mode 100644 index 1ed98d43f..000000000 --- a/tmt/tests/test-25-soft-reboot.fmf +++ /dev/null @@ -1,3 +0,0 @@ -summary: Execute soft reboot test -test: nu booted/test-soft-reboot.nu -duration: 30m diff --git a/tmt/tests/test-26-examples-build.fmf b/tmt/tests/test-26-examples-build.fmf deleted file mode 100644 index 7c556d74a..000000000 --- a/tmt/tests/test-26-examples-build.fmf +++ /dev/null @@ -1,18 +0,0 @@ -summary: Test bootc examples build scripts -test: | - #!/bin/bash - set -eux - - # Test bootc-bls example - echo "Testing bootc-bls example..." - cd examples/bootc-bls - ./build - - # Test bootc-uki example - echo "Testing bootc-uki example..." - cd ../bootc-uki - ./build.base - ./build.final - - echo "All example builds completed successfully" -duration: 45m diff --git a/tmt/tests/test-27-custom-selinux-policy.fmf b/tmt/tests/test-27-custom-selinux-policy.fmf deleted file mode 100644 index c77f2b011..000000000 --- a/tmt/tests/test-27-custom-selinux-policy.fmf +++ /dev/null @@ -1,3 +0,0 @@ -summary: Execute custom selinux policy test -test: nu booted/test-custom-selinux-policy.nu -duration: 30m diff --git a/tmt/tests/test-28-factory-reset.fmf b/tmt/tests/test-28-factory-reset.fmf deleted file mode 100644 index 4772798e0..000000000 --- a/tmt/tests/test-28-factory-reset.fmf +++ /dev/null @@ -1,3 +0,0 @@ -summary: Execute factory reset tests -test: nu booted/test-factory-reset.nu -duration: 30m diff --git a/tmt/tests/tests.fmf b/tmt/tests/tests.fmf new file mode 100644 index 000000000..69b98f332 --- /dev/null +++ b/tmt/tests/tests.fmf @@ -0,0 +1,66 @@ +# THIS IS GENERATED CODE - DO NOT EDIT +# Generated by: cargo xtask tmt + +/test-01-readonly: + summary: Execute booted readonly/nondestructive tests + duration: 30m + test: nu booted/test-01-readonly.nu + +/test-20-image-pushpull-upgrade: + summary: Execute local upgrade tests + duration: 30m + test: nu booted/test-image-pushpull-upgrade.nu + +/test-21-logically-bound-switch: + summary: Execute logically bound images tests for switching images + duration: 30m + test: nu booted/test-logically-bound-switch.nu + +/test-22-logically-bound-install: + summary: Execute logically bound images tests for installing image + duration: 30m + test: nu booted/test-logically-bound-install.nu + +/test-23-install-outside-container: + summary: Execute tests for installing outside of a container + duration: 30m + test: nu booted/test-install-outside-container.nu + +/test-23-usroverlay: + summary: Execute tests for bootc usrover + duration: 30m + test: nu booted/test-usroverlay.nu + +/test-24-image-upgrade-reboot: + summary: Execute local upgrade tests + duration: 30m + test: nu booted/test-image-upgrade-reboot.nu + +/test-25-soft-reboot: + summary: Execute soft reboot test + duration: 30m + test: nu booted/test-soft-reboot.nu + +/test-26-examples-build: + summary: Test bootc examples build scripts + duration: 45m + adjust: + - when: running_env != image_mode + enabled: false + because: packit tests use RPM bootc and does not install /usr/lib/bootc/initramfs-setup + test: bash booted/test-26-examples-build.sh + +/test-27-custom-selinux-policy: + summary: Execute custom selinux policy test + duration: 30m + adjust: + - when: running_env != image_mode + enabled: false + because: these tests require features only available in image mode + test: nu booted/test-custom-selinux-policy.nu + +/test-28-factory-reset: + summary: Execute factory reset tests + duration: 30m + test: nu booted/test-factory-reset.nu + From bd052a34c2b5dc72c645ce731651e06c2eec73da Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 21 Nov 2025 14:01:45 -0500 Subject: [PATCH 3/7] xtask: Move TMT infrastructure to tmt module and refactor YAML generation Move TMT test runner code from xtask.rs to tmt module: - `run_tmt()` and `tmt_provision()` functions - Helper functions for VM management and SSH connectivity - Related constants Also refactor `update_integration()` to use serde_yaml::Value for building YAML structures instead of string concatenation. Add detailed error reporting for failed TMT tests: - Assign run IDs using `tmt run --id` - Display verbose reports with `tmt run -i {id} report -vvv` Assisted-by: Claude Code (Sonnet 4.5) Signed-off-by: Colin Walters --- crates/xtask/src/tmt.rs | 695 ++++++++++++++++-- crates/xtask/src/xtask.rs | 494 +------------ tmt/plans/integration.fmf | 6 + .../booted/test-soft-reboot-selinux-policy.nu | 5 + .../test-29-soft-reboot-selinux-policy.fmf | 3 - tmt/tests/tests.fmf | 4 + 6 files changed, 660 insertions(+), 547 deletions(-) delete mode 100644 tmt/tests/test-29-soft-reboot-selinux-policy.fmf diff --git a/crates/xtask/src/tmt.rs b/crates/xtask/src/tmt.rs index 26d931c9e..019294cab 100644 --- a/crates/xtask/src/tmt.rs +++ b/crates/xtask/src/tmt.rs @@ -1,11 +1,534 @@ use anyhow::{Context, Result}; -use camino::Utf8Path; +use camino::{Utf8Path, Utf8PathBuf}; use fn_error_context::context; +use rand::Rng; +use xshell::{cmd, Shell}; // Generation markers for integration.fmf const PLAN_MARKER_BEGIN: &str = "# BEGIN GENERATED PLANS\n"; const PLAN_MARKER_END: &str = "# END GENERATED PLANS\n"; +// VM and SSH connectivity timeouts for bcvk integration +// Cloud-init can take 2-3 minutes to start SSH +const VM_READY_TIMEOUT_SECS: u64 = 60; +const SSH_CONNECTIVITY_MAX_ATTEMPTS: u32 = 60; +const SSH_CONNECTIVITY_RETRY_DELAY_SECS: u64 = 3; + +const COMMON_INST_ARGS: &[&str] = &[ + // TODO: Pass down the Secure Boot keys for tests if present + "--firmware=uefi-insecure", + "--label=bootc.test=1", +]; + +// Import the argument types from xtask.rs +use crate::{RunTmtArgs, TmtProvisionArgs}; + +/// Generate a random alphanumeric suffix for VM names +fn generate_random_suffix() -> String { + let mut rng = rand::rng(); + const CHARSET: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789"; + (0..8) + .map(|_| { + let idx = rng.random_range(0..CHARSET.len()); + CHARSET[idx] as char + }) + .collect() +} + +/// Sanitize a plan name for use in a VM name +/// Replaces non-alphanumeric characters (except - and _) with dashes +/// Returns "plan" if the result would be empty +fn sanitize_plan_name(plan: &str) -> String { + let sanitized = plan + .replace('/', "-") + .replace(|c: char| !c.is_alphanumeric() && c != '-' && c != '_', "-") + .trim_matches('-') + .to_string(); + + if sanitized.is_empty() { + "plan".to_string() + } else { + sanitized + } +} + +/// Check that required dependencies are available +#[context("Checking dependencies")] +fn check_dependencies(sh: &Shell) -> Result<()> { + for tool in ["bcvk", "tmt", "rsync"] { + cmd!(sh, "which {tool}") + .ignore_stdout() + .run() + .with_context(|| format!("{} is not available in PATH", tool))?; + } + Ok(()) +} + +/// Wait for a bcvk VM to be ready and return SSH connection info +#[context("Waiting for VM to be ready")] +fn wait_for_vm_ready(sh: &Shell, vm_name: &str) -> Result<(u16, String)> { + use std::thread; + use std::time::Duration; + + for attempt in 1..=VM_READY_TIMEOUT_SECS { + if let Ok(json_output) = cmd!(sh, "bcvk libvirt inspect {vm_name} --format=json") + .ignore_stderr() + .read() + { + if let Ok(json) = serde_json::from_str::(&json_output) { + if let (Some(ssh_port), Some(ssh_key)) = ( + json.get("ssh_port").and_then(|v| v.as_u64()), + json.get("ssh_private_key").and_then(|v| v.as_str()), + ) { + let ssh_port = ssh_port as u16; + return Ok((ssh_port, ssh_key.to_string())); + } + } + } + + if attempt < VM_READY_TIMEOUT_SECS { + thread::sleep(Duration::from_secs(1)); + } + } + + anyhow::bail!( + "VM {} did not become ready within {} seconds", + vm_name, + VM_READY_TIMEOUT_SECS + ) +} + +/// Verify SSH connectivity to the VM +/// Uses a more complex command similar to what TMT runs to ensure full readiness +#[context("Verifying SSH connectivity")] +fn verify_ssh_connectivity(sh: &Shell, port: u16, key_path: &Utf8Path) -> Result<()> { + use std::thread; + use std::time::Duration; + + let port_str = port.to_string(); + for attempt in 1..=SSH_CONNECTIVITY_MAX_ATTEMPTS { + // Test with a complex command like TMT uses (exports + whoami) + // Use IdentitiesOnly=yes to prevent ssh-agent from offering other keys + let result = cmd!( + sh, + "ssh -i {key_path} -p {port_str} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o IdentitiesOnly=yes root@localhost 'export TEST=value; whoami'" + ) + .ignore_stderr() + .read(); + + match &result { + Ok(output) if output.trim() == "root" => { + return Ok(()); + } + _ => {} + } + + if attempt % 10 == 0 { + println!( + "Waiting for SSH... attempt {}/{}", + attempt, SSH_CONNECTIVITY_MAX_ATTEMPTS + ); + } + + if attempt < SSH_CONNECTIVITY_MAX_ATTEMPTS { + thread::sleep(Duration::from_secs(SSH_CONNECTIVITY_RETRY_DELAY_SECS)); + } + } + + anyhow::bail!( + "SSH connectivity check failed after {} attempts", + SSH_CONNECTIVITY_MAX_ATTEMPTS + ) +} + +/// Run TMT tests using bcvk for VM management +/// This spawns a separate VM per test plan to avoid state leakage between tests. +#[context("Running TMT tests")] +pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { + // Check dependencies first + check_dependencies(sh)?; + + let image = &args.image; + let filter_args = &args.filters; + let context = args + .context + .iter() + .map(|v| v.as_str()) + .chain(std::iter::once("running_env=image_mode")) + .map(|v| format!("--context={v}")) + .collect::>(); + let preserve_vm = args.preserve_vm; + + println!("Using bcvk image: {}", image); + + // Create tmt-workdir and copy tmt bits to it + // This works around https://github.com/teemtee/tmt/issues/4062 + let workdir = Utf8Path::new("target/tmt-workdir"); + sh.create_dir(workdir) + .with_context(|| format!("Creating {}", workdir))?; + + // rsync .fmf and tmt directories to workdir + cmd!(sh, "rsync -a --delete --force .fmf tmt {workdir}/") + .run() + .with_context(|| format!("Copying tmt files to {}", workdir))?; + + // Change to workdir for running tmt commands + let _dir = sh.push_dir(workdir); + + // Get the list of plans + println!("Discovering test plans..."); + let plans_output = cmd!(sh, "tmt plan ls") + .read() + .context("Getting list of test plans")?; + + let mut plans: Vec<&str> = plans_output + .lines() + .map(|line| line.trim()) + .filter(|line| !line.is_empty() && line.starts_with("/")) + .collect(); + + // Filter plans based on user arguments + if !filter_args.is_empty() { + let original_count = plans.len(); + plans.retain(|plan| filter_args.iter().any(|arg| plan.contains(arg.as_str()))); + if plans.len() < original_count { + println!( + "Filtered from {} to {} plan(s) based on arguments: {:?}", + original_count, + plans.len(), + filter_args + ); + } + } + + if plans.is_empty() { + println!("No test plans found"); + return Ok(()); + } + + println!("Found {} test plan(s): {:?}", plans.len(), plans); + + // Generate a random suffix for VM names + let random_suffix = generate_random_suffix(); + + // Track overall success/failure + let mut all_passed = true; + let mut test_results: Vec<(String, bool, Option)> = Vec::new(); + + // Run each plan in its own VM + for plan in plans { + let plan_name = sanitize_plan_name(plan); + let vm_name = format!("bootc-tmt-{}-{}", random_suffix, plan_name); + + println!("\n========================================"); + println!("Running plan: {}", plan); + println!("VM name: {}", vm_name); + println!("========================================\n"); + + // Launch VM with bcvk + + let launch_result = cmd!( + sh, + "bcvk libvirt run --name {vm_name} --detach {COMMON_INST_ARGS...} {image}" + ) + .run() + .context("Launching VM with bcvk"); + + if let Err(e) = launch_result { + eprintln!("Failed to launch VM for plan {}: {:#}", plan, e); + all_passed = false; + test_results.push((plan.to_string(), false, None)); + continue; + } + + // Ensure VM cleanup happens even on error (unless --preserve-vm is set) + let cleanup_vm = || { + if preserve_vm { + return; + } + if let Err(e) = cmd!(sh, "bcvk libvirt rm --stop --force {vm_name}") + .ignore_stderr() + .ignore_status() + .run() + { + eprintln!("Warning: Failed to cleanup VM {}: {}", vm_name, e); + } + }; + + // Wait for VM to be ready and get SSH info + let vm_info = wait_for_vm_ready(sh, &vm_name); + let (ssh_port, ssh_key) = match vm_info { + Ok((port, key)) => (port, key), + Err(e) => { + eprintln!("Failed to get VM info for plan {}: {:#}", plan, e); + cleanup_vm(); + all_passed = false; + test_results.push((plan.to_string(), false, None)); + continue; + } + }; + + println!("VM ready, SSH port: {}", ssh_port); + + // Save SSH private key to a temporary file + let key_file = tempfile::NamedTempFile::new().context("Creating temporary SSH key file"); + + let key_file = match key_file { + Ok(f) => f, + Err(e) => { + eprintln!("Failed to create SSH key file for plan {}: {:#}", plan, e); + cleanup_vm(); + all_passed = false; + test_results.push((plan.to_string(), false, None)); + continue; + } + }; + + let key_path = Utf8PathBuf::try_from(key_file.path().to_path_buf()) + .context("Converting key path to UTF-8"); + + let key_path = match key_path { + Ok(p) => p, + Err(e) => { + eprintln!("Failed to convert key path for plan {}: {:#}", plan, e); + cleanup_vm(); + all_passed = false; + test_results.push((plan.to_string(), false, None)); + continue; + } + }; + + if let Err(e) = std::fs::write(&key_path, ssh_key) { + eprintln!("Failed to write SSH key for plan {}: {:#}", plan, e); + cleanup_vm(); + all_passed = false; + test_results.push((plan.to_string(), false, None)); + continue; + } + + // Set proper permissions on the key file (SSH requires 0600) + { + use std::os::unix::fs::PermissionsExt; + let perms = std::fs::Permissions::from_mode(0o600); + if let Err(e) = std::fs::set_permissions(&key_path, perms) { + eprintln!("Failed to set key permissions for plan {}: {:#}", plan, e); + cleanup_vm(); + all_passed = false; + test_results.push((plan.to_string(), false, None)); + continue; + } + } + + // Verify SSH connectivity + println!("Verifying SSH connectivity..."); + if let Err(e) = verify_ssh_connectivity(sh, ssh_port, &key_path) { + eprintln!("SSH verification failed for plan {}: {:#}", plan, e); + cleanup_vm(); + all_passed = false; + test_results.push((plan.to_string(), false, None)); + continue; + } + + println!("SSH connectivity verified"); + + let ssh_port_str = ssh_port.to_string(); + + // Run tmt for this specific plan using connect provisioner + println!("Running tmt tests for plan {}...", plan); + + // Generate a unique run ID for this test + // Use the VM name which already contains a random suffix for uniqueness + let run_id = vm_name.clone(); + + // Run tmt for this specific plan + // Note: provision must come before plan for connect to work properly + let context = context.clone(); + let how = ["--how=connect", "--guest=localhost", "--user=root"]; + let test_result = cmd!( + sh, + "tmt {context...} run --id {run_id} --all -e TMT_SCRIPTS_DIR=/var/lib/tmt/scripts provision {how...} --port {ssh_port_str} --key {key_path} plan --name {plan}" + ) + .run(); + + // Clean up VM regardless of test result (unless --preserve-vm is set) + cleanup_vm(); + + match test_result { + Ok(_) => { + println!("Plan {} completed successfully", plan); + test_results.push((plan.to_string(), true, Some(run_id))); + } + Err(e) => { + eprintln!("Plan {} failed: {:#}", plan, e); + all_passed = false; + test_results.push((plan.to_string(), false, Some(run_id))); + } + } + + // Print VM connection details if preserving + if preserve_vm { + // Copy SSH key to a persistent location + let persistent_key_path = Utf8Path::new("target").join(format!("{}.ssh-key", vm_name)); + if let Err(e) = std::fs::copy(&key_path, &persistent_key_path) { + eprintln!("Warning: Failed to save persistent SSH key: {}", e); + } else { + println!("\n========================================"); + println!("VM preserved for debugging:"); + println!("========================================"); + println!("VM name: {}", vm_name); + println!("SSH port: {}", ssh_port_str); + println!("SSH key: {}", persistent_key_path); + println!("\nTo connect via SSH:"); + println!( + " ssh -i {} -p {} -o IdentitiesOnly=yes root@localhost", + persistent_key_path, ssh_port_str + ); + println!("\nTo cleanup:"); + println!(" bcvk libvirt rm --stop --force {}", vm_name); + println!("========================================\n"); + } + } + } + + // Print summary + println!("\n========================================"); + println!("Test Summary"); + println!("========================================"); + for (plan, passed, _) in &test_results { + let status = if *passed { "PASSED" } else { "FAILED" }; + println!("{}: {}", plan, status); + } + println!("========================================\n"); + + // Print detailed error reports for failed tests + let failed_tests: Vec<_> = test_results + .iter() + .filter(|(_, passed, _)| !passed) + .collect(); + + if !failed_tests.is_empty() { + println!("\n========================================"); + println!("Detailed Error Reports"); + println!("========================================\n"); + + for (plan, _, run_id) in failed_tests { + println!("----------------------------------------"); + println!("Plan: {}", plan); + println!("----------------------------------------"); + + if let Some(id) = run_id { + println!("Run ID: {}\n", id); + + // Run tmt with the specific run ID and generate verbose report + let report_result = cmd!(sh, "tmt run -i {id} report -vvv") + .ignore_status() + .run(); + + match report_result { + Ok(_) => {} + Err(e) => { + eprintln!( + "Warning: Failed to generate detailed report for {}: {:#}", + plan, e + ); + } + } + } else { + println!("Run ID not available - cannot generate detailed report"); + } + + println!("\n"); + } + + println!("========================================\n"); + } + + if !all_passed { + anyhow::bail!("Some test plans failed"); + } + + Ok(()) +} + +/// Provision a VM for manual tmt testing +/// Wraps bcvk libvirt run and waits for SSH connectivity +/// +/// Prints SSH connection details for use with tmt provision --how connect +#[context("Provisioning VM for TMT")] +pub(crate) fn tmt_provision(sh: &Shell, args: &TmtProvisionArgs) -> Result<()> { + // Check for bcvk + if cmd!(sh, "which bcvk").ignore_status().read().is_err() { + anyhow::bail!("bcvk is not available in PATH"); + } + + let image = &args.image; + let vm_name = args + .vm_name + .clone() + .unwrap_or_else(|| format!("bootc-tmt-manual-{}", generate_random_suffix())); + + println!("Provisioning VM..."); + println!(" Image: {}", image); + println!(" VM name: {}\n", vm_name); + + // Launch VM with bcvk + // Use ds=iid-datasource-none to disable cloud-init for faster boot + cmd!( + sh, + "bcvk libvirt run --name {vm_name} --detach {COMMON_INST_ARGS...} {image}" + ) + .run() + .context("Launching VM with bcvk")?; + + println!("VM launched, waiting for SSH..."); + + // Wait for VM to be ready and get SSH info + let (ssh_port, ssh_key) = wait_for_vm_ready(sh, &vm_name)?; + + // Save SSH private key to target directory + let key_dir = Utf8Path::new("target"); + sh.create_dir(key_dir) + .context("Creating target directory")?; + let key_path = key_dir.join(format!("{}.ssh-key", vm_name)); + + std::fs::write(&key_path, ssh_key).context("Writing SSH key file")?; + + // Set proper permissions on key file (0600) + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .context("Setting SSH key file permissions")?; + } + + println!("SSH key saved to: {}", key_path); + + // Verify SSH connectivity + verify_ssh_connectivity(sh, ssh_port, &key_path)?; + + println!("\n========================================"); + println!("VM provisioned successfully!"); + println!("========================================"); + println!("VM name: {}", vm_name); + println!("SSH port: {}", ssh_port); + println!("SSH key: {}", key_path); + println!("\nTo use with tmt:"); + println!(" tmt run --all provision --how connect \\"); + println!(" --guest localhost --port {} \\", ssh_port); + println!(" --user root --key {} \\", key_path); + println!(" plan --name "); + println!("\nTo connect via SSH:"); + println!( + " ssh -i {} -p {} -o IdentitiesOnly=yes root@localhost", + key_path, ssh_port + ); + println!("\nTo cleanup:"); + println!(" bcvk libvirt rm --stop --force {}", vm_name); + println!("========================================\n"); + + Ok(()) +} + /// Parse tmt metadata from a test file /// Looks for: /// # number: N @@ -21,8 +544,11 @@ fn parse_tmt_metadata(content: &str) -> Result> { // Look for "# number: N" line if let Some(rest) = trimmed.strip_prefix("# number:") { - number = Some(rest.trim().parse::() - .context("Failed to parse number field")?); + number = Some( + rest.trim() + .parse::() + .context("Failed to parse number field")?, + ); continue; } @@ -100,8 +626,8 @@ pub(crate) fn update_integration() -> Result<()> { continue; }; - let content = std::fs::read_to_string(&path) - .with_context(|| format!("Reading {}", filename))?; + let content = + std::fs::read_to_string(&path).with_context(|| format!("Reading {}", filename))?; let metadata = parse_tmt_metadata(&content) .with_context(|| format!("Parsing tmt metadata from {}", filename))? @@ -146,45 +672,55 @@ pub(crate) fn update_integration() -> Result<()> { // Sort tests by number tests.sort_by_key(|t| t.number); - // Generate single tests.fmf file + // Generate single tests.fmf file using structured YAML let tests_dir = Utf8Path::new("tmt/tests"); let tests_fmf_path = tests_dir.join("tests.fmf"); - let mut tests_content = String::new(); - - // Add generated code marker - tests_content.push_str("# THIS IS GENERATED CODE - DO NOT EDIT\n"); - tests_content.push_str("# Generated by: cargo xtask tmt\n"); - tests_content.push_str("\n"); + // Build YAML structure + let mut tests_mapping = serde_yaml::Mapping::new(); for test in &tests { - tests_content.push_str(&format!("/test-{:02}-{}:\n", test.number, test.name)); + let test_key = format!("/test-{:02}-{}", test.number, test.name); - // Serialize all fmf attributes from metadata (summary, duration, adjust, etc.) - if let serde_yaml::Value::Mapping(map) = &test.extra { - if !map.is_empty() { - let extra_yaml = serde_yaml::to_string(&test.extra) - .context("Serializing extra metadata")?; - for line in extra_yaml.lines() { - if !line.trim().is_empty() { - tests_content.push_str(&format!(" {}\n", line)); - } - } - } - } + // Start with the extra metadata (summary, duration, adjust, etc.) + let mut test_value = if let serde_yaml::Value::Mapping(map) = &test.extra { + map.clone() + } else { + serde_yaml::Mapping::new() + }; // Add the test command (derived from file type, not in metadata) - if test.test_command.contains('\n') { - tests_content.push_str(" test: |\n"); - for line in test.test_command.lines() { - tests_content.push_str(&format!(" {}\n", line)); - } - } else { - tests_content.push_str(&format!(" test: {}\n", test.test_command)); - } + test_value.insert( + serde_yaml::Value::String("test".to_string()), + serde_yaml::Value::String(test.test_command.clone()), + ); - tests_content.push_str("\n"); + tests_mapping.insert( + serde_yaml::Value::String(test_key), + serde_yaml::Value::Mapping(test_value), + ); } + // Serialize to YAML + let tests_yaml = serde_yaml::to_string(&serde_yaml::Value::Mapping(tests_mapping)) + .context("Serializing tests to YAML")?; + + // Post-process YAML to add blank lines between tests for readability + let mut tests_yaml_formatted = String::new(); + for line in tests_yaml.lines() { + if line.starts_with("/test-") && !tests_yaml_formatted.is_empty() { + tests_yaml_formatted.push('\n'); + } + tests_yaml_formatted.push_str(line); + tests_yaml_formatted.push('\n'); + } + + // Build final content with header + let mut tests_content = String::new(); + tests_content.push_str("# THIS IS GENERATED CODE - DO NOT EDIT\n"); + tests_content.push_str("# Generated by: cargo xtask tmt\n"); + tests_content.push_str("\n"); + tests_content.push_str(&tests_yaml_formatted); + // Only write if content changed let needs_update = match std::fs::read_to_string(&tests_fmf_path) { Ok(existing) => existing != tests_content, @@ -192,58 +728,93 @@ pub(crate) fn update_integration() -> Result<()> { }; if needs_update { - std::fs::write(&tests_fmf_path, tests_content) - .context("Writing tests.fmf")?; + std::fs::write(&tests_fmf_path, tests_content).context("Writing tests.fmf")?; println!("Generated {}", tests_fmf_path); } else { println!("Unchanged: {}", tests_fmf_path); } - // Generate plans section (at root level, no indentation) - let mut plans_section = String::new(); + // Generate plans section using structured YAML + let mut plans_mapping = serde_yaml::Mapping::new(); for test in &tests { - plans_section.push_str(&format!("/plan-{:02}-{}:\n", test.number, test.name)); + let plan_key = format!("/plan-{:02}-{}", test.number, test.name); + let mut plan_value = serde_yaml::Mapping::new(); // Extract summary from extra metadata if let serde_yaml::Value::Mapping(map) = &test.extra { if let Some(summary) = map.get(&serde_yaml::Value::String("summary".to_string())) { - if let Some(summary_str) = summary.as_str() { - plans_section.push_str(&format!(" summary: {}\n", summary_str)); - } + plan_value.insert( + serde_yaml::Value::String("summary".to_string()), + summary.clone(), + ); } } - plans_section.push_str(" discover:\n"); - plans_section.push_str(" how: fmf\n"); - plans_section.push_str(" test:\n"); - plans_section.push_str(&format!(" - /tmt/tests/tests/test-{:02}-{}\n", test.number, test.name)); + // Build discover section + let mut discover = serde_yaml::Mapping::new(); + discover.insert( + serde_yaml::Value::String("how".to_string()), + serde_yaml::Value::String("fmf".to_string()), + ); + let test_path = format!("/tmt/tests/tests/test-{:02}-{}", test.number, test.name); + discover.insert( + serde_yaml::Value::String("test".to_string()), + serde_yaml::Value::Sequence(vec![serde_yaml::Value::String(test_path)]), + ); + plan_value.insert( + serde_yaml::Value::String("discover".to_string()), + serde_yaml::Value::Mapping(discover), + ); - // Extract and serialize adjust section if present + // Extract and add adjust section if present if let serde_yaml::Value::Mapping(map) = &test.extra { if let Some(adjust) = map.get(&serde_yaml::Value::String("adjust".to_string())) { - let adjust_yaml = serde_yaml::to_string(adjust) - .context("Serializing adjust metadata")?; - plans_section.push_str(" adjust:\n"); - for line in adjust_yaml.lines() { - if !line.trim().is_empty() { - plans_section.push_str(&format!(" {}\n", line)); - } - } + plan_value.insert( + serde_yaml::Value::String("adjust".to_string()), + adjust.clone(), + ); } } - plans_section.push_str("\n"); + plans_mapping.insert( + serde_yaml::Value::String(plan_key), + serde_yaml::Value::Mapping(plan_value), + ); + } + + // Serialize plans to YAML + let plans_yaml = serde_yaml::to_string(&serde_yaml::Value::Mapping(plans_mapping)) + .context("Serializing plans to YAML")?; + + // Post-process YAML to add blank lines between plans for readability + // and fix indentation for test list items + let mut plans_section = String::new(); + for line in plans_yaml.lines() { + if line.starts_with("/plan-") && !plans_section.is_empty() { + plans_section.push('\n'); + } + // Fix indentation: YAML serializer uses 2-space indent for list items, + // but we want them at 6 spaces (4 for discover + 2 for test) + if line.starts_with(" - /tmt/tests/") { + plans_section.push_str(" "); + plans_section.push_str(line.trim_start()); + } else { + plans_section.push_str(line); + } + plans_section.push('\n'); } // Update integration.fmf with generated plans let output_path = Utf8Path::new("tmt/plans/integration.fmf"); - let existing_content = std::fs::read_to_string(output_path) - .context("Reading integration.fmf")?; + let existing_content = + std::fs::read_to_string(output_path).context("Reading integration.fmf")?; // Replace plans section - let (before_plans, rest) = existing_content.split_once(PLAN_MARKER_BEGIN) + let (before_plans, rest) = existing_content + .split_once(PLAN_MARKER_BEGIN) .context("Missing # BEGIN GENERATED PLANS marker in integration.fmf")?; - let (_old_plans, after_plans) = rest.split_once(PLAN_MARKER_END) + let (_old_plans, after_plans) = rest + .split_once(PLAN_MARKER_END) .context("Missing # END GENERATED PLANS marker in integration.fmf")?; let new_content = format!( @@ -289,7 +860,9 @@ use tap.nu let extra = metadata.extra.as_mapping().unwrap(); assert_eq!( extra.get(&serde_yaml::Value::String("summary".to_string())), - Some(&serde_yaml::Value::String("Execute booted readonly/nondestructive tests".to_string())) + Some(&serde_yaml::Value::String( + "Execute booted readonly/nondestructive tests".to_string() + )) ); assert_eq!( extra.get(&serde_yaml::Value::String("duration".to_string())), diff --git a/crates/xtask/src/xtask.rs b/crates/xtask/src/xtask.rs index 69ff612a5..8ac734e69 100644 --- a/crates/xtask/src/xtask.rs +++ b/crates/xtask/src/xtask.rs @@ -12,7 +12,6 @@ use anyhow::{Context, Result}; use camino::{Utf8Path, Utf8PathBuf}; use clap::{Args, Parser, Subcommand}; use fn_error_context::context; -use rand::Rng; use xshell::{cmd, Shell}; mod man; @@ -27,12 +26,6 @@ const TAR_REPRODUCIBLE_OPTS: &[&str] = &[ "--pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime", ]; -// VM and SSH connectivity timeouts for bcvk integration -// Cloud-init can take 2-3 minutes to start SSH -const VM_READY_TIMEOUT_SECS: u64 = 60; -const SSH_CONNECTIVITY_MAX_ATTEMPTS: u32 = 60; -const SSH_CONNECTIVITY_RETRY_DELAY_SECS: u64 = 3; - /// Build tasks for bootc #[derive(Debug, Parser)] #[command(name = "xtask")] @@ -62,36 +55,36 @@ enum Commands { /// Arguments for run-tmt command #[derive(Debug, Args)] -struct RunTmtArgs { +pub(crate) struct RunTmtArgs { /// Image name (e.g., "localhost/bootc-integration") - image: String, + pub(crate) image: String, /// Test plan filters (e.g., "readonly") #[arg(value_name = "FILTER")] - filters: Vec, + pub(crate) filters: Vec, /// Include additional context values #[clap(long)] - context: Vec, + pub(crate) context: Vec, /// Set environment variables in the test #[clap(long)] - env: Vec, + pub(crate) env: Vec, /// Preserve VMs after test completion (useful for debugging) #[arg(long)] - preserve_vm: bool, + pub(crate) preserve_vm: bool, } /// Arguments for tmt-provision command #[derive(Debug, Args)] -struct TmtProvisionArgs { +pub(crate) struct TmtProvisionArgs { /// Image name (e.g., "localhost/bootc-integration") - image: String, + pub(crate) image: String, /// VM name (defaults to "bootc-tmt-manual-") #[arg(value_name = "VM_NAME")] - vm_name: Option, + pub(crate) vm_name: Option, } fn main() { @@ -136,8 +129,8 @@ fn try_main() -> Result<()> { Commands::Package => package(&sh), Commands::PackageSrpm => package_srpm(&sh), Commands::Spec => spec(&sh), - Commands::RunTmt(args) => run_tmt(&sh, &args), - Commands::TmtProvision(args) => tmt_provision(&sh, &args), + Commands::RunTmt(args) => tmt::run_tmt(&sh, &args), + Commands::TmtProvision(args) => tmt::tmt_provision(&sh, &args), } } @@ -405,468 +398,3 @@ fn update_generated(sh: &Shell) -> Result<()> { Ok(()) } - -/// Wait for a bcvk VM to be ready and return SSH connection info -#[context("Waiting for VM to be ready")] -fn wait_for_vm_ready(sh: &Shell, vm_name: &str) -> Result<(u16, String)> { - use std::thread; - use std::time::Duration; - - for attempt in 1..=VM_READY_TIMEOUT_SECS { - if let Ok(json_output) = cmd!(sh, "bcvk libvirt inspect {vm_name} --format=json") - .ignore_stderr() - .read() - { - if let Ok(json) = serde_json::from_str::(&json_output) { - if let (Some(ssh_port), Some(ssh_key)) = ( - json.get("ssh_port").and_then(|v| v.as_u64()), - json.get("ssh_private_key").and_then(|v| v.as_str()), - ) { - let ssh_port = ssh_port as u16; - return Ok((ssh_port, ssh_key.to_string())); - } - } - } - - if attempt < VM_READY_TIMEOUT_SECS { - thread::sleep(Duration::from_secs(1)); - } - } - - anyhow::bail!( - "VM {} did not become ready within {} seconds", - vm_name, - VM_READY_TIMEOUT_SECS - ) -} - -/// Verify SSH connectivity to the VM -/// Uses a more complex command similar to what TMT runs to ensure full readiness -#[context("Verifying SSH connectivity")] -fn verify_ssh_connectivity(sh: &Shell, port: u16, key_path: &Utf8Path) -> Result<()> { - use std::thread; - use std::time::Duration; - - let port_str = port.to_string(); - for attempt in 1..=SSH_CONNECTIVITY_MAX_ATTEMPTS { - // Test with a complex command like TMT uses (exports + whoami) - // Use IdentitiesOnly=yes to prevent ssh-agent from offering other keys - let result = cmd!( - sh, - "ssh -i {key_path} -p {port_str} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o IdentitiesOnly=yes root@localhost 'export TEST=value; whoami'" - ) - .ignore_stderr() - .read(); - - match &result { - Ok(output) if output.trim() == "root" => { - return Ok(()); - } - _ => {} - } - - if attempt % 10 == 0 { - println!( - "Waiting for SSH... attempt {}/{}", - attempt, SSH_CONNECTIVITY_MAX_ATTEMPTS - ); - } - - if attempt < SSH_CONNECTIVITY_MAX_ATTEMPTS { - thread::sleep(Duration::from_secs(SSH_CONNECTIVITY_RETRY_DELAY_SECS)); - } - } - - anyhow::bail!( - "SSH connectivity check failed after {} attempts", - SSH_CONNECTIVITY_MAX_ATTEMPTS - ) -} - -/// Generate a random alphanumeric suffix for VM names -fn generate_random_suffix() -> String { - let mut rng = rand::rng(); - const CHARSET: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789"; - (0..8) - .map(|_| { - let idx = rng.random_range(0..CHARSET.len()); - CHARSET[idx] as char - }) - .collect() -} - -/// Sanitize a plan name for use in a VM name -/// Replaces non-alphanumeric characters (except - and _) with dashes -/// Returns "plan" if the result would be empty -fn sanitize_plan_name(plan: &str) -> String { - let sanitized = plan - .replace('/', "-") - .replace(|c: char| !c.is_alphanumeric() && c != '-' && c != '_', "-") - .trim_matches('-') - .to_string(); - - if sanitized.is_empty() { - "plan".to_string() - } else { - sanitized - } -} - -/// Check that required dependencies are available -#[context("Checking dependencies")] -fn check_dependencies(sh: &Shell) -> Result<()> { - for tool in ["bcvk", "tmt", "rsync"] { - cmd!(sh, "which {tool}") - .ignore_stdout() - .run() - .with_context(|| format!("{} is not available in PATH", tool))?; - } - Ok(()) -} - -const COMMON_INST_ARGS: &[&str] = &[ - // TODO: Pass down the Secure Boot keys for tests if present - "--firmware=uefi-insecure", - "--label=bootc.test=1", -]; - -/// Run TMT tests using bcvk for VM management -/// This spawns a separate VM per test plan to avoid state leakage between tests. -#[context("Running TMT tests")] -fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { - // Check dependencies first - check_dependencies(sh)?; - - let image = &args.image; - let filter_args = &args.filters; - let context = args - .context - .iter() - .map(|v| v.as_str()) - .chain(std::iter::once("running_env=image_mode")) - .map(|v| format!("--context={v}")) - .collect::>(); - let preserve_vm = args.preserve_vm; - - println!("Using bcvk image: {}", image); - - // Create tmt-workdir and copy tmt bits to it - // This works around https://github.com/teemtee/tmt/issues/4062 - let workdir = Utf8Path::new("target/tmt-workdir"); - sh.create_dir(workdir) - .with_context(|| format!("Creating {}", workdir))?; - - // rsync .fmf and tmt directories to workdir - cmd!(sh, "rsync -a --delete --force .fmf tmt {workdir}/") - .run() - .with_context(|| format!("Copying tmt files to {}", workdir))?; - - // Change to workdir for running tmt commands - let _dir = sh.push_dir(workdir); - - // Get the list of plans - println!("Discovering test plans..."); - let plans_output = cmd!(sh, "tmt plan ls") - .read() - .context("Getting list of test plans")?; - - let mut plans: Vec<&str> = plans_output - .lines() - .map(|line| line.trim()) - .filter(|line| !line.is_empty() && line.starts_with("/")) - .collect(); - - // Filter plans based on user arguments - if !filter_args.is_empty() { - let original_count = plans.len(); - plans.retain(|plan| filter_args.iter().any(|arg| plan.contains(arg.as_str()))); - if plans.len() < original_count { - println!( - "Filtered from {} to {} plan(s) based on arguments: {:?}", - original_count, - plans.len(), - filter_args - ); - } - } - - if plans.is_empty() { - println!("No test plans found"); - return Ok(()); - } - - println!("Found {} test plan(s): {:?}", plans.len(), plans); - - // Generate a random suffix for VM names - let random_suffix = generate_random_suffix(); - - // Track overall success/failure - let mut all_passed = true; - let mut test_results = Vec::new(); - - // Run each plan in its own VM - for plan in plans { - let plan_name = sanitize_plan_name(plan); - let vm_name = format!("bootc-tmt-{}-{}", random_suffix, plan_name); - - println!("\n========================================"); - println!("Running plan: {}", plan); - println!("VM name: {}", vm_name); - println!("========================================\n"); - - // Launch VM with bcvk - - let launch_result = cmd!( - sh, - "bcvk libvirt run --name {vm_name} --detach {COMMON_INST_ARGS...} {image}" - ) - .run() - .context("Launching VM with bcvk"); - - if let Err(e) = launch_result { - eprintln!("Failed to launch VM for plan {}: {:#}", plan, e); - all_passed = false; - test_results.push((plan.to_string(), false)); - continue; - } - - // Ensure VM cleanup happens even on error (unless --preserve-vm is set) - let cleanup_vm = || { - if preserve_vm { - return; - } - if let Err(e) = cmd!(sh, "bcvk libvirt rm --stop --force {vm_name}") - .ignore_stderr() - .ignore_status() - .run() - { - eprintln!("Warning: Failed to cleanup VM {}: {}", vm_name, e); - } - }; - - // Wait for VM to be ready and get SSH info - let vm_info = wait_for_vm_ready(sh, &vm_name); - let (ssh_port, ssh_key) = match vm_info { - Ok((port, key)) => (port, key), - Err(e) => { - eprintln!("Failed to get VM info for plan {}: {:#}", plan, e); - cleanup_vm(); - all_passed = false; - test_results.push((plan.to_string(), false)); - continue; - } - }; - - println!("VM ready, SSH port: {}", ssh_port); - - // Save SSH private key to a temporary file - let key_file = tempfile::NamedTempFile::new().context("Creating temporary SSH key file"); - - let key_file = match key_file { - Ok(f) => f, - Err(e) => { - eprintln!("Failed to create SSH key file for plan {}: {:#}", plan, e); - cleanup_vm(); - all_passed = false; - test_results.push((plan.to_string(), false)); - continue; - } - }; - - let key_path = Utf8PathBuf::try_from(key_file.path().to_path_buf()) - .context("Converting key path to UTF-8"); - - let key_path = match key_path { - Ok(p) => p, - Err(e) => { - eprintln!("Failed to convert key path for plan {}: {:#}", plan, e); - cleanup_vm(); - all_passed = false; - test_results.push((plan.to_string(), false)); - continue; - } - }; - - if let Err(e) = std::fs::write(&key_path, ssh_key) { - eprintln!("Failed to write SSH key for plan {}: {:#}", plan, e); - cleanup_vm(); - all_passed = false; - test_results.push((plan.to_string(), false)); - continue; - } - - // Set proper permissions on the key file (SSH requires 0600) - { - use std::os::unix::fs::PermissionsExt; - let perms = std::fs::Permissions::from_mode(0o600); - if let Err(e) = std::fs::set_permissions(&key_path, perms) { - eprintln!("Failed to set key permissions for plan {}: {:#}", plan, e); - cleanup_vm(); - all_passed = false; - test_results.push((plan.to_string(), false)); - continue; - } - } - - // Verify SSH connectivity - println!("Verifying SSH connectivity..."); - if let Err(e) = verify_ssh_connectivity(sh, ssh_port, &key_path) { - eprintln!("SSH verification failed for plan {}: {:#}", plan, e); - cleanup_vm(); - all_passed = false; - test_results.push((plan.to_string(), false)); - continue; - } - - println!("SSH connectivity verified"); - - let ssh_port_str = ssh_port.to_string(); - - // Run tmt for this specific plan using connect provisioner - println!("Running tmt tests for plan {}...", plan); - - // Run tmt for this specific plan - // Note: provision must come before plan for connect to work properly - let context = context.clone(); - let how = ["--how=connect", "--guest=localhost", "--user=root"]; - let test_result = cmd!( - sh, - "tmt {context...} run --all -e TMT_SCRIPTS_DIR=/var/lib/tmt/scripts provision {how...} --port {ssh_port_str} --key {key_path} plan --name {plan}" - ) - .run(); - - // Clean up VM regardless of test result (unless --preserve-vm is set) - cleanup_vm(); - - match test_result { - Ok(_) => { - println!("Plan {} completed successfully", plan); - test_results.push((plan.to_string(), true)); - } - Err(e) => { - eprintln!("Plan {} failed: {:#}", plan, e); - all_passed = false; - test_results.push((plan.to_string(), false)); - } - } - - // Print VM connection details if preserving - if preserve_vm { - // Copy SSH key to a persistent location - let persistent_key_path = Utf8Path::new("target").join(format!("{}.ssh-key", vm_name)); - if let Err(e) = std::fs::copy(&key_path, &persistent_key_path) { - eprintln!("Warning: Failed to save persistent SSH key: {}", e); - } else { - println!("\n========================================"); - println!("VM preserved for debugging:"); - println!("========================================"); - println!("VM name: {}", vm_name); - println!("SSH port: {}", ssh_port_str); - println!("SSH key: {}", persistent_key_path); - println!("\nTo connect via SSH:"); - println!( - " ssh -i {} -p {} -o IdentitiesOnly=yes root@localhost", - persistent_key_path, ssh_port_str - ); - println!("\nTo cleanup:"); - println!(" bcvk libvirt rm --stop --force {}", vm_name); - println!("========================================\n"); - } - } - } - - // Print summary - println!("\n========================================"); - println!("Test Summary"); - println!("========================================"); - for (plan, passed) in &test_results { - let status = if *passed { "PASSED" } else { "FAILED" }; - println!("{}: {}", plan, status); - } - println!("========================================\n"); - - if !all_passed { - anyhow::bail!("Some test plans failed"); - } - - Ok(()) -} - -/// Provision a VM for manual tmt testing -/// Wraps bcvk libvirt run and waits for SSH connectivity -/// -/// Prints SSH connection details for use with tmt provision --how connect -#[context("Provisioning VM for TMT")] -fn tmt_provision(sh: &Shell, args: &TmtProvisionArgs) -> Result<()> { - // Check for bcvk - if cmd!(sh, "which bcvk").ignore_status().read().is_err() { - anyhow::bail!("bcvk is not available in PATH"); - } - - let image = &args.image; - let vm_name = args - .vm_name - .clone() - .unwrap_or_else(|| format!("bootc-tmt-manual-{}", generate_random_suffix())); - - println!("Provisioning VM..."); - println!(" Image: {}", image); - println!(" VM name: {}\n", vm_name); - - // Launch VM with bcvk - // Use ds=iid-datasource-none to disable cloud-init for faster boot - cmd!( - sh, - "bcvk libvirt run --name {vm_name} --detach {COMMON_INST_ARGS...} {image}" - ) - .run() - .context("Launching VM with bcvk")?; - - println!("VM launched, waiting for SSH..."); - - // Wait for VM to be ready and get SSH info - let (ssh_port, ssh_key) = wait_for_vm_ready(sh, &vm_name)?; - - // Save SSH private key to target directory - let key_dir = Utf8Path::new("target"); - sh.create_dir(key_dir) - .context("Creating target directory")?; - let key_path = key_dir.join(format!("{}.ssh-key", vm_name)); - - std::fs::write(&key_path, ssh_key).context("Writing SSH key file")?; - - // Set proper permissions on key file (0600) - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) - .context("Setting SSH key file permissions")?; - } - - println!("SSH key saved to: {}", key_path); - - // Verify SSH connectivity - verify_ssh_connectivity(sh, ssh_port, &key_path)?; - - println!("\n========================================"); - println!("VM provisioned successfully!"); - println!("========================================"); - println!("VM name: {}", vm_name); - println!("SSH port: {}", ssh_port); - println!("SSH key: {}", key_path); - println!("\nTo use with tmt:"); - println!(" tmt run --all provision --how connect \\"); - println!(" --guest localhost --port {} \\", ssh_port); - println!(" --user root --key {} \\", key_path); - println!(" plan --name "); - println!("\nTo connect via SSH:"); - println!( - " ssh -i {} -p {} -o IdentitiesOnly=yes root@localhost", - key_path, ssh_port - ); - println!("\nTo cleanup:"); - println!(" bcvk libvirt rm --stop --force {}", vm_name); - println!("========================================\n"); - - Ok(()) -} diff --git a/tmt/plans/integration.fmf b/tmt/plans/integration.fmf index 317ec1b81..5158cca36 100644 --- a/tmt/plans/integration.fmf +++ b/tmt/plans/integration.fmf @@ -122,4 +122,10 @@ execute: test: - /tmt/tests/tests/test-28-factory-reset +/plan-29-soft-reboot-selinux-policy: + summary: Test soft reboot with SELinux policy changes + discover: + how: fmf + test: + - /tmt/tests/tests/test-29-soft-reboot-selinux-policy # END GENERATED PLANS diff --git a/tmt/tests/booted/test-soft-reboot-selinux-policy.nu b/tmt/tests/booted/test-soft-reboot-selinux-policy.nu index 477dbdbdb..ca06efeae 100644 --- a/tmt/tests/booted/test-soft-reboot-selinux-policy.nu +++ b/tmt/tests/booted/test-soft-reboot-selinux-policy.nu @@ -1,3 +1,8 @@ +# number: 29 +# tmt: +# summary: Test soft reboot with SELinux policy changes +# duration: 30m +# # Verify that soft reboot is blocked when SELinux policies differ use std assert use tap.nu diff --git a/tmt/tests/test-29-soft-reboot-selinux-policy.fmf b/tmt/tests/test-29-soft-reboot-selinux-policy.fmf deleted file mode 100644 index 764e0602f..000000000 --- a/tmt/tests/test-29-soft-reboot-selinux-policy.fmf +++ /dev/null @@ -1,3 +0,0 @@ -summary: Test soft reboot with SELinux policy changes -test: nu booted/test-soft-reboot-selinux-policy.nu -duration: 30m diff --git a/tmt/tests/tests.fmf b/tmt/tests/tests.fmf index 69b98f332..b867456a4 100644 --- a/tmt/tests/tests.fmf +++ b/tmt/tests/tests.fmf @@ -64,3 +64,7 @@ duration: 30m test: nu booted/test-factory-reset.nu +/test-29-soft-reboot-selinux-policy: + summary: Test soft reboot with SELinux policy changes + duration: 30m + test: nu booted/test-soft-reboot-selinux-policy.nu From 1a89db671358dfb8cf5783fa4912a99b0d79e92d Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 20 Nov 2025 13:36:10 -0500 Subject: [PATCH 4/7] tests: Use --bind-storage-ro with bcvk To make it easier to do upgrade tests. Signed-off-by: Colin Walters Co-authored-by: Xiaofeng Wang Signed-off-by: Colin Walters --- crates/xtask/src/tmt.rs | 6 ++++- tmt/tests/booted/bootc_testlib.nu | 7 +++++ .../booted/readonly/017-test-bound-storage.nu | 27 +++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 tmt/tests/booted/readonly/017-test-bound-storage.nu diff --git a/crates/xtask/src/tmt.rs b/crates/xtask/src/tmt.rs index 019294cab..ce7b30360 100644 --- a/crates/xtask/src/tmt.rs +++ b/crates/xtask/src/tmt.rs @@ -344,9 +344,13 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { // Note: provision must come before plan for connect to work properly let context = context.clone(); let how = ["--how=connect", "--guest=localhost", "--user=root"]; + let env = ["TMT_SCRIPTS_DIR=/var/lib/tmt/scripts", "BCVK_EXPORT=1"] + .into_iter() + .chain(args.env.iter().map(|v| v.as_str())) + .flat_map(|v| ["--environment", v]); let test_result = cmd!( sh, - "tmt {context...} run --id {run_id} --all -e TMT_SCRIPTS_DIR=/var/lib/tmt/scripts provision {how...} --port {ssh_port_str} --key {key_path} plan --name {plan}" + "tmt {context...} run --id {run_id} --all {env...} provision {how...} --port {ssh_port_str} --key {key_path} plan --name {plan}" ) .run(); diff --git a/tmt/tests/booted/bootc_testlib.nu b/tmt/tests/booted/bootc_testlib.nu index 45089a358..5f15586ab 100644 --- a/tmt/tests/booted/bootc_testlib.nu +++ b/tmt/tests/booted/bootc_testlib.nu @@ -12,3 +12,10 @@ export def reboot [] { tmt-reboot } + +# True if we're running in bcvk with `--bind-storage-ro` and +# we can expect to be able to pull container images from the host. +# See xtask.rs +export def have_hostexports [] { + $env.BCVK_EXPORT? == "1" +} diff --git a/tmt/tests/booted/readonly/017-test-bound-storage.nu b/tmt/tests/booted/readonly/017-test-bound-storage.nu new file mode 100644 index 000000000..c87761853 --- /dev/null +++ b/tmt/tests/booted/readonly/017-test-bound-storage.nu @@ -0,0 +1,27 @@ +# Verify that we have host container storage with bcvk +use std assert +use tap.nu +use ../bootc_testlib.nu + +if not (bootc_testlib have_hostexports) { + print "No host exports, skipping" + exit 0 +} + +bootc status +let st = bootc status --json | from json +let is_composefs = ($st.status.booted.composefs? != null) +if $is_composefs { + # TODO we don't have imageDigest yet in status + exit 0 +} + +let booted = $st.status.booted +let imgref = $booted.image.image.image +let digest = $booted.image.imageDigest +let imgref_untagged = $imgref | split row ':' | first +let digested_imgref = $"($imgref_untagged)@($digest)" +systemd-run -dqP /bin/env +podman inspect $digested_imgref + +tap ok From d8b94b8d28dd15cae9f52095d18c4890b1dcc475 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 20 Nov 2025 13:52:45 -0500 Subject: [PATCH 5/7] tests: Change the upgrade test to support fetching from host This ensures it all can work much more elegantly/naturally with sealed UKI builds - we don't want to do the build-on-target thing. Signed-off-by: Colin Walters --- Justfile | 37 ++++++++++++++----- crates/xtask/src/tmt.rs | 1 + tmt/tests/Dockerfile.upgrade | 3 ++ tmt/tests/booted/test-image-upgrade-reboot.nu | 28 +++++++++----- 4 files changed, 50 insertions(+), 19 deletions(-) create mode 100644 tmt/tests/Dockerfile.upgrade diff --git a/Justfile b/Justfile index fa468a58a..0c4a05b6d 100644 --- a/Justfile +++ b/Justfile @@ -11,6 +11,13 @@ # -------------------------------------------------------------------- +# This image is just the base image plus our updated bootc binary +base_img := "localhost/bootc" +# Derives from the above and adds nushell, cloudinit etc. +integration_img := base_img + "-integration" +# Has a synthetic upgrade +integration_upgrade_img := integration_img + "-upgrade" + # ostree: The default # composefs-sealeduki-sdboot: A system with a sealed composefs using systemd-boot variant := env("BOOTC_variant", "ostree") @@ -33,8 +40,8 @@ buildargs := "--build-arg=base=" + base + " --build-arg=variant=" + variant # Note commonly you might want to override the base image via e.g. # `just build --build-arg=base=quay.io/fedora/fedora-bootc:42` build: - podman build {{base_buildargs}} -t localhost/bootc-bin {{buildargs}} . - ./tests/build-sealed {{variant}} localhost/bootc-bin localhost/bootc + podman build {{base_buildargs}} -t {{base_img}}-bin {{buildargs}} . + ./tests/build-sealed {{variant}} {{base_img}}-bin {{base_img}} # Build a sealed image from current sources. build-sealed: @@ -66,8 +73,8 @@ package: _packagecontainer # This container image has additional testing content and utilities build-integration-test-image: build - cd hack && podman build {{base_buildargs}} -t localhost/bootc-integration-bin -f Containerfile . - ./tests/build-sealed {{variant}} localhost/bootc-integration-bin localhost/bootc-integration + cd hack && podman build {{base_buildargs}} -t {{integration_img}}-bin -f Containerfile . + ./tests/build-sealed {{variant}} {{integration_img}}-bin {{integration_img}} # Keep these in sync with what's used in hack/lbi podman pull -q --retry 5 --retry-delay 5s quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.access.redhat.com/ubi9/podman:latest @@ -76,17 +83,17 @@ test-composefs: # These first two are currently a distinct test suite from tmt that directly # runs an integration test binary in the base image via bcvk just variant=composefs-sealeduki-sdboot build - cargo run --release -p tests-integration -- composefs-bcvk localhost/bootc + cargo run --release -p tests-integration -- composefs-bcvk {{base_img}} # We're trying to move more testing to tmt, so just variant=composefs-sealeduki-sdboot test-tmt readonly # Only used by ci.yml right now build-install-test-image: build-integration-test-image - cd hack && podman build {{base_buildargs}} -t localhost/bootc-integration-install -f Containerfile.drop-lbis + cd hack && podman build {{base_buildargs}} -t {{integration_img}}-install -f Containerfile.drop-lbis # These tests accept the container image as input, and may spawn it. run-container-external-tests: - ./tests/container/run localhost/bootc + ./tests/container/run {{base_img}} # We build the unit tests into a container image build-units: @@ -101,8 +108,18 @@ validate: # # To run an individual test, pass it as an argument like: # `just test-tmt readonly` -test-tmt *ARGS: build-integration-test-image - cargo xtask run-tmt --env=BOOTC_variant={{variant}} localhost/bootc-integration {{ARGS}} +test-tmt *ARGS: build-integration-test-image _build-upgrade-image + @just test-tmt-nobuild {{ARGS}} + +# Generate a local synthetic upgrade +_build-upgrade-image: + podman build -t {{integration_upgrade_img}}-bin --from={{integration_img}}-bin -f tmt/tests/Dockerfile.upgrade /usr/share/empty + ./tests/build-sealed {{variant}} {{integration_upgrade_img}}-bin {{integration_upgrade_img}} + +# Assume the localhost/bootc-integration image is up to date, and just run tests. +# Useful for iterating on tests quickly. +test-tmt-nobuild *ARGS: + cargo xtask run-tmt --env=BOOTC_variant={{variant}} --env=BOOTC_upgrade_image={{integration_upgrade_img}} {{integration_img}} {{ARGS}} # Cleanup all test VMs created by tmt tests tmt-vm-cleanup: @@ -112,7 +129,7 @@ tmt-vm-cleanup: test-container: build-units build-integration-test-image podman run --rm --read-only localhost/bootc-units /usr/bin/bootc-units # Pass these through for cross-checking - podman run --rm --env=BOOTC_variant={{variant}} --env=BOOTC_base={{base}} localhost/bootc-integration bootc-integration-tests container + podman run --rm --env=BOOTC_variant={{variant}} --env=BOOTC_base={{base}} {{integration_img}} bootc-integration-tests container # Remove all container images built (locally) via this Justfile, by matching a label clean-local-images: diff --git a/crates/xtask/src/tmt.rs b/crates/xtask/src/tmt.rs index ce7b30360..0d148ccd1 100644 --- a/crates/xtask/src/tmt.rs +++ b/crates/xtask/src/tmt.rs @@ -18,6 +18,7 @@ const COMMON_INST_ARGS: &[&str] = &[ // TODO: Pass down the Secure Boot keys for tests if present "--firmware=uefi-insecure", "--label=bootc.test=1", + "--bind-storage-ro", ]; // Import the argument types from xtask.rs diff --git a/tmt/tests/Dockerfile.upgrade b/tmt/tests/Dockerfile.upgrade new file mode 100644 index 000000000..ab3b73c7c --- /dev/null +++ b/tmt/tests/Dockerfile.upgrade @@ -0,0 +1,3 @@ +# Just creates a file as a new layer for a synthetic upgrade test +FROM localhost/bootc-integration +RUN touch --reference=/usr/bin/bash /usr/share/testing-bootc-upgrade-apply diff --git a/tmt/tests/booted/test-image-upgrade-reboot.nu b/tmt/tests/booted/test-image-upgrade-reboot.nu index a636b6303..553cae520 100644 --- a/tmt/tests/booted/test-image-upgrade-reboot.nu +++ b/tmt/tests/booted/test-image-upgrade-reboot.nu @@ -15,6 +15,8 @@ use tap.nu # This code runs on *each* boot. # Here we just capture information. bootc status +journalctl --list-boots + let st = bootc status --json | from json let booted = $st.status.booted.image @@ -25,30 +27,38 @@ def parse_cmdline [] { open /proc/cmdline | str trim | split row " " } +def imgsrc [] { + $env.BOOTC_upgrade_image? | default "localhost/bootc-derived-local" +} + # Run on the first boot def initial_build [] { tap begin "local image push + pull + upgrade" - bootc image copy-to-storage + let imgsrc = imgsrc + # For the packit case, we build locally right now + if ($imgsrc | str ends-with "-local") { + bootc image copy-to-storage - # A simple derived container that adds a file - "FROM localhost/bootc + # A simple derived container that adds a file + "FROM localhost/bootc RUN touch /usr/share/testing-bootc-upgrade-apply " | save Dockerfile - # Build it - podman build -t localhost/bootc-derived . + # Build it + podman build -t $imgsrc . + } # Now, switch into the new image - tmt-reboot -c "bootc switch --apply --transport containers-storage localhost/bootc-derived" - - # We cannot perform any other checks here since the system will be automatically rebooted + print $"Applying ($imgsrc)" + bootc switch --transport containers-storage ($imgsrc) + tmt-reboot } # Check we have the updated image def second_boot [] { print "verifying second boot" assert equal $booted.image.transport containers-storage - assert equal $booted.image.image localhost/bootc-derived + assert equal $booted.image.image $"(imgsrc)" # Verify the new file exists "/usr/share/testing-bootc-upgrade-apply" | path exists From eba71484a9cec0b399a6fcff340c21e8d1926e78 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 20 Nov 2025 17:02:06 -0500 Subject: [PATCH 6/7] ci: Expand composefs testing to include upgrade Signed-off-by: Colin Walters --- .github/workflows/ci.yml | 6 +++--- Justfile | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93de66024..4f71b3e28 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -207,9 +207,9 @@ jobs: - name: Unit and container integration tests run: just test-container - - name: Run readonly TMT tests - # TODO: expand to more tests - run: just test-tmt readonly + - name: Run TMT tests + # Note that this one only runs a subset of tests right now + run: just test-composefs - name: Archive TMT logs if: always() diff --git a/Justfile b/Justfile index 0c4a05b6d..f6f56138b 100644 --- a/Justfile +++ b/Justfile @@ -78,14 +78,14 @@ build-integration-test-image: build # Keep these in sync with what's used in hack/lbi podman pull -q --retry 5 --retry-delay 5s quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.access.redhat.com/ubi9/podman:latest -# Build+test composefs; compat alias +# Build+test using the `composefs-sealeduki-sdboot` variant. test-composefs: # These first two are currently a distinct test suite from tmt that directly # runs an integration test binary in the base image via bcvk just variant=composefs-sealeduki-sdboot build cargo run --release -p tests-integration -- composefs-bcvk {{base_img}} - # We're trying to move more testing to tmt, so - just variant=composefs-sealeduki-sdboot test-tmt readonly + # We're trying to move more testing to tmt + just variant=composefs-sealeduki-sdboot test-tmt readonly local-upgrade-reboot # Only used by ci.yml right now build-install-test-image: build-integration-test-image @@ -113,7 +113,7 @@ test-tmt *ARGS: build-integration-test-image _build-upgrade-image # Generate a local synthetic upgrade _build-upgrade-image: - podman build -t {{integration_upgrade_img}}-bin --from={{integration_img}}-bin -f tmt/tests/Dockerfile.upgrade /usr/share/empty + cat tmt/tests/Dockerfile.upgrade | podman build -t {{integration_upgrade_img}}-bin --from={{integration_img}}-bin - ./tests/build-sealed {{variant}} {{integration_upgrade_img}}-bin {{integration_upgrade_img}} # Assume the localhost/bootc-integration image is up to date, and just run tests. From 338aefe39cdfb83a65afa9a3b53434b5ecfd4bef Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 24 Nov 2025 16:35:48 -0500 Subject: [PATCH 7/7] xtask: Add distro-aware bind-storage-ro support for tmt tests CentOS 9 lacks systemd.extra-unit.* support which is required for --bind-storage-ro to work with bcvk. This was causing test failures on centos-9 while working fine on Fedora. Change the approach so tests express intent via `extra.try_bind_storage: true` metadata, and xtask handles the details: - Detect distro by running the container image and parsing os-release - Pass distro to tmt via --context=distro=- - Only add --bind-storage-ro when test wants it AND distro supports it - When bind storage is available, also set BOOTC_upgrade_image env var - Tests can detect missing $env.BOOTC_upgrade_image and fall back to building the upgrade image locally Add --upgrade-image CLI option to specify the upgrade image path, replacing the old --env=BOOTC_upgrade_image approach. Extract magic values to clear const declarations at the top of the file for better maintainability. Assisted-by: Claude Code (Sonnet 4.5) Signed-off-by: Colin Walters --- Justfile | 2 +- crates/xtask/src/tmt.rs | 283 +++++++++++++++--- crates/xtask/src/xtask.rs | 4 + tmt/plans/integration.fmf | 2 + .../booted/readonly/017-test-bound-storage.nu | 17 +- tmt/tests/booted/test-01-readonly.nu | 2 + tmt/tests/booted/test-image-upgrade-reboot.nu | 2 + 7 files changed, 262 insertions(+), 50 deletions(-) diff --git a/Justfile b/Justfile index f6f56138b..5b86c7aea 100644 --- a/Justfile +++ b/Justfile @@ -119,7 +119,7 @@ _build-upgrade-image: # Assume the localhost/bootc-integration image is up to date, and just run tests. # Useful for iterating on tests quickly. test-tmt-nobuild *ARGS: - cargo xtask run-tmt --env=BOOTC_variant={{variant}} --env=BOOTC_upgrade_image={{integration_upgrade_img}} {{integration_img}} {{ARGS}} + cargo xtask run-tmt --env=BOOTC_variant={{variant}} --upgrade-image={{integration_upgrade_img}} {{integration_img}} {{ARGS}} # Cleanup all test VMs created by tmt tests tmt-vm-cleanup: diff --git a/crates/xtask/src/tmt.rs b/crates/xtask/src/tmt.rs index 0d148ccd1..120b988e8 100644 --- a/crates/xtask/src/tmt.rs +++ b/crates/xtask/src/tmt.rs @@ -18,9 +18,20 @@ const COMMON_INST_ARGS: &[&str] = &[ // TODO: Pass down the Secure Boot keys for tests if present "--firmware=uefi-insecure", "--label=bootc.test=1", - "--bind-storage-ro", ]; +// Metadata field names +const FIELD_TRY_BIND_STORAGE: &str = "try_bind_storage"; +const FIELD_SUMMARY: &str = "summary"; +const FIELD_ADJUST: &str = "adjust"; + +// bcvk options +const BCVK_OPT_BIND_STORAGE_RO: &str = "--bind-storage-ro"; +const ENV_BOOTC_UPGRADE_IMAGE: &str = "BOOTC_upgrade_image"; + +// Distro identifiers +const DISTRO_CENTOS_9: &str = "centos-9"; + // Import the argument types from xtask.rs use crate::{RunTmtArgs, TmtProvisionArgs}; @@ -56,7 +67,7 @@ fn sanitize_plan_name(plan: &str) -> String { /// Check that required dependencies are available #[context("Checking dependencies")] fn check_dependencies(sh: &Shell) -> Result<()> { - for tool in ["bcvk", "tmt", "rsync"] { + for tool in ["bcvk", "tmt", "rsync", "podman"] { cmd!(sh, "which {tool}") .ignore_stdout() .run() @@ -65,6 +76,31 @@ fn check_dependencies(sh: &Shell) -> Result<()> { Ok(()) } +/// Detect distro from container image by reading os-release +/// Returns distro string like "centos-9" or "fedora-42" +#[context("Detecting distro from image")] +fn detect_distro_from_image(sh: &Shell, image: &str) -> Result { + let distro = cmd!( + sh, + "podman run --rm {image} bash -c '. /usr/lib/os-release && echo $ID-$VERSION_ID'" + ) + .read() + .context("Failed to run image as container to detect distro")?; + + let distro = distro.trim(); + if distro.is_empty() { + anyhow::bail!("Failed to extract distro from os-release"); + } + + Ok(distro.to_string()) +} + +/// Check if a distro supports --bind-storage-ro +/// CentOS 9 lacks systemd.extra-unit.* support required for bind-storage-ro +fn distro_supports_bind_storage_ro(distro: &str) -> bool { + !distro.starts_with(DISTRO_CENTOS_9) +} + /// Wait for a bcvk VM to be ready and return SSH connection info #[context("Waiting for VM to be ready")] fn wait_for_vm_ready(sh: &Shell, vm_name: &str) -> Result<(u16, String)> { @@ -142,6 +178,43 @@ fn verify_ssh_connectivity(sh: &Shell, port: u16, key_path: &Utf8Path) -> Result ) } +/// Parse integration.fmf to extract extra-try_bind_storage for all plans +#[context("Parsing integration.fmf")] +fn parse_plan_metadata(plans_file: &Utf8Path) -> Result> { + let content = std::fs::read_to_string(plans_file)?; + let yaml = serde_yaml::from_str::(&content) + .context("Failed to parse integration.fmf YAML")?; + + let Some(mapping) = yaml.as_mapping() else { + anyhow::bail!("Expected YAML mapping in integration.fmf"); + }; + + let mut plan_metadata = std::collections::HashMap::new(); + + for (key, value) in mapping { + let Some(plan_name) = key.as_str() else { + continue; + }; + if !plan_name.starts_with("/plan-") { + continue; + } + + let Some(plan_data) = value.as_mapping() else { + continue; + }; + if let Some(try_bind) = plan_data.get(&serde_yaml::Value::String(format!( + "extra-{}", + FIELD_TRY_BIND_STORAGE + ))) { + if let Some(b) = try_bind.as_bool() { + plan_metadata.insert(plan_name.to_string(), b); + } + } + } + + Ok(plan_metadata) +} + /// Run TMT tests using bcvk for VM management /// This spawns a separate VM per test plan to avoid state leakage between tests. #[context("Running TMT tests")] @@ -151,16 +224,21 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { let image = &args.image; let filter_args = &args.filters; + + // Detect distro from the image + let distro = detect_distro_from_image(sh, image)?; + let context = args .context .iter() - .map(|v| v.as_str()) - .chain(std::iter::once("running_env=image_mode")) - .map(|v| format!("--context={v}")) + .map(|v| format!("--context={}", v)) + .chain(std::iter::once(format!("--context=running_env=image_mode"))) + .chain(std::iter::once(format!("--context=distro={}", distro))) .collect::>(); let preserve_vm = args.preserve_vm; println!("Using bcvk image: {}", image); + println!("Detected distro: {}", distro); // Create tmt-workdir and copy tmt bits to it // This works around https://github.com/teemtee/tmt/issues/4062 @@ -176,6 +254,10 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { // Change to workdir for running tmt commands let _dir = sh.push_dir(workdir); + // Parse plan metadata from integration.fmf + let plans_file = Utf8Path::new("tmt/plans/integration.fmf"); + let plan_metadata = parse_plan_metadata(plans_file)?; + // Get the list of plans println!("Discovering test plans..."); let plans_output = cmd!(sh, "tmt plan ls") @@ -216,6 +298,9 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { let mut all_passed = true; let mut test_results: Vec<(String, bool, Option)> = Vec::new(); + // Environment variables to pass to tmt (in addition to args.env) + let mut tmt_env_vars = Vec::new(); + // Run each plan in its own VM for plan in plans { let plan_name = sanitize_plan_name(plan); @@ -226,11 +311,46 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { println!("VM name: {}", vm_name); println!("========================================\n"); - // Launch VM with bcvk + // Reset plan-specific environment variables + tmt_env_vars.clear(); + + // Get bcvk-opts based on plan metadata and distro support + let plan_bcvk_opts = { + let supports_bind_storage_ro = distro_supports_bind_storage_ro(&distro); + + // Plan names from tmt are like /tmt/plans/integration/plan-01-readonly + // but metadata keys are like /plan-01-readonly, so match on suffix + let try_bind_storage = plan_metadata + .iter() + .find(|(key, _)| plan.ends_with(key.as_str())) + .map(|(_, &v)| v) + .unwrap_or(false); + + let mut opts = Vec::new(); + + // If test wants bind storage and distro supports it, add --bind-storage-ro + if try_bind_storage && supports_bind_storage_ro { + opts.push(BCVK_OPT_BIND_STORAGE_RO.to_string()); + + // If upgrade image is provided, set it as an environment variable for tmt + // (not bcvk, as bcvk doesn't support --env) + if let Some(ref upgrade_img) = args.upgrade_image { + tmt_env_vars.push(format!("{}={}", ENV_BOOTC_UPGRADE_IMAGE, upgrade_img)); + } + } else if try_bind_storage && !supports_bind_storage_ro { + println!( + "Note: Test wants bind storage but skipping on {} (missing systemd.extra-unit.* support)", + distro + ); + } + + opts + }; + // Launch VM with bcvk let launch_result = cmd!( sh, - "bcvk libvirt run --name {vm_name} --detach {COMMON_INST_ARGS...} {image}" + "bcvk libvirt run --name {vm_name} --detach {COMMON_INST_ARGS...} {plan_bcvk_opts...} {image}" ) .run() .context("Launching VM with bcvk"); @@ -348,6 +468,7 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { let env = ["TMT_SCRIPTS_DIR=/var/lib/tmt/scripts", "BCVK_EXPORT=1"] .into_iter() .chain(args.env.iter().map(|v| v.as_str())) + .chain(tmt_env_vars.iter().map(|v| v.as_str())) .flat_map(|v| ["--environment", v]); let test_result = cmd!( sh, @@ -537,12 +658,16 @@ pub(crate) fn tmt_provision(sh: &Shell, args: &TmtProvisionArgs) -> Result<()> { /// Parse tmt metadata from a test file /// Looks for: /// # number: N +/// # extra: +/// # try_bind_storage: true /// # tmt: /// # fn parse_tmt_metadata(content: &str) -> Result> { let mut number = None; + let mut in_extra_block = false; let mut in_tmt_block = false; - let mut yaml_lines = Vec::new(); + let mut extra_yaml_lines = Vec::new(); + let mut tmt_yaml_lines = Vec::new(); for line in content.lines().take(50) { let trimmed = line.trim(); @@ -557,17 +682,28 @@ fn parse_tmt_metadata(content: &str) -> Result> { continue; } - if trimmed == "# tmt:" { + if trimmed == "# extra:" { + in_extra_block = true; + in_tmt_block = false; + continue; + } else if trimmed == "# tmt:" { in_tmt_block = true; + in_extra_block = false; continue; - } else if in_tmt_block { + } else if in_extra_block || in_tmt_block { // Stop if we hit a line that doesn't start with #, or is just "#" if !trimmed.starts_with('#') || trimmed == "#" { - break; + in_extra_block = false; + in_tmt_block = false; + continue; } // Remove the leading # and preserve indentation if let Some(yaml_line) = line.strip_prefix('#') { - yaml_lines.push(yaml_line); + if in_extra_block { + extra_yaml_lines.push(yaml_line); + } else { + tmt_yaml_lines.push(yaml_line); + } } } } @@ -576,15 +712,25 @@ fn parse_tmt_metadata(content: &str) -> Result> { return Ok(None); }; - let yaml_content = yaml_lines.join("\n"); - let extra: serde_yaml::Value = if yaml_content.trim().is_empty() { + // Parse extra metadata + let extra_yaml = extra_yaml_lines.join("\n"); + let extra: serde_yaml::Value = if extra_yaml.trim().is_empty() { serde_yaml::Value::Mapping(serde_yaml::Mapping::new()) } else { - serde_yaml::from_str(&yaml_content) - .with_context(|| format!("Failed to parse tmt metadata YAML:\n{}", yaml_content))? + serde_yaml::from_str(&extra_yaml) + .with_context(|| format!("Failed to parse extra metadata YAML:\n{}", extra_yaml))? }; - Ok(Some(TmtMetadata { number, extra })) + // Parse tmt metadata + let tmt_yaml = tmt_yaml_lines.join("\n"); + let tmt: serde_yaml::Value = if tmt_yaml.trim().is_empty() { + serde_yaml::Value::Mapping(serde_yaml::Mapping::new()) + } else { + serde_yaml::from_str(&tmt_yaml) + .with_context(|| format!("Failed to parse tmt metadata YAML:\n{}", tmt_yaml))? + }; + + Ok(Some(TmtMetadata { number, extra, tmt })) } #[derive(Debug, serde::Deserialize)] @@ -592,10 +738,10 @@ fn parse_tmt_metadata(content: &str) -> Result> { struct TmtMetadata { /// Test number for ordering and naming number: u32, - /// All other fmf attributes (summary, duration, adjust, require, etc.) - /// Note: summary and duration are typically required by fmf - #[serde(flatten)] + /// Extra metadata (try_bind_storage, etc.) extra: serde_yaml::Value, + /// TMT metadata (summary, duration, adjust, require, etc.) + tmt: serde_yaml::Value, } #[derive(Debug)] @@ -603,8 +749,10 @@ struct TestDef { number: u32, name: String, test_command: String, - /// All fmf attributes to pass through (summary, duration, adjust, etc.) - extra: serde_yaml::Value, + /// Whether this test wants to try bind storage (if distro supports it) + try_bind_storage: bool, + /// TMT fmf attributes to pass through (summary, duration, adjust, etc.) + tmt: serde_yaml::Value, } /// Generate tmt/plans/integration.fmf from test definitions @@ -666,11 +814,24 @@ pub(crate) fn update_integration() -> Result<()> { let test_command = format!("{} {}", extension, relative_path.display()); + // Check if test wants bind storage + let try_bind_storage = metadata + .extra + .as_mapping() + .and_then(|m| { + m.get(&serde_yaml::Value::String( + FIELD_TRY_BIND_STORAGE.to_string(), + )) + }) + .and_then(|v| v.as_bool()) + .unwrap_or(false); + tests.push(TestDef { number: metadata.number, name: display_name, test_command, - extra: metadata.extra, + try_bind_storage, + tmt: metadata.tmt, }); } @@ -686,8 +847,8 @@ pub(crate) fn update_integration() -> Result<()> { for test in &tests { let test_key = format!("/test-{:02}-{}", test.number, test.name); - // Start with the extra metadata (summary, duration, adjust, etc.) - let mut test_value = if let serde_yaml::Value::Mapping(map) = &test.extra { + // Start with the tmt metadata (summary, duration, adjust, etc.) + let mut test_value = if let serde_yaml::Value::Mapping(map) = &test.tmt { map.clone() } else { serde_yaml::Mapping::new() @@ -745,11 +906,11 @@ pub(crate) fn update_integration() -> Result<()> { let plan_key = format!("/plan-{:02}-{}", test.number, test.name); let mut plan_value = serde_yaml::Mapping::new(); - // Extract summary from extra metadata - if let serde_yaml::Value::Mapping(map) = &test.extra { - if let Some(summary) = map.get(&serde_yaml::Value::String("summary".to_string())) { + // Extract summary from tmt metadata + if let serde_yaml::Value::Mapping(map) = &test.tmt { + if let Some(summary) = map.get(&serde_yaml::Value::String(FIELD_SUMMARY.to_string())) { plan_value.insert( - serde_yaml::Value::String("summary".to_string()), + serde_yaml::Value::String(FIELD_SUMMARY.to_string()), summary.clone(), ); } @@ -772,15 +933,23 @@ pub(crate) fn update_integration() -> Result<()> { ); // Extract and add adjust section if present - if let serde_yaml::Value::Mapping(map) = &test.extra { - if let Some(adjust) = map.get(&serde_yaml::Value::String("adjust".to_string())) { + if let serde_yaml::Value::Mapping(map) = &test.tmt { + if let Some(adjust) = map.get(&serde_yaml::Value::String(FIELD_ADJUST.to_string())) { plan_value.insert( - serde_yaml::Value::String("adjust".to_string()), + serde_yaml::Value::String(FIELD_ADJUST.to_string()), adjust.clone(), ); } } + // Add extra-try_bind_storage if test wants it + if test.try_bind_storage { + plan_value.insert( + serde_yaml::Value::String(format!("extra-{}", FIELD_TRY_BIND_STORAGE)), + serde_yaml::Value::Bool(true), + ); + } + plans_mapping.insert( serde_yaml::Value::String(plan_key), serde_yaml::Value::Mapping(plan_value), @@ -861,16 +1030,16 @@ use tap.nu let metadata = parse_tmt_metadata(content).unwrap().unwrap(); assert_eq!(metadata.number, 1); - // Verify extra fields are captured - let extra = metadata.extra.as_mapping().unwrap(); + // Verify tmt fields are captured + let tmt = metadata.tmt.as_mapping().unwrap(); assert_eq!( - extra.get(&serde_yaml::Value::String("summary".to_string())), + tmt.get(&serde_yaml::Value::String("summary".to_string())), Some(&serde_yaml::Value::String( "Execute booted readonly/nondestructive tests".to_string() )) ); assert_eq!( - extra.get(&serde_yaml::Value::String("duration".to_string())), + tmt.get(&serde_yaml::Value::String("duration".to_string())), Some(&serde_yaml::Value::String("30m".to_string())) ); } @@ -892,9 +1061,9 @@ use std assert let metadata = parse_tmt_metadata(content).unwrap().unwrap(); assert_eq!(metadata.number, 27); - // Verify adjust section is in extra - let extra = metadata.extra.as_mapping().unwrap(); - assert!(extra.contains_key(&serde_yaml::Value::String("adjust".to_string()))); + // Verify adjust section is in tmt + let tmt = metadata.tmt.as_mapping().unwrap(); + assert!(tmt.contains_key(&serde_yaml::Value::String("adjust".to_string()))); } #[test] @@ -924,11 +1093,41 @@ set -eux let metadata = parse_tmt_metadata(content).unwrap().unwrap(); assert_eq!(metadata.number, 26); - let extra = metadata.extra.as_mapping().unwrap(); + let tmt = metadata.tmt.as_mapping().unwrap(); assert_eq!( - extra.get(&serde_yaml::Value::String("duration".to_string())), + tmt.get(&serde_yaml::Value::String("duration".to_string())), Some(&serde_yaml::Value::String("45m".to_string())) ); - assert!(extra.contains_key(&serde_yaml::Value::String("adjust".to_string()))); + assert!(tmt.contains_key(&serde_yaml::Value::String("adjust".to_string()))); + } + + #[test] + fn test_parse_tmt_metadata_with_try_bind_storage() { + let content = r#"# number: 24 +# extra: +# try_bind_storage: true +# tmt: +# summary: Execute local upgrade tests +# duration: 30m +# +use std assert +"#; + + let metadata = parse_tmt_metadata(content).unwrap().unwrap(); + assert_eq!(metadata.number, 24); + + let extra = metadata.extra.as_mapping().unwrap(); + assert_eq!( + extra.get(&serde_yaml::Value::String("try_bind_storage".to_string())), + Some(&serde_yaml::Value::Bool(true)) + ); + + let tmt = metadata.tmt.as_mapping().unwrap(); + assert_eq!( + tmt.get(&serde_yaml::Value::String("summary".to_string())), + Some(&serde_yaml::Value::String( + "Execute local upgrade tests".to_string() + )) + ); } } diff --git a/crates/xtask/src/xtask.rs b/crates/xtask/src/xtask.rs index 8ac734e69..76a5a9b95 100644 --- a/crates/xtask/src/xtask.rs +++ b/crates/xtask/src/xtask.rs @@ -71,6 +71,10 @@ pub(crate) struct RunTmtArgs { #[clap(long)] pub(crate) env: Vec, + /// Upgrade image to use when bind-storage-ro is available (e.g., localhost/bootc-integration-upgrade) + #[clap(long)] + pub(crate) upgrade_image: Option, + /// Preserve VMs after test completion (useful for debugging) #[arg(long)] pub(crate) preserve_vm: bool, diff --git a/tmt/plans/integration.fmf b/tmt/plans/integration.fmf index 5158cca36..87dfb10c3 100644 --- a/tmt/plans/integration.fmf +++ b/tmt/plans/integration.fmf @@ -43,6 +43,7 @@ execute: how: fmf test: - /tmt/tests/tests/test-01-readonly + extra-try_bind_storage: true /plan-20-image-pushpull-upgrade: summary: Execute local upgrade tests @@ -85,6 +86,7 @@ execute: how: fmf test: - /tmt/tests/tests/test-24-image-upgrade-reboot + extra-try_bind_storage: true /plan-25-soft-reboot: summary: Execute soft reboot test diff --git a/tmt/tests/booted/readonly/017-test-bound-storage.nu b/tmt/tests/booted/readonly/017-test-bound-storage.nu index c87761853..9d5640356 100644 --- a/tmt/tests/booted/readonly/017-test-bound-storage.nu +++ b/tmt/tests/booted/readonly/017-test-bound-storage.nu @@ -16,12 +16,15 @@ if $is_composefs { exit 0 } -let booted = $st.status.booted -let imgref = $booted.image.image.image -let digest = $booted.image.imageDigest -let imgref_untagged = $imgref | split row ':' | first -let digested_imgref = $"($imgref_untagged)@($digest)" -systemd-run -dqP /bin/env -podman inspect $digested_imgref +# If we have --bind-storage-ro, then verify it +if ($env.BOOTC_upgrade_image? != null) { + let booted = $st.status.booted + let imgref = $booted.image.image.image + let digest = $booted.image.imageDigest + let imgref_untagged = $imgref | split row ':' | first + let digested_imgref = $"($imgref_untagged)@($digest)" + systemd-run -dqP /bin/env + podman inspect $digested_imgref +} tap ok diff --git a/tmt/tests/booted/test-01-readonly.nu b/tmt/tests/booted/test-01-readonly.nu index 12e1315d4..68d48f406 100644 --- a/tmt/tests/booted/test-01-readonly.nu +++ b/tmt/tests/booted/test-01-readonly.nu @@ -1,4 +1,6 @@ # number: 1 +# extra: +# try_bind_storage: true # tmt: # summary: Execute booted readonly/nondestructive tests # duration: 30m diff --git a/tmt/tests/booted/test-image-upgrade-reboot.nu b/tmt/tests/booted/test-image-upgrade-reboot.nu index 553cae520..676605658 100644 --- a/tmt/tests/booted/test-image-upgrade-reboot.nu +++ b/tmt/tests/booted/test-image-upgrade-reboot.nu @@ -1,4 +1,6 @@ # number: 24 +# extra: +# try_bind_storage: true # tmt: # summary: Execute local upgrade tests # duration: 30m