diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2951602d7..0882ab882 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -207,9 +207,9 @@ jobs: - name: Unit and container integration tests run: just test-container - - name: Run readonly TMT tests - # TODO: expand to more tests - run: just test-tmt readonly + - name: Run TMT tests + # Note that this one only runs a subset of tests right now + run: just test-composefs - name: Archive TMT logs if: always() diff --git a/Justfile b/Justfile index fa468a58a..f6f56138b 100644 --- a/Justfile +++ b/Justfile @@ -11,6 +11,13 @@ # -------------------------------------------------------------------- +# This image is just the base image plus our updated bootc binary +base_img := "localhost/bootc" +# Derives from the above and adds nushell, cloudinit etc. +integration_img := base_img + "-integration" +# Has a synthetic upgrade +integration_upgrade_img := integration_img + "-upgrade" + # ostree: The default # composefs-sealeduki-sdboot: A system with a sealed composefs using systemd-boot variant := env("BOOTC_variant", "ostree") @@ -33,8 +40,8 @@ buildargs := "--build-arg=base=" + base + " --build-arg=variant=" + variant # Note commonly you might want to override the base image via e.g. # `just build --build-arg=base=quay.io/fedora/fedora-bootc:42` build: - podman build {{base_buildargs}} -t localhost/bootc-bin {{buildargs}} . - ./tests/build-sealed {{variant}} localhost/bootc-bin localhost/bootc + podman build {{base_buildargs}} -t {{base_img}}-bin {{buildargs}} . + ./tests/build-sealed {{variant}} {{base_img}}-bin {{base_img}} # Build a sealed image from current sources. build-sealed: @@ -66,27 +73,27 @@ package: _packagecontainer # This container image has additional testing content and utilities build-integration-test-image: build - cd hack && podman build {{base_buildargs}} -t localhost/bootc-integration-bin -f Containerfile . - ./tests/build-sealed {{variant}} localhost/bootc-integration-bin localhost/bootc-integration + cd hack && podman build {{base_buildargs}} -t {{integration_img}}-bin -f Containerfile . + ./tests/build-sealed {{variant}} {{integration_img}}-bin {{integration_img}} # Keep these in sync with what's used in hack/lbi podman pull -q --retry 5 --retry-delay 5s quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.access.redhat.com/ubi9/podman:latest -# Build+test composefs; compat alias +# Build+test using the `composefs-sealeduki-sdboot` variant. test-composefs: # These first two are currently a distinct test suite from tmt that directly # runs an integration test binary in the base image via bcvk just variant=composefs-sealeduki-sdboot build - cargo run --release -p tests-integration -- composefs-bcvk localhost/bootc - # We're trying to move more testing to tmt, so - just variant=composefs-sealeduki-sdboot test-tmt readonly + cargo run --release -p tests-integration -- composefs-bcvk {{base_img}} + # We're trying to move more testing to tmt + just variant=composefs-sealeduki-sdboot test-tmt readonly local-upgrade-reboot # Only used by ci.yml right now build-install-test-image: build-integration-test-image - cd hack && podman build {{base_buildargs}} -t localhost/bootc-integration-install -f Containerfile.drop-lbis + cd hack && podman build {{base_buildargs}} -t {{integration_img}}-install -f Containerfile.drop-lbis # These tests accept the container image as input, and may spawn it. run-container-external-tests: - ./tests/container/run localhost/bootc + ./tests/container/run {{base_img}} # We build the unit tests into a container image build-units: @@ -101,8 +108,18 @@ validate: # # To run an individual test, pass it as an argument like: # `just test-tmt readonly` -test-tmt *ARGS: build-integration-test-image - cargo xtask run-tmt --env=BOOTC_variant={{variant}} localhost/bootc-integration {{ARGS}} +test-tmt *ARGS: build-integration-test-image _build-upgrade-image + @just test-tmt-nobuild {{ARGS}} + +# Generate a local synthetic upgrade +_build-upgrade-image: + cat tmt/tests/Dockerfile.upgrade | podman build -t {{integration_upgrade_img}}-bin --from={{integration_img}}-bin - + ./tests/build-sealed {{variant}} {{integration_upgrade_img}}-bin {{integration_upgrade_img}} + +# Assume the localhost/bootc-integration image is up to date, and just run tests. +# Useful for iterating on tests quickly. +test-tmt-nobuild *ARGS: + cargo xtask run-tmt --env=BOOTC_variant={{variant}} --env=BOOTC_upgrade_image={{integration_upgrade_img}} {{integration_img}} {{ARGS}} # Cleanup all test VMs created by tmt tests tmt-vm-cleanup: @@ -112,7 +129,7 @@ tmt-vm-cleanup: test-container: build-units build-integration-test-image podman run --rm --read-only localhost/bootc-units /usr/bin/bootc-units # Pass these through for cross-checking - podman run --rm --env=BOOTC_variant={{variant}} --env=BOOTC_base={{base}} localhost/bootc-integration bootc-integration-tests container + podman run --rm --env=BOOTC_variant={{variant}} --env=BOOTC_base={{base}} {{integration_img}} bootc-integration-tests container # Remove all container images built (locally) via this Justfile, by matching a label clean-local-images: diff --git a/Makefile b/Makefile index daffe5199..bc523276b 100644 --- a/Makefile +++ b/Makefile @@ -30,15 +30,16 @@ prefix ?= /usr # the code is really tiny. # (Note we should also make installation of the units conditional on the rhsm feature) CARGO_FEATURES ?= $(shell . /usr/lib/os-release; if echo "$$ID_LIKE" |grep -qF rhel; then echo rhsm; fi) +CARGO_PROFILE ?= release all: bin manpages bin: - cargo build --release --features "$(CARGO_FEATURES)" + cargo build --profile $(CARGO_PROFILE) --features "$(CARGO_FEATURES)" .PHONY: manpages manpages: - cargo run --package xtask -- manpages + cargo run --profile $(CARGO_PROFILE) --package xtask -- manpages STORAGE_RELATIVE_PATH ?= $(shell realpath -m -s --relative-to="$(prefix)/lib/bootc/storage" /sysroot/ostree/bootc/storage) install: diff --git a/crates/xtask/src/xtask.rs b/crates/xtask/src/xtask.rs index bbe56ce6e..eabe8b8a4 100644 --- a/crates/xtask/src/xtask.rs +++ b/crates/xtask/src/xtask.rs @@ -491,18 +491,18 @@ fn generate_random_suffix() -> String { .collect() } -/// Sanitize a plan name for use in a VM name +/// Sanitize a name for use in a VM name /// Replaces non-alphanumeric characters (except - and _) with dashes -/// Returns "plan" if the result would be empty -fn sanitize_plan_name(plan: &str) -> String { - let sanitized = plan +/// Returns "tmt" if the result would be empty +fn sanitize_name(name: &str) -> String { + let sanitized = name .replace('/', "-") .replace(|c: char| !c.is_alphanumeric() && c != '-' && c != '_', "-") .trim_matches('-') .to_string(); if sanitized.is_empty() { - "plan".to_string() + "tmt".to_string() } else { sanitized } @@ -524,6 +524,7 @@ const COMMON_INST_ARGS: &[&str] = &[ // TODO: Pass down the Secure Boot keys for tests if present "--firmware=uefi-insecure", "--label=bootc.test=1", + "--bind-storage-ro", ]; /// Run TMT tests using bcvk for VM management @@ -560,38 +561,38 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { // Change to workdir for running tmt commands let _dir = sh.push_dir(workdir); - // Get the list of plans - println!("Discovering test plans..."); - let plans_output = cmd!(sh, "tmt plan ls") + // Get the list of tests + println!("Discovering tests..."); + let tests_output = cmd!(sh, "tmt test ls") .read() .context("Getting list of test plans")?; - let mut plans: Vec<&str> = plans_output + let mut tests: Vec<&str> = tests_output .lines() .map(|line| line.trim()) .filter(|line| !line.is_empty() && line.starts_with("/")) .collect(); - // Filter plans based on user arguments + // Filter tests based on user arguments if !filter_args.is_empty() { - let original_count = plans.len(); - plans.retain(|plan| filter_args.iter().any(|arg| plan.contains(arg.as_str()))); - if plans.len() < original_count { + let original_count = tests.len(); + tests.retain(|t| filter_args.iter().any(|arg| t.contains(arg.as_str()))); + if tests.len() < original_count { println!( - "Filtered from {} to {} plan(s) based on arguments: {:?}", + "Filtered from {} to {} test(s) based on arguments: {:?}", original_count, - plans.len(), + tests.len(), filter_args ); } } - if plans.is_empty() { - println!("No test plans found"); + if tests.is_empty() { + println!("No tests found"); return Ok(()); } - println!("Found {} test plan(s): {:?}", plans.len(), plans); + println!("Found {} test(s): {:?}", tests.len(), tests); // Generate a random suffix for VM names let random_suffix = generate_random_suffix(); @@ -600,18 +601,17 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { let mut all_passed = true; let mut test_results = Vec::new(); - // Run each plan in its own VM - for plan in plans { - let plan_name = sanitize_plan_name(plan); - let vm_name = format!("bootc-tmt-{}-{}", random_suffix, plan_name); + // Run each test in its own VM + for test in tests { + let test_name = sanitize_name(test); + let vm_name = format!("bootc-tmt-{}-{}", random_suffix, test_name); println!("\n========================================"); - println!("Running plan: {}", plan); + println!("Running test: {}", test); println!("VM name: {}", vm_name); println!("========================================\n"); // Launch VM with bcvk - let launch_result = cmd!( sh, "bcvk libvirt run --name {vm_name} --detach {COMMON_INST_ARGS...} {image}" @@ -620,9 +620,9 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { .context("Launching VM with bcvk"); if let Err(e) = launch_result { - eprintln!("Failed to launch VM for plan {}: {:#}", plan, e); + eprintln!("Failed to launch VM for test {test}: {e:#}"); all_passed = false; - test_results.push((plan.to_string(), false)); + test_results.push((test.to_string(), false)); continue; } @@ -645,10 +645,10 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { let (ssh_port, ssh_key) = match vm_info { Ok((port, key)) => (port, key), Err(e) => { - eprintln!("Failed to get VM info for plan {}: {:#}", plan, e); + eprintln!("Failed to get VM info for plan {test}: {e:#}"); cleanup_vm(); all_passed = false; - test_results.push((plan.to_string(), false)); + test_results.push((test.to_string(), false)); continue; } }; @@ -661,10 +661,10 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { let key_file = match key_file { Ok(f) => f, Err(e) => { - eprintln!("Failed to create SSH key file for plan {}: {:#}", plan, e); + eprintln!("Failed to create SSH key file for plan {test}: {e:#}"); cleanup_vm(); all_passed = false; - test_results.push((plan.to_string(), false)); + test_results.push((test.to_string(), false)); continue; } }; @@ -675,19 +675,19 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { let key_path = match key_path { Ok(p) => p, Err(e) => { - eprintln!("Failed to convert key path for plan {}: {:#}", plan, e); + eprintln!("Failed to convert key path for test {test}: {e:#}"); cleanup_vm(); all_passed = false; - test_results.push((plan.to_string(), false)); + test_results.push((test.to_string(), false)); continue; } }; if let Err(e) = std::fs::write(&key_path, ssh_key) { - eprintln!("Failed to write SSH key for plan {}: {:#}", plan, e); + eprintln!("Failed to write SSH key for test {test}: {e:#}"); cleanup_vm(); all_passed = false; - test_results.push((plan.to_string(), false)); + test_results.push((test.to_string(), false)); continue; } @@ -696,10 +696,10 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { use std::os::unix::fs::PermissionsExt; let perms = std::fs::Permissions::from_mode(0o600); if let Err(e) = std::fs::set_permissions(&key_path, perms) { - eprintln!("Failed to set key permissions for plan {}: {:#}", plan, e); + eprintln!("Failed to set key permissions for test {test}: {e:#}"); cleanup_vm(); all_passed = false; - test_results.push((plan.to_string(), false)); + test_results.push((test.to_string(), false)); continue; } } @@ -707,10 +707,10 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { // Verify SSH connectivity println!("Verifying SSH connectivity..."); if let Err(e) = verify_ssh_connectivity(sh, ssh_port, &key_path) { - eprintln!("SSH verification failed for plan {}: {:#}", plan, e); + eprintln!("SSH verification failed for test {test}: {e:#}"); cleanup_vm(); all_passed = false; - test_results.push((plan.to_string(), false)); + test_results.push((test.to_string(), false)); continue; } @@ -718,16 +718,20 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { let ssh_port_str = ssh_port.to_string(); - // Run tmt for this specific plan using connect provisioner - println!("Running tmt tests for plan {}...", plan); + // Run tmt for this specific test using connect provisioner + println!("Running tmt test {}...", test); - // Run tmt for this specific plan - // Note: provision must come before plan for connect to work properly + // Run tmt for this specific test + // Note: provision must come before tests for connect to work properly let context = context.clone(); let how = ["--how=connect", "--guest=localhost", "--user=root"]; + let env = ["TMT_SCRIPTS_DIR=/var/lib/tmt/scripts", "BCVK_EXPORT=1"] + .into_iter() + .chain(args.env.iter().map(|v| v.as_str())) + .flat_map(|v| ["--environment", v]); let test_result = cmd!( sh, - "tmt {context...} run --all -e TMT_SCRIPTS_DIR=/var/lib/tmt/scripts provision {how...} --port {ssh_port_str} --key {key_path} plan --name {plan}" + "tmt {context...} run --all {env...} provision {how...} --port {ssh_port_str} --key {key_path} tests --name {test}" ) .run(); @@ -736,13 +740,13 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { match test_result { Ok(_) => { - println!("Plan {} completed successfully", plan); - test_results.push((plan.to_string(), true)); + println!("Test {} completed successfully", test); + test_results.push((test.to_string(), true)); } Err(e) => { - eprintln!("Plan {} failed: {:#}", plan, e); + eprintln!("Test {} failed: {:#}", test, e); all_passed = false; - test_results.push((plan.to_string(), false)); + test_results.push((test.to_string(), false)); } } @@ -775,14 +779,15 @@ fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { println!("\n========================================"); println!("Test Summary"); println!("========================================"); - for (plan, passed) in &test_results { + for (test_name, passed) in &test_results { let status = if *passed { "PASSED" } else { "FAILED" }; - println!("{}: {}", plan, status); + println!("{}: {}", test_name, status); } println!("========================================\n"); if !all_passed { - anyhow::bail!("Some test plans failed"); + cmd!(sh, "tmt run -l report -vvv").ignore_status().run()?; + anyhow::bail!("Some tests failed"); } Ok(()) @@ -813,7 +818,7 @@ fn tmt_provision(sh: &Shell, args: &TmtProvisionArgs) -> Result<()> { // Use ds=iid-datasource-none to disable cloud-init for faster boot cmd!( sh, - "bcvk libvirt run --name {vm_name} --detach {COMMON_INST_ARGS...} {image}" + "bcvk libvirt run --bind-storage-ro --name {vm_name} --detach {COMMON_INST_ARGS...} {image}" ) .run() .context("Launching VM with bcvk")?; diff --git a/tmt/plans/integration.fmf b/tmt/plans/integration.fmf index 104f90feb..5cb2723d4 100644 --- a/tmt/plans/integration.fmf +++ b/tmt/plans/integration.fmf @@ -35,87 +35,20 @@ prepare: execute: how: tmt -/readonly-tests: - summary: Execute booted readonly/nondestructive tests +# Regular tests (no special requirements) +/all-tests: + summary: All integration tests discover: how: fmf - test: - - /tmt/tests/test-01-readonly + filter: tag:-image-mode-only -/test-20-local-upgrade: - summary: Execute local upgrade tests +# Image-mode-only tests +/image-mode-tests: + summary: Tests requiring image mode discover: how: fmf - test: - - /tmt/tests/test-20-local-upgrade - -/test-21-logically-bound-switch: - summary: Execute logically bound images tests for switching images - discover: - how: fmf - test: - - /tmt/tests/test-21-logically-bound-switch - -/test-22-logically-bound-install: - summary: Execute logically bound images tests for switching images - discover: - how: fmf - test: - - /tmt/tests/test-22-logically-bound-install - -/test-23-install-outside-container: - summary: Execute tests for installing outside of a container - discover: - how: fmf - test: - - /tmt/tests/test-23-install-outside-container - -/test-24-local-upgrade-reboot: - summary: Execute local upgrade tests with automated reboot - discover: - how: fmf - test: - - /tmt/tests/test-24-local-upgrade-reboot - -/test-25-soft-reboot: - summary: Soft reboot support - discover: - how: fmf - test: - - /tmt/tests/test-25-soft-reboot - -/test-26-examples-build: - summary: Test bootc examples build scripts - discover: - how: fmf - test: - - /tmt/tests/test-26-examples-build + filter: tag:image-mode-only adjust: - when: running_env != image_mode enabled: false - because: packit tests use RPM bootc and does not install /usr/lib/bootc/initramfs-setup - -/test-27-custom-selinux-policy: - summary: Execute restorecon test on system with custom selinux policy - discover: - how: fmf - test: - - /tmt/tests/test-27-custom-selinux-policy - adjust: - - when: running_env != image_mode - enabled: false - because: tmt-reboot does not work with systemd reboot in testing farm environment (see bug-soft-reboot.md) - -/test-23-usroverlay: - summary: Usroverlay - discover: - how: fmf - test: - - /tmt/tests/test-23-usroverlay - -/test-28-factory-reset: - summary: Factory reset - discover: - how: fmf - test: - - /tmt/tests/test-28-factory-reset + because: these tests require features only available in image mode diff --git a/tmt/tests/Dockerfile.upgrade b/tmt/tests/Dockerfile.upgrade new file mode 100644 index 000000000..ab3b73c7c --- /dev/null +++ b/tmt/tests/Dockerfile.upgrade @@ -0,0 +1,3 @@ +# Just creates a file as a new layer for a synthetic upgrade test +FROM localhost/bootc-integration +RUN touch --reference=/usr/bin/bash /usr/share/testing-bootc-upgrade-apply diff --git a/tmt/tests/booted/bootc_testlib.nu b/tmt/tests/booted/bootc_testlib.nu index 45089a358..5f15586ab 100644 --- a/tmt/tests/booted/bootc_testlib.nu +++ b/tmt/tests/booted/bootc_testlib.nu @@ -12,3 +12,10 @@ export def reboot [] { tmt-reboot } + +# True if we're running in bcvk with `--bind-storage-ro` and +# we can expect to be able to pull container images from the host. +# See xtask.rs +export def have_hostexports [] { + $env.BCVK_EXPORT? == "1" +} diff --git a/tmt/tests/booted/readonly/017-test-bound-storage.nu b/tmt/tests/booted/readonly/017-test-bound-storage.nu new file mode 100644 index 000000000..ad12a4df9 --- /dev/null +++ b/tmt/tests/booted/readonly/017-test-bound-storage.nu @@ -0,0 +1,27 @@ +# Verify that we have host container storage with bcvk +use std assert +use tap.nu +use ../bootc_testlib.nu + +if not (bootc_testlib have_hostexports) { + print "No host exports, skipping" + exit 0 +} + +let is_composefs = ($st.status.booted.composefs? != null) +if is_composefs { + # TODO we don't have imageDigest yet in status + exit 0 +} + +bootc status +let st = bootc status --json | from json +let booted = $st.status.booted +let imgref = $booted.image.image.image +let digest = $booted.image.imageDigest +let imgref_untagged = $imgref | split row ':' | first +let digested_imgref = $"($imgref_untagged)@($digest)" +systemd-run -dqP /bin/env +podman inspect $digested_imgref + +tap ok \ No newline at end of file diff --git a/tmt/tests/booted/test-image-upgrade-reboot.nu b/tmt/tests/booted/test-image-upgrade-reboot.nu index b2ad5bc60..59c7cdf38 100644 --- a/tmt/tests/booted/test-image-upgrade-reboot.nu +++ b/tmt/tests/booted/test-image-upgrade-reboot.nu @@ -10,6 +10,8 @@ use tap.nu # This code runs on *each* boot. # Here we just capture information. bootc status +journalctl --list-boots + let st = bootc status --json | from json let booted = $st.status.booted.image @@ -20,30 +22,38 @@ def parse_cmdline [] { open /proc/cmdline | str trim | split row " " } +def imgsrc [] { + $env.BOOTC_upgrade_image? | default "localhost/bootc-derived-local" +} + # Run on the first boot def initial_build [] { tap begin "local image push + pull + upgrade" - bootc image copy-to-storage + let imgsrc = imgsrc + # For the packit case, we build locally right now + if ($imgsrc | str ends-with "-local") { + bootc image copy-to-storage - # A simple derived container that adds a file - "FROM localhost/bootc + # A simple derived container that adds a file + "FROM localhost/bootc RUN touch /usr/share/testing-bootc-upgrade-apply " | save Dockerfile - # Build it - podman build -t localhost/bootc-derived . + # Build it + podman build -t $imgsrc . + } # Now, switch into the new image - tmt-reboot -c "bootc switch --apply --transport containers-storage localhost/bootc-derived" - - # We cannot perform any other checks here since the system will be automatically rebooted + print $"Applying ($imgsrc)" + bootc switch --transport containers-storage ($imgsrc) + tmt-reboot } # Check we have the updated image def second_boot [] { print "verifying second boot" assert equal $booted.image.transport containers-storage - assert equal $booted.image.image localhost/bootc-derived + assert equal $booted.image.image $"(imgsrc)" # Verify the new file exists "/usr/share/testing-bootc-upgrade-apply" | path exists diff --git a/tmt/tests/test-01-readonly.fmf b/tmt/tests/test-01-readonly.fmf index 7789daad7..c7c8b8150 100644 --- a/tmt/tests/test-01-readonly.fmf +++ b/tmt/tests/test-01-readonly.fmf @@ -1,3 +1,3 @@ summary: Execute booted readonly/nondestructive tests -test: ls booted/readonly/*-test-*.nu | sort -n | while read t; do nu $t; done +test: ls booted/readonly/*-test-*.nu | sort -n | while read t; do echo $t; nu $t; done duration: 30m diff --git a/tmt/tests/test-27-custom-selinux-policy.fmf b/tmt/tests/test-27-custom-selinux-policy.fmf index c77f2b011..e3e5c974e 100644 --- a/tmt/tests/test-27-custom-selinux-policy.fmf +++ b/tmt/tests/test-27-custom-selinux-policy.fmf @@ -1,3 +1,4 @@ summary: Execute custom selinux policy test +tag: [image-mode-only] test: nu booted/test-custom-selinux-policy.nu duration: 30m