-
Notifications
You must be signed in to change notification settings - Fork 293
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
WIP: tests/inst: Add destructive test framework
This adds infrastructure to the Rust test suite for destructive tests, and adds a new `transactionality` test which runs rpm-ostree in a loop (along with `ostree-finalize-staged`) and repeatedly kills them. The main goal here is to flush out any "logic errors". I plan to further extend this to reboots and then force poweroffs.
- Loading branch information
Showing
12 changed files
with
498 additions
and
60 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,259 @@ | ||
| //! Test that interrupting a deploy is safe | ||
| use anyhow::Result; | ||
| use commandspec::{sh_command, sh_execute}; | ||
| use rand::Rng; | ||
| use std::{fs, path::Path}; | ||
| use std::{process, thread, time}; | ||
|
|
||
| use crate::rpmostree; | ||
| use crate::test::*; | ||
|
|
||
| const ORIGREF: &'static str = "origref"; | ||
| const TESTREF: &'static str = "testref"; | ||
| const OVLROOT: &'static str = "overlay-root"; | ||
|
|
||
| /// TODO add readonly sysroot handling into base ostree | ||
| fn testinit() -> Result<()> { | ||
| assert!(std::path::Path::new("/run/ostree-booted").exists()); | ||
| sh_execute!( | ||
| r"if ! test -w /sysroot; then | ||
| mount -o remount,rw /sysroot | ||
| fi" | ||
| )?; | ||
| Ok(()) | ||
| } | ||
|
|
||
| enum Boot { | ||
| First, | ||
| Subsequent(u32), | ||
| } | ||
|
|
||
| fn boot() -> Result<Boot> { | ||
| let out = sh_command!("journalctl --list-boots | wc -l") | ||
| .unwrap() | ||
| .output()? | ||
| .stdout; | ||
| let v: u32 = std::str::from_utf8(&out)?.trim().parse()?; | ||
| match v { | ||
| 0 => anyhow::bail!("Unexpected zero boots from journalctl --list-boots"), | ||
| 1 => Ok(Boot::First), | ||
| n => Ok(Boot::Subsequent(n.checked_sub(1).unwrap())), | ||
| } | ||
| } | ||
|
|
||
| fn reboot() -> ! { | ||
| std::fs::write("/etc/kola-reboot", b"").expect("creating /etc/kola-reboot"); | ||
| let ecode = 0; | ||
| std::process::exit(ecode); | ||
| } | ||
|
|
||
| // This is like https://github.com/coreos/coreos-assembler/blob/a83e4c5f1191197132eee4bf570b661ce7e6bf1b/src/cmd-build-fast#L74 | ||
| fn consume_to_overlay_commit<P: AsRef<Path>, D: AsRef<Path>>( | ||
| repo: P, | ||
| base: &str, | ||
| branch: &str, | ||
| dir: D, | ||
| ) -> Result<()> { | ||
| let repo = repo.as_ref(); | ||
| let dir = dir.as_ref(); | ||
| let etc = dir.join("etc"); | ||
| let usretc = dir.join("usr/etc"); | ||
| if etc.exists() { | ||
| assert!(!usretc.exists()); | ||
| fs::create_dir_all(dir.join("usr"))?; | ||
| fs::rename(etc, usretc)?; | ||
| } | ||
| sh_execute!("ostree --repo={repo} commit --consume -b {branch} --base={base} --tree=dir={dir} --owner-uid 0 --owner-gid 0 --selinux-policy-from-base --link-checkout-speedup --no-bindings --no-xattrs", | ||
| repo = repo.to_str().unwrap(), | ||
| branch = branch, | ||
| base = base, | ||
| dir = dir.to_str().unwrap())?; | ||
| Ok(()) | ||
| } | ||
|
|
||
| fn update_srv_repo<P: AsRef<Path>>(srvrepo: P) -> Result<()> { | ||
| let verpath = Path::new("ovl-version"); | ||
| let v: u32 = if verpath.exists() { | ||
| let s = std::fs::read_to_string(&verpath)?; | ||
| let v: u32 = s.trim_end().parse()?; | ||
| v + 1 | ||
| } else { | ||
| 0 | ||
| }; | ||
| let ovldir = Path::new("ovl"); | ||
| mkvroot(&ovldir, v)?; | ||
| write_file(&verpath, &format!("{}", v))?; | ||
| consume_to_overlay_commit(srvrepo, TESTREF, TESTREF, &ovldir)?; | ||
| Ok(()) | ||
| } | ||
|
|
||
| /// Create an archive repository of current OS content. This is a bit expensive; | ||
| /// in the future we should try a trick using the `parent` property on this repo, | ||
| /// and then teach our webserver to redirect to the system for objects it doesn't | ||
| /// have. | ||
| fn generate_srv_repo<P: AsRef<Path>>(sysroot: &ostree::Sysroot, srvrepo: P) -> Result<()> { | ||
| let srvrepo = srvrepo.as_ref(); | ||
| let booted = sysroot.get_booted_deployment().expect("booted deployment"); | ||
| let booted_checksum = booted.get_csum().expect("booted csum"); | ||
| let booted_checksum = booted_checksum.as_str(); | ||
| sh_execute!( | ||
| r#" | ||
| ostree --repo={srvrepo} init --mode=archive | ||
| ostree --repo={srvrepo} pull-local /sysroot/ostree/repo {booted_checksum}"#, | ||
| srvrepo = srvrepo.to_str().unwrap(), | ||
| booted_checksum = booted_checksum | ||
| )?; | ||
| let ovlroot = Path::new(OVLROOT); | ||
| mkroot(&ovlroot)?; | ||
| consume_to_overlay_commit(srvrepo, booted_checksum, TESTREF, &ovlroot)?; | ||
| Ok(()) | ||
| } | ||
|
|
||
| fn run_cycle() -> Result<()> { | ||
| sh_execute!( | ||
| r#" | ||
| systemctl reset-failed rpm-ostreed | ||
| systemctl reset-failed ostree-finalize-staged | ||
| rpm-ostree cleanup -pr | ||
| rpm-ostree rebase testrepo:testref | ||
| systemctl stop ostree-finalize-staged"# | ||
| )?; | ||
| Ok(()) | ||
| } | ||
|
|
||
| fn time_cycle() -> Result<time::Duration> { | ||
| let start = time::Instant::now(); | ||
| run_cycle()?; | ||
| let end = time::Instant::now(); | ||
| Ok(end.duration_since(start)) | ||
| } | ||
|
|
||
| enum DeployResult { | ||
| NotDeployed, | ||
| Staged, | ||
| Finalized, | ||
| } | ||
|
|
||
| fn transactional_test(target_commit: &str) -> Result<()> { | ||
| sh_execute!("rpm-ostree cleanup -pr")?; | ||
| let status = rpmostree::query_status()?; | ||
| let firstdeploy = &status.deployments[0]; | ||
| let orig_commit = &firstdeploy.checksum; | ||
| let elapsed = time_cycle()?; | ||
| let iterations = 50u32; | ||
| println!( | ||
| "Using upgrade duration={:#?} with iterations={}", | ||
| elapsed, iterations | ||
| ); | ||
| let elapsed_div = elapsed / iterations; | ||
| let mut ok_staged = 0; | ||
| let mut ok_finalized = 0; | ||
| let mut errs_not_deployed = 0; | ||
| let mut errs_staged = 0; | ||
| let mut errs_finalized = 0; | ||
| let mut rng = rand::thread_rng(); | ||
| for i in 0..iterations { | ||
| println!("iteration={}", i); | ||
| let r = thread::spawn(run_cycle); | ||
|
|
||
| thread::sleep(rng.gen_range(0, iterations) * elapsed_div); | ||
| let _ = sh_execute!( | ||
| "systemctl kill -s KILL rpm-ostreed | ||
| systemctl kill -s KILL ostree-finalize-staged" | ||
| ); | ||
|
|
||
| let res = r.join().expect("join"); | ||
| // The systemctl stop above will exit successfully even if | ||
| // the process actually died from SIGKILL | ||
| let res = if let Ok(_) = res.as_ref() { | ||
| let finalize_res = process::Command::new("systemctl") | ||
| .args(&["is-failed", "ostree-finalize-staged.service"]) | ||
| .output()?; | ||
| if std::str::from_utf8(&finalize_res.stdout)? == "failed\n" { | ||
| Err(anyhow::anyhow!("ostree-finalize-staged.service failed")) | ||
| } else { | ||
| res | ||
| } | ||
| } else { | ||
| res | ||
| }; | ||
|
|
||
| // This effectively re-validates consistency; TODO | ||
| // add more validation periodically like e.g | ||
| // `ostree admin fsck`. | ||
| let status = rpmostree::query_status()?; | ||
| let firstdeploy = &status.deployments[0]; | ||
| let deployresult = if firstdeploy.checksum == target_commit { | ||
| if let Some(true) = firstdeploy.staged { | ||
| DeployResult::Staged | ||
| } else { | ||
| DeployResult::Finalized | ||
| } | ||
| } else if &firstdeploy.checksum == orig_commit { | ||
| DeployResult::NotDeployed | ||
| } else { | ||
| anyhow::bail!("Unexpected target commit: {}", firstdeploy.checksum); | ||
| }; | ||
| match (res, deployresult) { | ||
| (Ok(_), DeployResult::NotDeployed) => { | ||
| anyhow::bail!("Got successful result but not deployed!") | ||
| } | ||
| (Ok(_), DeployResult::Staged) => ok_staged += 1, | ||
| (Ok(_), DeployResult::Finalized) => ok_finalized += 1, | ||
| (Err(_), DeployResult::NotDeployed) => errs_not_deployed += 1, | ||
| (Err(_), DeployResult::Staged) => errs_staged += 1, | ||
| (Err(_), DeployResult::Finalized) => errs_finalized += 1, | ||
| }; | ||
| } | ||
| println!( | ||
| "iterations={} staged={} finalized={} errs=(undeployed={}, staged={}, finalized={})", | ||
| iterations, ok_staged, ok_finalized, errs_not_deployed, errs_staged, errs_finalized | ||
| ); | ||
| Ok(()) | ||
| } | ||
|
|
||
| #[itest(destructive = true)] | ||
| fn transactionality() -> Result<()> { | ||
| testinit()?; | ||
| let cancellable = Some(gio::Cancellable::new()); | ||
| let sysroot = ostree::Sysroot::new_default(); | ||
| sysroot.load(cancellable.as_ref())?; | ||
| assert!(sysroot.is_booted()); | ||
| let opts = Default::default(); | ||
| // We need this static across reboots | ||
| let srvrepo = Path::new("/var/tmp/ostree-test-srv"); | ||
| if !srvrepo.exists() { | ||
| let tmpdir = tempfile::tempdir_in("/var/tmp")?; | ||
| generate_srv_repo(&sysroot, &tmpdir)?; | ||
| fs::rename(tmpdir.into_path(), srvrepo)?; | ||
| } | ||
| let srvrepo_obj = ostree::Repo::new(&gio::File::new_for_path(srvrepo)); | ||
| srvrepo_obj.open(cancellable.as_ref())?; | ||
| let target_commit = srvrepo_obj.resolve_rev(TESTREF, false)?; | ||
| let target_commit: String = target_commit.into(); | ||
| with_webserver_in(&srvrepo, &opts, move |addr| { | ||
| let cancellable = Some(gio::Cancellable::new()); | ||
| let sysroot = ostree::Sysroot::new_default(); | ||
| sysroot.load(cancellable.as_ref())?; | ||
| let repo = sysroot.repo().unwrap(); | ||
| let status = rpmostree::query_status()?; | ||
|
|
||
| let url = format!("http://{}", addr); | ||
| sh_execute!( | ||
| "ostree remote delete --if-exists testrepo | ||
| ostree remote add --set=gpg-verify=false testrepo {url}", | ||
| url = url | ||
| )?; | ||
|
|
||
| transactional_test(&target_commit)?; | ||
|
|
||
| Ok(()) | ||
| })?; | ||
| Ok(()) | ||
| } | ||
|
|
||
| #[itest(destructive = true)] | ||
| fn other() -> Result<()> { | ||
| testinit()?; | ||
| Ok(()) | ||
| } |
Oops, something went wrong.