From f186a76aea2e3cd05d5142fa78ab900bad6d4aa1 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Fri, 22 Jul 2022 18:20:14 +0000 Subject: [PATCH 1/9] ci: instance launch test --- .../buildomat/jobs/build-end-to-end-tests.sh | 33 +++++ .github/buildomat/jobs/deploy.sh | 51 +++---- .github/buildomat/jobs/package.sh | 3 +- Cargo.lock | 60 +++++++++ Cargo.toml | 1 + end-to-end-tests/Cargo.toml | 13 ++ end-to-end-tests/README.adoc | 15 +++ end-to-end-tests/src/bin/bootstrap.rs | 98 ++++++++++++++ end-to-end-tests/src/helpers/ctx.rs | 125 ++++++++++++++++++ end-to-end-tests/src/helpers/mod.rs | 37 ++++++ end-to-end-tests/src/instance_launch.rs | 114 ++++++++++++++++ end-to-end-tests/src/lib.rs | 3 + sled-agent/src/lib.rs | 2 +- 13 files changed, 529 insertions(+), 26 deletions(-) create mode 100644 .github/buildomat/jobs/build-end-to-end-tests.sh create mode 100644 end-to-end-tests/Cargo.toml create mode 100644 end-to-end-tests/README.adoc create mode 100644 end-to-end-tests/src/bin/bootstrap.rs create mode 100644 end-to-end-tests/src/helpers/ctx.rs create mode 100644 end-to-end-tests/src/helpers/mod.rs create mode 100644 end-to-end-tests/src/instance_launch.rs create mode 100644 end-to-end-tests/src/lib.rs diff --git a/.github/buildomat/jobs/build-end-to-end-tests.sh b/.github/buildomat/jobs/build-end-to-end-tests.sh new file mode 100644 index 00000000000..b32e532f9e4 --- /dev/null +++ b/.github/buildomat/jobs/build-end-to-end-tests.sh @@ -0,0 +1,33 @@ +#!/bin/bash +#: +#: name = "helios / build-end-to-end-tests" +#: variety = "basic" +#: target = "helios-latest" +#: rust_toolchain = "nightly-2022-04-27" +#: output_rules = [ +#: "=/work/*.gz", +#: ] +#: + +set -o errexit +set -o pipefail +set -o xtrace + +cargo --version +rustc --version + +ptime -m ./tools/install_builder_prerequisites.sh -yp + +# +# Reduce debuginfo just to line tables. +# +export CARGO_PROFILE_DEV_DEBUG=1 +export CARGO_PROFILE_TEST_DEBUG=1 + +ptime -m cargo build -p end-to-end-tests --tests --bin bootstrap \ + --message-format json-render-diagnostics >/tmp/output.end-to-end.json + +for p in target/debug/bootstrap $(/opt/ooce/bin/jq -r 'select(.profile.test) | .executable' /tmp/output.end-to-end.json); do + # shellcheck disable=SC2094 + ptime -m gzip < "$p" > /work/"$(basename "$p").gz" +done diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index a12da12dbeb..746d7d997d8 100644 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -12,6 +12,9 @@ #: [dependencies.package] #: job = "helios / package" #: +#: [dependencies.build-end-to-end-tests] +#: job = "helios / build-end-to-end-tests" +#: set -o errexit set -o pipefail @@ -106,8 +109,29 @@ pfexec chown build:build /opt/oxide/work cd /opt/oxide/work ptime -m tar xvzf /input/package/work/package.tar.gz +mkdir tests +for p in /input/build-end-to-end-tests/work/*.gz; do + ptime -m gunzip < "$p" > "tests/$(basename "${p%.gz}")" + chmod a+x "tests/$(basename "${p%.gz}")" +done + ptime -m pfexec ./tools/create_virtual_hardware.sh +# +# Image-related tests use images served by catacomb. The lab network is +# IPv4-only; the propolis zones are IPv6-only. These steps set up tcpproxy +# configured to proxy to catacomb via port 54321 in the global zone. +# +pfexec mkdir -p /usr/oxide +pfexec rm -f /usr/oxide/tcpproxy +pfexec curl -sSfL -o /usr/oxide/tcpproxy \ + http://catacomb.eng.oxide.computer:12346/tcpproxy +pfexec chmod +x /usr/oxide/tcpproxy +pfexec rm -f /var/svc/manifest/site/tcpproxy.xml +pfexec curl -sSfL -o /var/svc/manifest/site/tcpproxy.xml \ + http://catacomb.eng.oxide.computer:12346/tcpproxy.xml +pfexec svccfg import /var/svc/manifest/site/tcpproxy.xml + # # XXX Right now, the Nexus external API is available on a specific IPv4 address # on a canned subnet. We need to create an address in the global zone such @@ -129,27 +153,8 @@ pfexec ipadm create-addr -T static -a 192.168.1.199/24 igb0/sidehatch OMICRON_NO_UNINSTALL=1 \ ptime -m pfexec ./target/release/omicron-package install -# Wait up to 5 minutes for RSS to say it's done -for _i in {1..30}; do - sleep 10 - grep "Finished setting up services" "$(svcs -L sled-agent)" && break +./tests/bootstrap +rm ./tests/bootstrap +for test_bin in tests/*; do + ./"$test_bin" done - -set +o xtrace - -start=$SECONDS -while :; do - if (( SECONDS - start > 60 )); then - printf 'FAILURE: NEXUS DID NOT BECOME AVAILABLE\n' >&2 - exit 1 - fi - - if curl --max-time 1 --fail-with-body -i http://192.168.1.20/spoof_login; then - printf 'ok; nexus became available!\n' - break - fi -done - -# -# XXX add tests here! -# diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh index e571755ae19..51fcde9211b 100644 --- a/.github/buildomat/jobs/package.sh +++ b/.github/buildomat/jobs/package.sh @@ -21,12 +21,11 @@ ptime -m ./tools/create_self_signed_cert.sh -yp ptime -m cargo run --locked --release --bin omicron-package -- package -# TODO: write tests and add the resulting test bin here files=( out/*.tar{,.gz} package-manifest.toml smf/sled-agent/config.toml target/release/omicron-package - tools/{create,destroy}_virtual_hardware.sh + tools/create_virtual_hardware.sh ) ptime -m tar cvzf /work/package.tar.gz "${files[@]}" diff --git a/Cargo.lock b/Cargo.lock index b641355450c..2a5494f892f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" +dependencies = [ + "gimli", +] + [[package]] name = "adler" version = "1.0.2" @@ -47,6 +56,9 @@ name = "anyhow" version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" +dependencies = [ + "backtrace", +] [[package]] name = "api_identity" @@ -174,6 +186,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "backtrace" +version = "0.3.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "bare-metal" version = "0.2.5" @@ -1420,6 +1447,18 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "end-to-end-tests" +version = "0.1.0" +dependencies = [ + "anyhow", + "omicron-sled-agent", + "oxide-client", + "rand 0.8.5", + "reqwest", + "tokio", +] + [[package]] name = "endian-type" version = "0.1.2" @@ -1830,6 +1869,12 @@ dependencies = [ "wasi 0.11.0+wasi-snapshot-preview1", ] +[[package]] +name = "gimli" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" + [[package]] name = "glob" version = "0.3.0" @@ -2908,6 +2953,15 @@ name = "nvpair-sys" version = "0.4.0" source = "git+https://github.com/jmesmon/rust-libzfs?branch=master#ecd5a922247a6c5acef55d76c5b8d115572bc850" +[[package]] +name = "object" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" +dependencies = [ + "memchr", +] + [[package]] name = "olpc-cjson" version = "0.1.2" @@ -4597,6 +4651,12 @@ dependencies = [ "regex", ] +[[package]] +name = "rustc-demangle" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" + [[package]] name = "rustc-hash" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 4e3fcc6263c..85e73235111 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ members = [ "oximeter-client", "test-utils", "wicket", + "end-to-end-tests", ] default-members = [ diff --git a/end-to-end-tests/Cargo.toml b/end-to-end-tests/Cargo.toml new file mode 100644 index 00000000000..f0306a0272d --- /dev/null +++ b/end-to-end-tests/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "end-to-end-tests" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" + +[dependencies] +anyhow = { version = "1.0.58", features = ["backtrace"] } +omicron-sled-agent = { path = "../sled-agent" } +oxide-client = { path = "../oxide-client" } +rand = "0.8.5" +reqwest = { version = "0.11.11", default-features = false } +tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] } diff --git a/end-to-end-tests/README.adoc b/end-to-end-tests/README.adoc new file mode 100644 index 00000000000..1e37b01b21f --- /dev/null +++ b/end-to-end-tests/README.adoc @@ -0,0 +1,15 @@ += End-to-end control plane tests + +These tests run in Buildomat. They are built by the xref:../.github/buildomat/jobs/package.sh[package] job and run by the xref:../.github/buildomat/jobs/deploy.sh[deploy] job. + +This package is not built or run by default (it is excluded from `default-members` in xref:../Cargo.toml[]). + +== Running these tests on your machine + +1. xref:../docs/how-to-run.adoc[Make yourself a Gimlet]. +2. Serve a Debian image over HTTP. The tests add a Debian image sourced from `http://[fd00:1122:3344:101::1]:54321/debian-11-genericcloud-amd64.raw`; that IP address belongs to the global zone. You can download the image from https://cloud.debian.org/images/cloud/bullseye/latest/. ++ +Some tools for running an HTTP server with `Range` support are: https://github.com/emikulic/darkhttpd[darkhttpd], https://github.com/joseluisq/static-web-server[static-web-server]. +3. Run the bootstrap bin target: `cargo run -p end-to-end-tests --bin bootstrap` + +Then you can `cargo test -p end-to-end-tests`. diff --git a/end-to-end-tests/src/bin/bootstrap.rs b/end-to-end-tests/src/bin/bootstrap.rs new file mode 100644 index 00000000000..75eea0c7175 --- /dev/null +++ b/end-to-end-tests/src/bin/bootstrap.rs @@ -0,0 +1,98 @@ +use anyhow::{bail, Result}; +use end_to_end_tests::helpers::ctx::{build_client, nexus_addr, Context}; +use end_to_end_tests::helpers::{generate_name, try_loop}; +use oxide_client::types::{ + ByteCount, DiskCreate, DiskSource, IpPoolCreate, IpRange, Ipv4Range, +}; +use oxide_client::{ClientDisksExt, ClientIpPoolsExt, ClientOrganizationsExt}; +use std::net::IpAddr; +use std::time::Duration; +use tokio::time::sleep; + +#[tokio::main] +async fn main() -> Result<()> { + let client = build_client()?; + + // ===== ENSURE NEXUS IS UP ===== // + eprintln!("waiting for nexus to come up..."); + try_loop( + || async { + sleep(Duration::from_secs(1)).await; + client.organization_list().send().await + }, + Duration::from_secs(300), + ) + .await?; + + // ===== CREATE IP POOL ===== // + eprintln!("creating IP pool..."); + let nexus_addr = match nexus_addr().ip() { + IpAddr::V4(addr) => addr.octets(), + IpAddr::V6(_) => bail!("not sure what to do about IPv6 here"), + }; + // TODO: not really sure about a good heuristic for selecting an IP address + // range here. in both my (iliana's) environment and the lab, the last octet + // is 20; in my environment the DHCP range is 100-249, and in the buildomat + // lab environment the network is currently private. + let first = [nexus_addr[0], nexus_addr[1], nexus_addr[2], 50].into(); + let last = [nexus_addr[0], nexus_addr[1], nexus_addr[2], 90].into(); + + let pool_name = client + .ip_pool_create() + .body(IpPoolCreate { + name: generate_name("ip-pool")?, + description: String::new(), + organization: None, + project: None, + }) + .send() + .await? + .name + .clone(); + client + .ip_pool_range_add() + .pool_name(pool_name) + .body(IpRange::V4(Ipv4Range { first, last })) + .send() + .await?; + + // ===== ENSURE DATASETS ARE READY ===== // + eprintln!("ensuring datasets are ready..."); + let ctx = Context::from_client(client).await?; + let disk_name = generate_name("disk")?; + try_loop( + || async { + sleep(Duration::from_secs(1)).await; + ctx.client + .disk_create() + .organization_name(ctx.org_name.clone()) + .project_name(ctx.project_name.clone()) + .body(DiskCreate { + name: disk_name.clone(), + description: String::new(), + disk_source: DiskSource::Blank { + block_size: 512 + .try_into() + .map_err(anyhow::Error::msg)?, + }, + size: ByteCount(1024 * 1024 * 1024), + }) + .send() + .await + .map_err(anyhow::Error::from) + }, + Duration::from_secs(120), + ) + .await?; + ctx.client + .disk_delete() + .organization_name(ctx.org_name.clone()) + .project_name(ctx.project_name.clone()) + .disk_name(disk_name) + .send() + .await?; + ctx.cleanup().await?; + + eprintln!("let's roll."); + Ok(()) +} diff --git a/end-to-end-tests/src/helpers/ctx.rs b/end-to-end-tests/src/helpers/ctx.rs new file mode 100644 index 00000000000..40b76b69910 --- /dev/null +++ b/end-to-end-tests/src/helpers/ctx.rs @@ -0,0 +1,125 @@ +use crate::helpers::generate_name; +use anyhow::{Context as _, Result}; +use omicron_sled_agent::params::ServiceType; +use omicron_sled_agent::rack_setup::config::SetupServiceConfig; +use oxide_client::types::{Name, OrganizationCreate, ProjectCreate}; +use oxide_client::{Client, ClientOrganizationsExt, ClientProjectsExt}; +use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION}; +use reqwest::Url; +use std::net::SocketAddr; +use std::path::Path; +use std::time::Duration; + +#[derive(Clone)] +pub struct Context { + pub client: Client, + pub org_name: Name, + pub project_name: Name, +} + +impl Context { + pub async fn new() -> Result { + Context::from_client(build_client()?).await + } + + pub async fn from_client(client: Client) -> Result { + let org_name = client + .organization_create() + .body(OrganizationCreate { + name: generate_name("org")?, + description: String::new(), + }) + .send() + .await? + .name + .clone(); + + let project_name = client + .project_create() + .organization_name(org_name.clone()) + .body(ProjectCreate { + name: generate_name("proj")?, + description: String::new(), + }) + .send() + .await? + .name + .clone(); + + Ok(Context { client, org_name, project_name }) + } + + pub async fn cleanup(self) -> Result<()> { + self.client + .project_delete() + .organization_name(self.org_name.clone()) + .project_name(self.project_name) + .send() + .await?; + self.client + .organization_delete() + .organization_name(self.org_name) + .send() + .await?; + Ok(()) + } +} + +pub fn build_client() -> Result { + let mut headers = HeaderMap::new(); + headers.insert( + AUTHORIZATION, + HeaderValue::from_static( + "Bearer oxide-spoof-001de000-05e4-4000-8000-000000004007", + ), + ); + + let client = reqwest::ClientBuilder::new() + .default_headers(headers) + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(60)) + .build()?; + Ok(Client::new_with_client(&get_base_url(), client)) +} + +pub fn nexus_addr() -> SocketAddr { + // Check $OXIDE_HOST first. + if let Ok(host) = + std::env::var("OXIDE_HOST").map_err(anyhow::Error::from).and_then(|s| { + Ok(Url::parse(&s)? + .host_str() + .context("no host in OXIDE_HOST url")? + .parse()?) + }) + { + return host; + } + + // If we can find config-rss.toml, look for an external_address. + let rss_config_path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../smf/sled-agent/config-rss.toml"); + if rss_config_path.exists() { + if let Ok(config) = SetupServiceConfig::from_file(rss_config_path) { + for request in config.requests { + for service in request.services { + if let ServiceType::Nexus { external_ip, .. } = + service.service_type + { + return (external_ip, 80).into(); + } + } + } + } + } + + ([192, 168, 1, 20], 80).into() +} + +fn get_base_url() -> String { + // Check $OXIDE_HOST first. + if let Ok(host) = std::env::var("OXIDE_HOST") { + return host; + } + + format!("http://{}", nexus_addr()) +} diff --git a/end-to-end-tests/src/helpers/mod.rs b/end-to-end-tests/src/helpers/mod.rs new file mode 100644 index 00000000000..1ca2eb57d0e --- /dev/null +++ b/end-to-end-tests/src/helpers/mod.rs @@ -0,0 +1,37 @@ +pub mod ctx; + +use anyhow::{Context, Result}; +use oxide_client::types::Name; +use rand::{thread_rng, Rng}; +use std::future::Future; +use std::time::{Duration, Instant}; + +pub fn generate_name(prefix: &str) -> Result { + format!("{}-{:x}", prefix, thread_rng().gen_range(0..0xfff_ffff_ffffu64)) + .try_into() + .map_err(anyhow::Error::msg) +} + +pub async fn try_loop(mut f: F, timeout: Duration) -> Result +where + F: FnMut() -> Fut, + Fut: Future>, + Result: Context, +{ + let start = Instant::now(); + loop { + match f().await { + Ok(t) => return Ok(t), + Err(err) => { + if Instant::now() - start > timeout { + return Err(err).with_context(|| { + format!( + "try_loop timed out after {} seconds", + timeout.as_secs_f64() + ) + }); + } + } + } + } +} diff --git a/end-to-end-tests/src/instance_launch.rs b/end-to-end-tests/src/instance_launch.rs new file mode 100644 index 00000000000..0a75c71f0ba --- /dev/null +++ b/end-to-end-tests/src/instance_launch.rs @@ -0,0 +1,114 @@ +#![cfg(test)] + +use crate::helpers::{ctx::Context, generate_name, try_loop}; +use anyhow::{ensure, Context as _, Result}; +use oxide_client::types::{ + ByteCount, DiskCreate, DiskSource, Distribution, ExternalIpCreate, + GlobalImageCreate, ImageSource, InstanceCpuCount, InstanceCreate, + InstanceDiskAttachment, InstanceNetworkInterfaceAttachment, +}; +use oxide_client::{ClientDisksExt, ClientImagesGlobalExt, ClientInstancesExt}; +use std::time::Duration; +use tokio::time::sleep; + +#[tokio::test] +async fn instance_launch() -> Result<()> { + let ctx = Context::new().await.unwrap(); + + let image_id = ctx + .client + .image_global_create() + .body(GlobalImageCreate { + name: generate_name("debian")?, + description: String::new(), + block_size: 512.try_into().map_err(anyhow::Error::msg)?, + distribution: Distribution { + name: "debian".try_into().map_err(anyhow::Error::msg)?, + version: "propolis-blob".into(), + }, + source: ImageSource::Url { + url: + "http://[fd00:1122:3344:101::1]:54321/debian-11-genericcloud-amd64.raw" + .into(), + }, + }) + .send() + .await? + .id; + + let disk_name = ctx + .client + .disk_create() + .organization_name(ctx.org_name.clone()) + .project_name(ctx.project_name.clone()) + .body(DiskCreate { + name: generate_name("disk")?, + description: String::new(), + disk_source: DiskSource::GlobalImage { image_id }, + size: ByteCount(2048 * 1024 * 1024), + }) + .send() + .await? + .name + .clone(); + + let instance = ctx + .client + .instance_create() + .organization_name(ctx.org_name.clone()) + .project_name(ctx.project_name.clone()) + .body(InstanceCreate { + name: generate_name("instance")?, + description: String::new(), + hostname: "localshark".into(), // 🦈 + memory: ByteCount(1024 * 1024 * 1024), + ncpus: InstanceCpuCount(2), + disks: vec![InstanceDiskAttachment::Attach { name: disk_name }], + network_interfaces: InstanceNetworkInterfaceAttachment::Default, + external_ips: vec![ExternalIpCreate::Ephemeral { pool_name: None }], + user_data: String::new(), + }) + .send() + .await?; + + // poll serial for login prompt, waiting 1 min max + let serial = try_loop( + || async { + sleep(Duration::from_secs(5)).await; + let data = String::from_utf8_lossy( + &ctx.client + .instance_serial_console() + .organization_name(ctx.org_name.clone()) + .project_name(ctx.project_name.clone()) + .instance_name(instance.name.clone()) + .from_start(0) + .max_bytes(10 * 1024 * 1024) + .send() + .await? + .data, + ) + .into_owned(); + ensure!(data.contains("localshark login:"), "not yet booted"); + Ok(data) + }, + Duration::from_secs(300), + ) + .await?; + + let host_keys = serial + .split_once("-----BEGIN SSH HOST KEY KEYS-----") + .and_then(|(_, s)| s.split_once("-----END SSH HOST KEY KEYS-----")) + .map(|(s, _)| s.trim()) + .context("failed to get SSH host keys from serial console")?; + println!("{}", host_keys); + + // tear-down + ctx.client + .instance_stop() + .organization_name(ctx.org_name.clone()) + .project_name(ctx.project_name.clone()) + .instance_name(instance.name.clone()) + .send() + .await?; + ctx.cleanup().await +} diff --git a/end-to-end-tests/src/lib.rs b/end-to-end-tests/src/lib.rs new file mode 100644 index 00000000000..1ee24262bdd --- /dev/null +++ b/end-to-end-tests/src/lib.rs @@ -0,0 +1,3 @@ +pub mod helpers; + +mod instance_launch; diff --git a/sled-agent/src/lib.rs b/sled-agent/src/lib.rs index 602293cab23..75601a142a5 100644 --- a/sled-agent/src/lib.rs +++ b/sled-agent/src/lib.rs @@ -27,7 +27,7 @@ mod instance; mod instance_manager; mod nexus; mod opte; -mod params; +pub mod params; pub mod rack_setup; mod serial; pub mod server; From 170461e0e41c780ca7241ce7eb794258f6d778c4 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Aug 2022 01:20:58 +0000 Subject: [PATCH 2/9] don't run end-to-end tests outside the lab --- .github/buildomat/jobs/build-and-test-linux.sh | 5 ++++- .github/buildomat/jobs/build-and-test.sh | 5 ++++- Cargo.toml | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/buildomat/jobs/build-and-test-linux.sh b/.github/buildomat/jobs/build-and-test-linux.sh index 2dccb84aee4..2ec39d26502 100644 --- a/.github/buildomat/jobs/build-and-test-linux.sh +++ b/.github/buildomat/jobs/build-and-test-linux.sh @@ -59,8 +59,11 @@ ptime -m cargo +'nightly-2022-09-27' build --locked --all-targets --verbose # NOTE: We're using using the same RUSTFLAGS and RUSTDOCFLAGS as above to avoid # having to rebuild here. # +# We also don't use `--workspace` here because we're not prepared to run tests +# from end-to-end-tests. +# banner test -ptime -m cargo +'nightly-2022-09-27' test --workspace --locked --verbose \ +ptime -m cargo +'nightly-2022-09-27' test --locked --verbose \ --no-fail-fast # diff --git a/.github/buildomat/jobs/build-and-test.sh b/.github/buildomat/jobs/build-and-test.sh index dc08c1126fc..15c39c8fa2a 100644 --- a/.github/buildomat/jobs/build-and-test.sh +++ b/.github/buildomat/jobs/build-and-test.sh @@ -59,8 +59,11 @@ ptime -m cargo +'nightly-2022-09-27' build --locked --all-targets --verbose # NOTE: We're using using the same RUSTFLAGS and RUSTDOCFLAGS as above to avoid # having to rebuild here. # +# We also don't use `--workspace` here because we're not prepared to run tests +# from end-to-end-tests. +# banner test -ptime -m cargo +'nightly-2022-09-27' test --workspace --locked --verbose \ +ptime -m cargo +'nightly-2022-09-27' test --locked --verbose \ --no-fail-fast # diff --git a/Cargo.toml b/Cargo.toml index 85e73235111..5295c4922b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ default-members = [ "nexus/db-model", "nexus/defaults", "nexus/types", + "nexus-client", "package", "rpaths", "sled-agent", From 6ea0d4529e28d5246a6de0ebf30dae478c152383 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Fri, 12 Aug 2022 08:05:55 +0000 Subject: [PATCH 3/9] ssh :) --- .github/buildomat/jobs/package.sh | 4 + Cargo.lock | 231 ++++++++++++++++++++++++ Cargo.toml | 10 + end-to-end-tests/Cargo.toml | 4 + end-to-end-tests/src/helpers/ctx.rs | 2 +- end-to-end-tests/src/instance_launch.rs | 158 +++++++++++++++- 6 files changed, 399 insertions(+), 10 deletions(-) diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh index 51fcde9211b..90f8df718e1 100644 --- a/.github/buildomat/jobs/package.sh +++ b/.github/buildomat/jobs/package.sh @@ -16,6 +16,10 @@ set -o xtrace cargo --version rustc --version +sed -e '/\[gateway\]/,/\[request\]/ s/^.*address =.*$/address = "192.168.1.199"/' \ + -e 's/^mac =.*$/mac = "18:c0:4d:0d:9f:b2"/' \ + -i smf/sled-agent/config-rss.toml + ptime -m ./tools/install_builder_prerequisites.sh -yp ptime -m ./tools/create_self_signed_cert.sh -yp diff --git a/Cargo.lock b/Cargo.lock index 2a5494f892f..32dd5e6632d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,6 +27,19 @@ dependencies = [ "heapless", ] +[[package]] +name = "aes" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" +dependencies = [ + "cfg-if 1.0.0", + "cipher", + "cpufeatures", + "ctr", + "opaque-debug", +] + [[package]] name = "aho-corasick" version = "0.7.19" @@ -222,6 +235,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64ct" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b645a089122eccb6111b4f81cbc1a49f5900ac4666bb93ac027feaecf15607bf" + [[package]] name = "bb8" version = "0.8.0" @@ -235,6 +254,19 @@ dependencies = [ "tokio", ] +[[package]] +name = "bcrypt-pbkdf" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12621b8e87feb183a6e5dbb315e49026b2229c4398797ee0ae2d1bc00aef41b9" +dependencies = [ + "blowfish", + "crypto-mac", + "pbkdf2", + "sha2 0.9.9", + "zeroize", +] + [[package]] name = "bcs" version = "0.1.4" @@ -340,6 +372,33 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-modes" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cb03d1bed155d89dce0f845b7899b18a9a163e148fd004e1c28421a783e2d8e" +dependencies = [ + "block-padding", + "cipher", +] + +[[package]] +name = "block-padding" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" + +[[package]] +name = "blowfish" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe3ff3fc1de48c1ac2e3341c4df38b0d1bfb8fdf04632a187c8b75aaa319a7ab" +dependencies = [ + "byteorder", + "cipher", + "opaque-debug", +] + [[package]] name = "bootstore" version = "0.1.0" @@ -954,6 +1013,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "cryptovec" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc7fa13a6bbb2322d325292c57f4c8e7291595506f8289968a0eb61c3130bdf" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "cstr-argument" version = "0.1.2" @@ -964,6 +1033,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctr" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea" +dependencies = [ + "cipher", +] + [[package]] name = "curve25519-dalek" version = "3.2.0" @@ -1232,6 +1310,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309" +dependencies = [ + "dirs-sys", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -1242,6 +1329,17 @@ dependencies = [ "dirs-sys-next", ] +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -1452,10 +1550,14 @@ name = "end-to-end-tests" version = "0.1.0" dependencies = [ "anyhow", + "base64", + "futures", "omicron-sled-agent", "oxide-client", "rand 0.8.5", "reqwest", + "thrussh", + "thrussh-keys", "tokio", ] @@ -2496,6 +2598,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "libsodium-sys" +version = "0.2.8" +source = "git+https://github.com/oxidecomputer/sodiumoxide?branch=oxide/omicron#c2546d157fe62f762300b129a2adcfdfb3bb0503" +dependencies = [ + "cc", + "libc", + "pkg-config", + "walkdir", +] + [[package]] name = "libsqlite3-sys" version = "0.25.1" @@ -2602,6 +2715,12 @@ dependencies = [ "digest 0.10.5", ] +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.5.0" @@ -2873,6 +2992,17 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-integer" version = "0.1.45" @@ -3685,6 +3815,17 @@ dependencies = [ "syn", ] +[[package]] +name = "password-hash" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77e0b28ace46c5a396546bcf443bf422b57049617433d8854227352a4a9b24e7" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "paste" version = "1.0.9" @@ -3709,6 +3850,19 @@ dependencies = [ "once_cell", ] +[[package]] +name = "pbkdf2" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95f5254224e617595d2cc3cc73ff0a5eaf2637519e25f03388154e9378b6ffa" +dependencies = [ + "base64ct", + "crypto-mac", + "hmac 0.11.0", + "password-hash", + "sha2 0.9.9", +] + [[package]] name = "peeking_take_while" version = "0.1.2" @@ -5861,6 +6015,73 @@ dependencies = [ "once_cell", ] +[[package]] +name = "thrussh" +version = "0.33.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e6540238a9adf83df6e66541c182a52acf892ab335595ca965c229ade8536f8" +dependencies = [ + "bitflags", + "byteorder", + "cryptovec", + "digest 0.9.0", + "flate2", + "futures", + "generic-array", + "log", + "rand 0.8.5", + "sha2 0.9.9", + "thiserror", + "thrussh-keys", + "thrussh-libsodium", + "tokio", +] + +[[package]] +name = "thrussh-keys" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a72cc51a2932b18d92f7289332d8564cec4a5014063722a9d3fdca52c5d8f5ab" +dependencies = [ + "aes", + "bcrypt-pbkdf", + "bit-vec", + "block-modes", + "byteorder", + "cryptovec", + "data-encoding", + "dirs", + "futures", + "hmac 0.11.0", + "log", + "md5", + "num-bigint", + "num-integer", + "pbkdf2", + "rand 0.8.5", + "serde", + "serde_derive", + "sha2 0.9.9", + "thiserror", + "thrussh-libsodium", + "tokio", + "tokio-stream", + "yasna", +] + +[[package]] +name = "thrussh-libsodium" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe89c70d27b1cb92e13bc8af63493e890d0de46dae4df0e28233f62b4ed9500" +dependencies = [ + "lazy_static", + "libc", + "libsodium-sys", + "pkg-config", + "vcpkg", +] + [[package]] name = "time" version = "0.1.44" @@ -6842,6 +7063,16 @@ dependencies = [ "libc", ] +[[package]] +name = "yasna" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e262a29d0e61ccf2b6190d7050d4b237535fc76ce4c1210d9caa316f71dffa75" +dependencies = [ + "bit-vec", + "num-bigint", +] + [[package]] name = "zerocopy" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index 5295c4922b0..499ebc7c15e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -116,3 +116,13 @@ panic = "abort" [patch.crates-io.pq-sys] git = 'https://github.com/oxidecomputer/pq-sys' branch = "oxide/omicron" + +# +# libsodium-sys is a dependency of thrussh, used in `end-to-end-tests`. We +# maintain a fork of the sodiumoxide repository to address an illumos build +# issue. See the README.oxide.md in the "oxide" branch of our fork for +# details. +# +[patch.crates-io.libsodium-sys] +git = 'https://github.com/oxidecomputer/sodiumoxide' +branch = "oxide/omicron" diff --git a/end-to-end-tests/Cargo.toml b/end-to-end-tests/Cargo.toml index f0306a0272d..a012b85fd27 100644 --- a/end-to-end-tests/Cargo.toml +++ b/end-to-end-tests/Cargo.toml @@ -6,8 +6,12 @@ license = "MPL-2.0" [dependencies] anyhow = { version = "1.0.58", features = ["backtrace"] } +base64 = "0.13.0" +futures = "0.3.21" omicron-sled-agent = { path = "../sled-agent" } oxide-client = { path = "../oxide-client" } rand = "0.8.5" reqwest = { version = "0.11.11", default-features = false } +thrussh = "0.33.5" +thrussh-keys = "0.21.0" tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] } diff --git a/end-to-end-tests/src/helpers/ctx.rs b/end-to-end-tests/src/helpers/ctx.rs index 40b76b69910..eb9d9dff2f0 100644 --- a/end-to-end-tests/src/helpers/ctx.rs +++ b/end-to-end-tests/src/helpers/ctx.rs @@ -76,7 +76,7 @@ pub fn build_client() -> Result { let client = reqwest::ClientBuilder::new() .default_headers(headers) - .connect_timeout(Duration::from_secs(5)) + .connect_timeout(Duration::from_secs(15)) .timeout(Duration::from_secs(60)) .build()?; Ok(Client::new_with_client(&get_base_url(), client)) diff --git a/end-to-end-tests/src/instance_launch.rs b/end-to-end-tests/src/instance_launch.rs index 0a75c71f0ba..aa0478bc279 100644 --- a/end-to-end-tests/src/instance_launch.rs +++ b/end-to-end-tests/src/instance_launch.rs @@ -2,19 +2,42 @@ use crate::helpers::{ctx::Context, generate_name, try_loop}; use anyhow::{ensure, Context as _, Result}; +use futures::future::Ready; use oxide_client::types::{ ByteCount, DiskCreate, DiskSource, Distribution, ExternalIpCreate, GlobalImageCreate, ImageSource, InstanceCpuCount, InstanceCreate, - InstanceDiskAttachment, InstanceNetworkInterfaceAttachment, + InstanceDiskAttachment, InstanceNetworkInterfaceAttachment, SshKeyCreate, }; -use oxide_client::{ClientDisksExt, ClientImagesGlobalExt, ClientInstancesExt}; +use oxide_client::{ + ClientDisksExt, ClientImagesGlobalExt, ClientInstancesExt, ClientSessionExt, +}; +use std::sync::Arc; use std::time::Duration; +use thrussh::{client::Session, ChannelMsg, Disconnect}; +use thrussh_keys::key::{KeyPair, PublicKey}; +use thrussh_keys::PublicKeyBase64; use tokio::time::sleep; #[tokio::test] async fn instance_launch() -> Result<()> { - let ctx = Context::new().await.unwrap(); + let ctx = Context::new().await?; + + eprintln!("generate SSH key"); + let key = + Arc::new(KeyPair::generate_ed25519().context("key generation failed")?); + let public_key_str = format!("ssh-ed25519 {}", key.public_key_base64()); + eprintln!("create SSH key: {}", public_key_str); + ctx.client + .session_sshkey_create() + .body(SshKeyCreate { + name: generate_name("key")?, + description: String::new(), + public_key: public_key_str, + }) + .send() + .await?; + eprintln!("create global image"); let image_id = ctx .client .image_global_create() @@ -36,6 +59,7 @@ async fn instance_launch() -> Result<()> { .await? .id; + eprintln!("create disk"); let disk_name = ctx .client .disk_create() @@ -52,6 +76,7 @@ async fn instance_launch() -> Result<()> { .name .clone(); + eprintln!("create instance"); let instance = ctx .client .instance_create() @@ -71,7 +96,23 @@ async fn instance_launch() -> Result<()> { .send() .await?; - // poll serial for login prompt, waiting 1 min max + let ip_addr = ctx + .client + .instance_external_ip_list() + .organization_name(ctx.org_name.clone()) + .project_name(ctx.project_name.clone()) + .instance_name(instance.name.clone()) + .send() + .await? + .items + .first() + .context("no external IPs")? + .ip; + eprintln!("instance external IP: {}", ip_addr); + + // poll serial for login prompt, waiting 5 min max + // (pulling disk blocks over HTTP is slow) + eprintln!("waiting for serial console"); let serial = try_loop( || async { sleep(Duration::from_secs(5)).await; @@ -88,21 +129,93 @@ async fn instance_launch() -> Result<()> { .data, ) .into_owned(); - ensure!(data.contains("localshark login:"), "not yet booted"); + ensure!( + data.contains("localshark login:"), + "not yet booted\n{}", + data + ); Ok(data) }, Duration::from_secs(300), ) .await?; - let host_keys = serial + let host_key = serial .split_once("-----BEGIN SSH HOST KEY KEYS-----") .and_then(|(_, s)| s.split_once("-----END SSH HOST KEY KEYS-----")) - .map(|(s, _)| s.trim()) - .context("failed to get SSH host keys from serial console")?; - println!("{}", host_keys); + .and_then(|(lines, _)| { + lines.trim().lines().find(|line| line.starts_with("ssh-ed25519")) + }) + .and_then(|line| line.split_whitespace().nth(1)) + .context("failed to get SSH host key from serial console")?; + eprintln!("host key: ssh-ed25519 {}", host_key); + let host_key = + PublicKey::parse(b"ssh-ed25519", &base64::decode(host_key)?)?; + + eprintln!("connecting ssh"); + let mut session = thrussh::client::connect( + Default::default(), + (ip_addr, 22), + SshClient { host_key }, + ) + .await?; + eprintln!("authenticating ssh"); + ensure!( + session.authenticate_publickey("debian", key).await?, + "authentication failed" + ); + + eprintln!("open session"); + let mut channel = session.channel_open_session().await?; + eprintln!("exec"); + channel.exec(true, "echo 'Hello, Oxide!' | sudo tee /dev/ttyS0").await?; + while let Some(msg) = channel.wait().await { + eprintln!("msg: {:?}", msg); + match msg { + ChannelMsg::Data { data } => { + ensure!( + data.as_ref() == b"Hello, Oxide!\n", + "wrong output: {:?}", + data + ); + } + ChannelMsg::ExitStatus { exit_status } => { + ensure!(exit_status == 0, "exit status {}", exit_status); + break; + } + _ => {} + } + } + + // sign off + eprintln!("disconnecting ssh"); + channel.eof().await?; + session.disconnect(Disconnect::ByApplication, "cya", "en").await?; + + // check that we saw it on the console + eprintln!("waiting for serial console"); + sleep(Duration::from_secs(5)).await; + let data = String::from_utf8_lossy( + &ctx.client + .instance_serial_console() + .organization_name(ctx.org_name.clone()) + .project_name(ctx.project_name.clone()) + .instance_name(instance.name.clone()) + .most_recent(1024 * 1024) + .max_bytes(1024 * 1024) + .send() + .await? + .data, + ) + .into_owned(); + ensure!( + data.contains("Hello, Oxide!"), + "string not seen on console\n{}", + data + ); // tear-down + eprintln!("tear-down"); ctx.client .instance_stop() .organization_name(ctx.org_name.clone()) @@ -112,3 +225,30 @@ async fn instance_launch() -> Result<()> { .await?; ctx.cleanup().await } + +#[derive(Debug)] +struct SshClient { + host_key: PublicKey, +} + +impl thrussh::client::Handler for SshClient { + type Error = anyhow::Error; + type FutureUnit = Ready>; + type FutureBool = Ready>; + + fn finished_bool(self, b: bool) -> Self::FutureBool { + futures::future::ready(Ok((self, b))) + } + + fn finished(self, session: Session) -> Self::FutureUnit { + futures::future::ready(Ok((self, session))) + } + + fn check_server_key( + self, + server_public_key: &PublicKey, + ) -> Self::FutureBool { + let b = &self.host_key == server_public_key; + self.finished_bool(b) + } +} From ec4778d7a44c49ff0efd4fcef75690f6a9e01924 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 16 Aug 2022 19:12:24 +0000 Subject: [PATCH 4/9] avoid hardcoding lab system's MAC address --- .github/buildomat/jobs/deploy.sh | 11 +++++++++++ .github/buildomat/jobs/package.sh | 4 ---- sled-agent/src/bootstrap/params.rs | 27 +++++++++++++++++++++++---- 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index 746d7d997d8..2e769737c86 100644 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -142,6 +142,17 @@ pfexec svccfg import /var/svc/manifest/site/tcpproxy.xml # pfexec ipadm create-addr -T static -a 192.168.1.199/24 igb0/sidehatch +# +# Modify config-rss.toml in the sled-agent zone to use our system's IP and MAC +# address for upstream connectivity. +# +tar xf out/sled-agent.tar pkg/config-rss.toml +sed -e '/\[gateway\]/,/\[request\]/ s/^.*address =.*$/address = "192.168.1.199"/' \ + -e "s/^mac =.*$/mac = \"$(dladm show-phys -m -p -o ADDRESS | head -n 1)\"/" \ + -i pkg/config-rss.toml +tar rf out/sled-agent.tar pkg/config-rss.toml +rm -rf pkg + # # This OMICRON_NO_UNINSTALL hack here is so that there is no implicit uninstall # before the install. This doesn't work right now because, above, we made diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh index 90f8df718e1..51fcde9211b 100644 --- a/.github/buildomat/jobs/package.sh +++ b/.github/buildomat/jobs/package.sh @@ -16,10 +16,6 @@ set -o xtrace cargo --version rustc --version -sed -e '/\[gateway\]/,/\[request\]/ s/^.*address =.*$/address = "192.168.1.199"/' \ - -e 's/^mac =.*$/mac = "18:c0:4d:0d:9f:b2"/' \ - -i smf/sled-agent/config-rss.toml - ptime -m ./tools/install_builder_prerequisites.sh -yp ptime -m ./tools/create_self_signed_cert.sh -yp diff --git a/sled-agent/src/bootstrap/params.rs b/sled-agent/src/bootstrap/params.rs index 8898ac7a02f..94db44729d4 100644 --- a/sled-agent/src/bootstrap/params.rs +++ b/sled-agent/src/bootstrap/params.rs @@ -7,9 +7,9 @@ use super::trust_quorum::SerializableShareDistribution; use macaddr::MacAddr6; use omicron_common::address::{Ipv6Subnet, SLED_PREFIX}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use serde_with::serde_as; -use serde_with::DisplayFromStr; +use serde_with::DeserializeAs; use serde_with::PickFirst; use std::borrow::Cow; use std::net::Ipv4Addr; @@ -29,11 +29,30 @@ pub struct Gateway { // This uses the `serde_with` crate's `serde_as` attribute, which tries // each of the listed serialization types (starting with the default) until // one succeeds. This supports deserialization from either an array of u8, - // or the display-string representation. - #[serde_as(as = "PickFirst<(_, DisplayFromStr)>")] + // or the display-string representation. (Our custom `ZeroPadded` adapter + // works around non-zero-padded MAC address bytes as seen in illumos + // `dladm`, which the macaddr crate refuses to parse.) + #[serde_as(as = "PickFirst<(_, ZeroPadded)>")] pub mac: MacAddr6, } +struct ZeroPadded; + +impl<'de> DeserializeAs<'de, MacAddr6> for ZeroPadded { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + <&str>::deserialize(deserializer)? + .split(':') + .map(|segment| format!("{:0>2}", segment)) + .collect::>() + .join(":") + .parse() + .map_err(serde::de::Error::custom) + } +} + /// Configuration information for launching a Sled Agent. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct SledAgentRequest { From e8a09f9322b5109abf708ac5e97da441161a283c Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Fri, 19 Aug 2022 22:51:30 +0000 Subject: [PATCH 5/9] use omicron_test_utils::dev::poll --- Cargo.lock | 1 + end-to-end-tests/Cargo.toml | 1 + end-to-end-tests/src/bin/bootstrap.rs | 28 +++++++++++++------------ end-to-end-tests/src/helpers/mod.rs | 28 +------------------------ end-to-end-tests/src/instance_launch.rs | 23 +++++++++++--------- 5 files changed, 31 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 32dd5e6632d..f69daaf4fdf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1553,6 +1553,7 @@ dependencies = [ "base64", "futures", "omicron-sled-agent", + "omicron-test-utils", "oxide-client", "rand 0.8.5", "reqwest", diff --git a/end-to-end-tests/Cargo.toml b/end-to-end-tests/Cargo.toml index a012b85fd27..94271c31fe4 100644 --- a/end-to-end-tests/Cargo.toml +++ b/end-to-end-tests/Cargo.toml @@ -9,6 +9,7 @@ anyhow = { version = "1.0.58", features = ["backtrace"] } base64 = "0.13.0" futures = "0.3.21" omicron-sled-agent = { path = "../sled-agent" } +omicron-test-utils = { path = "../test-utils" } oxide-client = { path = "../oxide-client" } rand = "0.8.5" reqwest = { version = "0.11.11", default-features = false } diff --git a/end-to-end-tests/src/bin/bootstrap.rs b/end-to-end-tests/src/bin/bootstrap.rs index 75eea0c7175..126207230e4 100644 --- a/end-to-end-tests/src/bin/bootstrap.rs +++ b/end-to-end-tests/src/bin/bootstrap.rs @@ -1,13 +1,13 @@ use anyhow::{bail, Result}; use end_to_end_tests::helpers::ctx::{build_client, nexus_addr, Context}; -use end_to_end_tests::helpers::{generate_name, try_loop}; +use end_to_end_tests::helpers::generate_name; +use omicron_test_utils::dev::poll::{wait_for_condition, CondCheckError}; use oxide_client::types::{ ByteCount, DiskCreate, DiskSource, IpPoolCreate, IpRange, Ipv4Range, }; use oxide_client::{ClientDisksExt, ClientIpPoolsExt, ClientOrganizationsExt}; use std::net::IpAddr; use std::time::Duration; -use tokio::time::sleep; #[tokio::main] async fn main() -> Result<()> { @@ -15,12 +15,16 @@ async fn main() -> Result<()> { // ===== ENSURE NEXUS IS UP ===== // eprintln!("waiting for nexus to come up..."); - try_loop( + wait_for_condition( || async { - sleep(Duration::from_secs(1)).await; - client.organization_list().send().await + client + .organization_list() + .send() + .await + .map_err(|_| CondCheckError::::NotYet) }, - Duration::from_secs(300), + &Duration::from_secs(1), + &Duration::from_secs(300), ) .await?; @@ -60,9 +64,8 @@ async fn main() -> Result<()> { eprintln!("ensuring datasets are ready..."); let ctx = Context::from_client(client).await?; let disk_name = generate_name("disk")?; - try_loop( + wait_for_condition( || async { - sleep(Duration::from_secs(1)).await; ctx.client .disk_create() .organization_name(ctx.org_name.clone()) @@ -71,17 +74,16 @@ async fn main() -> Result<()> { name: disk_name.clone(), description: String::new(), disk_source: DiskSource::Blank { - block_size: 512 - .try_into() - .map_err(anyhow::Error::msg)?, + block_size: 512.try_into().unwrap(), }, size: ByteCount(1024 * 1024 * 1024), }) .send() .await - .map_err(anyhow::Error::from) + .map_err(|_| CondCheckError::::NotYet) }, - Duration::from_secs(120), + &Duration::from_secs(1), + &Duration::from_secs(120), ) .await?; ctx.client diff --git a/end-to-end-tests/src/helpers/mod.rs b/end-to-end-tests/src/helpers/mod.rs index 1ca2eb57d0e..d042144da63 100644 --- a/end-to-end-tests/src/helpers/mod.rs +++ b/end-to-end-tests/src/helpers/mod.rs @@ -1,37 +1,11 @@ pub mod ctx; -use anyhow::{Context, Result}; +use anyhow::Result; use oxide_client::types::Name; use rand::{thread_rng, Rng}; -use std::future::Future; -use std::time::{Duration, Instant}; pub fn generate_name(prefix: &str) -> Result { format!("{}-{:x}", prefix, thread_rng().gen_range(0..0xfff_ffff_ffffu64)) .try_into() .map_err(anyhow::Error::msg) } - -pub async fn try_loop(mut f: F, timeout: Duration) -> Result -where - F: FnMut() -> Fut, - Fut: Future>, - Result: Context, -{ - let start = Instant::now(); - loop { - match f().await { - Ok(t) => return Ok(t), - Err(err) => { - if Instant::now() - start > timeout { - return Err(err).with_context(|| { - format!( - "try_loop timed out after {} seconds", - timeout.as_secs_f64() - ) - }); - } - } - } - } -} diff --git a/end-to-end-tests/src/instance_launch.rs b/end-to-end-tests/src/instance_launch.rs index aa0478bc279..6dfd8a7a986 100644 --- a/end-to-end-tests/src/instance_launch.rs +++ b/end-to-end-tests/src/instance_launch.rs @@ -1,8 +1,9 @@ #![cfg(test)] -use crate::helpers::{ctx::Context, generate_name, try_loop}; +use crate::helpers::{ctx::Context, generate_name}; use anyhow::{ensure, Context as _, Result}; use futures::future::Ready; +use omicron_test_utils::dev::poll::{wait_for_condition, CondCheckError}; use oxide_client::types::{ ByteCount, DiskCreate, DiskSource, Distribution, ExternalIpCreate, GlobalImageCreate, ImageSource, InstanceCpuCount, InstanceCreate, @@ -113,9 +114,11 @@ async fn instance_launch() -> Result<()> { // poll serial for login prompt, waiting 5 min max // (pulling disk blocks over HTTP is slow) eprintln!("waiting for serial console"); - let serial = try_loop( + let serial = wait_for_condition( || async { - sleep(Duration::from_secs(5)).await; + type Error = + CondCheckError>; + let data = String::from_utf8_lossy( &ctx.client .instance_serial_console() @@ -129,14 +132,14 @@ async fn instance_launch() -> Result<()> { .data, ) .into_owned(); - ensure!( - data.contains("localshark login:"), - "not yet booted\n{}", - data - ); - Ok(data) + if data.contains("localshark login:") { + Ok(data) + } else { + Err(Error::NotYet) + } }, - Duration::from_secs(300), + &Duration::from_secs(5), + &Duration::from_secs(300), ) .await?; From 433e30cdca5401f7fe5198fdcb03cf88369863c2 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Fri, 21 Oct 2022 00:26:58 +0000 Subject: [PATCH 6/9] changes after rebase --- end-to-end-tests/src/bin/bootstrap.rs | 2 +- end-to-end-tests/src/helpers/ctx.rs | 11 ++++++----- end-to-end-tests/src/instance_launch.rs | 7 ++++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/end-to-end-tests/src/bin/bootstrap.rs b/end-to-end-tests/src/bin/bootstrap.rs index 126207230e4..08f1ac14050 100644 --- a/end-to-end-tests/src/bin/bootstrap.rs +++ b/end-to-end-tests/src/bin/bootstrap.rs @@ -5,7 +5,7 @@ use omicron_test_utils::dev::poll::{wait_for_condition, CondCheckError}; use oxide_client::types::{ ByteCount, DiskCreate, DiskSource, IpPoolCreate, IpRange, Ipv4Range, }; -use oxide_client::{ClientDisksExt, ClientIpPoolsExt, ClientOrganizationsExt}; +use oxide_client::{ClientDisksExt, ClientOrganizationsExt, ClientSystemExt}; use std::net::IpAddr; use std::time::Duration; diff --git a/end-to-end-tests/src/helpers/ctx.rs b/end-to-end-tests/src/helpers/ctx.rs index eb9d9dff2f0..2708f7d5e2b 100644 --- a/end-to-end-tests/src/helpers/ctx.rs +++ b/end-to-end-tests/src/helpers/ctx.rs @@ -101,11 +101,12 @@ pub fn nexus_addr() -> SocketAddr { if rss_config_path.exists() { if let Ok(config) = SetupServiceConfig::from_file(rss_config_path) { for request in config.requests { - for service in request.services { - if let ServiceType::Nexus { external_ip, .. } = - service.service_type - { - return (external_ip, 80).into(); + for zone in request.service_zones { + for service in zone.services { + if let ServiceType::Nexus { external_ip, .. } = service + { + return (external_ip, 80).into(); + } } } } diff --git a/end-to-end-tests/src/instance_launch.rs b/end-to-end-tests/src/instance_launch.rs index 6dfd8a7a986..42eab8070f5 100644 --- a/end-to-end-tests/src/instance_launch.rs +++ b/end-to-end-tests/src/instance_launch.rs @@ -10,7 +10,7 @@ use oxide_client::types::{ InstanceDiskAttachment, InstanceNetworkInterfaceAttachment, SshKeyCreate, }; use oxide_client::{ - ClientDisksExt, ClientImagesGlobalExt, ClientInstancesExt, ClientSessionExt, + ClientDisksExt, ClientInstancesExt, ClientSessionExt, ClientSystemExt, }; use std::sync::Arc; use std::time::Duration; @@ -38,10 +38,10 @@ async fn instance_launch() -> Result<()> { .send() .await?; - eprintln!("create global image"); + eprintln!("create system image"); let image_id = ctx .client - .image_global_create() + .system_image_create() .body(GlobalImageCreate { name: generate_name("debian")?, description: String::new(), @@ -93,6 +93,7 @@ async fn instance_launch() -> Result<()> { network_interfaces: InstanceNetworkInterfaceAttachment::Default, external_ips: vec![ExternalIpCreate::Ephemeral { pool_name: None }], user_data: String::new(), + start: true, }) .send() .await?; From 60ef0186d934fbf3bd90585e2963fde5f48e9301 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Fri, 21 Oct 2022 03:58:45 +0000 Subject: [PATCH 7/9] further rebase fixes --- .github/buildomat/jobs/deploy.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index 2e769737c86..07b48ba9a4b 100644 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -146,11 +146,11 @@ pfexec ipadm create-addr -T static -a 192.168.1.199/24 igb0/sidehatch # Modify config-rss.toml in the sled-agent zone to use our system's IP and MAC # address for upstream connectivity. # -tar xf out/sled-agent.tar pkg/config-rss.toml +tar xf out/omicron-sled-agent.tar pkg/config-rss.toml sed -e '/\[gateway\]/,/\[request\]/ s/^.*address =.*$/address = "192.168.1.199"/' \ -e "s/^mac =.*$/mac = \"$(dladm show-phys -m -p -o ADDRESS | head -n 1)\"/" \ -i pkg/config-rss.toml -tar rf out/sled-agent.tar pkg/config-rss.toml +tar rf out/omicron-sled-agent.tar pkg/config-rss.toml rm -rf pkg # From 484d14bbf78626d8f901602185c5208b758614af Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 27 Oct 2022 11:09:16 -0700 Subject: [PATCH 8/9] remove darkhttpd from docs as it does not work --- end-to-end-tests/README.adoc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/end-to-end-tests/README.adoc b/end-to-end-tests/README.adoc index 1e37b01b21f..c6678699781 100644 --- a/end-to-end-tests/README.adoc +++ b/end-to-end-tests/README.adoc @@ -7,9 +7,7 @@ This package is not built or run by default (it is excluded from `default-member == Running these tests on your machine 1. xref:../docs/how-to-run.adoc[Make yourself a Gimlet]. -2. Serve a Debian image over HTTP. The tests add a Debian image sourced from `http://[fd00:1122:3344:101::1]:54321/debian-11-genericcloud-amd64.raw`; that IP address belongs to the global zone. You can download the image from https://cloud.debian.org/images/cloud/bullseye/latest/. -+ -Some tools for running an HTTP server with `Range` support are: https://github.com/emikulic/darkhttpd[darkhttpd], https://github.com/joseluisq/static-web-server[static-web-server]. +2. Serve a Debian image over HTTP. The tests add a Debian image sourced from `http://[fd00:1122:3344:101::1]:54321/debian-11-genericcloud-amd64.raw`; that IP address belongs to the global zone. You can download the image from https://cloud.debian.org/images/cloud/bullseye/latest/. https://github.com/joseluisq/static-web-server[static-web-server] works well for this. 3. Run the bootstrap bin target: `cargo run -p end-to-end-tests --bin bootstrap` Then you can `cargo test -p end-to-end-tests`. From 30d39dff7d7c0668b4b063cc71d993732935b628 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Fri, 28 Oct 2022 14:10:07 -0700 Subject: [PATCH 9/9] review nits --- Cargo.toml | 2 +- end-to-end-tests/src/bin/bootstrap.rs | 19 ++++--------------- end-to-end-tests/src/helpers/mod.rs | 21 ++++++++++++++++++++- end-to-end-tests/src/instance_launch.rs | 20 +++++++++++++++++++- 4 files changed, 44 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 499ebc7c15e..bcd1be2366e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -120,7 +120,7 @@ branch = "oxide/omicron" # # libsodium-sys is a dependency of thrussh, used in `end-to-end-tests`. We # maintain a fork of the sodiumoxide repository to address an illumos build -# issue. See the README.oxide.md in the "oxide" branch of our fork for +# issue. See the README.oxide.md in the "omicron/oxide" branch of our fork for # details. # [patch.crates-io.libsodium-sys] diff --git a/end-to-end-tests/src/bin/bootstrap.rs b/end-to-end-tests/src/bin/bootstrap.rs index 08f1ac14050..edbf9ac550c 100644 --- a/end-to-end-tests/src/bin/bootstrap.rs +++ b/end-to-end-tests/src/bin/bootstrap.rs @@ -1,12 +1,11 @@ -use anyhow::{bail, Result}; -use end_to_end_tests::helpers::ctx::{build_client, nexus_addr, Context}; -use end_to_end_tests::helpers::generate_name; +use anyhow::Result; +use end_to_end_tests::helpers::ctx::{build_client, Context}; +use end_to_end_tests::helpers::{generate_name, get_system_ip_pool}; use omicron_test_utils::dev::poll::{wait_for_condition, CondCheckError}; use oxide_client::types::{ ByteCount, DiskCreate, DiskSource, IpPoolCreate, IpRange, Ipv4Range, }; use oxide_client::{ClientDisksExt, ClientOrganizationsExt, ClientSystemExt}; -use std::net::IpAddr; use std::time::Duration; #[tokio::main] @@ -30,17 +29,7 @@ async fn main() -> Result<()> { // ===== CREATE IP POOL ===== // eprintln!("creating IP pool..."); - let nexus_addr = match nexus_addr().ip() { - IpAddr::V4(addr) => addr.octets(), - IpAddr::V6(_) => bail!("not sure what to do about IPv6 here"), - }; - // TODO: not really sure about a good heuristic for selecting an IP address - // range here. in both my (iliana's) environment and the lab, the last octet - // is 20; in my environment the DHCP range is 100-249, and in the buildomat - // lab environment the network is currently private. - let first = [nexus_addr[0], nexus_addr[1], nexus_addr[2], 50].into(); - let last = [nexus_addr[0], nexus_addr[1], nexus_addr[2], 90].into(); - + let (first, last) = get_system_ip_pool()?; let pool_name = client .ip_pool_create() .body(IpPoolCreate { diff --git a/end-to-end-tests/src/helpers/mod.rs b/end-to-end-tests/src/helpers/mod.rs index d042144da63..865ede37d19 100644 --- a/end-to-end-tests/src/helpers/mod.rs +++ b/end-to-end-tests/src/helpers/mod.rs @@ -1,11 +1,30 @@ pub mod ctx; -use anyhow::Result; +use self::ctx::nexus_addr; +use anyhow::{bail, Result}; use oxide_client::types::Name; use rand::{thread_rng, Rng}; +use std::net::{IpAddr, Ipv4Addr}; pub fn generate_name(prefix: &str) -> Result { format!("{}-{:x}", prefix, thread_rng().gen_range(0..0xfff_ffff_ffffu64)) .try_into() .map_err(anyhow::Error::msg) } + +/// HACK: we're picking a range that doesn't conflict with either iliana's or +/// the lab environment's IP ranges. This is not terribly robust. (in both +/// iliana's environment and the lab, the last octet is 20; in my environment +/// the DHCP range is 100-249, and in the buildomat lab environment the network +/// is currently private.) +pub fn get_system_ip_pool() -> Result<(Ipv4Addr, Ipv4Addr)> { + let nexus_addr = match nexus_addr().ip() { + IpAddr::V4(addr) => addr.octets(), + IpAddr::V6(_) => bail!("not sure what to do about IPv6 here"), + }; + + let first = [nexus_addr[0], nexus_addr[1], nexus_addr[2], 50].into(); + let last = [nexus_addr[0], nexus_addr[1], nexus_addr[2], 90].into(); + + Ok((first, last)) +} diff --git a/end-to-end-tests/src/instance_launch.rs b/end-to-end-tests/src/instance_launch.rs index 42eab8070f5..bbb5ff09473 100644 --- a/end-to-end-tests/src/instance_launch.rs +++ b/end-to-end-tests/src/instance_launch.rs @@ -219,7 +219,7 @@ async fn instance_launch() -> Result<()> { ); // tear-down - eprintln!("tear-down"); + eprintln!("stopping instance"); ctx.client .instance_stop() .organization_name(ctx.org_name.clone()) @@ -227,6 +227,24 @@ async fn instance_launch() -> Result<()> { .instance_name(instance.name.clone()) .send() .await?; + + eprintln!("deleting instance"); + wait_for_condition( + || async { + ctx.client + .instance_delete() + .organization_name(ctx.org_name.clone()) + .project_name(ctx.project_name.clone()) + .instance_name(instance.name.clone()) + .send() + .await + .map_err(|_| CondCheckError::::NotYet) + }, + &Duration::from_secs(1), + &Duration::from_secs(60), + ) + .await?; + ctx.cleanup().await }