From 082c009c197629cfe51de7a9ab1c6f5f2203a994 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 8 Dec 2021 14:01:24 -0500 Subject: [PATCH 01/50] [nexus] Refactor test-utilities to helper crate, add test benchmarks --- Cargo.lock | 233 +++++++++++++++--- nexus/Cargo.toml | 9 +- nexus/benches/setup_benchmark.rs | 59 +++++ nexus/test-utils/Cargo.toml | 26 ++ .../common => test-utils/src}/http_testing.rs | 0 .../common/mod.rs => test-utils/src/lib.rs} | 4 +- .../src}/resource_helpers.rs | 0 nexus/tests/test_authn_http.rs | 4 +- nexus/tests/test_authz.rs | 5 +- nexus/tests/test_basic.rs | 15 +- nexus/tests/test_console_api.rs | 5 +- nexus/tests/test_datasets.rs | 3 +- nexus/tests/test_disks.rs | 17 +- nexus/tests/test_instances.rs | 7 +- nexus/tests/test_organizations.rs | 9 +- nexus/tests/test_oximeter.rs | 24 +- nexus/tests/test_projects.rs | 5 +- nexus/tests/test_router_routes.rs | 5 +- nexus/tests/test_users_builtin.rs | 9 +- nexus/tests/test_vpc_firewall.rs | 5 +- nexus/tests/test_vpc_routers.rs | 7 +- nexus/tests/test_vpc_subnets.rs | 7 +- nexus/tests/test_vpcs.rs | 7 +- nexus/tests/test_zpools.rs | 3 +- 24 files changed, 354 insertions(+), 114 deletions(-) create mode 100644 nexus/benches/setup_benchmark.rs create mode 100644 nexus/test-utils/Cargo.toml rename nexus/{tests/common => test-utils/src}/http_testing.rs (100%) rename nexus/{tests/common/mod.rs => test-utils/src/lib.rs} (99%) rename nexus/{tests/common => test-utils/src}/resource_helpers.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 00c2e49d1df..29f3fe0f92e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -267,6 +267,18 @@ dependencies = [ "byte-tools", ] +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.8.0" @@ -294,6 +306,15 @@ dependencies = [ "serde", ] +[[package]] +name = "cast" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c24dab4283a142afa2fdca129b80ad2c6284e073930f964c3a1293c225ee39a" +dependencies = [ + "rustc_version 0.4.0", +] + [[package]] name = "cc" version = "1.0.72" @@ -397,6 +418,44 @@ dependencies = [ "libc", ] +[[package]] +name = "criterion" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1604dafd25fba2fe2d5895a9da139f8dc9b319a5fe5354ca137cbbce4e178d10" +dependencies = [ + "atty", + "cast", + "clap", + "criterion-plot", + "csv", + "futures", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "tokio", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d00996de9f2f7559f7f4dc286073197f83e92256a59ed395f9aac01fe717da57" +dependencies = [ + "cast", + "itertools", +] + [[package]] name = "crossbeam-channel" version = "0.5.1" @@ -444,7 +503,7 @@ dependencies = [ [[package]] name = "crucible" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?branch=main#cda0bf8b0fd8e53566d1918b4d14824103a1d410" +source = "git+https://github.com/oxidecomputer/crucible?branch=main#4f02540ab2557ad75e9f170d362dc889eefe1d1b" dependencies = [ "aes", "aes-gcm-siv", @@ -474,7 +533,7 @@ dependencies = [ [[package]] name = "crucible-common" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?branch=main#cda0bf8b0fd8e53566d1918b4d14824103a1d410" +source = "git+https://github.com/oxidecomputer/crucible?branch=main#4f02540ab2557ad75e9f170d362dc889eefe1d1b" dependencies = [ "anyhow", "serde", @@ -488,7 +547,7 @@ dependencies = [ [[package]] name = "crucible-protocol" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?branch=main#cda0bf8b0fd8e53566d1918b4d14824103a1d410" +source = "git+https://github.com/oxidecomputer/crucible?branch=main#4f02540ab2557ad75e9f170d362dc889eefe1d1b" dependencies = [ "anyhow", "bincode", @@ -502,7 +561,7 @@ dependencies = [ [[package]] name = "crucible-scope" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?branch=main#cda0bf8b0fd8e53566d1918b4d14824103a1d410" +source = "git+https://github.com/oxidecomputer/crucible?branch=main#4f02540ab2557ad75e9f170d362dc889eefe1d1b" dependencies = [ "anyhow", "futures", @@ -542,6 +601,28 @@ dependencies = [ "subtle", ] +[[package]] +name = "csv" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +dependencies = [ + "bstr", + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + [[package]] name = "ctr" version = "0.8.0" @@ -566,9 +647,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "757c0ded2af11d8e739c4daea1ac623dd1624b06c844cf3f5a39f1bdbd99bb12" +checksum = "d0d720b8683f8dd83c65155f0530560cba68cd2bf395f6513a483caee57ff7f4" dependencies = [ "darling_core", "darling_macro", @@ -576,9 +657,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c34d8efb62d0c2d7f60ece80f75e5c63c1588ba68032740494b0b9a996466e3" +checksum = "7a340f241d2ceed1deb47ae36c4144b2707ec7dd0b649f894cb39bb595986324" dependencies = [ "fnv", "ident_case", @@ -590,9 +671,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade7bff147130fe5e6d39f089c6bd49ec0250f35d70b2eebf72afdfc919f15cc" +checksum = "72c41b3b7352feb3211a0d743dc5700a4e3b60f51bd2b368892d1e0f9a95f44b" dependencies = [ "darling_core", "quote", @@ -610,9 +691,9 @@ dependencies = [ [[package]] name = "der" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28e98c534e9c8a0483aa01d6f6913bc063de254311bd267c9cf535e9b70e15b2" +checksum = "79b71cca7d95d7681a4b3b9cdf63c8dbc3730d0584c2c74e31416d64a90493f4" dependencies = [ "const-oid", ] @@ -741,7 +822,7 @@ checksum = "4bb454f0228b18c7f4c3b0ebbee346ed9c52e7443b0999cd543ff3571205701d" [[package]] name = "dropshot" version = "0.6.1-dev" -source = "git+https://github.com/oxidecomputer/dropshot?branch=main#ff33033ad3d7fcf32caa352d76fc243bd20db176" +source = "git+https://github.com/oxidecomputer/dropshot?branch=main#56f60da71be392de46dd576e9564ac1b5428dd5c" dependencies = [ "async-trait", "base64", @@ -776,7 +857,7 @@ dependencies = [ [[package]] name = "dropshot_endpoint" version = "0.6.1-dev" -source = "git+https://github.com/oxidecomputer/dropshot?branch=main#ff33033ad3d7fcf32caa352d76fc243bd20db176" +source = "git+https://github.com/oxidecomputer/dropshot?branch=main#56f60da71be392de46dd576e9564ac1b5428dd5c" dependencies = [ "proc-macro2", "quote", @@ -853,9 +934,9 @@ dependencies = [ [[package]] name = "encoding_rs" -version = "0.8.29" +version = "0.8.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a74ea89a0a1b98f6332de42c95baff457ada66d1cb4030f9ff151b2041a1c746" +checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df" dependencies = [ "cfg-if", ] @@ -1376,9 +1457,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69ddb889f9d0d08a67338271fa9b62996bc788c7796a5c18cf057420aaed5eaf" +checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" dependencies = [ "either", ] @@ -1520,9 +1601,9 @@ checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" [[package]] name = "memoffset" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" dependencies = [ "autocfg", ] @@ -1647,6 +1728,31 @@ dependencies = [ "uuid", ] +[[package]] +name = "nexus-test-utils" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "chrono", + "dropshot", + "http", + "hyper", + "omicron-common", + "omicron-nexus", + "omicron-sled-agent", + "omicron-test-utils", + "oximeter", + "oximeter-client", + "oximeter-collector", + "oximeter-producer", + "parse-display", + "serde", + "serde_json", + "slog", + "uuid", +] + [[package]] name = "normalize-line-endings" version = "0.3.0" @@ -1757,9 +1863,9 @@ dependencies = [ "async-bb8-diesel", "async-trait", "bb8", - "bytes", "chrono", "cookie", + "criterion", "db-macros", "diesel", "dropshot", @@ -1774,6 +1880,7 @@ dependencies = [ "macaddr", "mime_guess", "newtype_derive", + "nexus-test-utils", "omicron-common", "omicron-rpaths", "omicron-sled-agent", @@ -1783,7 +1890,6 @@ dependencies = [ "oso", "oximeter", "oximeter-client", - "oximeter-collector", "oximeter-db", "oximeter-instruments", "oximeter-producer", @@ -1909,6 +2015,12 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "opaque-debug" version = "0.2.3" @@ -2340,9 +2452,37 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.22" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1a3ea4f0dd7f1f3e512cf97bf100819aa547f36a6eccac8dbaae839eb92363e" + +[[package]] +name = "plotters" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a3fd9ec30b9749ce28cd91f255d569591cdf937fe280c312143e3c4bad6f2a" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12295df4f294471248581bc09bef3c38a5e46f1e36d6a37353621a0c6c357e1f" +checksum = "d88417318da0eaf0fdcdb51a0ee6c3bed624333bff8f946733049380be67ac1c" + +[[package]] +name = "plotters-svg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521fa9638fa597e1dc53e9412a4f9cefb01187ee1f7413076f9e6749e2885ba9" +dependencies = [ + "plotters-backend", +] [[package]] name = "polar-core" @@ -2755,6 +2895,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" + [[package]] name = "regex-syntax" version = "0.6.25" @@ -2852,6 +2998,15 @@ dependencies = [ "semver 0.9.0", ] +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver 1.0.4", +] + [[package]] name = "rustfmt-wrapper" version = "0.1.0" @@ -2886,9 +3041,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b3909d758bb75c79f23d4736fac9433868679d3ad2ea7a61e3c25cfda9a088" +checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" [[package]] name = "ryu" @@ -3014,6 +3169,12 @@ dependencies = [ "semver-parser 0.10.2", ] +[[package]] +name = "semver" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012" + [[package]] name = "semver-parser" version = "0.7.0" @@ -3235,9 +3396,9 @@ dependencies = [ [[package]] name = "signal-hook" -version = "0.3.10" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c98891d737e271a2954825ef19e46bd16bdb98e2746f2eec4f7a4ef7946efd1" +checksum = "c35dfd12afb7828318348b8c408383cf5071a086c1d4ab1c0f9840ec92dbb922" dependencies = [ "libc", "signal-hook-registry", @@ -3777,6 +3938,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.5.1" @@ -4015,7 +4186,7 @@ checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" [[package]] name = "typify" version = "0.0.6-dev" -source = "git+https://github.com/oxidecomputer/typify#5132e748f91311aadd56011b97f51c4f32373985" +source = "git+https://github.com/oxidecomputer/typify#80b510b02b1db22de463efcf6e7762243bcea67a" dependencies = [ "typify-impl", "typify-macro", @@ -4024,7 +4195,7 @@ dependencies = [ [[package]] name = "typify-impl" version = "0.0.6-dev" -source = "git+https://github.com/oxidecomputer/typify#5132e748f91311aadd56011b97f51c4f32373985" +source = "git+https://github.com/oxidecomputer/typify#80b510b02b1db22de463efcf6e7762243bcea67a" dependencies = [ "convert_case", "proc-macro2", @@ -4039,7 +4210,7 @@ dependencies = [ [[package]] name = "typify-macro" version = "0.0.6-dev" -source = "git+https://github.com/oxidecomputer/typify#5132e748f91311aadd56011b97f51c4f32373985" +source = "git+https://github.com/oxidecomputer/typify#80b510b02b1db22de463efcf6e7762243bcea67a" dependencies = [ "proc-macro2", "quote", diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index d6db89b61f8..57720d9daf7 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -100,8 +100,9 @@ version = "0.8" features = [ "serde", "v4" ] [dev-dependencies] -bytes = "1.0.1" +criterion = { version = "0.3", features = [ "async_tokio" ] } expectorate = "1.0.4" +nexus-test-utils = { path = "test-utils" } omicron-test-utils = { path = "../test-utils" } openapiv3 = "0.5.0" oximeter-db = { path = "../oximeter/db" } @@ -111,9 +112,9 @@ subprocess = "0.2.8" git = "https://github.com/oxidecomputer/openapi-lint" branch = "main" -[dev-dependencies.oximeter-collector] -version = "0.1.0" -path = "../oximeter/collector" +[[bench]] +name = "setup_benchmark" +harness = false # # Disable doc builds by default for our binaries to work around issue diff --git a/nexus/benches/setup_benchmark.rs b/nexus/benches/setup_benchmark.rs new file mode 100644 index 00000000000..738c2733109 --- /dev/null +++ b/nexus/benches/setup_benchmark.rs @@ -0,0 +1,59 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Benchmarks test setup/teardown. + +use criterion::{criterion_group, criterion_main, Criterion}; +use dropshot::test_util::LogContext; +use omicron_test_utils::dev; + +// This is the default wrapper around most Nexus integration tests. +// Benchmark how long an "empty test" would take. +async fn do_full_setup() { + let ctx = nexus_test_utils::test_setup("full_setup").await; + ctx.teardown().await; +} + +// Wraps exclusively the CockroachDB portion of setup/teardown. +async fn do_crdb_setup() { + let cfg = nexus_test_utils::load_test_config(); + let logctx = LogContext::new("crdb_setup", &cfg.log); + let mut db = dev::test_setup_database(&logctx.log).await; + db.cleanup().await.unwrap(); +} + +// Wraps exclusively the ClickhouseDB portion of setup/teardown. +async fn do_clickhouse_setup() { + let mut clickhouse = + dev::clickhouse::ClickHouseInstance::new(0).await.unwrap(); + clickhouse.cleanup().await.unwrap(); +} + +fn setup_benchmark(c: &mut Criterion) { + let mut group = c.benchmark_group("Test Setup"); + group.bench_function("do_full_setup", |b| { + b.to_async(tokio::runtime::Runtime::new().unwrap()) + .iter(|| do_full_setup()); + }); + group.bench_function("do_crdb_setup", |b| { + b.to_async(tokio::runtime::Runtime::new().unwrap()) + .iter(|| do_crdb_setup()); + }); + group.bench_function("do_clickhouse_setup", |b| { + b.to_async(tokio::runtime::Runtime::new().unwrap()) + .iter(|| do_clickhouse_setup()); + }); + group.finish(); +} + +criterion_group!( + name = benches; + // To accomodate the fact that these benchmarks are a bit bulky, + // we set the following: + // - Smaller sample size, to keep running time down + // - Higher noise threshold, to avoid avoid false positive change detection + config = Criterion::default().sample_size(10).noise_threshold(0.10); + targets = setup_benchmark +); +criterion_main!(benches); diff --git a/nexus/test-utils/Cargo.toml b/nexus/test-utils/Cargo.toml new file mode 100644 index 00000000000..5857bca9ab3 --- /dev/null +++ b/nexus/test-utils/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "nexus-test-utils" +version = "0.1.0" +edition = "2018" +license = "MPL-2.0" + +[dependencies] +anyhow = "1.0" +bytes = "1.0.1" +chrono = { version = "0.4", features = [ "serde" ] } +dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main" } +http = "0.2.5" +hyper = "0.14" +omicron-common = { path = "../../common" } +omicron-nexus = { path = ".." } +omicron-sled-agent = { path = "../../sled-agent" } +omicron-test-utils = { path = "../../test-utils" } +oximeter = { version = "0.1.0", path = "../../oximeter/oximeter" } +oximeter-client = { path = "../../oximeter-client" } +oximeter-collector = { version = "0.1.0", path = "../../oximeter/collector" } +oximeter-producer = { version = "0.1.0", path = "../../oximeter/producer" } +parse-display = "0.5.3" +serde = { version = "1.0", features = [ "derive" ] } +serde_json = "1.0" +slog = { version = "2.7", features = [ "max_level_trace", "release_max_level_debug" ] } +uuid = { version = "0.8", features = [ "serde", "v4" ] } diff --git a/nexus/tests/common/http_testing.rs b/nexus/test-utils/src/http_testing.rs similarity index 100% rename from nexus/tests/common/http_testing.rs rename to nexus/test-utils/src/http_testing.rs diff --git a/nexus/tests/common/mod.rs b/nexus/test-utils/src/lib.rs similarity index 99% rename from nexus/tests/common/mod.rs rename to nexus/test-utils/src/lib.rs index d47ca194652..46eff37f8a8 100644 --- a/nexus/tests/common/mod.rs +++ b/nexus/test-utils/src/lib.rs @@ -2,9 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -/*! - * Shared integration testing facilities - */ +//! Integration testing facilities for Nexus use dropshot::test_util::ClientTestContext; use dropshot::test_util::LogContext; diff --git a/nexus/tests/common/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs similarity index 100% rename from nexus/tests/common/resource_helpers.rs rename to nexus/test-utils/src/resource_helpers.rs diff --git a/nexus/tests/test_authn_http.rs b/nexus/tests/test_authn_http.rs index 07adc70268e..a68d6040d5a 100644 --- a/nexus/tests/test_authn_http.rs +++ b/nexus/tests/test_authn_http.rs @@ -8,8 +8,6 @@ // a lower-level interface, since our hyper Client will not allow us to send // such invalid requests. -pub mod common; - use async_trait::async_trait; use chrono::{DateTime, Duration, Utc}; use dropshot::endpoint; @@ -270,7 +268,7 @@ async fn start_whoami_server( authn_schemes_configured: Vec>>, sessions: HashMap, ) -> TestContext { - let config = common::load_test_config(); + let config = nexus_test_utils::load_test_config(); let logctx = LogContext::new(test_name, &config.log); let whoami_api = { diff --git a/nexus/tests/test_authz.rs b/nexus/tests/test_authz.rs index 928e83b9be5..adbad909f46 100644 --- a/nexus/tests/test_authz.rs +++ b/nexus/tests/test_authz.rs @@ -3,13 +3,12 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. //! Basic end-to-end tests for authorization -use common::http_testing::RequestBuilder; use dropshot::HttpErrorResponseBody; +use nexus_test_utils::http_testing::RequestBuilder; -pub mod common; -use common::test_setup; use http::method::Method; use http::StatusCode; +use nexus_test_utils::test_setup; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_nexus::authn::external::spoof::HTTP_HEADER_OXIDE_AUTHN_SPOOF; use omicron_nexus::external_api::params; diff --git a/nexus/tests/test_basic.rs b/nexus/tests/test_basic.rs index b71a8533307..8f779af533c 100644 --- a/nexus/tests/test_basic.rs +++ b/nexus/tests/test_basic.rs @@ -27,14 +27,13 @@ use omicron_nexus::external_api::{ use serde::Serialize; use uuid::Uuid; -pub mod common; -use common::http_testing::AuthnMode; -use common::http_testing::NexusRequest; -use common::http_testing::RequestBuilder; -use common::resource_helpers::create_organization; -use common::resource_helpers::create_project; -use common::start_sled_agent; -use common::test_setup; +use nexus_test_utils::http_testing::AuthnMode; +use nexus_test_utils::http_testing::NexusRequest; +use nexus_test_utils::http_testing::RequestBuilder; +use nexus_test_utils::resource_helpers::create_organization; +use nexus_test_utils::resource_helpers::create_project; +use nexus_test_utils::start_sled_agent; +use nexus_test_utils::test_setup; #[macro_use] extern crate slog; diff --git a/nexus/tests/test_console_api.rs b/nexus/tests/test_console_api.rs index 0a66c7aa5fe..ab6ba30101c 100644 --- a/nexus/tests/test_console_api.rs +++ b/nexus/tests/test_console_api.rs @@ -7,9 +7,8 @@ use http::header::HeaderName; use http::{header, method::Method, StatusCode}; use std::env::current_dir; -pub mod common; -use common::http_testing::{RequestBuilder, TestResponse}; -use common::{load_test_config, test_setup, test_setup_with_config}; +use nexus_test_utils::http_testing::{RequestBuilder, TestResponse}; +use nexus_test_utils::{load_test_config, test_setup, test_setup_with_config}; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_nexus::external_api::console_api::LoginParams; use omicron_nexus::external_api::params::OrganizationCreate; diff --git a/nexus/tests/test_datasets.rs b/nexus/tests/test_datasets.rs index 9d61323a10a..f25e129c2fd 100644 --- a/nexus/tests/test_datasets.rs +++ b/nexus/tests/test_datasets.rs @@ -11,8 +11,7 @@ use omicron_nexus::internal_api::params::{ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use uuid::Uuid; -pub mod common; -use common::{test_setup, SLED_AGENT_UUID}; +use nexus_test_utils::{test_setup, SLED_AGENT_UUID}; extern crate slog; diff --git a/nexus/tests/test_disks.rs b/nexus/tests/test_disks.rs index e3067644a4e..9bac547b227 100644 --- a/nexus/tests/test_disks.rs +++ b/nexus/tests/test_disks.rs @@ -27,14 +27,13 @@ use dropshot::test_util::objects_post; use dropshot::test_util::read_json; use dropshot::test_util::ClientTestContext; -pub mod common; -use common::http_testing::AuthnMode; -use common::http_testing::NexusRequest; -use common::http_testing::RequestBuilder; -use common::identity_eq; -use common::resource_helpers::create_organization; -use common::resource_helpers::create_project; -use common::test_setup; +use nexus_test_utils::http_testing::AuthnMode; +use nexus_test_utils::http_testing::NexusRequest; +use nexus_test_utils::http_testing::RequestBuilder; +use nexus_test_utils::identity_eq; +use nexus_test_utils::resource_helpers::create_organization; +use nexus_test_utils::resource_helpers::create_project; +use nexus_test_utils::test_setup; /* * TODO-cleanup the mess of URLs used here and in test_instances.rs ought to @@ -88,7 +87,9 @@ async fn test_disks() { snapshot_id: None, size: ByteCount::from_gibibytes_u32(1), }; + println!("POSTING A DISK"); let disk: Disk = objects_post(&client, &url_disks, new_disk.clone()).await; + println!("POSTED A DISK"); assert_eq!(disk.identity.name, "just-rainsticks"); assert_eq!(disk.identity.description, "sells rainsticks"); assert_eq!(disk.project_id, project.identity.id); diff --git a/nexus/tests/test_instances.rs b/nexus/tests/test_instances.rs index 305dc013846..0a6725871f8 100644 --- a/nexus/tests/test_instances.rs +++ b/nexus/tests/test_instances.rs @@ -26,10 +26,9 @@ use dropshot::test_util::objects_post; use dropshot::test_util::read_json; use dropshot::test_util::ClientTestContext; -pub mod common; -use common::identity_eq; -use common::resource_helpers::{create_organization, create_project}; -use common::test_setup; +use nexus_test_utils::identity_eq; +use nexus_test_utils::resource_helpers::{create_organization, create_project}; +use nexus_test_utils::test_setup; static ORGANIZATION_NAME: &str = "test-org"; static PROJECT_NAME: &str = "springfield-squidport"; diff --git a/nexus/tests/test_organizations.rs b/nexus/tests/test_organizations.rs index 96cbbeb518a..396e82ea9d7 100644 --- a/nexus/tests/test_organizations.rs +++ b/nexus/tests/test_organizations.rs @@ -6,13 +6,12 @@ use omicron_nexus::external_api::views::Organization; use dropshot::test_util::{object_delete, object_get}; -pub mod common; -use common::resource_helpers::{ - create_organization, create_project, objects_list_page_authz, -}; -use common::test_setup; use http::method::Method; use http::StatusCode; +use nexus_test_utils::resource_helpers::{ + create_organization, create_project, objects_list_page_authz, +}; +use nexus_test_utils::test_setup; extern crate slog; diff --git a/nexus/tests/test_oximeter.rs b/nexus/tests/test_oximeter.rs index 17b65e6d09c..fdaef0d8e0b 100644 --- a/nexus/tests/test_oximeter.rs +++ b/nexus/tests/test_oximeter.rs @@ -4,8 +4,6 @@ //! Integration tests for oximeter collectors and producers. -pub mod common; - use omicron_test_utils::dev::poll::{wait_for_condition, CondCheckError}; use oximeter_db::DbWrite; use std::net; @@ -14,7 +12,8 @@ use uuid::Uuid; #[tokio::test] async fn test_oximeter_database_records() { - let context = common::test_setup("test_oximeter_database_records").await; + let context = + nexus_test_utils::test_setup("test_oximeter_database_records").await; let db = &context.database; // Get a handle to the DB, for various tests @@ -33,7 +32,7 @@ async fn test_oximeter_database_records() { let actual_id = result[0].get::<&str, Uuid>("id"); assert_eq!( actual_id, - common::OXIMETER_UUID.parse().unwrap(), + nexus_test_utils::OXIMETER_UUID.parse().unwrap(), "Oximeter ID does not match the ID returned from the database" ); @@ -50,13 +49,13 @@ async fn test_oximeter_database_records() { let actual_id = result[0].get::<&str, Uuid>("id"); assert_eq!( actual_id, - common::PRODUCER_UUID.parse().unwrap(), + nexus_test_utils::PRODUCER_UUID.parse().unwrap(), "Producer ID does not match the ID returned from the database" ); let actual_oximeter_id = result[0].get::<&str, Uuid>("oximeter_id"); assert_eq!( actual_oximeter_id, - common::OXIMETER_UUID.parse().unwrap(), + nexus_test_utils::OXIMETER_UUID.parse().unwrap(), "Producer's oximeter ID returned from the database does not match the expected ID" ); @@ -65,10 +64,11 @@ async fn test_oximeter_database_records() { #[tokio::test] async fn test_oximeter_reregistration() { - let mut context = common::test_setup("test_oximeter_reregistration").await; + let mut context = + nexus_test_utils::test_setup("test_oximeter_reregistration").await; let db = &context.database; - let producer_id = common::PRODUCER_UUID.parse().unwrap(); - let oximeter_id = common::OXIMETER_UUID.parse().unwrap(); + let producer_id = nexus_test_utils::PRODUCER_UUID.parse().unwrap(); + let oximeter_id = nexus_test_utils::OXIMETER_UUID.parse().unwrap(); // Get a handle to the DB, for various tests let conn = db.connect().await.unwrap(); @@ -157,9 +157,9 @@ async fn test_oximeter_reregistration() { // Restart the producer, and verify that we have _more_ data than before // Set up a test metric producer server - context.producer = common::start_producer_server( + context.producer = nexus_test_utils::start_producer_server( context.server.http_server_internal.local_addr(), - common::PRODUCER_UUID.parse().unwrap(), + nexus_test_utils::PRODUCER_UUID.parse().unwrap(), ) .await .expect("Failed to restart metric producer server"); @@ -229,7 +229,7 @@ async fn test_oximeter_reregistration() { let timeseries = new_timeseries; // Restart oximeter again, and verify that we have even more new data. - context.oximeter = common::start_oximeter( + context.oximeter = nexus_test_utils::start_oximeter( context.server.http_server_internal.local_addr(), context.clickhouse.port(), oximeter_id, diff --git a/nexus/tests/test_projects.rs b/nexus/tests/test_projects.rs index 05288ee142e..7efa39a0909 100644 --- a/nexus/tests/test_projects.rs +++ b/nexus/tests/test_projects.rs @@ -7,9 +7,8 @@ use omicron_nexus::external_api::views::Project; use dropshot::test_util::object_get; use dropshot::test_util::objects_list_page; -pub mod common; -use common::resource_helpers::{create_organization, create_project}; -use common::test_setup; +use nexus_test_utils::resource_helpers::{create_organization, create_project}; +use nexus_test_utils::test_setup; extern crate slog; diff --git a/nexus/tests/test_router_routes.rs b/nexus/tests/test_router_routes.rs index e7ae6e72df5..44ba08941b2 100644 --- a/nexus/tests/test_router_routes.rs +++ b/nexus/tests/test_router_routes.rs @@ -2,22 +2,21 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -pub mod common; use std::net::{IpAddr, Ipv4Addr}; -use common::test_setup; use dropshot::test_util::{ object_delete, object_get, objects_list_page, objects_post, }; use dropshot::Method; use http::StatusCode; +use nexus_test_utils::test_setup; use omicron_common::api::external::{ IdentityMetadataCreateParams, IdentityMetadataUpdateParams, Name, RouteDestination, RouteTarget, RouterRoute, RouterRouteCreateParams, RouterRouteKind, RouterRouteUpdateParams, }; -use crate::common::resource_helpers::{ +use nexus_test_utils::resource_helpers::{ create_organization, create_project, create_router, create_vpc, }; diff --git a/nexus/tests/test_users_builtin.rs b/nexus/tests/test_users_builtin.rs index c75842ac07e..245d25672e8 100644 --- a/nexus/tests/test_users_builtin.rs +++ b/nexus/tests/test_users_builtin.rs @@ -5,11 +5,10 @@ use http::Method; use http::StatusCode; use std::collections::BTreeMap; -pub mod common; -use common::http_testing::AuthnMode; -use common::http_testing::NexusRequest; -use common::http_testing::RequestBuilder; -use common::test_setup; +use nexus_test_utils::http_testing::AuthnMode; +use nexus_test_utils::http_testing::NexusRequest; +use nexus_test_utils::http_testing::RequestBuilder; +use nexus_test_utils::test_setup; use omicron_nexus::authn; use omicron_nexus::external_api::views::User; diff --git a/nexus/tests/test_vpc_firewall.rs b/nexus/tests/test_vpc_firewall.rs index e966c836bf7..6af4501bc06 100644 --- a/nexus/tests/test_vpc_firewall.rs +++ b/nexus/tests/test_vpc_firewall.rs @@ -16,11 +16,10 @@ use std::convert::TryFrom; use dropshot::test_util::{object_delete, objects_list_page}; -pub mod common; -use common::resource_helpers::{ +use nexus_test_utils::resource_helpers::{ create_organization, create_project, create_vpc, }; -use common::test_setup; +use nexus_test_utils::test_setup; extern crate slog; diff --git a/nexus/tests/test_vpc_routers.rs b/nexus/tests/test_vpc_routers.rs index 81c81f9f2ec..c5ade382093 100644 --- a/nexus/tests/test_vpc_routers.rs +++ b/nexus/tests/test_vpc_routers.rs @@ -14,12 +14,11 @@ use dropshot::test_util::object_get; use dropshot::test_util::objects_list_page; use dropshot::test_util::objects_post; -pub mod common; -use common::identity_eq; -use common::resource_helpers::{ +use nexus_test_utils::identity_eq; +use nexus_test_utils::resource_helpers::{ create_organization, create_project, create_vpc, }; -use common::test_setup; +use nexus_test_utils::test_setup; extern crate slog; diff --git a/nexus/tests/test_vpc_subnets.rs b/nexus/tests/test_vpc_subnets.rs index d93bf8b0e31..25324e8c93a 100644 --- a/nexus/tests/test_vpc_subnets.rs +++ b/nexus/tests/test_vpc_subnets.rs @@ -16,12 +16,11 @@ use dropshot::test_util::objects_list_page; use dropshot::test_util::objects_post; use dropshot::test_util::ClientTestContext; -pub mod common; -use common::identity_eq; -use common::resource_helpers::{ +use nexus_test_utils::identity_eq; +use nexus_test_utils::resource_helpers::{ create_organization, create_project, create_vpc, }; -use common::test_setup; +use nexus_test_utils::test_setup; extern crate slog; diff --git a/nexus/tests/test_vpcs.rs b/nexus/tests/test_vpcs.rs index 2e4355edc96..3a2a2a4c49f 100644 --- a/nexus/tests/test_vpcs.rs +++ b/nexus/tests/test_vpcs.rs @@ -11,12 +11,11 @@ use dropshot::test_util::object_get; use dropshot::test_util::objects_list_page; use dropshot::test_util::ClientTestContext; -pub mod common; -use common::identity_eq; -use common::resource_helpers::{ +use nexus_test_utils::identity_eq; +use nexus_test_utils::resource_helpers::{ create_organization, create_project, create_vpc, create_vpc_with_error, }; -use common::test_setup; +use nexus_test_utils::test_setup; extern crate slog; diff --git a/nexus/tests/test_zpools.rs b/nexus/tests/test_zpools.rs index ae88667f42e..67095973bd4 100644 --- a/nexus/tests/test_zpools.rs +++ b/nexus/tests/test_zpools.rs @@ -8,8 +8,7 @@ use omicron_common::api::external::ByteCount; use omicron_nexus::internal_api::params::ZpoolPutRequest; use uuid::Uuid; -pub mod common; -use common::{test_setup, SLED_AGENT_UUID}; +use nexus_test_utils::{test_setup, SLED_AGENT_UUID}; extern crate slog; From 444aaa054c0da62b6a820487358b05fda537d01d Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 8 Dec 2021 14:24:03 -0500 Subject: [PATCH 02/50] No need to be screamy about disk posting, that's for another PR --- nexus/tests/test_disks.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/nexus/tests/test_disks.rs b/nexus/tests/test_disks.rs index 9bac547b227..88142097742 100644 --- a/nexus/tests/test_disks.rs +++ b/nexus/tests/test_disks.rs @@ -87,9 +87,7 @@ async fn test_disks() { snapshot_id: None, size: ByteCount::from_gibibytes_u32(1), }; - println!("POSTING A DISK"); let disk: Disk = objects_post(&client, &url_disks, new_disk.clone()).await; - println!("POSTED A DISK"); assert_eq!(disk.identity.name, "just-rainsticks"); assert_eq!(disk.identity.description, "sells rainsticks"); assert_eq!(disk.project_id, project.identity.id); From 42727a7eb1453ea2471a299b8a452c29d0a6d54d Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 8 Dec 2021 17:48:06 -0500 Subject: [PATCH 03/50] Optimize CRDB setup by using 'compile-time' population of a seed database --- Cargo.lock | 1 + nexus/test-utils/Cargo.toml | 5 +++ nexus/test-utils/build.rs | 26 ++++++++++++ test-utils/src/dev/db.rs | 10 ++++- test-utils/src/dev/mod.rs | 85 ++++++++++++++++++++++++++++++++++--- 5 files changed, 119 insertions(+), 8 deletions(-) create mode 100644 nexus/test-utils/build.rs diff --git a/Cargo.lock b/Cargo.lock index 29f3fe0f92e..9cff56c99b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1750,6 +1750,7 @@ dependencies = [ "serde", "serde_json", "slog", + "tokio", "uuid", ] diff --git a/nexus/test-utils/Cargo.toml b/nexus/test-utils/Cargo.toml index 5857bca9ab3..ff0324ad174 100644 --- a/nexus/test-utils/Cargo.toml +++ b/nexus/test-utils/Cargo.toml @@ -24,3 +24,8 @@ serde = { version = "1.0", features = [ "derive" ] } serde_json = "1.0" slog = { version = "2.7", features = [ "max_level_trace", "release_max_level_debug" ] } uuid = { version = "0.8", features = [ "serde", "v4" ] } + +[build-dependencies] +dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main" } +omicron-test-utils = { path = "../../test-utils" } +tokio = { version = "1.14" } diff --git a/nexus/test-utils/build.rs b/nexus/test-utils/build.rs new file mode 100644 index 00000000000..0ec894a2a4c --- /dev/null +++ b/nexus/test-utils/build.rs @@ -0,0 +1,26 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use dropshot::{test_util::LogContext, ConfigLogging, ConfigLoggingLevel}; +use omicron_test_utils::dev::{test_setup_database_seed, SEED_DB_DIR}; + +// Creates a "pre-populated" CockroachDB storage directory, which +// subsequent tests can copy instead of creating themselves. +// +// Refer to the documentation of [`test_setup_database_seed`] for +// more context. +#[tokio::main] +async fn main() { + println!("cargo:rerun-if-changed=../../common/src/sql/dbinit.sql"); + println!("cargo:rerun-if-changed=../../tools/cockroachdb_checksums"); + println!("cargo:rerun-if-changed=../../tools/cockroachdb_version"); + + let logctx = LogContext::new( + "crdb_seeding", + &ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Info }, + ); + + test_setup_database_seed(&logctx.log).await; + println!("cargo:warning=Created 'seed' CRDB for tests in {}", SEED_DB_DIR); +} diff --git a/test-utils/src/dev/db.rs b/test-utils/src/dev/db.rs index 1ee9e9f5703..cb1a89dd1a3 100644 --- a/test-utils/src/dev/db.rs +++ b/test-utils/src/dev/db.rs @@ -204,7 +204,7 @@ impl CockroachStarterBuilder { CockroachStarterBuilder::temp_path(&temp_dir, "listen-url"); let listen_arg = format!("127.0.0.1:{}", self.listen_port); self.arg("--store") - .arg(store_dir) + .arg(&store_dir) .arg("--listen-addr") .arg(&listen_arg) .arg("--listening-url-file") @@ -222,6 +222,7 @@ impl CockroachStarterBuilder { Ok(CockroachStarter { temp_dir, + store_dir: store_dir.into(), listen_url_file, args: self.args, cmd_builder: self.cmd_builder, @@ -260,6 +261,8 @@ impl CockroachStarterBuilder { pub struct CockroachStarter { /// temporary directory used for URL file and potentially data storage temp_dir: TempDir, + /// path to storage directory + store_dir: PathBuf, /// path to listen URL file (inside temp_dir) listen_url_file: PathBuf, /// command-line arguments, mirrored here for reporting to the user @@ -283,6 +286,11 @@ impl CockroachStarter { self.temp_dir.path() } + /// Returns the path to the storage directory created for this execution. + pub fn store_dir(&self) -> &Path { + self.store_dir.as_path() + } + /** * Spawns a new process to run the configured command * diff --git a/test-utils/src/dev/mod.rs b/test-utils/src/dev/mod.rs index 48fa7e9f3fc..f3d6224f195 100644 --- a/test-utils/src/dev/mod.rs +++ b/test-utils/src/dev/mod.rs @@ -17,6 +17,35 @@ use dropshot::ConfigLogging; use dropshot::ConfigLoggingIfExists; use dropshot::ConfigLoggingLevel; use slog::Logger; +use std::path::Path; + +/// Path to the "seed" CockroachDB directory. +/// +/// Populating CockroachDB unfortunately isn't free - creation of +/// tables, indices, and users takes several seconds to complete. +/// +/// By creating a "seed" version of the database, we can cut down +/// on the time spent performing this operation. Instead, we opt +/// to copy the database from this seed location. +pub const SEED_DB_DIR: &str = "/tmp/crdb-base"; + +// Helper for copying all the files in one directory to another. +fn copy_dir( + src: impl AsRef, + dst: impl AsRef, +) -> std::io::Result<()> { + std::fs::create_dir_all(&dst)?; + for entry in std::fs::read_dir(src)? { + let entry = entry?; + let ty = entry.file_type()?; + if ty.is_dir() { + copy_dir(entry.path(), dst.as_ref().join(entry.file_name()))?; + } else { + std::fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?; + } + } + Ok(()) +} /** * Set up a [`dropshot::test_util::LogContext`] appropriate for a test named @@ -35,11 +64,39 @@ pub fn test_setup_log(test_name: &str) -> LogContext { LogContext::new(test_name, &log_config) } -/** - * Set up a [`db::CockroachInstance`] for running tests against. - */ +enum StorageSource { + Populate, + CopyFromSeed, +} + +/// Creates a [`db::CockroachInstance`] with a populated storage directory. +/// +/// This is intended to optimize subsequent calls to [`test_setup_database`] +/// by reducing the latency of populating the storage directory. +pub async fn test_setup_database_seed(log: &Logger) { + std::fs::remove_dir_all(SEED_DB_DIR).unwrap(); + std::fs::create_dir_all(SEED_DB_DIR).unwrap(); + let mut db = + setup_database(log, Some(SEED_DB_DIR), StorageSource::Populate).await; + db.cleanup().await.unwrap(); +} + +/// Set up a [`db::CockroachInstance`] for running tests. pub async fn test_setup_database(log: &Logger) -> db::CockroachInstance { - let mut builder = db::CockroachStarterBuilder::new(); + setup_database(log, None, StorageSource::CopyFromSeed).await +} + +async fn setup_database( + log: &Logger, + store_dir: Option<&str>, + storage_source: StorageSource, +) -> db::CockroachInstance { + let builder = db::CockroachStarterBuilder::new(); + let mut builder = if let Some(store_dir) = store_dir { + builder.store_dir(store_dir) + } else { + builder + }; builder.redirect_stdio_to_files(); let starter = builder.build().unwrap(); info!( @@ -47,14 +104,28 @@ pub async fn test_setup_database(log: &Logger) -> db::CockroachInstance { "cockroach temporary directory: {}", starter.temp_dir().display() ); + + // If we're going to copy the storage directory from the seed, + // it is critical we do so before starting the DB. + if matches!(storage_source, StorageSource::CopyFromSeed) { + info!(&log, "cockroach: copying from seed directory"); + copy_dir(SEED_DB_DIR, starter.store_dir()) + .expect("Cannot copy storage from seed directory"); + } + info!(&log, "cockroach command line: {}", starter.cmdline()); let database = starter.start().await.unwrap(); info!(&log, "cockroach pid: {}", database.pid()); let db_url = database.pg_config(); info!(&log, "cockroach listen URL: {}", db_url); - info!(&log, "cockroach: populating"); - database.populate().await.expect("failed to populate database"); - info!(&log, "cockroach: populated"); + + // If we populate the storage directory by importing the '.sql' + // file, we must do so after the DB has started. + if matches!(storage_source, StorageSource::Populate) { + info!(&log, "cockroach: populating"); + database.populate().await.expect("failed to populate database"); + info!(&log, "cockroach: populated"); + } database } From 54b6f68364914dead2c683ca9600cce295406d59 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 8 Dec 2021 20:36:46 -0500 Subject: [PATCH 04/50] Add to top-level workspace --- Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 336b9e22b90..5435afd0169 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "common", "nexus", "nexus/src/db/db-macros", + "nexus/test-utils", "nexus-client", "package", "rpaths", @@ -22,6 +23,7 @@ default-members = [ "common", "nexus", "nexus/src/db/db-macros", + "nexus/test-utils", "package", "rpaths", "sled-agent", From c736fe078f53e5d70f76be8ae8163ddf652ec425 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 8 Dec 2021 20:38:56 -0500 Subject: [PATCH 05/50] Add some explanations --- nexus/test-utils/build.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nexus/test-utils/build.rs b/nexus/test-utils/build.rs index 0ec894a2a4c..0c8ec246359 100644 --- a/nexus/test-utils/build.rs +++ b/nexus/test-utils/build.rs @@ -8,6 +8,11 @@ use omicron_test_utils::dev::{test_setup_database_seed, SEED_DB_DIR}; // Creates a "pre-populated" CockroachDB storage directory, which // subsequent tests can copy instead of creating themselves. // +// Is it critical this happens at build-time? No. However, it +// makes it more convenient for tests to assume this seeded +// directory exists, rather than all attempting to create it +// concurrently. +// // Refer to the documentation of [`test_setup_database_seed`] for // more context. #[tokio::main] From 05aed1b8ca5ae0b29fd7d008985802af32beb4a4 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 8 Dec 2021 20:51:33 -0500 Subject: [PATCH 06/50] re-run if build.rs changes --- nexus/test-utils/build.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nexus/test-utils/build.rs b/nexus/test-utils/build.rs index 0c8ec246359..a6f967bf9dc 100644 --- a/nexus/test-utils/build.rs +++ b/nexus/test-utils/build.rs @@ -3,7 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use dropshot::{test_util::LogContext, ConfigLogging, ConfigLoggingLevel}; -use omicron_test_utils::dev::{test_setup_database_seed, SEED_DB_DIR}; +use omicron_test_utils::dev::test_setup_database_seed; // Creates a "pre-populated" CockroachDB storage directory, which // subsequent tests can copy instead of creating themselves. @@ -17,6 +17,7 @@ use omicron_test_utils::dev::{test_setup_database_seed, SEED_DB_DIR}; // more context. #[tokio::main] async fn main() { + println!("cargo:rerun-if-changed=build.rs"); println!("cargo:rerun-if-changed=../../common/src/sql/dbinit.sql"); println!("cargo:rerun-if-changed=../../tools/cockroachdb_checksums"); println!("cargo:rerun-if-changed=../../tools/cockroachdb_version"); @@ -27,5 +28,4 @@ async fn main() { ); test_setup_database_seed(&logctx.log).await; - println!("cargo:warning=Created 'seed' CRDB for tests in {}", SEED_DB_DIR); } From 304d03df52b8c4325b54b779450aa4ebddac302b Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 8 Dec 2021 21:08:54 -0500 Subject: [PATCH 07/50] The temporary directory might not exist --- test-utils/src/dev/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-utils/src/dev/mod.rs b/test-utils/src/dev/mod.rs index f3d6224f195..e7de5363779 100644 --- a/test-utils/src/dev/mod.rs +++ b/test-utils/src/dev/mod.rs @@ -74,7 +74,7 @@ enum StorageSource { /// This is intended to optimize subsequent calls to [`test_setup_database`] /// by reducing the latency of populating the storage directory. pub async fn test_setup_database_seed(log: &Logger) { - std::fs::remove_dir_all(SEED_DB_DIR).unwrap(); + let _ = std::fs::remove_dir_all(SEED_DB_DIR); std::fs::create_dir_all(SEED_DB_DIR).unwrap(); let mut db = setup_database(log, Some(SEED_DB_DIR), StorageSource::Populate).await; From c5427aa9283a9c614203fd2f81ccad4d159d34c3 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 8 Dec 2021 21:29:58 -0500 Subject: [PATCH 08/50] proby dropshot --- nexus/test-utils/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nexus/test-utils/Cargo.toml b/nexus/test-utils/Cargo.toml index ff0324ad174..b53469d4b27 100644 --- a/nexus/test-utils/Cargo.toml +++ b/nexus/test-utils/Cargo.toml @@ -8,7 +8,7 @@ license = "MPL-2.0" anyhow = "1.0" bytes = "1.0.1" chrono = { version = "0.4", features = [ "serde" ] } -dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main" } +dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] } http = "0.2.5" hyper = "0.14" omicron-common = { path = "../../common" } @@ -26,6 +26,6 @@ slog = { version = "2.7", features = [ "max_level_trace", "release_max_level_de uuid = { version = "0.8", features = [ "serde", "v4" ] } [build-dependencies] -dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main" } +dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] } omicron-test-utils = { path = "../../test-utils" } tokio = { version = "1.14" } From fea3cd94109e33b1e161667badf0793df3f03ffd Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 8 Dec 2021 23:00:18 -0500 Subject: [PATCH 09/50] Remove test-utils from default workspace --- Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 5435afd0169..39d8ee8369f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,6 @@ default-members = [ "common", "nexus", "nexus/src/db/db-macros", - "nexus/test-utils", "package", "rpaths", "sled-agent", From 0153e1954dd87a66782562fd8a823193bf70f5e7 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 8 Dec 2021 23:16:44 -0500 Subject: [PATCH 10/50] Download database executables *before* we build/test all targets --- .github/buildomat/jobs/build-and-test.sh | 24 ++++++++++++------------ .github/workflows/rust.yml | 12 ++++++------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/buildomat/jobs/build-and-test.sh b/.github/buildomat/jobs/build-and-test.sh index 0115b5e3f6a..92547e63aaa 100644 --- a/.github/buildomat/jobs/build-and-test.sh +++ b/.github/buildomat/jobs/build-and-test.sh @@ -14,6 +14,18 @@ set -o xtrace cargo --version rustc --version +banner clickhouse +ptime -m ./tools/ci_download_clickhouse + +banner cockroach +ptime -m bash ./tools/ci_download_cockroachdb + +# +# Put "./cockroachdb/bin" and "./clickhouse" on the PATH for the test +# suite. +# +export PATH="$PATH:$PWD/cockroachdb/bin:$PWD/clickhouse" + # # We build with: # @@ -33,18 +45,6 @@ export RUSTFLAGS="-D warnings" export RUSTDOCFLAGS="-D warnings" ptime -m cargo +'nightly-2021-11-24' build --locked --all-targets --verbose -banner clickhouse -ptime -m ./tools/ci_download_clickhouse - -banner cockroach -ptime -m bash ./tools/ci_download_cockroachdb - -# -# Put "./cockroachdb/bin" and "./clickhouse" on the PATH for the test -# suite. -# -export PATH="$PATH:$PWD/cockroachdb/bin:$PWD/clickhouse" - # # NOTE: We're using using the same RUSTFLAGS and RUSTDOCFLAGS as above to avoid # having to rebuild here. diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index c817c74c98b..89538b1d15f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -100,6 +100,12 @@ jobs: with: key: ${{ runner.os }}-clickhouse-binary-${{ hashFiles('tools/clickhouse_checksums') }} path: "clickhouse" + - name: Download ClickHouse + if: steps.cache-clickhouse.outputs.cache-hit != 'true' + run: ./tools/ci_download_clickhouse + - name: Download CockroachDB binary + if: steps.cache-cockroachdb.outputs.cache-hit != 'true' + run: bash ./tools/ci_download_cockroachdb - name: Build # We build with: # - RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings": disallow warnings @@ -112,12 +118,6 @@ jobs: # run. Building with `--locked` ensures that the checked-in Cargo.lock # is up to date. run: RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo +${{ matrix.toolchain }} build --locked --all-targets --verbose - - name: Download ClickHouse - if: steps.cache-clickhouse.outputs.cache-hit != 'true' - run: ./tools/ci_download_clickhouse - - name: Download CockroachDB binary - if: steps.cache-cockroachdb.outputs.cache-hit != 'true' - run: bash ./tools/ci_download_cockroachdb - name: Run tests # Use the same RUSTFLAGS and RUSTDOCFLAGS as above to avoid having to # rebuild here. From a0c88a4a040d0674171e5ccf2c2374724a7203c0 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 8 Dec 2021 23:33:42 -0500 Subject: [PATCH 11/50] Adjust path when building, we want that executable for our build script --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 89538b1d15f..516db86f9db 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -117,7 +117,7 @@ jobs: # also gives us a record of which dependencies were used for each CI # run. Building with `--locked` ensures that the checked-in Cargo.lock # is up to date. - run: RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo +${{ matrix.toolchain }} build --locked --all-targets --verbose + run: PATH="$PATH:$PWD/cockroachdb/bin:$PWD/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo +${{ matrix.toolchain }} build --locked --all-targets --verbose - name: Run tests # Use the same RUSTFLAGS and RUSTDOCFLAGS as above to avoid having to # rebuild here. From 56718af614d60208dce9febbf2252fef32ad0d90 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 9 Dec 2021 00:53:42 -0500 Subject: [PATCH 12/50] Add skeleton of saga-based structure --- nexus/src/db/db-macros/src/lib.rs | 4 +- nexus/src/db/model.rs | 12 +-- nexus/src/db/schema.rs | 3 + nexus/src/nexus.rs | 44 +++------- nexus/src/sagas.rs | 140 ++++++++++++++++++++++++++++-- 5 files changed, 161 insertions(+), 42 deletions(-) diff --git a/nexus/src/db/db-macros/src/lib.rs b/nexus/src/db/db-macros/src/lib.rs index 1bd49481468..a045174fd9f 100644 --- a/nexus/src/db/db-macros/src/lib.rs +++ b/nexus/src/db/db-macros/src/lib.rs @@ -186,7 +186,7 @@ fn build_resource_identity( let identity_name = format_ident!("{}Identity", struct_name); quote! { #[doc = #identity_doc] - #[derive(Clone, Debug, Selectable, Queryable, Insertable)] + #[derive(Clone, Debug, Selectable, Queryable, Insertable, serde::Serialize, serde::Deserialize)] #[table_name = #table_name ] pub struct #identity_name { pub id: ::uuid::Uuid, @@ -225,7 +225,7 @@ fn build_asset_identity(struct_name: &Ident, table_name: &Lit) -> TokenStream { let identity_name = format_ident!("{}Identity", struct_name); quote! { #[doc = #identity_doc] - #[derive(Clone, Debug, Selectable, Queryable, Insertable)] + #[derive(Clone, Debug, Selectable, Queryable, Insertable, serde::Serialize, serde::Deserialize)] #[table_name = #table_name ] pub struct #identity_name { pub id: ::uuid::Uuid, diff --git a/nexus/src/db/model.rs b/nexus/src/db/model.rs index 4411eaaa75c..305358e2785 100644 --- a/nexus/src/db/model.rs +++ b/nexus/src/db/model.rs @@ -26,7 +26,7 @@ use omicron_common::api::internal; use parse_display::Display; use ref_cast::RefCast; use schemars::JsonSchema; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use std::convert::TryFrom; use std::net::SocketAddr; use uuid::Uuid; @@ -105,8 +105,9 @@ macro_rules! impl_enum_type { Ord, PartialOrd, RefCast, - Deserialize, JsonSchema, + Serialize, + Deserialize, )] #[sql_type = "sql_types::Text"] #[serde(transparent)] @@ -141,7 +142,7 @@ where } } -#[derive(Copy, Clone, Debug, AsExpression, FromSqlRow)] +#[derive(Copy, Clone, Debug, AsExpression, FromSqlRow, Serialize, Deserialize)] #[sql_type = "sql_types::BigInt"] pub struct ByteCount(pub external::ByteCount); @@ -175,6 +176,7 @@ where #[derive( Copy, Clone, Debug, Eq, Ord, PartialEq, PartialOrd, AsExpression, FromSqlRow, + Serialize, Deserialize )] #[sql_type = "sql_types::BigInt"] #[repr(transparent)] @@ -829,7 +831,7 @@ where } /// A Disk (network block device). -#[derive(Queryable, Insertable, Clone, Debug, Selectable, Resource)] +#[derive(Queryable, Insertable, Clone, Debug, Selectable, Resource, Serialize, Deserialize)] #[table_name = "disk"] pub struct Disk { #[diesel(embed)] @@ -909,7 +911,7 @@ impl Into for Disk { } } -#[derive(AsChangeset, Clone, Debug, Queryable, Insertable, Selectable)] +#[derive(AsChangeset, Clone, Debug, Queryable, Insertable, Selectable, Serialize, Deserialize)] #[table_name = "disk"] // When "attach_instance_id" is set to None, we'd like to // clear it from the DB, rather than ignore the update. diff --git a/nexus/src/db/schema.rs b/nexus/src/db/schema.rs index acb6b44666a..4dcbcf5675f 100644 --- a/nexus/src/db/schema.rs +++ b/nexus/src/db/schema.rs @@ -301,6 +301,7 @@ table! { } allow_tables_to_appear_in_same_query!( + dataset, disk, instance, metric_producer, @@ -308,6 +309,7 @@ allow_tables_to_appear_in_same_query!( organization, oximeter, project, + region, saga, saga_node_event, console_session, @@ -318,4 +320,5 @@ allow_tables_to_appear_in_same_query!( vpc_router, vpc_firewall_rule, user_builtin, + zpool, ); diff --git a/nexus/src/nexus.rs b/nexus/src/nexus.rs index 976fee1e415..47017edd047 100644 --- a/nexus/src/nexus.rs +++ b/nexus/src/nexus.rs @@ -647,7 +647,7 @@ impl Nexus { } pub async fn project_create_disk( - &self, + self: &Arc, organization_name: &Name, project_name: &Name, params: ¶ms::DiskCreate, @@ -666,37 +666,21 @@ impl Nexus { }); } - let disk_id = Uuid::new_v4(); - let disk = db::model::Disk::new( - disk_id, - project.id(), - params.clone(), - db::model::DiskRuntimeState::new(), - ); - let disk_created = self.db_datastore.project_create_disk(disk).await?; - - // TODO: Here, we should ensure the disk is backed by appropriate - // regions. This is blocked behind actually having Crucible agents - // running in zones for dedicated zpools. - // - // TODO: Performing this operation, alongside "create" and "update - // state from create to detach", should be executed in a Saga. + let saga_params = Arc::new(sagas::ParamsDiskCreate { + project_id: project.id(), + create_params: params.clone(), + }); - /* - * This is a little hokey. We'd like to simulate an asynchronous - * transition from "Creating" to "Detached". For instances, the - * simulation lives in a simulated sled agent. Here, the analog might - * be a simulated storage control plane. But that doesn't exist yet, - * and we don't even know what APIs it would provide yet. So we just - * carry out the simplest possible "simulation" here: we'll return to - * the client a structure describing a disk in state "Creating", but by - * the time we do so, we've already updated the internal representation - * to "Created". - */ - self.db_datastore - .disk_update_runtime(&disk_id, &disk_created.runtime().detach()) + let saga_outputs = self + .execute_saga( + Arc::clone(&sagas::SAGA_DISK_CREATE_TEMPLATE), + sagas::SAGA_DISK_CREATE_NAME, + saga_params, + ) .await?; - + let disk_created = saga_outputs.lookup_output::("created_disk").map_err(|e| { + Error::InternalError { internal_message: e.to_string() } + })?; Ok(disk_created) } diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index 6888cf82792..8723da35cdf 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -39,9 +39,12 @@ use uuid::Uuid; * We'll need a richer mechanism for registering sagas, but this works for now. */ pub const SAGA_INSTANCE_CREATE_NAME: &'static str = "instance-create"; +pub const SAGA_DISK_CREATE_NAME: &'static str = "disk-create"; lazy_static! { pub static ref SAGA_INSTANCE_CREATE_TEMPLATE: Arc> = Arc::new(saga_instance_create()); + pub static ref SAGA_DISK_CREATE_TEMPLATE: Arc> = + Arc::new(saga_disk_create()); } lazy_static! { @@ -51,11 +54,18 @@ lazy_static! { fn all_templates( ) -> BTreeMap<&'static str, Arc>>> { - vec![( - SAGA_INSTANCE_CREATE_NAME, - Arc::clone(&SAGA_INSTANCE_CREATE_TEMPLATE) - as Arc>>, - )] + vec![ + ( + SAGA_INSTANCE_CREATE_NAME, + Arc::clone(&SAGA_INSTANCE_CREATE_TEMPLATE) + as Arc>>, + ), + ( + SAGA_DISK_CREATE_NAME, + Arc::clone(&SAGA_DISK_CREATE_TEMPLATE) + as Arc>>, + ), + ] .into_iter() .collect() } @@ -223,3 +233,123 @@ async fn sic_instance_ensure( .map(|_| ()) .map_err(ActionError::action_failed) } + +#[derive(Debug, Deserialize, Serialize)] +pub struct ParamsDiskCreate { + pub project_id: Uuid, + pub create_params: params::DiskCreate, +} + +#[derive(Debug)] +pub struct SagaDiskCreate; +impl SagaType for SagaDiskCreate { + type SagaParamsType = Arc; + type ExecContextType = Arc; +} + +pub fn saga_disk_create() -> SagaTemplate { + let mut template_builder = SagaTemplateBuilder::new(); + + template_builder.append( + "disk_id", + "GenerateDiskId", + new_action_noop_undo(sdc_generate_uuid), + ); + + template_builder.append( + "created_disk", + "CreateDiskRecord", + // TODO: Needs undo action. + new_action_noop_undo(sdc_create_disk_record), + ); + + template_builder.append( + "datasets_and_regions", + "AllocRegions", + // TODO: Needs undo action. + new_action_noop_undo(sdc_alloc_regions), + ); + + template_builder.append( + "regions_ensure", + "RegionsEnsure", + // TODO: Needs undo action. + new_action_noop_undo(sdc_regions_ensure), + ); + + template_builder.append( + "disk_runtime", + "FinalizeDiskRecord", + // TODO: Needs undo action. + new_action_noop_undo(sdc_finalize_disk_record), + ); + + template_builder.build() +} + +async fn sdc_generate_uuid( + _: ActionContext, +) -> Result { + Ok(Uuid::new_v4()) +} + +async fn sdc_create_disk_record( + sagactx: ActionContext, +) -> Result { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params(); + + let disk_id = sagactx.lookup::("disk_id")?; + let disk = db::model::Disk::new( + disk_id, + params.project_id, + params.create_params.clone(), + db::model::DiskRuntimeState::new(), + ); + let disk_created = osagactx.datastore() + .project_create_disk(disk) + .await + .map_err(ActionError::action_failed)?; + Ok(disk_created) +} + +async fn sdc_alloc_regions( + sagactx: ActionContext, +) -> Result<(), ActionError> { + let _osagactx = sagactx.user_data(); + let _params = sagactx.saga_params(); + // TODO: Here, we should ensure the disk is backed by appropriate + // regions. This is blocked behind actually having Crucible agents + // running in zones for dedicated zpools. + // + // TODO: I believe this was a join of dataset + region, group by dataset + // sum region sizes, sort by ascending? Something like that? + todo!(); +} + +async fn sdc_regions_ensure( + sagactx: ActionContext, +) -> Result<(), ActionError> { + let _osagactx = sagactx.user_data(); + let _params = sagactx.saga_params(); + + // TODO: Make the calls to crucible agents. + // TODO: Figure out how we're testing this - setup fake endpoints + // in the simulated sled agent, or do something else? + todo!(); +} + +async fn sdc_finalize_disk_record( + sagactx: ActionContext, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let _params = sagactx.saga_params(); + + let disk_id = sagactx.lookup::("disk_id")?; + let disk_created = sagactx.lookup::("disk_created")?; + osagactx.datastore() + .disk_update_runtime(&disk_id, &disk_created.runtime().detach()) + .await + .map_err(ActionError::action_failed)?; + Ok(()) +} From a00cd7c68aecc3ed345d1200a5077e5cb3f4e2a2 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 9 Dec 2021 15:47:50 -0500 Subject: [PATCH 13/50] Skeleton of test --- nexus/src/db/datastore.rs | 100 ++++++++++++++++++++++++++++--- nexus/src/db/model.rs | 6 +- nexus/src/external_api/params.rs | 14 +++++ nexus/src/sagas.rs | 27 ++++++--- 4 files changed, 127 insertions(+), 20 deletions(-) diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index b5d8b72c475..207e8415458 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -62,12 +62,12 @@ use crate::db::{ public_error_from_diesel_pool_shouldnt_fail, }, model::{ - ConsoleSession, Dataset, Disk, DiskAttachment, DiskRuntimeState, + ConsoleSession, Dataset, DatasetKind, Disk, DiskAttachment, DiskRuntimeState, Generation, Instance, InstanceRuntimeState, Name, Organization, OrganizationUpdate, OximeterInfo, ProducerEndpoint, Project, - ProjectUpdate, RouterRoute, RouterRouteUpdate, Sled, UserBuiltin, Vpc, - VpcFirewallRule, VpcRouter, VpcRouterUpdate, VpcSubnet, - VpcSubnetUpdate, VpcUpdate, Zpool, + ProjectUpdate, Region, RouterRoute, RouterRouteUpdate, Sled, + UserBuiltin, Vpc, VpcFirewallRule, VpcRouter, VpcRouterUpdate, + VpcSubnet, VpcSubnetUpdate, VpcUpdate, Zpool, }, pagination::paginated, update_and_check::{UpdateAndCheck, UpdateStatus}, @@ -230,6 +230,23 @@ impl DataStore { }) } + /// Allocates enough regions to back a disk. + /// + /// Returns the allocated regions, as well as the datasets to which they + /// belong. + pub async fn region_allocate( + &self, + params: ¶ms::DiskCreate, + ) -> Result, Error> { + use db::schema::region::dsl as region_dsl; + use db::schema::dataset::dsl as dataset_dsl; + + + // TODO: I believe this was a join of dataset + region, group by dataset + // sum region sizes, sort by ascending? Something like that + todo!(); + } + /// Create a organization pub async fn organization_create( &self, @@ -2030,22 +2047,23 @@ impl DataStore { #[cfg(test)] mod test { + use super::*; use crate::authz; use crate::context::OpContext; use crate::db; use crate::db::identity::Resource; use crate::db::model::{ConsoleSession, Organization, Project}; - use crate::db::DataStore; use crate::external_api::params; use chrono::{Duration, Utc}; - use omicron_common::api::external::{Error, IdentityMetadataCreateParams}; + use omicron_common::api::external::{ByteCount, Name, Error, IdentityMetadataCreateParams}; use omicron_test_utils::dev; use std::sync::Arc; + use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use uuid::Uuid; #[tokio::test] async fn test_project_creation() { - let logctx = dev::test_setup_log("test_collection_not_present"); + let logctx = dev::test_setup_log("test_project_creation"); let opctx = OpContext::for_unit_tests(logctx.log.new(o!())); let mut db = dev::test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; @@ -2081,7 +2099,7 @@ mod test { #[tokio::test] async fn test_session_methods() { - let logctx = dev::test_setup_log("test_collection_not_present"); + let logctx = dev::test_setup_log("test_session_methods"); let mut db = dev::test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&cfg); @@ -2132,6 +2150,72 @@ mod test { let delete_again = datastore.session_hard_delete(token.clone()).await; assert_eq!(delete_again, Ok(())); + let _ = db.cleanup().await; + } + + #[tokio::test] + async fn test_region_allocation() { + let logctx = dev::test_setup_log("test_region_allocation"); + let opctx = OpContext::for_unit_tests(logctx.log.new(o!())); + let mut db = dev::test_setup_database(&logctx.log).await; + let cfg = db::Config { url: db.pg_config().clone() }; + let pool = db::Pool::new(&cfg); + let datastore = DataStore::new(Arc::new(pool)); + + // TODO: Refactor org / project creation to a helper. + let organization = Organization::new(params::OrganizationCreate { + identity: IdentityMetadataCreateParams { + name: "org".parse().unwrap(), + description: "desc".to_string(), + }, + }); + let organization = + datastore.organization_create(&opctx, organization).await.unwrap(); + let project = Project::new( + organization.id(), + params::ProjectCreate { + identity: IdentityMetadataCreateParams { + name: "project".parse().unwrap(), + description: "desc".to_string(), + }, + }, + ); + let org = authz::FLEET.organization(organization.id()); + datastore.project_create(&opctx, &org, project).await.unwrap(); + + // Create Datasets, from which regions will be allocated. + + // XXX This pool doesn't exist... + let pool_id = Uuid::new_v4(); + let dataset_id = Uuid::new_v4(); + // XXX This address is a lie, but that doesn't matter - we're here + // to test the database interaction, not the networking aspect. + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); + let kind = DatasetKind(crate::internal_api::params::DatasetKind::Crucible); + + let dataset_ids: Vec = (0..6).map(|_| Uuid::new_v4()).collect(); + for id in &dataset_ids { + let dataset = Dataset::new(*id, pool_id, addr, kind.clone()); + datastore.dataset_upsert(dataset).await.unwrap(); + } + + // Allocate regions from the datasets. + + let disk_create_params = params::DiskCreate { + identity: IdentityMetadataCreateParams { + name: Name::try_from("disk1".to_string()).unwrap(), + description: "my-disk".to_string(), + }, + snapshot_id: None, + size: ByteCount::from_mebibytes_u32(500), + }; + let dataset_and_regions = datastore.region_allocate(&disk_create_params).await.unwrap(); + + // Verify the allocation we just performed. + assert_eq!(3, dataset_and_regions.len()); + // TODO: verify allocated regions make sense + + let _ = db.cleanup().await; } } diff --git a/nexus/src/db/model.rs b/nexus/src/db/model.rs index 305358e2785..5afcce90a9d 100644 --- a/nexus/src/db/model.rs +++ b/nexus/src/db/model.rs @@ -488,7 +488,7 @@ impl_enum_type!( #[postgres(type_name = "dataset_kind", type_schema = "public")] pub struct DatasetKindEnum; - #[derive(Clone, Debug, AsExpression, FromSqlRow)] + #[derive(Clone, Debug, AsExpression, FromSqlRow, Serialize, Deserialize)] #[sql_type = "DatasetKindEnum"] pub struct DatasetKind(pub internal_api::params::DatasetKind); @@ -508,7 +508,7 @@ impl From for DatasetKind { /// /// A dataset represents a portion of a Zpool, which is then made /// available to a service on the Sled. -#[derive(Queryable, Insertable, Debug, Clone, Selectable, Asset)] +#[derive(Queryable, Insertable, Debug, Clone, Selectable, Asset, Deserialize, Serialize)] #[table_name = "dataset"] pub struct Dataset { #[diesel(embed)] @@ -568,7 +568,7 @@ impl DatastoreCollection for Disk { /// /// A region represents a portion of a Crucible Downstairs dataset /// allocated within a volume. -#[derive(Queryable, Insertable, Debug, Clone, Selectable, Asset)] +#[derive(Queryable, Insertable, Debug, Clone, Selectable, Asset, Serialize, Deserialize)] #[table_name = "region"] pub struct Region { #[diesel(embed)] diff --git a/nexus/src/external_api/params.rs b/nexus/src/external_api/params.rs index 515058ff235..5c330c068df 100644 --- a/nexus/src/external_api/params.rs +++ b/nexus/src/external_api/params.rs @@ -168,6 +168,20 @@ pub struct DiskCreate { pub size: ByteCount, } +impl DiskCreate { + pub fn block_size(&self) -> u64 { + 512 + } + + pub fn extent_size(&self) -> u64 { + 1 << 20 + } + + pub fn extent_count(&self) -> u64 { + (self.size.to_bytes() + self.extent_size() - 1) / self.extent_size() + } +} + /* * BUILT-IN USERS * diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index 8723da35cdf..354d385b006 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -300,6 +300,11 @@ async fn sdc_create_disk_record( let params = sagactx.saga_params(); let disk_id = sagactx.lookup::("disk_id")?; + + // NOTE: This could be done in a txn with region allocation? + // + // Unclear if it's a problem to let this disk exist without any backing + // regions for a brief period of time. let disk = db::model::Disk::new( disk_id, params.project_id, @@ -315,16 +320,20 @@ async fn sdc_create_disk_record( async fn sdc_alloc_regions( sagactx: ActionContext, -) -> Result<(), ActionError> { - let _osagactx = sagactx.user_data(); - let _params = sagactx.saga_params(); - // TODO: Here, we should ensure the disk is backed by appropriate - // regions. This is blocked behind actually having Crucible agents - // running in zones for dedicated zpools. +) -> Result, ActionError> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params(); + // Ensure the disk is backed by appropriate regions. // - // TODO: I believe this was a join of dataset + region, group by dataset - // sum region sizes, sort by ascending? Something like that? - todo!(); + // This allocates regions in the database, but the disk state is still + // "creating" - the respective Crucible Agents must be instructed to + // allocate the necessary regions before we can mark the disk as "ready to + // be used". + let datasets_and_regions = osagactx.datastore() + .region_allocate(¶ms.create_params) + .await + .map_err(ActionError::action_failed)?; + Ok(datasets_and_regions) } async fn sdc_regions_ensure( From 1b19c578c7a17e179be8b977c3ae296d52348172 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 9 Dec 2021 17:23:02 -0500 Subject: [PATCH 14/50] fixup --- .envrc | 2 -- demo-instance.sh | 11 ----------- demo.sh | 22 ---------------------- logs.sh | 34 ---------------------------------- 4 files changed, 69 deletions(-) delete mode 100644 .envrc delete mode 100755 demo-instance.sh delete mode 100755 demo.sh delete mode 100755 logs.sh diff --git a/.envrc b/.envrc deleted file mode 100644 index 57aff8d5ae0..00000000000 --- a/.envrc +++ /dev/null @@ -1,2 +0,0 @@ -PATH="$PWD/clickhouse:$PATH" -PATH="$PWD/cockroachdb/bin:$PATH" diff --git a/demo-instance.sh b/demo-instance.sh deleted file mode 100755 index bfbfe1f8a9b..00000000000 --- a/demo-instance.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -eu - -trap "kill 0" EXIT - -set -x -# ./tools/oxapi_demo disk_create_demo myorg myproject mydisk -./tools/oxapi_demo instance_create_demo myorg myproject myinstance -# ./tools/oxapi_demo instance_attach_disk myorg myproject myinstance mydisk -# ./tools/oxapi_demo instance_start myorg myproject myinstance -set +x diff --git a/demo.sh b/demo.sh deleted file mode 100755 index b4f7e06fa81..00000000000 --- a/demo.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -eu - -trap "kill 0" EXIT - -cargo build --release --package omicron-package -# cargo build --release --package omicron-common --bin omicron-package -echo "Packaging..." -./target/release/omicron-package package - -echo "Launching DB..." -cargo run --bin=omicron-dev -- db-run &> /dev/null & -cargo run --bin=omicron-dev -- ch-run &> /dev/null & - -echo "Installing..." -pfexec ./target/release/omicron-package install - -echo "Sled Agent and Nexus Online" -sleep 8 -./tools/oxapi_demo organization_create_demo myorg -./tools/oxapi_demo project_create_demo myorg myproject -wait diff --git a/logs.sh b/logs.sh deleted file mode 100755 index 6bd6adfc679..00000000000 --- a/logs.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -set -eu - -trap "kill 0" EXIT - -if { [ "$TERM" = "screen" ] && [ -n "$TMUX" ]; } then - echo "This script should be run in tmux! It splits panes..." - exit 1 -fi - -# First of all, split the lower half of the screen into logs. -services=(nexus sled-agent oximeter) - -pane_height=$(tmux list-pane -F "#{pane_id}:#{pane_height}" | rg "$TMUX_PANE:" | cut -d: -f2) -pane_count=$(("${#services[@]}" + 1 )) -height_per_svc=$(( "$pane_height" / "$pane_count" )) -height_remaining=$(( "$pane_height" - "$height_per_svc" )) - -pane="$TMUX_PANE" -for index in "${!services[@]}"; do - service="${services[index]}" - logfile="$(svcs -L "$service")" - pane=$(tmux split-window \ - -t "$pane" \ - -l "$height_remaining" \ - -P -F "#{pane_id}" \ - -v \ - "tail -F $logfile") - - height_remaining=$(( "$height_remaining" - "$height_per_svc" )) -done - -# Then just keep watching the services. -watch -n 2 "svcs -a | rg illumos" From 4f9feefd58b1446b15106a9105c6a049eb6539a2 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 9 Dec 2021 17:27:45 -0500 Subject: [PATCH 15/50] review feedback --- README.adoc | 2 +- test-utils/src/dev/mod.rs | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/README.adoc b/README.adoc index 47ac1b0b253..d1e5b2b5df7 100644 --- a/README.adoc +++ b/README.adoc @@ -83,7 +83,7 @@ example, on Helios, you'd want `/usr/bin` on your PATH. -- . CockroachDB v21.1.10. + -The test suite expects to be able to start a single-node CockroachDB cluster using the `cockroach` executable on your PATH. +The build and test suite expects to be able to start a single-node CockroachDB cluster using the `cockroach` executable on your PATH. On illumos, MacOS, and Linux, you should be able to use the `tools/ci_download_cockroachdb` script to fetch the official CockroachDB binary. It will be put into `./cockroachdb/bin/cockroach`. Alternatively, you can follow the https://www.cockroachlabs.com/docs/stable/install-cockroachdb.html[official CockroachDB installation instructions for your platform]. diff --git a/test-utils/src/dev/mod.rs b/test-utils/src/dev/mod.rs index e7de5363779..55791199015 100644 --- a/test-utils/src/dev/mod.rs +++ b/test-utils/src/dev/mod.rs @@ -17,7 +17,7 @@ use dropshot::ConfigLogging; use dropshot::ConfigLoggingIfExists; use dropshot::ConfigLoggingLevel; use slog::Logger; -use std::path::Path; +use std::path::{Path, PathBuf}; /// Path to the "seed" CockroachDB directory. /// @@ -27,7 +27,9 @@ use std::path::Path; /// By creating a "seed" version of the database, we can cut down /// on the time spent performing this operation. Instead, we opt /// to copy the database from this seed location. -pub const SEED_DB_DIR: &str = "/tmp/crdb-base"; +fn seed_dir() -> PathBuf { + std::env::temp_dir().join("crdb-base") +} // Helper for copying all the files in one directory to another. fn copy_dir( @@ -74,10 +76,10 @@ enum StorageSource { /// This is intended to optimize subsequent calls to [`test_setup_database`] /// by reducing the latency of populating the storage directory. pub async fn test_setup_database_seed(log: &Logger) { - let _ = std::fs::remove_dir_all(SEED_DB_DIR); - std::fs::create_dir_all(SEED_DB_DIR).unwrap(); - let mut db = - setup_database(log, Some(SEED_DB_DIR), StorageSource::Populate).await; + let dir = seed_dir(); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let mut db = setup_database(log, Some(&dir), StorageSource::Populate).await; db.cleanup().await.unwrap(); } @@ -88,7 +90,7 @@ pub async fn test_setup_database(log: &Logger) -> db::CockroachInstance { async fn setup_database( log: &Logger, - store_dir: Option<&str>, + store_dir: Option<&Path>, storage_source: StorageSource, ) -> db::CockroachInstance { let builder = db::CockroachStarterBuilder::new(); @@ -109,7 +111,7 @@ async fn setup_database( // it is critical we do so before starting the DB. if matches!(storage_source, StorageSource::CopyFromSeed) { info!(&log, "cockroach: copying from seed directory"); - copy_dir(SEED_DB_DIR, starter.store_dir()) + copy_dir(seed_dir(), starter.store_dir()) .expect("Cannot copy storage from seed directory"); } From 8eb0549ff08592fbf1b4de2d2e20e43d80e32f4a Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 9 Dec 2021 17:39:23 -0500 Subject: [PATCH 16/50] OUT of the default members again gah --- Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 5435afd0169..39d8ee8369f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,6 @@ default-members = [ "common", "nexus", "nexus/src/db/db-macros", - "nexus/test-utils", "package", "rpaths", "sled-agent", From 0d673badd5564151c6eab5153134a5b756e2b1ff Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 10 Dec 2021 17:24:09 -0500 Subject: [PATCH 17/50] ... allocation query still WIP --- nexus/src/db/datastore.rs | 42 +++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index 207e8415458..2a6ebfd522d 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -241,6 +241,24 @@ impl DataStore { use db::schema::region::dsl as region_dsl; use db::schema::dataset::dsl as dataset_dsl; + let datasets = dataset_dsl::dataset + // First, we look for valid datasets. + .filter(dataset_dsl::time_deleted.is_null()) + // We tally up the regions within those datasets + .inner_join( + region_dsl::region.on( + dataset_dsl::id.eq(region_dsl::dataset_id) + ) + ) +// .group_by(region_dsl::dataset_id) +// .select((Dataset::as_select(), diesel::dsl::sum(region_dsl::extent_count))) +// .order((region_dsl::extent_size * region_dsl::extent_count).asc()) +// .select((Dataset::as_select(), diesel::dsl::sum(region_dsl::extent_size * region_dsl::extent_count))) + .select(Dataset::as_select()) + .get_results_async::(self.pool()) + .await + .unwrap(); // TODO: Handle errors + // TODO: I believe this was a join of dataset + region, group by dataset // sum region sizes, sort by ascending? Something like that @@ -2183,24 +2201,28 @@ mod test { let org = authz::FLEET.organization(organization.id()); datastore.project_create(&opctx, &org, project).await.unwrap(); - // Create Datasets, from which regions will be allocated. - - // XXX This pool doesn't exist... - let pool_id = Uuid::new_v4(); - let dataset_id = Uuid::new_v4(); - // XXX This address is a lie, but that doesn't matter - we're here - // to test the database interaction, not the networking aspect. + // Create a sled... let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); - let kind = DatasetKind(crate::internal_api::params::DatasetKind::Crucible); + let sled_id = Uuid::new_v4(); + let sled = Sled::new(sled_id, addr.clone()); + datastore.sled_upsert(sled).await.unwrap(); + + // ... and a zpool within that sled... + let zpool_id = Uuid::new_v4(); + let zpool = Zpool::new(zpool_id, sled_id, &crate::internal_api::params::ZpoolPutRequest { + size: ByteCount::from_gibibytes_u32(100), + }); + datastore.zpool_upsert(zpool).await.unwrap(); + // ... and datasets within that zpool. + let kind = DatasetKind(crate::internal_api::params::DatasetKind::Crucible); let dataset_ids: Vec = (0..6).map(|_| Uuid::new_v4()).collect(); for id in &dataset_ids { - let dataset = Dataset::new(*id, pool_id, addr, kind.clone()); + let dataset = Dataset::new(*id, zpool_id, addr, kind.clone()); datastore.dataset_upsert(dataset).await.unwrap(); } // Allocate regions from the datasets. - let disk_create_params = params::DiskCreate { identity: IdentityMetadataCreateParams { name: Name::try_from("disk1".to_string()).unwrap(), From 95be792c3c7f6e97fd85857bf8e9b9a302ed8150 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 13 Dec 2021 21:07:30 -0500 Subject: [PATCH 18/50] Still workin towards a test --- Cargo.lock | 27 +++++++ nexus/Cargo.toml | 5 +- nexus/src/db/datastore.rs | 160 ++++++++++++++++++++++++++------------ nexus/src/db/model.rs | 17 +++- nexus/src/db/schema.rs | 3 + nexus/src/sagas.rs | 3 +- 6 files changed, 162 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8d2124152fe..aec3e5b5b40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -175,6 +175,17 @@ dependencies = [ "num_enum", ] +[[package]] +name = "bigdecimal" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aaf33151a6429fe9211d1b276eafdf70cdff28b071e76c0b0e1503221ea3744" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "bincode" version = "1.3.3" @@ -714,6 +725,7 @@ name = "diesel" version = "2.0.0" source = "git+https://github.com/diesel-rs/diesel?rev=ce77c382#ce77c382d2836f6b385225991cf58cb2d2dd65d6" dependencies = [ + "bigdecimal", "bitflags", "byteorder", "chrono", @@ -721,6 +733,9 @@ dependencies = [ "ipnetwork", "itoa", "libc", + "num-bigint", + "num-integer", + "num-traits", "pq-sys", "r2d2", "serde_json", @@ -1780,6 +1795,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -1875,6 +1901,7 @@ dependencies = [ "async-bb8-diesel", "async-trait", "bb8", + "bigdecimal", "chrono", "cookie", "criterion", diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index 14122c0f52b..9dd206924a6 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -9,12 +9,13 @@ path = "../rpaths" [dependencies] anyhow = "1.0" +async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", rev = "22c26ef" } async-trait = "0.1.51" bb8 = "0.7.1" -async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", rev = "22c26ef" } +bigdecimal = "0.3" cookie = "0.15" # Tracking pending 2.0 version. -diesel = { git = "https://github.com/diesel-rs/diesel", rev = "ce77c382", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] } +diesel = { git = "https://github.com/diesel-rs/diesel", rev = "ce77c382", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "numeric", "uuid"] } futures = "0.3.18" hex = "0.4.3" http = "0.2.5" diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index 51ceb80a4a7..9207324c6c3 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -34,6 +34,7 @@ use crate::authz; use crate::context::OpContext; use crate::external_api::params; use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl, ConnectionManager}; +use bigdecimal::ToPrimitive; use chrono::Utc; use diesel::prelude::*; use diesel::upsert::excluded; @@ -51,7 +52,7 @@ use omicron_common::api::external::{ CreateResult, IdentityMetadataCreateParams, }; use omicron_common::bail_unless; -use std::convert::TryFrom; +use std::convert::{TryFrom, TryInto}; use std::sync::Arc; use uuid::Uuid; @@ -236,33 +237,100 @@ impl DataStore { /// belong. pub async fn region_allocate( &self, + disk_id: Uuid, params: ¶ms::DiskCreate, ) -> Result, Error> { use db::schema::region::dsl as region_dsl; use db::schema::dataset::dsl as dataset_dsl; - let datasets = dataset_dsl::dataset - // First, we look for valid datasets. + println!("region_allocate: Allocating region: {:#?}", params); + + // Allocation Policy + // + // NOTE: This policy can - and should! - be changed. + // It is currently acting as a placeholder, showing a feasible + // interaction between datasets and regions. + // + // This policy allocates regions to distinct Crucible datasets, + // favoring datasets with the smallest existing (summed) region + // sizes. + // + // + // + let datasets: Vec<(Dataset, u64)> = dataset_dsl::dataset + // First, we look for valid datasets (non-deleted crucible datasets). .filter(dataset_dsl::time_deleted.is_null()) - // We tally up the regions within those datasets - .inner_join( + .filter(dataset_dsl::kind.eq(DatasetKind(crate::internal_api::params::DatasetKind::Crucible))) + // Next, observe all the regions allocated to each dataset, and + // determine how much space they're using. + // + // NOTE: We *could* store "free/allocated" space per-dataset, and + // work hard to keep them up-to-date, rather than trying to + // recompute this. + .left_outer_join( region_dsl::region.on( dataset_dsl::id.eq(region_dsl::dataset_id) ) ) -// .group_by(region_dsl::dataset_id) -// .select((Dataset::as_select(), diesel::dsl::sum(region_dsl::extent_count))) -// .order((region_dsl::extent_size * region_dsl::extent_count).asc()) -// .select((Dataset::as_select(), diesel::dsl::sum(region_dsl::extent_size * region_dsl::extent_count))) - .select(Dataset::as_select()) - .get_results_async::(self.pool()) + .group_by(dataset_dsl::id) + .select( + ( + Dataset::as_select(), + diesel::dsl::sum(region_dsl::extent_count * region_dsl::extent_size).nullable() + ) + ) + .order(diesel::dsl::sum(region_dsl::extent_size * region_dsl::extent_count).asc()) + .get_results_async::<(Dataset, Option)>(self.pool()) .await - .unwrap(); // TODO: Handle errors + .map_err(|e| { + public_error_from_diesel_pool_shouldnt_fail(e) + })? + .into_iter() + .map(|(dataset, total_allocated)| { + // TODO: If there aren't any regions, zero is a reasonable + // default. But if the size * count is too big, we probably + // want safer handling here. + // + // Do we need to convert to u64 if this is internal? + (dataset, total_allocated.map_or(0, |value| value.to_u64().unwrap())) + }).collect(); + + println!("region_allocate: Observed datasets: {:#?}", datasets); + + // TODO: We don't actually need to return the allc'd space right now... + // maybe we should tho to compare it with a total size or something. + let datasets: Vec = datasets.into_iter().map(|(d, _)| d).collect(); + + // TODO: magic num + let threshold = 3; + if datasets.len() < threshold { + return Err(Error::internal_error("Not enough datasets for replicated allocation")); + } + let source_datasets = &datasets[0..threshold]; + let regions: Vec = source_datasets.iter() + .map(|dataset| { + Region::new( + dataset.id(), + disk_id, + params.block_size().try_into().unwrap(), + params.extent_size().try_into().unwrap(), + params.extent_count().try_into().unwrap(), + ) + }) + .collect(); + + let regions = diesel::insert_into(region_dsl::region) + .values(regions) + .returning(Region::as_returning()) + .get_results_async(self.pool()) + .await + .map_err(|e| { + public_error_from_diesel_pool_shouldnt_fail(e) + })?; - // TODO: I believe this was a join of dataset + region, group by dataset - // sum region sizes, sort by ascending? Something like that - todo!(); + // TODO: also, make this concurrency-safe. Txns? + Ok(source_datasets.into_iter().map(|d| d.clone()).zip(regions).collect()) } /// Create a organization @@ -2173,54 +2241,46 @@ mod test { logctx.cleanup_successful(); } + // Creates a test sled, returns its UUID. + async fn create_test_sled(datastore: &DataStore) -> Uuid { + let bogus_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); + let sled_id = Uuid::new_v4(); + let sled = Sled::new(sled_id, bogus_addr.clone()); + datastore.sled_upsert(sled).await.unwrap(); + sled_id + } + + // Creates a test zpool, returns its UUID. + async fn create_test_zpool(datastore: &DataStore, sled_id: Uuid) -> Uuid { + let zpool_id = Uuid::new_v4(); + let zpool = Zpool::new(zpool_id, sled_id, &crate::internal_api::params::ZpoolPutRequest { + size: ByteCount::from_gibibytes_u32(100), + }); + datastore.zpool_upsert(zpool).await.unwrap(); + zpool_id + } + #[tokio::test] async fn test_region_allocation() { let logctx = dev::test_setup_log("test_region_allocation"); - let opctx = OpContext::for_unit_tests(logctx.log.new(o!())); let mut db = dev::test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&cfg); let datastore = DataStore::new(Arc::new(pool)); - // TODO: Refactor org / project creation to a helper. - let organization = Organization::new(params::OrganizationCreate { - identity: IdentityMetadataCreateParams { - name: "org".parse().unwrap(), - description: "desc".to_string(), - }, - }); - let organization = - datastore.organization_create(&opctx, organization).await.unwrap(); - let project = Project::new( - organization.id(), - params::ProjectCreate { - identity: IdentityMetadataCreateParams { - name: "project".parse().unwrap(), - description: "desc".to_string(), - }, - }, - ); - let org = authz::FLEET.organization(organization.id()); - datastore.project_create(&opctx, &org, project).await.unwrap(); - // Create a sled... - let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); - let sled_id = Uuid::new_v4(); - let sled = Sled::new(sled_id, addr.clone()); - datastore.sled_upsert(sled).await.unwrap(); + let sled_id = create_test_sled(&datastore).await; // ... and a zpool within that sled... - let zpool_id = Uuid::new_v4(); - let zpool = Zpool::new(zpool_id, sled_id, &crate::internal_api::params::ZpoolPutRequest { - size: ByteCount::from_gibibytes_u32(100), - }); - datastore.zpool_upsert(zpool).await.unwrap(); + let zpool_id = create_test_zpool(&datastore, sled_id).await; // ... and datasets within that zpool. + let bogus_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); let kind = DatasetKind(crate::internal_api::params::DatasetKind::Crucible); let dataset_ids: Vec = (0..6).map(|_| Uuid::new_v4()).collect(); for id in &dataset_ids { - let dataset = Dataset::new(*id, zpool_id, addr, kind.clone()); + let dataset = Dataset::new(*id, zpool_id, bogus_addr, kind.clone()); + println!("test: inserting {:#?}", dataset); datastore.dataset_upsert(dataset).await.unwrap(); } @@ -2233,12 +2293,16 @@ mod test { snapshot_id: None, size: ByteCount::from_mebibytes_u32(500), }; - let dataset_and_regions = datastore.region_allocate(&disk_create_params).await.unwrap(); + + let disk1_id = Uuid::new_v4(); + let dataset_and_regions = datastore.region_allocate(disk1_id, &disk_create_params).await.unwrap(); // Verify the allocation we just performed. assert_eq!(3, dataset_and_regions.len()); // TODO: verify allocated regions make sense + let disk2_id = Uuid::new_v4(); + let dataset_and_regions = datastore.region_allocate(disk2_id, &disk_create_params).await.unwrap(); let _ = db.cleanup().await; } diff --git a/nexus/src/db/model.rs b/nexus/src/db/model.rs index 5afcce90a9d..602a5753fbd 100644 --- a/nexus/src/db/model.rs +++ b/nexus/src/db/model.rs @@ -484,7 +484,7 @@ impl DatastoreCollection for Zpool { } impl_enum_type!( - #[derive(SqlType, Debug)] + #[derive(SqlType, Debug, QueryId)] #[postgres(type_name = "dataset_kind", type_schema = "public")] pub struct DatasetKindEnum; @@ -582,6 +582,19 @@ pub struct Region { extent_count: i64, } +impl Region { + pub fn new(dataset_id: Uuid, disk_id: Uuid, block_size: i64, extent_size: i64, extent_count: i64) -> Self { + Self { + identity: RegionIdentity::new(Uuid::new_v4()), + dataset_id, + disk_id, + block_size, + extent_size, + extent_count, + } + } +} + /// Describes an organization within the database. #[derive(Queryable, Insertable, Debug, Resource, Selectable)] #[table_name = "organization"] @@ -645,7 +658,7 @@ impl Project { pub fn new(organization_id: Uuid, params: params::ProjectCreate) -> Self { Self { identity: ProjectIdentity::new(Uuid::new_v4(), params.identity), - organization_id: organization_id, + organization_id, } } } diff --git a/nexus/src/db/schema.rs b/nexus/src/db/schema.rs index 4dcbcf5675f..5d62efefb17 100644 --- a/nexus/src/db/schema.rs +++ b/nexus/src/db/schema.rs @@ -176,6 +176,9 @@ table! { } } +// TODO: Free/allocated space here? How do we know we're okay to alloc? +// +// Maybe just "total size" of dataset, and we can figure out the rest? table! { dataset (id) { id -> Uuid, diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index 354d385b006..c57c0cfdaba 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -323,6 +323,7 @@ async fn sdc_alloc_regions( ) -> Result, ActionError> { let osagactx = sagactx.user_data(); let params = sagactx.saga_params(); + let disk_id = sagactx.lookup::("disk_id")?; // Ensure the disk is backed by appropriate regions. // // This allocates regions in the database, but the disk state is still @@ -330,7 +331,7 @@ async fn sdc_alloc_regions( // allocate the necessary regions before we can mark the disk as "ready to // be used". let datasets_and_regions = osagactx.datastore() - .region_allocate(¶ms.create_params) + .region_allocate(disk_id, ¶ms.create_params) .await .map_err(ActionError::action_failed)?; Ok(datasets_and_regions) From 0276b731e8079d082c1484d2fb41cc910f9bec99 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 14 Dec 2021 12:01:00 -0500 Subject: [PATCH 19/50] Unit test OK --- Cargo.lock | 27 ---------- nexus/Cargo.toml | 3 +- nexus/src/db/datastore.rs | 110 +++++++++++++++++++++----------------- nexus/src/db/model.rs | 6 +++ 4 files changed, 67 insertions(+), 79 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aec3e5b5b40..8d2124152fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -175,17 +175,6 @@ dependencies = [ "num_enum", ] -[[package]] -name = "bigdecimal" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aaf33151a6429fe9211d1b276eafdf70cdff28b071e76c0b0e1503221ea3744" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "bincode" version = "1.3.3" @@ -725,7 +714,6 @@ name = "diesel" version = "2.0.0" source = "git+https://github.com/diesel-rs/diesel?rev=ce77c382#ce77c382d2836f6b385225991cf58cb2d2dd65d6" dependencies = [ - "bigdecimal", "bitflags", "byteorder", "chrono", @@ -733,9 +721,6 @@ dependencies = [ "ipnetwork", "itoa", "libc", - "num-bigint", - "num-integer", - "num-traits", "pq-sys", "r2d2", "serde_json", @@ -1795,17 +1780,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "num-bigint" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - [[package]] name = "num-integer" version = "0.1.44" @@ -1901,7 +1875,6 @@ dependencies = [ "async-bb8-diesel", "async-trait", "bb8", - "bigdecimal", "chrono", "cookie", "criterion", diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index 9dd206924a6..89b3a3e38a6 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -12,10 +12,9 @@ anyhow = "1.0" async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", rev = "22c26ef" } async-trait = "0.1.51" bb8 = "0.7.1" -bigdecimal = "0.3" cookie = "0.15" # Tracking pending 2.0 version. -diesel = { git = "https://github.com/diesel-rs/diesel", rev = "ce77c382", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "numeric", "uuid"] } +diesel = { git = "https://github.com/diesel-rs/diesel", rev = "ce77c382", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] } futures = "0.3.18" hex = "0.4.3" http = "0.2.5" diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index 9207324c6c3..9bf65995ee1 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -34,7 +34,6 @@ use crate::authz; use crate::context::OpContext; use crate::external_api::params; use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl, ConnectionManager}; -use bigdecimal::ToPrimitive; use chrono::Utc; use diesel::prelude::*; use diesel::upsert::excluded; @@ -243,9 +242,7 @@ impl DataStore { use db::schema::region::dsl as region_dsl; use db::schema::dataset::dsl as dataset_dsl; - println!("region_allocate: Allocating region: {:#?}", params); - - // Allocation Policy + // ALLOCATION POLICY // // NOTE: This policy can - and should! - be changed. // It is currently acting as a placeholder, showing a feasible @@ -253,53 +250,35 @@ impl DataStore { // // This policy allocates regions to distinct Crucible datasets, // favoring datasets with the smallest existing (summed) region - // sizes. - // + // sizes. Basically, "pick the datasets with the smallest load first". // - // - let datasets: Vec<(Dataset, u64)> = dataset_dsl::dataset + // Longer-term, we should consider: + // - Storage size + remaining free space + // - Sled placement of datasets + // - What sort of loads we'd like to create (even split across all disks + // may not be preferable, especially if maintenance is expected) + let datasets: Vec = dataset_dsl::dataset // First, we look for valid datasets (non-deleted crucible datasets). .filter(dataset_dsl::time_deleted.is_null()) .filter(dataset_dsl::kind.eq(DatasetKind(crate::internal_api::params::DatasetKind::Crucible))) // Next, observe all the regions allocated to each dataset, and // determine how much space they're using. // - // NOTE: We *could* store "free/allocated" space per-dataset, and - // work hard to keep them up-to-date, rather than trying to - // recompute this. + // NOTE: We could store "free/allocated" space per-dataset, and keep + // them up-to-date, rather than trying to recompute this. .left_outer_join( region_dsl::region.on( dataset_dsl::id.eq(region_dsl::dataset_id) ) ) .group_by(dataset_dsl::id) - .select( - ( - Dataset::as_select(), - diesel::dsl::sum(region_dsl::extent_count * region_dsl::extent_size).nullable() - ) - ) + .select(Dataset::as_select()) .order(diesel::dsl::sum(region_dsl::extent_size * region_dsl::extent_count).asc()) - .get_results_async::<(Dataset, Option)>(self.pool()) + .get_results_async::(self.pool()) .await .map_err(|e| { public_error_from_diesel_pool_shouldnt_fail(e) - })? - .into_iter() - .map(|(dataset, total_allocated)| { - // TODO: If there aren't any regions, zero is a reasonable - // default. But if the size * count is too big, we probably - // want safer handling here. - // - // Do we need to convert to u64 if this is internal? - (dataset, total_allocated.map_or(0, |value| value.to_u64().unwrap())) - }).collect(); - - println!("region_allocate: Observed datasets: {:#?}", datasets); - - // TODO: We don't actually need to return the allc'd space right now... - // maybe we should tho to compare it with a total size or something. - let datasets: Vec = datasets.into_iter().map(|(d, _)| d).collect(); + })?; // TODO: magic num let threshold = 3; @@ -2143,6 +2122,7 @@ mod test { use chrono::{Duration, Utc}; use omicron_common::api::external::{ByteCount, Name, Error, IdentityMetadataCreateParams}; use omicron_test_utils::dev; + use std::collections::HashSet; use std::sync::Arc; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use uuid::Uuid; @@ -2260,6 +2240,21 @@ mod test { zpool_id } + fn create_test_disk_create_params(name: &str, size: ByteCount) -> params::DiskCreate { + params::DiskCreate { + identity: IdentityMetadataCreateParams { + name: Name::try_from(name.to_string()).unwrap(), + description: name.to_string(), + }, + snapshot_id: None, + size, + } + } + + // TODO: Test region allocation when not enough datasets exist (below + // threshold) + // TODO: Test region allocation when running out of space. + #[tokio::test] async fn test_region_allocation() { let logctx = dev::test_setup_log("test_region_allocation"); @@ -2275,34 +2270,49 @@ mod test { let zpool_id = create_test_zpool(&datastore, sled_id).await; // ... and datasets within that zpool. + let dataset_count = 6; let bogus_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); let kind = DatasetKind(crate::internal_api::params::DatasetKind::Crucible); - let dataset_ids: Vec = (0..6).map(|_| Uuid::new_v4()).collect(); + let dataset_ids: Vec = (0..dataset_count).map(|_| Uuid::new_v4()).collect(); for id in &dataset_ids { let dataset = Dataset::new(*id, zpool_id, bogus_addr, kind.clone()); - println!("test: inserting {:#?}", dataset); datastore.dataset_upsert(dataset).await.unwrap(); } - // Allocate regions from the datasets. - let disk_create_params = params::DiskCreate { - identity: IdentityMetadataCreateParams { - name: Name::try_from("disk1".to_string()).unwrap(), - description: "my-disk".to_string(), - }, - snapshot_id: None, - size: ByteCount::from_mebibytes_u32(500), - }; - + // Allocate regions from the datasets for this disk. + let params = create_test_disk_create_params("disk1", ByteCount::from_mebibytes_u32(500)); let disk1_id = Uuid::new_v4(); - let dataset_and_regions = datastore.region_allocate(disk1_id, &disk_create_params).await.unwrap(); + let dataset_and_regions = datastore.region_allocate(disk1_id, ¶ms).await.unwrap(); - // Verify the allocation we just performed. + // Verify the allocation. assert_eq!(3, dataset_and_regions.len()); - // TODO: verify allocated regions make sense + let mut disk1_datasets = HashSet::new(); + for (dataset, region) in dataset_and_regions { + assert!(disk1_datasets.insert(dataset.id())); + assert_eq!(disk1_id, region.disk_id()); + assert_eq!(params.block_size(), region.block_size()); + assert_eq!(params.extent_size(), region.extent_size()); + assert_eq!(params.extent_count(), region.extent_count()); + } + // Allocate regions for a second disk. Observe that we allocate from + // the three previously unused datasets. + let params = create_test_disk_create_params("disk2", ByteCount::from_mebibytes_u32(500)); let disk2_id = Uuid::new_v4(); - let dataset_and_regions = datastore.region_allocate(disk2_id, &disk_create_params).await.unwrap(); + let dataset_and_regions = datastore.region_allocate(disk2_id, ¶ms).await.unwrap(); + assert_eq!(3, dataset_and_regions.len()); + let mut disk2_datasets = HashSet::new(); + for (dataset, region) in dataset_and_regions { + assert!(disk2_datasets.insert(dataset.id())); + assert_eq!(disk2_id, region.disk_id()); + assert_eq!(params.block_size(), region.block_size()); + assert_eq!(params.extent_size(), region.extent_size()); + assert_eq!(params.extent_count(), region.extent_count()); + } + + // Double-check that the datasets used for the first disk weren't + // used when allocating the second disk. + assert_eq!(0, disk1_datasets.intersection(&disk2_datasets).count()); let _ = db.cleanup().await; } diff --git a/nexus/src/db/model.rs b/nexus/src/db/model.rs index 602a5753fbd..807aec6c48a 100644 --- a/nexus/src/db/model.rs +++ b/nexus/src/db/model.rs @@ -593,6 +593,12 @@ impl Region { extent_count, } } + + pub fn disk_id(&self) -> Uuid { self.disk_id } + pub fn dataset_id(&self) -> Uuid { self.dataset_id } + pub fn block_size(&self) -> u64 { self.block_size as u64 } + pub fn extent_size(&self) -> u64 { self.extent_size as u64 } + pub fn extent_count(&self) -> u64 { self.extent_count as u64 } } /// Describes an organization within the database. From 39aa5b9a2a4c894489d521758e1506efedc5b640 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 14 Dec 2021 12:21:22 -0500 Subject: [PATCH 20/50] Add 'below threshold' test --- nexus/src/db/datastore.rs | 156 ++++++++++++++++++++++++++++---------- nexus/src/db/model.rs | 89 ++++++++++++++++++---- nexus/src/nexus.rs | 8 +- nexus/src/sagas.rs | 9 ++- 4 files changed, 201 insertions(+), 61 deletions(-) diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index 9bf65995ee1..5a175ec3675 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -62,10 +62,10 @@ use crate::db::{ public_error_from_diesel_pool_shouldnt_fail, }, model::{ - ConsoleSession, Dataset, DatasetKind, Disk, DiskAttachment, DiskRuntimeState, - Generation, Instance, InstanceRuntimeState, Name, Organization, - OrganizationUpdate, OximeterInfo, ProducerEndpoint, Project, - ProjectUpdate, Region, RouterRoute, RouterRouteUpdate, Sled, + ConsoleSession, Dataset, DatasetKind, Disk, DiskAttachment, + DiskRuntimeState, Generation, Instance, InstanceRuntimeState, Name, + Organization, OrganizationUpdate, OximeterInfo, ProducerEndpoint, + Project, ProjectUpdate, Region, RouterRoute, RouterRouteUpdate, Sled, UserBuiltin, Vpc, VpcFirewallRule, VpcRouter, VpcRouterUpdate, VpcSubnet, VpcSubnetUpdate, VpcUpdate, Zpool, }, @@ -73,6 +73,9 @@ use crate::db::{ update_and_check::{UpdateAndCheck, UpdateStatus}, }; +// Number of unique datasets required to back a region. +const REGION_REDUNDANCY_THRESHOLD: usize = 3; + pub struct DataStore { pool: Arc, } @@ -239,8 +242,8 @@ impl DataStore { disk_id: Uuid, params: ¶ms::DiskCreate, ) -> Result, Error> { - use db::schema::region::dsl as region_dsl; use db::schema::dataset::dsl as dataset_dsl; + use db::schema::region::dsl as region_dsl; // ALLOCATION POLICY // @@ -260,34 +263,39 @@ impl DataStore { let datasets: Vec = dataset_dsl::dataset // First, we look for valid datasets (non-deleted crucible datasets). .filter(dataset_dsl::time_deleted.is_null()) - .filter(dataset_dsl::kind.eq(DatasetKind(crate::internal_api::params::DatasetKind::Crucible))) + .filter(dataset_dsl::kind.eq(DatasetKind( + crate::internal_api::params::DatasetKind::Crucible, + ))) // Next, observe all the regions allocated to each dataset, and // determine how much space they're using. // // NOTE: We could store "free/allocated" space per-dataset, and keep // them up-to-date, rather than trying to recompute this. .left_outer_join( - region_dsl::region.on( - dataset_dsl::id.eq(region_dsl::dataset_id) - ) + region_dsl::region + .on(dataset_dsl::id.eq(region_dsl::dataset_id)), ) .group_by(dataset_dsl::id) .select(Dataset::as_select()) - .order(diesel::dsl::sum(region_dsl::extent_size * region_dsl::extent_count).asc()) + .order( + diesel::dsl::sum( + region_dsl::extent_size * region_dsl::extent_count, + ) + .asc(), + ) .get_results_async::(self.pool()) .await - .map_err(|e| { - public_error_from_diesel_pool_shouldnt_fail(e) - })?; + .map_err(|e| public_error_from_diesel_pool_shouldnt_fail(e))?; - // TODO: magic num - let threshold = 3; - if datasets.len() < threshold { - return Err(Error::internal_error("Not enough datasets for replicated allocation")); + if datasets.len() < REGION_REDUNDANCY_THRESHOLD { + return Err(Error::internal_error( + "Not enough datasets for replicated allocation", + )); } - let source_datasets = &datasets[0..threshold]; - let regions: Vec = source_datasets.iter() + let source_datasets = &datasets[0..REGION_REDUNDANCY_THRESHOLD]; + let regions: Vec = source_datasets + .iter() .map(|dataset| { Region::new( dataset.id(), @@ -304,12 +312,14 @@ impl DataStore { .returning(Region::as_returning()) .get_results_async(self.pool()) .await - .map_err(|e| { - public_error_from_diesel_pool_shouldnt_fail(e) - })?; + .map_err(|e| public_error_from_diesel_pool_shouldnt_fail(e))?; // TODO: also, make this concurrency-safe. Txns? - Ok(source_datasets.into_iter().map(|d| d.clone()).zip(regions).collect()) + Ok(source_datasets + .into_iter() + .map(|d| d.clone()) + .zip(regions) + .collect()) } /// Create a organization @@ -2120,11 +2130,13 @@ mod test { use crate::db::model::{ConsoleSession, Organization, Project}; use crate::external_api::params; use chrono::{Duration, Utc}; - use omicron_common::api::external::{ByteCount, Name, Error, IdentityMetadataCreateParams}; + use omicron_common::api::external::{ + ByteCount, Error, IdentityMetadataCreateParams, Name, + }; use omicron_test_utils::dev; use std::collections::HashSet; - use std::sync::Arc; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + use std::sync::Arc; use uuid::Uuid; #[tokio::test] @@ -2223,7 +2235,8 @@ mod test { // Creates a test sled, returns its UUID. async fn create_test_sled(datastore: &DataStore) -> Uuid { - let bogus_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); + let bogus_addr = + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); let sled_id = Uuid::new_v4(); let sled = Sled::new(sled_id, bogus_addr.clone()); datastore.sled_upsert(sled).await.unwrap(); @@ -2233,14 +2246,21 @@ mod test { // Creates a test zpool, returns its UUID. async fn create_test_zpool(datastore: &DataStore, sled_id: Uuid) -> Uuid { let zpool_id = Uuid::new_v4(); - let zpool = Zpool::new(zpool_id, sled_id, &crate::internal_api::params::ZpoolPutRequest { - size: ByteCount::from_gibibytes_u32(100), - }); + let zpool = Zpool::new( + zpool_id, + sled_id, + &crate::internal_api::params::ZpoolPutRequest { + size: ByteCount::from_gibibytes_u32(100), + }, + ); datastore.zpool_upsert(zpool).await.unwrap(); zpool_id } - fn create_test_disk_create_params(name: &str, size: ByteCount) -> params::DiskCreate { + fn create_test_disk_create_params( + name: &str, + size: ByteCount, + ) -> params::DiskCreate { params::DiskCreate { identity: IdentityMetadataCreateParams { name: Name::try_from(name.to_string()).unwrap(), @@ -2251,8 +2271,6 @@ mod test { } } - // TODO: Test region allocation when not enough datasets exist (below - // threshold) // TODO: Test region allocation when running out of space. #[tokio::test] @@ -2270,22 +2288,29 @@ mod test { let zpool_id = create_test_zpool(&datastore, sled_id).await; // ... and datasets within that zpool. - let dataset_count = 6; - let bogus_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); - let kind = DatasetKind(crate::internal_api::params::DatasetKind::Crucible); - let dataset_ids: Vec = (0..dataset_count).map(|_| Uuid::new_v4()).collect(); + let dataset_count = REGION_REDUNDANCY_THRESHOLD * 2; + let bogus_addr = + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); + let kind = + DatasetKind(crate::internal_api::params::DatasetKind::Crucible); + let dataset_ids: Vec = + (0..dataset_count).map(|_| Uuid::new_v4()).collect(); for id in &dataset_ids { let dataset = Dataset::new(*id, zpool_id, bogus_addr, kind.clone()); datastore.dataset_upsert(dataset).await.unwrap(); } // Allocate regions from the datasets for this disk. - let params = create_test_disk_create_params("disk1", ByteCount::from_mebibytes_u32(500)); + let params = create_test_disk_create_params( + "disk1", + ByteCount::from_mebibytes_u32(500), + ); let disk1_id = Uuid::new_v4(); - let dataset_and_regions = datastore.region_allocate(disk1_id, ¶ms).await.unwrap(); + let dataset_and_regions = + datastore.region_allocate(disk1_id, ¶ms).await.unwrap(); // Verify the allocation. - assert_eq!(3, dataset_and_regions.len()); + assert_eq!(REGION_REDUNDANCY_THRESHOLD, dataset_and_regions.len()); let mut disk1_datasets = HashSet::new(); for (dataset, region) in dataset_and_regions { assert!(disk1_datasets.insert(dataset.id())); @@ -2297,10 +2322,14 @@ mod test { // Allocate regions for a second disk. Observe that we allocate from // the three previously unused datasets. - let params = create_test_disk_create_params("disk2", ByteCount::from_mebibytes_u32(500)); + let params = create_test_disk_create_params( + "disk2", + ByteCount::from_mebibytes_u32(500), + ); let disk2_id = Uuid::new_v4(); - let dataset_and_regions = datastore.region_allocate(disk2_id, ¶ms).await.unwrap(); - assert_eq!(3, dataset_and_regions.len()); + let dataset_and_regions = + datastore.region_allocate(disk2_id, ¶ms).await.unwrap(); + assert_eq!(REGION_REDUNDANCY_THRESHOLD, dataset_and_regions.len()); let mut disk2_datasets = HashSet::new(); for (dataset, region) in dataset_and_regions { assert!(disk2_datasets.insert(dataset.id())); @@ -2316,4 +2345,47 @@ mod test { let _ = db.cleanup().await; } + + #[tokio::test] + async fn test_region_allocation_not_enough_datasets() { + let logctx = + dev::test_setup_log("test_region_allocation_not_enough_datasets"); + let mut db = dev::test_setup_database(&logctx.log).await; + let cfg = db::Config { url: db.pg_config().clone() }; + let pool = db::Pool::new(&cfg); + let datastore = DataStore::new(Arc::new(pool)); + + // Create a sled... + let sled_id = create_test_sled(&datastore).await; + + // ... and a zpool within that sled... + let zpool_id = create_test_zpool(&datastore, sled_id).await; + + // ... and datasets within that zpool. + let dataset_count = REGION_REDUNDANCY_THRESHOLD - 1; + let bogus_addr = + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); + let kind = + DatasetKind(crate::internal_api::params::DatasetKind::Crucible); + let dataset_ids: Vec = + (0..dataset_count).map(|_| Uuid::new_v4()).collect(); + for id in &dataset_ids { + let dataset = Dataset::new(*id, zpool_id, bogus_addr, kind.clone()); + datastore.dataset_upsert(dataset).await.unwrap(); + } + + // Allocate regions from the datasets for this disk. + let params = create_test_disk_create_params( + "disk1", + ByteCount::from_mebibytes_u32(500), + ); + let disk1_id = Uuid::new_v4(); + let err = + datastore.region_allocate(disk1_id, ¶ms).await.unwrap_err(); + assert!(err + .to_string() + .contains("Not enough datasets for replicated allocation")); + + let _ = db.cleanup().await; + } } diff --git a/nexus/src/db/model.rs b/nexus/src/db/model.rs index 807aec6c48a..aa9bf616c53 100644 --- a/nexus/src/db/model.rs +++ b/nexus/src/db/model.rs @@ -142,7 +142,9 @@ where } } -#[derive(Copy, Clone, Debug, AsExpression, FromSqlRow, Serialize, Deserialize)] +#[derive( + Copy, Clone, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, +)] #[sql_type = "sql_types::BigInt"] pub struct ByteCount(pub external::ByteCount); @@ -175,8 +177,17 @@ where } #[derive( - Copy, Clone, Debug, Eq, Ord, PartialEq, PartialOrd, AsExpression, FromSqlRow, - Serialize, Deserialize + Copy, + Clone, + Debug, + Eq, + Ord, + PartialEq, + PartialOrd, + AsExpression, + FromSqlRow, + Serialize, + Deserialize, )] #[sql_type = "sql_types::BigInt"] #[repr(transparent)] @@ -508,7 +519,16 @@ impl From for DatasetKind { /// /// A dataset represents a portion of a Zpool, which is then made /// available to a service on the Sled. -#[derive(Queryable, Insertable, Debug, Clone, Selectable, Asset, Deserialize, Serialize)] +#[derive( + Queryable, + Insertable, + Debug, + Clone, + Selectable, + Asset, + Deserialize, + Serialize, +)] #[table_name = "dataset"] pub struct Dataset { #[diesel(embed)] @@ -568,7 +588,16 @@ impl DatastoreCollection for Disk { /// /// A region represents a portion of a Crucible Downstairs dataset /// allocated within a volume. -#[derive(Queryable, Insertable, Debug, Clone, Selectable, Asset, Serialize, Deserialize)] +#[derive( + Queryable, + Insertable, + Debug, + Clone, + Selectable, + Asset, + Serialize, + Deserialize, +)] #[table_name = "region"] pub struct Region { #[diesel(embed)] @@ -583,7 +612,13 @@ pub struct Region { } impl Region { - pub fn new(dataset_id: Uuid, disk_id: Uuid, block_size: i64, extent_size: i64, extent_count: i64) -> Self { + pub fn new( + dataset_id: Uuid, + disk_id: Uuid, + block_size: i64, + extent_size: i64, + extent_count: i64, + ) -> Self { Self { identity: RegionIdentity::new(Uuid::new_v4()), dataset_id, @@ -594,11 +629,21 @@ impl Region { } } - pub fn disk_id(&self) -> Uuid { self.disk_id } - pub fn dataset_id(&self) -> Uuid { self.dataset_id } - pub fn block_size(&self) -> u64 { self.block_size as u64 } - pub fn extent_size(&self) -> u64 { self.extent_size as u64 } - pub fn extent_count(&self) -> u64 { self.extent_count as u64 } + pub fn disk_id(&self) -> Uuid { + self.disk_id + } + pub fn dataset_id(&self) -> Uuid { + self.dataset_id + } + pub fn block_size(&self) -> u64 { + self.block_size as u64 + } + pub fn extent_size(&self) -> u64 { + self.extent_size as u64 + } + pub fn extent_count(&self) -> u64 { + self.extent_count as u64 + } } /// Describes an organization within the database. @@ -850,7 +895,16 @@ where } /// A Disk (network block device). -#[derive(Queryable, Insertable, Clone, Debug, Selectable, Resource, Serialize, Deserialize)] +#[derive( + Queryable, + Insertable, + Clone, + Debug, + Selectable, + Resource, + Serialize, + Deserialize, +)] #[table_name = "disk"] pub struct Disk { #[diesel(embed)] @@ -930,7 +984,16 @@ impl Into for Disk { } } -#[derive(AsChangeset, Clone, Debug, Queryable, Insertable, Selectable, Serialize, Deserialize)] +#[derive( + AsChangeset, + Clone, + Debug, + Queryable, + Insertable, + Selectable, + Serialize, + Deserialize, +)] #[table_name = "disk"] // When "attach_instance_id" is set to None, we'd like to // clear it from the DB, rather than ignore the update. diff --git a/nexus/src/nexus.rs b/nexus/src/nexus.rs index 47017edd047..6fc5cff5138 100644 --- a/nexus/src/nexus.rs +++ b/nexus/src/nexus.rs @@ -678,9 +678,11 @@ impl Nexus { saga_params, ) .await?; - let disk_created = saga_outputs.lookup_output::("created_disk").map_err(|e| { - Error::InternalError { internal_message: e.to_string() } - })?; + let disk_created = saga_outputs + .lookup_output::("created_disk") + .map_err(|e| Error::InternalError { + internal_message: e.to_string(), + })?; Ok(disk_created) } diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index c57c0cfdaba..ddc10e98789 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -311,7 +311,8 @@ async fn sdc_create_disk_record( params.create_params.clone(), db::model::DiskRuntimeState::new(), ); - let disk_created = osagactx.datastore() + let disk_created = osagactx + .datastore() .project_create_disk(disk) .await .map_err(ActionError::action_failed)?; @@ -330,7 +331,8 @@ async fn sdc_alloc_regions( // "creating" - the respective Crucible Agents must be instructed to // allocate the necessary regions before we can mark the disk as "ready to // be used". - let datasets_and_regions = osagactx.datastore() + let datasets_and_regions = osagactx + .datastore() .region_allocate(disk_id, ¶ms.create_params) .await .map_err(ActionError::action_failed)?; @@ -357,7 +359,8 @@ async fn sdc_finalize_disk_record( let disk_id = sagactx.lookup::("disk_id")?; let disk_created = sagactx.lookup::("disk_created")?; - osagactx.datastore() + osagactx + .datastore() .disk_update_runtime(&disk_id, &disk_created.runtime().detach()) .await .map_err(ActionError::action_failed)?; From 791138869ad9315702d6e5cf644d9b9321674d67 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 15 Dec 2021 13:54:40 -0500 Subject: [PATCH 21/50] First steps towards txn --- nexus/src/db/datastore.rs | 123 ++++++++++++++++++++------------------ 1 file changed, 64 insertions(+), 59 deletions(-) diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index 5a175ec3675..1fea601999f 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -260,66 +260,71 @@ impl DataStore { // - Sled placement of datasets // - What sort of loads we'd like to create (even split across all disks // may not be preferable, especially if maintenance is expected) - let datasets: Vec = dataset_dsl::dataset - // First, we look for valid datasets (non-deleted crucible datasets). - .filter(dataset_dsl::time_deleted.is_null()) - .filter(dataset_dsl::kind.eq(DatasetKind( - crate::internal_api::params::DatasetKind::Crucible, - ))) - // Next, observe all the regions allocated to each dataset, and - // determine how much space they're using. - // - // NOTE: We could store "free/allocated" space per-dataset, and keep - // them up-to-date, rather than trying to recompute this. - .left_outer_join( - region_dsl::region - .on(dataset_dsl::id.eq(region_dsl::dataset_id)), - ) - .group_by(dataset_dsl::id) - .select(Dataset::as_select()) - .order( - diesel::dsl::sum( - region_dsl::extent_size * region_dsl::extent_count, - ) - .asc(), - ) - .get_results_async::(self.pool()) - .await - .map_err(|e| public_error_from_diesel_pool_shouldnt_fail(e))?; - - if datasets.len() < REGION_REDUNDANCY_THRESHOLD { - return Err(Error::internal_error( - "Not enough datasets for replicated allocation", - )); - } - - let source_datasets = &datasets[0..REGION_REDUNDANCY_THRESHOLD]; - let regions: Vec = source_datasets - .iter() - .map(|dataset| { - Region::new( - dataset.id(), - disk_id, - params.block_size().try_into().unwrap(), - params.extent_size().try_into().unwrap(), - params.extent_count().try_into().unwrap(), - ) - }) - .collect(); - - let regions = diesel::insert_into(region_dsl::region) - .values(regions) - .returning(Region::as_returning()) - .get_results_async(self.pool()) - .await - .map_err(|e| public_error_from_diesel_pool_shouldnt_fail(e))?; + let params: params::DiskCreate = params.clone(); + self.pool().transaction(move |conn| { + let datasets: Vec = dataset_dsl::dataset + // First, we look for valid datasets (non-deleted crucible datasets). + .filter(dataset_dsl::time_deleted.is_null()) + .filter(dataset_dsl::kind.eq(DatasetKind( + crate::internal_api::params::DatasetKind::Crucible, + ))) + // Next, observe all the regions allocated to each dataset, and + // determine how much space they're using. + // + // NOTE: We could store "free/allocated" space per-dataset, and keep + // them up-to-date, rather than trying to recompute this. + .left_outer_join( + region_dsl::region + .on(dataset_dsl::id.eq(region_dsl::dataset_id)), + ) + .group_by(dataset_dsl::id) + .select(Dataset::as_select()) + .order( + diesel::dsl::sum( + region_dsl::extent_size * region_dsl::extent_count, + ) + .asc(), + ) + .get_results::(conn)?; +// .map_err(|e| Error::internal_error(&format!("Database error: {:#}", e)))?; + + if datasets.len() < REGION_REDUNDANCY_THRESHOLD { + // TODO: Want a better error type here... + return Err(diesel::result::Error::NotFound); +// return Err(Error::internal_error( +// "Not enough datasets for replicated allocation", +// )); + } - // TODO: also, make this concurrency-safe. Txns? - Ok(source_datasets - .into_iter() - .map(|d| d.clone()) - .zip(regions) - .collect()) + // Create identical regions on each of the following datasets. + let source_datasets = &datasets[0..REGION_REDUNDANCY_THRESHOLD]; + let regions: Vec = source_datasets + .iter() + .map(|dataset| { + Region::new( + dataset.id(), + disk_id, + params.block_size().try_into().unwrap(), + params.extent_size().try_into().unwrap(), + params.extent_count().try_into().unwrap(), + ) + }) + .collect(); + let regions = diesel::insert_into(region_dsl::region) + .values(regions) + .returning(Region::as_returning()) + .get_results(conn)?; +// .map_err(|e| Error::internal_error(&format!("Database error: {:#}", e)))?; + + // TODO: also, make this concurrency-safe. Txns? + + // Return the regions with the datasets to which they were allocated. + Ok(source_datasets + .into_iter() + .map(|d| d.clone()) + .zip(regions) + .collect()) + }).await.map_err(|e| Error::internal_error(&format!("Database error: {:#?}", e))) } /// Create a organization From ab2bf3aa87c5670f2952988f7c2050818620f3d8 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 21 Dec 2021 16:39:28 -0500 Subject: [PATCH 22/50] Txns now compiling w/better errors, woo --- nexus/src/db/datastore.rs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index 3bc3f3234a5..a6bef838d3a 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -266,6 +266,11 @@ impl DataStore { // - Sled placement of datasets // - What sort of loads we'd like to create (even split across all disks // may not be preferable, especially if maintenance is expected) + #[derive(Debug)] + enum RegionAllocateError { + NotEnoughDatasets(usize), + } + type TxnError = TransactionError; let params: params::DiskCreate = params.clone(); self.pool().transaction(move |conn| { let datasets: Vec = dataset_dsl::dataset @@ -292,14 +297,9 @@ impl DataStore { .asc(), ) .get_results::(conn)?; -// .map_err(|e| Error::internal_error(&format!("Database error: {:#}", e)))?; if datasets.len() < REGION_REDUNDANCY_THRESHOLD { - // TODO: Want a better error type here... - return Err(diesel::result::Error::NotFound); -// return Err(Error::internal_error( -// "Not enough datasets for replicated allocation", -// )); + return Err(TxnError::CustomError(RegionAllocateError::NotEnoughDatasets(datasets.len()))); } // Create identical regions on each of the following datasets. @@ -320,9 +320,6 @@ impl DataStore { .values(regions) .returning(Region::as_returning()) .get_results(conn)?; -// .map_err(|e| Error::internal_error(&format!("Database error: {:#}", e)))?; - - // TODO: also, make this concurrency-safe. Txns? // Return the regions with the datasets to which they were allocated. Ok(source_datasets @@ -330,7 +327,7 @@ impl DataStore { .map(|d| d.clone()) .zip(regions) .collect()) - }).await.map_err(|e| Error::internal_error(&format!("Database error: {:#?}", e))) + }).await.map_err(|e| Error::internal_error(&format!("Transaction error: {:#?}", e))) } /// Create a organization From 1b1510cd7821773cd3db7cd6453d504e7d883568 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 27 Dec 2021 20:23:31 -0500 Subject: [PATCH 23/50] making requests to crucible agent --- nexus/Cargo.toml | 1 + nexus/src/sagas.rs | 64 +++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index 84903d387c1..e6cb476778b 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -13,6 +13,7 @@ async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", async-trait = "0.1.51" bb8 = "0.7.1" cookie = "0.15" +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "de022b8a" } # Tracking pending 2.0 version. diesel = { git = "https://github.com/diesel-rs/diesel", rev = "ce77c382", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] } futures = "0.3.18" diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index df27962c472..d023d1077b8 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -13,10 +13,15 @@ * easier it will be to test, version, and update in deployed systems. */ +use anyhow::anyhow; use crate::db; -use crate::db::identity::Resource; +use crate::db::identity::{Asset, Resource}; use crate::external_api::params; use crate::saga_interface::SagaContext; +use crucible_agent_client::{ + Client as CrucibleAgentClient, + types::{CreateRegion, RegionId, State as RegionState} +}; use chrono::Utc; use lazy_static::lazy_static; use omicron_common::api::external::Generation; @@ -26,6 +31,7 @@ use omicron_common::api::external::Name; use omicron_common::api::external::NetworkInterface; use omicron_common::api::internal::nexus::InstanceRuntimeState; use omicron_common::api::internal::sled_agent::InstanceHardware; +use omicron_common::backoff::{self, BackoffError}; use serde::Deserialize; use serde::Serialize; use std::collections::BTreeMap; @@ -424,16 +430,60 @@ async fn sdc_alloc_regions( Ok(datasets_and_regions) } +async fn allocate_region_from_dataset(dataset: &db::model::Dataset, region: &db::model::Region) + -> Result +{ + let url = format!("http://{}", dataset.address()); + let client = CrucibleAgentClient::new(&url); + + let region_request = CreateRegion { + block_size: region.block_size(), + extent_count: region.extent_count(), + extent_size: region.extent_size(), + // TODO: Can we avoid casting from UUID to string? + // NOTE: This'll require updating the crucible agent client. + id: RegionId(region.id().to_string()), + volume_id: region.disk_id().to_string(), + }; + + let create_region = || async { + let region = client.region_create(®ion_request) + .await + .map_err(|e| { + BackoffError::Permanent(e) + })?; + match region.state { + RegionState::Requested => Err(BackoffError::Transient(anyhow!("Region creation in progress"))), + RegionState::Created => Ok(region), + _ => Err(BackoffError::Permanent(anyhow!("Failed to create region, unexpected state: {:?}", region.state))), + } + }; + + let log_create_failure = |_, delay| { + // TODO: Log pls + eprintln!("Region requested, not yet created. Retrying in {:?}", delay); + }; + + let region = backoff::retry_notify( + backoff::internal_service_policy(), + create_region, + log_create_failure, + ).await.map_err(|e| ActionError::action_failed(e.to_string()))?; + + Ok(region) +} + async fn sdc_regions_ensure( sagactx: ActionContext, ) -> Result<(), ActionError> { - let _osagactx = sagactx.user_data(); - let _params = sagactx.saga_params(); + let datasets_and_regions = sagactx.lookup::>("datasets_and_regions")?; + // TODO: parallelize these requests + for (dataset, region) in &datasets_and_regions { + let _ = allocate_region_from_dataset(dataset, region).await?; + // TODO: Region has a port value, we could store this in the DB? + } - // TODO: Make the calls to crucible agents. - // TODO: Figure out how we're testing this - setup fake endpoints - // in the simulated sled agent, or do something else? - todo!(); + Ok(()) } async fn sdc_finalize_disk_record( From 8bc078c0863bc80a0472f401aef2b977a5f1fcf7 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 27 Dec 2021 20:53:42 -0500 Subject: [PATCH 24/50] Allocations now requested in parallel --- nexus/src/sagas.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index d023d1077b8..95d418a4f56 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -23,6 +23,7 @@ use crucible_agent_client::{ types::{CreateRegion, RegionId, State as RegionState} }; use chrono::Utc; +use futures::StreamExt; use lazy_static::lazy_static; use omicron_common::api::external::Generation; use omicron_common::api::external::IdentityMetadataCreateParams; @@ -477,11 +478,19 @@ async fn sdc_regions_ensure( sagactx: ActionContext, ) -> Result<(), ActionError> { let datasets_and_regions = sagactx.lookup::>("datasets_and_regions")?; - // TODO: parallelize these requests - for (dataset, region) in &datasets_and_regions { - let _ = allocate_region_from_dataset(dataset, region).await?; - // TODO: Region has a port value, we could store this in the DB? - } + let request_count = datasets_and_regions.len(); + futures::stream::iter(datasets_and_regions) + .map(|(dataset, region)| async move { + allocate_region_from_dataset(&dataset, ®ion).await + }) + // Execute the allocation requests concurrently. + .buffer_unordered(request_count) + .collect::>>() + .await + .into_iter() + .collect::, _>>()?; + + // TODO: Region has a port value, we could store this in the DB? Ok(()) } From 285dd04cd3414e16ab171ab381767fe65d56c066 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 27 Dec 2021 20:54:22 -0500 Subject: [PATCH 25/50] fmt --- nexus/src/sagas.rs | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index 95d418a4f56..c6e790ae9e5 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -13,16 +13,16 @@ * easier it will be to test, version, and update in deployed systems. */ -use anyhow::anyhow; use crate::db; use crate::db::identity::{Asset, Resource}; use crate::external_api::params; use crate::saga_interface::SagaContext; +use anyhow::anyhow; +use chrono::Utc; use crucible_agent_client::{ + types::{CreateRegion, RegionId, State as RegionState}, Client as CrucibleAgentClient, - types::{CreateRegion, RegionId, State as RegionState} }; -use chrono::Utc; use futures::StreamExt; use lazy_static::lazy_static; use omicron_common::api::external::Generation; @@ -431,9 +431,10 @@ async fn sdc_alloc_regions( Ok(datasets_and_regions) } -async fn allocate_region_from_dataset(dataset: &db::model::Dataset, region: &db::model::Region) - -> Result -{ +async fn allocate_region_from_dataset( + dataset: &db::model::Dataset, + region: &db::model::Region, +) -> Result { let url = format!("http://{}", dataset.address()); let client = CrucibleAgentClient::new(&url); @@ -448,15 +449,19 @@ async fn allocate_region_from_dataset(dataset: &db::model::Dataset, region: &db: }; let create_region = || async { - let region = client.region_create(®ion_request) + let region = client + .region_create(®ion_request) .await - .map_err(|e| { - BackoffError::Permanent(e) - })?; + .map_err(|e| BackoffError::Permanent(e))?; match region.state { - RegionState::Requested => Err(BackoffError::Transient(anyhow!("Region creation in progress"))), + RegionState::Requested => Err(BackoffError::Transient(anyhow!( + "Region creation in progress" + ))), RegionState::Created => Ok(region), - _ => Err(BackoffError::Permanent(anyhow!("Failed to create region, unexpected state: {:?}", region.state))), + _ => Err(BackoffError::Permanent(anyhow!( + "Failed to create region, unexpected state: {:?}", + region.state + ))), } }; @@ -469,7 +474,9 @@ async fn allocate_region_from_dataset(dataset: &db::model::Dataset, region: &db: backoff::internal_service_policy(), create_region, log_create_failure, - ).await.map_err(|e| ActionError::action_failed(e.to_string()))?; + ) + .await + .map_err(|e| ActionError::action_failed(e.to_string()))?; Ok(region) } @@ -477,7 +484,10 @@ async fn allocate_region_from_dataset(dataset: &db::model::Dataset, region: &db: async fn sdc_regions_ensure( sagactx: ActionContext, ) -> Result<(), ActionError> { - let datasets_and_regions = sagactx.lookup::>("datasets_and_regions")?; + let datasets_and_regions = sagactx + .lookup::>( + "datasets_and_regions", + )?; let request_count = datasets_and_regions.len(); futures::stream::iter(datasets_and_regions) .map(|(dataset, region)| async move { From ee84a12097426d683d2e801b322399eab9b76107 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 27 Dec 2021 21:06:06 -0500 Subject: [PATCH 26/50] Plumbing log --- nexus/src/nexus.rs | 4 ++++ nexus/src/saga_interface.rs | 5 +++++ nexus/src/sagas.rs | 8 +++++--- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/nexus/src/nexus.rs b/nexus/src/nexus.rs index c5e8efbb7ee..08f87d82cd1 100644 --- a/nexus/src/nexus.rs +++ b/nexus/src/nexus.rs @@ -228,6 +228,10 @@ impl Nexus { nexus_arc } + pub fn log(&self) -> &Logger { + &self.log + } + pub async fn wait_for_populate(&self) -> Result<(), anyhow::Error> { let mut my_rx = self.populate_status.clone(); loop { diff --git a/nexus/src/saga_interface.rs b/nexus/src/saga_interface.rs index 6f20243eb44..6793cb7656b 100644 --- a/nexus/src/saga_interface.rs +++ b/nexus/src/saga_interface.rs @@ -11,6 +11,7 @@ use crate::external_api::params; use crate::Nexus; use omicron_common::api::external::Error; use sled_agent_client::Client as SledAgentClient; +use slog::Logger; use std::fmt; use std::sync::Arc; use uuid::Uuid; @@ -35,6 +36,10 @@ impl SagaContext { SagaContext { nexus } } + pub fn log(&self) -> &Logger { + self.nexus.log() + } + /* * TODO-design This interface should not exist. Instead, sleds should be * represented in the database. Reservations will wind up writing to the diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index c6e790ae9e5..ed896b86a4a 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -46,6 +46,7 @@ use steno::SagaTemplate; use steno::SagaTemplateBuilder; use steno::SagaTemplateGeneric; use steno::SagaType; +use slog::Logger; use uuid::Uuid; /* @@ -432,6 +433,7 @@ async fn sdc_alloc_regions( } async fn allocate_region_from_dataset( + log: &Logger, dataset: &db::model::Dataset, region: &db::model::Region, ) -> Result { @@ -466,8 +468,7 @@ async fn allocate_region_from_dataset( }; let log_create_failure = |_, delay| { - // TODO: Log pls - eprintln!("Region requested, not yet created. Retrying in {:?}", delay); + warn!(log, "Region requested, not yet created. Retrying in {:?}", delay); }; let region = backoff::retry_notify( @@ -484,6 +485,7 @@ async fn allocate_region_from_dataset( async fn sdc_regions_ensure( sagactx: ActionContext, ) -> Result<(), ActionError> { + let log = sagactx.user_data().log(); let datasets_and_regions = sagactx .lookup::>( "datasets_and_regions", @@ -491,7 +493,7 @@ async fn sdc_regions_ensure( let request_count = datasets_and_regions.len(); futures::stream::iter(datasets_and_regions) .map(|(dataset, region)| async move { - allocate_region_from_dataset(&dataset, ®ion).await + allocate_region_from_dataset(log, &dataset, ®ion).await }) // Execute the allocation requests concurrently. .buffer_unordered(request_count) From fa1691be8793ce3f8ec42189091af6dcb0a3d4a0 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 3 Jan 2022 11:51:21 -0500 Subject: [PATCH 27/50] More fleshed out simulated server (still just a HashMap) --- Cargo.lock | 15 ++ nexus/test-utils/src/lib.rs | 5 +- sled-agent/Cargo.toml | 3 + .../src/sim/http_entrypoints_storage.rs | 164 +++++++++++++++++ sled-agent/src/sim/mod.rs | 2 + sled-agent/src/sim/sled_agent.rs | 25 ++- sled-agent/src/sim/storage.rs | 167 ++++++++++++++++++ 7 files changed, 377 insertions(+), 4 deletions(-) create mode 100644 sled-agent/src/sim/http_entrypoints_storage.rs create mode 100644 sled-agent/src/sim/storage.rs diff --git a/Cargo.lock b/Cargo.lock index b09460ab051..438b097e324 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -539,6 +539,19 @@ dependencies = [ "xts-mode", ] +[[package]] +name = "crucible-agent-client" +version = "0.0.1" +source = "git+https://github.com/oxidecomputer/crucible?rev=de022b8a#de022b8ae24fb10cec6b024437aabc3acf249e43" +dependencies = [ + "anyhow", + "percent-encoding", + "progenitor", + "reqwest", + "serde", + "serde_json", +] + [[package]] name = "crucible-common" version = "0.0.0" @@ -1923,6 +1936,7 @@ dependencies = [ "chrono", "cookie", "criterion", + "crucible-agent-client", "db-macros", "diesel", "diesel-dtrace", @@ -2009,6 +2023,7 @@ dependencies = [ "bytes", "cfg-if", "chrono", + "crucible-agent-client", "dropshot", "expectorate", "futures", diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index bb33fe24122..cef80372871 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -123,7 +123,7 @@ pub async fn test_setup_with_config( /* Set up a single sled agent. */ let sa_id = Uuid::parse_str(SLED_AGENT_UUID).unwrap(); - let sa = start_sled_agent( + let sled_agent = start_sled_agent( logctx.log.new(o!( "component" => "omicron_sled_agent::sim::Server", "sled_id" => sa_id.to_string(), @@ -159,13 +159,14 @@ pub async fn test_setup_with_config( internal_client: testctx_internal, database, clickhouse, - sled_agent: sa, + sled_agent, oximeter, producer, logctx, } } +// TODO: We probably want to have the ability to expand this config. pub async fn start_sled_agent( log: Logger, nexus_address: SocketAddr, diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 81c3bf7117f..2affd833134 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -11,6 +11,9 @@ bincode = "1.3.3" bytes = "1.1" cfg-if = "1.0" chrono = { version = "0.4", features = [ "serde" ] } +# Simulated sled agent only. +# TODO: Can we avoid having this in the deps list for non-sim SA? +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "de022b8a" } dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] } futures = "0.3.18" ipnetwork = "0.18" diff --git a/sled-agent/src/sim/http_entrypoints_storage.rs b/sled-agent/src/sim/http_entrypoints_storage.rs new file mode 100644 index 00000000000..568a23c257e --- /dev/null +++ b/sled-agent/src/sim/http_entrypoints_storage.rs @@ -0,0 +1,164 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! HTTP entrypoint functions for simulating the storage agent API. + +// use crucible_agent_client::types::{CreateRegion, RegionId}; +use dropshot::{ + endpoint, + ApiDescription, + HttpError, + HttpResponseDeleted, + HttpResponseOk, + Path as TypedPath, + RequestContext, + TypedBody, +}; +use schemars::JsonSchema; +use serde::{Serialize, Deserialize}; +use std::sync::Arc; + +use super::storage::CrucibleData; + +type CrucibleAgentApiDescription = ApiDescription>; + +/** + * Returns a description of the sled agent API + */ +pub fn api() -> CrucibleAgentApiDescription { + fn register_endpoints(api: &mut CrucibleAgentApiDescription) -> Result<(), String> { + api.register(region_list)?; + api.register(region_create)?; + api.register(region_get)?; + api.register(region_delete)?; + Ok(()) + } + + let mut api = CrucibleAgentApiDescription::new(); + if let Err(err) = register_endpoints(&mut api) { + panic!("failed to register entrypoints: {}", err); + } + api +} + +// XXX XXX XXX THIS SUCKS XXX XXX XXX +// +// I need to re-define all structs used in the crucible agent +// API to ensure they have the traits I need. The ones re-exported +// through the client bindings, i.e., crucible_agent_client::types, +// don't implement what I need. +// +// I'd like them to! If we could ensure the generated client +// also implemented e.g. JsonSchema, this might work? +// +// TODO: Try w/RegionId or State first? + +#[derive( + Serialize, + Deserialize, + JsonSchema, + Debug, + PartialEq, + Eq, + Clone, + PartialOrd, + Ord, +)] +pub struct RegionId(pub String); + +#[derive(Serialize, Deserialize, JsonSchema, Debug, PartialEq, Clone)] +#[serde(rename_all = "lowercase")] +pub enum State { + Requested, + Created, + Tombstoned, + Destroyed, + Failed, +} + +#[derive(Serialize, Deserialize, JsonSchema, Debug, PartialEq, Clone)] +pub struct CreateRegion { + pub id: RegionId, + pub volume_id: String, + + pub block_size: u64, + pub extent_size: u64, + pub extent_count: u64, +} + +#[derive(Serialize, Deserialize, JsonSchema, Debug, PartialEq, Clone)] +pub struct Region { + pub id: RegionId, + pub volume_id: String, + + pub block_size: u64, + pub extent_size: u64, + pub extent_count: u64, + + pub port_number: u16, + pub state: State, +} + +#[endpoint { + method = GET, + path = "/crucible/0/regions", +}] +async fn region_list( + rc: Arc>>, +) -> Result>, HttpError> { + let crucible = rc.context(); + Ok(HttpResponseOk(crucible.list().await)) +} + +#[endpoint { + method = POST, + path = "/crucible/0/regions", +}] +async fn region_create( + rc: Arc>>, + body: TypedBody, +) -> Result, HttpError> { + let params = body.into_inner(); + let crucible = rc.context(); + + Ok(HttpResponseOk(crucible.create(params).await)) +} + +#[derive(Deserialize, JsonSchema)] +struct RegionPath { + id: RegionId, +} + +#[endpoint { + method = GET, + path = "/crucible/0/regions/{id}", +}] +async fn region_get( + rc: Arc>>, + path: TypedPath, +) -> Result, HttpError> { + let id = path.into_inner().id; + let crucible = rc.context(); + match crucible.get(id).await { + Some(region) => Ok(HttpResponseOk(region)), + None => Err(HttpError::for_not_found(None, "Region not found".to_string())), + } +} + +#[endpoint { + method = DELETE, + path = "/crucible/0/regions/{id}", +}] +async fn region_delete( + rc: Arc>>, + path: TypedPath, +) -> Result { + let id = path.into_inner().id; + let crucible = rc.context(); + + match crucible.delete(id).await { + Some(_) => Ok(HttpResponseDeleted()), + None => Err(HttpError::for_not_found(None, "Region not found".to_string())), + } +} diff --git a/sled-agent/src/sim/mod.rs b/sled-agent/src/sim/mod.rs index 0cad7673f32..f485b6ef507 100644 --- a/sled-agent/src/sim/mod.rs +++ b/sled-agent/src/sim/mod.rs @@ -10,10 +10,12 @@ mod collection; mod config; mod disk; mod http_entrypoints; +mod http_entrypoints_storage; mod instance; mod server; mod simulatable; mod sled_agent; +mod storage; pub use config::{Config, SimMode}; pub use server::{run_server, Server}; diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index 664d4aef5ea..8d3ea5357c5 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -7,6 +7,7 @@ */ use crate::params::DiskStateRequested; +use futures::lock::Mutex; use nexus_client::Client as NexusClient; use omicron_common::api::external::Error; use omicron_common::api::internal::nexus::DiskRuntimeState; @@ -21,6 +22,7 @@ use super::collection::SimCollection; use super::config::SimMode; use super::disk::SimDisk; use super::instance::SimInstance; +use super::storage::Storage; /** * Simulates management of the control plane on a sled @@ -33,12 +35,13 @@ use super::instance::SimInstance; */ pub struct SledAgent { /** unique id for this server */ - pub id: Uuid, + _id: Uuid, /** collection of simulated instances, indexed by instance uuid */ instances: Arc>, /** collection of simulated disks, indexed by disk uuid */ disks: Arc>, + storage: Mutex, } impl SledAgent { @@ -57,9 +60,10 @@ impl SledAgent { let instance_log = log.new(o!("kind" => "instances")); let disk_log = log.new(o!("kind" => "disks")); + let storage_log = log.new(o!("kind" => "storage")); SledAgent { - id: *id, + _id: *id, instances: Arc::new(SimCollection::new( Arc::clone(&ctlsc), instance_log, @@ -70,6 +74,7 @@ impl SledAgent { disk_log, sim_mode, )), + storage: Mutex::new(Storage::new(storage_log)), } } @@ -111,4 +116,20 @@ impl SledAgent { pub async fn disk_poke(&self, id: Uuid) { self.disks.sim_poke(id).await; } + + /// Adds a Zpool to the simulated sled agent. + pub async fn create_zpool(&self, id: Uuid) { + self.storage.lock() + .await + .insert_zpool(id); + } + + /// Adds a Crucible Dataset within a zpool. + pub async fn create_crucible_dataset(&self, zpool_id: Uuid, dataset_id: Uuid) { + let mut storage = self.storage.lock().await; + let log = storage.log().clone(); + storage + .get_zpool_mut(zpool_id) + .insert_dataset(&log, dataset_id); + } } diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs new file mode 100644 index 00000000000..09f110ff2d7 --- /dev/null +++ b/sled-agent/src/sim/storage.rs @@ -0,0 +1,167 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Simulated sled agent storage implementation + +use futures::lock::Mutex; +use slog::Logger; +use std::collections::HashMap; +use std::net::SocketAddr; +use std::str::FromStr; +use std::sync::Arc; +use uuid::Uuid; + +// XXX Don't really like this import. +// +// Maybe refactor the "types" used by the HTTP +// service to a separate file. +use super::http_entrypoints_storage::{CreateRegion, State, RegionId, Region}; + +pub struct CrucibleDataInner { + regions: HashMap, +} + +impl CrucibleDataInner { + fn new() -> Self { + Self { + regions: HashMap::new(), + } + } + + fn list(&self) -> Vec { + self.regions.values().cloned().collect() + } + + fn create(&mut self, params: CreateRegion) -> Region { + let id = Uuid::from_str(¶ms.id.0).unwrap(); + let region = Region { + id: params.id, + volume_id: params.volume_id, + block_size: params.block_size, + extent_size: params.extent_size, + extent_count: params.extent_count, + // NOTE: This is a lie, obviously. No server is running. + port_number: 0, + state: State::Requested, + }; + self.regions.insert(id, region.clone()); + region + } + + fn get(&self, id: RegionId) -> Option { + let id = Uuid::from_str(&id.0).unwrap(); + self.regions.get(&id).cloned() + } + + fn delete(&mut self, id: RegionId) -> Option { + let id = Uuid::from_str(&id.0).unwrap(); + self.regions.remove(&id) + } +} + +pub struct CrucibleData { + inner: Mutex, +} + +impl CrucibleData { + fn new() -> Self { + Self { + inner: Mutex::new(CrucibleDataInner::new()), + } + } + + pub async fn list(&self) -> Vec { + self.inner.lock().await.list() + } + + pub async fn create(&self, params: CreateRegion) -> Region { + self.inner.lock().await.create(params) + } + + pub async fn get(&self, id: RegionId) -> Option { + self.inner.lock().await.get(id) + } + + pub async fn delete(&self, id: RegionId) -> Option{ + self.inner.lock().await.delete(id) + } +} + +/// A simulated Crucible Dataset. +/// +/// Contains both the data and the HTTP server. +pub struct CrucibleDataset { + _server: dropshot::HttpServer>, + _data: Arc, +} + +impl CrucibleDataset { + fn new(log: &Logger) -> Self { + let data = Arc::new(CrucibleData::new()); + let config = dropshot::ConfigDropshot { + bind_address: SocketAddr::new("127.0.0.1".parse().unwrap(), 0), + ..Default::default() + }; + let dropshot_log = log.new(o!("component" => "Simulated CrucibleAgent Dropshot Server")); + let server = dropshot::HttpServerStarter::new( + &config, + super::http_entrypoints_storage::api(), + data.clone(), + &dropshot_log, + ) + .expect("Could not initialize server") + .start(); + + CrucibleDataset { + _server: server, + _data: data, + } + } +} + +pub struct Zpool { + datasets: HashMap, +} + +impl Zpool { + pub fn new() -> Self { + Zpool { + datasets: HashMap::new(), + } + } + + pub fn insert_dataset(&mut self, log: &Logger, id: Uuid) { + self.datasets.insert(id, CrucibleDataset::new(log)); + } +} + +/// Simulated representation of all storage on a sled. +pub struct Storage { + log: Logger, + zpools: HashMap, +} + +impl Storage { + pub fn new(log: Logger) -> Self { + Self { + log, + zpools: HashMap::new(), + } + } + + pub fn log(&self) -> &Logger { + &self.log + } + + /// Adds a Zpool to the sled's simulated storage. + /// + /// The Zpool is originally empty. + pub fn insert_zpool(&mut self, id: Uuid) { + self.zpools.insert(id, Zpool::new()); + } + + pub fn get_zpool_mut(&mut self, id: Uuid) -> &mut Zpool { + self.zpools.get_mut(&id).expect("Zpool does not exist") + } +} From e38bb4a458e8c515b3be25380a12474ea189694a Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 3 Jan 2022 12:06:08 -0500 Subject: [PATCH 28/50] Fix broken test (test_region_allocation_not_enough_datasets) --- nexus/src/db/datastore.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index 302271211d5..f34ab996a5a 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -268,8 +268,9 @@ impl DataStore { // - Sled placement of datasets // - What sort of loads we'd like to create (even split across all disks // may not be preferable, especially if maintenance is expected) - #[derive(Debug)] + #[derive(Debug, thiserror::Error)] enum RegionAllocateError { + #[error("Not enough datasets for replicated allocation: {0}")] NotEnoughDatasets(usize), } type TxnError = TransactionError; @@ -335,7 +336,7 @@ impl DataStore { }) .await .map_err(|e| { - Error::internal_error(&format!("Transaction error: {:#?}", e)) + Error::internal_error(&format!("Transaction error: {}", e)) }) } From 48f3488aca79fcfbabf61758634c28de854eb669 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 3 Jan 2022 12:09:59 -0500 Subject: [PATCH 29/50] fmt --- .../src/sim/http_entrypoints_storage.rs | 24 +++++++-------- sled-agent/src/sim/sled_agent.rs | 14 ++++----- sled-agent/src/sim/storage.rs | 29 ++++++------------- 3 files changed, 28 insertions(+), 39 deletions(-) diff --git a/sled-agent/src/sim/http_entrypoints_storage.rs b/sled-agent/src/sim/http_entrypoints_storage.rs index 568a23c257e..471878c718f 100644 --- a/sled-agent/src/sim/http_entrypoints_storage.rs +++ b/sled-agent/src/sim/http_entrypoints_storage.rs @@ -6,17 +6,11 @@ // use crucible_agent_client::types::{CreateRegion, RegionId}; use dropshot::{ - endpoint, - ApiDescription, - HttpError, - HttpResponseDeleted, - HttpResponseOk, - Path as TypedPath, - RequestContext, - TypedBody, + endpoint, ApiDescription, HttpError, HttpResponseDeleted, HttpResponseOk, + Path as TypedPath, RequestContext, TypedBody, }; use schemars::JsonSchema; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; use std::sync::Arc; use super::storage::CrucibleData; @@ -27,7 +21,9 @@ type CrucibleAgentApiDescription = ApiDescription>; * Returns a description of the sled agent API */ pub fn api() -> CrucibleAgentApiDescription { - fn register_endpoints(api: &mut CrucibleAgentApiDescription) -> Result<(), String> { + fn register_endpoints( + api: &mut CrucibleAgentApiDescription, + ) -> Result<(), String> { api.register(region_list)?; api.register(region_create)?; api.register(region_get)?; @@ -142,7 +138,9 @@ async fn region_get( let crucible = rc.context(); match crucible.get(id).await { Some(region) => Ok(HttpResponseOk(region)), - None => Err(HttpError::for_not_found(None, "Region not found".to_string())), + None => { + Err(HttpError::for_not_found(None, "Region not found".to_string())) + } } } @@ -159,6 +157,8 @@ async fn region_delete( match crucible.delete(id).await { Some(_) => Ok(HttpResponseDeleted()), - None => Err(HttpError::for_not_found(None, "Region not found".to_string())), + None => { + Err(HttpError::for_not_found(None, "Region not found".to_string())) + } } } diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index 8d3ea5357c5..1828ce98f0c 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -119,17 +119,17 @@ impl SledAgent { /// Adds a Zpool to the simulated sled agent. pub async fn create_zpool(&self, id: Uuid) { - self.storage.lock() - .await - .insert_zpool(id); + self.storage.lock().await.insert_zpool(id); } /// Adds a Crucible Dataset within a zpool. - pub async fn create_crucible_dataset(&self, zpool_id: Uuid, dataset_id: Uuid) { + pub async fn create_crucible_dataset( + &self, + zpool_id: Uuid, + dataset_id: Uuid, + ) { let mut storage = self.storage.lock().await; let log = storage.log().clone(); - storage - .get_zpool_mut(zpool_id) - .insert_dataset(&log, dataset_id); + storage.get_zpool_mut(zpool_id).insert_dataset(&log, dataset_id); } } diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index 09f110ff2d7..a48fbfe7085 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -16,7 +16,7 @@ use uuid::Uuid; // // Maybe refactor the "types" used by the HTTP // service to a separate file. -use super::http_entrypoints_storage::{CreateRegion, State, RegionId, Region}; +use super::http_entrypoints_storage::{CreateRegion, Region, RegionId, State}; pub struct CrucibleDataInner { regions: HashMap, @@ -24,9 +24,7 @@ pub struct CrucibleDataInner { impl CrucibleDataInner { fn new() -> Self { - Self { - regions: HashMap::new(), - } + Self { regions: HashMap::new() } } fn list(&self) -> Vec { @@ -66,9 +64,7 @@ pub struct CrucibleData { impl CrucibleData { fn new() -> Self { - Self { - inner: Mutex::new(CrucibleDataInner::new()), - } + Self { inner: Mutex::new(CrucibleDataInner::new()) } } pub async fn list(&self) -> Vec { @@ -83,7 +79,7 @@ impl CrucibleData { self.inner.lock().await.get(id) } - pub async fn delete(&self, id: RegionId) -> Option{ + pub async fn delete(&self, id: RegionId) -> Option { self.inner.lock().await.delete(id) } } @@ -103,7 +99,8 @@ impl CrucibleDataset { bind_address: SocketAddr::new("127.0.0.1".parse().unwrap(), 0), ..Default::default() }; - let dropshot_log = log.new(o!("component" => "Simulated CrucibleAgent Dropshot Server")); + let dropshot_log = log + .new(o!("component" => "Simulated CrucibleAgent Dropshot Server")); let server = dropshot::HttpServerStarter::new( &config, super::http_entrypoints_storage::api(), @@ -113,10 +110,7 @@ impl CrucibleDataset { .expect("Could not initialize server") .start(); - CrucibleDataset { - _server: server, - _data: data, - } + CrucibleDataset { _server: server, _data: data } } } @@ -126,9 +120,7 @@ pub struct Zpool { impl Zpool { pub fn new() -> Self { - Zpool { - datasets: HashMap::new(), - } + Zpool { datasets: HashMap::new() } } pub fn insert_dataset(&mut self, log: &Logger, id: Uuid) { @@ -144,10 +136,7 @@ pub struct Storage { impl Storage { pub fn new(log: Logger) -> Self { - Self { - log, - zpools: HashMap::new(), - } + Self { log, zpools: HashMap::new() } } pub fn log(&self) -> &Logger { From e497dd3522d57d372a2120133cb8cc102f75dcf7 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 3 Jan 2022 18:22:09 -0500 Subject: [PATCH 30/50] Heyo, the disk integration tests are passing --- nexus/src/db/datastore.rs | 4 + nexus/src/sagas.rs | 11 ++- nexus/tests/integration_tests/disks.rs | 117 ++++++++++++++++--------- sled-agent/src/sim/sled_agent.rs | 24 +++-- sled-agent/src/sim/storage.rs | 67 ++++++++++---- 5 files changed, 148 insertions(+), 75 deletions(-) diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index f34ab996a5a..243a6e02f0d 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -256,6 +256,10 @@ impl DataStore { // ALLOCATION POLICY // // NOTE: This policy can - and should! - be changed. + // + // See https://rfd.shared.oxide.computer/rfd/0205 for a more + // complete discussion. + // // It is currently acting as a placeholder, showing a feasible // interaction between datasets and regions. // diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index ed896b86a4a..8594cf61171 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -340,7 +340,7 @@ impl SagaType for SagaDiskCreate { type ExecContextType = Arc; } -pub fn saga_disk_create() -> SagaTemplate { +fn saga_disk_create() -> SagaTemplate { let mut template_builder = SagaTemplateBuilder::new(); template_builder.append( @@ -424,11 +424,13 @@ async fn sdc_alloc_regions( // "creating" - the respective Crucible Agents must be instructed to // allocate the necessary regions before we can mark the disk as "ready to // be used". + eprintln!("SAGA: Allocating datasets + regions..."); let datasets_and_regions = osagactx .datastore() .region_allocate(disk_id, ¶ms.create_params) .await .map_err(ActionError::action_failed)?; + eprintln!("SAGA: Allocating datasets + regions... {:?}", datasets_and_regions); Ok(datasets_and_regions) } @@ -437,6 +439,8 @@ async fn allocate_region_from_dataset( dataset: &db::model::Dataset, region: &db::model::Region, ) -> Result { + eprintln!("SAGA: Allocating region from dataset"); + let url = format!("http://{}", dataset.address()); let client = CrucibleAgentClient::new(&url); @@ -485,6 +489,8 @@ async fn allocate_region_from_dataset( async fn sdc_regions_ensure( sagactx: ActionContext, ) -> Result<(), ActionError> { + eprintln!("SAGA: Ensuring regions exist"); + let log = sagactx.user_data().log(); let datasets_and_regions = sagactx .lookup::>( @@ -504,6 +510,7 @@ async fn sdc_regions_ensure( // TODO: Region has a port value, we could store this in the DB? + eprintln!("SAGA: Ensuring regions exist - OK"); Ok(()) } @@ -514,7 +521,7 @@ async fn sdc_finalize_disk_record( let _params = sagactx.saga_params(); let disk_id = sagactx.lookup::("disk_id")?; - let disk_created = sagactx.lookup::("disk_created")?; + let disk_created = sagactx.lookup::("created_disk")?; osagactx .datastore() .disk_update_runtime(&disk_id, &disk_created.runtime().detach()) diff --git a/nexus/tests/integration_tests/disks.rs b/nexus/tests/integration_tests/disks.rs index a6f2d118d88..6576c6a3c87 100644 --- a/nexus/tests/integration_tests/disks.rs +++ b/nexus/tests/integration_tests/disks.rs @@ -35,30 +35,58 @@ use nexus_test_utils::resource_helpers::create_project; use nexus_test_utils::ControlPlaneTestContext; use nexus_test_utils_macros::nexus_test; +const ORG_NAME: &str = "test-org"; +const PROJECT_NAME: &str = "springfield-squidport-disks"; + +fn get_project_url() -> String { + format!("/organizations/{}/projects/{}", ORG_NAME, PROJECT_NAME) +} + +fn get_disks_url() -> String { + format!("{}/disks", get_project_url()) +} + +fn get_instances_url() -> String { + format!("{}/instances", get_project_url()) +} + +fn get_instance_disks_url(instance_name: &str) -> String { + format!("{}/{}/disks", get_instances_url(), instance_name) +} + +fn get_disk_attach_url(instance_name: &str) -> String { + format!("{}/attach", get_instance_disks_url(instance_name)) +} + +fn get_disk_detach_url(instance_name: &str) -> String { + format!("{}/detach", get_instance_disks_url(instance_name)) +} + +async fn create_org_and_project(client: &ClientTestContext) -> Uuid { + create_organization(&client, ORG_NAME).await; + let project = create_project(client, ORG_NAME, PROJECT_NAME).await; + project.identity.id +} + /* * TODO-cleanup the mess of URLs used here and in test_instances.rs ought to * come from common code. */ + #[nexus_test] -async fn test_disks(cptestctx: &ControlPlaneTestContext) { +async fn test_disks_not_found_before_creation(cptestctx: &ControlPlaneTestContext) { let client = &cptestctx.external_client; - let apictx = &cptestctx.server.apictx; - let nexus = &apictx.nexus; /* Create a project for testing. */ - let org_name = "test-org"; - create_organization(&client, &org_name).await; - let project_name = "springfield-squidport-disks"; - let url_disks = - format!("/organizations/{}/projects/{}/disks", org_name, project_name); - let project = create_project(client, &org_name, &project_name).await; + create_org_and_project(&client).await; + let disks_url = get_disks_url(); /* List disks. There aren't any yet. */ - let disks = disks_list(&client, &url_disks).await; + let disks = disks_list(&client, &disks_url).await; assert_eq!(disks.len(), 0); /* Make sure we get a 404 if we fetch one. */ - let disk_url = format!("{}/just-rainsticks", url_disks); + let disk_url = format!("{}/just-rainsticks", disks_url); let error = client .make_request_error(Method::GET, &disk_url, StatusCode::NOT_FOUND) .await; @@ -76,8 +104,33 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { .parsed_body::() .unwrap(); assert_eq!(error.message, "not found: disk with name \"just-rainsticks\""); +} + +#[nexus_test] +async fn test_disks(cptestctx: &ControlPlaneTestContext) { + let client = &cptestctx.external_client; + let apictx = &cptestctx.server.apictx; + let nexus = &apictx.nexus; + let sled_agent = &cptestctx.sled_agent.sled_agent; + + // Create a Zpool. + let zpool_id = Uuid::new_v4(); + let zpool_size = 10 * 1024 * 1024 * 1024; + sled_agent.create_zpool(zpool_id, zpool_size).await; + + // Create multiple Datasets within that Zpool. + let dataset_count = 3; + let dataset_ids: Vec<_> = (0..dataset_count).map(|_| Uuid::new_v4()).collect(); + for id in &dataset_ids { + sled_agent.create_crucible_dataset(zpool_id, *id).await; + } + + /* Create a project for testing. */ + let project_id = create_org_and_project(&client).await; + let disks_url = get_disks_url(); /* Create a disk. */ + let disk_url = format!("{}/just-rainsticks", disks_url); let new_disk = params::DiskCreate { identity: IdentityMetadataCreateParams { name: "just-rainsticks".parse().unwrap(), @@ -86,10 +139,10 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { snapshot_id: None, size: ByteCount::from_gibibytes_u32(1), }; - let disk: Disk = objects_post(&client, &url_disks, new_disk.clone()).await; + let disk: Disk = objects_post(&client, &disks_url, new_disk.clone()).await; assert_eq!(disk.identity.name, "just-rainsticks"); assert_eq!(disk.identity.description, "sells rainsticks"); - assert_eq!(disk.project_id, project.identity.id); + assert_eq!(disk.project_id, project_id); assert_eq!(disk.snapshot_id, None); assert_eq!(disk.size.to_whole_mebibytes(), 1024); assert_eq!(disk.state, DiskState::Creating); @@ -102,7 +155,7 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { let disk = disk_get(&client, &disk_url).await; assert_eq!(disk.identity.name, "just-rainsticks"); assert_eq!(disk.identity.description, "sells rainsticks"); - assert_eq!(disk.project_id, project.identity.id); + assert_eq!(disk.project_id, project_id); assert_eq!(disk.snapshot_id, None); assert_eq!(disk.size.to_whole_mebibytes(), 1024); assert_eq!(disk.state, DiskState::Detached); @@ -111,7 +164,7 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { let error = client .make_request_error_body( Method::POST, - &url_disks, + &disks_url, new_disk, StatusCode::BAD_REQUEST, ) @@ -119,15 +172,12 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { assert_eq!(error.message, "already exists: disk \"just-rainsticks\""); /* List disks again and expect to find the one we just created. */ - let disks = disks_list(&client, &url_disks).await; + let disks = disks_list(&client, &disks_url).await; assert_eq!(disks.len(), 1); disks_eq(&disks[0], &disk); /* Create an instance to attach the disk. */ - let url_instances = format!( - "/organizations/{}/projects/{}/instances", - org_name, project_name - ); + let url_instances = get_instances_url(); let instance: Instance = objects_post( &client, &url_instances, @@ -147,25 +197,16 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { * Verify that there are no disks attached to the instance, and specifically * that our disk is not attached to this instance. */ - let url_instance_disks = format!( - "/organizations/{}/projects/{}/instances/{}/disks", - org_name, - project_name, + let url_instance_disks = get_instance_disks_url( instance.identity.name.as_str() ); let disks = objects_list_page::(&client, &url_instance_disks).await; assert_eq!(disks.items.len(), 0); - let url_instance_attach_disk = format!( - "/organizations/{}/projects/{}/instances/{}/disks/attach", - org_name, - project_name, + let url_instance_attach_disk = get_disk_attach_url( instance.identity.name.as_str(), ); - let url_instance_detach_disk = format!( - "/organizations/{}/projects/{}/instances/{}/disks/detach", - org_name, - project_name, + let url_instance_detach_disk = get_disk_detach_url( instance.identity.name.as_str(), ); @@ -229,16 +270,10 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { }, ) .await; - let url_instance2_attach_disk = format!( - "/organizations/{}/projects/{}/instances/{}/disks/attach", - org_name, - project_name, + let url_instance2_attach_disk = get_disk_attach_url( instance2.identity.name.as_str(), ); - let url_instance2_detach_disk = format!( - "/organizations/{}/projects/{}/instances/{}/disks/detach", - org_name, - project_name, + let url_instance2_detach_disk = get_disk_detach_url( instance2.identity.name.as_str(), ); let error = client @@ -448,7 +483,7 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { .expect("failed to delete disk"); /* It should no longer be present in our list of disks. */ - assert_eq!(disks_list(&client, &url_disks).await.len(), 0); + assert_eq!(disks_list(&client, &disks_url).await.len(), 0); /* We shouldn't find it if we request it explicitly. */ let error = client .make_request_error(Method::GET, &disk_url, StatusCode::NOT_FOUND) diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index 1828ce98f0c..94f1a4ec2a8 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -34,9 +34,6 @@ use super::storage::Storage; * move later. */ pub struct SledAgent { - /** unique id for this server */ - _id: Uuid, - /** collection of simulated instances, indexed by instance uuid */ instances: Arc>, /** collection of simulated disks, indexed by disk uuid */ @@ -54,7 +51,7 @@ impl SledAgent { id: &Uuid, sim_mode: SimMode, log: Logger, - ctlsc: Arc, + nexus_client: Arc, ) -> SledAgent { info!(&log, "created simulated sled agent"; "sim_mode" => ?sim_mode); @@ -63,18 +60,21 @@ impl SledAgent { let storage_log = log.new(o!("kind" => "storage")); SledAgent { - _id: *id, instances: Arc::new(SimCollection::new( - Arc::clone(&ctlsc), + Arc::clone(&nexus_client), instance_log, sim_mode, )), disks: Arc::new(SimCollection::new( - Arc::clone(&ctlsc), + Arc::clone(&nexus_client), disk_log, sim_mode, )), - storage: Mutex::new(Storage::new(storage_log)), + storage: Mutex::new(Storage::new( + *id, + Arc::clone(&nexus_client), + storage_log, + )), } } @@ -118,8 +118,8 @@ impl SledAgent { } /// Adds a Zpool to the simulated sled agent. - pub async fn create_zpool(&self, id: Uuid) { - self.storage.lock().await.insert_zpool(id); + pub async fn create_zpool(&self, id: Uuid, size: u64) { + self.storage.lock().await.insert_zpool(id, size).await; } /// Adds a Crucible Dataset within a zpool. @@ -128,8 +128,6 @@ impl SledAgent { zpool_id: Uuid, dataset_id: Uuid, ) { - let mut storage = self.storage.lock().await; - let log = storage.log().clone(); - storage.get_zpool_mut(zpool_id).insert_dataset(&log, dataset_id); + self.storage.lock().await.insert_dataset(zpool_id, dataset_id).await; } } diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index a48fbfe7085..60810ba8a85 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -5,6 +5,8 @@ //! Simulated sled agent storage implementation use futures::lock::Mutex; +use nexus_client::Client as NexusClient; +use nexus_client::types::{ByteCount, DatasetKind, DatasetPutRequest, ZpoolPutRequest}; use slog::Logger; use std::collections::HashMap; use std::net::SocketAddr; @@ -39,9 +41,13 @@ impl CrucibleDataInner { block_size: params.block_size, extent_size: params.extent_size, extent_count: params.extent_count, - // NOTE: This is a lie, obviously. No server is running. + // NOTE: This is a lie - no server is running. port_number: 0, - state: State::Requested, + + // TODO: This should be "Started", we should control state + // transitions. +// state: State::Requested, + state: State::Created, }; self.regions.insert(id, region.clone()); region @@ -88,7 +94,7 @@ impl CrucibleData { /// /// Contains both the data and the HTTP server. pub struct CrucibleDataset { - _server: dropshot::HttpServer>, + server: dropshot::HttpServer>, _data: Arc, } @@ -110,7 +116,11 @@ impl CrucibleDataset { .expect("Could not initialize server") .start(); - CrucibleDataset { _server: server, _data: data } + CrucibleDataset { server, _data: data } + } + + fn address(&self) -> SocketAddr { + self.server.local_addr() } } @@ -123,34 +133,53 @@ impl Zpool { Zpool { datasets: HashMap::new() } } - pub fn insert_dataset(&mut self, log: &Logger, id: Uuid) { + pub fn insert_dataset(&mut self, log: &Logger, id: Uuid) -> &CrucibleDataset { self.datasets.insert(id, CrucibleDataset::new(log)); + self.datasets.get(&id).expect("Failed to get the dataset we just inserted") } } /// Simulated representation of all storage on a sled. pub struct Storage { + sled_id: Uuid, + nexus_client: Arc, log: Logger, zpools: HashMap, } impl Storage { - pub fn new(log: Logger) -> Self { - Self { log, zpools: HashMap::new() } - } - - pub fn log(&self) -> &Logger { - &self.log + pub fn new(sled_id: Uuid, nexus_client: Arc, log: Logger) -> Self { + Self { sled_id, nexus_client, log, zpools: HashMap::new() } } - /// Adds a Zpool to the sled's simulated storage. - /// - /// The Zpool is originally empty. - pub fn insert_zpool(&mut self, id: Uuid) { - self.zpools.insert(id, Zpool::new()); - } + /// Adds a Zpool to the sled's simulated storage and notifies Nexus. + pub async fn insert_zpool(&mut self, zpool_id: Uuid, size: u64) { + // Update our local data + self.zpools.insert(zpool_id, Zpool::new()); - pub fn get_zpool_mut(&mut self, id: Uuid) -> &mut Zpool { - self.zpools.get_mut(&id).expect("Zpool does not exist") + // Notify Nexus + let request = ZpoolPutRequest { + size: ByteCount(size), + }; + self.nexus_client.zpool_put(&self.sled_id, &zpool_id, &request) + .await + .expect("Failed to notify Nexus about new Zpool"); + } + + /// Adds a Dataset to the sled's simulated storage and notifies Nexus. + pub async fn insert_dataset(&mut self, zpool_id: Uuid, dataset_id: Uuid) { + // Update our local data + let dataset = self.zpools.get_mut(&zpool_id) + .expect("Zpool does not exist") + .insert_dataset(&self.log, dataset_id); + + // Notify Nexus + let request = DatasetPutRequest { + address: dataset.address().to_string(), + kind: DatasetKind::Crucible, + }; + self.nexus_client.dataset_put(&zpool_id, &dataset_id, &request) + .await + .expect("Failed to notify Nexus about new Dataset"); } } From d459cab83b8afe95bf93c3e6e146d80c2dde6323 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 5 Jan 2022 21:14:43 -0500 Subject: [PATCH 31/50] Idempotent region allocation, adding undo saga actions, more tests --- nexus/src/db/datastore.rs | 157 +++++- nexus/src/db/db-macros/src/lib.rs | 4 +- nexus/src/db/model.rs | 4 +- nexus/src/internal_api/params.rs | 2 +- nexus/src/nexus.rs | 3 + nexus/src/sagas.rs | 88 +++- nexus/tests/integration_tests/disks.rs | 488 ++++++++++++++---- .../src/sim/http_entrypoints_storage.rs | 4 +- sled-agent/src/sim/mod.rs | 2 + sled-agent/src/sim/sled_agent.rs | 11 +- sled-agent/src/sim/storage.rs | 82 ++- 11 files changed, 686 insertions(+), 159 deletions(-) diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index 243a6e02f0d..48e75ca7df0 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -241,7 +241,7 @@ impl DataStore { }) } - /// Allocates enough regions to back a disk. + /// Idempotently allocates enough regions to back a disk. /// /// Returns the allocated regions, as well as the datasets to which they /// belong. @@ -281,8 +281,25 @@ impl DataStore { let params: params::DiskCreate = params.clone(); self.pool() .transaction(move |conn| { + // First, for idempotency, check if regions are already + // allocated to this disk. + // + // If they are, return those regions and the associated + // datasets. + let datasets_and_regions = region_dsl::region + .filter(region_dsl::disk_id.eq(disk_id)) + .inner_join( + dataset_dsl::dataset + .on(region_dsl::dataset_id.eq(dataset_dsl::id)) + ) + .select((Dataset::as_select(), Region::as_select())) + .get_results::<(Dataset, Region)>(conn)?; + if !datasets_and_regions.is_empty() { + return Ok(datasets_and_regions); + } + let datasets: Vec = dataset_dsl::dataset - // First, we look for valid datasets (non-deleted crucible datasets). + // We look for valid datasets (non-deleted crucible datasets). .filter(dataset_dsl::time_deleted.is_null()) .filter(dataset_dsl::kind.eq(DatasetKind( crate::internal_api::params::DatasetKind::Crucible, @@ -344,6 +361,26 @@ impl DataStore { }) } + /// Deletes all regions backing a disk. + pub async fn regions_hard_delete( + &self, + disk_id: Uuid, + ) -> DeleteResult { + use db::schema::region::dsl as dsl; + + diesel::delete(dsl::region) + .filter(dsl::disk_id.eq(disk_id)) + .execute_async(self.pool()) + .await + .map(|_| ()) + .map_err(|e| { + Error::internal_error(&format!( + "error deleting regions: {:?}", + e + )) + }) + } + /// Create a organization pub async fn organization_create( &self, @@ -1118,23 +1155,55 @@ impl DataStore { opctx: &OpContext, disk_authz: authz::ProjectChild, ) -> DeleteResult { - use db::schema::disk::dsl; - let now = Utc::now(); - let disk_id = disk_authz.id(); opctx.authorize(authz::Action::Delete, disk_authz).await?; + self.project_delete_disk_internal(disk_id, self.pool_authorized(opctx).await?).await + } + + // TODO: Delete me (this function, not the disk!), ensure all datastore + // access is auth-checked. + // + // Here's the deal: We have auth checks on access to the database - at the + // time of writing this comment, only a subset of access is protected, and + // "Delete Disk" is actually one of the first targets of this auth check. + // + // However, there are contexts where we want to delete disks *outside* of + // calling the HTTP API-layer "delete disk" endpoint. As one example, during + // the "undo" part of the disk creation saga, we want to allow users to + // delete the disk they (partially) created. + // + // This gets a little tricky mapping back to user permissions - a user + // SHOULD be able to create a disk with the "create" permission, without the + // "delete" permission. To still make the call internally, we'd basically + // need to manufacture a token that identifies the ability to "create a + // disk, or delete a very specific disk with ID = ...". + pub async fn project_delete_disk_no_auth( + &self, + disk_id: &Uuid, + ) -> DeleteResult { + self.project_delete_disk_internal(disk_id, self.pool()).await + } + + async fn project_delete_disk_internal( + &self, + disk_id: &Uuid, + pool: &bb8::Pool>, + ) -> DeleteResult { + use db::schema::disk::dsl; + let now = Utc::now(); let destroyed = api::external::DiskState::Destroyed.label(); let detached = api::external::DiskState::Detached.label(); let faulted = api::external::DiskState::Faulted.label(); + let creating = api::external::DiskState::Creating.label(); let result = diesel::update(dsl::disk) .filter(dsl::time_deleted.is_null()) .filter(dsl::id.eq(*disk_id)) - .filter(dsl::disk_state.eq_any(vec![detached, faulted])) + .filter(dsl::disk_state.eq_any(vec![detached, faulted, creating])) .set((dsl::disk_state.eq(destroyed), dsl::time_deleted.eq(now))) .check_if_exists::(*disk_id) - .execute_and_check(self.pool_authorized(opctx).await?) + .execute_and_check(pool) .await .map_err(|e| { public_error_from_diesel_pool( @@ -1144,6 +1213,14 @@ impl DataStore { ) })?; + // TODO: We need to also de-allocate the regions associated + // with a disk - see `regions_hard_delete` - but this should only + // happen when undoing deletion is no longer possible. + // + // This will be necessary - in addition to actually sending + // requests to the Crucible Agents, requesting that the regions + // be destroyed - + match result.status { UpdateStatus::Updated => Ok(()), UpdateStatus::NotUpdatedButExists => Err(Error::InvalidRequest { @@ -2668,6 +2745,72 @@ mod test { let _ = db.cleanup().await; } + #[tokio::test] + async fn test_region_allocation_is_idempotent() { + let logctx = + dev::test_setup_log("test_region_allocation_is_idempotent"); + let mut db = dev::test_setup_database(&logctx.log).await; + let cfg = db::Config { url: db.pg_config().clone() }; + let pool = db::Pool::new(&cfg); + let datastore = DataStore::new(Arc::new(pool)); + + // Create a sled... + let sled_id = create_test_sled(&datastore).await; + + // ... and a zpool within that sled... + let zpool_id = create_test_zpool(&datastore, sled_id).await; + + // ... and datasets within that zpool. + let dataset_count = REGION_REDUNDANCY_THRESHOLD; + let bogus_addr = + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); + let kind = + DatasetKind(crate::internal_api::params::DatasetKind::Crucible); + let dataset_ids: Vec = + (0..dataset_count).map(|_| Uuid::new_v4()).collect(); + for id in &dataset_ids { + let dataset = Dataset::new(*id, zpool_id, bogus_addr, kind.clone()); + datastore.dataset_upsert(dataset).await.unwrap(); + } + + // Allocate regions from the datasets for this disk. + let params = create_test_disk_create_params( + "disk", + ByteCount::from_mebibytes_u32(500), + ); + let disk_id = Uuid::new_v4(); + let mut dataset_and_regions1 = datastore.region_allocate(disk_id, ¶ms) + .await + .unwrap(); + let mut dataset_and_regions2 = datastore.region_allocate(disk_id, ¶ms) + .await + .unwrap(); + + // Give them a consistent order so we can easily compare them. + let sort_vec = |v: &mut Vec<(Dataset, Region)>| { + v.sort_by(|(d1, r1), (d2, r2)| { + let order = d1.id().cmp(&d2.id()); + match order { + std::cmp::Ordering::Equal => r1.id().cmp(&r2.id()), + _ => order + } + }); + }; + sort_vec(&mut dataset_and_regions1); + sort_vec(&mut dataset_and_regions2); + + // Validate that the two calls to allocate return the same data. + assert_eq!(dataset_and_regions1.len(), dataset_and_regions2.len()); + for i in 0..dataset_and_regions1.len() { + assert_eq!( + dataset_and_regions1[i], + dataset_and_regions2[i], + ); + } + + let _ = db.cleanup().await; + } + #[tokio::test] async fn test_region_allocation_not_enough_datasets() { let logctx = diff --git a/nexus/src/db/db-macros/src/lib.rs b/nexus/src/db/db-macros/src/lib.rs index a045174fd9f..3b29099c5dc 100644 --- a/nexus/src/db/db-macros/src/lib.rs +++ b/nexus/src/db/db-macros/src/lib.rs @@ -186,7 +186,7 @@ fn build_resource_identity( let identity_name = format_ident!("{}Identity", struct_name); quote! { #[doc = #identity_doc] - #[derive(Clone, Debug, Selectable, Queryable, Insertable, serde::Serialize, serde::Deserialize)] + #[derive(Clone, Debug, PartialEq, Selectable, Queryable, Insertable, serde::Serialize, serde::Deserialize)] #[table_name = #table_name ] pub struct #identity_name { pub id: ::uuid::Uuid, @@ -225,7 +225,7 @@ fn build_asset_identity(struct_name: &Ident, table_name: &Lit) -> TokenStream { let identity_name = format_ident!("{}Identity", struct_name); quote! { #[doc = #identity_doc] - #[derive(Clone, Debug, Selectable, Queryable, Insertable, serde::Serialize, serde::Deserialize)] + #[derive(Clone, Debug, PartialEq, Selectable, Queryable, Insertable, serde::Serialize, serde::Deserialize)] #[table_name = #table_name ] pub struct #identity_name { pub id: ::uuid::Uuid, diff --git a/nexus/src/db/model.rs b/nexus/src/db/model.rs index 3cb746361d7..5b375cc37de 100644 --- a/nexus/src/db/model.rs +++ b/nexus/src/db/model.rs @@ -521,7 +521,7 @@ impl_enum_type!( #[postgres(type_name = "dataset_kind", type_schema = "public")] pub struct DatasetKindEnum; - #[derive(Clone, Debug, AsExpression, FromSqlRow, Serialize, Deserialize)] + #[derive(Clone, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq)] #[sql_type = "DatasetKindEnum"] pub struct DatasetKind(pub internal_api::params::DatasetKind); @@ -550,6 +550,7 @@ impl From for DatasetKind { Asset, Deserialize, Serialize, + PartialEq, )] #[table_name = "dataset"] pub struct Dataset { @@ -619,6 +620,7 @@ impl DatastoreCollection for Disk { Asset, Serialize, Deserialize, + PartialEq, )] #[table_name = "region"] pub struct Region { diff --git a/nexus/src/internal_api/params.rs b/nexus/src/internal_api/params.rs index 5d073cce5b2..1e4f420d676 100644 --- a/nexus/src/internal_api/params.rs +++ b/nexus/src/internal_api/params.rs @@ -33,7 +33,7 @@ pub struct ZpoolPutRequest { pub struct ZpoolPutResponse {} /// Describes the purpose of the dataset. -#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, Copy)] +#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, Copy, PartialEq)] pub enum DatasetKind { Crucible, Cockroach, diff --git a/nexus/src/nexus.rs b/nexus/src/nexus.rs index 08f87d82cd1..80dd5af92c1 100644 --- a/nexus/src/nexus.rs +++ b/nexus/src/nexus.rs @@ -767,6 +767,9 @@ impl Nexus { * address that. */ self.db_datastore.project_delete_disk(opctx, authz_disk).await + + // TODO: Call to crucible agents, requesting deletion. + // TODO: self.db_datastore.regions_hard_delete(disk.id) } /* diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index 8594cf61171..6fe39783ff2 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -25,6 +25,7 @@ use crucible_agent_client::{ }; use futures::StreamExt; use lazy_static::lazy_static; +use omicron_common::api::external::Error; use omicron_common::api::external::Generation; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::InstanceState; @@ -352,28 +353,33 @@ fn saga_disk_create() -> SagaTemplate { template_builder.append( "created_disk", "CreateDiskRecord", - // TODO: Needs undo action. - new_action_noop_undo(sdc_create_disk_record), + ActionFunc::new_action( + sdc_create_disk_record, + sdc_create_disk_record_undo, + ), ); template_builder.append( "datasets_and_regions", "AllocRegions", - // TODO: Needs undo action. - new_action_noop_undo(sdc_alloc_regions), + ActionFunc::new_action( + sdc_alloc_regions, + sdc_alloc_regions_undo, + ), ); template_builder.append( "regions_ensure", "RegionsEnsure", - // TODO: Needs undo action. - new_action_noop_undo(sdc_regions_ensure), + ActionFunc::new_action( + sdc_regions_ensure, + sdc_regions_ensure_undo, + ), ); template_builder.append( "disk_runtime", "FinalizeDiskRecord", - // TODO: Needs undo action. new_action_noop_undo(sdc_finalize_disk_record), ); @@ -412,6 +418,19 @@ async fn sdc_create_disk_record( Ok(disk_created) } +async fn sdc_create_disk_record_undo( + sagactx: ActionContext, +) -> Result<(), anyhow::Error> { + let osagactx = sagactx.user_data(); + + let disk_id = sagactx.lookup::("disk_id")?; + osagactx + .datastore() + .project_delete_disk_no_auth(&disk_id) + .await?; + Ok(()) +} + async fn sdc_alloc_regions( sagactx: ActionContext, ) -> Result, ActionError> { @@ -424,23 +443,31 @@ async fn sdc_alloc_regions( // "creating" - the respective Crucible Agents must be instructed to // allocate the necessary regions before we can mark the disk as "ready to // be used". - eprintln!("SAGA: Allocating datasets + regions..."); let datasets_and_regions = osagactx .datastore() .region_allocate(disk_id, ¶ms.create_params) .await .map_err(ActionError::action_failed)?; - eprintln!("SAGA: Allocating datasets + regions... {:?}", datasets_and_regions); Ok(datasets_and_regions) } -async fn allocate_region_from_dataset( +async fn sdc_alloc_regions_undo( + sagactx: ActionContext, +) -> Result<(), anyhow::Error> { + let osagactx = sagactx.user_data(); + + let disk_id = sagactx.lookup::("disk_id")?; + osagactx.datastore() + .regions_hard_delete(disk_id) + .await?; + Ok(()) +} + +async fn ensure_region_in_dataset( log: &Logger, dataset: &db::model::Dataset, region: &db::model::Region, -) -> Result { - eprintln!("SAGA: Allocating region from dataset"); - +) -> Result { let url = format!("http://{}", dataset.address()); let client = CrucibleAgentClient::new(&url); @@ -480,8 +507,7 @@ async fn allocate_region_from_dataset( create_region, log_create_failure, ) - .await - .map_err(|e| ActionError::action_failed(e.to_string()))?; + .await?; Ok(region) } @@ -489,8 +515,6 @@ async fn allocate_region_from_dataset( async fn sdc_regions_ensure( sagactx: ActionContext, ) -> Result<(), ActionError> { - eprintln!("SAGA: Ensuring regions exist"); - let log = sagactx.user_data().log(); let datasets_and_regions = sagactx .lookup::>( @@ -499,18 +523,42 @@ async fn sdc_regions_ensure( let request_count = datasets_and_regions.len(); futures::stream::iter(datasets_and_regions) .map(|(dataset, region)| async move { - allocate_region_from_dataset(log, &dataset, ®ion).await + ensure_region_in_dataset(log, &dataset, ®ion).await }) // Execute the allocation requests concurrently. .buffer_unordered(request_count) .collect::>>() .await .into_iter() - .collect::, _>>()?; + .collect::, _>>() + .map_err(ActionError::action_failed)?; // TODO: Region has a port value, we could store this in the DB? + Ok(()) +} + +async fn sdc_regions_ensure_undo( + sagactx: ActionContext, +) -> Result<(), anyhow::Error> { + let datasets_and_regions = sagactx + .lookup::>( + "datasets_and_regions", + )?; - eprintln!("SAGA: Ensuring regions exist - OK"); + let request_count = datasets_and_regions.len(); + futures::stream::iter(datasets_and_regions) + .map(|(dataset, region)| async move { + let url = format!("http://{}", dataset.address()); + let client = CrucibleAgentClient::new(&url); + let id = RegionId(region.id().to_string()); + client.region_delete(&id).await + }) + // Execute the allocation requests concurrently. + .buffer_unordered(request_count) + .collect::>>() + .await + .into_iter() + .collect::, _>>()?; Ok(()) } diff --git a/nexus/tests/integration_tests/disks.rs b/nexus/tests/integration_tests/disks.rs index 6576c6a3c87..bfec51e8a1d 100644 --- a/nexus/tests/integration_tests/disks.rs +++ b/nexus/tests/integration_tests/disks.rs @@ -2,9 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -/*! - * Tests basic disk support in the API - */ +//! Tests basic disk support in the API use http::method::Method; use http::StatusCode; @@ -16,6 +14,7 @@ use omicron_common::api::external::Instance; use omicron_common::api::external::InstanceCpuCount; use omicron_nexus::TestInterfaces as _; use omicron_nexus::{external_api::params, Nexus}; +use omicron_sled_agent::sim::{RegionState, SledAgent}; use sled_agent_client::TestInterfaces as _; use std::sync::Arc; use uuid::Uuid; @@ -37,6 +36,7 @@ use nexus_test_utils_macros::nexus_test; const ORG_NAME: &str = "test-org"; const PROJECT_NAME: &str = "springfield-squidport-disks"; +const DISK_NAME: &str = "just-rainsticks"; fn get_project_url() -> String { format!("/organizations/{}/projects/{}", ORG_NAME, PROJECT_NAME) @@ -68,31 +68,81 @@ async fn create_org_and_project(client: &ClientTestContext) -> Uuid { project.identity.id } -/* - * TODO-cleanup the mess of URLs used here and in test_instances.rs ought to - * come from common code. - */ +struct DiskTest { + sled_agent: Arc, + zpool_id: Uuid, + dataset_ids: Vec, + project_id: Uuid, +} + +impl DiskTest { + // Creates fake physical storage, an organization, and a project. + async fn new(cptestctx: &ControlPlaneTestContext) -> Self { + let client = &cptestctx.external_client; + let sled_agent = cptestctx.sled_agent.sled_agent.clone(); + + // Create a Zpool. + let zpool_id = Uuid::new_v4(); + let zpool_size = 10 * 1024 * 1024 * 1024; + sled_agent.create_zpool(zpool_id, zpool_size).await; + + // Create multiple Datasets within that Zpool. + let dataset_count = 3; + let dataset_ids: Vec<_> = (0..dataset_count).map(|_| Uuid::new_v4()).collect(); + for id in &dataset_ids { + sled_agent.create_crucible_dataset(zpool_id, *id).await; + + // By default, regions are created immediately. + let crucible = sled_agent.get_crucible_dataset(zpool_id, *id).await; + crucible.set_create_callback(Box::new(|_| { + RegionState::Created + })).await; + } + + // Create a project for testing. + let project_id = create_org_and_project(&client).await; + + Self { + sled_agent, + zpool_id, + dataset_ids, + project_id, + } + } + + async fn finish_region_allocation( + &self, + ) { + for dataset_id in &self.dataset_ids { + let dataset = self.sled_agent.get_crucible_dataset(self.zpool_id, *dataset_id).await; + let regions = dataset.list().await; + + for region in ®ions { + dataset.set_state(®ion.id, RegionState::Created).await; + } + } + } + +} #[nexus_test] -async fn test_disks_not_found_before_creation(cptestctx: &ControlPlaneTestContext) { +async fn test_disk_not_found_before_creation(cptestctx: &ControlPlaneTestContext) { let client = &cptestctx.external_client; - - /* Create a project for testing. */ - create_org_and_project(&client).await; + DiskTest::new(&cptestctx).await; let disks_url = get_disks_url(); - /* List disks. There aren't any yet. */ + // List disks. There aren't any yet. let disks = disks_list(&client, &disks_url).await; assert_eq!(disks.len(), 0); - /* Make sure we get a 404 if we fetch one. */ - let disk_url = format!("{}/just-rainsticks", disks_url); + // Make sure we get a 404 if we fetch one. + let disk_url = format!("{}/{}", disks_url, DISK_NAME); let error = client .make_request_error(Method::GET, &disk_url, StatusCode::NOT_FOUND) .await; - assert_eq!(error.message, "not found: disk with name \"just-rainsticks\""); + assert_eq!(error.message, format!("not found: disk with name \"{}\"", DISK_NAME)); - /* We should also get a 404 if we delete one. */ + // We should also get a 404 if we delete one. let error = NexusRequest::new( RequestBuilder::new(client, Method::DELETE, &disk_url) .expect_status(Some(StatusCode::NOT_FOUND)), @@ -103,64 +153,186 @@ async fn test_disks_not_found_before_creation(cptestctx: &ControlPlaneTestContex .expect("unexpected success") .parsed_body::() .unwrap(); - assert_eq!(error.message, "not found: disk with name \"just-rainsticks\""); + assert_eq!(error.message, format!("not found: disk with name \"{}\"", DISK_NAME)); } #[nexus_test] -async fn test_disks(cptestctx: &ControlPlaneTestContext) { +async fn test_disk_create_attach_detach_delete(cptestctx: &ControlPlaneTestContext) { let client = &cptestctx.external_client; - let apictx = &cptestctx.server.apictx; - let nexus = &apictx.nexus; - let sled_agent = &cptestctx.sled_agent.sled_agent; - - // Create a Zpool. - let zpool_id = Uuid::new_v4(); - let zpool_size = 10 * 1024 * 1024 * 1024; - sled_agent.create_zpool(zpool_id, zpool_size).await; - - // Create multiple Datasets within that Zpool. - let dataset_count = 3; - let dataset_ids: Vec<_> = (0..dataset_count).map(|_| Uuid::new_v4()).collect(); - for id in &dataset_ids { - sled_agent.create_crucible_dataset(zpool_id, *id).await; - } - - /* Create a project for testing. */ - let project_id = create_org_and_project(&client).await; + let test = DiskTest::new(&cptestctx).await; + let nexus = &cptestctx.server.apictx.nexus; let disks_url = get_disks_url(); - /* Create a disk. */ - let disk_url = format!("{}/just-rainsticks", disks_url); + // Create a disk. + let disk_url = format!("{}/{}", disks_url, DISK_NAME); let new_disk = params::DiskCreate { identity: IdentityMetadataCreateParams { - name: "just-rainsticks".parse().unwrap(), + name: DISK_NAME.parse().unwrap(), description: String::from("sells rainsticks"), }, snapshot_id: None, size: ByteCount::from_gibibytes_u32(1), }; let disk: Disk = objects_post(&client, &disks_url, new_disk.clone()).await; - assert_eq!(disk.identity.name, "just-rainsticks"); + assert_eq!(disk.identity.name, DISK_NAME); assert_eq!(disk.identity.description, "sells rainsticks"); - assert_eq!(disk.project_id, project_id); + assert_eq!(disk.project_id, test.project_id); assert_eq!(disk.snapshot_id, None); assert_eq!(disk.size.to_whole_mebibytes(), 1024); assert_eq!(disk.state, DiskState::Creating); - /* - * Fetch the disk and expect it to match what we just created except that - * the state will now be "Detached", as the server has simulated the create - * process. - */ + // Fetch the disk and expect it to match what we just created except that + // the state will now be "Detached", as the server has simulated the create + // process. let disk = disk_get(&client, &disk_url).await; - assert_eq!(disk.identity.name, "just-rainsticks"); + assert_eq!(disk.identity.name, DISK_NAME); assert_eq!(disk.identity.description, "sells rainsticks"); - assert_eq!(disk.project_id, project_id); + assert_eq!(disk.project_id, test.project_id); assert_eq!(disk.snapshot_id, None); assert_eq!(disk.size.to_whole_mebibytes(), 1024); assert_eq!(disk.state, DiskState::Detached); - /* Attempt to create a second disk with a conflicting name. */ + // List disks again and expect to find the one we just created. + let disks = disks_list(&client, &disks_url).await; + assert_eq!(disks.len(), 1); + disks_eq(&disks[0], &disk); + + // Create an instance to attach the disk. + let url_instances = get_instances_url(); + let instance: Instance = objects_post( + &client, + &url_instances, + params::InstanceCreate { + identity: IdentityMetadataCreateParams { + name: "instance1".parse().unwrap(), + description: "instance1".to_string(), + }, + ncpus: InstanceCpuCount(4), + memory: ByteCount::from_mebibytes_u32(256), + hostname: "instance1".to_string(), + }, + ) + .await; + + // Verify that there are no disks attached to the instance, and specifically + // that our disk is not attached to this instance. + let url_instance_disks = get_instance_disks_url( + instance.identity.name.as_str() + ); + let disks = objects_list_page::(&client, &url_instance_disks).await; + assert_eq!(disks.items.len(), 0); + + let url_instance_attach_disk = get_disk_attach_url( + instance.identity.name.as_str(), + ); + let url_instance_detach_disk = get_disk_detach_url( + instance.identity.name.as_str(), + ); + + // Start attaching the disk to the instance. + let mut response = client + .make_request( + Method::POST, + &url_instance_attach_disk, + Some(params::DiskIdentifier { disk: disk.identity.name.clone() }), + StatusCode::ACCEPTED, + ) + .await + .unwrap(); + let attached_disk: Disk = read_json(&mut response).await; + let instance_id = &instance.identity.id; + assert_eq!(attached_disk.identity.name, disk.identity.name); + assert_eq!(attached_disk.identity.id, disk.identity.id); + assert_eq!(attached_disk.state, DiskState::Attaching(instance_id.clone())); + + // Finish simulation of the attachment and verify the new state, both on the + // attachment and the disk itself. + disk_simulate(nexus, &disk.identity.id).await; + let attached_disk: Disk = disk_get(&client, &disk_url).await; + assert_eq!(attached_disk.identity.name, disk.identity.name); + assert_eq!(attached_disk.identity.id, disk.identity.id); + assert_eq!(attached_disk.state, DiskState::Attached(instance_id.clone())); + + // Attach the disk to the same instance. This should complete immediately + // with no state change. + client + .make_request( + Method::POST, + &url_instance_attach_disk, + Some(params::DiskIdentifier { disk: disk.identity.name }), + StatusCode::ACCEPTED, + ) + .await + .unwrap(); + let disk = disk_get(&client, &disk_url).await; + assert_eq!(disk.state, DiskState::Attached(instance_id.clone())); + + // Begin detaching the disk. + client + .make_request( + Method::POST, + &url_instance_detach_disk, + Some(params::DiskIdentifier { disk: disk.identity.name.clone() }), + StatusCode::ACCEPTED, + ) + .await + .unwrap(); + let disk: Disk = disk_get(&client, &disk_url).await; + assert_eq!(disk.state, DiskState::Detaching(instance_id.clone())); + + // Finish the detachment. + disk_simulate(nexus, &disk.identity.id).await; + let disk = disk_get(&client, &disk_url).await; + assert_eq!(disk.state, DiskState::Detached); + + // Since detach is idempotent, we can detach it again. + client + .make_request( + Method::POST, + &url_instance_detach_disk, + Some(params::DiskIdentifier { disk: disk.identity.name.clone() }), + StatusCode::ACCEPTED, + ) + .await + .unwrap(); + + // A priveleged user should be able to delete the disk. + NexusRequest::object_delete(client, &disk_url) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("failed to delete disk"); + + // It should no longer be present in our list of disks. + assert_eq!(disks_list(&client, &disks_url).await.len(), 0); + + // We shouldn't find it if we request it explicitly. + let error = client + .make_request_error(Method::GET, &disk_url, StatusCode::NOT_FOUND) + .await; + assert_eq!(error.message, format!("not found: disk with name \"{}\"", DISK_NAME)); +} + +#[nexus_test] +async fn test_disk_create_disk_that_already_exists_fails(cptestctx: &ControlPlaneTestContext) { + let client = &cptestctx.external_client; + DiskTest::new(&cptestctx).await; + let disks_url = get_disks_url(); + + // Create a disk. + let new_disk = params::DiskCreate { + identity: IdentityMetadataCreateParams { + name: DISK_NAME.parse().unwrap(), + description: String::from("sells rainsticks"), + }, + snapshot_id: None, + size: ByteCount::from_gibibytes_u32(1), + }; + let _: Disk = objects_post(&client, &disks_url, new_disk.clone()).await; + let disk_url = format!("{}/{}", disks_url, DISK_NAME); + let disk = disk_get(&client, &disk_url).await; + + // Attempt to create a second disk with a conflicting name. let error = client .make_request_error_body( Method::POST, @@ -169,21 +341,41 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { StatusCode::BAD_REQUEST, ) .await; - assert_eq!(error.message, "already exists: disk \"just-rainsticks\""); + assert_eq!(error.message, format!("already exists: disk \"{}\"", DISK_NAME)); - /* List disks again and expect to find the one we just created. */ + // List disks again and expect to find the one we just created. let disks = disks_list(&client, &disks_url).await; assert_eq!(disks.len(), 1); disks_eq(&disks[0], &disk); +} - /* Create an instance to attach the disk. */ +#[nexus_test] +async fn test_disk_move_between_instances(cptestctx: &ControlPlaneTestContext) { + let client = &cptestctx.external_client; + let nexus = &cptestctx.server.apictx.nexus; + DiskTest::new(&cptestctx).await; + let disks_url = get_disks_url(); + + // Create a disk. + let disk_url = format!("{}/{}", disks_url, DISK_NAME); + let new_disk = params::DiskCreate { + identity: IdentityMetadataCreateParams { + name: DISK_NAME.parse().unwrap(), + description: String::from("sells rainsticks"), + }, + snapshot_id: None, + size: ByteCount::from_gibibytes_u32(1), + }; + let disk: Disk = objects_post(&client, &disks_url, new_disk.clone()).await; + + // Create an instance to attach the disk. let url_instances = get_instances_url(); let instance: Instance = objects_post( &client, &url_instances, params::InstanceCreate { identity: IdentityMetadataCreateParams { - name: "just-rainsticks".parse().unwrap(), + name: DISK_NAME.parse().unwrap(), description: String::from("sells rainsticks"), }, ncpus: InstanceCpuCount(4), @@ -193,10 +385,8 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { ) .await; - /* - * Verify that there are no disks attached to the instance, and specifically - * that our disk is not attached to this instance. - */ + // Verify that there are no disks attached to the instance, and specifically + // that our disk is not attached to this instance. let url_instance_disks = get_instance_disks_url( instance.identity.name.as_str() ); @@ -210,7 +400,7 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { instance.identity.name.as_str(), ); - /* Start attaching the disk to the instance. */ + // Start attaching the disk to the instance. let mut response = client .make_request( Method::POST, @@ -226,20 +416,16 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { assert_eq!(attached_disk.identity.id, disk.identity.id); assert_eq!(attached_disk.state, DiskState::Attaching(instance_id.clone())); - /* - * Finish simulation of the attachment and verify the new state, both on the - * attachment and the disk itself. - */ + // Finish simulation of the attachment and verify the new state, both on the + // attachment and the disk itself. disk_simulate(nexus, &disk.identity.id).await; let attached_disk: Disk = disk_get(&client, &disk_url).await; assert_eq!(attached_disk.identity.name, disk.identity.name); assert_eq!(attached_disk.identity.id, disk.identity.id); assert_eq!(attached_disk.state, DiskState::Attached(instance_id.clone())); - /* - * Attach the disk to the same instance. This should complete immediately - * with no state change. - */ + // Attach the disk to the same instance. This should complete immediately + // with no state change. client .make_request( Method::POST, @@ -252,10 +438,8 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { let disk = disk_get(&client, &disk_url).await; assert_eq!(disk.state, DiskState::Attached(instance_id.clone())); - /* - * Create a second instance and try to attach the disk to that. This should - * fail and the disk should remain attached to the first instance. - */ + // Create a second instance and try to attach the disk to that. This should + // fail and the disk should remain attached to the first instance. let instance2: Instance = objects_post( &client, &url_instances, @@ -286,16 +470,14 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { .await; assert_eq!( error.message, - "cannot attach disk \"just-rainsticks\": disk is attached to another \ - instance" + format!("cannot attach disk \"{}\": disk is attached to another \ + instance", DISK_NAME) ); let attached_disk = disk_get(&client, &disk_url).await; assert_eq!(attached_disk.state, DiskState::Attached(instance_id.clone())); - /* - * Begin detaching the disk. - */ + // Begin detaching the disk. client .make_request( Method::POST, @@ -308,7 +490,7 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { let disk: Disk = disk_get(&client, &disk_url).await; assert_eq!(disk.state, DiskState::Detaching(instance_id.clone())); - /* It's still illegal to attach this disk elsewhere. */ + // It's still illegal to attach this disk elsewhere. let error = client .make_request_error_body( Method::POST, @@ -319,11 +501,11 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { .await; assert_eq!( error.message, - "cannot attach disk \"just-rainsticks\": disk is attached to another \ - instance" + format!("cannot attach disk \"{}\": disk is attached to another \ + instance", DISK_NAME) ); - /* It's even illegal to attach this disk back to the same instance. */ + // It's even illegal to attach this disk back to the same instance. let error = client .make_request_error_body( Method::POST, @@ -332,14 +514,14 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { StatusCode::BAD_REQUEST, ) .await; - /* TODO-debug the error message here is misleading. */ + // TODO-debug the error message here is misleading. assert_eq!( error.message, - "cannot attach disk \"just-rainsticks\": disk is attached to another \ - instance" + format!("cannot attach disk \"{}\": disk is attached to another \ + instance", DISK_NAME) ); - /* However, there's no problem attempting to detach it again. */ + // However, there's no problem attempting to detach it again. client .make_request( Method::POST, @@ -352,12 +534,12 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { let disk = disk_get(&client, &disk_url).await; assert_eq!(disk.state, DiskState::Detaching(instance_id.clone())); - /* Finish the detachment. */ + // Finish the detachment. disk_simulate(nexus, &disk.identity.id).await; let disk = disk_get(&client, &disk_url).await; assert_eq!(disk.state, DiskState::Detached); - /* Since delete is idempotent, we can detach it again -- from either one. */ + // Since delete is idempotent, we can detach it again -- from either one. client .make_request( Method::POST, @@ -377,7 +559,7 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { .await .unwrap(); - /* Now, start attaching it again to the second instance. */ + // Now, start attaching it again to the second instance. let mut response = client .make_request( Method::POST, @@ -396,10 +578,8 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { let disk = disk_get(&client, &disk_url).await; assert_eq!(disk.state, DiskState::Attaching(instance2_id.clone())); - /* - * At this point, it's not legal to attempt to attach it to a different - * instance (the first one). - */ + // At this point, it's not legal to attempt to attach it to a different + // instance (the first one). let error = client .make_request_error_body( Method::POST, @@ -410,11 +590,11 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { .await; assert_eq!( error.message, - "cannot attach disk \"just-rainsticks\": disk is attached to another \ - instance" + format!("cannot attach disk \"{}\": disk is attached to another \ + instance", DISK_NAME) ); - /* It's fine to attempt another attachment to the same instance. */ + // It's fine to attempt another attachment to the same instance. client .make_request( Method::POST, @@ -427,13 +607,13 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { let disk = disk_get(&client, &disk_url).await; assert_eq!(disk.state, DiskState::Attaching(instance2_id.clone())); - /* It's not allowed to delete a disk that's attaching. */ + // It's not allowed to delete a disk that's attaching. let error = client .make_request_error(Method::DELETE, &disk_url, StatusCode::BAD_REQUEST) .await; assert_eq!(error.message, "disk is attached"); - /* Now, begin a detach while the disk is still being attached. */ + // Now, begin a detach while the disk is still being attached. client .make_request( Method::POST, @@ -446,21 +626,53 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { let disk: Disk = disk_get(&client, &disk_url).await; assert_eq!(disk.state, DiskState::Detaching(instance2_id.clone())); - /* It's not allowed to delete a disk that's detaching, either. */ + // It's not allowed to delete a disk that's detaching, either. let error = client .make_request_error(Method::DELETE, &disk_url, StatusCode::BAD_REQUEST) .await; assert_eq!(error.message, "disk is attached"); - /* Finish detachment. */ + // Finish detachment. disk_simulate(nexus, &disk.identity.id).await; let disk = disk_get(&client, &disk_url).await; assert_eq!(disk.state, DiskState::Detached); - /* - * If we're not authenticated, or authenticated as an unprivileged user, we - * shouldn't be able to delete this disk. - */ + NexusRequest::object_delete(client, &disk_url) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("failed to delete disk"); + + // It should no longer be present in our list of disks. + assert_eq!(disks_list(&client, &disks_url).await.len(), 0); + + // We shouldn't find it if we request it explicitly. + let error = client + .make_request_error(Method::GET, &disk_url, StatusCode::NOT_FOUND) + .await; + assert_eq!(error.message, format!("not found: disk with name \"{}\"", DISK_NAME)); +} + +#[nexus_test] +async fn test_disk_deletion_requires_authentication(cptestctx: &ControlPlaneTestContext) { + let client = &cptestctx.external_client; + DiskTest::new(&cptestctx).await; + let disks_url = get_disks_url(); + + // Create a disk. + let disk_url = format!("{}/{}", disks_url, DISK_NAME); + let new_disk = params::DiskCreate { + identity: IdentityMetadataCreateParams { + name: DISK_NAME.parse().unwrap(), + description: String::from("sells rainsticks"), + }, + snapshot_id: None, + size: ByteCount::from_gibibytes_u32(1), + }; + let _: Disk = objects_post(&client, &disks_url, new_disk.clone()).await; + + // If we're not authenticated, or authenticated as an unprivileged user, we + // shouldn't be able to delete this disk. NexusRequest::new( RequestBuilder::new(client, Method::DELETE, &disk_url) .expect_status(Some(StatusCode::UNAUTHORIZED)), @@ -481,14 +693,76 @@ async fn test_disks(cptestctx: &ControlPlaneTestContext) { .execute() .await .expect("failed to delete disk"); +} - /* It should no longer be present in our list of disks. */ - assert_eq!(disks_list(&client, &disks_url).await.len(), 0); - /* We shouldn't find it if we request it explicitly. */ - let error = client - .make_request_error(Method::GET, &disk_url, StatusCode::NOT_FOUND) +// Nexus-side allocation: +// TODO: Disk allocation failure (not enough space!) +// TODO: Transition from Requested -> Started in simulated storage. + +#[nexus_test] +async fn test_disk_creation_region_requested_then_started(cptestctx: &ControlPlaneTestContext) { + let client = &cptestctx.external_client; + let test = DiskTest::new(&cptestctx).await; + let nexus = &cptestctx.server.apictx.nexus; + let disks_url = get_disks_url(); + + // TODO: TODO: TODO +} + + +// Tests that region allocation failure causes disk allocation to fail. +#[nexus_test] +async fn test_disk_region_creation_failure(cptestctx: &ControlPlaneTestContext) { + let client = &cptestctx.external_client; + let test = DiskTest::new(&cptestctx).await; + + // Before we create a disk, set the response from the Crucible Agent: + // no matter what regions get requested, they'll always fail. + for id in &test.dataset_ids { + let crucible = test.sled_agent.get_crucible_dataset(test.zpool_id, *id).await; + crucible.set_create_callback(Box::new(|_| { + RegionState::Failed + })).await; + } + + // Attempt to allocate the disk, observe a server error. + let disks_url = get_disks_url(); + let new_disk = params::DiskCreate { + identity: IdentityMetadataCreateParams { + name: DISK_NAME.parse().unwrap(), + description: String::from("sells rainsticks"), + }, + snapshot_id: None, + size: ByteCount::from_gibibytes_u32(1), + }; + + // Unfortunately, the error message is only posted internally to the + // logs, and it not returned to the client. + // + // TODO: Maybe consider making this a more informative error? + // Should we propagate this to the client? + client + .make_request_error_body( + Method::POST, + &disks_url, + new_disk, + StatusCode::INTERNAL_SERVER_ERROR, + ) .await; - assert_eq!(error.message, "not found: disk with name \"just-rainsticks\""); + + // After the failed allocation, the disk should not exist. + let disks = disks_list(&client, &disks_url).await; + assert_eq!(disks.len(), 0); + + // After the failed allocation, regions will exist, but be destroyed. + // + // TODO: Query DB? Maybe allocate when *almost* full? + for id in &test.dataset_ids { + let crucible = test.sled_agent.get_crucible_dataset(test.zpool_id, *id).await; + let regions = crucible.list().await; + assert_eq!(regions.len(), 1); + assert_eq!(regions[0].state, RegionState::Destroyed); + } } async fn disk_get(client: &ClientTestContext, disk_url: &str) -> Disk { diff --git a/sled-agent/src/sim/http_entrypoints_storage.rs b/sled-agent/src/sim/http_entrypoints_storage.rs index 471878c718f..f23e1955d95 100644 --- a/sled-agent/src/sim/http_entrypoints_storage.rs +++ b/sled-agent/src/sim/http_entrypoints_storage.rs @@ -17,9 +17,7 @@ use super::storage::CrucibleData; type CrucibleAgentApiDescription = ApiDescription>; -/** - * Returns a description of the sled agent API - */ +/// Returns a description of the Crucible Agent API. pub fn api() -> CrucibleAgentApiDescription { fn register_endpoints( api: &mut CrucibleAgentApiDescription, diff --git a/sled-agent/src/sim/mod.rs b/sled-agent/src/sim/mod.rs index f485b6ef507..a331f0d1b82 100644 --- a/sled-agent/src/sim/mod.rs +++ b/sled-agent/src/sim/mod.rs @@ -19,3 +19,5 @@ mod storage; pub use config::{Config, SimMode}; pub use server::{run_server, Server}; +pub use sled_agent::SledAgent; +pub use http_entrypoints_storage::State as RegionState; diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index 94f1a4ec2a8..71e0c692152 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -22,7 +22,7 @@ use super::collection::SimCollection; use super::config::SimMode; use super::disk::SimDisk; use super::instance::SimInstance; -use super::storage::Storage; +use super::storage::{CrucibleData, Storage}; /** * Simulates management of the control plane on a sled @@ -130,4 +130,13 @@ impl SledAgent { ) { self.storage.lock().await.insert_dataset(zpool_id, dataset_id).await; } + + /// Returns a crucible dataset within a particular zpool. + pub async fn get_crucible_dataset( + &self, + zpool_id: Uuid, + dataset_id: Uuid, + ) -> Arc { + self.storage.lock().await.get_dataset(zpool_id, dataset_id).await + } } diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index 60810ba8a85..258e3fa6fff 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -20,13 +20,23 @@ use uuid::Uuid; // service to a separate file. use super::http_entrypoints_storage::{CreateRegion, Region, RegionId, State}; -pub struct CrucibleDataInner { +type CreateCallback = Box State + Send + 'static>; + +struct CrucibleDataInner { regions: HashMap, + on_create: Option, } impl CrucibleDataInner { fn new() -> Self { - Self { regions: HashMap::new() } + Self { + regions: HashMap::new(), + on_create: None, + } + } + + fn set_create_callback(&mut self, callback: CreateCallback) { + self.on_create = Some(callback); } fn list(&self) -> Vec { @@ -35,6 +45,13 @@ impl CrucibleDataInner { fn create(&mut self, params: CreateRegion) -> Region { let id = Uuid::from_str(¶ms.id.0).unwrap(); + + let state = if let Some(on_create) = &self.on_create { + on_create(¶ms) + } else { + State::Requested + }; + let region = Region { id: params.id, volume_id: params.volume_id, @@ -43,13 +60,12 @@ impl CrucibleDataInner { extent_count: params.extent_count, // NOTE: This is a lie - no server is running. port_number: 0, - - // TODO: This should be "Started", we should control state - // transitions. -// state: State::Requested, - state: State::Created, + state, }; - self.regions.insert(id, region.clone()); + let old = self.regions.insert(id, region.clone()); + if let Some(old) = old { + assert_eq!(old.id, region.id, "Region already exists, but with a different ID"); + } region } @@ -58,19 +74,33 @@ impl CrucibleDataInner { self.regions.get(&id).cloned() } + fn get_mut(&mut self, id: &RegionId) -> Option<&mut Region> { + let id = Uuid::from_str(&id.0).unwrap(); + self.regions.get_mut(&id) + } + fn delete(&mut self, id: RegionId) -> Option { let id = Uuid::from_str(&id.0).unwrap(); - self.regions.remove(&id) + let mut region = self.regions.get_mut(&id)?; + region.state = State::Destroyed; + Some(region.clone()) } } +/// Represents a running Crucible Agent. Contains regions. pub struct CrucibleData { inner: Mutex, } impl CrucibleData { fn new() -> Self { - Self { inner: Mutex::new(CrucibleDataInner::new()) } + Self { + inner: Mutex::new(CrucibleDataInner::new()) + } + } + + pub async fn set_create_callback(&self, callback: CreateCallback) { + self.inner.lock().await.set_create_callback(callback); } pub async fn list(&self) -> Vec { @@ -88,17 +118,21 @@ impl CrucibleData { pub async fn delete(&self, id: RegionId) -> Option { self.inner.lock().await.delete(id) } + + pub async fn set_state(&self, id: &RegionId, state: State) { + self.inner.lock().await.get_mut(id).expect("region does not exist").state = state; + } } /// A simulated Crucible Dataset. /// /// Contains both the data and the HTTP server. -pub struct CrucibleDataset { +pub struct CrucibleServer { server: dropshot::HttpServer>, - _data: Arc, + data: Arc, } -impl CrucibleDataset { +impl CrucibleServer { fn new(log: &Logger) -> Self { let data = Arc::new(CrucibleData::new()); let config = dropshot::ConfigDropshot { @@ -116,7 +150,7 @@ impl CrucibleDataset { .expect("Could not initialize server") .start(); - CrucibleDataset { server, _data: data } + CrucibleServer { server, data } } fn address(&self) -> SocketAddr { @@ -125,7 +159,7 @@ impl CrucibleDataset { } pub struct Zpool { - datasets: HashMap, + datasets: HashMap, } impl Zpool { @@ -133,8 +167,8 @@ impl Zpool { Zpool { datasets: HashMap::new() } } - pub fn insert_dataset(&mut self, log: &Logger, id: Uuid) -> &CrucibleDataset { - self.datasets.insert(id, CrucibleDataset::new(log)); + pub fn insert_dataset(&mut self, log: &Logger, id: Uuid) -> &CrucibleServer { + self.datasets.insert(id, CrucibleServer::new(log)); self.datasets.get(&id).expect("Failed to get the dataset we just inserted") } } @@ -182,4 +216,18 @@ impl Storage { .await .expect("Failed to notify Nexus about new Dataset"); } + + pub async fn get_dataset( + &self, + zpool_id: Uuid, + dataset_id: Uuid + ) -> Arc { + self.zpools.get(&zpool_id) + .expect("Zpool does not exist") + .datasets + .get(&dataset_id) + .expect("Dataset does not exist") + .data + .clone() + } } From 291f3dad91a642500213c0a9d1fd01af8ded188f Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 6 Jan 2022 00:14:54 -0500 Subject: [PATCH 32/50] Cleaned up integration tests for disks --- nexus/src/db/datastore.rs | 73 +++++++++++---- nexus/src/nexus.rs | 51 +++++------ nexus/src/sagas.rs | 113 +++++++++++++++++++++--- nexus/tests/integration_tests/disks.rs | 117 +++++++++++++++++-------- 4 files changed, 259 insertions(+), 95 deletions(-) diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index 48e75ca7df0..f3dbc21d6e5 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -241,6 +241,31 @@ impl DataStore { }) } + /// Gets allocated regions for a disk, and the datasets to which those + /// regions belong. + /// + /// Note that this function does not validate liveness of the Disk, so it + /// may be used in a context where the disk is being deleted. + pub async fn get_allocated_regions( + &self, + disk_id: Uuid, + ) -> Result, Error> { + use db::schema::dataset::dsl as dataset_dsl; + use db::schema::region::dsl as region_dsl; + region_dsl::region + .filter(region_dsl::disk_id.eq(disk_id)) + .inner_join( + dataset_dsl::dataset + .on(region_dsl::dataset_id.eq(dataset_dsl::id)) + ) + .select((Dataset::as_select(), Region::as_select())) + .get_results_async::<(Dataset, Region)>(self.pool()) + .await + .map_err(|e| { + public_error_from_diesel_pool_shouldnt_fail(e) + }) + } + /// Idempotently allocates enough regions to back a disk. /// /// Returns the allocated regions, as well as the datasets to which they @@ -1192,15 +1217,20 @@ impl DataStore { use db::schema::disk::dsl; let now = Utc::now(); + let ok_to_delete_states = vec![ + api::external::DiskState::Detached, + api::external::DiskState::Faulted, + api::external::DiskState::Creating, + ]; + + let ok_to_delete_state_labels: Vec<_> = ok_to_delete_states.iter().map(|s| s.label()).collect(); let destroyed = api::external::DiskState::Destroyed.label(); - let detached = api::external::DiskState::Detached.label(); - let faulted = api::external::DiskState::Faulted.label(); - let creating = api::external::DiskState::Creating.label(); let result = diesel::update(dsl::disk) .filter(dsl::time_deleted.is_null()) .filter(dsl::id.eq(*disk_id)) - .filter(dsl::disk_state.eq_any(vec![detached, faulted, creating])) + .filter(dsl::disk_state.eq_any(ok_to_delete_state_labels)) + .filter(dsl::attach_instance_id.is_null()) .set((dsl::disk_state.eq(destroyed), dsl::time_deleted.eq(now))) .check_if_exists::(*disk_id) .execute_and_check(pool) @@ -1213,22 +1243,29 @@ impl DataStore { ) })?; - // TODO: We need to also de-allocate the regions associated - // with a disk - see `regions_hard_delete` - but this should only - // happen when undoing deletion is no longer possible. - // - // This will be necessary - in addition to actually sending - // requests to the Crucible Agents, requesting that the regions - // be destroyed - - match result.status { UpdateStatus::Updated => Ok(()), - UpdateStatus::NotUpdatedButExists => Err(Error::InvalidRequest { - message: format!( - "disk cannot be deleted in state \"{}\"", - result.found.runtime_state.disk_state - ), - }), + UpdateStatus::NotUpdatedButExists => { + let disk_state = result.found.state(); + if !ok_to_delete_states.contains(disk_state.state()) { + return Err(Error::InvalidRequest { + message: format!( + "disk cannot be deleted in state \"{}\"", + result.found.runtime_state.disk_state + ), + }); + } else if disk_state.is_attached() { + return Err(Error::InvalidRequest { + message: String::from("disk is attached"), + }); + } else { + // NOTE: This is a "catch-all" error case, more specific + // errors should be preferred as they're more actionable. + return Err(Error::InvalidRequest { + message: String::from("disk exists, but cannot be deleted"), + }); + } + } } } diff --git a/nexus/src/nexus.rs b/nexus/src/nexus.rs index 80dd5af92c1..f0a06f5f9f2 100644 --- a/nexus/src/nexus.rs +++ b/nexus/src/nexus.rs @@ -730,7 +730,7 @@ impl Nexus { } pub async fn project_delete_disk( - &self, + self: &Arc, opctx: &OpContext, organization_name: &Name, project_name: &Name, @@ -739,37 +739,28 @@ impl Nexus { let (disk, authz_disk) = self .project_lookup_disk(organization_name, project_name, disk_name) .await?; - let runtime = disk.runtime(); - bail_unless!(runtime.state().state() != &DiskState::Destroyed); - - if runtime.state().is_attached() { - return Err(Error::InvalidRequest { - message: String::from("disk is attached"), - }); - } - /* - * TODO-correctness It's not clear how this handles the case where we - * begin this delete operation while some other request is ongoing to - * attach the disk. We won't be able to see that in the state here. We - * might be able to detect this when we go update the disk's state to - * Attaching (because a SQL UPDATE will update 0 rows), but we'd sort of - * already be in a bad state because the destroyed disk will be - * attaching (and eventually attached) on some sled, and if the wrong - * combination of components crash at this point, we could wind up not - * fixing that state. - * - * This is a consequence of the choice _not_ to record the Attaching - * state in the database before beginning the attach process. If we did - * that, we wouldn't have this problem, but we'd have a similar problem - * of dealing with the case of a crash after recording this state and - * before actually beginning the attach process. Sagas can maybe - * address that. - */ - self.db_datastore.project_delete_disk(opctx, authz_disk).await + // TODO: We need to sort out the authorization checks. + // + // Normally, this would be coupled alongside access to the + // datastore, but now that disk deletion exists within a Saga, + // this would require OpContext to be serialized (which is + // not trivial). + opctx.authorize(authz::Action::Query, authz::DATABASE).await?; + opctx.authorize(authz::Action::Delete, authz_disk).await?; + + let saga_params = Arc::new(sagas::ParamsDiskDelete { + disk_id: disk.id() + }); + self + .execute_saga( + Arc::clone(&sagas::SAGA_DISK_DELETE_TEMPLATE), + sagas::SAGA_DISK_DELETE_NAME, + saga_params, + ) + .await?; - // TODO: Call to crucible agents, requesting deletion. - // TODO: self.db_datastore.regions_hard_delete(disk.id) + Ok(()) } /* diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index 6fe39783ff2..7fa7392f87c 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -55,11 +55,14 @@ use uuid::Uuid; */ pub const SAGA_INSTANCE_CREATE_NAME: &'static str = "instance-create"; pub const SAGA_DISK_CREATE_NAME: &'static str = "disk-create"; +pub const SAGA_DISK_DELETE_NAME: &'static str = "disk-delete"; lazy_static! { pub static ref SAGA_INSTANCE_CREATE_TEMPLATE: Arc> = Arc::new(saga_instance_create()); pub static ref SAGA_DISK_CREATE_TEMPLATE: Arc> = Arc::new(saga_disk_create()); + pub static ref SAGA_DISK_DELETE_TEMPLATE: Arc> = + Arc::new(saga_disk_delete()); } lazy_static! { @@ -80,6 +83,11 @@ fn all_templates( Arc::clone(&SAGA_DISK_CREATE_TEMPLATE) as Arc>>, ), + ( + SAGA_DISK_DELETE_NAME, + Arc::clone(&SAGA_DISK_DELETE_TEMPLATE) + as Arc>>, + ), ] .into_iter() .collect() @@ -400,10 +408,11 @@ async fn sdc_create_disk_record( let disk_id = sagactx.lookup::("disk_id")?; - // NOTE: This could be done in a txn with region allocation? + // NOTE: This could be done in a transaction alongside region allocation? // // Unclear if it's a problem to let this disk exist without any backing - // regions for a brief period of time. + // regions for a brief period of time, or if that's under the valid + // jurisdiction of "Creating". let disk = db::model::Disk::new( disk_id, params.project_id, @@ -537,14 +546,9 @@ async fn sdc_regions_ensure( Ok(()) } -async fn sdc_regions_ensure_undo( - sagactx: ActionContext, -) -> Result<(), anyhow::Error> { - let datasets_and_regions = sagactx - .lookup::>( - "datasets_and_regions", - )?; - +async fn delete_regions( + datasets_and_regions: Vec<(db::model::Dataset, db::model::Region)>, +) -> Result<(), Error> { let request_count = datasets_and_regions.len(); futures::stream::iter(datasets_and_regions) .map(|(dataset, region)| async move { @@ -562,6 +566,17 @@ async fn sdc_regions_ensure_undo( Ok(()) } +async fn sdc_regions_ensure_undo( + sagactx: ActionContext, +) -> Result<(), anyhow::Error> { + let datasets_and_regions = sagactx + .lookup::>( + "datasets_and_regions", + )?; + delete_regions(datasets_and_regions).await?; + Ok(()) +} + async fn sdc_finalize_disk_record( sagactx: ActionContext, ) -> Result<(), ActionError> { @@ -577,3 +592,81 @@ async fn sdc_finalize_disk_record( .map_err(ActionError::action_failed)?; Ok(()) } + +#[derive(Debug, Deserialize, Serialize)] +pub struct ParamsDiskDelete { + pub disk_id: Uuid, +} + +#[derive(Debug)] +pub struct SagaDiskDelete; +impl SagaType for SagaDiskDelete { + type SagaParamsType = Arc; + type ExecContextType = Arc; +} + + +fn saga_disk_delete() -> SagaTemplate { + let mut template_builder = SagaTemplateBuilder::new(); + + template_builder.append( + "no_result", + "DeleteDiskRecord", + new_action_noop_undo(sdd_delete_disk_record), + ); + + template_builder.append( + "no_result", + "DeleteRegions", + new_action_noop_undo(sdd_delete_regions), + ); + + template_builder.append( + "no_result", + "DeleteRegionRecords", + new_action_noop_undo(sdd_delete_region_records), + ); + + template_builder.build() +} + +async fn sdd_delete_disk_record( + sagactx: ActionContext, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params(); + + osagactx.datastore() + .project_delete_disk_no_auth(¶ms.disk_id) + .await + .map_err(ActionError::action_failed)?; + Ok(()) +} + +async fn sdd_delete_regions( + sagactx: ActionContext, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params(); + + let datasets_and_regions = osagactx.datastore() + .get_allocated_regions(params.disk_id) + .await + .map_err(ActionError::action_failed)?; + delete_regions(datasets_and_regions) + .await + .map_err(ActionError::action_failed)?; + Ok(()) +} + +async fn sdd_delete_region_records( + sagactx: ActionContext, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params(); + osagactx.datastore() + .regions_hard_delete(params.disk_id) + .await + .map_err(ActionError::action_failed)?; + Ok(()) +} diff --git a/nexus/tests/integration_tests/disks.rs b/nexus/tests/integration_tests/disks.rs index bfec51e8a1d..82edde4cfa4 100644 --- a/nexus/tests/integration_tests/disks.rs +++ b/nexus/tests/integration_tests/disks.rs @@ -71,6 +71,7 @@ async fn create_org_and_project(client: &ClientTestContext) -> Uuid { struct DiskTest { sled_agent: Arc, zpool_id: Uuid, + zpool_size: ByteCount, dataset_ids: Vec, project_id: Uuid, } @@ -83,8 +84,8 @@ impl DiskTest { // Create a Zpool. let zpool_id = Uuid::new_v4(); - let zpool_size = 10 * 1024 * 1024 * 1024; - sled_agent.create_zpool(zpool_id, zpool_size).await; + let zpool_size = ByteCount::from_gibibytes_u32(10); + sled_agent.create_zpool(zpool_id, zpool_size.to_bytes()).await; // Create multiple Datasets within that Zpool. let dataset_count = 3; @@ -105,24 +106,11 @@ impl DiskTest { Self { sled_agent, zpool_id, + zpool_size, dataset_ids, project_id, } } - - async fn finish_region_allocation( - &self, - ) { - for dataset_id in &self.dataset_ids { - let dataset = self.sled_agent.get_crucible_dataset(self.zpool_id, *dataset_id).await; - let regions = dataset.list().await; - - for region in ®ions { - dataset.set_state(®ion.id, RegionState::Created).await; - } - } - } - } #[nexus_test] @@ -608,10 +596,17 @@ async fn test_disk_move_between_instances(cptestctx: &ControlPlaneTestContext) { assert_eq!(disk.state, DiskState::Attaching(instance2_id.clone())); // It's not allowed to delete a disk that's attaching. - let error = client - .make_request_error(Method::DELETE, &disk_url, StatusCode::BAD_REQUEST) - .await; - assert_eq!(error.message, "disk is attached"); + let error = NexusRequest::new( + RequestBuilder::new(client, Method::DELETE, &disk_url) + .expect_status(Some(StatusCode::BAD_REQUEST)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("expected request to fail") + .parsed_body::() + .expect("cannot parse"); + assert_eq!(error.message, "disk cannot be deleted in state \"attaching\""); // Now, begin a detach while the disk is still being attached. client @@ -627,10 +622,17 @@ async fn test_disk_move_between_instances(cptestctx: &ControlPlaneTestContext) { assert_eq!(disk.state, DiskState::Detaching(instance2_id.clone())); // It's not allowed to delete a disk that's detaching, either. - let error = client - .make_request_error(Method::DELETE, &disk_url, StatusCode::BAD_REQUEST) - .await; - assert_eq!(error.message, "disk is attached"); + let error = NexusRequest::new( + RequestBuilder::new(client, Method::DELETE, &disk_url) + .expect_status(Some(StatusCode::BAD_REQUEST)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("expected request to fail") + .parsed_body::() + .expect("cannot parse"); + assert_eq!(error.message, "disk cannot be deleted in state \"detaching\""); // Finish detachment. disk_simulate(nexus, &disk.identity.id).await; @@ -695,18 +697,39 @@ async fn test_disk_deletion_requires_authentication(cptestctx: &ControlPlaneTest .expect("failed to delete disk"); } -// Nexus-side allocation: -// TODO: Disk allocation failure (not enough space!) -// TODO: Transition from Requested -> Started in simulated storage. - #[nexus_test] async fn test_disk_creation_region_requested_then_started(cptestctx: &ControlPlaneTestContext) { let client = &cptestctx.external_client; let test = DiskTest::new(&cptestctx).await; - let nexus = &cptestctx.server.apictx.nexus; let disks_url = get_disks_url(); - // TODO: TODO: TODO + // Before we create a disk, set the response from the Crucible Agent: + // no matter what regions get requested, they'll always *start* as + // "Requested", and transition to "Created" on the second call. + for id in &test.dataset_ids { + let crucible = test.sled_agent.get_crucible_dataset(test.zpool_id, *id).await; + let called = std::sync::atomic::AtomicBool::new(false); + crucible.set_create_callback(Box::new(move |_| { + if !called.load(std::sync::atomic::Ordering::SeqCst) { + called.store(true, std::sync::atomic::Ordering::SeqCst); + RegionState::Requested + } else { + RegionState::Created + } + })).await; + } + + // The disk is created successfully, even when this "requested" -> "started" + // transition occurs. + let new_disk = params::DiskCreate { + identity: IdentityMetadataCreateParams { + name: DISK_NAME.parse().unwrap(), + description: String::from("sells rainsticks"), + }, + snapshot_id: None, + size: ByteCount::from_gibibytes_u32(1), + }; + let _: Disk = objects_post(&client, &disks_url, new_disk.clone()).await; } @@ -725,6 +748,17 @@ async fn test_disk_region_creation_failure(cptestctx: &ControlPlaneTestContext) })).await; } + let disk_size = ByteCount::from_gibibytes_u32(3); + let dataset_count = test.dataset_ids.len() as u64; + assert!( + disk_size.to_bytes() * dataset_count < test.zpool_size.to_bytes(), + "Disk size too big for Zpool size" + ); + assert!( + 2 * disk_size.to_bytes() * dataset_count > test.zpool_size.to_bytes(), + "(test constraint) Zpool needs to be smaller (to store only one disk)", + ); + // Attempt to allocate the disk, observe a server error. let disks_url = get_disks_url(); let new_disk = params::DiskCreate { @@ -733,19 +767,19 @@ async fn test_disk_region_creation_failure(cptestctx: &ControlPlaneTestContext) description: String::from("sells rainsticks"), }, snapshot_id: None, - size: ByteCount::from_gibibytes_u32(1), + size: disk_size, }; // Unfortunately, the error message is only posted internally to the // logs, and it not returned to the client. // // TODO: Maybe consider making this a more informative error? - // Should we propagate this to the client? + // How should we propagate this to the client? client .make_request_error_body( Method::POST, &disks_url, - new_disk, + new_disk.clone(), StatusCode::INTERNAL_SERVER_ERROR, ) .await; @@ -754,15 +788,24 @@ async fn test_disk_region_creation_failure(cptestctx: &ControlPlaneTestContext) let disks = disks_list(&client, &disks_url).await; assert_eq!(disks.len(), 0); - // After the failed allocation, regions will exist, but be destroyed. - // - // TODO: Query DB? Maybe allocate when *almost* full? + // After the failed allocation, regions will exist, but be "Failed". for id in &test.dataset_ids { let crucible = test.sled_agent.get_crucible_dataset(test.zpool_id, *id).await; let regions = crucible.list().await; assert_eq!(regions.len(), 1); - assert_eq!(regions[0].state, RegionState::Destroyed); + assert_eq!(regions[0].state, RegionState::Failed); } + + // Validate that the underlying regions were released as a part of + // unwinding the failed disk allocation, by performing another disk + // allocation that should succeed. + for id in &test.dataset_ids { + let crucible = test.sled_agent.get_crucible_dataset(test.zpool_id, *id).await; + crucible.set_create_callback(Box::new(|_| { + RegionState::Created + })).await; + } + let _: Disk = objects_post(&client, &disks_url, new_disk.clone()).await; } async fn disk_get(client: &ClientTestContext, disk_url: &str) -> Disk { From ad1defdb44b8fc90a06bc407cd3290517bd6a5ec Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 6 Jan 2022 00:20:09 -0500 Subject: [PATCH 33/50] fmt --- nexus/src/db/datastore.rs | 45 +++---- nexus/src/nexus.rs | 18 ++- nexus/src/sagas.rs | 36 +++-- nexus/tests/integration_tests/disks.rs | 173 ++++++++++++++----------- sled-agent/src/sim/mod.rs | 2 +- sled-agent/src/sim/storage.rs | 60 ++++++--- 6 files changed, 184 insertions(+), 150 deletions(-) diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index ef492f8f334..2e5427b9bb4 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -256,14 +256,12 @@ impl DataStore { .filter(region_dsl::disk_id.eq(disk_id)) .inner_join( dataset_dsl::dataset - .on(region_dsl::dataset_id.eq(dataset_dsl::id)) + .on(region_dsl::dataset_id.eq(dataset_dsl::id)), ) .select((Dataset::as_select(), Region::as_select())) .get_results_async::<(Dataset, Region)>(self.pool()) .await - .map_err(|e| { - public_error_from_diesel_pool_shouldnt_fail(e) - }) + .map_err(|e| public_error_from_diesel_pool_shouldnt_fail(e)) } /// Idempotently allocates enough regions to back a disk. @@ -315,7 +313,7 @@ impl DataStore { .filter(region_dsl::disk_id.eq(disk_id)) .inner_join( dataset_dsl::dataset - .on(region_dsl::dataset_id.eq(dataset_dsl::id)) + .on(region_dsl::dataset_id.eq(dataset_dsl::id)), ) .select((Dataset::as_select(), Region::as_select())) .get_results::<(Dataset, Region)>(conn)?; @@ -387,11 +385,8 @@ impl DataStore { } /// Deletes all regions backing a disk. - pub async fn regions_hard_delete( - &self, - disk_id: Uuid, - ) -> DeleteResult { - use db::schema::region::dsl as dsl; + pub async fn regions_hard_delete(&self, disk_id: Uuid) -> DeleteResult { + use db::schema::region::dsl; diesel::delete(dsl::region) .filter(dsl::disk_id.eq(disk_id)) @@ -1182,7 +1177,11 @@ impl DataStore { ) -> DeleteResult { let disk_id = disk_authz.id(); opctx.authorize(authz::Action::Delete, disk_authz).await?; - self.project_delete_disk_internal(disk_id, self.pool_authorized(opctx).await?).await + self.project_delete_disk_internal( + disk_id, + self.pool_authorized(opctx).await?, + ) + .await } // TODO: Delete me (this function, not the disk!), ensure all datastore @@ -1223,7 +1222,8 @@ impl DataStore { api::external::DiskState::Creating, ]; - let ok_to_delete_state_labels: Vec<_> = ok_to_delete_states.iter().map(|s| s.label()).collect(); + let ok_to_delete_state_labels: Vec<_> = + ok_to_delete_states.iter().map(|s| s.label()).collect(); let destroyed = api::external::DiskState::Destroyed.label(); let result = diesel::update(dsl::disk) @@ -1262,7 +1262,9 @@ impl DataStore { // NOTE: This is a "catch-all" error case, more specific // errors should be preferred as they're more actionable. return Err(Error::InvalidRequest { - message: String::from("disk exists, but cannot be deleted"), + message: String::from( + "disk exists, but cannot be deleted", + ), }); } } @@ -2817,12 +2819,10 @@ mod test { ByteCount::from_mebibytes_u32(500), ); let disk_id = Uuid::new_v4(); - let mut dataset_and_regions1 = datastore.region_allocate(disk_id, ¶ms) - .await - .unwrap(); - let mut dataset_and_regions2 = datastore.region_allocate(disk_id, ¶ms) - .await - .unwrap(); + let mut dataset_and_regions1 = + datastore.region_allocate(disk_id, ¶ms).await.unwrap(); + let mut dataset_and_regions2 = + datastore.region_allocate(disk_id, ¶ms).await.unwrap(); // Give them a consistent order so we can easily compare them. let sort_vec = |v: &mut Vec<(Dataset, Region)>| { @@ -2830,7 +2830,7 @@ mod test { let order = d1.id().cmp(&d2.id()); match order { std::cmp::Ordering::Equal => r1.id().cmp(&r2.id()), - _ => order + _ => order, } }); }; @@ -2840,10 +2840,7 @@ mod test { // Validate that the two calls to allocate return the same data. assert_eq!(dataset_and_regions1.len(), dataset_and_regions2.len()); for i in 0..dataset_and_regions1.len() { - assert_eq!( - dataset_and_regions1[i], - dataset_and_regions2[i], - ); + assert_eq!(dataset_and_regions1[i], dataset_and_regions2[i],); } let _ = db.cleanup().await; diff --git a/nexus/src/nexus.rs b/nexus/src/nexus.rs index f0a06f5f9f2..15d3bb4b9d1 100644 --- a/nexus/src/nexus.rs +++ b/nexus/src/nexus.rs @@ -749,16 +749,14 @@ impl Nexus { opctx.authorize(authz::Action::Query, authz::DATABASE).await?; opctx.authorize(authz::Action::Delete, authz_disk).await?; - let saga_params = Arc::new(sagas::ParamsDiskDelete { - disk_id: disk.id() - }); - self - .execute_saga( - Arc::clone(&sagas::SAGA_DISK_DELETE_TEMPLATE), - sagas::SAGA_DISK_DELETE_NAME, - saga_params, - ) - .await?; + let saga_params = + Arc::new(sagas::ParamsDiskDelete { disk_id: disk.id() }); + self.execute_saga( + Arc::clone(&sagas::SAGA_DISK_DELETE_TEMPLATE), + sagas::SAGA_DISK_DELETE_NAME, + saga_params, + ) + .await?; Ok(()) } diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index 7fa7392f87c..f4ce09904e1 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -36,6 +36,7 @@ use omicron_common::api::internal::sled_agent::InstanceHardware; use omicron_common::backoff::{self, BackoffError}; use serde::Deserialize; use serde::Serialize; +use slog::Logger; use std::collections::BTreeMap; use std::convert::TryFrom; use std::sync::Arc; @@ -47,7 +48,6 @@ use steno::SagaTemplate; use steno::SagaTemplateBuilder; use steno::SagaTemplateGeneric; use steno::SagaType; -use slog::Logger; use uuid::Uuid; /* @@ -370,19 +370,13 @@ fn saga_disk_create() -> SagaTemplate { template_builder.append( "datasets_and_regions", "AllocRegions", - ActionFunc::new_action( - sdc_alloc_regions, - sdc_alloc_regions_undo, - ), + ActionFunc::new_action(sdc_alloc_regions, sdc_alloc_regions_undo), ); template_builder.append( "regions_ensure", "RegionsEnsure", - ActionFunc::new_action( - sdc_regions_ensure, - sdc_regions_ensure_undo, - ), + ActionFunc::new_action(sdc_regions_ensure, sdc_regions_ensure_undo), ); template_builder.append( @@ -433,10 +427,7 @@ async fn sdc_create_disk_record_undo( let osagactx = sagactx.user_data(); let disk_id = sagactx.lookup::("disk_id")?; - osagactx - .datastore() - .project_delete_disk_no_auth(&disk_id) - .await?; + osagactx.datastore().project_delete_disk_no_auth(&disk_id).await?; Ok(()) } @@ -466,9 +457,7 @@ async fn sdc_alloc_regions_undo( let osagactx = sagactx.user_data(); let disk_id = sagactx.lookup::("disk_id")?; - osagactx.datastore() - .regions_hard_delete(disk_id) - .await?; + osagactx.datastore().regions_hard_delete(disk_id).await?; Ok(()) } @@ -508,7 +497,10 @@ async fn ensure_region_in_dataset( }; let log_create_failure = |_, delay| { - warn!(log, "Region requested, not yet created. Retrying in {:?}", delay); + warn!( + log, + "Region requested, not yet created. Retrying in {:?}", delay + ); }; let region = backoff::retry_notify( @@ -605,7 +597,6 @@ impl SagaType for SagaDiskDelete { type ExecContextType = Arc; } - fn saga_disk_delete() -> SagaTemplate { let mut template_builder = SagaTemplateBuilder::new(); @@ -636,7 +627,8 @@ async fn sdd_delete_disk_record( let osagactx = sagactx.user_data(); let params = sagactx.saga_params(); - osagactx.datastore() + osagactx + .datastore() .project_delete_disk_no_auth(¶ms.disk_id) .await .map_err(ActionError::action_failed)?; @@ -649,7 +641,8 @@ async fn sdd_delete_regions( let osagactx = sagactx.user_data(); let params = sagactx.saga_params(); - let datasets_and_regions = osagactx.datastore() + let datasets_and_regions = osagactx + .datastore() .get_allocated_regions(params.disk_id) .await .map_err(ActionError::action_failed)?; @@ -664,7 +657,8 @@ async fn sdd_delete_region_records( ) -> Result<(), ActionError> { let osagactx = sagactx.user_data(); let params = sagactx.saga_params(); - osagactx.datastore() + osagactx + .datastore() .regions_hard_delete(params.disk_id) .await .map_err(ActionError::action_failed)?; diff --git a/nexus/tests/integration_tests/disks.rs b/nexus/tests/integration_tests/disks.rs index 82edde4cfa4..d47dee2fb55 100644 --- a/nexus/tests/integration_tests/disks.rs +++ b/nexus/tests/integration_tests/disks.rs @@ -89,32 +89,29 @@ impl DiskTest { // Create multiple Datasets within that Zpool. let dataset_count = 3; - let dataset_ids: Vec<_> = (0..dataset_count).map(|_| Uuid::new_v4()).collect(); + let dataset_ids: Vec<_> = + (0..dataset_count).map(|_| Uuid::new_v4()).collect(); for id in &dataset_ids { sled_agent.create_crucible_dataset(zpool_id, *id).await; // By default, regions are created immediately. let crucible = sled_agent.get_crucible_dataset(zpool_id, *id).await; - crucible.set_create_callback(Box::new(|_| { - RegionState::Created - })).await; + crucible + .set_create_callback(Box::new(|_| RegionState::Created)) + .await; } // Create a project for testing. let project_id = create_org_and_project(&client).await; - Self { - sled_agent, - zpool_id, - zpool_size, - dataset_ids, - project_id, - } + Self { sled_agent, zpool_id, zpool_size, dataset_ids, project_id } } } #[nexus_test] -async fn test_disk_not_found_before_creation(cptestctx: &ControlPlaneTestContext) { +async fn test_disk_not_found_before_creation( + cptestctx: &ControlPlaneTestContext, +) { let client = &cptestctx.external_client; DiskTest::new(&cptestctx).await; let disks_url = get_disks_url(); @@ -128,7 +125,10 @@ async fn test_disk_not_found_before_creation(cptestctx: &ControlPlaneTestContext let error = client .make_request_error(Method::GET, &disk_url, StatusCode::NOT_FOUND) .await; - assert_eq!(error.message, format!("not found: disk with name \"{}\"", DISK_NAME)); + assert_eq!( + error.message, + format!("not found: disk with name \"{}\"", DISK_NAME) + ); // We should also get a 404 if we delete one. let error = NexusRequest::new( @@ -141,11 +141,16 @@ async fn test_disk_not_found_before_creation(cptestctx: &ControlPlaneTestContext .expect("unexpected success") .parsed_body::() .unwrap(); - assert_eq!(error.message, format!("not found: disk with name \"{}\"", DISK_NAME)); + assert_eq!( + error.message, + format!("not found: disk with name \"{}\"", DISK_NAME) + ); } #[nexus_test] -async fn test_disk_create_attach_detach_delete(cptestctx: &ControlPlaneTestContext) { +async fn test_disk_create_attach_detach_delete( + cptestctx: &ControlPlaneTestContext, +) { let client = &cptestctx.external_client; let test = DiskTest::new(&cptestctx).await; let nexus = &cptestctx.server.apictx.nexus; @@ -204,18 +209,15 @@ async fn test_disk_create_attach_detach_delete(cptestctx: &ControlPlaneTestConte // Verify that there are no disks attached to the instance, and specifically // that our disk is not attached to this instance. - let url_instance_disks = get_instance_disks_url( - instance.identity.name.as_str() - ); + let url_instance_disks = + get_instance_disks_url(instance.identity.name.as_str()); let disks = objects_list_page::(&client, &url_instance_disks).await; assert_eq!(disks.items.len(), 0); - let url_instance_attach_disk = get_disk_attach_url( - instance.identity.name.as_str(), - ); - let url_instance_detach_disk = get_disk_detach_url( - instance.identity.name.as_str(), - ); + let url_instance_attach_disk = + get_disk_attach_url(instance.identity.name.as_str()); + let url_instance_detach_disk = + get_disk_detach_url(instance.identity.name.as_str()); // Start attaching the disk to the instance. let mut response = client @@ -298,11 +300,16 @@ async fn test_disk_create_attach_detach_delete(cptestctx: &ControlPlaneTestConte let error = client .make_request_error(Method::GET, &disk_url, StatusCode::NOT_FOUND) .await; - assert_eq!(error.message, format!("not found: disk with name \"{}\"", DISK_NAME)); + assert_eq!( + error.message, + format!("not found: disk with name \"{}\"", DISK_NAME) + ); } #[nexus_test] -async fn test_disk_create_disk_that_already_exists_fails(cptestctx: &ControlPlaneTestContext) { +async fn test_disk_create_disk_that_already_exists_fails( + cptestctx: &ControlPlaneTestContext, +) { let client = &cptestctx.external_client; DiskTest::new(&cptestctx).await; let disks_url = get_disks_url(); @@ -329,7 +336,10 @@ async fn test_disk_create_disk_that_already_exists_fails(cptestctx: &ControlPlan StatusCode::BAD_REQUEST, ) .await; - assert_eq!(error.message, format!("already exists: disk \"{}\"", DISK_NAME)); + assert_eq!( + error.message, + format!("already exists: disk \"{}\"", DISK_NAME) + ); // List disks again and expect to find the one we just created. let disks = disks_list(&client, &disks_url).await; @@ -375,18 +385,15 @@ async fn test_disk_move_between_instances(cptestctx: &ControlPlaneTestContext) { // Verify that there are no disks attached to the instance, and specifically // that our disk is not attached to this instance. - let url_instance_disks = get_instance_disks_url( - instance.identity.name.as_str() - ); + let url_instance_disks = + get_instance_disks_url(instance.identity.name.as_str()); let disks = objects_list_page::(&client, &url_instance_disks).await; assert_eq!(disks.items.len(), 0); - let url_instance_attach_disk = get_disk_attach_url( - instance.identity.name.as_str(), - ); - let url_instance_detach_disk = get_disk_detach_url( - instance.identity.name.as_str(), - ); + let url_instance_attach_disk = + get_disk_attach_url(instance.identity.name.as_str()); + let url_instance_detach_disk = + get_disk_detach_url(instance.identity.name.as_str()); // Start attaching the disk to the instance. let mut response = client @@ -442,12 +449,10 @@ async fn test_disk_move_between_instances(cptestctx: &ControlPlaneTestContext) { }, ) .await; - let url_instance2_attach_disk = get_disk_attach_url( - instance2.identity.name.as_str(), - ); - let url_instance2_detach_disk = get_disk_detach_url( - instance2.identity.name.as_str(), - ); + let url_instance2_attach_disk = + get_disk_attach_url(instance2.identity.name.as_str()); + let url_instance2_detach_disk = + get_disk_detach_url(instance2.identity.name.as_str()); let error = client .make_request_error_body( Method::POST, @@ -458,8 +463,11 @@ async fn test_disk_move_between_instances(cptestctx: &ControlPlaneTestContext) { .await; assert_eq!( error.message, - format!("cannot attach disk \"{}\": disk is attached to another \ - instance", DISK_NAME) + format!( + "cannot attach disk \"{}\": disk is attached to another \ + instance", + DISK_NAME + ) ); let attached_disk = disk_get(&client, &disk_url).await; @@ -489,8 +497,11 @@ async fn test_disk_move_between_instances(cptestctx: &ControlPlaneTestContext) { .await; assert_eq!( error.message, - format!("cannot attach disk \"{}\": disk is attached to another \ - instance", DISK_NAME) + format!( + "cannot attach disk \"{}\": disk is attached to another \ + instance", + DISK_NAME + ) ); // It's even illegal to attach this disk back to the same instance. @@ -505,8 +516,11 @@ async fn test_disk_move_between_instances(cptestctx: &ControlPlaneTestContext) { // TODO-debug the error message here is misleading. assert_eq!( error.message, - format!("cannot attach disk \"{}\": disk is attached to another \ - instance", DISK_NAME) + format!( + "cannot attach disk \"{}\": disk is attached to another \ + instance", + DISK_NAME + ) ); // However, there's no problem attempting to detach it again. @@ -578,8 +592,11 @@ async fn test_disk_move_between_instances(cptestctx: &ControlPlaneTestContext) { .await; assert_eq!( error.message, - format!("cannot attach disk \"{}\": disk is attached to another \ - instance", DISK_NAME) + format!( + "cannot attach disk \"{}\": disk is attached to another \ + instance", + DISK_NAME + ) ); // It's fine to attempt another attachment to the same instance. @@ -652,11 +669,16 @@ async fn test_disk_move_between_instances(cptestctx: &ControlPlaneTestContext) { let error = client .make_request_error(Method::GET, &disk_url, StatusCode::NOT_FOUND) .await; - assert_eq!(error.message, format!("not found: disk with name \"{}\"", DISK_NAME)); + assert_eq!( + error.message, + format!("not found: disk with name \"{}\"", DISK_NAME) + ); } #[nexus_test] -async fn test_disk_deletion_requires_authentication(cptestctx: &ControlPlaneTestContext) { +async fn test_disk_deletion_requires_authentication( + cptestctx: &ControlPlaneTestContext, +) { let client = &cptestctx.external_client; DiskTest::new(&cptestctx).await; let disks_url = get_disks_url(); @@ -698,7 +720,9 @@ async fn test_disk_deletion_requires_authentication(cptestctx: &ControlPlaneTest } #[nexus_test] -async fn test_disk_creation_region_requested_then_started(cptestctx: &ControlPlaneTestContext) { +async fn test_disk_creation_region_requested_then_started( + cptestctx: &ControlPlaneTestContext, +) { let client = &cptestctx.external_client; let test = DiskTest::new(&cptestctx).await; let disks_url = get_disks_url(); @@ -707,16 +731,19 @@ async fn test_disk_creation_region_requested_then_started(cptestctx: &ControlPla // no matter what regions get requested, they'll always *start* as // "Requested", and transition to "Created" on the second call. for id in &test.dataset_ids { - let crucible = test.sled_agent.get_crucible_dataset(test.zpool_id, *id).await; + let crucible = + test.sled_agent.get_crucible_dataset(test.zpool_id, *id).await; let called = std::sync::atomic::AtomicBool::new(false); - crucible.set_create_callback(Box::new(move |_| { - if !called.load(std::sync::atomic::Ordering::SeqCst) { - called.store(true, std::sync::atomic::Ordering::SeqCst); - RegionState::Requested - } else { - RegionState::Created - } - })).await; + crucible + .set_create_callback(Box::new(move |_| { + if !called.load(std::sync::atomic::Ordering::SeqCst) { + called.store(true, std::sync::atomic::Ordering::SeqCst); + RegionState::Requested + } else { + RegionState::Created + } + })) + .await; } // The disk is created successfully, even when this "requested" -> "started" @@ -732,20 +759,20 @@ async fn test_disk_creation_region_requested_then_started(cptestctx: &ControlPla let _: Disk = objects_post(&client, &disks_url, new_disk.clone()).await; } - // Tests that region allocation failure causes disk allocation to fail. #[nexus_test] -async fn test_disk_region_creation_failure(cptestctx: &ControlPlaneTestContext) { +async fn test_disk_region_creation_failure( + cptestctx: &ControlPlaneTestContext, +) { let client = &cptestctx.external_client; let test = DiskTest::new(&cptestctx).await; // Before we create a disk, set the response from the Crucible Agent: // no matter what regions get requested, they'll always fail. for id in &test.dataset_ids { - let crucible = test.sled_agent.get_crucible_dataset(test.zpool_id, *id).await; - crucible.set_create_callback(Box::new(|_| { - RegionState::Failed - })).await; + let crucible = + test.sled_agent.get_crucible_dataset(test.zpool_id, *id).await; + crucible.set_create_callback(Box::new(|_| RegionState::Failed)).await; } let disk_size = ByteCount::from_gibibytes_u32(3); @@ -790,7 +817,8 @@ async fn test_disk_region_creation_failure(cptestctx: &ControlPlaneTestContext) // After the failed allocation, regions will exist, but be "Failed". for id in &test.dataset_ids { - let crucible = test.sled_agent.get_crucible_dataset(test.zpool_id, *id).await; + let crucible = + test.sled_agent.get_crucible_dataset(test.zpool_id, *id).await; let regions = crucible.list().await; assert_eq!(regions.len(), 1); assert_eq!(regions[0].state, RegionState::Failed); @@ -800,10 +828,9 @@ async fn test_disk_region_creation_failure(cptestctx: &ControlPlaneTestContext) // unwinding the failed disk allocation, by performing another disk // allocation that should succeed. for id in &test.dataset_ids { - let crucible = test.sled_agent.get_crucible_dataset(test.zpool_id, *id).await; - crucible.set_create_callback(Box::new(|_| { - RegionState::Created - })).await; + let crucible = + test.sled_agent.get_crucible_dataset(test.zpool_id, *id).await; + crucible.set_create_callback(Box::new(|_| RegionState::Created)).await; } let _: Disk = objects_post(&client, &disks_url, new_disk.clone()).await; } diff --git a/sled-agent/src/sim/mod.rs b/sled-agent/src/sim/mod.rs index a331f0d1b82..3e26a1bbfc3 100644 --- a/sled-agent/src/sim/mod.rs +++ b/sled-agent/src/sim/mod.rs @@ -18,6 +18,6 @@ mod sled_agent; mod storage; pub use config::{Config, SimMode}; +pub use http_entrypoints_storage::State as RegionState; pub use server::{run_server, Server}; pub use sled_agent::SledAgent; -pub use http_entrypoints_storage::State as RegionState; diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index 258e3fa6fff..115cd8e4a55 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -5,8 +5,10 @@ //! Simulated sled agent storage implementation use futures::lock::Mutex; +use nexus_client::types::{ + ByteCount, DatasetKind, DatasetPutRequest, ZpoolPutRequest, +}; use nexus_client::Client as NexusClient; -use nexus_client::types::{ByteCount, DatasetKind, DatasetPutRequest, ZpoolPutRequest}; use slog::Logger; use std::collections::HashMap; use std::net::SocketAddr; @@ -29,10 +31,7 @@ struct CrucibleDataInner { impl CrucibleDataInner { fn new() -> Self { - Self { - regions: HashMap::new(), - on_create: None, - } + Self { regions: HashMap::new(), on_create: None } } fn set_create_callback(&mut self, callback: CreateCallback) { @@ -64,7 +63,10 @@ impl CrucibleDataInner { }; let old = self.regions.insert(id, region.clone()); if let Some(old) = old { - assert_eq!(old.id, region.id, "Region already exists, but with a different ID"); + assert_eq!( + old.id, region.id, + "Region already exists, but with a different ID" + ); } region } @@ -94,9 +96,7 @@ pub struct CrucibleData { impl CrucibleData { fn new() -> Self { - Self { - inner: Mutex::new(CrucibleDataInner::new()) - } + Self { inner: Mutex::new(CrucibleDataInner::new()) } } pub async fn set_create_callback(&self, callback: CreateCallback) { @@ -120,7 +120,12 @@ impl CrucibleData { } pub async fn set_state(&self, id: &RegionId, state: State) { - self.inner.lock().await.get_mut(id).expect("region does not exist").state = state; + self.inner + .lock() + .await + .get_mut(id) + .expect("region does not exist") + .state = state; } } @@ -167,9 +172,15 @@ impl Zpool { Zpool { datasets: HashMap::new() } } - pub fn insert_dataset(&mut self, log: &Logger, id: Uuid) -> &CrucibleServer { + pub fn insert_dataset( + &mut self, + log: &Logger, + id: Uuid, + ) -> &CrucibleServer { self.datasets.insert(id, CrucibleServer::new(log)); - self.datasets.get(&id).expect("Failed to get the dataset we just inserted") + self.datasets + .get(&id) + .expect("Failed to get the dataset we just inserted") } } @@ -182,7 +193,11 @@ pub struct Storage { } impl Storage { - pub fn new(sled_id: Uuid, nexus_client: Arc, log: Logger) -> Self { + pub fn new( + sled_id: Uuid, + nexus_client: Arc, + log: Logger, + ) -> Self { Self { sled_id, nexus_client, log, zpools: HashMap::new() } } @@ -192,10 +207,9 @@ impl Storage { self.zpools.insert(zpool_id, Zpool::new()); // Notify Nexus - let request = ZpoolPutRequest { - size: ByteCount(size), - }; - self.nexus_client.zpool_put(&self.sled_id, &zpool_id, &request) + let request = ZpoolPutRequest { size: ByteCount(size) }; + self.nexus_client + .zpool_put(&self.sled_id, &zpool_id, &request) .await .expect("Failed to notify Nexus about new Zpool"); } @@ -203,7 +217,9 @@ impl Storage { /// Adds a Dataset to the sled's simulated storage and notifies Nexus. pub async fn insert_dataset(&mut self, zpool_id: Uuid, dataset_id: Uuid) { // Update our local data - let dataset = self.zpools.get_mut(&zpool_id) + let dataset = self + .zpools + .get_mut(&zpool_id) .expect("Zpool does not exist") .insert_dataset(&self.log, dataset_id); @@ -212,7 +228,8 @@ impl Storage { address: dataset.address().to_string(), kind: DatasetKind::Crucible, }; - self.nexus_client.dataset_put(&zpool_id, &dataset_id, &request) + self.nexus_client + .dataset_put(&zpool_id, &dataset_id, &request) .await .expect("Failed to notify Nexus about new Dataset"); } @@ -220,9 +237,10 @@ impl Storage { pub async fn get_dataset( &self, zpool_id: Uuid, - dataset_id: Uuid + dataset_id: Uuid, ) -> Arc { - self.zpools.get(&zpool_id) + self.zpools + .get(&zpool_id) .expect("Zpool does not exist") .datasets .get(&dataset_id) From cfb923ca77beb2cbb0ee7364d60a4009e150f634 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 6 Jan 2022 00:49:36 -0500 Subject: [PATCH 34/50] Cleanup comments --- nexus/src/db/datastore.rs | 14 ++++++++--- nexus/src/db/schema.rs | 3 --- nexus/test-utils/src/lib.rs | 1 - sled-agent/Cargo.toml | 3 +-- .../src/sim/http_entrypoints_storage.rs | 25 ++++++++++++------- sled-agent/src/sim/storage.rs | 4 --- 6 files changed, 27 insertions(+), 23 deletions(-) diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index 2e5427b9bb4..9d8419fc221 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -82,6 +82,7 @@ use crate::db::{ }; // Number of unique datasets required to back a region. +// TODO: This should likely turn into a configuration option. const REGION_REDUNDANCY_THRESHOLD: usize = 3; pub struct DataStore { @@ -330,8 +331,15 @@ impl DataStore { // Next, observe all the regions allocated to each dataset, and // determine how much space they're using. // - // NOTE: We could store "free/allocated" space per-dataset, and keep - // them up-to-date, rather than trying to recompute this. + // TODO: We could store "free/allocated" space per-dataset, + // and keep them up-to-date, rather than trying to recompute + // this. + // + // TODO: We admittedly don't actually *fail* any request for + // running out of space - we try to send the request down to + // crucible agents, and expect them to fail on our behalf in + // out-of-storage conditions. This should undoubtedly be + // handled more explicitly. .left_outer_join( region_dsl::region .on(dataset_dsl::id.eq(region_dsl::dataset_id)), @@ -2710,8 +2718,6 @@ mod test { } } - // TODO: Test region allocation when running out of space. - #[tokio::test] async fn test_region_allocation() { let logctx = dev::test_setup_log("test_region_allocation"); diff --git a/nexus/src/db/schema.rs b/nexus/src/db/schema.rs index d7ebc38fe91..0067899e2be 100644 --- a/nexus/src/db/schema.rs +++ b/nexus/src/db/schema.rs @@ -177,9 +177,6 @@ table! { } } -// TODO: Free/allocated space here? How do we know we're okay to alloc? -// -// Maybe just "total size" of dataset, and we can figure out the rest? table! { dataset (id) { id -> Uuid, diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index 20d85de7398..fb7bc555e82 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -167,7 +167,6 @@ pub async fn test_setup_with_config( } } -// TODO: We probably want to have the ability to expand this config. pub async fn start_sled_agent( log: Logger, nexus_address: SocketAddr, diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 2affd833134..c9066bb7eda 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -11,8 +11,7 @@ bincode = "1.3.3" bytes = "1.1" cfg-if = "1.0" chrono = { version = "0.4", features = [ "serde" ] } -# Simulated sled agent only. -# TODO: Can we avoid having this in the deps list for non-sim SA? +# Only used by the simulated sled agent. crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "de022b8a" } dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] } futures = "0.3.18" diff --git a/sled-agent/src/sim/http_entrypoints_storage.rs b/sled-agent/src/sim/http_entrypoints_storage.rs index f23e1955d95..a8f79f0d213 100644 --- a/sled-agent/src/sim/http_entrypoints_storage.rs +++ b/sled-agent/src/sim/http_entrypoints_storage.rs @@ -4,7 +4,6 @@ //! HTTP entrypoint functions for simulating the storage agent API. -// use crucible_agent_client::types::{CreateRegion, RegionId}; use dropshot::{ endpoint, ApiDescription, HttpError, HttpResponseDeleted, HttpResponseOk, Path as TypedPath, RequestContext, TypedBody, @@ -41,12 +40,20 @@ pub fn api() -> CrucibleAgentApiDescription { // I need to re-define all structs used in the crucible agent // API to ensure they have the traits I need. The ones re-exported // through the client bindings, i.e., crucible_agent_client::types, -// don't implement what I need. +// don't implement the "JsonSchema" trait, and cannot be used as +// parameters to these Dropshot endpoints. // // I'd like them to! If we could ensure the generated client // also implemented e.g. JsonSchema, this might work? -// -// TODO: Try w/RegionId or State first? + +// To quickly test the type compatibility, uncomment the lines below, +// and remove the hand-rolled implementations. + +// pub type RegionId = crucible_agent_client::types::RegionId; +// pub type State = crucible_agent_client::types::State; +// pub type CreateRegion = crucible_agent_client::types::CreateRegion; +// pub type Region = crucible_agent_client::types::Region; +// pub type RegionPath = crucible_agent_client::types::RegionPath; #[derive( Serialize, @@ -94,6 +101,11 @@ pub struct Region { pub state: State, } +#[derive(Deserialize, JsonSchema)] +struct RegionPath { + id: RegionId, +} + #[endpoint { method = GET, path = "/crucible/0/regions", @@ -119,11 +131,6 @@ async fn region_create( Ok(HttpResponseOk(crucible.create(params).await)) } -#[derive(Deserialize, JsonSchema)] -struct RegionPath { - id: RegionId, -} - #[endpoint { method = GET, path = "/crucible/0/regions/{id}", diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index 115cd8e4a55..f6049a17de1 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -16,10 +16,6 @@ use std::str::FromStr; use std::sync::Arc; use uuid::Uuid; -// XXX Don't really like this import. -// -// Maybe refactor the "types" used by the HTTP -// service to a separate file. use super::http_entrypoints_storage::{CreateRegion, Region, RegionId, State}; type CreateCallback = Box State + Send + 'static>; From c3bd755ec87ff2436923719475bb2140caf810c5 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 7 Jan 2022 10:01:21 -0500 Subject: [PATCH 35/50] Make use of the #derive(JsonSchema) addition to progenitor --- Cargo.lock | 15 ++-- Cargo.toml | 6 +- nexus-client/Cargo.toml | 1 + nexus/tests/integration_tests/disks.rs | 3 +- oximeter-client/Cargo.toml | 1 + oximeter/producer/src/lib.rs | 3 +- sled-agent-client/Cargo.toml | 1 + sled-agent/src/sim/disk.rs | 3 +- .../src/sim/http_entrypoints_storage.rs | 70 ++----------------- sled-agent/src/sim/instance.rs | 3 +- sled-agent/src/sim/mod.rs | 1 - sled-agent/src/sim/storage.rs | 5 +- 12 files changed, 29 insertions(+), 83 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73919fe6f1b..687a2ceb8d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -530,12 +530,12 @@ dependencies = [ [[package]] name = "crucible-agent-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=de022b8a#de022b8ae24fb10cec6b024437aabc3acf249e43" dependencies = [ "anyhow", "percent-encoding", "progenitor", "reqwest", + "schemars", "serde", "serde_json", ] @@ -1765,6 +1765,7 @@ dependencies = [ "percent-encoding", "progenitor", "reqwest", + "schemars", "serde", "serde_json", "slog", @@ -2197,6 +2198,7 @@ dependencies = [ "percent-encoding", "progenitor", "reqwest", + "schemars", "serde", "slog", "uuid", @@ -2692,7 +2694,6 @@ dependencies = [ [[package]] name = "progenitor" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/progenitor#66b41ba301793b8d720770b2210bee8884446d3f" dependencies = [ "anyhow", "getopts", @@ -2706,7 +2707,6 @@ dependencies = [ [[package]] name = "progenitor-impl" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/progenitor#66b41ba301793b8d720770b2210bee8884446d3f" dependencies = [ "anyhow", "convert_case", @@ -2727,7 +2727,6 @@ dependencies = [ [[package]] name = "progenitor-macro" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/progenitor#66b41ba301793b8d720770b2210bee8884446d3f" dependencies = [ "openapiv3", "proc-macro2", @@ -3474,6 +3473,7 @@ dependencies = [ "percent-encoding", "progenitor", "reqwest", + "schemars", "serde", "slog", "uuid", @@ -4135,7 +4135,7 @@ checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" [[package]] name = "typify" version = "0.0.6-dev" -source = "git+https://github.com/oxidecomputer/typify#80b510b02b1db22de463efcf6e7762243bcea67a" +source = "git+https://github.com/oxidecomputer/typify?branch=json-schema#0be9b35a5007e3f797e43ce41eba03dcee63e503" dependencies = [ "typify-impl", "typify-macro", @@ -4144,9 +4144,10 @@ dependencies = [ [[package]] name = "typify-impl" version = "0.0.6-dev" -source = "git+https://github.com/oxidecomputer/typify#80b510b02b1db22de463efcf6e7762243bcea67a" +source = "git+https://github.com/oxidecomputer/typify?branch=json-schema#0be9b35a5007e3f797e43ce41eba03dcee63e503" dependencies = [ "convert_case", + "log", "proc-macro2", "quote", "rustfmt-wrapper", @@ -4159,7 +4160,7 @@ dependencies = [ [[package]] name = "typify-macro" version = "0.0.6-dev" -source = "git+https://github.com/oxidecomputer/typify#80b510b02b1db22de463efcf6e7762243bcea67a" +source = "git+https://github.com/oxidecomputer/typify?branch=json-schema#0be9b35a5007e3f797e43ce41eba03dcee63e503" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 5853d11a0b3..aaa4eed6633 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,8 +57,10 @@ panic = "abort" # # Local client generation during development. # -#[patch."https://github.com/oxidecomputer/progenitor"] -#progenitor = { path = "../progenitor/progenitor" } +[patch."https://github.com/oxidecomputer/progenitor"] +progenitor = { path = "../progenitor/progenitor" } +[patch."https://github.com/oxidecomputer/crucible"] +crucible-agent-client = { path = "../crucible/agent-client" } #[patch."https://github.com/oxidecomputer/typify"] #typify = { path = "../typify/typify" } diff --git a/nexus-client/Cargo.toml b/nexus-client/Cargo.toml index 0ef31114868..2ceb8bbb54f 100644 --- a/nexus-client/Cargo.toml +++ b/nexus-client/Cargo.toml @@ -9,6 +9,7 @@ anyhow = "1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor" } reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] } percent-encoding = "2.1.0" +schemars = { version = "0.8" } serde_json = "1.0" [dependencies.chrono] diff --git a/nexus/tests/integration_tests/disks.rs b/nexus/tests/integration_tests/disks.rs index d47dee2fb55..e2602359989 100644 --- a/nexus/tests/integration_tests/disks.rs +++ b/nexus/tests/integration_tests/disks.rs @@ -4,6 +4,7 @@ //! Tests basic disk support in the API +use crucible_agent_client::types::State as RegionState; use http::method::Method; use http::StatusCode; use omicron_common::api::external::ByteCount; @@ -14,7 +15,7 @@ use omicron_common::api::external::Instance; use omicron_common::api::external::InstanceCpuCount; use omicron_nexus::TestInterfaces as _; use omicron_nexus::{external_api::params, Nexus}; -use omicron_sled_agent::sim::{RegionState, SledAgent}; +use omicron_sled_agent::sim::SledAgent; use sled_agent_client::TestInterfaces as _; use std::sync::Arc; use uuid::Uuid; diff --git a/oximeter-client/Cargo.toml b/oximeter-client/Cargo.toml index d2165b95f83..2ee69f74aeb 100644 --- a/oximeter-client/Cargo.toml +++ b/oximeter-client/Cargo.toml @@ -8,6 +8,7 @@ license = "MPL-2.0" anyhow = "1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor" } reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] } +schemars = { version = "0.8" } percent-encoding = "2.1.0" [dependencies.chrono] diff --git a/oximeter/producer/src/lib.rs b/oximeter/producer/src/lib.rs index 29c8b3818cc..e10e9e944dd 100644 --- a/oximeter/producer/src/lib.rs +++ b/oximeter/producer/src/lib.rs @@ -180,7 +180,8 @@ pub async fn register( client .cpapi_producers_post(&server_info.into()) .await - .map_err(|msg| Error::RegistrationError(msg.to_string())) + .map_err(|msg| Error::RegistrationError(msg.to_string()))?; + Ok(()) } /// Handle a request to pull available metric data from a [`ProducerRegistry`]. diff --git a/sled-agent-client/Cargo.toml b/sled-agent-client/Cargo.toml index 2f233c4ebdc..ebf1f35dfd1 100644 --- a/sled-agent-client/Cargo.toml +++ b/sled-agent-client/Cargo.toml @@ -9,6 +9,7 @@ anyhow = "1.0" async-trait = "0.1" progenitor = { git = "https://github.com/oxidecomputer/progenitor" } reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] } +schemars = { version = "0.8" } percent-encoding = "2.1.0" [dependencies.chrono] diff --git a/sled-agent/src/sim/disk.rs b/sled-agent/src/sim/disk.rs index 95ef478dcc5..8bbbbaa2025 100644 --- a/sled-agent/src/sim/disk.rs +++ b/sled-agent/src/sim/disk.rs @@ -95,6 +95,7 @@ impl Simulatable for SimDisk { &nexus_client::types::DiskRuntimeState::from(current), ) .await - .map_err(Error::from) + .map_err(Error::from)?; + Ok(()) } } diff --git a/sled-agent/src/sim/http_entrypoints_storage.rs b/sled-agent/src/sim/http_entrypoints_storage.rs index a8f79f0d213..6ef08a9a373 100644 --- a/sled-agent/src/sim/http_entrypoints_storage.rs +++ b/sled-agent/src/sim/http_entrypoints_storage.rs @@ -4,12 +4,13 @@ //! HTTP entrypoint functions for simulating the storage agent API. +use crucible_agent_client::types::{CreateRegion, Region, RegionId}; use dropshot::{ endpoint, ApiDescription, HttpError, HttpResponseDeleted, HttpResponseOk, Path as TypedPath, RequestContext, TypedBody, }; use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use std::sync::Arc; use super::storage::CrucibleData; @@ -35,71 +36,8 @@ pub fn api() -> CrucibleAgentApiDescription { api } -// XXX XXX XXX THIS SUCKS XXX XXX XXX -// -// I need to re-define all structs used in the crucible agent -// API to ensure they have the traits I need. The ones re-exported -// through the client bindings, i.e., crucible_agent_client::types, -// don't implement the "JsonSchema" trait, and cannot be used as -// parameters to these Dropshot endpoints. -// -// I'd like them to! If we could ensure the generated client -// also implemented e.g. JsonSchema, this might work? - -// To quickly test the type compatibility, uncomment the lines below, -// and remove the hand-rolled implementations. - -// pub type RegionId = crucible_agent_client::types::RegionId; -// pub type State = crucible_agent_client::types::State; -// pub type CreateRegion = crucible_agent_client::types::CreateRegion; -// pub type Region = crucible_agent_client::types::Region; -// pub type RegionPath = crucible_agent_client::types::RegionPath; - -#[derive( - Serialize, - Deserialize, - JsonSchema, - Debug, - PartialEq, - Eq, - Clone, - PartialOrd, - Ord, -)] -pub struct RegionId(pub String); - -#[derive(Serialize, Deserialize, JsonSchema, Debug, PartialEq, Clone)] -#[serde(rename_all = "lowercase")] -pub enum State { - Requested, - Created, - Tombstoned, - Destroyed, - Failed, -} - -#[derive(Serialize, Deserialize, JsonSchema, Debug, PartialEq, Clone)] -pub struct CreateRegion { - pub id: RegionId, - pub volume_id: String, - - pub block_size: u64, - pub extent_size: u64, - pub extent_count: u64, -} - -#[derive(Serialize, Deserialize, JsonSchema, Debug, PartialEq, Clone)] -pub struct Region { - pub id: RegionId, - pub volume_id: String, - - pub block_size: u64, - pub extent_size: u64, - pub extent_count: u64, - - pub port_number: u16, - pub state: State, -} +// TODO: We'd like to de-duplicate as much as possible with the +// real crucible agent here, to avoid skew. #[derive(Deserialize, JsonSchema)] struct RegionPath { diff --git a/sled-agent/src/sim/instance.rs b/sled-agent/src/sim/instance.rs index b7414d4c877..684606719d6 100644 --- a/sled-agent/src/sim/instance.rs +++ b/sled-agent/src/sim/instance.rs @@ -102,6 +102,7 @@ impl Simulatable for SimInstance { &nexus_client::types::InstanceRuntimeState::from(current), ) .await - .map_err(Error::from) + .map_err(Error::from)?; + Ok(()) } } diff --git a/sled-agent/src/sim/mod.rs b/sled-agent/src/sim/mod.rs index 3e26a1bbfc3..3824951d854 100644 --- a/sled-agent/src/sim/mod.rs +++ b/sled-agent/src/sim/mod.rs @@ -18,6 +18,5 @@ mod sled_agent; mod storage; pub use config::{Config, SimMode}; -pub use http_entrypoints_storage::State as RegionState; pub use server::{run_server, Server}; pub use sled_agent::SledAgent; diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index f6049a17de1..55b2cb80f45 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -4,6 +4,7 @@ //! Simulated sled agent storage implementation +use crucible_agent_client::types::{CreateRegion, Region, RegionId, State}; use futures::lock::Mutex; use nexus_client::types::{ ByteCount, DatasetKind, DatasetPutRequest, ZpoolPutRequest, @@ -16,8 +17,6 @@ use std::str::FromStr; use std::sync::Arc; use uuid::Uuid; -use super::http_entrypoints_storage::{CreateRegion, Region, RegionId, State}; - type CreateCallback = Box State + Send + 'static>; struct CrucibleDataInner { @@ -60,7 +59,7 @@ impl CrucibleDataInner { let old = self.regions.insert(id, region.clone()); if let Some(old) = old { assert_eq!( - old.id, region.id, + old.id.0, region.id.0, "Region already exists, but with a different ID" ); } From 124fbad6ff877d6b666fbcc3be5e5a968800d727 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 13 Jan 2022 22:01:16 -0500 Subject: [PATCH 36/50] Add indices --- common/src/sql/dbinit.sql | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/common/src/sql/dbinit.sql b/common/src/sql/dbinit.sql index 2fe6a42afb0..2cbe7410bc8 100644 --- a/common/src/sql/dbinit.sql +++ b/common/src/sql/dbinit.sql @@ -133,6 +133,13 @@ CREATE TABLE omicron.public.Region ( extent_count INT NOT NULL ); +/* + * Allow all regions within one disk to be accessed quickly. + */ +CREATE INDEX on omicron.public.Region ( + disk_id +); + /* * Organizations */ From 830c254d6e3b08e0c00104c02d5947c881e326ed Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 13 Jan 2022 22:02:24 -0500 Subject: [PATCH 37/50] Add an Explainable trait to EXPLAIN Diesel statements --- nexus/src/db/explain.rs | 102 ++++++++++++++++++++++++++++++++++++++++ nexus/src/db/mod.rs | 1 + 2 files changed, 103 insertions(+) create mode 100644 nexus/src/db/explain.rs diff --git a/nexus/src/db/explain.rs b/nexus/src/db/explain.rs new file mode 100644 index 00000000000..a8485a0e30d --- /dev/null +++ b/nexus/src/db/explain.rs @@ -0,0 +1,102 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Utility allowing Diesel to EXPLAIN queries. + +use super::pool::DbConnection; +use async_bb8_diesel::{ + AsyncRunQueryDsl, ConnectionManager, PoolError, +}; +use async_trait::async_trait; +use diesel::pg::Pg; +use diesel::prelude::*; +use diesel::query_builder::*; + +/// A wrapper around a runnable Diesel query, which EXPLAINs what it is doing. +/// +/// Q: The Query we're explaining. +/// +/// EXPLAIN: https://www.cockroachlabs.com/docs/stable/explain.html +pub trait Explainable { + /// Syncronously issues an explain statement. + fn explain( + self, + conn: &mut DbConnection, + ) -> Result; +} + +impl Explainable for Q +where + Q: QueryFragment + RunQueryDsl + Sized, +{ + fn explain(self, conn: &mut DbConnection) -> Result { + Ok( + ExplainStatement { + query: self, + }.get_results::(conn)?.join("\n") + ) + } +} + +/// An async variant of [`Explainable`]. +#[async_trait] +pub trait ExplainableAsync { + /// Asynchronously issues an explain statement. + async fn explain_async( + self, + pool: &bb8::Pool>, + ) -> Result; +} + +#[async_trait] +impl ExplainableAsync for Q +where + Q: QueryFragment + RunQueryDsl + Sized + Send + 'static, +{ + async fn explain_async( + self, + pool: &bb8::Pool>, + ) -> Result { + Ok( + ExplainStatement { + query: self, + }.get_results_async::(pool).await?.join("\n") + ) + } +} + +// An EXPLAIN statement, wrapping an underlying query. +// +// This isn't `pub` because it's kinda weird to access "part" of the EXPLAIN +// output, which would be possible by calling "get_result" instead of +// "get_results". We'd like to be able to constrain callers such that they get +// all of the output or none of it. +// +// See the [`Explainable`] trait for why this exists. +struct ExplainStatement { + query: Q, +} + +impl QueryId for ExplainStatement { + type QueryId = (); + const HAS_STATIC_QUERY_ID: bool = false; +} + +impl Query for ExplainStatement { + type SqlType = diesel::sql_types::Text; +} + +impl RunQueryDsl for ExplainStatement {} + +impl QueryFragment for ExplainStatement +where + Q: QueryFragment +{ + fn walk_ast(&self, mut out: AstPass) -> QueryResult<()> { + out.push_sql("EXPLAIN ("); + self.query.walk_ast(out.reborrow())?; + out.push_sql(")"); + Ok(()) + } +} diff --git a/nexus/src/db/mod.rs b/nexus/src/db/mod.rs index c0df61ac95a..84b80802638 100644 --- a/nexus/src/db/mod.rs +++ b/nexus/src/db/mod.rs @@ -14,6 +14,7 @@ mod config; // This is marked public for use by the integration tests pub mod datastore; mod error; +mod explain; pub mod fixed_data; mod pagination; mod pool; From 23fc024a174c47400a4cfa343e287c5cf5cb0167 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 13 Jan 2022 22:05:52 -0500 Subject: [PATCH 38/50] fmt --- nexus/src/db/explain.rs | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/nexus/src/db/explain.rs b/nexus/src/db/explain.rs index a8485a0e30d..0c24fe1d039 100644 --- a/nexus/src/db/explain.rs +++ b/nexus/src/db/explain.rs @@ -5,9 +5,7 @@ //! Utility allowing Diesel to EXPLAIN queries. use super::pool::DbConnection; -use async_bb8_diesel::{ - AsyncRunQueryDsl, ConnectionManager, PoolError, -}; +use async_bb8_diesel::{AsyncRunQueryDsl, ConnectionManager, PoolError}; use async_trait::async_trait; use diesel::pg::Pg; use diesel::prelude::*; @@ -30,12 +28,13 @@ impl Explainable for Q where Q: QueryFragment + RunQueryDsl + Sized, { - fn explain(self, conn: &mut DbConnection) -> Result { - Ok( - ExplainStatement { - query: self, - }.get_results::(conn)?.join("\n") - ) + fn explain( + self, + conn: &mut DbConnection, + ) -> Result { + Ok(ExplainStatement { query: self } + .get_results::(conn)? + .join("\n")) } } @@ -58,11 +57,10 @@ where self, pool: &bb8::Pool>, ) -> Result { - Ok( - ExplainStatement { - query: self, - }.get_results_async::(pool).await?.join("\n") - ) + Ok(ExplainStatement { query: self } + .get_results_async::(pool) + .await? + .join("\n")) } } @@ -91,7 +89,7 @@ impl RunQueryDsl for ExplainStatement {} impl QueryFragment for ExplainStatement where - Q: QueryFragment + Q: QueryFragment, { fn walk_ast(&self, mut out: AstPass) -> QueryResult<()> { out.push_sql("EXPLAIN ("); From 067c97c551a1c123e20cdc8facb57cbb67e708e0 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 14 Jan 2022 10:52:26 -0500 Subject: [PATCH 39/50] Add tests --- nexus/src/db/explain.rs | 139 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) diff --git a/nexus/src/db/explain.rs b/nexus/src/db/explain.rs index 0c24fe1d039..33595aa44a3 100644 --- a/nexus/src/db/explain.rs +++ b/nexus/src/db/explain.rs @@ -98,3 +98,142 @@ where Ok(()) } } + +#[cfg(test)] +mod test { + use super::*; + + use crate::db; + use async_bb8_diesel::{AsyncConnection, AsyncSimpleConnection}; + use diesel::SelectableHelper; + use nexus_test_utils::db::test_setup_database; + use omicron_test_utils::dev; + use uuid::Uuid; + + mod schema { + use diesel::prelude::*; + + table! { + test_users { + id -> Uuid, + age -> Int8, + height -> Int8, + } + } + } + + use schema::test_users; + + #[derive(Clone, Debug, Queryable, Insertable, PartialEq, Selectable)] + #[table_name = "test_users"] + struct User { + id: Uuid, + age: i64, + height: i64, + } + + async fn create_schema(pool: &db::Pool) { + pool.pool() + .get() + .await + .unwrap() + .batch_execute_async( + "CREATE TABLE test_users ( + id UUID PRIMARY KEY, + age INT NOT NULL, + height INT NOT NULL + )", + ) + .await + .unwrap(); + } + + // Tests the ".explain()" method in a synchronous context. + // + // This is often done when calling from transactions, which we demonstrate. + #[tokio::test] + async fn test_explain() { + let logctx = dev::test_setup_log("test_explain"); + let db = test_setup_database(&logctx.log).await; + let cfg = db::Config { url: db.pg_config().clone() }; + let pool = db::Pool::new(&cfg); + + create_schema(&pool).await; + + use schema::test_users::dsl; + pool.pool() + .transaction(move |conn| -> Result<(), db::error::TransactionError<()>> { + let explanation = dsl::test_users + .filter(dsl::id.eq(Uuid::nil())) + .select(User::as_select()) + .explain(conn) + .unwrap(); + assert_eq!(r#"distribution: local +vectorized: true + +• scan + missing stats + table: test_users@primary + spans: [/'00000000-0000-0000-0000-000000000000' - /'00000000-0000-0000-0000-000000000000']"#, + explanation); + Ok(()) + }) + .await + .unwrap(); + } + + // Tests the ".explain_async()" method in an asynchronous context. + #[tokio::test] + async fn test_explain_async() { + let logctx = dev::test_setup_log("test_explain_async"); + let db = test_setup_database(&logctx.log).await; + let cfg = db::Config { url: db.pg_config().clone() }; + let pool = db::Pool::new(&cfg); + + create_schema(&pool).await; + + use schema::test_users::dsl; + let explanation = dsl::test_users + .filter(dsl::id.eq(Uuid::nil())) + .select(User::as_select()) + .explain_async(pool.pool()) + .await + .unwrap(); + + assert_eq!( + r#"distribution: local +vectorized: true + +• scan + missing stats + table: test_users@primary + spans: [/'00000000-0000-0000-0000-000000000000' - /'00000000-0000-0000-0000-000000000000']"#, + explanation + ); + } + + // Tests that ".explain()" can tell us when we're doing full table scans. + #[tokio::test] + async fn test_explain_full_table_scan() { + let logctx = dev::test_setup_log("test_explain_full_table_scan"); + let db = test_setup_database(&logctx.log).await; + let cfg = db::Config { url: db.pg_config().clone() }; + let pool = db::Pool::new(&cfg); + + create_schema(&pool).await; + + use schema::test_users::dsl; + let explanation = dsl::test_users + .filter(dsl::age.eq(2)) + .select(User::as_select()) + .explain_async(pool.pool()) + .await + .unwrap(); + + assert!( + explanation.contains("FULL SCAN"), + "Expected [{}] to contain 'FULL SCAN'", + explanation + ); + } +} From 1ce9c6a5281a709a6a8a60e9f84f59f5873021fb Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 14 Jan 2022 16:15:06 -0500 Subject: [PATCH 40/50] Testing queries --- nexus/src/db/datastore.rs | 161 +++++++++++++++++++++++++++----------- 1 file changed, 116 insertions(+), 45 deletions(-) diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index b2e4c47bf11..60204dc4744 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -43,6 +43,9 @@ use chrono::Utc; use diesel::prelude::*; use diesel::upsert::excluded; use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; +use diesel::query_builder::QueryFragment; +use diesel::query_dsl::methods::LoadQuery; +use diesel::pg::Pg; use omicron_common::api; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::DeleteResult; @@ -85,6 +88,18 @@ use crate::db::{ // TODO: This should likely turn into a configuration option. const REGION_REDUNDANCY_THRESHOLD: usize = 3; +// Represents a query that is ready to be executed. +// +// This is a helper trait which lets the statement +// either be executed or explained. +// +// U: The output type of executing the statement. +trait RunnableQuery: RunQueryDsl + QueryFragment + LoadQuery {} + +impl RunnableQuery for T +where T: RunQueryDsl + QueryFragment + LoadQuery + {} + pub struct DataStore { pool: Arc, } @@ -242,15 +257,9 @@ impl DataStore { }) } - /// Gets allocated regions for a disk, and the datasets to which those - /// regions belong. - /// - /// Note that this function does not validate liveness of the Disk, so it - /// may be used in a context where the disk is being deleted. - pub async fn get_allocated_regions( - &self, + fn get_allocated_regions_query( disk_id: Uuid, - ) -> Result, Error> { + ) -> impl RunnableQuery<(Dataset, Region)> { use db::schema::dataset::dsl as dataset_dsl; use db::schema::region::dsl as region_dsl; region_dsl::region @@ -260,11 +269,60 @@ impl DataStore { .on(region_dsl::dataset_id.eq(dataset_dsl::id)), ) .select((Dataset::as_select(), Region::as_select())) + } + + /// Gets allocated regions for a disk, and the datasets to which those + /// regions belong. + /// + /// Note that this function does not validate liveness of the Disk, so it + /// may be used in a context where the disk is being deleted. + pub async fn get_allocated_regions( + &self, + disk_id: Uuid, + ) -> Result, Error> { + Self::get_allocated_regions_query(disk_id) .get_results_async::<(Dataset, Region)>(self.pool()) .await .map_err(|e| public_error_from_diesel_pool_shouldnt_fail(e)) } + fn get_allocatable_datasets_query( + ) -> impl RunnableQuery { + use db::schema::dataset::dsl as dataset_dsl; + use db::schema::region::dsl as region_dsl; + + dataset_dsl::dataset + // We look for valid datasets (non-deleted crucible datasets). + .filter(dataset_dsl::time_deleted.is_null()) + .filter(dataset_dsl::kind.eq(DatasetKind( + crate::internal_api::params::DatasetKind::Crucible, + ))) + // Next, observe all the regions allocated to each dataset, and + // determine how much space they're using. + // + // TODO: We could store "free/allocated" space per-dataset, + // and keep them up-to-date, rather than trying to recompute + // this. + // + // TODO: We admittedly don't actually *fail* any request for + // running out of space - we try to send the request down to + // crucible agents, and expect them to fail on our behalf in + // out-of-storage conditions. This should undoubtedly be + // handled more explicitly. + .left_outer_join( + region_dsl::region + .on(dataset_dsl::id.eq(region_dsl::dataset_id)), + ) + .group_by(dataset_dsl::id) + .select(Dataset::as_select()) + .order( + diesel::dsl::sum( + region_dsl::extent_size * region_dsl::extent_count, + ) + .asc(), + ) + } + /// Idempotently allocates enough regions to back a disk. /// /// Returns the allocated regions, as well as the datasets to which they @@ -310,48 +368,13 @@ impl DataStore { // // If they are, return those regions and the associated // datasets. - let datasets_and_regions = region_dsl::region - .filter(region_dsl::disk_id.eq(disk_id)) - .inner_join( - dataset_dsl::dataset - .on(region_dsl::dataset_id.eq(dataset_dsl::id)), - ) - .select((Dataset::as_select(), Region::as_select())) + let datasets_and_regions = Self::get_allocated_regions_query(disk_id) .get_results::<(Dataset, Region)>(conn)?; if !datasets_and_regions.is_empty() { return Ok(datasets_and_regions); } - let datasets: Vec = dataset_dsl::dataset - // We look for valid datasets (non-deleted crucible datasets). - .filter(dataset_dsl::time_deleted.is_null()) - .filter(dataset_dsl::kind.eq(DatasetKind( - crate::internal_api::params::DatasetKind::Crucible, - ))) - // Next, observe all the regions allocated to each dataset, and - // determine how much space they're using. - // - // TODO: We could store "free/allocated" space per-dataset, - // and keep them up-to-date, rather than trying to recompute - // this. - // - // TODO: We admittedly don't actually *fail* any request for - // running out of space - we try to send the request down to - // crucible agents, and expect them to fail on our behalf in - // out-of-storage conditions. This should undoubtedly be - // handled more explicitly. - .left_outer_join( - region_dsl::region - .on(dataset_dsl::id.eq(region_dsl::dataset_id)), - ) - .group_by(dataset_dsl::id) - .select(Dataset::as_select()) - .order( - diesel::dsl::sum( - region_dsl::extent_size * region_dsl::extent_count, - ) - .asc(), - ) + let datasets: Vec = Self::get_allocatable_datasets_query() .get_results::(conn)?; if datasets.len() < REGION_REDUNDANCY_THRESHOLD { @@ -2615,6 +2638,7 @@ pub async fn datastore_test( mod test { use super::*; use crate::authz; + use crate::db::explain::ExplainableAsync; use crate::db::identity::Resource; use crate::db::model::{ConsoleSession, Organization, Project}; use crate::external_api::params; @@ -2932,4 +2956,51 @@ mod test { let _ = db.cleanup().await; } + + // Validate that queries which should be executable without a full table + // scan are, in fact, runnable without a FULL SCAN. + #[tokio::test] + async fn test_queries_do_not_require_full_table_scan() { + let logctx = + dev::test_setup_log("test_queries_do_not_require_full_table_scan"); + let mut db = test_setup_database(&logctx.log).await; + let cfg = db::Config { url: db.pg_config().clone() }; + let pool = db::Pool::new(&cfg); + let datastore = DataStore::new(Arc::new(pool)); + + let explanation = DataStore::get_allocated_regions_query(Uuid::nil()) + .explain_async(datastore.pool()) + .await + .unwrap(); + assert!(!explanation.contains("FULL SCAN"), "Found an unexpected FULL SCAN: {}", explanation); + + let _ = db.cleanup().await; + } + + // Welp, life isn't perfect - sometimes, we take shortcuts, and implement + // queries that DO require FULL SCAN. + // + // These are problematic scans from a performance point-of-view, but we can + // keep track of which queries do so with this test! + // + // NOTE: If this test is failing because a table scan is no longer required, + // congratulations, move that query into the more appropriate test: + // `test_queries_do_not_require_full_table_scan`. + #[tokio::test] + async fn test_queries_that_do_require_full_table_scan() { + let logctx = + dev::test_setup_log("test_queries_that_do_require_full_table_scan"); + let mut db = test_setup_database(&logctx.log).await; + let cfg = db::Config { url: db.pg_config().clone() }; + let pool = db::Pool::new(&cfg); + let datastore = DataStore::new(Arc::new(pool)); + + let explanation = DataStore::get_allocatable_datasets_query() + .explain_async(datastore.pool()) + .await + .unwrap(); + assert!(explanation.contains("FULL SCAN"), "Found an unexpected FULL SCAN: {}", explanation); + + let _ = db.cleanup().await; + } } From d574f67fc95dc5ef31b27e025a1bafe0a181aa90 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 17 Jan 2022 16:12:16 -0500 Subject: [PATCH 41/50] review feedback --- common/src/api/external/mod.rs | 6 ++-- common/src/sql/dbinit.sql | 2 +- nexus/src/db/datastore.rs | 37 +++++----------------- nexus/src/db/model.rs | 24 +++++++------- nexus/src/external_api/params.rs | 54 ++++++++++++++++++++++++++++---- nexus/src/sagas.rs | 8 ++--- 6 files changed, 76 insertions(+), 55 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 10e7b302168..eacbc00caf3 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -399,7 +399,7 @@ impl JsonSchema for RoleName { * the database as an i64. Constraining it here ensures that we can't fail to * serialize the value. */ -#[derive(Copy, Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[derive(Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] pub struct ByteCount(u64); impl ByteCount { @@ -468,8 +468,8 @@ impl From for ByteCount { } } -impl From<&ByteCount> for i64 { - fn from(b: &ByteCount) -> Self { +impl From for i64 { + fn from(b: ByteCount) -> Self { /* We have already validated that this value is in range. */ i64::try_from(b.0).unwrap() } diff --git a/common/src/sql/dbinit.sql b/common/src/sql/dbinit.sql index 2cbe7410bc8..55c4d19cc10 100644 --- a/common/src/sql/dbinit.sql +++ b/common/src/sql/dbinit.sql @@ -134,7 +134,7 @@ CREATE TABLE omicron.public.Region ( ); /* - * Allow all regions within one disk to be accessed quickly. + * Allow all regions belonging to a disk to be accessed quickly. */ CREATE INDEX on omicron.public.Region ( disk_id diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index 60204dc4744..054270f0343 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -321,6 +321,7 @@ impl DataStore { ) .asc(), ) + .limit(REGION_REDUNDANCY_THRESHOLD.try_into().unwrap()) } /// Idempotently allocates enough regions to back a disk. @@ -332,7 +333,6 @@ impl DataStore { disk_id: Uuid, params: ¶ms::DiskCreate, ) -> Result, Error> { - use db::schema::dataset::dsl as dataset_dsl; use db::schema::region::dsl as region_dsl; // ALLOCATION POLICY @@ -391,9 +391,9 @@ impl DataStore { Region::new( dataset.id(), disk_id, - params.block_size().try_into().unwrap(), - params.extent_size().try_into().unwrap(), - params.extent_count().try_into().unwrap(), + params.block_size().into(), + params.extent_size().into(), + params.extent_count(), ) }) .collect(); @@ -1237,20 +1237,6 @@ impl DataStore { }) } - pub async fn project_delete_disk( - &self, - opctx: &OpContext, - disk_authz: authz::ProjectChild, - ) -> DeleteResult { - let disk_id = disk_authz.id(); - opctx.authorize(authz::Action::Delete, disk_authz).await?; - self.project_delete_disk_internal( - disk_id, - self.pool_authorized(opctx).await?, - ) - .await - } - // TODO: Delete me (this function, not the disk!), ensure all datastore // access is auth-checked. // @@ -1271,16 +1257,9 @@ impl DataStore { pub async fn project_delete_disk_no_auth( &self, disk_id: &Uuid, - ) -> DeleteResult { - self.project_delete_disk_internal(disk_id, self.pool()).await - } - - async fn project_delete_disk_internal( - &self, - disk_id: &Uuid, - pool: &bb8::Pool>, ) -> DeleteResult { use db::schema::disk::dsl; + let pool = self.pool(); let now = Utc::now(); let ok_to_delete_states = vec![ @@ -1328,8 +1307,8 @@ impl DataStore { } else { // NOTE: This is a "catch-all" error case, more specific // errors should be preferred as they're more actionable. - return Err(Error::InvalidRequest { - message: String::from( + return Err(Error::InternalError{ + internal_message: String::from( "disk exists, but cannot be deleted", ), }); @@ -2999,7 +2978,7 @@ mod test { .explain_async(datastore.pool()) .await .unwrap(); - assert!(explanation.contains("FULL SCAN"), "Found an unexpected FULL SCAN: {}", explanation); + assert!(explanation.contains("FULL SCAN"), "Expected FULL SCAN: {}", explanation); let _ = db.cleanup().await; } diff --git a/nexus/src/db/model.rs b/nexus/src/db/model.rs index 5b375cc37de..7279410cf49 100644 --- a/nexus/src/db/model.rs +++ b/nexus/src/db/model.rs @@ -144,7 +144,7 @@ where } #[derive( - Copy, Clone, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, + Copy, Clone, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq, )] #[sql_type = "sql_types::BigInt"] pub struct ByteCount(pub external::ByteCount); @@ -161,7 +161,7 @@ where &self, out: &mut serialize::Output, ) -> serialize::Result { - i64::from(&self.0).to_sql(out) + i64::from(self.0).to_sql(out) } } @@ -630,8 +630,8 @@ pub struct Region { dataset_id: Uuid, disk_id: Uuid, - block_size: i64, - extent_size: i64, + block_size: ByteCount, + extent_size: ByteCount, extent_count: i64, } @@ -639,8 +639,8 @@ impl Region { pub fn new( dataset_id: Uuid, disk_id: Uuid, - block_size: i64, - extent_size: i64, + block_size: ByteCount, + extent_size: ByteCount, extent_count: i64, ) -> Self { Self { @@ -659,14 +659,14 @@ impl Region { pub fn dataset_id(&self) -> Uuid { self.dataset_id } - pub fn block_size(&self) -> u64 { - self.block_size as u64 + pub fn block_size(&self) -> external::ByteCount { + self.block_size.0 } - pub fn extent_size(&self) -> u64 { - self.extent_size as u64 + pub fn extent_size(&self) -> external::ByteCount { + self.extent_size.0 } - pub fn extent_count(&self) -> u64 { - self.extent_count as u64 + pub fn extent_count(&self) -> i64 { + self.extent_count } } diff --git a/nexus/src/external_api/params.rs b/nexus/src/external_api/params.rs index bbcb5acf56d..e87ebdad265 100644 --- a/nexus/src/external_api/params.rs +++ b/nexus/src/external_api/params.rs @@ -12,6 +12,7 @@ use omicron_common::api::external::{ }; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use std::convert::TryFrom; use uuid::Uuid; /* @@ -182,17 +183,21 @@ pub struct DiskCreate { pub size: ByteCount, } +const BLOCK_SIZE: u32 = 512_u32; +const EXTENT_SIZE: u32 = 1_u32 << 20; + impl DiskCreate { - pub fn block_size(&self) -> u64 { - 512 + pub fn block_size(&self) -> ByteCount { + ByteCount::from(BLOCK_SIZE) } - pub fn extent_size(&self) -> u64 { - 1 << 20 + pub fn extent_size(&self) -> ByteCount { + ByteCount::from(EXTENT_SIZE) } - pub fn extent_count(&self) -> u64 { - (self.size.to_bytes() + self.extent_size() - 1) / self.extent_size() + pub fn extent_count(&self) -> i64 { + let extent_size = self.extent_size().to_bytes(); + i64::try_from((self.size.to_bytes() + extent_size - 1) / extent_size).unwrap() } } @@ -222,3 +227,40 @@ pub struct UserBuiltinCreate { #[serde(flatten)] pub identity: IdentityMetadataCreateParams, } + +#[cfg(test)] +mod test { + use super::*; + + fn new_disk_create_params(size: ByteCount) -> DiskCreate { + DiskCreate { + identity: IdentityMetadataCreateParams { + name: Name::try_from("myobject".to_string()).unwrap(), + description: "desc".to_string(), + }, + snapshot_id: None, + size, + } + } + + #[test] + fn test_extent_count() { + let params = new_disk_create_params(ByteCount::try_from(0u64).unwrap()); + assert_eq!(0, params.extent_count()); + + let params = new_disk_create_params(ByteCount::try_from(1u64).unwrap()); + assert_eq!(1, params.extent_count()); + let params = new_disk_create_params(ByteCount::try_from(EXTENT_SIZE - 1).unwrap()); + assert_eq!(1, params.extent_count()); + let params = new_disk_create_params(ByteCount::try_from(EXTENT_SIZE).unwrap()); + assert_eq!(1, params.extent_count()); + + let params = new_disk_create_params(ByteCount::try_from(EXTENT_SIZE + 1).unwrap()); + assert_eq!(2, params.extent_count()); + + // Mostly just checking we don't blow up on an unwrap here. + let params = new_disk_create_params(ByteCount::try_from(i64::MAX).unwrap()); + assert!(params.size.to_bytes() < (params.extent_count() as u64) * params.extent_size().to_bytes()); + } +} + diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index f4ce09904e1..6f7051da364 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -38,7 +38,7 @@ use serde::Deserialize; use serde::Serialize; use slog::Logger; use std::collections::BTreeMap; -use std::convert::TryFrom; +use std::convert::{TryInto, TryFrom}; use std::sync::Arc; use steno::new_action_noop_undo; use steno::ActionContext; @@ -470,9 +470,9 @@ async fn ensure_region_in_dataset( let client = CrucibleAgentClient::new(&url); let region_request = CreateRegion { - block_size: region.block_size(), - extent_count: region.extent_count(), - extent_size: region.extent_size(), + block_size: region.block_size().to_bytes(), + extent_count: region.extent_count().try_into().unwrap(), + extent_size: region.extent_size().to_bytes(), // TODO: Can we avoid casting from UUID to string? // NOTE: This'll require updating the crucible agent client. id: RegionId(region.id().to_string()), From 20eaafa43b96f86ab6f736f0cf5274d51e3852bd Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 19 Jan 2022 00:04:40 -0500 Subject: [PATCH 42/50] Implement QueryID --- nexus/src/db/explain.rs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/nexus/src/db/explain.rs b/nexus/src/db/explain.rs index 33595aa44a3..3ce77919597 100644 --- a/nexus/src/db/explain.rs +++ b/nexus/src/db/explain.rs @@ -26,7 +26,11 @@ pub trait Explainable { impl Explainable for Q where - Q: QueryFragment + RunQueryDsl + Sized, + Q: QueryFragment + + QueryId + + RunQueryDsl + + Sized + + 'static, { fn explain( self, @@ -51,7 +55,12 @@ pub trait ExplainableAsync { #[async_trait] impl ExplainableAsync for Q where - Q: QueryFragment + RunQueryDsl + Sized + Send + 'static, + Q: QueryFragment + + QueryId + + RunQueryDsl + + Sized + + Send + + 'static, { async fn explain_async( self, @@ -76,9 +85,12 @@ struct ExplainStatement { query: Q, } -impl QueryId for ExplainStatement { - type QueryId = (); - const HAS_STATIC_QUERY_ID: bool = false; +impl QueryId for ExplainStatement +where + Q: QueryId + 'static, +{ + type QueryId = ExplainStatement; + const HAS_STATIC_QUERY_ID: bool = Q::HAS_STATIC_QUERY_ID; } impl Query for ExplainStatement { From 18add7188591716ae6df8410ec2b02dbd6f0aaf6 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 19 Jan 2022 13:52:07 -0500 Subject: [PATCH 43/50] Make explained Queries work (requiring QueryId), update tests, fmt --- nexus/src/db/datastore.rs | 116 +++++++++++++++++++++++++------ nexus/src/db/model.rs | 9 ++- nexus/src/external_api/params.rs | 24 +++++-- nexus/src/sagas.rs | 14 +++- 4 files changed, 132 insertions(+), 31 deletions(-) diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index da3b54888ff..6cba8530dc6 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -40,12 +40,12 @@ use async_bb8_diesel::{ PoolError, }; use chrono::Utc; +use diesel::pg::Pg; use diesel::prelude::*; +use diesel::query_builder::{QueryFragment, QueryId}; +use diesel::query_dsl::methods::LoadQuery; use diesel::upsert::excluded; use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; -use diesel::query_builder::QueryFragment; -use diesel::query_dsl::methods::LoadQuery; -use diesel::pg::Pg; use omicron_common::api; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::DeleteResult; @@ -90,15 +90,24 @@ const REGION_REDUNDANCY_THRESHOLD: usize = 3; // Represents a query that is ready to be executed. // -// This is a helper trait which lets the statement -// either be executed or explained. +// This helper trait lets the statement either be executed or explained. // // U: The output type of executing the statement. -trait RunnableQuery: RunQueryDsl + QueryFragment + LoadQuery {} +trait RunnableQuery: + RunQueryDsl + + QueryFragment + + LoadQuery + + QueryId +{ +} -impl RunnableQuery for T -where T: RunQueryDsl + QueryFragment + LoadQuery - {} +impl RunnableQuery for T where + T: RunQueryDsl + + QueryFragment + + LoadQuery + + QueryId +{ +} pub struct DataStore { pool: Arc, @@ -286,8 +295,7 @@ impl DataStore { .map_err(|e| public_error_from_diesel_pool_shouldnt_fail(e)) } - fn get_allocatable_datasets_query( - ) -> impl RunnableQuery { + fn get_allocatable_datasets_query() -> impl RunnableQuery { use db::schema::dataset::dsl as dataset_dsl; use db::schema::region::dsl as region_dsl; @@ -368,14 +376,17 @@ impl DataStore { // // If they are, return those regions and the associated // datasets. - let datasets_and_regions = Self::get_allocated_regions_query(disk_id) - .get_results::<(Dataset, Region)>(conn)?; + let datasets_and_regions = Self::get_allocated_regions_query( + disk_id, + ) + .get_results::<(Dataset, Region)>(conn)?; if !datasets_and_regions.is_empty() { return Ok(datasets_and_regions); } - let datasets: Vec = Self::get_allocatable_datasets_query() - .get_results::(conn)?; + let datasets: Vec = + Self::get_allocatable_datasets_query() + .get_results::(conn)?; if datasets.len() < REGION_REDUNDANCY_THRESHOLD { return Err(TxnError::CustomError( @@ -1244,6 +1255,8 @@ impl DataStore { }) } + /// Attempts to delete a disk. Returns the disk (prior to deletion) + /// if successful. // TODO: Delete me (this function, not the disk!), ensure all datastore // access is auth-checked. // @@ -1264,7 +1277,7 @@ impl DataStore { pub async fn project_delete_disk_no_auth( &self, disk_id: &Uuid, - ) -> DeleteResult { + ) -> Result<(), Error> { use db::schema::disk::dsl; let pool = self.pool(); let now = Utc::now(); @@ -1314,7 +1327,7 @@ impl DataStore { } else { // NOTE: This is a "catch-all" error case, more specific // errors should be preferred as they're more actionable. - return Err(Error::InternalError{ + return Err(Error::InternalError { internal_message: String::from( "disk exists, but cannot be deleted", ), @@ -2751,6 +2764,10 @@ mod test { sled_id } + fn test_zpool_size() -> ByteCount { + ByteCount::from_gibibytes_u32(100) + } + // Creates a test zpool, returns its UUID. async fn create_test_zpool(datastore: &DataStore, sled_id: Uuid) -> Uuid { let zpool_id = Uuid::new_v4(); @@ -2758,7 +2775,7 @@ mod test { zpool_id, sled_id, &crate::internal_api::params::ZpoolPutRequest { - size: ByteCount::from_gibibytes_u32(100), + size: test_zpool_size(), }, ); datastore.zpool_upsert(zpool).await.unwrap(); @@ -2812,11 +2829,13 @@ mod test { ByteCount::from_mebibytes_u32(500), ); let disk1_id = Uuid::new_v4(); + // Currently, we only allocate one Region Set per disk. + let expected_region_count = REGION_REDUNDANCY_THRESHOLD; let dataset_and_regions = datastore.region_allocate(disk1_id, ¶ms).await.unwrap(); // Verify the allocation. - assert_eq!(REGION_REDUNDANCY_THRESHOLD, dataset_and_regions.len()); + assert_eq!(expected_region_count, dataset_and_regions.len()); let mut disk1_datasets = HashSet::new(); for (dataset, region) in dataset_and_regions { assert!(disk1_datasets.insert(dataset.id())); @@ -2835,7 +2854,7 @@ mod test { let disk2_id = Uuid::new_v4(); let dataset_and_regions = datastore.region_allocate(disk2_id, ¶ms).await.unwrap(); - assert_eq!(REGION_REDUNDANCY_THRESHOLD, dataset_and_regions.len()); + assert_eq!(expected_region_count, dataset_and_regions.len()); let mut disk2_datasets = HashSet::new(); for (dataset, region) in dataset_and_regions { assert!(disk2_datasets.insert(dataset.id())); @@ -2956,6 +2975,51 @@ mod test { let _ = db.cleanup().await; } + // TODO: This test should be updated when the correct handling + // of this out-of-space case is implemented. + #[tokio::test] + async fn test_region_allocation_out_of_space_does_not_fail_yet() { + let logctx = dev::test_setup_log( + "test_region_allocation_out_of_space_does_not_fail_yet", + ); + let mut db = test_setup_database(&logctx.log).await; + let cfg = db::Config { url: db.pg_config().clone() }; + let pool = db::Pool::new(&cfg); + let datastore = DataStore::new(Arc::new(pool)); + + // Create a sled... + let sled_id = create_test_sled(&datastore).await; + + // ... and a zpool within that sled... + let zpool_id = create_test_zpool(&datastore, sled_id).await; + + // ... and datasets within that zpool. + let dataset_count = REGION_REDUNDANCY_THRESHOLD; + let bogus_addr = + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); + let kind = + DatasetKind(crate::internal_api::params::DatasetKind::Crucible); + let dataset_ids: Vec = + (0..dataset_count).map(|_| Uuid::new_v4()).collect(); + for id in &dataset_ids { + let dataset = Dataset::new(*id, zpool_id, bogus_addr, kind.clone()); + datastore.dataset_upsert(dataset).await.unwrap(); + } + + // Allocate regions from the datasets for this disk. + // + // Note that we ask for a disk which is as large as the zpool, + // so we shouldn't have space for redundancy. + let disk_size = test_zpool_size(); + let params = create_test_disk_create_params("disk1", disk_size); + let disk1_id = Uuid::new_v4(); + + // NOTE: This *should* be an error, rather than succeeding. + datastore.region_allocate(disk1_id, ¶ms).await.unwrap(); + + let _ = db.cleanup().await; + } + // Validate that queries which should be executable without a full table // scan are, in fact, runnable without a FULL SCAN. #[tokio::test] @@ -2971,7 +3035,11 @@ mod test { .explain_async(datastore.pool()) .await .unwrap(); - assert!(!explanation.contains("FULL SCAN"), "Found an unexpected FULL SCAN: {}", explanation); + assert!( + !explanation.contains("FULL SCAN"), + "Found an unexpected FULL SCAN: {}", + explanation + ); let _ = db.cleanup().await; } @@ -2998,7 +3066,11 @@ mod test { .explain_async(datastore.pool()) .await .unwrap(); - assert!(explanation.contains("FULL SCAN"), "Expected FULL SCAN: {}", explanation); + assert!( + explanation.contains("FULL SCAN"), + "Expected FULL SCAN: {}", + explanation + ); let _ = db.cleanup().await; } diff --git a/nexus/src/db/model.rs b/nexus/src/db/model.rs index 7279410cf49..5b1961ccec6 100644 --- a/nexus/src/db/model.rs +++ b/nexus/src/db/model.rs @@ -144,7 +144,14 @@ where } #[derive( - Copy, Clone, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq, + Copy, + Clone, + Debug, + AsExpression, + FromSqlRow, + Serialize, + Deserialize, + PartialEq, )] #[sql_type = "sql_types::BigInt"] pub struct ByteCount(pub external::ByteCount); diff --git a/nexus/src/external_api/params.rs b/nexus/src/external_api/params.rs index e87ebdad265..6a0f46b48df 100644 --- a/nexus/src/external_api/params.rs +++ b/nexus/src/external_api/params.rs @@ -197,7 +197,8 @@ impl DiskCreate { pub fn extent_count(&self) -> i64 { let extent_size = self.extent_size().to_bytes(); - i64::try_from((self.size.to_bytes() + extent_size - 1) / extent_size).unwrap() + i64::try_from((self.size.to_bytes() + extent_size - 1) / extent_size) + .unwrap() } } @@ -250,17 +251,26 @@ mod test { let params = new_disk_create_params(ByteCount::try_from(1u64).unwrap()); assert_eq!(1, params.extent_count()); - let params = new_disk_create_params(ByteCount::try_from(EXTENT_SIZE - 1).unwrap()); + let params = new_disk_create_params( + ByteCount::try_from(EXTENT_SIZE - 1).unwrap(), + ); assert_eq!(1, params.extent_count()); - let params = new_disk_create_params(ByteCount::try_from(EXTENT_SIZE).unwrap()); + let params = + new_disk_create_params(ByteCount::try_from(EXTENT_SIZE).unwrap()); assert_eq!(1, params.extent_count()); - let params = new_disk_create_params(ByteCount::try_from(EXTENT_SIZE + 1).unwrap()); + let params = new_disk_create_params( + ByteCount::try_from(EXTENT_SIZE + 1).unwrap(), + ); assert_eq!(2, params.extent_count()); // Mostly just checking we don't blow up on an unwrap here. - let params = new_disk_create_params(ByteCount::try_from(i64::MAX).unwrap()); - assert!(params.size.to_bytes() < (params.extent_count() as u64) * params.extent_size().to_bytes()); + let params = + new_disk_create_params(ByteCount::try_from(i64::MAX).unwrap()); + assert!( + params.size.to_bytes() + < (params.extent_count() as u64) + * params.extent_size().to_bytes() + ); } } - diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index 6f7051da364..f2c72b690f0 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -38,7 +38,7 @@ use serde::Deserialize; use serde::Serialize; use slog::Logger; use std::collections::BTreeMap; -use std::convert::{TryInto, TryFrom}; +use std::convert::{TryFrom, TryInto}; use std::sync::Arc; use steno::new_action_noop_undo; use steno::ActionContext; @@ -603,12 +603,24 @@ fn saga_disk_delete() -> SagaTemplate { template_builder.append( "no_result", "DeleteDiskRecord", + // TODO: See the comment on the "DeleteRegions" step, + // we may want to un-delete the disk if we cannot remove + // underlying regions. new_action_noop_undo(sdd_delete_disk_record), ); template_builder.append( "no_result", "DeleteRegions", + // TODO(https://github.com/oxidecomputer/omicron/issues/612): + // We need a way to deal with this operation failing, aside from + // propagating the error to the user. + // + // What if the Sled goes offline? Nexus must ultimately be + // responsible for reconciling this scenario. + // + // The current behavior causes the disk deletion saga to + // fail, but still marks the disk as destroyed. new_action_noop_undo(sdd_delete_regions), ); From 39d2c6bf1660bfe97204fcff7e6feade92533dae Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 19 Jan 2022 15:23:46 -0500 Subject: [PATCH 44/50] logs, idempotency, and setting concurrency limits --- nexus/src/db/datastore.rs | 20 +++++++++++++++----- nexus/src/nexus.rs | 20 +++++++++++--------- nexus/src/saga_interface.rs | 7 ++++--- nexus/src/sagas.rs | 25 +++++++++++++++++-------- 4 files changed, 47 insertions(+), 25 deletions(-) diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index 6cba8530dc6..c5c567d41cc 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -1255,8 +1255,10 @@ impl DataStore { }) } - /// Attempts to delete a disk. Returns the disk (prior to deletion) - /// if successful. + /// Updates a disk record to indicate it has been deleted. + /// + /// Does not attempt to modify any resources (e.g. regions) which may + /// belong to the disk. // TODO: Delete me (this function, not the disk!), ensure all datastore // access is auth-checked. // @@ -1312,12 +1314,20 @@ impl DataStore { match result.status { UpdateStatus::Updated => Ok(()), UpdateStatus::NotUpdatedButExists => { - let disk_state = result.found.state(); - if !ok_to_delete_states.contains(disk_state.state()) { + let disk = result.found; + let disk_state = disk.state(); + if disk.time_deleted().is_some() + && disk_state.state() + == &api::external::DiskState::Destroyed + { + // To maintain idempotency, if the disk has already been + // destroyed, don't throw an error. + return Ok(()); + } else if !ok_to_delete_states.contains(disk_state.state()) { return Err(Error::InvalidRequest { message: format!( "disk cannot be deleted in state \"{}\"", - result.found.runtime_state.disk_state + disk.runtime_state.disk_state ), }); } else if disk_state.is_attached() { diff --git a/nexus/src/nexus.rs b/nexus/src/nexus.rs index 04898368715..f19a753b38d 100644 --- a/nexus/src/nexus.rs +++ b/nexus/src/nexus.rs @@ -209,28 +209,28 @@ impl Nexus { }; /* TODO-cleanup all the extra Arcs here seems wrong */ - let nexus_arc = Arc::new(nexus); + let nexus = Arc::new(nexus); let opctx = OpContext::for_background( log.new(o!("component" => "SagaRecoverer")), authz, authn::Context::internal_saga_recovery(), Arc::clone(&db_datastore), ); + let saga_logger = nexus.log.new(o!("saga_type" => "recovery")); let recovery_task = db::recover( opctx, my_sec_id, - Arc::new(Arc::new(SagaContext::new(Arc::clone(&nexus_arc)))), + Arc::new(Arc::new(SagaContext::new( + Arc::clone(&nexus), + saga_logger, + ))), db_datastore, Arc::clone(&sec_client), &sagas::ALL_TEMPLATES, ); - *nexus_arc.recovery_task.lock().unwrap() = Some(recovery_task); - nexus_arc - } - - pub fn log(&self) -> &Logger { - &self.log + *nexus.recovery_task.lock().unwrap() = Some(recovery_task); + nexus } pub async fn wait_for_populate(&self) -> Result<(), anyhow::Error> { @@ -442,8 +442,10 @@ impl Nexus { P: serde::Serialize, { let saga_id = SagaId(Uuid::new_v4()); + let saga_logger = + self.log.new(o!("template_name" => template_name.to_owned())); let saga_context = - Arc::new(Arc::new(SagaContext::new(Arc::clone(self)))); + Arc::new(Arc::new(SagaContext::new(Arc::clone(self), saga_logger))); let future = self .sec_client .saga_create( diff --git a/nexus/src/saga_interface.rs b/nexus/src/saga_interface.rs index 6793cb7656b..a3d54961797 100644 --- a/nexus/src/saga_interface.rs +++ b/nexus/src/saga_interface.rs @@ -23,6 +23,7 @@ use uuid::Uuid; */ pub struct SagaContext { nexus: Arc, + log: Logger, } impl fmt::Debug for SagaContext { @@ -32,12 +33,12 @@ impl fmt::Debug for SagaContext { } impl SagaContext { - pub fn new(nexus: Arc) -> SagaContext { - SagaContext { nexus } + pub fn new(nexus: Arc, log: Logger) -> SagaContext { + SagaContext { nexus, log } } pub fn log(&self) -> &Logger { - self.nexus.log() + &self.log } /* diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index f2c72b690f0..30f19e68330 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -401,12 +401,6 @@ async fn sdc_create_disk_record( let params = sagactx.saga_params(); let disk_id = sagactx.lookup::("disk_id")?; - - // NOTE: This could be done in a transaction alongside region allocation? - // - // Unclear if it's a problem to let this disk exist without any backing - // regions for a brief period of time, or if that's under the valid - // jurisdiction of "Creating". let disk = db::model::Disk::new( disk_id, params.project_id, @@ -443,6 +437,11 @@ async fn sdc_alloc_regions( // "creating" - the respective Crucible Agents must be instructed to // allocate the necessary regions before we can mark the disk as "ready to // be used". + // + // TODO: Depending on the result of + // https://github.com/oxidecomputer/omicron/issues/613 , we + // should consider using a paginated API to access regions, rather than + // returning all of them at once. let datasets_and_regions = osagactx .datastore() .region_allocate(disk_id, ¶ms.create_params) @@ -513,6 +512,10 @@ async fn ensure_region_in_dataset( Ok(region) } +// Arbitrary limit on concurrency, for operations issued +// on multiple regions within a disk at the same time. +const MAX_CONCURRENT_REGION_REQUESTS: usize = 3; + async fn sdc_regions_ensure( sagactx: ActionContext, ) -> Result<(), ActionError> { @@ -527,7 +530,10 @@ async fn sdc_regions_ensure( ensure_region_in_dataset(log, &dataset, ®ion).await }) // Execute the allocation requests concurrently. - .buffer_unordered(request_count) + .buffer_unordered(std::cmp::min( + request_count, + MAX_CONCURRENT_REGION_REQUESTS, + )) .collect::>>() .await .into_iter() @@ -550,7 +556,10 @@ async fn delete_regions( client.region_delete(&id).await }) // Execute the allocation requests concurrently. - .buffer_unordered(request_count) + .buffer_unordered(std::cmp::min( + request_count, + MAX_CONCURRENT_REGION_REQUESTS, + )) .collect::>>() .await .into_iter() From 64ad4f24bf7aef84dae4a429f4af3a99eeeaaa42 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 19 Jan 2022 17:08:32 -0500 Subject: [PATCH 45/50] extent_size -> blocks_per_extent (everywhere but sled-agent/, which is still acting like crucible-agent) --- common/src/sql/dbinit.sql | 2 +- nexus/src/db/datastore.rs | 10 ++++++---- nexus/src/db/model.rs | 10 +++++----- nexus/src/db/schema.rs | 2 +- nexus/src/external_api/params.rs | 20 +++++++++++--------- nexus/src/sagas.rs | 2 +- 6 files changed, 25 insertions(+), 21 deletions(-) diff --git a/common/src/sql/dbinit.sql b/common/src/sql/dbinit.sql index 55c4d19cc10..43ced3ad94e 100644 --- a/common/src/sql/dbinit.sql +++ b/common/src/sql/dbinit.sql @@ -129,7 +129,7 @@ CREATE TABLE omicron.public.Region ( /* Metadata describing the region */ block_size INT NOT NULL, - extent_size INT NOT NULL, + blocks_per_extent INT NOT NULL, extent_count INT NOT NULL ); diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index c5c567d41cc..8db93160753 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -325,7 +325,9 @@ impl DataStore { .select(Dataset::as_select()) .order( diesel::dsl::sum( - region_dsl::extent_size * region_dsl::extent_count, + region_dsl::blocks_per_extent + * region_dsl::block_size + * region_dsl::extent_count, ) .asc(), ) @@ -403,7 +405,7 @@ impl DataStore { dataset.id(), disk_id, params.block_size().into(), - params.extent_size().into(), + params.blocks_per_extent(), params.extent_count(), ) }) @@ -2851,7 +2853,7 @@ mod test { assert!(disk1_datasets.insert(dataset.id())); assert_eq!(disk1_id, region.disk_id()); assert_eq!(params.block_size(), region.block_size()); - assert_eq!(params.extent_size(), region.extent_size()); + assert_eq!(params.blocks_per_extent(), region.blocks_per_extent()); assert_eq!(params.extent_count(), region.extent_count()); } @@ -2870,7 +2872,7 @@ mod test { assert!(disk2_datasets.insert(dataset.id())); assert_eq!(disk2_id, region.disk_id()); assert_eq!(params.block_size(), region.block_size()); - assert_eq!(params.extent_size(), region.extent_size()); + assert_eq!(params.blocks_per_extent(), region.blocks_per_extent()); assert_eq!(params.extent_count(), region.extent_count()); } diff --git a/nexus/src/db/model.rs b/nexus/src/db/model.rs index 5b1961ccec6..438a11b1a5a 100644 --- a/nexus/src/db/model.rs +++ b/nexus/src/db/model.rs @@ -638,7 +638,7 @@ pub struct Region { disk_id: Uuid, block_size: ByteCount, - extent_size: ByteCount, + blocks_per_extent: i64, extent_count: i64, } @@ -647,7 +647,7 @@ impl Region { dataset_id: Uuid, disk_id: Uuid, block_size: ByteCount, - extent_size: ByteCount, + blocks_per_extent: i64, extent_count: i64, ) -> Self { Self { @@ -655,7 +655,7 @@ impl Region { dataset_id, disk_id, block_size, - extent_size, + blocks_per_extent, extent_count, } } @@ -669,8 +669,8 @@ impl Region { pub fn block_size(&self) -> external::ByteCount { self.block_size.0 } - pub fn extent_size(&self) -> external::ByteCount { - self.extent_size.0 + pub fn blocks_per_extent(&self) -> i64 { + self.blocks_per_extent } pub fn extent_count(&self) -> i64 { self.extent_count diff --git a/nexus/src/db/schema.rs b/nexus/src/db/schema.rs index 0067899e2be..99996166ce7 100644 --- a/nexus/src/db/schema.rs +++ b/nexus/src/db/schema.rs @@ -204,7 +204,7 @@ table! { disk_id -> Uuid, block_size -> Int8, - extent_size -> Int8, + blocks_per_extent -> Int8, extent_count -> Int8, } } diff --git a/nexus/src/external_api/params.rs b/nexus/src/external_api/params.rs index 6a0f46b48df..043dc4fb4ab 100644 --- a/nexus/src/external_api/params.rs +++ b/nexus/src/external_api/params.rs @@ -12,7 +12,6 @@ use omicron_common::api::external::{ }; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use std::convert::TryFrom; use uuid::Uuid; /* @@ -183,7 +182,7 @@ pub struct DiskCreate { pub size: ByteCount, } -const BLOCK_SIZE: u32 = 512_u32; +const BLOCK_SIZE: u32 = 1_u32 << 12; const EXTENT_SIZE: u32 = 1_u32 << 20; impl DiskCreate { @@ -191,14 +190,15 @@ impl DiskCreate { ByteCount::from(BLOCK_SIZE) } - pub fn extent_size(&self) -> ByteCount { - ByteCount::from(EXTENT_SIZE) + pub fn blocks_per_extent(&self) -> i64 { + EXTENT_SIZE as i64 / BLOCK_SIZE as i64 } pub fn extent_count(&self) -> i64 { - let extent_size = self.extent_size().to_bytes(); - i64::try_from((self.size.to_bytes() + extent_size - 1) / extent_size) - .unwrap() + let extent_size = EXTENT_SIZE as i64; + let size = self.size.to_bytes() as i64; + size / extent_size + + ((size % extent_size) + extent_size - 1) / extent_size } } @@ -232,6 +232,7 @@ pub struct UserBuiltinCreate { #[cfg(test)] mod test { use super::*; + use std::convert::TryFrom; fn new_disk_create_params(size: ByteCount) -> DiskCreate { DiskCreate { @@ -269,8 +270,9 @@ mod test { new_disk_create_params(ByteCount::try_from(i64::MAX).unwrap()); assert!( params.size.to_bytes() - < (params.extent_count() as u64) - * params.extent_size().to_bytes() + <= (params.extent_count() as u64) + * (params.blocks_per_extent() as u64) + * params.block_size().to_bytes() ); } } diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index 30f19e68330..34c52b124ba 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -471,7 +471,7 @@ async fn ensure_region_in_dataset( let region_request = CreateRegion { block_size: region.block_size().to_bytes(), extent_count: region.extent_count().try_into().unwrap(), - extent_size: region.extent_size().to_bytes(), + extent_size: region.blocks_per_extent().try_into().unwrap(), // TODO: Can we avoid casting from UUID to string? // NOTE: This'll require updating the crucible agent client. id: RegionId(region.id().to_string()), From e89325363435234e442dbeb2b353811ef77f8c92 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 19 Jan 2022 22:51:51 -0500 Subject: [PATCH 46/50] Store used size of datasets, fix indices, avoid full table scans --- common/src/sql/dbinit.sql | 12 +++- nexus/src/db/datastore.rs | 118 ++++++++++++++++++++------------------ nexus/src/db/model.rs | 6 ++ nexus/src/db/schema.rs | 2 + 4 files changed, 80 insertions(+), 58 deletions(-) diff --git a/common/src/sql/dbinit.sql b/common/src/sql/dbinit.sql index 43ced3ad94e..6780574a181 100644 --- a/common/src/sql/dbinit.sql +++ b/common/src/sql/dbinit.sql @@ -105,13 +105,21 @@ CREATE TABLE omicron.public.Dataset ( /* FK into the Pool table */ pool_id UUID NOT NULL, - /* Contact information for the downstairs region */ + /* Contact information for the dataset */ ip INET NOT NULL, port INT4 NOT NULL, - kind omicron.public.dataset_kind NOT NULL + kind omicron.public.dataset_kind NOT NULL, + + /* An upper bound on the amount of space that might be in-use */ + size_used INT ); +/* Create an index on the size usage for Crucible's allocation */ +CREATE INDEX on omicron.public.Dataset ( + size_used +) WHERE size_used IS NOT NULL AND time_deleted IS NULL AND kind = 'crucible'; + /* * A region of space allocated to Crucible Downstairs, within a dataset. */ diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index 8db93160753..831eb3800aa 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -296,41 +296,22 @@ impl DataStore { } fn get_allocatable_datasets_query() -> impl RunnableQuery { - use db::schema::dataset::dsl as dataset_dsl; - use db::schema::region::dsl as region_dsl; + use db::schema::dataset::dsl; - dataset_dsl::dataset + dsl::dataset // We look for valid datasets (non-deleted crucible datasets). - .filter(dataset_dsl::time_deleted.is_null()) - .filter(dataset_dsl::kind.eq(DatasetKind( + .filter(dsl::size_used.is_not_null()) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::kind.eq(DatasetKind( crate::internal_api::params::DatasetKind::Crucible, ))) - // Next, observe all the regions allocated to each dataset, and - // determine how much space they're using. - // - // TODO: We could store "free/allocated" space per-dataset, - // and keep them up-to-date, rather than trying to recompute - // this. - // + .order(dsl::size_used.asc()) // TODO: We admittedly don't actually *fail* any request for // running out of space - we try to send the request down to // crucible agents, and expect them to fail on our behalf in // out-of-storage conditions. This should undoubtedly be // handled more explicitly. - .left_outer_join( - region_dsl::region - .on(dataset_dsl::id.eq(region_dsl::dataset_id)), - ) - .group_by(dataset_dsl::id) .select(Dataset::as_select()) - .order( - diesel::dsl::sum( - region_dsl::blocks_per_extent - * region_dsl::block_size - * region_dsl::extent_count, - ) - .asc(), - ) .limit(REGION_REDUNDANCY_THRESHOLD.try_into().unwrap()) } @@ -343,6 +324,7 @@ impl DataStore { disk_id: Uuid, params: ¶ms::DiskCreate, ) -> Result, Error> { + use db::schema::dataset::dsl as dataset_dsl; use db::schema::region::dsl as region_dsl; // ALLOCATION POLICY @@ -386,7 +368,7 @@ impl DataStore { return Ok(datasets_and_regions); } - let datasets: Vec = + let mut datasets: Vec = Self::get_allocatable_datasets_query() .get_results::(conn)?; @@ -397,7 +379,8 @@ impl DataStore { } // Create identical regions on each of the following datasets. - let source_datasets = &datasets[0..REGION_REDUNDANCY_THRESHOLD]; + let source_datasets = + &mut datasets[0..REGION_REDUNDANCY_THRESHOLD]; let regions: Vec = source_datasets .iter() .map(|dataset| { @@ -415,6 +398,26 @@ impl DataStore { .returning(Region::as_returning()) .get_results(conn)?; + // Update the tallied sizes in the source datasets containing + // those regions. + let region_size = i64::from(params.block_size()) + * params.blocks_per_extent() + * params.extent_count(); + for dataset in source_datasets.iter_mut() { + dataset.size_used = + dataset.size_used.map(|v| v + region_size); + } + + let dataset_ids: Vec = + source_datasets.iter().map(|ds| ds.id()).collect(); + diesel::update(dataset_dsl::dataset) + .filter(dataset_dsl::id.eq_any(dataset_ids)) + .set( + dataset_dsl::size_used + .eq(dataset_dsl::size_used + region_size), + ) + .execute(conn)?; + // Return the regions with the datasets to which they were allocated. Ok(source_datasets .into_iter() @@ -429,20 +432,44 @@ impl DataStore { } /// Deletes all regions backing a disk. + /// + /// Also updates the storage usage on their corresponding datasets. pub async fn regions_hard_delete(&self, disk_id: Uuid) -> DeleteResult { - use db::schema::region::dsl; + use db::schema::dataset::dsl as dataset_dsl; + use db::schema::region::dsl as region_dsl; - diesel::delete(dsl::region) - .filter(dsl::disk_id.eq(disk_id)) - .execute_async(self.pool()) + // Remove the regions, collecting datasets they're from. + let (dataset_id, size) = diesel::delete(region_dsl::region) + .filter(region_dsl::disk_id.eq(disk_id)) + .returning(( + region_dsl::dataset_id, + region_dsl::block_size + * region_dsl::blocks_per_extent + * region_dsl::extent_count, + )) + .get_result_async::<(Uuid, i64)>(self.pool()) .await - .map(|_| ()) .map_err(|e| { Error::internal_error(&format!( "error deleting regions: {:?}", e )) - }) + })?; + + // Update those datasets to which the regions belonged. + diesel::update(dataset_dsl::dataset) + .filter(dataset_dsl::id.eq(dataset_id)) + .set(dataset_dsl::size_used.eq(dataset_dsl::size_used - size)) + .execute_async(self.pool()) + .await + .map_err(|e| { + Error::internal_error(&format!( + "error updating dataset space: {:?}", + e + )) + })?; + + Ok(()) } /// Create a organization @@ -3053,34 +3080,13 @@ mod test { explanation ); - let _ = db.cleanup().await; - } - - // Welp, life isn't perfect - sometimes, we take shortcuts, and implement - // queries that DO require FULL SCAN. - // - // These are problematic scans from a performance point-of-view, but we can - // keep track of which queries do so with this test! - // - // NOTE: If this test is failing because a table scan is no longer required, - // congratulations, move that query into the more appropriate test: - // `test_queries_do_not_require_full_table_scan`. - #[tokio::test] - async fn test_queries_that_do_require_full_table_scan() { - let logctx = - dev::test_setup_log("test_queries_that_do_require_full_table_scan"); - let mut db = test_setup_database(&logctx.log).await; - let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&cfg); - let datastore = DataStore::new(Arc::new(pool)); - let explanation = DataStore::get_allocatable_datasets_query() .explain_async(datastore.pool()) .await .unwrap(); assert!( - explanation.contains("FULL SCAN"), - "Expected FULL SCAN: {}", + !explanation.contains("FULL SCAN"), + "Found an unexpected FULL SCAN: {}", explanation ); diff --git a/nexus/src/db/model.rs b/nexus/src/db/model.rs index 438a11b1a5a..1fb308610d9 100644 --- a/nexus/src/db/model.rs +++ b/nexus/src/db/model.rs @@ -572,6 +572,7 @@ pub struct Dataset { port: i32, kind: DatasetKind, + pub size_used: Option, } impl Dataset { @@ -581,6 +582,10 @@ impl Dataset { addr: SocketAddr, kind: DatasetKind, ) -> Self { + let size_used = match kind { + DatasetKind(internal_api::params::DatasetKind::Crucible) => Some(0), + _ => None, + }; Self { identity: DatasetIdentity::new(id), time_deleted: None, @@ -589,6 +594,7 @@ impl Dataset { ip: addr.ip().into(), port: addr.port().into(), kind, + size_used, } } diff --git a/nexus/src/db/schema.rs b/nexus/src/db/schema.rs index 99996166ce7..fc7424f4883 100644 --- a/nexus/src/db/schema.rs +++ b/nexus/src/db/schema.rs @@ -191,6 +191,8 @@ table! { port -> Int4, kind -> crate::db::model::DatasetKindEnum, + + size_used -> Nullable, } } From 2d918fb0d0e12477f12643b6c63006e928d3f822 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 21 Jan 2022 10:24:59 -0500 Subject: [PATCH 47/50] Add expectorate --- nexus/src/db/explain.rs | 42 ++++++++++---------------- nexus/tests/output/test-explain-output | 7 +++++ 2 files changed, 23 insertions(+), 26 deletions(-) create mode 100644 nexus/tests/output/test-explain-output diff --git a/nexus/src/db/explain.rs b/nexus/src/db/explain.rs index 3ce77919597..9b03904e592 100644 --- a/nexus/src/db/explain.rs +++ b/nexus/src/db/explain.rs @@ -118,6 +118,7 @@ mod test { use crate::db; use async_bb8_diesel::{AsyncConnection, AsyncSimpleConnection}; use diesel::SelectableHelper; + use expectorate::assert_contents; use nexus_test_utils::db::test_setup_database; use omicron_test_utils::dev; use uuid::Uuid; @@ -174,22 +175,20 @@ mod test { use schema::test_users::dsl; pool.pool() - .transaction(move |conn| -> Result<(), db::error::TransactionError<()>> { - let explanation = dsl::test_users - .filter(dsl::id.eq(Uuid::nil())) - .select(User::as_select()) - .explain(conn) - .unwrap(); - assert_eq!(r#"distribution: local -vectorized: true - -• scan - missing stats - table: test_users@primary - spans: [/'00000000-0000-0000-0000-000000000000' - /'00000000-0000-0000-0000-000000000000']"#, - explanation); - Ok(()) - }) + .transaction( + move |conn| -> Result<(), db::error::TransactionError<()>> { + let explanation = dsl::test_users + .filter(dsl::id.eq(Uuid::nil())) + .select(User::as_select()) + .explain(conn) + .unwrap(); + assert_contents( + "tests/output/test-explain-output", + &explanation, + ); + Ok(()) + }, + ) .await .unwrap(); } @@ -212,16 +211,7 @@ vectorized: true .await .unwrap(); - assert_eq!( - r#"distribution: local -vectorized: true - -• scan - missing stats - table: test_users@primary - spans: [/'00000000-0000-0000-0000-000000000000' - /'00000000-0000-0000-0000-000000000000']"#, - explanation - ); + assert_contents("tests/output/test-explain-output", &explanation); } // Tests that ".explain()" can tell us when we're doing full table scans. diff --git a/nexus/tests/output/test-explain-output b/nexus/tests/output/test-explain-output new file mode 100644 index 00000000000..e2574ab7014 --- /dev/null +++ b/nexus/tests/output/test-explain-output @@ -0,0 +1,7 @@ +distribution: local +vectorized: true + +• scan + missing stats + table: test_users@primary + spans: [/'00000000-0000-0000-0000-000000000000' - /'00000000-0000-0000-0000-000000000000'] From 08bcfdca8004482287c53aff8b98c69700a63b00 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 21 Jan 2022 10:26:29 -0500 Subject: [PATCH 48/50] I forgot to EXPECTORATE the right newline --- nexus/tests/output/test-explain-output | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/tests/output/test-explain-output b/nexus/tests/output/test-explain-output index e2574ab7014..0f065162712 100644 --- a/nexus/tests/output/test-explain-output +++ b/nexus/tests/output/test-explain-output @@ -4,4 +4,4 @@ vectorized: true • scan missing stats table: test_users@primary - spans: [/'00000000-0000-0000-0000-000000000000' - /'00000000-0000-0000-0000-000000000000'] + spans: [/'00000000-0000-0000-0000-000000000000' - /'00000000-0000-0000-0000-000000000000'] \ No newline at end of file From 8e2f3a5d10d0f0a62d29b3e2734e33858a02dc78 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 21 Jan 2022 10:50:47 -0500 Subject: [PATCH 49/50] Update deps, rely on new implied JsonSchema derives --- Cargo.lock | 4766 ++++++++++++++++++++++++++++++++++ Cargo.toml | 6 +- nexus-client/Cargo.toml | 1 - nexus/Cargo.toml | 2 +- oximeter-client/Cargo.toml | 1 - sled-agent-client/Cargo.toml | 1 - sled-agent/Cargo.toml | 2 +- 7 files changed, 4770 insertions(+), 9 deletions(-) create mode 100644 Cargo.lock diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 00000000000..38b7d60617f --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,4766 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aead" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b613b8e1e3cf911a086f53f03bf286f52fd7a7258e4fa606f0ef220d39d8877" +dependencies = [ + "generic-array 0.14.4", +] + +[[package]] +name = "aes" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", + "opaque-debug 0.3.0", +] + +[[package]] +name = "aes-gcm-siv" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589c637f0e68c877bbd59a4599bbe849cac8e5f3e4b5a3ebae8f528cd218dcdc" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "polyval", + "subtle", + "zeroize", +] + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "anyhow" +version = "1.0.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84450d0b4a8bd1ba4144ce8ce718fbc5d071358b1e5384bace6536b3d1f2d5b3" + +[[package]] +name = "api_identity" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "array-init" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6945cc5422176fc5e602e590c2878d2c2acd9a4fe20a4baa7c28022521698ec6" + +[[package]] +name = "ascii-canvas" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6" +dependencies = [ + "term", +] + +[[package]] +name = "async-bb8-diesel" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/async-bb8-diesel?rev=c849b717be#c849b717be1e33528c7c3950fefcb431a65ef469" +dependencies = [ + "async-trait", + "bb8", + "diesel", + "thiserror", + "tokio", +] + +[[package]] +name = "async-stream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171374e7e3b2504e0e5236e3b59260560f9fe94bfe9ac39ba5e4e929c5590625" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "061a7acccaa286c011ddc30970520b98fa40e00c9d644633fb26b5fc63a265e3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "backoff" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fe17f59a06fe8b87a6fc8bf53bb70b3aba76d7685f432487a68cd5552853625" +dependencies = [ + "futures-core", + "getrandom", + "instant", + "pin-project", + "rand", + "tokio", +] + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bb8" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9f4fa9768efd269499d8fba693260cfc670891cf6de3adc935588447a77cc8" +dependencies = [ + "async-trait", + "futures-channel", + "futures-util", + "parking_lot", + "tokio", +] + +[[package]] +name = "bhyve_api" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/propolis?rev=00ec8cf18f6a2311b0907f0b16b0ff8a327944d1#00ec8cf18f6a2311b0907f0b16b0ff8a327944d1" +dependencies = [ + "bitflags", + "libc", + "num_enum", +] + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bit-set" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e11e16035ea35e4e5997b393eacbf6f63983188f7a2ad25bfb13465f5ad59de" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitstruct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1b10c3912af09af44ea1dafe307edb5ed374b2a32658eb610e372270c9017b4" +dependencies = [ + "bitstruct_derive", +] + +[[package]] +name = "bitstruct_derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35fd19022c2b750d14eb9724c204d08ab7544570105b3b466d8a9f2f3feded27" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "bitvec" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5237f00a8c86130a0cc317830e558b966dd7850d48a953d998c813f01a41b527" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array 0.12.4", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array 0.14.4", +] + +[[package]] +name = "block-buffer" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d36a02058e76b040de25a4464ba1c80935655595b661505c8b39b664828b95" +dependencies = [ + "generic-array 0.14.4", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1e260c3a9040a7c19a12468758f4c16f31a81a1fe087482be9570ec864bb6c" + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +dependencies = [ + "serde", +] + +[[package]] +name = "cast" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c24dab4283a142afa2fdca129b80ad2c6284e073930f964c3a1293c225ee39a" +dependencies = [ + "rustc_version 0.4.0", +] + +[[package]] +name = "cc" +version = "1.0.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "serde", + "time 0.1.44", + "winapi", +] + +[[package]] +name = "cipher" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" +dependencies = [ + "generic-array 0.14.4", +] + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim 0.8.0", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "const-oid" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6f2aa4d0537bcc1c74df8755072bd31c1ef1a3a1b85a68e8404a8c353b7b8b" + +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + +[[package]] +name = "convert_case" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb4a24b1aaf0fd0ce8b45161144d6f42cd91677fd5940fd431183eb023b3a2b8" + +[[package]] +name = "cookie" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94d4706de1b0fa5b132270cddffa8585166037822e260a944fe161acd137ca05" +dependencies = [ + "time 0.3.5", + "version_check", +] + +[[package]] +name = "core-foundation" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6888e10551bb93e424d8df1d07f1a8b4fceb0001a3a4b048bfc47554946f47b3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "cpufeatures" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" +dependencies = [ + "libc", +] + +[[package]] +name = "criterion" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1604dafd25fba2fe2d5895a9da139f8dc9b319a5fe5354ca137cbbce4e178d10" +dependencies = [ + "atty", + "cast", + "clap", + "criterion-plot", + "csv", + "futures", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "tokio", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d00996de9f2f7559f7f4dc286073197f83e92256a59ed395f9aac01fe717da57" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae5588f6b3c3cb05239e90bd110f257254aecd01e4635400391aeae07497845" +dependencies = [ + "cfg-if", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b10ddc024425c88c2ad148c1b0fd53f4c6d38db9697c9f1588381212fa657c9" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" +dependencies = [ + "cfg-if", + "lazy_static", +] + +[[package]] +name = "crucible" +version = "0.0.1" +source = "git+https://github.com/oxidecomputer/crucible?branch=main#4f02540ab2557ad75e9f170d362dc889eefe1d1b" +dependencies = [ + "aes", + "aes-gcm-siv", + "anyhow", + "base64", + "bytes", + "crucible-common", + "crucible-protocol", + "crucible-scope", + "futures", + "futures-core", + "rand", + "rand_chacha", + "ringbuffer", + "serde", + "serde_json", + "structopt", + "tokio", + "tokio-util", + "toml", + "tracing", + "usdt 0.2.1", + "uuid", + "xts-mode", +] + +[[package]] +name = "crucible-agent-client" +version = "0.0.1" +source = "git+https://github.com/oxidecomputer/crucible?rev=078d364e14d57d5faa3a44001c65709935419779#078d364e14d57d5faa3a44001c65709935419779" +dependencies = [ + "anyhow", + "percent-encoding", + "progenitor", + "reqwest", + "schemars", + "serde", + "serde_json", +] + +[[package]] +name = "crucible-common" +version = "0.0.0" +source = "git+https://github.com/oxidecomputer/crucible?branch=main#4f02540ab2557ad75e9f170d362dc889eefe1d1b" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "tempfile", + "thiserror", + "toml", + "uuid", +] + +[[package]] +name = "crucible-protocol" +version = "0.0.0" +source = "git+https://github.com/oxidecomputer/crucible?branch=main#4f02540ab2557ad75e9f170d362dc889eefe1d1b" +dependencies = [ + "anyhow", + "bincode", + "bytes", + "crucible-common", + "serde", + "tokio-util", + "uuid", +] + +[[package]] +name = "crucible-scope" +version = "0.0.0" +source = "git+https://github.com/oxidecomputer/crucible?branch=main#4f02540ab2557ad75e9f170d362dc889eefe1d1b" +dependencies = [ + "anyhow", + "futures", + "futures-core", + "serde", + "serde_json", + "tokio", + "tokio-util", + "toml", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-bigint" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83bd3bb4314701c568e340cd8cf78c975aa0ca79e03d3f6d1677d5b0c9c0c03" +dependencies = [ + "generic-array 0.14.4", + "rand_core", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d6b536309245c849479fba3da410962a43ed8e51c26b729208ec0ac2798d0" +dependencies = [ + "generic-array 0.14.4", +] + +[[package]] +name = "crypto-mac" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" +dependencies = [ + "generic-array 0.14.4", + "subtle", +] + +[[package]] +name = "csv" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +dependencies = [ + "bstr", + "csv-core", + "itoa 0.4.8", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + +[[package]] +name = "ctr" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea" +dependencies = [ + "cipher", +] + +[[package]] +name = "curve25519-dalek" +version = "4.0.0-pre.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4033478fbf70d6acf2655ac70da91ee65852d69daf7a67bf7a2f518fb47aafcf" +dependencies = [ + "byteorder", + "digest 0.9.0", + "rand_core", + "subtle", + "zeroize", +] + +[[package]] +name = "darling" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d720b8683f8dd83c65155f0530560cba68cd2bf395f6513a483caee57ff7f4" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a340f241d2ceed1deb47ae36c4144b2707ec7dd0b649f894cb39bb595986324" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72c41b3b7352feb3211a0d743dc5700a4e3b60f51bd2b368892d1e0f9a95f44b" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "db-macros" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "der" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79b71cca7d95d7681a4b3b9cdf63c8dbc3730d0584c2c74e31416d64a90493f4" +dependencies = [ + "const-oid", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "diesel" +version = "2.0.0" +source = "git+https://github.com/diesel-rs/diesel?rev=ce77c382#ce77c382d2836f6b385225991cf58cb2d2dd65d6" +dependencies = [ + "bitflags", + "byteorder", + "chrono", + "diesel_derives", + "ipnetwork", + "itoa 0.4.8", + "libc", + "pq-sys", + "r2d2", + "serde_json", + "uuid", +] + +[[package]] +name = "diesel-dtrace" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/diesel-dtrace?branch=main#f74c1db52e86f53d9234969541673d1d9aecb101" +dependencies = [ + "diesel", + "serde", + "usdt 0.3.1", + "uuid", +] + +[[package]] +name = "diesel_derives" +version = "2.0.0" +source = "git+https://github.com/diesel-rs/diesel?rev=ce77c382#ce77c382d2836f6b385225991cf58cb2d2dd65d6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "diff" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499" + +[[package]] +name = "difference" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array 0.12.4", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array 0.14.4", +] + +[[package]] +name = "digest" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b697d66081d42af4fba142d56918a3cb21dc8eb63372c6b85d14f44fb9c5979b" +dependencies = [ + "block-buffer 0.10.0", + "crypto-common", + "generic-array 0.14.4", + "subtle", +] + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "dladm" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/propolis?rev=00ec8cf18f6a2311b0907f0b16b0ff8a327944d1#00ec8cf18f6a2311b0907f0b16b0ff8a327944d1" +dependencies = [ + "libc", + "num_enum", +] + +[[package]] +name = "dof" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e6b21a1211455e82b1245d6e1b024f30606afbb734c114515d40d0e0b34ce81" +dependencies = [ + "thiserror", + "zerocopy", +] + +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + +[[package]] +name = "dropshot" +version = "0.6.1-dev" +source = "git+https://github.com/oxidecomputer/dropshot?branch=main#8e4af93207fb79998eea90bd094ff7a5475673e5" +dependencies = [ + "async-stream", + "async-trait", + "base64", + "bytes", + "chrono", + "dropshot_endpoint", + "futures", + "hostname", + "http", + "hyper", + "indexmap", + "openapiv3", + "paste", + "percent-encoding", + "proc-macro2", + "rustls", + "rustls-pemfile", + "schemars", + "serde", + "serde_json", + "serde_urlencoded", + "slog", + "slog-async", + "slog-bunyan", + "slog-json", + "slog-term", + "syn", + "tokio", + "tokio-rustls", + "toml", + "usdt 0.3.1", + "uuid", +] + +[[package]] +name = "dropshot_endpoint" +version = "0.6.1-dev" +source = "git+https://github.com/oxidecomputer/dropshot?branch=main#8e4af93207fb79998eea90bd094ff7a5475673e5" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_tokenstream", + "syn", +] + +[[package]] +name = "dtrace-parser" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bbb93fb1a0c517bf20f37caaf5d1f7d20f144c6c35a7d751ecad077b6c042e8" +dependencies = [ + "pest", + "pest_derive", + "thiserror", +] + +[[package]] +name = "dyn-clone" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2626afccd7561a06cf1367e2950c4718ea04565e20fb5029b6c7d8ad09abcf" + +[[package]] +name = "ecdsa" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43ee23aa5b4f68c7a092b5c3beb25f50c406adc75e2363634f242f28ab255372" +dependencies = [ + "der", + "elliptic-curve", + "hmac 0.11.0", + "signature", +] + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "elliptic-curve" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beca177dcb8eb540133e7680baff45e7cc4d93bf22002676cec549f82343721b" +dependencies = [ + "crypto-bigint", + "ff", + "generic-array 0.14.4", + "group", + "pkcs8", + "rand_core", + "subtle", + "zeroize", +] + +[[package]] +name = "ena" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7402b94a93c24e742487327a7cd839dc9d36fec9de9fb25b09f2dae459f36c3" +dependencies = [ + "log", +] + +[[package]] +name = "encoding_rs" +version = "0.8.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "erased-serde" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3de9ad4541d99dc22b59134e7ff8dc3d6c988c89ecd7324bf10a8362b07a2afa" +dependencies = [ + "serde", +] + +[[package]] +name = "expectorate" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804d601ea8a13ddbecf5ab4b6cf75b5d6d0539479c6fb2aea1596e352a5ee27e" +dependencies = [ + "difference", + "newline-converter", +] + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fastrand" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "779d043b6a0b90cc4c0ed7ee380a6504394cee7efd7db050e3774eee387324b2" +dependencies = [ + "instant", +] + +[[package]] +name = "ff" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0f40b2dcd8bc322217a5f6559ae5f9e9d1de202a2ecee2e9eafcbece7562a4f" +dependencies = [ + "bitvec", + "rand_core", + "subtle", +] + +[[package]] +name = "filetime" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "975ccf83d8d9d0d84682850a38c8169027be83368805971cc4f238c2b245bc98" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "winapi", +] + +[[package]] +name = "fixedbitset" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" + +[[package]] +name = "fixedbitset" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "398ea4fabe40b9b0d885340a2a991a44c8a645624075ad966d21f88688e2b69e" + +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "fragile" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69a039c3498dc930fe810151a34ba0c1c70b02b8625035592e74432f678591f2" + +[[package]] +name = "funty" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1847abb9cb65d566acd5942e94aea9c8f547ad02c98e1649326fc0e8910b8b1e" + +[[package]] +name = "futures" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28560757fe2bb34e79f907794bb6b22ae8b0e5c669b638a1132f2592b19035b4" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3dda0b6588335f360afc675d0564c17a77a2bda81ca178a4b6081bd86c7f0b" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0c8ff0461b82559810cdccfde3215c3f373807f5e5232b71479bff7bb2583d7" + +[[package]] +name = "futures-executor" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29d6d2ff5bb10fb95c85b8ce46538a2e5f5e7fdc755623a7d4529ab8a4ed9d2a" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9d34af5a1aac6fb380f735fe510746c38067c5bf16c7fd250280503c971b2" + +[[package]] +name = "futures-macro" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbd947adfffb0efc70599b3ddcf7b5597bb5fa9e245eb99f62b3a5f7bb8bd3c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3055baccb68d74ff6480350f8d6eb8fcfa3aa11bdc1a1ae3afdd0514617d508" + +[[package]] +name = "futures-task" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ee7c6485c30167ce4dfb83ac568a849fe53274c831081476ee13e0dce1aad72" + +[[package]] +name = "futures-util" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b5cf40b47a271f77a8b1bec03ca09044d99d2372c0de244e66430761127164" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" +dependencies = [ + "typenum", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "group" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c363a5301b8f153d80747126a04b3c82073b9fe3130571a9d170cacdeaf7912" +dependencies = [ + "byteorder", + "ff", + "rand_core", + "subtle", +] + +[[package]] +name = "h2" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9de88456263e249e241fcd211d3954e2c9b0ef7ccfc235a444eb367cae3689" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7a30908dbce072eca83216eab939d2290080e00ca71611b96a09e5cdce5f3fa" +dependencies = [ + "libc", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" +dependencies = [ + "crypto-mac", + "digest 0.9.0", +] + +[[package]] +name = "hmac" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddca131f3e7f2ce2df364b57949a9d47915cfbd35e46cfee355ccebbf794d6a2" +dependencies = [ + "digest 0.10.1", +] + +[[package]] +name = "home" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2456aef2e6b6a9784192ae780c0f15bc57df0e918585282325e8c8ac27737654" +dependencies = [ + "winapi", +] + +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi", +] + +[[package]] +name = "http" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" +dependencies = [ + "bytes", + "fnv", + "itoa 1.0.1", +] + +[[package]] +name = "http-body" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "hyper" +version = "0.14.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ec3e62bdc98a2f0393a5048e4c30ef659440ea6e0e572965103e72bd836f55" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa 0.4.8", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac" +dependencies = [ + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "impl-trait-for-tuples" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5dacb10c5b3bb92d46ba347505a9041e676bb20ad220101326bffb0c93031ee" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "indexmap" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" +dependencies = [ + "autocfg", + "hashbrown", + "serde", +] + +[[package]] +name = "indoc" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a75aeaaef0ce18b58056d306c27b07436fbb34b8816c53094b76dd81803136" +dependencies = [ + "unindent", +] + +[[package]] +name = "input_buffer" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f97967975f448f1a7ddb12b0bc41069d09ed6a1c161a92687e057325db35d413" +dependencies = [ + "bytes", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "ipnet" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" + +[[package]] +name = "ipnetwork" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4088d739b183546b239688ddbc79891831df421773df95e236daf7867866d355" +dependencies = [ + "serde", +] + +[[package]] +name = "itertools" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + +[[package]] +name = "itoa" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" + +[[package]] +name = "js-sys" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cc9ffccd38c451a86bf13657df244e9c3f37493cce8e5e21e940963777acc84" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "k256" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "903ae2481bcdfdb7b68e0a9baa4b7c9aff600b9ae2e8e5bb5833b8c91ab851ea" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "sha2 0.9.8", +] + +[[package]] +name = "lalrpop" +version = "0.19.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15174f1c529af5bf1283c3bc0058266b483a67156f79589fab2a25e23cf8988" +dependencies = [ + "ascii-canvas", + "atty", + "bit-set", + "diff", + "ena", + "itertools", + "lalrpop-util", + "petgraph 0.5.1", + "regex", + "regex-syntax", + "string_cache", + "term", + "tiny-keccak", + "unicode-xid", +] + +[[package]] +name = "lalrpop-util" +version = "0.19.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e58cce361efcc90ba8a0a5f982c741ff86b603495bb15a998412e957dcd278" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.112" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125" + +[[package]] +name = "lock_api" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "macaddr" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baee0bbc17ce759db233beb01648088061bf678383130602a298e6998eedb2d8" +dependencies = [ + "serde", +] + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + +[[package]] +name = "md-5" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6a38fc55c8bbc10058782919516f88826e70320db6d206aebc49611d24216ae" +dependencies = [ + "digest 0.10.1", +] + +[[package]] +name = "memchr" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "mio" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc" +dependencies = [ + "libc", + "log", + "miow", + "ntapi", + "winapi", +] + +[[package]] +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi", +] + +[[package]] +name = "mockall" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d4d70639a72f972725db16350db56da68266ca368b2a1fe26724a903ad3d6b8" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79ef208208a0dea3f72221e26e904cdc6db2e481d9ade89081ddd494f1dbaa6b" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "native-tls" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" + +[[package]] +name = "newline-converter" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f81c2b19eebbc4249b3ca6aff70ae05bf18d6a99b7cc63cf0248774e640565" + +[[package]] +name = "newtype_derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac8cd24d9f185bb7223958d8c1ff7a961b74b1953fd05dba7cc568a63b3861ec" +dependencies = [ + "rustc_version 0.1.7", +] + +[[package]] +name = "nexus-client" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "omicron-common", + "percent-encoding", + "progenitor", + "reqwest", + "serde", + "serde_json", + "slog", + "uuid", +] + +[[package]] +name = "nexus-test-utils" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "chrono", + "dropshot", + "http", + "hyper", + "omicron-common", + "omicron-nexus", + "omicron-sled-agent", + "omicron-test-utils", + "oximeter", + "oximeter-client", + "oximeter-collector", + "oximeter-producer", + "parse-display", + "serde", + "serde_json", + "slog", + "tokio", + "uuid", +] + +[[package]] +name = "nexus-test-utils-macros" +version = "0.1.0" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "num_enum" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9bd055fb730c4f8f4f57d45d35cd6b3f0980535b056dc7ff119cee6a66ed6f" +dependencies = [ + "derivative", + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486ea01961c4a818096de679a8b740b26d9033146ac5291b1c98557658f8cdd9" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "omicron-common" +version = "0.1.0" +dependencies = [ + "anyhow", + "api_identity", + "backoff", + "chrono", + "dropshot", + "expectorate", + "futures", + "http", + "hyper", + "ipnetwork", + "macaddr", + "parse-display", + "progenitor", + "reqwest", + "ring", + "schemars", + "serde", + "serde_derive", + "serde_json", + "serde_urlencoded", + "serde_with", + "slog", + "smf", + "steno", + "structopt", + "thiserror", + "tokio", + "tokio-postgres", + "uuid", +] + +[[package]] +name = "omicron-nexus" +version = "0.1.0" +dependencies = [ + "anyhow", + "api_identity", + "async-bb8-diesel", + "async-trait", + "bb8", + "chrono", + "cookie", + "criterion", + "crucible-agent-client", + "db-macros", + "diesel", + "diesel-dtrace", + "dropshot", + "expectorate", + "futures", + "hex", + "http", + "hyper", + "ipnetwork", + "lazy_static", + "libc", + "macaddr", + "mime_guess", + "newtype_derive", + "nexus-test-utils", + "nexus-test-utils-macros", + "omicron-common", + "omicron-rpaths", + "omicron-sled-agent", + "omicron-test-utils", + "openapi-lint", + "openapiv3", + "oso", + "oximeter", + "oximeter-client", + "oximeter-db", + "oximeter-instruments", + "oximeter-producer", + "parse-display", + "pq-sys", + "rand", + "ref-cast", + "reqwest", + "schemars", + "serde", + "serde_json", + "serde_urlencoded", + "serde_with", + "sled-agent-client", + "slog", + "slog-dtrace", + "steno", + "structopt", + "subprocess", + "thiserror", + "tokio", + "tokio-postgres", + "toml", + "usdt 0.3.1", + "uuid", +] + +[[package]] +name = "omicron-package" +version = "0.1.0" +dependencies = [ + "anyhow", + "crossbeam", + "omicron-common", + "propolis-server", + "rayon", + "reqwest", + "serde", + "serde_derive", + "smf", + "structopt", + "tar", + "thiserror", + "tokio", + "toml", + "walkdir", +] + +[[package]] +name = "omicron-rpaths" +version = "0.1.0" + +[[package]] +name = "omicron-sled-agent" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "bincode", + "bytes", + "cfg-if", + "chrono", + "crucible-agent-client", + "dropshot", + "expectorate", + "futures", + "ipnetwork", + "mockall", + "nexus-client", + "omicron-common", + "omicron-test-utils", + "openapi-lint", + "openapiv3", + "p256", + "percent-encoding", + "progenitor", + "propolis-client", + "rand", + "reqwest", + "schemars", + "serde", + "serde_json", + "serial_test", + "slog", + "slog-async", + "slog-dtrace", + "slog-term", + "smf", + "socket2", + "spdm", + "structopt", + "subprocess", + "tar", + "tempfile", + "thiserror", + "tokio", + "tokio-util", + "toml", + "uuid", + "vsss-rs", + "zone", +] + +[[package]] +name = "omicron-test-utils" +version = "0.1.0" +dependencies = [ + "anyhow", + "dropshot", + "expectorate", + "futures", + "libc", + "omicron-common", + "postgres-protocol", + "signal-hook", + "signal-hook-tokio", + "slog", + "structopt", + "subprocess", + "tempfile", + "thiserror", + "tokio", + "tokio-postgres", +] + +[[package]] +name = "once_cell" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" + +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "openapi-lint" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/openapi-lint?branch=main#025fc867b7da62b33474d0a765823653a437f362" +dependencies = [ + "convert_case 0.5.0", + "indexmap", + "openapiv3", +] + +[[package]] +name = "openapiv3" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9de1b830d6f0f82e832f5a173d54f827f233e75b30f0f787c1289cca956879f8" +dependencies = [ + "indexmap", + "serde", + "serde_json", +] + +[[package]] +name = "openssl" +version = "0.10.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" + +[[package]] +name = "openssl-sys" +version = "0.9.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df13d165e607909b363a4757a6f133f8a818a74e9d3a98d09c6128e15fa4c73" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "oso" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3130188f8467dd6d904e692f3d31a3aa6e63902449a1b02d056e8e2633b863d1" +dependencies = [ + "impl-trait-for-tuples", + "lazy_static", + "maplit", + "oso-derive", + "polar-core", + "thiserror", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "oso-derive" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b03f10ad1ba555f82c0ccfe497010beccd873651604cb69999ba8c570d312fda" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "oximeter" +version = "0.1.0" +dependencies = [ + "bytes", + "chrono", + "num-traits", + "oximeter-macro-impl", + "schemars", + "serde", + "thiserror", + "trybuild", + "uuid", +] + +[[package]] +name = "oximeter-client" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "omicron-common", + "percent-encoding", + "progenitor", + "reqwest", + "serde", + "slog", + "uuid", +] + +[[package]] +name = "oximeter-collector" +version = "0.1.0" +dependencies = [ + "dropshot", + "expectorate", + "nexus-client", + "omicron-common", + "omicron-test-utils", + "openapi-lint", + "openapiv3", + "oximeter", + "oximeter-db", + "reqwest", + "serde", + "serde_json", + "slog", + "slog-dtrace", + "structopt", + "subprocess", + "thiserror", + "tokio", + "toml", + "uuid", +] + +[[package]] +name = "oximeter-db" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "dropshot", + "itertools", + "omicron-test-utils", + "oximeter", + "regex", + "reqwest", + "schemars", + "serde", + "serde_json", + "slog", + "slog-async", + "slog-dtrace", + "slog-term", + "structopt", + "thiserror", + "tokio", + "uuid", +] + +[[package]] +name = "oximeter-instruments" +version = "0.1.0" +dependencies = [ + "chrono", + "dropshot", + "futures", + "http", + "oximeter", + "uuid", +] + +[[package]] +name = "oximeter-macro-impl" +version = "0.1.0" +dependencies = [ + "bytes", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "oximeter-producer" +version = "0.1.0" +dependencies = [ + "chrono", + "dropshot", + "nexus-client", + "omicron-common", + "oximeter", + "reqwest", + "schemars", + "serde", + "slog", + "slog-dtrace", + "thiserror", + "tokio", + "uuid", +] + +[[package]] +name = "p256" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d053368e1bae4c8a672953397bd1bd7183dde1c72b0b7612a15719173148d186" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2 0.9.8", +] + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + +[[package]] +name = "parse-display" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "898bf4c2a569dedbfd4e6c3f0bbd0ae825e5b6b0b69bae3e3c1000158689334a" +dependencies = [ + "once_cell", + "parse-display-derive", + "regex", +] + +[[package]] +name = "parse-display-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1779d1e28ab04568223744c2af4aa4e642e67b92c76bdce0929a6d2c36267199" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "regex", + "regex-syntax", + "structmeta", + "syn", +] + +[[package]] +name = "paste" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0744126afe1a6dd7f394cb50a716dbe086cb06e255e53d8d0185d82828358fb5" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "833d1ae558dc601e9a60366421196a8d94bc0ac980476d0b67e1d0988d72b2d0" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54be6e404f5317079812fc8f9f5279de376d8856929e21c184ecf6bbd692a11d" +dependencies = [ + "maplit", + "pest", + "sha-1 0.8.2", +] + +[[package]] +name = "petgraph" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7" +dependencies = [ + "fixedbitset 0.2.0", + "indexmap", +] + +[[package]] +name = "petgraph" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a13a2fa9d0b63e5f22328828741e523766fff0ee9e779316902290dff3f824f" +dependencies = [ + "fixedbitset 0.4.0", + "indexmap", +] + +[[package]] +name = "phf" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9fc3db1018c4b59d7d582a739436478b6035138b6aecbce989fc91c3e98409f" +dependencies = [ + "phf_shared 0.10.0", +] + +[[package]] +name = "phf_shared" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "576bc800220cc65dac09e99e97b08b358cfab6e17078de8dc5fee223bd2d0c08" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkcs8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee3ef9b64d26bad0536099c816c6734379e45bbd5f14798def6809e5cc350447" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1a3ea4f0dd7f1f3e512cf97bf100819aa547f36a6eccac8dbaae839eb92363e" + +[[package]] +name = "plotters" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a3fd9ec30b9749ce28cd91f255d569591cdf937fe280c312143e3c4bad6f2a" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d88417318da0eaf0fdcdb51a0ee6c3bed624333bff8f946733049380be67ac1c" + +[[package]] +name = "plotters-svg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521fa9638fa597e1dc53e9412a4f9cefb01187ee1f7413076f9e6749e2885ba9" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "polar-core" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b469dacc2c69faed6b71972dac8fb9fbddf18ae22ecce48f97a4643e58860fa3" +dependencies = [ + "indoc", + "js-sys", + "lalrpop", + "lalrpop-util", + "serde", + "serde_derive", + "wasm-bindgen", +] + +[[package]] +name = "polyval" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8419d2b623c7c0896ff2d5d96e2cb4ede590fed28fcc34934f4c33c036e620a1" +dependencies = [ + "cfg-if", + "cpufeatures", + "opaque-debug 0.3.0", + "universal-hash", +] + +[[package]] +name = "postgres-protocol" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79ec03bce71f18b4a27c4c64c6ba2ddf74686d69b91d8714fb32ead3adaed713" +dependencies = [ + "base64", + "byteorder", + "bytes", + "fallible-iterator", + "hmac 0.12.0", + "md-5", + "memchr", + "rand", + "sha2 0.10.0", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04619f94ba0cc80999f4fc7073607cb825bc739a883cb6d20900fc5e009d6b0d" +dependencies = [ + "bytes", + "chrono", + "fallible-iterator", + "postgres-protocol", + "serde", + "serde_json", + "uuid", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed0cfbc8191465bed66e1718596ee0b0b35d5ee1f41c5df2189d0fe8bde535ba" + +[[package]] +name = "pq-sys" +version = "0.4.6" +source = "git+https://github.com/oxidecomputer/pq-sys?branch=oxide/omicron#b1194c190f4d4a103c2280908cd1e97628c5c1cb" +dependencies = [ + "vcpkg", +] + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "predicates" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95e5a7689e456ab905c22c2b48225bb921aba7c8dfa58440d68ba13f6222a715" +dependencies = [ + "difflib", + "float-cmp", + "itertools", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57e35a3326b75e49aa85f5dc6ec15b41108cf5aee58eabb1f274dd18b73c2451" + +[[package]] +name = "predicates-tree" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338c7be2905b732ae3984a2f40032b5e94fd8f52505b186c7d4d68d193445df7" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "proc-macro-crate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebace6889caf889b4d3f76becee12e90353f2b8c7d875534a71e5742f8f6f83" +dependencies = [ + "thiserror", + "toml", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "progenitor" +version = "0.0.0" +source = "git+https://github.com/oxidecomputer/progenitor#f1f9e2e93850713908f4e6494808a07f3b253108" +dependencies = [ + "anyhow", + "getopts", + "openapiv3", + "progenitor-impl", + "progenitor-macro", + "serde", + "serde_json", +] + +[[package]] +name = "progenitor-impl" +version = "0.0.0" +source = "git+https://github.com/oxidecomputer/progenitor#f1f9e2e93850713908f4e6494808a07f3b253108" +dependencies = [ + "anyhow", + "convert_case 0.4.0", + "getopts", + "indexmap", + "openapiv3", + "proc-macro2", + "quote", + "regex", + "rustfmt-wrapper", + "schemars", + "serde", + "serde_json", + "syn", + "thiserror", + "typify", + "unicode-xid", +] + +[[package]] +name = "progenitor-macro" +version = "0.0.0" +source = "git+https://github.com/oxidecomputer/progenitor#f1f9e2e93850713908f4e6494808a07f3b253108" +dependencies = [ + "openapiv3", + "proc-macro2", + "progenitor-impl", + "quote", + "serde", + "serde_json", + "serde_tokenstream", + "syn", +] + +[[package]] +name = "propolis" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/propolis?rev=00ec8cf18f6a2311b0907f0b16b0ff8a327944d1#00ec8cf18f6a2311b0907f0b16b0ff8a327944d1" +dependencies = [ + "anyhow", + "bhyve_api", + "bitflags", + "bitstruct", + "byteorder", + "crucible", + "dladm", + "erased-serde", + "futures", + "lazy_static", + "libc", + "num_enum", + "serde", + "serde_arrays", + "slog", + "thiserror", + "tokio", + "usdt 0.2.1", + "viona_api", +] + +[[package]] +name = "propolis-client" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/propolis?rev=00ec8cf18f6a2311b0907f0b16b0ff8a327944d1#00ec8cf18f6a2311b0907f0b16b0ff8a327944d1" +dependencies = [ + "reqwest", + "ring", + "schemars", + "serde", + "serde_json", + "slog", + "structopt", + "thiserror", + "uuid", +] + +[[package]] +name = "propolis-server" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/propolis?rev=00ec8cf18f6a2311b0907f0b16b0ff8a327944d1#00ec8cf18f6a2311b0907f0b16b0ff8a327944d1" +dependencies = [ + "anyhow", + "dropshot", + "futures", + "hyper", + "propolis", + "propolis-client", + "serde", + "serde_derive", + "slog", + "structopt", + "thiserror", + "tokio", + "tokio-tungstenite", + "toml", + "uuid", +] + +[[package]] +name = "quote" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47aa80447ce4daf1717500037052af176af5d38cc3e571d9ec1c7353fc10c87d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r2d2" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" +dependencies = [ + "log", + "parking_lot", + "scheduled-thread-pool", +] + +[[package]] +name = "radium" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643f8f41a8ebc4c5dc4515c82bb8abd397b527fc20fd681b7c011c2aee5d44fb" + +[[package]] +name = "rand" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rayon" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + +[[package]] +name = "redox_syscall" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +dependencies = [ + "getrandom", + "redox_syscall", +] + +[[package]] +name = "ref-cast" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300f2a835d808734ee295d45007adacb9ebb29dd3ae2424acfa17930cae541da" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c38e3aecd2b21cb3959637b883bb3714bc7e43f0268b9a29d3743ee3e55cdd2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "reqwest" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f242f1488a539a79bac6dbe7c8609ae43b7914b7736210f239a37cccb32525" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-rustls", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tokio-rustls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", + "winreg", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "ringbuffer" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72dfeba9c94fdc308dcfe165efa6855c698f317484b736605ff859c5fa81d992" +dependencies = [ + "array-init", +] + +[[package]] +name = "rustc_version" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f5376ea5e30ce23c03eb77cbe4962b988deead10910c372b226388b594c084" +dependencies = [ + "semver 0.1.20", +] + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver 1.0.4", +] + +[[package]] +name = "rustfmt-wrapper" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7733577fb5b13c8b256232e7ca84aa424f915efae6ec980082d60a03f99da3f8" +dependencies = [ + "tempfile", + "thiserror", + "toolchain_find", +] + +[[package]] +name = "rustls" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d37e5e2290f3e040b594b1a9e04377c2c671f1a1cfd9bfdef82106ac1c113f84" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-pemfile" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eebeaeb360c87bfb72e84abdb3447159c0eaececf1bef2aecd65a8be949d1c9" +dependencies = [ + "base64", +] + +[[package]] +name = "rustversion" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" + +[[package]] +name = "ryu" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c9613b5a66ab9ba26415184cfc41156594925a9cf3a2057e57f31ff145f6568" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + +[[package]] +name = "scheduled-thread-pool" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" +dependencies = [ + "parking_lot", +] + +[[package]] +name = "schemars" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6b5a3c80cea1ab61f4260238409510e814e38b4b563c06044edf91e7dc070e3" +dependencies = [ + "bytes", + "chrono", + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", + "uuid", +] + +[[package]] +name = "schemars_derive" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41ae4dce13e8614c46ac3c38ef1c0d668b101df6ac39817aebdaa26642ddae9b" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "security-framework" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525bc1abfda2e1998d152c45cf13e696f76d0a4972310b22fac1658b05df7c87" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9dd14d83160b528b7bfd66439110573efcfbe281b17fc2ca9f39f550d619c7e" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4f410fedcf71af0345d7607d246e7ad15faaadd49d240ee3b24e5dc21a820ac" + +[[package]] +name = "semver" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012" + +[[package]] +name = "semver-parser" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" +dependencies = [ + "pest", +] + +[[package]] +name = "serde" +version = "1.0.133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97565067517b60e2d1ea8b268e59ce036de907ac523ad83a0475da04e818989a" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde-big-array" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18b20e7752957bbe9661cff4e0bb04d183d0948cdab2ea58cdb9df36a61dfe62" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "serde_arrays" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38636132857f68ec3d5f3eb121166d2af33cb55174c4d5ff645db6165cbef0fd" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_cbor" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" +dependencies = [ + "half", + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed201699328568d8d08208fdd080e3ff594e6c422e438b6705905da01005d537" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_derive_internals" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dbab34ca63057a1f15280bdf3c39f2b1eb1b54c17e98360e511637aef7418c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c059c05b48c5c0067d4b4b2b4f0732dd65feb52daf7e0ea09cd87e7dadc1af79" +dependencies = [ + "itoa 1.0.1", + "ryu", + "serde", +] + +[[package]] +name = "serde_tokenstream" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6deb15c3a535e81438110111d90168d91721652f502abb147f31cde129f683d" +dependencies = [ + "proc-macro2", + "serde", + "syn", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa 1.0.1", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad6056b4cb69b6e43e3a0f055def223380baecc99da683884f205bf347f7c4b3" +dependencies = [ + "rustversion", + "serde", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12e47be9471c72889ebafb5e14d5ff930d89ae7a67bbdb5f8abb564f845a927e" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serial_test" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0bccbcf40c8938196944a3da0e133e031a33f4d6b72db3bda3cc556e361905d" +dependencies = [ + "lazy_static", + "parking_lot", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2acd6defeddb41eb60bb468f8825d0cfd0c2a76bc03bfd235b6a1dc4f6a1ad5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha-1" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +dependencies = [ + "block-buffer 0.7.3", + "digest 0.8.1", + "fake-simd", + "opaque-debug 0.2.3", +] + +[[package]] +name = "sha-1" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if", + "cpufeatures", + "digest 0.9.0", + "opaque-debug 0.3.0", +] + +[[package]] +name = "sha2" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if", + "cpufeatures", + "digest 0.9.0", + "opaque-debug 0.3.0", +] + +[[package]] +name = "sha2" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d964dd36bb15bcf2f2b35694c072feab74969a54f2bbeec7a2d725d2bdcb6" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.1", +] + +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "647c97df271007dcea485bb74ffdb57f2e683f1306c854f468a0c244badabf2d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +dependencies = [ + "libc", +] + +[[package]] +name = "signal-hook-tokio" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213241f76fb1e37e27de3b6aa1b068a2c333233b59cca6634f634b80a27ecf1e" +dependencies = [ + "futures-core", + "libc", + "signal-hook", + "tokio", +] + +[[package]] +name = "signature" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2807892cfa58e081aa1f1111391c7a0649d4fa127a4ffbe34bcbfb35a1171a4" +dependencies = [ + "digest 0.9.0", + "rand_core", +] + +[[package]] +name = "siphasher" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "533494a8f9b724d33625ab53c6c4800f7cc445895924a8ef649222dcb76e938b" + +[[package]] +name = "slab" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" + +[[package]] +name = "sled-agent-client" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "omicron-common", + "percent-encoding", + "progenitor", + "reqwest", + "serde", + "slog", + "uuid", +] + +[[package]] +name = "slog" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" + +[[package]] +name = "slog-async" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "766c59b252e62a34651412870ff55d8c4e6d04df19b43eecb2703e417b097ffe" +dependencies = [ + "crossbeam-channel", + "slog", + "take_mut", + "thread_local", +] + +[[package]] +name = "slog-bunyan" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924f5e8b5a1069e484f6ad80024322990e21ee8399f12ba289e48cd30bc0dda0" +dependencies = [ + "chrono", + "hostname", + "slog", + "slog-json", +] + +[[package]] +name = "slog-dtrace" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4ec6883908a7f628be97d107387c45de771d8a2fa8af147337ec38549f93cdd" +dependencies = [ + "chrono", + "serde", + "serde_json", + "slog", + "usdt 0.2.1", +] + +[[package]] +name = "slog-json" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e9b96fb6b5e80e371423b4aca6656eb537661ce8f82c2697e619f8ca85d043" +dependencies = [ + "chrono", + "serde", + "serde_json", + "slog", +] + +[[package]] +name = "slog-term" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95c1e7e5aab61ced6006149ea772770b84a0d16ce0f7885def313e4829946d76" +dependencies = [ + "atty", + "chrono", + "slog", + "term", + "thread_local", +] + +[[package]] +name = "smallvec" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309" + +[[package]] +name = "smf" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f19d427ae89311c2770c49fdcfa14627577c499311fe8f4cc8fcfde2dd3c4e2e" +dependencies = [ + "thiserror", +] + +[[package]] +name = "socket2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "spdm" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/spdm?rev=9742f6e#9742f6eae7b86cc8bc8bc2fb0feeb44f770a1fb6" +dependencies = [ + "bitflags", + "rand", + "ring", + "webpki", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "spki" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c01a0c15da1b0b0e1494112e7af814a678fec9bd157881b49beac661e9b6f32" +dependencies = [ + "der", +] + +[[package]] +name = "steno" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/steno?branch=main#c6ea85cfd268668a5974741b308d89acfae76063" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "futures", + "newtype_derive", + "petgraph 0.6.0", + "schemars", + "serde", + "serde_json", + "slog", + "thiserror", + "tokio", + "uuid", +] + +[[package]] +name = "string_cache" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923f0f39b6267d37d23ce71ae7235602134b250ace715dd2c90421998ddac0c6" +dependencies = [ + "lazy_static", + "new_debug_unreachable", + "parking_lot", + "phf_shared 0.8.0", + "precomputed-hash", +] + +[[package]] +name = "stringprep" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "structmeta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59915b528a896f2e3bfa1a6ace65f7bb0ff9f9863de6213b0c01cb6fd3c3ac71" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn", +] + +[[package]] +name = "structmeta-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73800bcca56045d5ab138a48cd28a96093335335deaa916f22b5749c4150c79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "structopt" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" +dependencies = [ + "clap", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "subprocess" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "055cf3ebc2981ad8f0a5a17ef6652f652d87831f79fddcba2ac57bcb9a0aa407" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + +[[package]] +name = "syn" +version = "1.0.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a684ac3dcd8913827e18cd09a68384ee66c1de24157e3c556c9ab16d85695fb7" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tar" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "termtree" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a4ec180a2de59b57434704ccfad967f789b12737738798fa08798cd5824c16" + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread-id" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fdfe0627923f7411a43ec9ec9c39c3a9b4151be313e0922042581fb6c9b717f" +dependencies = [ + "libc", + "redox_syscall", + "winapi", +] + +[[package]] +name = "thread_local" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" +dependencies = [ + "once_cell", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi", + "winapi", +] + +[[package]] +name = "time" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41effe7cfa8af36f439fac33861b66b049edc6f9a32331e2312660529c1c24ad" +dependencies = [ + "itoa 0.4.8", + "libc", + "time-macros", +] + +[[package]] +name = "time-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25eb0ca3468fc0acc11828786797f6ef9aa1555e4a211a60d64cc8e4d1be47d6" + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tinyvec" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbbf1c778ec206785635ce8ad57fe52b3009ae9e0c9f574a728f3049d3e55838" +dependencies = [ + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "once_cell", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-macros" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-postgres" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b6c8b33df661b548dcd8f9bf87debb8c56c05657ed291122e1188698c2ece95" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "socket2", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-rustls" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a27d5f2b839802bd8267fa19b0530f5a08b9c08cd417976be2a65d130fe1c11b" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e96bb520beab540ab664bd5a9cfeaa1fcd846fa68c830b42e2c8963071251d2" +dependencies = [ + "futures-util", + "log", + "pin-project", + "tokio", + "tungstenite", +] + +[[package]] +name = "tokio-util" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde", +] + +[[package]] +name = "toolchain_find" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e85654a10e7a07a47c6f19d93818f3f343e22927f2fa280c84f7c8042743413" +dependencies = [ + "home", + "lazy_static", + "regex", + "semver 0.11.0", + "walkdir", +] + +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + +[[package]] +name = "tracing" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f480b8f81512e825f337ad51e94c1eb5d3bbdf2b363dcd01e2b19a9ffe3f8e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f4ed65637b8390770814083d20756f87bfa2c21bf2f110babdc5438351746e4" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245da694cc7fc4729f3f418b304cb57789f1bed2a78c575407ab8a23f53cb4d3" +dependencies = [ + "sharded-slab", + "thread_local", + "tracing-core", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "trybuild" +version = "1.0.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d664de8ea7e531ad4c0f5a834f20b8cb2b8e6dfe88d05796ee7887518ed67b9" +dependencies = [ + "glob", + "lazy_static", + "serde", + "serde_json", + "termcolor", + "toml", +] + +[[package]] +name = "tungstenite" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fe8dada8c1a3aeca77d6b51a4f1314e0f4b8e438b7b1b71e3ddaca8080e4093" +dependencies = [ + "base64", + "byteorder", + "bytes", + "http", + "httparse", + "input_buffer", + "log", + "rand", + "sha-1 0.9.8", + "thiserror", + "url", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" + +[[package]] +name = "typify" +version = "0.0.6-dev" +source = "git+https://github.com/oxidecomputer/typify#9afa917671b29fc231bc9ce304e041bdd685af09" +dependencies = [ + "typify-impl", + "typify-macro", +] + +[[package]] +name = "typify-impl" +version = "0.0.6-dev" +source = "git+https://github.com/oxidecomputer/typify#9afa917671b29fc231bc9ce304e041bdd685af09" +dependencies = [ + "convert_case 0.4.0", + "log", + "proc-macro2", + "quote", + "rustfmt-wrapper", + "schemars", + "serde_json", + "syn", + "thiserror", + "unicode-xid", +] + +[[package]] +name = "typify-macro" +version = "0.0.6-dev" +source = "git+https://github.com/oxidecomputer/typify#9afa917671b29fc231bc9ce304e041bdd685af09" +dependencies = [ + "proc-macro2", + "quote", + "schemars", + "serde", + "serde_json", + "serde_tokenstream", + "syn", + "typify-impl", +] + +[[package]] +name = "ucd-trie" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" + +[[package]] +name = "unicode-normalization" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" + +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "unindent" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7" + +[[package]] +name = "universal-hash" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f214e8f697e925001e66ec2c6e37a4ef93f0f78c2eed7814394e10c62025b05" +dependencies = [ + "generic-array 0.14.4", + "subtle", +] + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + +[[package]] +name = "usdt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ede7ff58821fee096f60fd731d56f61b56883a4808a290cac15df29baf1bc50" +dependencies = [ + "dof", + "dtrace-parser", + "serde", + "usdt-attr-macro 0.2.1", + "usdt-impl 0.2.1", + "usdt-macro 0.2.1", +] + +[[package]] +name = "usdt" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3877c300bc107d8d05455e8e2bec271ca9e1d24fbc17e3ef27d18b9b53c684e2" +dependencies = [ + "dof", + "dtrace-parser", + "serde", + "usdt-attr-macro 0.3.1", + "usdt-impl 0.3.1", + "usdt-macro 0.3.1", +] + +[[package]] +name = "usdt-attr-macro" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8233e148b2767bf47aa1620bfe72c982da8abe2da05090c3b716f2fd1865e94d" +dependencies = [ + "dtrace-parser", + "proc-macro2", + "quote", + "serde_tokenstream", + "syn", + "usdt-impl 0.2.1", +] + +[[package]] +name = "usdt-attr-macro" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dedf62a6eab9c6da23fa64353382ef2b73dce8891eadda8755bfed179cb2103d" +dependencies = [ + "dtrace-parser", + "proc-macro2", + "quote", + "serde_tokenstream", + "syn", + "usdt-impl 0.3.1", +] + +[[package]] +name = "usdt-impl" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6e158f113f80db319e54f62c59ea4329a31190a43ed7db07578982021209014" +dependencies = [ + "byteorder", + "dof", + "dtrace-parser", + "libc", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn", + "thiserror", + "thread-id", +] + +[[package]] +name = "usdt-impl" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dda9311a9f1452ebc752d264e9d1e3c2dac9807219e19471bcef3bd1b7f90cb" +dependencies = [ + "byteorder", + "dof", + "dtrace-parser", + "libc", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn", + "thiserror", + "thread-id", +] + +[[package]] +name = "usdt-macro" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af8dae3bb94dfc25189d8566b8da97ad05003fe24346b118c697c6671b4b7d10" +dependencies = [ + "dtrace-parser", + "proc-macro2", + "quote", + "serde_tokenstream", + "syn", + "usdt-impl 0.2.1", +] + +[[package]] +name = "usdt-macro" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64160eb8f7ee61fcaef542b2f75fa807e8c3ed4b4f075e6ca0b17ea6fcea6f8" +dependencies = [ + "dtrace-parser", + "proc-macro2", + "quote", + "serde_tokenstream", + "syn", + "usdt-impl 0.3.1", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom", + "serde", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" + +[[package]] +name = "viona_api" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/propolis?rev=00ec8cf18f6a2311b0907f0b16b0ff8a327944d1#00ec8cf18f6a2311b0907f0b16b0ff8a327944d1" + +[[package]] +name = "vsss-rs" +version = "2.0.0-pre0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99cd0cd00b7a1e29f0a87aaeeb264cf87925829e5293c7f09a6d5addc4f52780" +dependencies = [ + "curve25519-dalek", + "elliptic-curve", + "ff", + "group", + "k256", + "rand_chacha", + "rand_core", + "serde", + "serde-big-array", + "serde_cbor", + "sha2 0.9.8", + "subtle", + "zeroize", +] + +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasm-bindgen" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "632f73e236b219150ea279196e54e610f5dbafa5d61786303d4da54f84e47fce" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a317bf8f9fba2476b4b2c85ef4c4af8ff39c3c7f0cdfeed4f82c34a880aa837b" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e8d7523cb1f2a4c96c1317ca690031b714a51cc14e05f712446691f413f5d39" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0237232789cf037d5480773fe568aac745bfe2afbc11a863e97901780a6b47cc" + +[[package]] +name = "web-sys" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38eb105f1c59d9eaa6b5cdc92b859d85b926e82cb2e0945cd0c9259faa6fe9fb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c475786c6f47219345717a043a37ec04cb4bc185e28853adcc4fa0a947eba630" +dependencies = [ + "webpki", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi", +] + +[[package]] +name = "wyz" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "129e027ad65ce1453680623c3fb5163cbf7107bfe1aa32257e7d0e63f9ced188" +dependencies = [ + "tap", +] + +[[package]] +name = "xattr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" +dependencies = [ + "libc", +] + +[[package]] +name = "xts-mode" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a099a2f21d48275314733f85bc43b6c6213b66394233aaea573fc7a520dcd9" +dependencies = [ + "byteorder", + "cipher", +] + +[[package]] +name = "zerocopy" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6580539ad917b7c026220c4b3f2c08d52ce54d6ce0dc491e66002e35388fab46" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" +dependencies = [ + "proc-macro2", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65f1a51723ec88c66d5d1fe80c841f17f63587d6691901d66be9bec6c3b51f73" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zone" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3596bbc963cd9dbaa69b02e349af4d061c56c41d211ba64150a2cedb2f722707" +dependencies = [ + "itertools", + "thiserror", + "zone_cfg_derive", +] + +[[package]] +name = "zone_cfg_derive" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ac2a898023d86613a7efa7a4195e4b75240009e559acb17ddd7fa876d19527" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index aaa4eed6633..5853d11a0b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,10 +57,8 @@ panic = "abort" # # Local client generation during development. # -[patch."https://github.com/oxidecomputer/progenitor"] -progenitor = { path = "../progenitor/progenitor" } -[patch."https://github.com/oxidecomputer/crucible"] -crucible-agent-client = { path = "../crucible/agent-client" } +#[patch."https://github.com/oxidecomputer/progenitor"] +#progenitor = { path = "../progenitor/progenitor" } #[patch."https://github.com/oxidecomputer/typify"] #typify = { path = "../typify/typify" } diff --git a/nexus-client/Cargo.toml b/nexus-client/Cargo.toml index 2ceb8bbb54f..0ef31114868 100644 --- a/nexus-client/Cargo.toml +++ b/nexus-client/Cargo.toml @@ -9,7 +9,6 @@ anyhow = "1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor" } reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] } percent-encoding = "2.1.0" -schemars = { version = "0.8" } serde_json = "1.0" [dependencies.chrono] diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index cd3225640e9..d0b72ea8799 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -13,7 +13,7 @@ async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", async-trait = "0.1.51" bb8 = "0.7.1" cookie = "0.16" -crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "de022b8a" } +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "078d364e14d57d5faa3a44001c65709935419779" } # Tracking pending 2.0 version. diesel = { git = "https://github.com/diesel-rs/diesel", rev = "ce77c382", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] } futures = "0.3.18" diff --git a/oximeter-client/Cargo.toml b/oximeter-client/Cargo.toml index 2ee69f74aeb..d2165b95f83 100644 --- a/oximeter-client/Cargo.toml +++ b/oximeter-client/Cargo.toml @@ -8,7 +8,6 @@ license = "MPL-2.0" anyhow = "1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor" } reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] } -schemars = { version = "0.8" } percent-encoding = "2.1.0" [dependencies.chrono] diff --git a/sled-agent-client/Cargo.toml b/sled-agent-client/Cargo.toml index ebf1f35dfd1..2f233c4ebdc 100644 --- a/sled-agent-client/Cargo.toml +++ b/sled-agent-client/Cargo.toml @@ -9,7 +9,6 @@ anyhow = "1.0" async-trait = "0.1" progenitor = { git = "https://github.com/oxidecomputer/progenitor" } reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] } -schemars = { version = "0.8" } percent-encoding = "2.1.0" [dependencies.chrono] diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index ffab14ff4c9..8875c22802f 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -12,7 +12,7 @@ bytes = "1.1" cfg-if = "1.0" chrono = { version = "0.4", features = [ "serde" ] } # Only used by the simulated sled agent. -crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "de022b8a" } +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "078d364e14d57d5faa3a44001c65709935419779" } dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] } futures = "0.3.18" ipnetwork = "0.18" From c24a46ea8f3a88f0aac74c4c978337d8f7032147 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 21 Jan 2022 14:51:50 -0500 Subject: [PATCH 50/50] Clarifying comment --- sled-agent/src/sim/storage.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index 55b2cb80f45..4e178d410ac 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -3,6 +3,10 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. //! Simulated sled agent storage implementation +//! +//! Note, this refers to the "storage which exists on the Sled", rather +//! than the representation of "virtual disks" which would be presented +//! through Nexus' external API. use crucible_agent_client::types::{CreateRegion, Region, RegionId, State}; use futures::lock::Mutex;