From c3bd755ec87ff2436923719475bb2140caf810c5 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 7 Jan 2022 10:01:21 -0500 Subject: [PATCH 1/6] Make use of the #derive(JsonSchema) addition to progenitor --- Cargo.lock | 15 ++-- Cargo.toml | 6 +- nexus-client/Cargo.toml | 1 + nexus/tests/integration_tests/disks.rs | 3 +- oximeter-client/Cargo.toml | 1 + oximeter/producer/src/lib.rs | 3 +- sled-agent-client/Cargo.toml | 1 + sled-agent/src/sim/disk.rs | 3 +- .../src/sim/http_entrypoints_storage.rs | 70 ++----------------- sled-agent/src/sim/instance.rs | 3 +- sled-agent/src/sim/mod.rs | 1 - sled-agent/src/sim/storage.rs | 5 +- 12 files changed, 29 insertions(+), 83 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73919fe6f1b..687a2ceb8d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -530,12 +530,12 @@ dependencies = [ [[package]] name = "crucible-agent-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=de022b8a#de022b8ae24fb10cec6b024437aabc3acf249e43" dependencies = [ "anyhow", "percent-encoding", "progenitor", "reqwest", + "schemars", "serde", "serde_json", ] @@ -1765,6 +1765,7 @@ dependencies = [ "percent-encoding", "progenitor", "reqwest", + "schemars", "serde", "serde_json", "slog", @@ -2197,6 +2198,7 @@ dependencies = [ "percent-encoding", "progenitor", "reqwest", + "schemars", "serde", "slog", "uuid", @@ -2692,7 +2694,6 @@ dependencies = [ [[package]] name = "progenitor" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/progenitor#66b41ba301793b8d720770b2210bee8884446d3f" dependencies = [ "anyhow", "getopts", @@ -2706,7 +2707,6 @@ dependencies = [ [[package]] name = "progenitor-impl" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/progenitor#66b41ba301793b8d720770b2210bee8884446d3f" dependencies = [ "anyhow", "convert_case", @@ -2727,7 +2727,6 @@ dependencies = [ [[package]] name = "progenitor-macro" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/progenitor#66b41ba301793b8d720770b2210bee8884446d3f" dependencies = [ "openapiv3", "proc-macro2", @@ -3474,6 +3473,7 @@ dependencies = [ "percent-encoding", "progenitor", "reqwest", + "schemars", "serde", "slog", "uuid", @@ -4135,7 +4135,7 @@ checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" [[package]] name = "typify" version = "0.0.6-dev" -source = "git+https://github.com/oxidecomputer/typify#80b510b02b1db22de463efcf6e7762243bcea67a" +source = "git+https://github.com/oxidecomputer/typify?branch=json-schema#0be9b35a5007e3f797e43ce41eba03dcee63e503" dependencies = [ "typify-impl", "typify-macro", @@ -4144,9 +4144,10 @@ dependencies = [ [[package]] name = "typify-impl" version = "0.0.6-dev" -source = "git+https://github.com/oxidecomputer/typify#80b510b02b1db22de463efcf6e7762243bcea67a" +source = "git+https://github.com/oxidecomputer/typify?branch=json-schema#0be9b35a5007e3f797e43ce41eba03dcee63e503" dependencies = [ "convert_case", + "log", "proc-macro2", "quote", "rustfmt-wrapper", @@ -4159,7 +4160,7 @@ dependencies = [ [[package]] name = "typify-macro" version = "0.0.6-dev" -source = "git+https://github.com/oxidecomputer/typify#80b510b02b1db22de463efcf6e7762243bcea67a" +source = "git+https://github.com/oxidecomputer/typify?branch=json-schema#0be9b35a5007e3f797e43ce41eba03dcee63e503" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 5853d11a0b3..aaa4eed6633 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,8 +57,10 @@ panic = "abort" # # Local client generation during development. # -#[patch."https://github.com/oxidecomputer/progenitor"] -#progenitor = { path = "../progenitor/progenitor" } +[patch."https://github.com/oxidecomputer/progenitor"] +progenitor = { path = "../progenitor/progenitor" } +[patch."https://github.com/oxidecomputer/crucible"] +crucible-agent-client = { path = "../crucible/agent-client" } #[patch."https://github.com/oxidecomputer/typify"] #typify = { path = "../typify/typify" } diff --git a/nexus-client/Cargo.toml b/nexus-client/Cargo.toml index 0ef31114868..2ceb8bbb54f 100644 --- a/nexus-client/Cargo.toml +++ b/nexus-client/Cargo.toml @@ -9,6 +9,7 @@ anyhow = "1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor" } reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] } percent-encoding = "2.1.0" +schemars = { version = "0.8" } serde_json = "1.0" [dependencies.chrono] diff --git a/nexus/tests/integration_tests/disks.rs b/nexus/tests/integration_tests/disks.rs index d47dee2fb55..e2602359989 100644 --- a/nexus/tests/integration_tests/disks.rs +++ b/nexus/tests/integration_tests/disks.rs @@ -4,6 +4,7 @@ //! Tests basic disk support in the API +use crucible_agent_client::types::State as RegionState; use http::method::Method; use http::StatusCode; use omicron_common::api::external::ByteCount; @@ -14,7 +15,7 @@ use omicron_common::api::external::Instance; use omicron_common::api::external::InstanceCpuCount; use omicron_nexus::TestInterfaces as _; use omicron_nexus::{external_api::params, Nexus}; -use omicron_sled_agent::sim::{RegionState, SledAgent}; +use omicron_sled_agent::sim::SledAgent; use sled_agent_client::TestInterfaces as _; use std::sync::Arc; use uuid::Uuid; diff --git a/oximeter-client/Cargo.toml b/oximeter-client/Cargo.toml index d2165b95f83..2ee69f74aeb 100644 --- a/oximeter-client/Cargo.toml +++ b/oximeter-client/Cargo.toml @@ -8,6 +8,7 @@ license = "MPL-2.0" anyhow = "1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor" } reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] } +schemars = { version = "0.8" } percent-encoding = "2.1.0" [dependencies.chrono] diff --git a/oximeter/producer/src/lib.rs b/oximeter/producer/src/lib.rs index 29c8b3818cc..e10e9e944dd 100644 --- a/oximeter/producer/src/lib.rs +++ b/oximeter/producer/src/lib.rs @@ -180,7 +180,8 @@ pub async fn register( client .cpapi_producers_post(&server_info.into()) .await - .map_err(|msg| Error::RegistrationError(msg.to_string())) + .map_err(|msg| Error::RegistrationError(msg.to_string()))?; + Ok(()) } /// Handle a request to pull available metric data from a [`ProducerRegistry`]. diff --git a/sled-agent-client/Cargo.toml b/sled-agent-client/Cargo.toml index 2f233c4ebdc..ebf1f35dfd1 100644 --- a/sled-agent-client/Cargo.toml +++ b/sled-agent-client/Cargo.toml @@ -9,6 +9,7 @@ anyhow = "1.0" async-trait = "0.1" progenitor = { git = "https://github.com/oxidecomputer/progenitor" } reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] } +schemars = { version = "0.8" } percent-encoding = "2.1.0" [dependencies.chrono] diff --git a/sled-agent/src/sim/disk.rs b/sled-agent/src/sim/disk.rs index 95ef478dcc5..8bbbbaa2025 100644 --- a/sled-agent/src/sim/disk.rs +++ b/sled-agent/src/sim/disk.rs @@ -95,6 +95,7 @@ impl Simulatable for SimDisk { &nexus_client::types::DiskRuntimeState::from(current), ) .await - .map_err(Error::from) + .map_err(Error::from)?; + Ok(()) } } diff --git a/sled-agent/src/sim/http_entrypoints_storage.rs b/sled-agent/src/sim/http_entrypoints_storage.rs index a8f79f0d213..6ef08a9a373 100644 --- a/sled-agent/src/sim/http_entrypoints_storage.rs +++ b/sled-agent/src/sim/http_entrypoints_storage.rs @@ -4,12 +4,13 @@ //! HTTP entrypoint functions for simulating the storage agent API. +use crucible_agent_client::types::{CreateRegion, Region, RegionId}; use dropshot::{ endpoint, ApiDescription, HttpError, HttpResponseDeleted, HttpResponseOk, Path as TypedPath, RequestContext, TypedBody, }; use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use std::sync::Arc; use super::storage::CrucibleData; @@ -35,71 +36,8 @@ pub fn api() -> CrucibleAgentApiDescription { api } -// XXX XXX XXX THIS SUCKS XXX XXX XXX -// -// I need to re-define all structs used in the crucible agent -// API to ensure they have the traits I need. The ones re-exported -// through the client bindings, i.e., crucible_agent_client::types, -// don't implement the "JsonSchema" trait, and cannot be used as -// parameters to these Dropshot endpoints. -// -// I'd like them to! If we could ensure the generated client -// also implemented e.g. JsonSchema, this might work? - -// To quickly test the type compatibility, uncomment the lines below, -// and remove the hand-rolled implementations. - -// pub type RegionId = crucible_agent_client::types::RegionId; -// pub type State = crucible_agent_client::types::State; -// pub type CreateRegion = crucible_agent_client::types::CreateRegion; -// pub type Region = crucible_agent_client::types::Region; -// pub type RegionPath = crucible_agent_client::types::RegionPath; - -#[derive( - Serialize, - Deserialize, - JsonSchema, - Debug, - PartialEq, - Eq, - Clone, - PartialOrd, - Ord, -)] -pub struct RegionId(pub String); - -#[derive(Serialize, Deserialize, JsonSchema, Debug, PartialEq, Clone)] -#[serde(rename_all = "lowercase")] -pub enum State { - Requested, - Created, - Tombstoned, - Destroyed, - Failed, -} - -#[derive(Serialize, Deserialize, JsonSchema, Debug, PartialEq, Clone)] -pub struct CreateRegion { - pub id: RegionId, - pub volume_id: String, - - pub block_size: u64, - pub extent_size: u64, - pub extent_count: u64, -} - -#[derive(Serialize, Deserialize, JsonSchema, Debug, PartialEq, Clone)] -pub struct Region { - pub id: RegionId, - pub volume_id: String, - - pub block_size: u64, - pub extent_size: u64, - pub extent_count: u64, - - pub port_number: u16, - pub state: State, -} +// TODO: We'd like to de-duplicate as much as possible with the +// real crucible agent here, to avoid skew. #[derive(Deserialize, JsonSchema)] struct RegionPath { diff --git a/sled-agent/src/sim/instance.rs b/sled-agent/src/sim/instance.rs index b7414d4c877..684606719d6 100644 --- a/sled-agent/src/sim/instance.rs +++ b/sled-agent/src/sim/instance.rs @@ -102,6 +102,7 @@ impl Simulatable for SimInstance { &nexus_client::types::InstanceRuntimeState::from(current), ) .await - .map_err(Error::from) + .map_err(Error::from)?; + Ok(()) } } diff --git a/sled-agent/src/sim/mod.rs b/sled-agent/src/sim/mod.rs index 3e26a1bbfc3..3824951d854 100644 --- a/sled-agent/src/sim/mod.rs +++ b/sled-agent/src/sim/mod.rs @@ -18,6 +18,5 @@ mod sled_agent; mod storage; pub use config::{Config, SimMode}; -pub use http_entrypoints_storage::State as RegionState; pub use server::{run_server, Server}; pub use sled_agent::SledAgent; diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index f6049a17de1..55b2cb80f45 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -4,6 +4,7 @@ //! Simulated sled agent storage implementation +use crucible_agent_client::types::{CreateRegion, Region, RegionId, State}; use futures::lock::Mutex; use nexus_client::types::{ ByteCount, DatasetKind, DatasetPutRequest, ZpoolPutRequest, @@ -16,8 +17,6 @@ use std::str::FromStr; use std::sync::Arc; use uuid::Uuid; -use super::http_entrypoints_storage::{CreateRegion, Region, RegionId, State}; - type CreateCallback = Box State + Send + 'static>; struct CrucibleDataInner { @@ -60,7 +59,7 @@ impl CrucibleDataInner { let old = self.regions.insert(id, region.clone()); if let Some(old) = old { assert_eq!( - old.id, region.id, + old.id.0, region.id.0, "Region already exists, but with a different ID" ); } From 6d02abdadd95217e59588586603e76002051b7fc Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 14 Jan 2022 17:18:07 -0500 Subject: [PATCH 2/6] Add Rust Cache Action to Github Actions (#498) Co-authored-by: David Crespo --- .github/workflows/rust.yml | 8 ++++++++ README.adoc | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index dc1a2e8f158..091b5dbefd6 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -43,6 +43,8 @@ jobs: steps: # actions/checkout@v2 - uses: actions/checkout@28c7f3d2b5162b5ddd3dfd9a45aa55eaf396478b + - uses: Swatinem/rust-cache@v1 + if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version run: cargo --version - name: Check build of deployed Omicron packages @@ -55,6 +57,8 @@ jobs: steps: # actions/checkout@v2 - uses: actions/checkout@28c7f3d2b5162b5ddd3dfd9a45aa55eaf396478b + - uses: Swatinem/rust-cache@v1 + if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version run: cargo --version - name: Report Clippy version @@ -76,6 +80,8 @@ jobs: steps: # actions/checkout@v2 - uses: actions/checkout@28c7f3d2b5162b5ddd3dfd9a45aa55eaf396478b + - uses: Swatinem/rust-cache@v1 + if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version run: cargo --version - name: Test build documentation @@ -94,6 +100,8 @@ jobs: steps: # actions/checkout@v2 - uses: actions/checkout@28c7f3d2b5162b5ddd3dfd9a45aa55eaf396478b + - uses: Swatinem/rust-cache@v1 + if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version run: cargo --version - name: Configure GitHub cache for CockroachDB binaries diff --git a/README.adoc b/README.adoc index 85bbb73caa6..47ee24a6f10 100644 --- a/README.adoc +++ b/README.adoc @@ -34,7 +34,7 @@ The code here is still rough, but the following are implemented: * actual API: ** projects: CRUD -** instances: CRUD + boot/halt/reboot (simulated) +** instances: CRUD + boot/halt/reboot. ** disks: CRD + attach/detach (simulated) ** racks: list + get ** sleds: list + get From 2e5c819883c01636f4030e23ebf081891c4272d3 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Fri, 14 Jan 2022 18:58:42 -0500 Subject: [PATCH 3/6] Introduce `thing-flinger` (#534) `thing-flinger` is a remote deployment system for installing Omicron. It's intended to be used to simplify and shorten the testing cycle. More information is found in the [documentation](package/src/README.adoc). --- Cargo.lock | 25 + package/Cargo.toml | 6 + package/README.adoc | 142 ++++++ package/src/bin/deployment-example.toml | 46 ++ package/src/bin/omicron-package.rs | 77 +-- package/src/bin/thing-flinger.rs | 634 ++++++++++++++++++++++++ package/src/lib.rs | 77 +++ 7 files changed, 937 insertions(+), 70 deletions(-) create mode 100644 package/README.adoc create mode 100644 package/src/bin/deployment-example.toml create mode 100644 package/src/bin/thing-flinger.rs create mode 100644 package/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index e13ceb4041e..3c598f49a2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -453,6 +453,20 @@ dependencies = [ "itertools", ] +[[package]] +name = "crossbeam" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae5588f6b3c3cb05239e90bd110f257254aecd01e4635400391aeae07497845" +dependencies = [ + "cfg-if", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + [[package]] name = "crossbeam-channel" version = "0.5.1" @@ -487,6 +501,16 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b10ddc024425c88c2ad148c1b0fd53f4c6d38db9697c9f1588381212fa657c9" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.5" @@ -1972,6 +1996,7 @@ name = "omicron-package" version = "0.1.0" dependencies = [ "anyhow", + "crossbeam", "omicron-common", "propolis-server", "rayon", diff --git a/package/Cargo.toml b/package/Cargo.toml index 83fe8e4df19..9a256c84e78 100644 --- a/package/Cargo.toml +++ b/package/Cargo.toml @@ -6,7 +6,9 @@ license = "MPL-2.0" [dependencies] anyhow = "1.0" +crossbeam = "0.8" omicron-common = { path = "../common" } + # We depend on the propolis-server here -- a binary, not a library -- to # make it visible to the packaging tool, which can compile it and shove # it in a tarball. @@ -28,3 +30,7 @@ walkdir = "2.3" [[bin]] name = "omicron-package" doc = false + +[[bin]] +name = "thing-flinger" +doc = false diff --git a/package/README.adoc b/package/README.adoc new file mode 100644 index 00000000000..eaf17ca782a --- /dev/null +++ b/package/README.adoc @@ -0,0 +1,142 @@ +Omicron is a complex piece of software consisting of many build and install-time dependencies. It's +intended to run primarily on illumos based systems, and as such is built to use runtime facilities +of illumos, such as https://illumos.org/man/5/smf[SMF]. Furthermore, Omicron is fundamentally a +distributed system, with its components intended to run on multiple servers communicating over the +network. In order to secure the system, certain cryptographic primitives, such as asymmetric key +pairs and shared secrets are required. Due to the nature of these cryptographic primitives, there is +a requirement for the distribution or creation of files unique to a specific server, such that no +other server has access to those files. Examples of this are private keys, and threshold key +shares, although other non-cryptographic unique files may also become necessary over time. + +In order to satisfy the above requirements of building and deploying a complex distributed system +consisting of unique, private files, two CLI tools have been created: + + . link:src/bin/omicron-package.rs[omicron-package] - build, package, install on local machine + . link:src/bin/thing-flinger.rs[thing-flinger] - build, package, deploy to remote machines + + +If a user is working on their local illumos based machine, and only wants to run +omicron in single node mode, they should follow the install instruction in +the link:../README.adoc[Omicron README] and use `omicron-package`. If the user +wishes for a more complete workflow, where they can code on their local laptop, +use a remote build machine, and install to multiple machines for a more realistic +deployment, they should use `thing-flinger`. + +The remainder of this document will describe a typical workflow for using +thing-flinger, pointing out room for improvement. + +== Environment and Configuration + + + +------------------+ +------------------+ + | | | | + | | | | + | Client |----------------> Builder | + | | | | + | | | | + +------------------+ +------------------+ + | + | + | + | + +---------------------------+--------------------------+ + | | | + | | | + | | | + +--------v---------+ +---------v--------+ +---------v--------+ + | | | | | | + | | | | | | + | Deployed Server | | Deployed Server | | Deployed Server | + | | | | | | + | | | | | | + +------------------+ +------------------+ +------------------+ + + +`thing-flinger` defines three types of nodes: + + * Client - Where a user typically edits their code and runs thing-flinger. This can run any OS. + * Builder - A Helios box where Omicron is built and packaged + * Deployed Server - Helios machines where Omicron will be installed and run + +It's not at all necessary for these to be separate nodes. For example, a client and builder can be +the same machine, as long as it's a Helios box. Same goes for Builder and a deployment server. The +benefit of this separation though, is that it allows editing on something like a laptop, without +having to worry about setting up a development environment on an illumos based host. + +Machine topology is configured in a `TOML` file that is passed on the command line. All illumos +machines are listed under `servers`, and just the names are used for configuring a builder and +deployment servers. An link:src/bin/deployment-example.toml[example] is provided. + +Thing flinger works over SSH, and so the user must have the public key of their client configured +for their account on all servers. SSH agent forwarding is used to prevent the need for the keys of +the builder to also be on the other servers, thus minimizing needed server configuration. + +== Typical Workflow + +=== Prerequisites + +Ensure you have an account on all illumos boxes, with the client public key in +`~/.ssh/authorized_keys`. + +.The build machine must have Rust and cargo installed, as well as +all the dependencies for Omicron installed. Following the *prerequisites* in the +https://github.com/oxidecomputer/omicron/#build-and-run[Build and run] section of the main Omicron +README is probably a good idea. + +=== Command Based Workflow + +==== Build thing-flinger on client +`thing-flinger` is part of the `omicron-package` crate. + +`cargo build -p omicron-package` + +==== sync +Copy your source code to the builder. Note that this copies over your `.git` subdirectory on purpose so +that a branch can be configured for building with the `git_treeish` field in the toml `builder` +table. + +`./target/debug/thing-flinger -c sync` + +==== build-minimal +Build necessary parts of omicron on the builder, required for future use by thing-flinger. + +`./target/debug/thing-flinger -c build-minimal` + +==== package +Build and package omicron using `omicron-package` on the builder. + +`./target/debug/thing-flinger -c package` + +==== overlay +Create files that are unique to each deployment server. + +`./target/debug/thing-flinger -c overlay` + +==== install +Install omicron to all machines, in parallel. This consists of copying the packaged omicron tarballs +along with overlay files, and omicron-package and its manifest to a `staging` directory on each +deployment server, and then running omicron-package, installing overlay files, and restarting +services. + +`./target/debug/thing-flinger -c install` + +=== Current Limitations + +`thing-flinger` is an early prototype. It has served so far to demonstrate that unique files, +specifically secret shares, can be created and distributed over ssh, and that omicron can be +installed remotely using `omicron-package`. It is not currently complete enough to fully test a +distributed omicron setup, as the underlying dependencies are not configured yet. Specifically, +`CockroachDB` and perhaps `Clickhouse`, need to be configured to run in multiple server mode. It's +anticipated that the `overlay` feature of `thing-flinger` can be used to generate and distribute +configs for this. + +=== Design rationale + +`thing-flinger` is a command line program written in rust. It was written this way to build upon +`omicron-package`, which is also in rust, as that is our default language of choice at Oxide. +`thing-flinger` is based around SSH, as that is the minimal viable requirement for a test tool such +as this. Additionally, it provides for the most straightforward implementation, and takes the least +effort to use securely. This particular implementation wraps the openssh ssh client via +`std::process::Command`, rather than using the `ssh2` crate, because ssh2, as a wrapper around +`libssh`, does not support agent-forwarding. + diff --git a/package/src/bin/deployment-example.toml b/package/src/bin/deployment-example.toml new file mode 100644 index 00000000000..62296fa4d0f --- /dev/null +++ b/package/src/bin/deployment-example.toml @@ -0,0 +1,46 @@ +# This manifest describes the servers that omicron will be installed to, along +# with any ancillary information specific to a given server. +# +# It is ingested by the `thing-flinger` tool. + +# This must be an absolute path +local_source = "/Users/ajs/oxide/omicron" + +[builder] +# `server` must refer to one of the `servers` in the servers table +server = "atrium" + +# This must be an absolute path +omicron_path = "/home/andrew/oxide/omicron" + +# Git branch, sha, etc... +git_treeish = "thing-flinger2" + +[deployment] +servers = ["sock", "buskin"] +rack_secret_threshold = 2 + +# Location where files to install will be placed before running +# `omicron-package install` +# +# This must be an absolute path +# We specifically allow for $HOME in validating the absolute path +staging_dir = "$HOME/omicron_staging" + +[servers.tarz] +username = "ajs" +addr = "tarz.local" + +[servers.atrium] +username = "andrew" +addr = "atrium.eng.oxide.computer" + +[servers.sock] +username = "andrew" +addr = "sock.eng.oxide.computer" + +[servers.buskin] +username = "andrew" +addr = "buskin.eng.oxide.computer" + + diff --git a/package/src/bin/omicron-package.rs b/package/src/bin/omicron-package.rs index 10f1b64bb52..f9131400dba 100644 --- a/package/src/bin/omicron-package.rs +++ b/package/src/bin/omicron-package.rs @@ -9,6 +9,7 @@ use omicron_common::packaging::sha256_digest; use anyhow::{anyhow, bail, Context, Result}; +use omicron_package::{parse, SubCommand}; use rayon::prelude::*; use reqwest; use serde_derive::Deserialize; @@ -19,7 +20,6 @@ use std::path::{Path, PathBuf}; use std::process::Command; use structopt::StructOpt; use tar::Builder; -use thiserror::Error; use tokio::io::AsyncWriteExt; const S3_BUCKET: &str = "https://oxide-omicron-build.s3.amazonaws.com"; @@ -29,59 +29,6 @@ const PKG: &str = "pkg"; // Name for the directory component where downloaded blobs are stored. const BLOB: &str = "blob"; -#[derive(Debug, StructOpt)] -enum SubCommand { - /// Builds the packages specified in a manifest, and places them into a target - /// directory. - Package { - /// The output directory, where artifacts should be placed. - /// - /// Defaults to "out". - #[structopt(long = "out", default_value = "out")] - artifact_dir: PathBuf, - - /// The binary profile to package. - /// - /// True: release, False: debug (default). - #[structopt( - short, - long, - help = "True if bundling release-mode binaries" - )] - release: bool, - }, - /// Checks the packages specified in a manifest, without building. - Check, - /// Installs the packages to a target machine. - Install { - /// The directory from which artifacts will be pulled. - /// - /// Should match the format from the Package subcommand. - #[structopt(long = "in", default_value = "out")] - artifact_dir: PathBuf, - - /// The directory to which artifacts will be installed. - /// - /// Defaults to "/opt/oxide". - #[structopt(long = "out", default_value = "/opt/oxide")] - install_dir: PathBuf, - }, - /// Removes the packages from the target machine. - Uninstall { - /// The directory from which artifacts were be pulled. - /// - /// Should match the format from the Package subcommand. - #[structopt(long = "in", default_value = "out")] - artifact_dir: PathBuf, - - /// The directory to which artifacts were installed. - /// - /// Defaults to "/opt/oxide". - #[structopt(long = "out", default_value = "/opt/oxide")] - install_dir: PathBuf, - }, -} - #[derive(Debug, StructOpt)] #[structopt(name = "packaging tool")] struct Args { @@ -121,15 +68,6 @@ fn run_cargo_on_package( Ok(()) } -/// Errors which may be returned when parsing the server configuration. -#[derive(Error, Debug)] -enum ParseError { - #[error("Cannot parse toml: {0}")] - Toml(#[from] toml::de::Error), - #[error("IO error: {0}")] - Io(#[from] std::io::Error), -} - #[derive(Deserialize, Debug)] #[serde(rename_all = "lowercase")] enum Build { @@ -200,12 +138,6 @@ struct Config { packages: BTreeMap, } -fn parse>(path: P) -> Result { - let contents = std::fs::read_to_string(path.as_ref())?; - let cfg = toml::from_str::(&contents)?; - Ok(cfg) -} - async fn do_check(config: &Config) -> Result<()> { for (package_name, package) in &config.packages { println!("Checking {}", package_name); @@ -356,6 +288,11 @@ fn do_install( anyhow!("Cannot create installation directory: {}", err) })?; + println!( + "Copying digest.toml from {} to {}", + artifact_dir.to_string_lossy(), + install_dir.to_string_lossy() + ); // Move the digest of expected packages. std::fs::copy( artifact_dir.join("digest.toml"), @@ -454,7 +391,7 @@ fn do_uninstall( #[tokio::main] async fn main() -> Result<()> { let args = Args::from_args_safe().map_err(|err| anyhow!(err))?; - let config = parse(&args.manifest)?; + let config = parse::<_, Config>(&args.manifest)?; // Use a CWD that is the root of the Omicron repository. if let Ok(manifest) = env::var("CARGO_MANIFEST_DIR") { diff --git a/package/src/bin/thing-flinger.rs b/package/src/bin/thing-flinger.rs new file mode 100644 index 00000000000..284a59226d3 --- /dev/null +++ b/package/src/bin/thing-flinger.rs @@ -0,0 +1,634 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Utility for deploying Omicron to remote machines + +use omicron_package::{parse, SubCommand as PackageSubCommand}; + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Write; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{anyhow, Context, Result}; +use crossbeam::thread::{self, ScopedJoinHandle}; +use serde_derive::Deserialize; +use structopt::StructOpt; +use thiserror::Error; + +#[derive(Deserialize, Debug)] +struct Builder { + pub server: String, + pub omicron_path: PathBuf, + pub git_treeish: String, +} + +// A server on which an omicron package is deployed +#[derive(Deserialize, Debug)] +struct Server { + pub username: String, + pub addr: String, +} + +#[derive(Deserialize, Debug)] +struct Deployment { + pub servers: BTreeSet, + pub rack_secret_threshold: usize, + pub staging_dir: PathBuf, +} + +#[derive(Debug, Deserialize)] +struct Config { + pub local_source: PathBuf, + pub builder: Builder, + pub servers: BTreeMap, + pub deployment: Deployment, +} + +fn parse_into_set(src: &str) -> BTreeSet { + src.split_whitespace().map(|s| s.to_owned()).collect() +} + +#[derive(Debug, StructOpt)] +enum SubCommand { + /// Run the given command on the given servers, or all servers if none are + /// given. + /// + /// Be careful! + Exec { + /// The command to run + #[structopt(short, long)] + cmd: String, + + /// The servers to run the command on + #[structopt(short, long, parse(from_str = parse_into_set))] + servers: Option>, + }, + + /// Sync our local source to the build host + Sync, + + /// Build omicron-package and everything needed to run thing-flinger + /// commands on the build host. + /// + /// Package always builds everything, but it can be set in release mode, and + /// we expect the existing tools to run from 'target/debug'. Additionally, + // you can't run `Package` until you have actually built `omicron-package`, + // which `BuildMinimal` does. + BuildMinimal, + + /// Use all subcommands from omicron-package + #[structopt(flatten)] + Package(PackageSubCommand), + + /// Create an overlay directory tree for each deployment server + /// + /// Each directory tree contains unique files for the given server that will + /// be populated in the svc/pkg dir. + /// + /// This is a separate subcommand so that we can reconstruct overlays + /// without rebuilding or repackaging. + Overlay, +} + +#[derive(Debug, StructOpt)] +#[structopt(name = "thing-flinger")] +struct Args { + /// The path to the deployment manifest TOML file + #[structopt(short, long, help = "Path to deployment manifest toml file")] + config: PathBuf, + + #[structopt(subcommand)] + subcommand: SubCommand, +} + +/// Errors which can be returned when executing subcommands +#[derive(Error, Debug)] +enum FlingError { + #[error("Servers not listed in configuration: {0:?}")] + InvalidServers(Vec), + + /// The parameter should be the name of the argument that could not be + /// properly converted to a string. + #[error("{0} is not valid UTF-8")] + BadString(String), + + /// Failed to rsync omicron to build host + #[error("Failed to sync {src} with {dst}")] + FailedSync { src: String, dst: String }, + + /// The given path must be absolute + #[error("Path for {field} must be absolute")] + NotAbsolutePath { field: &'static str }, +} + +// TODO: run in parallel when that option is given +fn do_exec( + config: &Config, + cmd: String, + servers: Option>, +) -> Result<()> { + if let Some(ref servers) = servers { + validate_servers(servers, &config.servers)?; + + for name in servers { + let server = &config.servers[name]; + ssh_exec(&server, &cmd, false)?; + } + } else { + for (_, server) in config.servers.iter() { + ssh_exec(&server, &cmd, false)?; + } + } + Ok(()) +} + +fn do_sync(config: &Config) -> Result<()> { + let server = + config.servers.get(&config.builder.server).ok_or_else(|| { + FlingError::InvalidServers(vec![config.builder.server.clone()]) + })?; + + // For rsync to copy from the source appropriately we must guarantee a + // trailing slash. + let src = + format!("{}/", config.local_source.canonicalize()?.to_string_lossy()); + let dst = format!( + "{}@{}:{}", + server.username, + server.addr, + config.builder.omicron_path.to_str().unwrap() + ); + let mut cmd = Command::new("rsync"); + cmd.arg("-az") + .arg("-e") + .arg("ssh") + .arg("--delete") + .arg("--progress") + .arg("--exclude") + .arg("target/") + .arg("--exclude") + .arg("out/") + .arg("--exclude") + .arg("cockroachdb/") + .arg("--exclude") + .arg("clickhouse/") + .arg("--exclude") + .arg("*.swp") + .arg("--out-format") + .arg("File changed: %o %t %f") + .arg(&src) + .arg(&dst); + let status = + cmd.status().context(format!("Failed to run command: ({:?})", cmd))?; + if !status.success() { + return Err(FlingError::FailedSync { src, dst }.into()); + } + + Ok(()) +} + +// Build omicron-package and omicron-sled-agent on the builder +// +// We need to build omicron-sled-agent for overlay file generation +fn do_build_minimal(config: &Config) -> Result<()> { + let server = &config.servers[&config.builder.server]; + let cmd = format!( + "cd {} && git checkout {} && cargo build -p {} -p {}", + config.builder.omicron_path.to_string_lossy(), + config.builder.git_treeish, + "omicron-package", + "omicron-sled-agent" + ); + ssh_exec(&server, &cmd, false) +} + +fn do_package( + config: &Config, + artifact_dir: PathBuf, + release: bool, +) -> Result<()> { + let server = &config.servers[&config.builder.server]; + let mut cmd = String::new(); + let mut release_flag = ""; + if release { + release_flag = "--release"; + } + let cmd_path = "./target/debug/omicron-package"; + let artifact_dir = artifact_dir + .to_str() + .ok_or_else(|| FlingError::BadString("artifact_dir".to_string()))?; + + // We use a bash login shell to get a proper environment, so we have a path to + // postgres, and $DEP_PQ_LIBDIRS is filled in. This is required for building + // nexus. + // + // See https://github.com/oxidecomputer/omicron/blob/8757ec542ea4ffbadd6f26094ed4ba357715d70d/rpaths/src/lib.rs + write!( + &mut cmd, + "bash -lc 'cd {} && git checkout {} && {} package --out {} {}'", + config.builder.omicron_path.to_string_lossy(), + config.builder.git_treeish, + cmd_path, + &artifact_dir, + release_flag + )?; + + ssh_exec(&server, &cmd, false) +} + +fn do_check(config: &Config) -> Result<()> { + let server = &config.servers[&config.builder.server]; + let mut cmd = String::new(); + let cmd_path = "./target/debug/omicron-package"; + + write!( + &mut cmd, + "bash -lc 'cd {} && git checkout {} && {} check'", + config.builder.omicron_path.to_string_lossy(), + config.builder.git_treeish, + cmd_path, + )?; + + ssh_exec(&server, &cmd, false) +} + +fn do_uninstall( + config: &Config, + artifact_dir: PathBuf, + install_dir: PathBuf, +) -> Result<()> { + let mut deployment_src = PathBuf::from(&config.deployment.staging_dir); + deployment_src.push(&artifact_dir); + for server_name in &config.deployment.servers { + let server = &config.servers[server_name]; + // Run `omicron-package uninstall` on the deployment server + let cmd = format!( + "cd {} && pfexec ./omicron-package uninstall --in {} --out {}", + config.deployment.staging_dir.to_string_lossy(), + deployment_src.to_string_lossy(), + install_dir.to_string_lossy() + ); + println!("$ {}", cmd); + ssh_exec(&server, &cmd, true)?; + } + Ok(()) +} + +fn do_install(config: &Config, artifact_dir: &Path, install_dir: &Path) { + let builder = &config.servers[&config.builder.server]; + let mut pkg_dir = PathBuf::from(&config.builder.omicron_path); + pkg_dir.push(artifact_dir); + let pkg_dir = pkg_dir.to_string_lossy(); + let pkg_dir = &pkg_dir; + + thread::scope(|s| { + let mut handles = + Vec::<(String, ScopedJoinHandle<'_, Result<()>>)>::new(); + + // Spawn a thread for each server install + for server_name in &config.deployment.servers { + handles.push(( + server_name.to_owned(), + s.spawn(move |_| -> Result<()> { + single_server_install( + config, + &artifact_dir, + &install_dir, + &pkg_dir, + builder, + server_name, + ) + }), + )); + } + + // Join all the handles and print the install status + for (server_name, handle) in handles { + match handle.join() { + Ok(Ok(())) => { + println!("Install completed for server: {}", server_name) + } + Ok(Err(e)) => { + println!( + "Install failed for server: {} with error: {}", + server_name, e + ) + } + Err(_) => { + println!( + "Install failed for server: {}. Thread panicked.", + server_name + ) + } + } + } + }) + .unwrap(); +} + +fn do_overlay(config: &Config) -> Result<()> { + let mut root_path = PathBuf::from(&config.builder.omicron_path); + // TODO: This needs to match the artifact_dir in `package` + root_path.push("out/overlay"); + let server_dirs = dir_per_deploy_server(config, &root_path); + let server = &config.servers[&config.builder.server]; + overlay_sled_agent(&server, config, &server_dirs) +} + +fn overlay_sled_agent( + server: &Server, + config: &Config, + server_dirs: &[PathBuf], +) -> Result<()> { + let sled_agent_dirs: Vec = server_dirs + .iter() + .map(|dir| { + let mut dir = PathBuf::from(dir); + dir.push("sled-agent/pkg"); + dir + }) + .collect(); + + // Create directories on builder + let dirs = dir_string(&sled_agent_dirs); + let cmd = format!("sh -c 'for dir in {}; do mkdir -p $dir; done'", dirs); + + let cmd = format!( + "{} && cd {} && ./target/debug/sled-agent-overlay-files \ + --threshold {} --directories {}", + cmd, + config.builder.omicron_path.to_string_lossy(), + config.deployment.rack_secret_threshold, + dirs + ); + ssh_exec(server, &cmd, false) +} + +fn single_server_install( + config: &Config, + artifact_dir: &Path, + install_dir: &Path, + pkg_dir: &str, + builder: &Server, + server_name: &str, +) -> Result<()> { + let server = &config.servers[server_name]; + + copy_package_artifacts_to_staging(config, pkg_dir, builder, server)?; + copy_omicron_package_binary_to_staging(config, builder, server)?; + copy_package_manifest_to_staging(config, builder, server)?; + run_omicron_package_from_staging( + config, + server, + &artifact_dir, + &install_dir, + )?; + copy_overlay_files_to_staging( + config, + pkg_dir, + builder, + server, + server_name, + )?; + install_overlay_files_from_staging(config, server, &install_dir)?; + restart_services(server) +} + +// Copy package artifacts as a result of `omicron-package package` from the +// builder to the deployment server staging directory. +fn copy_package_artifacts_to_staging( + config: &Config, + pkg_dir: &str, + builder: &Server, + destination: &Server, +) -> Result<()> { + let cmd = format!( + "rsync -avz -e 'ssh -o StrictHostKeyChecking=no' \ + --exclude overlay/ {} {}@{}:{}", + pkg_dir, + destination.username, + destination.addr, + config.deployment.staging_dir.to_string_lossy() + ); + println!("$ {}", cmd); + ssh_exec(builder, &cmd, true) +} + +fn copy_omicron_package_binary_to_staging( + config: &Config, + builder: &Server, + destination: &Server, +) -> Result<()> { + let mut bin_path = PathBuf::from(&config.builder.omicron_path); + bin_path.push("target/debug/omicron-package"); + let cmd = format!( + "rsync -avz {} {}@{}:{}", + bin_path.to_string_lossy(), + destination.username, + destination.addr, + config.deployment.staging_dir.to_string_lossy() + ); + println!("$ {}", cmd); + ssh_exec(builder, &cmd, true) +} + +fn copy_package_manifest_to_staging( + config: &Config, + builder: &Server, + destination: &Server, +) -> Result<()> { + let mut path = PathBuf::from(&config.builder.omicron_path); + path.push("package-manifest.toml"); + let cmd = format!( + "rsync {} {}@{}:{}", + path.to_string_lossy(), + destination.username, + destination.addr, + config.deployment.staging_dir.to_string_lossy() + ); + println!("$ {}", cmd); + ssh_exec(builder, &cmd, true) +} + +fn run_omicron_package_from_staging( + config: &Config, + destination: &Server, + artifact_dir: &Path, + install_dir: &Path, +) -> Result<()> { + let mut deployment_src = PathBuf::from(&config.deployment.staging_dir); + deployment_src.push(&artifact_dir); + + // Run `omicron-package install` on the deployment server + let cmd = format!( + "cd {} && pfexec ./omicron-package install --in {} --out {}", + config.deployment.staging_dir.to_string_lossy(), + deployment_src.to_string_lossy(), + install_dir.to_string_lossy() + ); + println!("$ {}", cmd); + ssh_exec(destination, &cmd, true) +} + +fn copy_overlay_files_to_staging( + config: &Config, + pkg_dir: &str, + builder: &Server, + destination: &Server, + destination_name: &str, +) -> Result<()> { + let cmd = format!( + "rsync -avz {}/overlay/{}/ {}@{}:{}/overlay/", + pkg_dir, + destination_name, + destination.username, + destination.addr, + config.deployment.staging_dir.to_string_lossy() + ); + println!("$ {}", cmd); + ssh_exec(builder, &cmd, true) +} + +fn install_overlay_files_from_staging( + config: &Config, + destination: &Server, + install_dir: &Path, +) -> Result<()> { + let cmd = format!( + "pfexec cp -r {}/overlay/* {}", + config.deployment.staging_dir.to_string_lossy(), + install_dir.to_string_lossy() + ); + println!("$ {}", cmd); + ssh_exec(&destination, &cmd, false) +} + +// For now, we just restart sled-agent, as that's the only service with an +// overlay file. +fn restart_services(destination: &Server) -> Result<()> { + ssh_exec(destination, "svcadm restart sled-agent", false) +} + +fn dir_string(dirs: &[PathBuf]) -> String { + dirs.iter().map(|dir| dir.to_string_lossy().to_string() + " ").collect() +} + +fn dir_per_deploy_server(config: &Config, root: &Path) -> Vec { + config + .deployment + .servers + .iter() + .map(|server_dir| { + let mut dir = PathBuf::from(root); + dir.push(server_dir); + dir + }) + .collect() +} + +fn ssh_exec( + server: &Server, + remote_cmd: &str, + forward_agent: bool, +) -> Result<()> { + // Source .profile, so we have access to cargo. Rustup installs knowledge + // about the cargo path here. + let remote_cmd = String::from(". $HOME/.profile && ") + remote_cmd; + let auth_sock = std::env::var("SSH_AUTH_SOCK")?; + let mut cmd = Command::new("ssh"); + if forward_agent { + cmd.arg("-A"); + } + cmd.arg("-o") + .arg("StrictHostKeyChecking=no") + .arg("-l") + .arg(&server.username) + .arg(&server.addr) + .arg(&remote_cmd); + cmd.env("SSH_AUTH_SOCK", auth_sock); + cmd.status() + .context(format!("Failed to run {} on {}", remote_cmd, server.addr))?; + + Ok(()) +} + +fn validate_servers( + chosen: &BTreeSet, + all: &BTreeMap, +) -> Result<(), FlingError> { + let all = all.keys().cloned().collect(); + let diff: Vec = chosen.difference(&all).cloned().collect(); + if !diff.is_empty() { + Err(FlingError::InvalidServers(diff)) + } else { + Ok(()) + } +} + +fn validate_absolute_path( + path: &Path, + field: &'static str, +) -> Result<(), FlingError> { + if path.is_absolute() || path.starts_with("$HOME") { + Ok(()) + } else { + Err(FlingError::NotAbsolutePath { field }) + } +} + +fn validate(config: &Config) -> Result<(), FlingError> { + validate_absolute_path(&config.local_source, "local_source")?; + validate_absolute_path( + &config.builder.omicron_path, + "builder.omicron_path", + )?; + validate_absolute_path( + &config.deployment.staging_dir, + "deployment.staging_dir", + )?; + + validate_servers(&config.deployment.servers, &config.servers)?; + + validate_servers( + &BTreeSet::from([config.builder.server.clone()]), + &config.servers, + ) +} + +fn main() -> Result<()> { + let args = Args::from_args_safe().map_err(|err| anyhow!(err))?; + let config = parse::<_, Config>(args.config)?; + + validate(&config)?; + + match args.subcommand { + SubCommand::Exec { cmd, servers } => { + do_exec(&config, cmd, servers)?; + } + SubCommand::Sync => do_sync(&config)?, + SubCommand::BuildMinimal => do_build_minimal(&config)?, + SubCommand::Package(PackageSubCommand::Package { + artifact_dir, + release, + }) => { + do_package(&config, artifact_dir, release)?; + } + SubCommand::Package(PackageSubCommand::Install { + artifact_dir, + install_dir, + }) => { + do_install(&config, &artifact_dir, &install_dir); + } + SubCommand::Package(PackageSubCommand::Uninstall { + artifact_dir, + install_dir, + }) => { + do_uninstall(&config, artifact_dir, install_dir)?; + } + SubCommand::Package(PackageSubCommand::Check) => do_check(&config)?, + SubCommand::Overlay => do_overlay(&config)?, + } + Ok(()) +} diff --git a/package/src/lib.rs b/package/src/lib.rs new file mode 100644 index 00000000000..f4a84573fab --- /dev/null +++ b/package/src/lib.rs @@ -0,0 +1,77 @@ +//! Common code shared between `omicron-package` and `thing-flinger` binaries. + +use serde::de::DeserializeOwned; +use std::path::Path; +use std::path::PathBuf; +use structopt::StructOpt; +use thiserror::Error; + +/// Errors which may be returned when parsing the server configuration. +#[derive(Error, Debug)] +pub enum ParseError { + #[error("Cannot parse toml: {0}")] + Toml(#[from] toml::de::Error), + #[error("IO error: {0}")] + Io(#[from] std::io::Error), +} + +pub fn parse, C: DeserializeOwned>( + path: P, +) -> Result { + let contents = std::fs::read_to_string(path.as_ref())?; + let cfg = toml::from_str::(&contents)?; + Ok(cfg) +} + +#[derive(Debug, StructOpt)] +pub enum SubCommand { + /// Builds the packages specified in a manifest, and places them into a target + /// directory. + Package { + /// The output directory, where artifacts should be placed. + /// + /// Defaults to "out". + #[structopt(long = "out", default_value = "out")] + artifact_dir: PathBuf, + + /// The binary profile to package. + /// + /// True: release, False: debug (default). + #[structopt( + short, + long, + help = "True if bundling release-mode binaries" + )] + release: bool, + }, + /// Checks the packages specified in a manifest, without building. + Check, + /// Installs the packages to a target machine. + Install { + /// The directory from which artifacts will be pulled. + /// + /// Should match the format from the Package subcommand. + #[structopt(long = "in", default_value = "out")] + artifact_dir: PathBuf, + + /// The directory to which artifacts will be installed. + /// + /// Defaults to "/opt/oxide". + #[structopt(long = "out", default_value = "/opt/oxide")] + install_dir: PathBuf, + }, + /// Removes the packages from the target machine. + Uninstall { + /// The directory from which artifacts were be pulled. + /// + /// Should match the format from the Package subcommand. + #[structopt(long = "in", default_value = "out")] + artifact_dir: PathBuf, + + /// The directory to which artifacts were installed. + /// + /// Defaults to "/opt/oxide". + #[structopt(long = "out", default_value = "/opt/oxide")] + install_dir: PathBuf, + }, +} From 2a760e3a172fe8c2bec369ba458d66ac7b332fef Mon Sep 17 00:00:00 2001 From: Adam Leventhal Date: Tue, 18 Jan 2022 09:18:24 -0800 Subject: [PATCH 4/6] update dropshot and progenitor (#602) --- Cargo.lock | 74 +++++++++++++++++++------- common/src/lib.rs | 12 ++--- oximeter/producer/examples/producer.rs | 7 ++- oximeter/producer/src/lib.rs | 1 + sled-agent/src/sim/disk.rs | 1 + sled-agent/src/sim/instance.rs | 1 + 6 files changed, 68 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c598f49a2a..9c2371a8c18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -98,6 +98,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-stream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171374e7e3b2504e0e5236e3b59260560f9fe94bfe9ac39ba5e4e929c5590625" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.52" @@ -875,8 +896,9 @@ checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" [[package]] name = "dropshot" version = "0.6.1-dev" -source = "git+https://github.com/oxidecomputer/dropshot?branch=main#3ced9b099d72e9b975e9a6551989b027415b849b" +source = "git+https://github.com/oxidecomputer/dropshot?branch=main#8e4af93207fb79998eea90bd094ff7a5475673e5" dependencies = [ + "async-stream", "async-trait", "base64", "bytes", @@ -891,6 +913,8 @@ dependencies = [ "paste", "percent-encoding", "proc-macro2", + "rustls", + "rustls-pemfile", "schemars", "serde", "serde_json", @@ -902,6 +926,7 @@ dependencies = [ "slog-term", "syn", "tokio", + "tokio-rustls", "toml", "usdt 0.3.1", "uuid", @@ -910,7 +935,7 @@ dependencies = [ [[package]] name = "dropshot_endpoint" version = "0.6.1-dev" -source = "git+https://github.com/oxidecomputer/dropshot?branch=main#3ced9b099d72e9b975e9a6551989b027415b849b" +source = "git+https://github.com/oxidecomputer/dropshot?branch=main#8e4af93207fb79998eea90bd094ff7a5475673e5" dependencies = [ "proc-macro2", "quote", @@ -1263,9 +1288,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f072413d126e57991455e0a922b31e4c8ba7c2ffbebf6b78b4f8521397d65cd" +checksum = "0c9de88456263e249e241fcd211d3954e2c9b0ef7ccfc235a444eb367cae3689" dependencies = [ "bytes", "fnv", @@ -1469,9 +1494,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" +checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" dependencies = [ "autocfg", "hashbrown", @@ -2711,7 +2736,7 @@ dependencies = [ [[package]] name = "progenitor" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/progenitor#66b41ba301793b8d720770b2210bee8884446d3f" +source = "git+https://github.com/oxidecomputer/progenitor#f1f9e2e93850713908f4e6494808a07f3b253108" dependencies = [ "anyhow", "getopts", @@ -2725,7 +2750,7 @@ dependencies = [ [[package]] name = "progenitor-impl" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/progenitor#66b41ba301793b8d720770b2210bee8884446d3f" +source = "git+https://github.com/oxidecomputer/progenitor#f1f9e2e93850713908f4e6494808a07f3b253108" dependencies = [ "anyhow", "convert_case", @@ -2739,20 +2764,24 @@ dependencies = [ "schemars", "serde", "serde_json", + "syn", "thiserror", "typify", + "unicode-xid", ] [[package]] name = "progenitor-macro" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/progenitor#66b41ba301793b8d720770b2210bee8884446d3f" +source = "git+https://github.com/oxidecomputer/progenitor#f1f9e2e93850713908f4e6494808a07f3b253108" dependencies = [ "openapiv3", "proc-macro2", "progenitor-impl", "quote", + "serde", "serde_json", + "serde_tokenstream", "syn", ] @@ -2984,15 +3013,16 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c4e0a76dc12a116108933f6301b95e83634e0c47b0afbed6abbaa0601e99258" +checksum = "87f242f1488a539a79bac6dbe7c8609ae43b7914b7736210f239a37cccb32525" dependencies = [ "base64", "bytes", "encoding_rs", "futures-core", "futures-util", + "h2", "http", "http-body", "hyper", @@ -3294,9 +3324,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.74" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2bb9cd061c5865d345bb02ca49fcef1391741b672b54a0bf7b679badec3142" +checksum = "c059c05b48c5c0067d4b4b2b4f0732dd65feb52daf7e0ea09cd87e7dadc1af79" dependencies = [ "itoa 1.0.1", "ryu", @@ -3305,9 +3335,9 @@ dependencies = [ [[package]] name = "serde_tokenstream" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3ce95257fba42a656f558db28d56a9fac5aa6e4f29c5ef607f32f524fab0ab" +checksum = "d6deb15c3a535e81438110111d90168d91721652f502abb147f31cde129f683d" dependencies = [ "proc-macro2", "serde", @@ -3997,9 +4027,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.23.1" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4baa378e417d780beff82bf54ceb0d195193ea6a00c14e22359e7f39456b5689" +checksum = "a27d5f2b839802bd8267fa19b0530f5a08b9c08cd417976be2a65d130fe1c11b" dependencies = [ "rustls", "tokio", @@ -4154,7 +4184,7 @@ checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" [[package]] name = "typify" version = "0.0.6-dev" -source = "git+https://github.com/oxidecomputer/typify#80b510b02b1db22de463efcf6e7762243bcea67a" +source = "git+https://github.com/oxidecomputer/typify#9afa917671b29fc231bc9ce304e041bdd685af09" dependencies = [ "typify-impl", "typify-macro", @@ -4163,9 +4193,10 @@ dependencies = [ [[package]] name = "typify-impl" version = "0.0.6-dev" -source = "git+https://github.com/oxidecomputer/typify#80b510b02b1db22de463efcf6e7762243bcea67a" +source = "git+https://github.com/oxidecomputer/typify#9afa917671b29fc231bc9ce304e041bdd685af09" dependencies = [ "convert_case", + "log", "proc-macro2", "quote", "rustfmt-wrapper", @@ -4173,17 +4204,20 @@ dependencies = [ "serde_json", "syn", "thiserror", + "unicode-xid", ] [[package]] name = "typify-macro" version = "0.0.6-dev" -source = "git+https://github.com/oxidecomputer/typify#80b510b02b1db22de463efcf6e7762243bcea67a" +source = "git+https://github.com/oxidecomputer/typify#9afa917671b29fc231bc9ce304e041bdd685af09" dependencies = [ "proc-macro2", "quote", "schemars", + "serde", "serde_json", + "serde_tokenstream", "syn", "typify-impl", ] diff --git a/common/src/lib.rs b/common/src/lib.rs index 8222f7f2c75..1d62c70d60b 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -34,18 +34,18 @@ pub mod packaging; macro_rules! generate_logging_api { ($path:literal) => { progenitor::generate_api!( - $path, - slog::Logger, - |log: &slog::Logger, request: &reqwest::Request| { + spec = $path, + inner_type = slog::Logger, + pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { slog::debug!(log, "client request"; "method" => %request.method(), "uri" => %request.url(), "body" => ?&request.body(), ); - }, - |log: &slog::Logger, result: &Result<_, _>| { + }), + post_hook = (|log: &slog::Logger, result: &Result<_, _>| { slog::debug!(log, "client response"; "result" => ?result); - }, + }), ); }; } diff --git a/oximeter/producer/examples/producer.rs b/oximeter/producer/examples/producer.rs index ef20bb87623..c137f533bcb 100644 --- a/oximeter/producer/examples/producer.rs +++ b/oximeter/producer/examples/producer.rs @@ -86,8 +86,11 @@ impl Producer for CpuBusyProducer { #[tokio::main] async fn main() { let address = "[::1]:0".parse().unwrap(); - let dropshot_config = - ConfigDropshot { bind_address: address, request_body_max_bytes: 2048 }; + let dropshot_config = ConfigDropshot { + bind_address: address, + request_body_max_bytes: 2048, + tls: None, + }; let logging_config = ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Debug }; let server_info = ProducerEndpoint { diff --git a/oximeter/producer/src/lib.rs b/oximeter/producer/src/lib.rs index 29c8b3818cc..f7cb59e16f1 100644 --- a/oximeter/producer/src/lib.rs +++ b/oximeter/producer/src/lib.rs @@ -180,6 +180,7 @@ pub async fn register( client .cpapi_producers_post(&server_info.into()) .await + .map(|_| ()) .map_err(|msg| Error::RegistrationError(msg.to_string())) } diff --git a/sled-agent/src/sim/disk.rs b/sled-agent/src/sim/disk.rs index 95ef478dcc5..1bf395d7701 100644 --- a/sled-agent/src/sim/disk.rs +++ b/sled-agent/src/sim/disk.rs @@ -95,6 +95,7 @@ impl Simulatable for SimDisk { &nexus_client::types::DiskRuntimeState::from(current), ) .await + .map(|_| ()) .map_err(Error::from) } } diff --git a/sled-agent/src/sim/instance.rs b/sled-agent/src/sim/instance.rs index b7414d4c877..2e57353f878 100644 --- a/sled-agent/src/sim/instance.rs +++ b/sled-agent/src/sim/instance.rs @@ -102,6 +102,7 @@ impl Simulatable for SimInstance { &nexus_client::types::InstanceRuntimeState::from(current), ) .await + .map(|_| ()) .map_err(Error::from) } } From 1beee373284675cfce9204bf11ee697d404e2768 Mon Sep 17 00:00:00 2001 From: "Adam H. Leventhal" Date: Tue, 18 Jan 2022 10:40:04 -0800 Subject: [PATCH 5/6] update to use the new progenitor interface --- Cargo.lock | 18 ++++++++++++++++++ Cargo.toml | 8 ++++---- common/src/lib.rs | 1 + 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c2371a8c18..7ddd19adae3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -572,6 +572,19 @@ dependencies = [ "xts-mode", ] +[[package]] +name = "crucible-agent-client" +version = "0.0.1" +dependencies = [ + "anyhow", + "percent-encoding", + "progenitor", + "reqwest", + "schemars", + "serde", + "serde_json", +] + [[package]] name = "crucible-common" version = "0.0.0" @@ -1810,6 +1823,7 @@ dependencies = [ "percent-encoding", "progenitor", "reqwest", + "schemars", "serde", "serde_json", "slog", @@ -1963,6 +1977,7 @@ dependencies = [ "chrono", "cookie", "criterion", + "crucible-agent-client", "db-macros", "diesel", "diesel-dtrace", @@ -2051,6 +2066,7 @@ dependencies = [ "bytes", "cfg-if", "chrono", + "crucible-agent-client", "dropshot", "expectorate", "futures", @@ -2241,6 +2257,7 @@ dependencies = [ "percent-encoding", "progenitor", "reqwest", + "schemars", "serde", "slog", "uuid", @@ -3523,6 +3540,7 @@ dependencies = [ "percent-encoding", "progenitor", "reqwest", + "schemars", "serde", "slog", "uuid", diff --git a/Cargo.toml b/Cargo.toml index aaa4eed6633..2438e6df956 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,10 +57,8 @@ panic = "abort" # # Local client generation during development. # -[patch."https://github.com/oxidecomputer/progenitor"] -progenitor = { path = "../progenitor/progenitor" } -[patch."https://github.com/oxidecomputer/crucible"] -crucible-agent-client = { path = "../crucible/agent-client" } +#[patch."https://github.com/oxidecomputer/progenitor"] +#progenitor = { path = "../progenitor/progenitor" } #[patch."https://github.com/oxidecomputer/typify"] #typify = { path = "../typify/typify" } @@ -72,3 +70,5 @@ crucible-agent-client = { path = "../crucible/agent-client" } git = 'https://github.com/oxidecomputer/pq-sys' branch = "oxide/omicron" +[patch."https://github.com/oxidecomputer/crucible"] +crucible-agent-client = { path = "../crucible/agent-client" } diff --git a/common/src/lib.rs b/common/src/lib.rs index 1d62c70d60b..3f734d20159 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -46,6 +46,7 @@ macro_rules! generate_logging_api { post_hook = (|log: &slog::Logger, result: &Result<_, _>| { slog::debug!(log, "client response"; "result" => ?result); }), + derives = [schemars::JsonSchema], ); }; } From e6ba5bf58aeeb108c999c8537df6b9ebcc5d5b2a Mon Sep 17 00:00:00 2001 From: "Adam H. Leventhal" Date: Tue, 18 Jan 2022 16:00:55 -0800 Subject: [PATCH 6/6] update for crucible-agent-client --- Cargo.lock | 1 + Cargo.toml | 3 --- nexus/Cargo.toml | 2 +- sled-agent/Cargo.toml | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ddd19adae3..9fbf71fb944 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -575,6 +575,7 @@ dependencies = [ [[package]] name = "crucible-agent-client" version = "0.0.1" +source = "git+https://github.com/oxidecomputer/crucible?rev=7e8d0c31#7e8d0c314adb43698e8a65d1c55abf2225d35c2c" dependencies = [ "anyhow", "percent-encoding", diff --git a/Cargo.toml b/Cargo.toml index 2438e6df956..040a8505a63 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,6 +69,3 @@ panic = "abort" [patch.crates-io.pq-sys] git = 'https://github.com/oxidecomputer/pq-sys' branch = "oxide/omicron" - -[patch."https://github.com/oxidecomputer/crucible"] -crucible-agent-client = { path = "../crucible/agent-client" } diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index a4e6326d007..1d5f105f643 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -13,7 +13,7 @@ async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", async-trait = "0.1.51" bb8 = "0.7.1" cookie = "0.16" -crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "de022b8a" } +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "7e8d0c31" } # Tracking pending 2.0 version. diesel = { git = "https://github.com/diesel-rs/diesel", rev = "ce77c382", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] } futures = "0.3.18" diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 44ffa145b4e..26057fa6358 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -12,7 +12,7 @@ bytes = "1.1" cfg-if = "1.0" chrono = { version = "0.4", features = [ "serde" ] } # Only used by the simulated sled agent. -crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "de022b8a" } +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "7e8d0c31" } dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] } futures = "0.3.18" ipnetwork = "0.18"