From 123ef3ec1486e74d00bb6adcef9bac4bc5049c13 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 2 Dec 2021 10:44:57 -0500 Subject: [PATCH 1/3] Split 'packaging' tool out of omicron-common --- Cargo.lock | 40 +-- Cargo.toml | 2 + common/Cargo.toml | 77 +---- common/src/bin/omicron-package.rs | 458 ------------------------------ 4 files changed, 38 insertions(+), 539 deletions(-) delete mode 100644 common/src/bin/omicron-package.rs diff --git a/Cargo.lock b/Cargo.lock index 54d55c9b948..a6d168a85ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -444,7 +444,7 @@ dependencies = [ [[package]] name = "crucible" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?branch=main#9df1c1230f125e4ea20700d9f0630f90b18a3c70" +source = "git+https://github.com/oxidecomputer/crucible?branch=main#cda0bf8b0fd8e53566d1918b4d14824103a1d410" dependencies = [ "aes", "aes-gcm-siv", @@ -474,7 +474,7 @@ dependencies = [ [[package]] name = "crucible-common" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?branch=main#9df1c1230f125e4ea20700d9f0630f90b18a3c70" +source = "git+https://github.com/oxidecomputer/crucible?branch=main#cda0bf8b0fd8e53566d1918b4d14824103a1d410" dependencies = [ "anyhow", "serde", @@ -488,7 +488,7 @@ dependencies = [ [[package]] name = "crucible-protocol" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?branch=main#9df1c1230f125e4ea20700d9f0630f90b18a3c70" +source = "git+https://github.com/oxidecomputer/crucible?branch=main#cda0bf8b0fd8e53566d1918b4d14824103a1d410" dependencies = [ "anyhow", "bincode", @@ -502,7 +502,7 @@ dependencies = [ [[package]] name = "crucible-scope" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?branch=main#9df1c1230f125e4ea20700d9f0630f90b18a3c70" +source = "git+https://github.com/oxidecomputer/crucible?branch=main#cda0bf8b0fd8e53566d1918b4d14824103a1d410" dependencies = [ "anyhow", "futures", @@ -1364,9 +1364,6 @@ name = "ipnet" version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" -dependencies = [ - "serde", -] [[package]] name = "ipnetwork" @@ -1722,7 +1719,6 @@ version = "0.1.0" dependencies = [ "anyhow", "api_identity", - "async-trait", "backoff", "chrono", "dropshot", @@ -1730,14 +1726,10 @@ dependencies = [ "futures", "http", "hyper", - "ipnet", "ipnetwork", "macaddr", "parse-display", - "percent-encoding", "progenitor", - "propolis-server", - "rayon", "reqwest", "ring", "schemars", @@ -1750,14 +1742,10 @@ dependencies = [ "smf", "steno", "structopt", - "tar", - "tempfile", "thiserror", "tokio", "tokio-postgres", - "toml", "uuid", - "walkdir", ] [[package]] @@ -1820,6 +1808,26 @@ dependencies = [ "uuid", ] +[[package]] +name = "omicron-package" +version = "0.1.0" +dependencies = [ + "anyhow", + "omicron-common", + "propolis-server", + "rayon", + "reqwest", + "serde", + "serde_derive", + "smf", + "structopt", + "tar", + "thiserror", + "tokio", + "toml", + "walkdir", +] + [[package]] name = "omicron-rpaths" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 174c5070bf3..336b9e22b90 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "nexus", "nexus/src/db/db-macros", "nexus-client", + "package", "rpaths", "sled-agent", "sled-agent-client", @@ -21,6 +22,7 @@ default-members = [ "common", "nexus", "nexus/src/db/db-macros", + "package", "rpaths", "sled-agent", "sled-agent-client", diff --git a/common/Cargo.toml b/common/Cargo.toml index ecce8b577f2..db0bdc58fcc 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -6,87 +6,34 @@ license = "MPL-2.0" [dependencies] anyhow = "1.0" -async-trait = "0.1.51" +api_identity = { path = "../api_identity" } +backoff = { version = "0.3.0", features = [ "tokio" ] } +chrono = { version = "0.4", features = [ "serde" ] } +dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main" } futures = "0.3.18" http = "0.2.5" hyper = "0.14" ipnetwork = "0.18" -propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "00ec8cf18f6a2311b0907f0b16b0ff8a327944d1" } -rayon = "1.5" +macaddr = { version = "1.0.1", features = [ "serde_std" ] } reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] } ring = "0.16" +schemars = { version = "0.8", features = [ "chrono", "uuid" ] } +serde = { version = "1.0", features = [ "derive" ] } serde_derive = "1.0" serde_json = "1.0" serde_with = "1.11.0" +slog = { version = "2.5", features = [ "max_level_trace", "release_max_level_debug" ] } smf = "0.2" +steno = { git = "https://github.com/oxidecomputer/steno", branch = "main" } structopt = "0.3" -tar = "0.4" -tempfile = "3.0" thiserror = "1.0" -toml = "0.5.6" -walkdir = "2.3" +tokio = { version = "1.14", features = [ "full" ] } +tokio-postgres = { version = "0.7", features = [ "with-chrono-0_4", "with-uuid-0_8" ] } +uuid = { version = "0.8", features = [ "serde", "v4" ] } parse-display = "0.5.3" progenitor = { git = "https://github.com/oxidecomputer/progenitor" } -percent-encoding = "2.1.0" - -[dependencies.api_identity] -path = "../api_identity" - -[dependencies.backoff] -version = "0.3.0" -features = [ "tokio" ] - -[dependencies.chrono] -version = "0.4" -features = [ "serde" ] - -[dependencies.dropshot] -git = "https://github.com/oxidecomputer/dropshot" -branch = "main" - -[dependencies.ipnet] -version = "2.3.1" -features = [ "serde" ] - -[dependencies.macaddr] -version = "1.0.1" -features = [ "serde_std" ] - -[dependencies.schemars] -version = "0.8" -features = [ "chrono", "uuid" ] - -[dependencies.serde] -version = "1.0" -features = [ "derive" ] - -[dependencies.slog] -version = "2.5" -features = [ "max_level_trace", "release_max_level_debug" ] - -[dependencies.steno] -git = "https://github.com/oxidecomputer/steno" -branch = "main" - -[dependencies.tokio] -version = "1.14" -features = [ "full" ] - -[dependencies.tokio-postgres] -version = "0.7" -features = [ "with-chrono-0_4", "with-uuid-0_8" ] - -[dependencies.uuid] -version = "0.8" -features = [ "serde", "v4" ] [dev-dependencies] expectorate = "1.0.4" serde_urlencoded = "0.7.0" tokio = { version = "1.14", features = [ "test-util" ] } - -# Disable doc builds by default for our binaries to work around issue -# rust-lang/cargo#8373. These docs would not be very useful anyway. -[[bin]] -name = "omicron-package" -doc = false diff --git a/common/src/bin/omicron-package.rs b/common/src/bin/omicron-package.rs deleted file mode 100644 index 20b440f6c30..00000000000 --- a/common/src/bin/omicron-package.rs +++ /dev/null @@ -1,458 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -/*! - * Utility for bundling target binaries as tarfiles. - */ - -use omicron_common::packaging::sha256_digest; - -use anyhow::{anyhow, bail, Context, Result}; -use rayon::prelude::*; -use reqwest; -use serde_derive::Deserialize; -use std::collections::{BTreeMap, HashMap}; -use std::env; -use std::fs::{create_dir_all, OpenOptions}; -use std::path::{Path, PathBuf}; -use std::process::Command; -use structopt::StructOpt; -use tar::Builder; -use thiserror::Error; -use tokio::io::AsyncWriteExt; - -const S3_BUCKET: &str = "https://oxide-omicron-build.s3.amazonaws.com"; - -// Name for the directory component where additional packaged files are stored. -const PKG: &str = "pkg"; -// Name for the directory component where downloaded blobs are stored. -const BLOB: &str = "blob"; - -#[derive(Debug, StructOpt)] -enum SubCommand { - /// Builds the packages specified in a manifest, and places them into a target - /// directory. - Package { - /// The output directory, where artifacts should be placed. - /// - /// Defaults to "out". - #[structopt(long = "out", default_value = "out")] - artifact_dir: PathBuf, - - /// The binary profile to package. - /// - /// True: release, False: debug (default). - #[structopt( - short, - long, - help = "True if bundling release-mode binaries" - )] - release: bool, - }, - /// Installs the packages to a target machine. - Install { - /// The directory from which artifacts will be pulled. - /// - /// Should match the format from the Package subcommand. - #[structopt(long = "in", default_value = "out")] - artifact_dir: PathBuf, - - /// The directory to which artifacts will be installed. - /// - /// Defaults to "/opt/oxide". - #[structopt(long = "out", default_value = "/opt/oxide")] - install_dir: PathBuf, - }, - /// Removes the packages from the target machine. - Uninstall { - /// The directory from which artifacts were be pulled. - /// - /// Should match the format from the Package subcommand. - #[structopt(long = "in", default_value = "out")] - artifact_dir: PathBuf, - - /// The directory to which artifacts were installed. - /// - /// Defaults to "/opt/oxide". - #[structopt(long = "out", default_value = "/opt/oxide")] - install_dir: PathBuf, - }, -} - -#[derive(Debug, StructOpt)] -#[structopt(name = "packaging tool")] -struct Args { - /// The path to the build manifest TOML file. - /// - /// Defaults to "package-manifest.toml". - #[structopt( - short, - long, - default_value = "package-manifest.toml", - help = "Path to package manifest toml file" - )] - manifest: PathBuf, - - #[structopt(subcommand)] - subcommand: SubCommand, -} - -fn build_rust_package(package: &str, release: bool) -> Result<()> { - let mut cmd = Command::new("cargo"); - // We rely on the rust-toolchain.toml file for toolchain information, - // rather than specifying one within the packaging tool. - cmd.arg("build").arg("-p").arg(package); - if release { - cmd.arg("--release"); - } - let status = - cmd.status().context(format!("Failed to run command: ({:?})", cmd))?; - if !status.success() { - bail!("Failed to build package: {}", package); - } - - Ok(()) -} - -/// Errors which may be returned when parsing the server configuration. -#[derive(Error, Debug)] -enum ParseError { - #[error("Cannot parse toml: {0}")] - Toml(#[from] toml::de::Error), - #[error("IO error: {0}")] - Io(#[from] std::io::Error), -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "lowercase")] -enum Build { - /// The package is a rust binary, which should be compiled and extracted. - Rust, -} - -#[derive(Deserialize, Debug)] -struct PackageInfo { - // The name of the compiled binary to be used. - // TODO: Could be extrapolated to "produced build artifacts", we don't - // really care about the individual binary file. - binary_name: String, - // The name of the service name to be used on the target OS. - // Also used as a lookup within the "smf/" directory. - service_name: String, - // Instructs how this package should be compiled. - build: Build, - // A list of blobs from the Omicron build S3 bucket which should be placed - // within this package. - blobs: Option>, - // If present, this service is necessary for bootstrapping, and should - // be durably installed to the target system. - // - // This is typically set to None - most services are bootstrapped - // by the bootstrap agent. - bootstrap: Option, -} - -impl PackageInfo { - fn binary_name(&self) -> &str { - &self.binary_name - } - - // Returns the path to the compiled binary. - fn binary_path(&self, release: bool) -> PathBuf { - match &self.build { - Build::Rust => format!( - "target/{}/{}", - if release { "release" } else { "debug" }, - self.binary_name() - ) - .into(), - } - } - - // Builds the requested package. - fn build(&self, package_name: &str, release: bool) -> Result<()> { - match &self.build { - Build::Rust => build_rust_package(package_name, release), - } - } -} - -#[derive(Deserialize, Debug)] -struct Config { - // Path to SMF manifest directory. - smf: PathBuf, - - #[serde(default, rename = "package")] - packages: BTreeMap, -} - -fn parse>(path: P) -> Result { - let contents = std::fs::read_to_string(path.as_ref())?; - let cfg = toml::from_str::(&contents)?; - Ok(cfg) -} - -async fn do_package( - config: &Config, - output_directory: &Path, - release: bool, -) -> Result<()> { - // Create the output directory, if it does not already exist. - create_dir_all(&output_directory) - .map_err(|err| anyhow!("Cannot create output directory: {}", err))?; - - // As we emit each package bundle, capture their digests for - // verification purposes. - let mut digests = HashMap::>::new(); - - for (package_name, package) in &config.packages { - println!("Building {}", package_name); - package.build(&package_name, release)?; - - let tarfile = - output_directory.join(format!("{}.tar", package.service_name)); - let file = OpenOptions::new() - .write(true) - .read(true) - .truncate(true) - .create(true) - .open(&tarfile) - .map_err(|err| anyhow!("Cannot create tarfile: {}", err))?; - - // Create an archive filled with: - // - The binary - // - The corresponding SMF directory - // - // TODO: We could add compression here, if we'd like? - let mut archive = Builder::new(file); - archive.mode(tar::HeaderMode::Deterministic); - - // Add binary - archive - .append_path_with_name( - package.binary_path(release), - &package.binary_name, - ) - .map_err(|err| { - anyhow!("Cannot append binary to tarfile: {}", err) - })?; - - // Add SMF directory - let smf_path: PathBuf = format!( - "{}/{}", - config.smf.to_string_lossy(), - package.service_name - ) - .into(); - add_path_to_archive(&mut archive, &smf_path, Path::new(PKG))?; - - // Add (and possibly download) blobs - add_blobs(&mut archive, package_name, package, output_directory) - .await?; - - let mut file = archive - .into_inner() - .map_err(|err| anyhow!("Failed to finalize archive: {}", err))?; - - // Once we've created the archive, acquire a digest which can - // later be used for verification. - let digest = sha256_digest(&mut file)?; - digests.insert(package.binary_name().into(), digest.as_ref().into()); - } - - let toml = toml::to_string(&digests)?; - std::fs::write(output_directory.join("digest.toml"), &toml)?; - Ok(()) -} - -// Adds all files within "path" to "archive". -// -// Within the archive, all files are renamed to "dst_prefix/". -fn add_path_to_archive( - archive: &mut Builder, - path: &Path, - dst_prefix: &Path, -) -> Result<()> { - for entry in walkdir::WalkDir::new(&path) { - let entry = - entry.map_err(|err| anyhow!("Cannot access entry: {}", err))?; - if entry.file_name().to_string_lossy().starts_with('.') { - // Omit hidden files - we don't want to include text editor - // artifacts in the tarfile. - continue; - } - let dst = dst_prefix.join(entry.path().strip_prefix(&path)?); - println!( - "Archiving {} -> {}", - entry.path().to_string_lossy(), - dst.to_string_lossy() - ); - archive - .append_path_with_name(entry.path(), dst) - .map_err(|err| anyhow!("Cannot append file to tarfile: {}", err))?; - } - Ok(()) -} - -async fn add_blobs( - archive: &mut Builder, - name: &String, - package: &PackageInfo, - output_directory: &Path, -) -> Result<()> { - if let Some(blobs) = &package.blobs { - let blobs_path = output_directory.join(name); - std::fs::create_dir_all(&blobs_path)?; - for blob in blobs { - let blob_path = blobs_path.join(blob); - // TODO: Check against hash, download if mismatch (i.e., - // corruption/update). - if !blob_path.exists() { - download(&blob.to_string_lossy(), &blob_path).await?; - } - } - add_path_to_archive(archive, blobs_path.as_path(), Path::new(BLOB))?; - } - Ok(()) -} - -// Downloads "source" from S3_BUCKET to "destination". -async fn download(source: &str, destination: &Path) -> Result<()> { - println!("Downloading {} to {}", source, destination.to_string_lossy()); - let response = reqwest::get(format!("{}/{}", S3_BUCKET, source)).await?; - let mut file = tokio::fs::File::create(destination).await?; - file.write_all(&response.bytes().await?).await?; - Ok(()) -} - -fn do_install( - config: &Config, - artifact_dir: &Path, - install_dir: &Path, -) -> Result<()> { - create_dir_all(&install_dir).map_err(|err| { - anyhow!("Cannot create installation directory: {}", err) - })?; - - // Move the digest of expected packages. - std::fs::copy( - artifact_dir.join("digest.toml"), - install_dir.join("digest.toml"), - )?; - - // Copy all packages to the install location in parallel. - let packages: Vec<(&String, &PackageInfo)> = - config.packages.iter().collect(); - packages.into_par_iter().try_for_each(|(_, package)| -> Result<()> { - let tarfile = - artifact_dir.join(format!("{}.tar", package.service_name)); - let src = tarfile.as_path(); - let dst = install_dir.join(src.strip_prefix(artifact_dir)?); - println!( - "Installing {} -> {}", - src.to_string_lossy(), - dst.to_string_lossy() - ); - std::fs::copy(&src, &dst)?; - Ok(()) - })?; - - // Ensure we start from a clean slate - remove all packages. - uninstall_all_packages(config); - - // Extract and install the bootstrap service, which itself extracts and - // installs other services. - for (_, package) in &config.packages { - if let Some(manifest) = &package.bootstrap { - let tar_path = - install_dir.join(format!("{}.tar", package.service_name)); - let service_path = install_dir.join(&package.service_name); - println!( - "Unpacking {} to {}", - tar_path.to_string_lossy(), - service_path.to_string_lossy() - ); - - let tar_file = std::fs::File::open(&tar_path)?; - let _ = std::fs::remove_dir_all(&service_path); - std::fs::create_dir_all(&service_path)?; - let mut archive = tar::Archive::new(tar_file); - archive.unpack(&service_path)?; - - let mut manifest_path = install_dir.to_path_buf(); - manifest_path.push(&package.service_name); - manifest_path.push(PKG); - manifest_path.push(manifest); - println!( - "Installing bootstrap service from {}", - manifest_path.to_string_lossy() - ); - smf::Config::import().run(&manifest_path)?; - } - } - - Ok(()) -} - -// Attempts to both disable and delete all requested packages. -fn uninstall_all_packages(config: &Config) { - for package in config.packages.values() { - let _ = smf::Adm::new() - .disable() - .synchronous() - .run(smf::AdmSelection::ByPattern(&[&package.service_name])); - let _ = smf::Config::delete().force().run(&package.service_name); - } -} - -fn remove_all_unless_already_removed>(path: P) -> Result<()> { - if let Err(e) = std::fs::remove_dir_all(path.as_ref()) { - match e.kind() { - std::io::ErrorKind::NotFound => {} - _ => bail!(e), - } - } - Ok(()) -} - -fn do_uninstall( - config: &Config, - artifact_dir: &Path, - install_dir: &Path, -) -> Result<()> { - println!("Uninstalling all packages"); - uninstall_all_packages(config); - println!("Removing: {}", artifact_dir.to_string_lossy()); - remove_all_unless_already_removed(artifact_dir)?; - println!("Removing: {}", install_dir.to_string_lossy()); - remove_all_unless_already_removed(install_dir)?; - Ok(()) -} - -#[tokio::main] -async fn main() -> Result<()> { - let args = Args::from_args_safe().map_err(|err| anyhow!(err))?; - let config = parse(&args.manifest)?; - - // Use a CWD that is the root of the Omicron repository. - if let Ok(manifest) = env::var("CARGO_MANIFEST_DIR") { - let manifest_dir = PathBuf::from(manifest); - let root = manifest_dir.parent().unwrap(); - env::set_current_dir(&root)?; - } - - match &args.subcommand { - SubCommand::Package { artifact_dir, release } => { - do_package(&config, &artifact_dir, *release).await?; - } - SubCommand::Install { artifact_dir, install_dir } => { - do_install(&config, &artifact_dir, &install_dir)?; - } - SubCommand::Uninstall { artifact_dir, install_dir } => { - do_uninstall(&config, &artifact_dir, &install_dir)?; - } - } - - Ok(()) -} From 3a092569514451fdfb0abf7b6e5a106050f81738 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 2 Dec 2021 10:46:31 -0500 Subject: [PATCH 2/3] Oops actually add the package crate --- package/.gitignore | 1 + package/Cargo.toml | 27 ++ package/src/bin/omicron-package.rs | 458 +++++++++++++++++++++++++++++ 3 files changed, 486 insertions(+) create mode 100644 package/.gitignore create mode 100644 package/Cargo.toml create mode 100644 package/src/bin/omicron-package.rs diff --git a/package/.gitignore b/package/.gitignore new file mode 100644 index 00000000000..ea8c4bf7f35 --- /dev/null +++ b/package/.gitignore @@ -0,0 +1 @@ +/target diff --git a/package/Cargo.toml b/package/Cargo.toml new file mode 100644 index 00000000000..dc975a17603 --- /dev/null +++ b/package/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "omicron-package" +version = "0.1.0" +edition = "2018" +license = "MPL-2.0" + +[dependencies] +anyhow = "1.0" +omicron-common = { path = "../common" } +propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "00ec8cf18f6a2311b0907f0b16b0ff8a327944d1" } +rayon = "1.5" +reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] } +serde = { version = "1.0", features = [ "derive" ] } +serde_derive = "1.0" +smf = "0.2" +structopt = "0.3" +tar = "0.4" +thiserror = "1.0" +tokio = { version = "1.14", features = [ "full" ] } +toml = "0.5.6" +walkdir = "2.3" + +# Disable doc builds by default for our binaries to work around issue +# rust-lang/cargo#8373. These docs would not be very useful anyway. +[[bin]] +name = "omicron-package" +doc = false diff --git a/package/src/bin/omicron-package.rs b/package/src/bin/omicron-package.rs new file mode 100644 index 00000000000..20b440f6c30 --- /dev/null +++ b/package/src/bin/omicron-package.rs @@ -0,0 +1,458 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +/*! + * Utility for bundling target binaries as tarfiles. + */ + +use omicron_common::packaging::sha256_digest; + +use anyhow::{anyhow, bail, Context, Result}; +use rayon::prelude::*; +use reqwest; +use serde_derive::Deserialize; +use std::collections::{BTreeMap, HashMap}; +use std::env; +use std::fs::{create_dir_all, OpenOptions}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use structopt::StructOpt; +use tar::Builder; +use thiserror::Error; +use tokio::io::AsyncWriteExt; + +const S3_BUCKET: &str = "https://oxide-omicron-build.s3.amazonaws.com"; + +// Name for the directory component where additional packaged files are stored. +const PKG: &str = "pkg"; +// Name for the directory component where downloaded blobs are stored. +const BLOB: &str = "blob"; + +#[derive(Debug, StructOpt)] +enum SubCommand { + /// Builds the packages specified in a manifest, and places them into a target + /// directory. + Package { + /// The output directory, where artifacts should be placed. + /// + /// Defaults to "out". + #[structopt(long = "out", default_value = "out")] + artifact_dir: PathBuf, + + /// The binary profile to package. + /// + /// True: release, False: debug (default). + #[structopt( + short, + long, + help = "True if bundling release-mode binaries" + )] + release: bool, + }, + /// Installs the packages to a target machine. + Install { + /// The directory from which artifacts will be pulled. + /// + /// Should match the format from the Package subcommand. + #[structopt(long = "in", default_value = "out")] + artifact_dir: PathBuf, + + /// The directory to which artifacts will be installed. + /// + /// Defaults to "/opt/oxide". + #[structopt(long = "out", default_value = "/opt/oxide")] + install_dir: PathBuf, + }, + /// Removes the packages from the target machine. + Uninstall { + /// The directory from which artifacts were be pulled. + /// + /// Should match the format from the Package subcommand. + #[structopt(long = "in", default_value = "out")] + artifact_dir: PathBuf, + + /// The directory to which artifacts were installed. + /// + /// Defaults to "/opt/oxide". + #[structopt(long = "out", default_value = "/opt/oxide")] + install_dir: PathBuf, + }, +} + +#[derive(Debug, StructOpt)] +#[structopt(name = "packaging tool")] +struct Args { + /// The path to the build manifest TOML file. + /// + /// Defaults to "package-manifest.toml". + #[structopt( + short, + long, + default_value = "package-manifest.toml", + help = "Path to package manifest toml file" + )] + manifest: PathBuf, + + #[structopt(subcommand)] + subcommand: SubCommand, +} + +fn build_rust_package(package: &str, release: bool) -> Result<()> { + let mut cmd = Command::new("cargo"); + // We rely on the rust-toolchain.toml file for toolchain information, + // rather than specifying one within the packaging tool. + cmd.arg("build").arg("-p").arg(package); + if release { + cmd.arg("--release"); + } + let status = + cmd.status().context(format!("Failed to run command: ({:?})", cmd))?; + if !status.success() { + bail!("Failed to build package: {}", package); + } + + Ok(()) +} + +/// Errors which may be returned when parsing the server configuration. +#[derive(Error, Debug)] +enum ParseError { + #[error("Cannot parse toml: {0}")] + Toml(#[from] toml::de::Error), + #[error("IO error: {0}")] + Io(#[from] std::io::Error), +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "lowercase")] +enum Build { + /// The package is a rust binary, which should be compiled and extracted. + Rust, +} + +#[derive(Deserialize, Debug)] +struct PackageInfo { + // The name of the compiled binary to be used. + // TODO: Could be extrapolated to "produced build artifacts", we don't + // really care about the individual binary file. + binary_name: String, + // The name of the service name to be used on the target OS. + // Also used as a lookup within the "smf/" directory. + service_name: String, + // Instructs how this package should be compiled. + build: Build, + // A list of blobs from the Omicron build S3 bucket which should be placed + // within this package. + blobs: Option>, + // If present, this service is necessary for bootstrapping, and should + // be durably installed to the target system. + // + // This is typically set to None - most services are bootstrapped + // by the bootstrap agent. + bootstrap: Option, +} + +impl PackageInfo { + fn binary_name(&self) -> &str { + &self.binary_name + } + + // Returns the path to the compiled binary. + fn binary_path(&self, release: bool) -> PathBuf { + match &self.build { + Build::Rust => format!( + "target/{}/{}", + if release { "release" } else { "debug" }, + self.binary_name() + ) + .into(), + } + } + + // Builds the requested package. + fn build(&self, package_name: &str, release: bool) -> Result<()> { + match &self.build { + Build::Rust => build_rust_package(package_name, release), + } + } +} + +#[derive(Deserialize, Debug)] +struct Config { + // Path to SMF manifest directory. + smf: PathBuf, + + #[serde(default, rename = "package")] + packages: BTreeMap, +} + +fn parse>(path: P) -> Result { + let contents = std::fs::read_to_string(path.as_ref())?; + let cfg = toml::from_str::(&contents)?; + Ok(cfg) +} + +async fn do_package( + config: &Config, + output_directory: &Path, + release: bool, +) -> Result<()> { + // Create the output directory, if it does not already exist. + create_dir_all(&output_directory) + .map_err(|err| anyhow!("Cannot create output directory: {}", err))?; + + // As we emit each package bundle, capture their digests for + // verification purposes. + let mut digests = HashMap::>::new(); + + for (package_name, package) in &config.packages { + println!("Building {}", package_name); + package.build(&package_name, release)?; + + let tarfile = + output_directory.join(format!("{}.tar", package.service_name)); + let file = OpenOptions::new() + .write(true) + .read(true) + .truncate(true) + .create(true) + .open(&tarfile) + .map_err(|err| anyhow!("Cannot create tarfile: {}", err))?; + + // Create an archive filled with: + // - The binary + // - The corresponding SMF directory + // + // TODO: We could add compression here, if we'd like? + let mut archive = Builder::new(file); + archive.mode(tar::HeaderMode::Deterministic); + + // Add binary + archive + .append_path_with_name( + package.binary_path(release), + &package.binary_name, + ) + .map_err(|err| { + anyhow!("Cannot append binary to tarfile: {}", err) + })?; + + // Add SMF directory + let smf_path: PathBuf = format!( + "{}/{}", + config.smf.to_string_lossy(), + package.service_name + ) + .into(); + add_path_to_archive(&mut archive, &smf_path, Path::new(PKG))?; + + // Add (and possibly download) blobs + add_blobs(&mut archive, package_name, package, output_directory) + .await?; + + let mut file = archive + .into_inner() + .map_err(|err| anyhow!("Failed to finalize archive: {}", err))?; + + // Once we've created the archive, acquire a digest which can + // later be used for verification. + let digest = sha256_digest(&mut file)?; + digests.insert(package.binary_name().into(), digest.as_ref().into()); + } + + let toml = toml::to_string(&digests)?; + std::fs::write(output_directory.join("digest.toml"), &toml)?; + Ok(()) +} + +// Adds all files within "path" to "archive". +// +// Within the archive, all files are renamed to "dst_prefix/". +fn add_path_to_archive( + archive: &mut Builder, + path: &Path, + dst_prefix: &Path, +) -> Result<()> { + for entry in walkdir::WalkDir::new(&path) { + let entry = + entry.map_err(|err| anyhow!("Cannot access entry: {}", err))?; + if entry.file_name().to_string_lossy().starts_with('.') { + // Omit hidden files - we don't want to include text editor + // artifacts in the tarfile. + continue; + } + let dst = dst_prefix.join(entry.path().strip_prefix(&path)?); + println!( + "Archiving {} -> {}", + entry.path().to_string_lossy(), + dst.to_string_lossy() + ); + archive + .append_path_with_name(entry.path(), dst) + .map_err(|err| anyhow!("Cannot append file to tarfile: {}", err))?; + } + Ok(()) +} + +async fn add_blobs( + archive: &mut Builder, + name: &String, + package: &PackageInfo, + output_directory: &Path, +) -> Result<()> { + if let Some(blobs) = &package.blobs { + let blobs_path = output_directory.join(name); + std::fs::create_dir_all(&blobs_path)?; + for blob in blobs { + let blob_path = blobs_path.join(blob); + // TODO: Check against hash, download if mismatch (i.e., + // corruption/update). + if !blob_path.exists() { + download(&blob.to_string_lossy(), &blob_path).await?; + } + } + add_path_to_archive(archive, blobs_path.as_path(), Path::new(BLOB))?; + } + Ok(()) +} + +// Downloads "source" from S3_BUCKET to "destination". +async fn download(source: &str, destination: &Path) -> Result<()> { + println!("Downloading {} to {}", source, destination.to_string_lossy()); + let response = reqwest::get(format!("{}/{}", S3_BUCKET, source)).await?; + let mut file = tokio::fs::File::create(destination).await?; + file.write_all(&response.bytes().await?).await?; + Ok(()) +} + +fn do_install( + config: &Config, + artifact_dir: &Path, + install_dir: &Path, +) -> Result<()> { + create_dir_all(&install_dir).map_err(|err| { + anyhow!("Cannot create installation directory: {}", err) + })?; + + // Move the digest of expected packages. + std::fs::copy( + artifact_dir.join("digest.toml"), + install_dir.join("digest.toml"), + )?; + + // Copy all packages to the install location in parallel. + let packages: Vec<(&String, &PackageInfo)> = + config.packages.iter().collect(); + packages.into_par_iter().try_for_each(|(_, package)| -> Result<()> { + let tarfile = + artifact_dir.join(format!("{}.tar", package.service_name)); + let src = tarfile.as_path(); + let dst = install_dir.join(src.strip_prefix(artifact_dir)?); + println!( + "Installing {} -> {}", + src.to_string_lossy(), + dst.to_string_lossy() + ); + std::fs::copy(&src, &dst)?; + Ok(()) + })?; + + // Ensure we start from a clean slate - remove all packages. + uninstall_all_packages(config); + + // Extract and install the bootstrap service, which itself extracts and + // installs other services. + for (_, package) in &config.packages { + if let Some(manifest) = &package.bootstrap { + let tar_path = + install_dir.join(format!("{}.tar", package.service_name)); + let service_path = install_dir.join(&package.service_name); + println!( + "Unpacking {} to {}", + tar_path.to_string_lossy(), + service_path.to_string_lossy() + ); + + let tar_file = std::fs::File::open(&tar_path)?; + let _ = std::fs::remove_dir_all(&service_path); + std::fs::create_dir_all(&service_path)?; + let mut archive = tar::Archive::new(tar_file); + archive.unpack(&service_path)?; + + let mut manifest_path = install_dir.to_path_buf(); + manifest_path.push(&package.service_name); + manifest_path.push(PKG); + manifest_path.push(manifest); + println!( + "Installing bootstrap service from {}", + manifest_path.to_string_lossy() + ); + smf::Config::import().run(&manifest_path)?; + } + } + + Ok(()) +} + +// Attempts to both disable and delete all requested packages. +fn uninstall_all_packages(config: &Config) { + for package in config.packages.values() { + let _ = smf::Adm::new() + .disable() + .synchronous() + .run(smf::AdmSelection::ByPattern(&[&package.service_name])); + let _ = smf::Config::delete().force().run(&package.service_name); + } +} + +fn remove_all_unless_already_removed>(path: P) -> Result<()> { + if let Err(e) = std::fs::remove_dir_all(path.as_ref()) { + match e.kind() { + std::io::ErrorKind::NotFound => {} + _ => bail!(e), + } + } + Ok(()) +} + +fn do_uninstall( + config: &Config, + artifact_dir: &Path, + install_dir: &Path, +) -> Result<()> { + println!("Uninstalling all packages"); + uninstall_all_packages(config); + println!("Removing: {}", artifact_dir.to_string_lossy()); + remove_all_unless_already_removed(artifact_dir)?; + println!("Removing: {}", install_dir.to_string_lossy()); + remove_all_unless_already_removed(install_dir)?; + Ok(()) +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::from_args_safe().map_err(|err| anyhow!(err))?; + let config = parse(&args.manifest)?; + + // Use a CWD that is the root of the Omicron repository. + if let Ok(manifest) = env::var("CARGO_MANIFEST_DIR") { + let manifest_dir = PathBuf::from(manifest); + let root = manifest_dir.parent().unwrap(); + env::set_current_dir(&root)?; + } + + match &args.subcommand { + SubCommand::Package { artifact_dir, release } => { + do_package(&config, &artifact_dir, *release).await?; + } + SubCommand::Install { artifact_dir, install_dir } => { + do_install(&config, &artifact_dir, &install_dir)?; + } + SubCommand::Uninstall { artifact_dir, install_dir } => { + do_uninstall(&config, &artifact_dir, &install_dir)?; + } + } + + Ok(()) +} From 814376e66efac790f1bd9a5cc9156b7ebeb727ca Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 2 Dec 2021 11:14:10 -0500 Subject: [PATCH 3/3] Explain propolis-server dep --- package/Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/package/Cargo.toml b/package/Cargo.toml index dc975a17603..5dcf30cd2dc 100644 --- a/package/Cargo.toml +++ b/package/Cargo.toml @@ -7,6 +7,9 @@ license = "MPL-2.0" [dependencies] anyhow = "1.0" omicron-common = { path = "../common" } +# We depend on the propolis-server here -- a binary, not a library -- to +# make it visible to the packaging tool, which can compile it and shove +# it in a tarball. propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "00ec8cf18f6a2311b0907f0b16b0ff8a327944d1" } rayon = "1.5" reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] }