From 793c0826ff367a866261fbf3f301470e47813b69 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Thu, 9 Dec 2021 02:30:46 -0500 Subject: [PATCH 1/7] Introduce `thing-flinger` `thing-flinger` is a remote deployment system for installing Omicron. It's intended to be used to simplify and shorten the testing cycle. More information is found in the [documentation](package/src/README.adoc). --- Cargo.lock | 25 + package/Cargo.toml | 6 + package/README.adoc | 140 ++++++ package/src/bin/deployment-example.toml | 42 ++ package/src/bin/omicron-package.rs | 77 +--- package/src/bin/thing-flinger.rs | 590 ++++++++++++++++++++++++ package/src/lib.rs | 77 ++++ 7 files changed, 887 insertions(+), 70 deletions(-) create mode 100644 package/README.adoc create mode 100644 package/src/bin/deployment-example.toml create mode 100644 package/src/bin/thing-flinger.rs create mode 100644 package/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index ce7d1d682f4..43ece8dc830 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -465,6 +465,20 @@ dependencies = [ "itertools", ] +[[package]] +name = "crossbeam" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae5588f6b3c3cb05239e90bd110f257254aecd01e4635400391aeae07497845" +dependencies = [ + "cfg-if", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + [[package]] name = "crossbeam-channel" version = "0.5.1" @@ -499,6 +513,16 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b10ddc024425c88c2ad148c1b0fd53f4c6d38db9697c9f1588381212fa657c9" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.5" @@ -1986,6 +2010,7 @@ name = "omicron-package" version = "0.1.0" dependencies = [ "anyhow", + "crossbeam", "omicron-common", "propolis-server", "rayon", diff --git a/package/Cargo.toml b/package/Cargo.toml index 5dcf30cd2dc..051fd62d4e0 100644 --- a/package/Cargo.toml +++ b/package/Cargo.toml @@ -6,7 +6,9 @@ license = "MPL-2.0" [dependencies] anyhow = "1.0" +crossbeam = "0.8" omicron-common = { path = "../common" } + # We depend on the propolis-server here -- a binary, not a library -- to # make it visible to the packaging tool, which can compile it and shove # it in a tarball. @@ -28,3 +30,7 @@ walkdir = "2.3" [[bin]] name = "omicron-package" doc = false + +[[bin]] +name = "thing-flinger" +doc = false diff --git a/package/README.adoc b/package/README.adoc new file mode 100644 index 00000000000..3f6acb7e520 --- /dev/null +++ b/package/README.adoc @@ -0,0 +1,140 @@ +Omicron is a complex piece of software consisting of many build and install-time dependencies. It's +intended to run primarily on illumos based systems, and as such is built to use runtime facilities +of illumos, such as https://illumos.org/man/5/smf[SMF]. Furthermore, Omicron is fundamentally a +distributed system, with its components intended to run on multiple servers communicating over the +network. In order to secure the system, certain cryptographic primitives, such as asymmetric key +pairs and shared secrets are required. Due to the nature of these cryptographic primitives, there is +a requirement for the distribution or creation of files unique to a specific server, such that no +other server has access to those files. Examples of this are private keys, and threshold key +shares, although other non-cryptographic unique files may also become necessary over time. + +In order to satisfy the above requirements of building and deploying a complex distributed system +consisting of unique, private files, two CLI tools have been created: + + . link:src/bin/omicron-package.rs[omicron-package] - build, package, install on local machine + . link:src/bin/thing-flinger.rs[thing-flinger] - build, package, deploy to remote machines + + +If a user is working on their local illumos based machine, and only wants to run +omicron in single node mode, they should follow the install instruction in +the link:../README.adoc[Omicron README] and use `omicron-package`. If the user +wishes for a more complete workflow, where they can code on their local laptop, +use a remote build machine, and install to multiple machines for a more realistic +deployment, they should use `thing-flinger`. + +The remainder of this document will describe a typical workflow for using +thing-flinger, pointing out room for improvement. + +== Environment and Configuration + + + +------------------+ +------------------+ + | | | | + | | | | + | Client |----------------> Builder | + | | | | + | | | | + +------------------+ +------------------+ + | + | + | + | + +---------------------------+--------------------------+ + | | | + | | | + | | | + +--------v---------+ +---------v--------+ +---------v--------+ + | | | | | | + | | | | | | + | Deployed Server | | Deployed Server | | Deployed Server | + | | | | | | + | | | | | | + +------------------+ +------------------+ +------------------+ + + +`thing-flinger` defines three types of nodes: + * Client - Where a user typically edits their code and runs thing-flinger. This can run any OS. + * Builder - A Helios box where Omicron is built and packaged + * Deployment - Helios machines where Omicron will be installed and run + +It's not at all necessary for these to be separate nodes. For example, a client and builder can be +the same machine, as long as it's a Helios box. Same goes for Builder and a deployment server. The +benefit of this separation though, is that it allows editing on something like a laptop, without +having to worry about setting up a development environment on an illumos based host. + +Machine topology is configured in a `TOML` file that is passed on the command line. All illumos +machines are listed under `servers`, and just the names are used to for configuring a builder and +deployment servers. An link:src/bin/deployment-example.toml[example] is provided. + +Thing flinger works over SSH, and so the user must have the public key of their client configured +for their account on all servers. SSH agent forwarding is used to prevent the need for the keys of +the builder to also be on the other servers, thus minimizing needed server configuration. + +== Typical Workflow + +=== Prerequisites + +Ensure you have an account on all illumos boxes, with the client public key in +`~/.ssh/authorized_keys`. + +.The build machine should have a rust and cargo installed, and it's probably a good idea to also have +all the dependencies for Omicron installed. Following the *prerequisites* in the +https://github.com/oxidecomputer/omicron/#build-and-run[Build and run] section of the main Omicron +README is probably a good idea. + +=== Command Based Workflow + +==== Build omicron-package on client. + +`cargo build -p omicron-package` + +==== sync +Use rsync to copy your source code to the builder. Note that this copies over your `.git` subdirectory on purpose so +that a branch can be configured for building with the `git_treeish` field in the toml `builder` +table. + +`./target/debug/thing-flinger -c sync` + +==== build-minimal +Build necessary parts of omicron on the builder, required for future use by thing-flinger. + +`./target/debug/thing-flinger -c build-minimal` + +==== package +Build and package omicron using `omicron-package` on the builder. + +`./target/debug/thing-flinger -c package` + +==== overlay +Create files that are unique to each deployment server. + +`./target/debug/thing-flinger -c overlay` + +==== install +Install omicron to all machines, in parallel. This consists copying the packaged omicron tarballs +along with overlay files, and omicron-package and it's manifest to a `staging` directory on each +deployment server, and then running omicron-package, installing overlay files, and restarting +services. + +`./target/debug/thing-flinger -c install` + +=== Current Limitations + +`thing-flinger` is an early prototype. It has served so far to demonstrate that unique files, +specifically secret shares, can be created and distributed over ssh, and that omicron can be +installed remotely using `omicron-package`. It is not currently complete enough to fully test a +distributed omicron setup, as the underlying dependencies are not configured yet. Specifically, +`CockroachDB` and perhaps `Clickhouse`, need to be configured to run in multiple server mode. It's +anticipated that the `overlay` feature of `thing-flinger` can be used to generate and distribute +configs for this. + +=== Design rationale + +`thing-flinger` is a command line program written in rust. It was written this way to build upon and +`omicron-package`, which is also in rust, as that is our default language of choice at Oxide. +`thing-flinger` is based around SSH, as that is the minimal viable requirement for a test tool such +as this. Additionally, it provides for the most straightforward implementation, and takes the least +effort to use securely. This particular implementation wraps the openssh ssh client via +`std::process::Command`, rather than using the `ssh2` crate, because ssh2, as a wrapper around +`libssh`, does not support agent-forwarding. + diff --git a/package/src/bin/deployment-example.toml b/package/src/bin/deployment-example.toml new file mode 100644 index 00000000000..c75098cdb80 --- /dev/null +++ b/package/src/bin/deployment-example.toml @@ -0,0 +1,42 @@ +# This manifest describes the servers that omicron will be installed to, along +# with any ancillary information specific to a given server. +# +# It is ingested by the `thing-flinger` tool. + +# This must be an absolute path +local_source = "/Users/ajs/oxide/omicron" + +[builder] +# `server` must refer to one of the `servers` in the servers table +server = "atrium" +omicron_path = "/home/andrew/oxide/omicron" +git_treeish = "thing-flinger" + +[deployment] +servers = ["sock", "buskin"] +rack_secret_threshold = 2 + +# Location where files to install will be placed before running +# `omicron-package install` +# +# Since usernames may vary per server, this directory is relative to the user's +# home directory. +staging_dir = "omicron_staging" + +[servers.tarz] +username = "ajs" +addr = "tarz.local" + +[servers.atrium] +username = "andrew" +addr = "atrium.eng.oxide.computer" + +[servers.sock] +username = "andrew" +addr = "sock.eng.oxide.computer" + +[servers.buskin] +username = "andrew" +addr = "buskin.eng.oxide.computer" + + diff --git a/package/src/bin/omicron-package.rs b/package/src/bin/omicron-package.rs index 10f1b64bb52..f9131400dba 100644 --- a/package/src/bin/omicron-package.rs +++ b/package/src/bin/omicron-package.rs @@ -9,6 +9,7 @@ use omicron_common::packaging::sha256_digest; use anyhow::{anyhow, bail, Context, Result}; +use omicron_package::{parse, SubCommand}; use rayon::prelude::*; use reqwest; use serde_derive::Deserialize; @@ -19,7 +20,6 @@ use std::path::{Path, PathBuf}; use std::process::Command; use structopt::StructOpt; use tar::Builder; -use thiserror::Error; use tokio::io::AsyncWriteExt; const S3_BUCKET: &str = "https://oxide-omicron-build.s3.amazonaws.com"; @@ -29,59 +29,6 @@ const PKG: &str = "pkg"; // Name for the directory component where downloaded blobs are stored. const BLOB: &str = "blob"; -#[derive(Debug, StructOpt)] -enum SubCommand { - /// Builds the packages specified in a manifest, and places them into a target - /// directory. - Package { - /// The output directory, where artifacts should be placed. - /// - /// Defaults to "out". - #[structopt(long = "out", default_value = "out")] - artifact_dir: PathBuf, - - /// The binary profile to package. - /// - /// True: release, False: debug (default). - #[structopt( - short, - long, - help = "True if bundling release-mode binaries" - )] - release: bool, - }, - /// Checks the packages specified in a manifest, without building. - Check, - /// Installs the packages to a target machine. - Install { - /// The directory from which artifacts will be pulled. - /// - /// Should match the format from the Package subcommand. - #[structopt(long = "in", default_value = "out")] - artifact_dir: PathBuf, - - /// The directory to which artifacts will be installed. - /// - /// Defaults to "/opt/oxide". - #[structopt(long = "out", default_value = "/opt/oxide")] - install_dir: PathBuf, - }, - /// Removes the packages from the target machine. - Uninstall { - /// The directory from which artifacts were be pulled. - /// - /// Should match the format from the Package subcommand. - #[structopt(long = "in", default_value = "out")] - artifact_dir: PathBuf, - - /// The directory to which artifacts were installed. - /// - /// Defaults to "/opt/oxide". - #[structopt(long = "out", default_value = "/opt/oxide")] - install_dir: PathBuf, - }, -} - #[derive(Debug, StructOpt)] #[structopt(name = "packaging tool")] struct Args { @@ -121,15 +68,6 @@ fn run_cargo_on_package( Ok(()) } -/// Errors which may be returned when parsing the server configuration. -#[derive(Error, Debug)] -enum ParseError { - #[error("Cannot parse toml: {0}")] - Toml(#[from] toml::de::Error), - #[error("IO error: {0}")] - Io(#[from] std::io::Error), -} - #[derive(Deserialize, Debug)] #[serde(rename_all = "lowercase")] enum Build { @@ -200,12 +138,6 @@ struct Config { packages: BTreeMap, } -fn parse>(path: P) -> Result { - let contents = std::fs::read_to_string(path.as_ref())?; - let cfg = toml::from_str::(&contents)?; - Ok(cfg) -} - async fn do_check(config: &Config) -> Result<()> { for (package_name, package) in &config.packages { println!("Checking {}", package_name); @@ -356,6 +288,11 @@ fn do_install( anyhow!("Cannot create installation directory: {}", err) })?; + println!( + "Copying digest.toml from {} to {}", + artifact_dir.to_string_lossy(), + install_dir.to_string_lossy() + ); // Move the digest of expected packages. std::fs::copy( artifact_dir.join("digest.toml"), @@ -454,7 +391,7 @@ fn do_uninstall( #[tokio::main] async fn main() -> Result<()> { let args = Args::from_args_safe().map_err(|err| anyhow!(err))?; - let config = parse(&args.manifest)?; + let config = parse::<_, Config>(&args.manifest)?; // Use a CWD that is the root of the Omicron repository. if let Ok(manifest) = env::var("CARGO_MANIFEST_DIR") { diff --git a/package/src/bin/thing-flinger.rs b/package/src/bin/thing-flinger.rs new file mode 100644 index 00000000000..53bae34825b --- /dev/null +++ b/package/src/bin/thing-flinger.rs @@ -0,0 +1,590 @@ +use omicron_package::{parse, SubCommand as PackageSubCommand}; + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Write; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{anyhow, Context, Result}; +use crossbeam::thread::{self, ScopedJoinHandle}; +use serde_derive::Deserialize; +use structopt::StructOpt; +use thiserror::Error; + +#[derive(Deserialize, Debug)] +struct Builder { + pub server: String, + pub omicron_path: String, + pub git_treeish: String, +} + +// A server on which an omicron package is deployed +#[derive(Deserialize, Debug)] +struct Server { + pub username: String, + pub addr: String, +} + +#[derive(Deserialize, Debug)] +struct Deployment { + pub servers: BTreeSet, + pub rack_secret_threshold: usize, + pub staging_dir: String, +} + +#[derive(Debug, Deserialize)] +struct Config { + pub local_source: String, + pub builder: Builder, + pub servers: BTreeMap, + pub deployment: Deployment, +} + +fn parse_into_set(src: &str) -> BTreeSet { + src.split_whitespace().map(|s| s.to_owned()).collect() +} + +#[derive(Debug, StructOpt)] +enum SubCommand { + /// Run the given command on the given servers, or all servers if none are + /// given. + /// + /// Be careful! + Exec { + /// The command to run + #[structopt(short, long)] + cmd: String, + + /// The servers to run the command on + #[structopt(short, long, parse(from_str = parse_into_set))] + servers: Option>, + }, + + /// Sync our local source to the build host + Sync, + + /// Build omicron-package and everything needed to run thing-flinger + /// commands on the build host. + /// + /// Package always builds everything, but it can be set in release mode, and + /// we expect the existing tools to run from 'target/debug'. Additionally, + // you can't run `Package` until you have actually built `omicron-package`, + // which `BuildMinimal` does. + BuildMinimal, + + /// Use all subcommands from omicron-package + #[structopt(flatten)] + Package(PackageSubCommand), + + /// Create an overlay directory tree for each deployment server + /// + /// Each directory tree contains unique files for the given server that will + /// be populated in the svc/pkg dir. + /// + /// This is a separate subcommand so that we can reconstruct overlays + /// without rebuilding or repackaging. + Overlay, +} + +#[derive(Debug, StructOpt)] +#[structopt(name = "thing-flinger")] +struct Args { + /// The path to the deployment manifest TOML file + #[structopt(short, long, help = "Path to deployment manifest toml file")] + config: PathBuf, + + #[structopt(subcommand)] + subcommand: SubCommand, +} + +/// Errors which can be returned when executing subcommands +#[derive(Error, Debug)] +enum FlingError { + #[error("Servers not listed in configuration: {0:?}")] + InvalidServers(Vec), + + /// The parameter should be the name of the argument that could not be + /// properly converted to a string. + #[error("{0} is not valid UTF-8")] + BadString(String), + + /// Failed to rsync omicron to build host + #[error("Failed to sync {src} with {dst}")] + FailedSync { src: String, dst: String }, +} + +fn validate_servers( + chosen: &BTreeSet, + all: &BTreeMap, +) -> Result<(), FlingError> { + let all = all.keys().cloned().collect(); + let diff: Vec = chosen.difference(&all).cloned().collect(); + if !diff.is_empty() { + Err(FlingError::InvalidServers(diff)) + } else { + Ok(()) + } +} + +// TODO: run in parallel when that option is given +fn do_exec( + config: &Config, + cmd: String, + servers: Option>, +) -> Result<()> { + if servers.is_some() { + validate_servers(&servers.as_ref().unwrap(), &config.servers)?; + + for name in &servers.unwrap() { + let server = &config.servers[name]; + ssh_exec(&server, &cmd, false)?; + } + } else { + for (_, server) in config.servers.iter() { + ssh_exec(&server, &cmd, false)?; + } + } + Ok(()) +} + +fn do_sync(config: &Config) -> Result<()> { + let server = + config.servers.get(&config.builder.server).ok_or_else(|| { + FlingError::InvalidServers(vec![config.builder.server.clone()]) + })?; + let src = config.local_source.clone() + "/"; + let dst = format!( + "{}@{}:{}", + server.username, server.addr, config.builder.omicron_path + ); + let mut cmd = Command::new("rsync"); + cmd.arg("-az") + .arg("-e") + .arg("ssh") + .arg("--delete") + .arg("--progress") + .arg("--exclude") + .arg("target/") + .arg("--exclude") + .arg("out/") + .arg("--exclude") + .arg("cockroachdb/") + .arg("--exclude") + .arg("clickhouse/") + .arg("--exclude") + .arg("*.swp") + .arg("--out-format") + .arg("File changed: %o %t %f") + .arg(&src) + .arg(&dst); + let status = + cmd.status().context(format!("Failed to run command: ({:?})", cmd))?; + if !status.success() { + return Err(FlingError::FailedSync { src, dst }.into()); + } + + Ok(()) +} + +// Build omicron-package and omicron-sled-agent on the builder +// +// We need to build omicron-sled-agent for overlay file generation +fn do_build_minimal(config: &Config) -> Result<()> { + let server = &config.servers[&config.builder.server]; + let cmd = format!( + "cd {} && git checkout {} && cargo build -p {} -p {}", + config.builder.omicron_path, + config.builder.git_treeish, + "omicron-package", + "omicron-sled-agent" + ); + ssh_exec(&server, &cmd, false) +} + +fn do_package( + config: &Config, + artifact_dir: PathBuf, + release: bool, +) -> Result<()> { + let server = &config.servers[&config.builder.server]; + let mut cmd = String::new(); + let mut release_flag = ""; + if release { + release_flag = "--release"; + } + let cmd_path = "./target/debug/omicron-package"; + let artifact_dir = artifact_dir + .to_str() + .ok_or_else(|| FlingError::BadString("artifact_dir".to_string()))?; + + // We use a bash login shell to get a proper environment, so we have a path to + // postgres, and $DEP_PQ_LIBDIRS is filled in. This is required for building + // nexus. + // + // See https://github.com/oxidecomputer/omicron/blob/8757ec542ea4ffbadd6f26094ed4ba357715d70d/rpaths/src/lib.rs + write!( + &mut cmd, + "bash -lc 'cd {} && git checkout {} && {} package --out {} {}'", + config.builder.omicron_path, + config.builder.git_treeish, + cmd_path, + &artifact_dir, + release_flag + )?; + + ssh_exec(&server, &cmd, false) +} + +fn do_check(config: &Config) -> Result<()> { + let server = &config.servers[&config.builder.server]; + let mut cmd = String::new(); + let cmd_path = "./target/debug/omicron-package"; + + write!( + &mut cmd, + "bash -lc 'cd {} && git checkout {} && {} check'", + config.builder.omicron_path, config.builder.git_treeish, cmd_path, + )?; + + ssh_exec(&server, &cmd, false) +} + +fn do_uninstall( + config: &Config, + artifact_dir: PathBuf, + install_dir: PathBuf, +) -> Result<()> { + let mut deployment_src = PathBuf::from(&config.deployment.staging_dir); + deployment_src.push(&artifact_dir); + for server_name in &config.deployment.servers { + let server = &config.servers[server_name]; + // Run `omicron-package uninstall` on the deployment server + let cmd = format!( + "cd ~/{} && pfexec ./omicron-package uninstall --in ~/{} --out {}", + config.deployment.staging_dir, + deployment_src.to_string_lossy(), + install_dir.to_string_lossy() + ); + println!("$ {}", cmd); + ssh_exec(&server, &cmd, true)?; + } + Ok(()) +} + +fn do_install(config: &Config, artifact_dir: &Path, install_dir: &Path) { + let builder = &config.servers[&config.builder.server]; + let mut pkg_dir = PathBuf::from(&config.builder.omicron_path); + pkg_dir.push(artifact_dir); + let pkg_dir = pkg_dir.to_string_lossy(); + let pkg_dir = &pkg_dir; + + thread::scope(|s| { + let mut handles = + Vec::<(String, ScopedJoinHandle<'_, Result<()>>)>::new(); + + // Spawn a thread for each server install + for server_name in &config.deployment.servers { + handles.push(( + server_name.to_owned(), + s.spawn(move |_| -> Result<()> { + single_server_install( + config, + &artifact_dir, + &install_dir, + &pkg_dir, + builder, + server_name, + ) + }), + )); + } + + // Join all the handles and print the install status + for (server_name, handle) in handles { + match handle.join() { + Ok(Ok(())) => { + println!("Install completed for server: {}", server_name) + } + Ok(Err(e)) => { + println!( + "Install failed for server: {} with error: {}", + server_name, e + ) + } + Err(_) => { + println!( + "Install failed for server: {}. Thread panicked.", + server_name + ) + } + } + } + }) + .unwrap(); +} + +fn do_overlay(config: &Config) -> Result<()> { + let mut root_path = PathBuf::from(&config.builder.omicron_path); + // TODO: This needs to match the artifact_dir in `package` + root_path.push("out/overlay"); + let server_dirs = dir_per_deploy_server(config, &root_path); + let server = &config.servers[&config.builder.server]; + overlay_sled_agent(&server, config, &server_dirs) +} + +fn overlay_sled_agent( + server: &Server, + config: &Config, + server_dirs: &[PathBuf], +) -> Result<()> { + let sled_agent_dirs: Vec = server_dirs + .iter() + .map(|dir| { + let mut dir = PathBuf::from(dir); + dir.push("sled-agent/pkg"); + dir + }) + .collect(); + + // Create directories on builder + let dirs = dir_string(&sled_agent_dirs); + let cmd = format!("sh -c 'for dir in {}; do mkdir -p $dir; done'", dirs); + + let cmd = format!( + "{} && cd {} && ./target/debug/sled-agent-overlay-files \ + --threshold {} --directories {}", + cmd, + config.builder.omicron_path, + config.deployment.rack_secret_threshold, + dirs + ); + ssh_exec(server, &cmd, false) +} + +fn single_server_install( + config: &Config, + artifact_dir: &Path, + install_dir: &Path, + pkg_dir: &str, + builder: &Server, + server_name: &str, +) -> Result<()> { + let server = &config.servers[server_name]; + + copy_package_artifacts_to_staging(config, pkg_dir, builder, server)?; + copy_omicron_package_binary_to_staging(config, builder, server)?; + copy_package_manifest_to_staging(config, builder, server)?; + run_omicron_package_from_staging( + config, + server, + &artifact_dir, + &install_dir, + )?; + copy_overlay_files_to_staging( + config, + pkg_dir, + builder, + server, + server_name, + )?; + install_overlay_files_from_staging(config, server, &install_dir)?; + restart_services(server) +} + +// Copy package artifacts as a result of `omicron-package package` from the +// builder to the deployment server staging directory. +fn copy_package_artifacts_to_staging( + config: &Config, + pkg_dir: &str, + builder: &Server, + destination: &Server, +) -> Result<()> { + let cmd = format!( + "rsync -avz -e 'ssh -o StrictHostKeyChecking=no' \ + --exclude overlay/ {} {}@{}:~/{}", + pkg_dir, + destination.username, + destination.addr, + config.deployment.staging_dir + ); + println!("$ {}", cmd); + ssh_exec(builder, &cmd, true) +} + +fn copy_omicron_package_binary_to_staging( + config: &Config, + builder: &Server, + destination: &Server, +) -> Result<()> { + let mut bin_path = PathBuf::from(&config.builder.omicron_path); + bin_path.push("target/debug/omicron-package"); + let cmd = format!( + "rsync -avz {} {}@{}:~/{}", + bin_path.to_string_lossy(), + destination.username, + destination.addr, + config.deployment.staging_dir + ); + println!("$ {}", cmd); + ssh_exec(builder, &cmd, true) +} + +fn copy_package_manifest_to_staging( + config: &Config, + builder: &Server, + destination: &Server, +) -> Result<()> { + let mut path = PathBuf::from(&config.builder.omicron_path); + path.push("package-manifest.toml"); + let cmd = format!( + "rsync {} {}@{}:~/{}", + path.to_string_lossy(), + destination.username, + destination.addr, + config.deployment.staging_dir + ); + println!("$ {}", cmd); + ssh_exec(builder, &cmd, true) +} + +fn run_omicron_package_from_staging( + config: &Config, + destination: &Server, + artifact_dir: &Path, + install_dir: &Path, +) -> Result<()> { + let mut deployment_src = PathBuf::from(&config.deployment.staging_dir); + deployment_src.push(&artifact_dir); + + // Run `omicron-package install` on the deployment server + let cmd = format!( + "cd ~/{} && pfexec ./omicron-package install --in ~/{} --out {}", + config.deployment.staging_dir, + deployment_src.to_string_lossy(), + install_dir.to_string_lossy() + ); + println!("$ {}", cmd); + ssh_exec(destination, &cmd, true) +} + +fn copy_overlay_files_to_staging( + config: &Config, + pkg_dir: &str, + builder: &Server, + destination: &Server, + destination_name: &str, +) -> Result<()> { + let cmd = format!( + "rsync -avz {}/overlay/{}/ {}@{}:~/{}/overlay/", + pkg_dir, + destination_name, + destination.username, + destination.addr, + config.deployment.staging_dir + ); + println!("$ {}", cmd); + ssh_exec(builder, &cmd, true) +} + +fn install_overlay_files_from_staging( + config: &Config, + destination: &Server, + install_dir: &Path, +) -> Result<()> { + let cmd = format!( + "pfexec cp -r ~/{}/overlay/* {}", + config.deployment.staging_dir, + install_dir.to_string_lossy() + ); + println!("$ {}", cmd); + ssh_exec(&destination, &cmd, false) +} + +// For now, we just restart sled-agent, as that's the only service with an +// overlay file. +fn restart_services(destination: &Server) -> Result<()> { + ssh_exec(destination, "svcadm restart sled-agent", false) +} + +fn dir_string(dirs: &[PathBuf]) -> String { + dirs.iter().map(|dir| dir.to_string_lossy().to_string() + " ").collect() +} + +fn dir_per_deploy_server(config: &Config, root: &Path) -> Vec { + config + .deployment + .servers + .iter() + .map(|server_dir| { + let mut dir = PathBuf::from(root); + dir.push(server_dir); + dir + }) + .collect() +} + +fn ssh_exec( + server: &Server, + remote_cmd: &str, + forward_agent: bool, +) -> Result<()> { + // Source .profile, so we have access to cargo. Rustup installs knowledge + // about the cargo path here. + let remote_cmd = String::from(". ~/.profile && ") + remote_cmd; + let auth_sock = std::env::var("SSH_AUTH_SOCK")?; + let mut cmd = Command::new("ssh"); + if forward_agent { + cmd.arg("-A"); + } + cmd.arg("-o") + .arg("StrictHostKeyChecking=no") + .arg("-l") + .arg(&server.username) + .arg(&server.addr) + .arg(&remote_cmd); + cmd.env("SSH_AUTH_SOCK", auth_sock); + cmd.status() + .context(format!("Failed to run {} on {}", remote_cmd, server.addr))?; + + Ok(()) +} + +fn main() -> Result<()> { + let args = Args::from_args_safe().map_err(|err| anyhow!(err))?; + let config = parse::<_, Config>(args.config)?; + + validate_servers(&config.deployment.servers, &config.servers)?; + validate_servers( + &BTreeSet::from([config.builder.server.clone()]), + &config.servers, + )?; + + match args.subcommand { + SubCommand::Exec { cmd, servers } => { + do_exec(&config, cmd, servers)?; + } + SubCommand::Sync => do_sync(&config)?, + SubCommand::BuildMinimal => do_build_minimal(&config)?, + SubCommand::Package(PackageSubCommand::Package { + artifact_dir, + release, + }) => { + do_package(&config, artifact_dir, release)?; + } + SubCommand::Package(PackageSubCommand::Install { + artifact_dir, + install_dir, + }) => { + do_install(&config, &artifact_dir, &install_dir); + } + SubCommand::Package(PackageSubCommand::Uninstall { + artifact_dir, + install_dir, + }) => { + do_uninstall(&config, artifact_dir, install_dir)?; + } + SubCommand::Package(PackageSubCommand::Check) => do_check(&config)?, + SubCommand::Overlay => do_overlay(&config)?, + } + Ok(()) +} diff --git a/package/src/lib.rs b/package/src/lib.rs new file mode 100644 index 00000000000..f4a84573fab --- /dev/null +++ b/package/src/lib.rs @@ -0,0 +1,77 @@ +//! Common code shared between `omicron-package` and `thing-flinger` binaries. + +use serde::de::DeserializeOwned; +use std::path::Path; +use std::path::PathBuf; +use structopt::StructOpt; +use thiserror::Error; + +/// Errors which may be returned when parsing the server configuration. +#[derive(Error, Debug)] +pub enum ParseError { + #[error("Cannot parse toml: {0}")] + Toml(#[from] toml::de::Error), + #[error("IO error: {0}")] + Io(#[from] std::io::Error), +} + +pub fn parse, C: DeserializeOwned>( + path: P, +) -> Result { + let contents = std::fs::read_to_string(path.as_ref())?; + let cfg = toml::from_str::(&contents)?; + Ok(cfg) +} + +#[derive(Debug, StructOpt)] +pub enum SubCommand { + /// Builds the packages specified in a manifest, and places them into a target + /// directory. + Package { + /// The output directory, where artifacts should be placed. + /// + /// Defaults to "out". + #[structopt(long = "out", default_value = "out")] + artifact_dir: PathBuf, + + /// The binary profile to package. + /// + /// True: release, False: debug (default). + #[structopt( + short, + long, + help = "True if bundling release-mode binaries" + )] + release: bool, + }, + /// Checks the packages specified in a manifest, without building. + Check, + /// Installs the packages to a target machine. + Install { + /// The directory from which artifacts will be pulled. + /// + /// Should match the format from the Package subcommand. + #[structopt(long = "in", default_value = "out")] + artifact_dir: PathBuf, + + /// The directory to which artifacts will be installed. + /// + /// Defaults to "/opt/oxide". + #[structopt(long = "out", default_value = "/opt/oxide")] + install_dir: PathBuf, + }, + /// Removes the packages from the target machine. + Uninstall { + /// The directory from which artifacts were be pulled. + /// + /// Should match the format from the Package subcommand. + #[structopt(long = "in", default_value = "out")] + artifact_dir: PathBuf, + + /// The directory to which artifacts were installed. + /// + /// Defaults to "/opt/oxide". + #[structopt(long = "out", default_value = "/opt/oxide")] + install_dir: PathBuf, + }, +} From dd958e0576b724c4a5282d6612d5531a4acc431b Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Mon, 10 Jan 2022 13:18:20 -0500 Subject: [PATCH 2/7] fix clippy --- package/src/bin/thing-flinger.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package/src/bin/thing-flinger.rs b/package/src/bin/thing-flinger.rs index 53bae34825b..7b901e53049 100644 --- a/package/src/bin/thing-flinger.rs +++ b/package/src/bin/thing-flinger.rs @@ -132,10 +132,10 @@ fn do_exec( cmd: String, servers: Option>, ) -> Result<()> { - if servers.is_some() { - validate_servers(&servers.as_ref().unwrap(), &config.servers)?; + if let Some(ref servers) = servers { + validate_servers(servers, &config.servers)?; - for name in &servers.unwrap() { + for name in servers { let server = &config.servers[name]; ssh_exec(&server, &cmd, false)?; } From 8d4d4aa1b8949be88a59daa815c81d27fd67f707 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Mon, 10 Jan 2022 19:21:16 -0500 Subject: [PATCH 3/7] some more fixes for review --- package/README.adoc | 10 +-- package/src/bin/deployment-example.toml | 2 + package/src/bin/thing-flinger.rs | 95 ++++++++++++++++--------- 3 files changed, 70 insertions(+), 37 deletions(-) diff --git a/package/README.adoc b/package/README.adoc index 3f6acb7e520..2bd3de57293 100644 --- a/package/README.adoc +++ b/package/README.adoc @@ -55,7 +55,7 @@ thing-flinger, pointing out room for improvement. `thing-flinger` defines three types of nodes: * Client - Where a user typically edits their code and runs thing-flinger. This can run any OS. * Builder - A Helios box where Omicron is built and packaged - * Deployment - Helios machines where Omicron will be installed and run + * Deployed Server - Helios machines where Omicron will be installed and run It's not at all necessary for these to be separate nodes. For example, a client and builder can be the same machine, as long as it's a Helios box. Same goes for Builder and a deployment server. The @@ -77,7 +77,7 @@ the builder to also be on the other servers, thus minimizing needed server confi Ensure you have an account on all illumos boxes, with the client public key in `~/.ssh/authorized_keys`. -.The build machine should have a rust and cargo installed, and it's probably a good idea to also have +.The build machine must have Rust and cargo installed, as well as all the dependencies for Omicron installed. Following the *prerequisites* in the https://github.com/oxidecomputer/omicron/#build-and-run[Build and run] section of the main Omicron README is probably a good idea. @@ -89,7 +89,7 @@ README is probably a good idea. `cargo build -p omicron-package` ==== sync -Use rsync to copy your source code to the builder. Note that this copies over your `.git` subdirectory on purpose so +Copy your source code to the builder. Note that this copies over your `.git` subdirectory on purpose so that a branch can be configured for building with the `git_treeish` field in the toml `builder` table. @@ -111,8 +111,8 @@ Create files that are unique to each deployment server. `./target/debug/thing-flinger -c overlay` ==== install -Install omicron to all machines, in parallel. This consists copying the packaged omicron tarballs -along with overlay files, and omicron-package and it's manifest to a `staging` directory on each +Install omicron to all machines, in parallel. This consists of copying the packaged omicron tarballs +along with overlay files, and omicron-package and its manifest to a `staging` directory on each deployment server, and then running omicron-package, installing overlay files, and restarting services. diff --git a/package/src/bin/deployment-example.toml b/package/src/bin/deployment-example.toml index c75098cdb80..929c5090487 100644 --- a/package/src/bin/deployment-example.toml +++ b/package/src/bin/deployment-example.toml @@ -10,6 +10,8 @@ local_source = "/Users/ajs/oxide/omicron" # `server` must refer to one of the `servers` in the servers table server = "atrium" omicron_path = "/home/andrew/oxide/omicron" + +# Git branch, sha, etc... git_treeish = "thing-flinger" [deployment] diff --git a/package/src/bin/thing-flinger.rs b/package/src/bin/thing-flinger.rs index 7b901e53049..0e0dd714cf9 100644 --- a/package/src/bin/thing-flinger.rs +++ b/package/src/bin/thing-flinger.rs @@ -1,3 +1,9 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Utility for deploying Omicron to remote machines + use omicron_package::{parse, SubCommand as PackageSubCommand}; use std::collections::{BTreeMap, BTreeSet}; @@ -14,7 +20,7 @@ use thiserror::Error; #[derive(Deserialize, Debug)] struct Builder { pub server: String, - pub omicron_path: String, + pub omicron_path: PathBuf, pub git_treeish: String, } @@ -29,12 +35,12 @@ struct Server { struct Deployment { pub servers: BTreeSet, pub rack_secret_threshold: usize, - pub staging_dir: String, + pub staging_dir: PathBuf, } #[derive(Debug, Deserialize)] struct Config { - pub local_source: String, + pub local_source: PathBuf, pub builder: Builder, pub servers: BTreeMap, pub deployment: Deployment, @@ -111,19 +117,10 @@ enum FlingError { /// Failed to rsync omicron to build host #[error("Failed to sync {src} with {dst}")] FailedSync { src: String, dst: String }, -} -fn validate_servers( - chosen: &BTreeSet, - all: &BTreeMap, -) -> Result<(), FlingError> { - let all = all.keys().cloned().collect(); - let diff: Vec = chosen.difference(&all).cloned().collect(); - if !diff.is_empty() { - Err(FlingError::InvalidServers(diff)) - } else { - Ok(()) - } + /// The given path must be absolute + #[error("Path for {field} must be absolute")] + NotAbsolutePath { field: &'static str }, } // TODO: run in parallel when that option is given @@ -152,10 +149,16 @@ fn do_sync(config: &Config) -> Result<()> { config.servers.get(&config.builder.server).ok_or_else(|| { FlingError::InvalidServers(vec![config.builder.server.clone()]) })?; - let src = config.local_source.clone() + "/"; + + // For rsync to copy from the source appropriately we must guarantee a + // trailing slash. + let src = + format!("{}/", config.local_source.canonicalize()?.to_string_lossy()); let dst = format!( "{}@{}:{}", - server.username, server.addr, config.builder.omicron_path + server.username, + server.addr, + config.builder.omicron_path.to_str().unwrap() ); let mut cmd = Command::new("rsync"); cmd.arg("-az") @@ -193,7 +196,7 @@ fn do_build_minimal(config: &Config) -> Result<()> { let server = &config.servers[&config.builder.server]; let cmd = format!( "cd {} && git checkout {} && cargo build -p {} -p {}", - config.builder.omicron_path, + config.builder.omicron_path.to_string_lossy(), config.builder.git_treeish, "omicron-package", "omicron-sled-agent" @@ -225,7 +228,7 @@ fn do_package( write!( &mut cmd, "bash -lc 'cd {} && git checkout {} && {} package --out {} {}'", - config.builder.omicron_path, + config.builder.omicron_path.to_string_lossy(), config.builder.git_treeish, cmd_path, &artifact_dir, @@ -243,7 +246,9 @@ fn do_check(config: &Config) -> Result<()> { write!( &mut cmd, "bash -lc 'cd {} && git checkout {} && {} check'", - config.builder.omicron_path, config.builder.git_treeish, cmd_path, + config.builder.omicron_path.to_string_lossy(), + config.builder.git_treeish, + cmd_path, )?; ssh_exec(&server, &cmd, false) @@ -261,7 +266,7 @@ fn do_uninstall( // Run `omicron-package uninstall` on the deployment server let cmd = format!( "cd ~/{} && pfexec ./omicron-package uninstall --in ~/{} --out {}", - config.deployment.staging_dir, + config.deployment.staging_dir.to_string_lossy(), deployment_src.to_string_lossy(), install_dir.to_string_lossy() ); @@ -354,7 +359,7 @@ fn overlay_sled_agent( "{} && cd {} && ./target/debug/sled-agent-overlay-files \ --threshold {} --directories {}", cmd, - config.builder.omicron_path, + config.builder.omicron_path.to_string_lossy(), config.deployment.rack_secret_threshold, dirs ); @@ -405,7 +410,7 @@ fn copy_package_artifacts_to_staging( pkg_dir, destination.username, destination.addr, - config.deployment.staging_dir + config.deployment.staging_dir.to_string_lossy() ); println!("$ {}", cmd); ssh_exec(builder, &cmd, true) @@ -423,7 +428,7 @@ fn copy_omicron_package_binary_to_staging( bin_path.to_string_lossy(), destination.username, destination.addr, - config.deployment.staging_dir + config.deployment.staging_dir.to_string_lossy() ); println!("$ {}", cmd); ssh_exec(builder, &cmd, true) @@ -441,7 +446,7 @@ fn copy_package_manifest_to_staging( path.to_string_lossy(), destination.username, destination.addr, - config.deployment.staging_dir + config.deployment.staging_dir.to_string_lossy() ); println!("$ {}", cmd); ssh_exec(builder, &cmd, true) @@ -459,7 +464,7 @@ fn run_omicron_package_from_staging( // Run `omicron-package install` on the deployment server let cmd = format!( "cd ~/{} && pfexec ./omicron-package install --in ~/{} --out {}", - config.deployment.staging_dir, + config.deployment.staging_dir.to_string_lossy(), deployment_src.to_string_lossy(), install_dir.to_string_lossy() ); @@ -480,7 +485,7 @@ fn copy_overlay_files_to_staging( destination_name, destination.username, destination.addr, - config.deployment.staging_dir + config.deployment.staging_dir.to_string_lossy() ); println!("$ {}", cmd); ssh_exec(builder, &cmd, true) @@ -493,7 +498,7 @@ fn install_overlay_files_from_staging( ) -> Result<()> { let cmd = format!( "pfexec cp -r ~/{}/overlay/* {}", - config.deployment.staging_dir, + config.deployment.staging_dir.to_string_lossy(), install_dir.to_string_lossy() ); println!("$ {}", cmd); @@ -549,15 +554,41 @@ fn ssh_exec( Ok(()) } -fn main() -> Result<()> { - let args = Args::from_args_safe().map_err(|err| anyhow!(err))?; - let config = parse::<_, Config>(args.config)?; +fn validate_servers( + chosen: &BTreeSet, + all: &BTreeMap, +) -> Result<(), FlingError> { + let all = all.keys().cloned().collect(); + let diff: Vec = chosen.difference(&all).cloned().collect(); + if !diff.is_empty() { + Err(FlingError::InvalidServers(diff)) + } else { + Ok(()) + } +} +fn validate(config: &Config) -> Result<(), FlingError> { + if !config.local_source.is_absolute() { + return Err(FlingError::NotAbsolutePath { field: "local_source" }); + } + if !config.builder.omicron_path.is_absolute() { + return Err(FlingError::NotAbsolutePath { + field: "builder.omicron_path", + }); + } validate_servers(&config.deployment.servers, &config.servers)?; + validate_servers( &BTreeSet::from([config.builder.server.clone()]), &config.servers, - )?; + ) +} + +fn main() -> Result<()> { + let args = Args::from_args_safe().map_err(|err| anyhow!(err))?; + let config = parse::<_, Config>(args.config)?; + + validate(&config)?; match args.subcommand { SubCommand::Exec { cmd, servers } => { From ea613a607e658dfb68a6f623d5a8e9eef21e0e68 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Mon, 10 Jan 2022 19:28:30 -0500 Subject: [PATCH 4/7] branch update for testing --- package/src/bin/deployment-example.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/src/bin/deployment-example.toml b/package/src/bin/deployment-example.toml index 929c5090487..ad2eb54a3ac 100644 --- a/package/src/bin/deployment-example.toml +++ b/package/src/bin/deployment-example.toml @@ -12,7 +12,7 @@ server = "atrium" omicron_path = "/home/andrew/oxide/omicron" # Git branch, sha, etc... -git_treeish = "thing-flinger" +git_treeish = "thing-flinger2" [deployment] servers = ["sock", "buskin"] From b4331af0cae27d3b866f8814435651445aaacd29 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Mon, 10 Jan 2022 20:33:55 -0500 Subject: [PATCH 5/7] Use $HOME instead of ~ --- package/src/bin/thing-flinger.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/package/src/bin/thing-flinger.rs b/package/src/bin/thing-flinger.rs index 0e0dd714cf9..31d4a9dcb7c 100644 --- a/package/src/bin/thing-flinger.rs +++ b/package/src/bin/thing-flinger.rs @@ -265,7 +265,7 @@ fn do_uninstall( let server = &config.servers[server_name]; // Run `omicron-package uninstall` on the deployment server let cmd = format!( - "cd ~/{} && pfexec ./omicron-package uninstall --in ~/{} --out {}", + "cd $HOME/{} && pfexec ./omicron-package uninstall --in $HOME/{} --out {}", config.deployment.staging_dir.to_string_lossy(), deployment_src.to_string_lossy(), install_dir.to_string_lossy() @@ -406,7 +406,7 @@ fn copy_package_artifacts_to_staging( ) -> Result<()> { let cmd = format!( "rsync -avz -e 'ssh -o StrictHostKeyChecking=no' \ - --exclude overlay/ {} {}@{}:~/{}", + --exclude overlay/ {} {}@{}:$HOME/{}", pkg_dir, destination.username, destination.addr, @@ -424,7 +424,7 @@ fn copy_omicron_package_binary_to_staging( let mut bin_path = PathBuf::from(&config.builder.omicron_path); bin_path.push("target/debug/omicron-package"); let cmd = format!( - "rsync -avz {} {}@{}:~/{}", + "rsync -avz {} {}@{}:$HOME/{}", bin_path.to_string_lossy(), destination.username, destination.addr, @@ -442,7 +442,7 @@ fn copy_package_manifest_to_staging( let mut path = PathBuf::from(&config.builder.omicron_path); path.push("package-manifest.toml"); let cmd = format!( - "rsync {} {}@{}:~/{}", + "rsync {} {}@{}:$HOME/{}", path.to_string_lossy(), destination.username, destination.addr, @@ -463,7 +463,7 @@ fn run_omicron_package_from_staging( // Run `omicron-package install` on the deployment server let cmd = format!( - "cd ~/{} && pfexec ./omicron-package install --in ~/{} --out {}", + "cd $HOME/{} && pfexec ./omicron-package install --in $HOME/{} --out {}", config.deployment.staging_dir.to_string_lossy(), deployment_src.to_string_lossy(), install_dir.to_string_lossy() @@ -480,7 +480,7 @@ fn copy_overlay_files_to_staging( destination_name: &str, ) -> Result<()> { let cmd = format!( - "rsync -avz {}/overlay/{}/ {}@{}:~/{}/overlay/", + "rsync -avz {}/overlay/{}/ {}@{}:$HOME/{}/overlay/", pkg_dir, destination_name, destination.username, @@ -497,7 +497,7 @@ fn install_overlay_files_from_staging( install_dir: &Path, ) -> Result<()> { let cmd = format!( - "pfexec cp -r ~/{}/overlay/* {}", + "pfexec cp -r $HOME/{}/overlay/* {}", config.deployment.staging_dir.to_string_lossy(), install_dir.to_string_lossy() ); @@ -535,7 +535,7 @@ fn ssh_exec( ) -> Result<()> { // Source .profile, so we have access to cargo. Rustup installs knowledge // about the cargo path here. - let remote_cmd = String::from(". ~/.profile && ") + remote_cmd; + let remote_cmd = String::from(". $HOME/.profile && ") + remote_cmd; let auth_sock = std::env::var("SSH_AUTH_SOCK")?; let mut cmd = Command::new("ssh"); if forward_agent { From d49618ce7f2c0524671dd5917fcc4cfe5d291454 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Fri, 14 Jan 2022 17:03:37 -0500 Subject: [PATCH 6/7] Fixes for review by Luqman --- package/README.adoc | 8 +++-- package/src/bin/deployment-example.toml | 7 ++-- package/src/bin/thing-flinger.rs | 43 ++++++++++++++++--------- 3 files changed, 37 insertions(+), 21 deletions(-) diff --git a/package/README.adoc b/package/README.adoc index 2bd3de57293..eaf17ca782a 100644 --- a/package/README.adoc +++ b/package/README.adoc @@ -53,6 +53,7 @@ thing-flinger, pointing out room for improvement. `thing-flinger` defines three types of nodes: + * Client - Where a user typically edits their code and runs thing-flinger. This can run any OS. * Builder - A Helios box where Omicron is built and packaged * Deployed Server - Helios machines where Omicron will be installed and run @@ -63,7 +64,7 @@ benefit of this separation though, is that it allows editing on something like a having to worry about setting up a development environment on an illumos based host. Machine topology is configured in a `TOML` file that is passed on the command line. All illumos -machines are listed under `servers`, and just the names are used to for configuring a builder and +machines are listed under `servers`, and just the names are used for configuring a builder and deployment servers. An link:src/bin/deployment-example.toml[example] is provided. Thing flinger works over SSH, and so the user must have the public key of their client configured @@ -84,7 +85,8 @@ README is probably a good idea. === Command Based Workflow -==== Build omicron-package on client. +==== Build thing-flinger on client +`thing-flinger` is part of the `omicron-package` crate. `cargo build -p omicron-package` @@ -130,7 +132,7 @@ configs for this. === Design rationale -`thing-flinger` is a command line program written in rust. It was written this way to build upon and +`thing-flinger` is a command line program written in rust. It was written this way to build upon `omicron-package`, which is also in rust, as that is our default language of choice at Oxide. `thing-flinger` is based around SSH, as that is the minimal viable requirement for a test tool such as this. Additionally, it provides for the most straightforward implementation, and takes the least diff --git a/package/src/bin/deployment-example.toml b/package/src/bin/deployment-example.toml index ad2eb54a3ac..3b256dbaa37 100644 --- a/package/src/bin/deployment-example.toml +++ b/package/src/bin/deployment-example.toml @@ -9,6 +9,8 @@ local_source = "/Users/ajs/oxide/omicron" [builder] # `server` must refer to one of the `servers` in the servers table server = "atrium" + +# This must be an absolute path omicron_path = "/home/andrew/oxide/omicron" # Git branch, sha, etc... @@ -21,9 +23,8 @@ rack_secret_threshold = 2 # Location where files to install will be placed before running # `omicron-package install` # -# Since usernames may vary per server, this directory is relative to the user's -# home directory. -staging_dir = "omicron_staging" +# This must be an absolute path +staging_dir = "$HOME/omicron_staging" [servers.tarz] username = "ajs" diff --git a/package/src/bin/thing-flinger.rs b/package/src/bin/thing-flinger.rs index 31d4a9dcb7c..284a59226d3 100644 --- a/package/src/bin/thing-flinger.rs +++ b/package/src/bin/thing-flinger.rs @@ -265,7 +265,7 @@ fn do_uninstall( let server = &config.servers[server_name]; // Run `omicron-package uninstall` on the deployment server let cmd = format!( - "cd $HOME/{} && pfexec ./omicron-package uninstall --in $HOME/{} --out {}", + "cd {} && pfexec ./omicron-package uninstall --in {} --out {}", config.deployment.staging_dir.to_string_lossy(), deployment_src.to_string_lossy(), install_dir.to_string_lossy() @@ -406,7 +406,7 @@ fn copy_package_artifacts_to_staging( ) -> Result<()> { let cmd = format!( "rsync -avz -e 'ssh -o StrictHostKeyChecking=no' \ - --exclude overlay/ {} {}@{}:$HOME/{}", + --exclude overlay/ {} {}@{}:{}", pkg_dir, destination.username, destination.addr, @@ -424,7 +424,7 @@ fn copy_omicron_package_binary_to_staging( let mut bin_path = PathBuf::from(&config.builder.omicron_path); bin_path.push("target/debug/omicron-package"); let cmd = format!( - "rsync -avz {} {}@{}:$HOME/{}", + "rsync -avz {} {}@{}:{}", bin_path.to_string_lossy(), destination.username, destination.addr, @@ -442,7 +442,7 @@ fn copy_package_manifest_to_staging( let mut path = PathBuf::from(&config.builder.omicron_path); path.push("package-manifest.toml"); let cmd = format!( - "rsync {} {}@{}:$HOME/{}", + "rsync {} {}@{}:{}", path.to_string_lossy(), destination.username, destination.addr, @@ -463,7 +463,7 @@ fn run_omicron_package_from_staging( // Run `omicron-package install` on the deployment server let cmd = format!( - "cd $HOME/{} && pfexec ./omicron-package install --in $HOME/{} --out {}", + "cd {} && pfexec ./omicron-package install --in {} --out {}", config.deployment.staging_dir.to_string_lossy(), deployment_src.to_string_lossy(), install_dir.to_string_lossy() @@ -480,7 +480,7 @@ fn copy_overlay_files_to_staging( destination_name: &str, ) -> Result<()> { let cmd = format!( - "rsync -avz {}/overlay/{}/ {}@{}:$HOME/{}/overlay/", + "rsync -avz {}/overlay/{}/ {}@{}:{}/overlay/", pkg_dir, destination_name, destination.username, @@ -497,7 +497,7 @@ fn install_overlay_files_from_staging( install_dir: &Path, ) -> Result<()> { let cmd = format!( - "pfexec cp -r $HOME/{}/overlay/* {}", + "pfexec cp -r {}/overlay/* {}", config.deployment.staging_dir.to_string_lossy(), install_dir.to_string_lossy() ); @@ -567,15 +567,28 @@ fn validate_servers( } } -fn validate(config: &Config) -> Result<(), FlingError> { - if !config.local_source.is_absolute() { - return Err(FlingError::NotAbsolutePath { field: "local_source" }); - } - if !config.builder.omicron_path.is_absolute() { - return Err(FlingError::NotAbsolutePath { - field: "builder.omicron_path", - }); +fn validate_absolute_path( + path: &Path, + field: &'static str, +) -> Result<(), FlingError> { + if path.is_absolute() || path.starts_with("$HOME") { + Ok(()) + } else { + Err(FlingError::NotAbsolutePath { field }) } +} + +fn validate(config: &Config) -> Result<(), FlingError> { + validate_absolute_path(&config.local_source, "local_source")?; + validate_absolute_path( + &config.builder.omicron_path, + "builder.omicron_path", + )?; + validate_absolute_path( + &config.deployment.staging_dir, + "deployment.staging_dir", + )?; + validate_servers(&config.deployment.servers, &config.servers)?; validate_servers( From b07c3c2cf684a971e7a20dea28daba4584f210f8 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Fri, 14 Jan 2022 17:06:08 -0500 Subject: [PATCH 7/7] slight update to comment in deployment-example.toml --- package/src/bin/deployment-example.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/package/src/bin/deployment-example.toml b/package/src/bin/deployment-example.toml index 3b256dbaa37..62296fa4d0f 100644 --- a/package/src/bin/deployment-example.toml +++ b/package/src/bin/deployment-example.toml @@ -24,6 +24,7 @@ rack_secret_threshold = 2 # `omicron-package install` # # This must be an absolute path +# We specifically allow for $HOME in validating the absolute path staging_dir = "$HOME/omicron_staging" [servers.tarz]