From a54cb20fad7fe135c29fc895514dc3dd6998ae64 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Sun, 12 May 2024 06:35:01 +0000 Subject: [PATCH] hubris bits, stamping, job limits --- Cargo.lock | 6 ++ Cargo.toml | 1 + dev-tools/releng/Cargo.toml | 6 ++ dev-tools/releng/src/hubris.rs | 180 +++++++++++++++++++++++++++++++++ dev-tools/releng/src/job.rs | 24 ++++- dev-tools/releng/src/main.rs | 109 ++++++++++++++++++-- 6 files changed, 318 insertions(+), 8 deletions(-) create mode 100644 dev-tools/releng/src/hubris.rs diff --git a/Cargo.lock b/Cargo.lock index a627b0a79b9..b85ae46cc61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5592,16 +5592,22 @@ dependencies = [ "clap", "fs-err", "futures", + "num_cpus", + "omicron-common", "omicron-workspace-hack", "omicron-zone-package", "once_cell", + "reqwest", "semver 1.0.22", + "serde", "shell-words", "slog", "slog-async", "slog-term", "tar", "tokio", + "toml 0.8.12", + "tufaceous-lib", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9cc8244bfeb..2f4cc4c8f44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -338,6 +338,7 @@ nexus-test-utils = { path = "nexus/test-utils" } nexus-types = { path = "nexus/types" } num-integer = "0.1.46" num = { version = "0.4.2", default-features = false, features = [ "libm" ] } +num_cpus = "1.16.0" omicron-common = { path = "common" } omicron-gateway = { path = "gateway" } omicron-nexus = { path = "nexus" } diff --git a/dev-tools/releng/Cargo.toml b/dev-tools/releng/Cargo.toml index 25a7b09cdf1..9e57db3e92f 100644 --- a/dev-tools/releng/Cargo.toml +++ b/dev-tools/releng/Cargo.toml @@ -12,16 +12,22 @@ chrono.workspace = true clap.workspace = true fs-err = { workspace = true, features = ["tokio"] } futures.workspace = true +num_cpus.workspace = true +omicron-common.workspace = true omicron-workspace-hack.workspace = true omicron-zone-package.workspace = true once_cell.workspace = true +reqwest.workspace = true semver.workspace = true +serde.workspace = true shell-words.workspace = true slog.workspace = true slog-async.workspace = true slog-term.workspace = true tar.workspace = true tokio = { workspace = true, features = ["full"] } +toml.workspace = true +tufaceous-lib.workspace = true [lints] workspace = true diff --git a/dev-tools/releng/src/hubris.rs b/dev-tools/releng/src/hubris.rs new file mode 100644 index 00000000000..89a680214bd --- /dev/null +++ b/dev-tools/releng/src/hubris.rs @@ -0,0 +1,180 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::HashMap; + +use anyhow::Context; +use anyhow::Result; +use camino::Utf8PathBuf; +use fs_err::tokio as fs; +use fs_err::tokio::File; +use futures::future::TryFutureExt; +use futures::stream::FuturesUnordered; +use futures::stream::TryStreamExt; +use omicron_common::api::external::SemverVersion; +use omicron_common::api::internal::nexus::KnownArtifactKind; +use semver::Version; +use serde::Deserialize; +use serde::Serialize; +use tokio::io::AsyncWriteExt; +use tufaceous_lib::assemble::DeserializedArtifactData; +use tufaceous_lib::assemble::DeserializedArtifactSource; +use tufaceous_lib::assemble::DeserializedFileArtifactSource; + +async fn fetch_one( + base_url: &'static str, + client: reqwest::Client, + hash: &str, +) -> Result> { + client + .get(format!("{}/artifact/{}", base_url, hash)) + .send() + .and_then(|response| response.json()) + .await + .with_context(|| { + format!( + "failed to fetch hubris artifact {} from {}", + hash, base_url + ) + }) +} + +pub(crate) async fn fetch_hubris_artifacts( + base_url: &'static str, + client: reqwest::Client, + manifest_list: Utf8PathBuf, + output_dir: Utf8PathBuf, +) -> Result<()> { + fs::create_dir_all(&output_dir).await?; + + let (manifests, hashes) = fs::read_to_string(manifest_list) + .await? + .lines() + .filter_map(|line| line.split_whitespace().next()) + .map(|hash| { + let hash = hash.to_owned(); + let client = client.clone(); + async move { + let data = fetch_one(base_url, client, &hash).await?; + let str = String::from_utf8(data) + .context("hubris artifact manifest was not UTF-8")?; + let hash_manifest: Manifest = toml::from_str(&str) + .context( + "failed to deserialize hubris artifact manifest", + )?; + + let mut hashes = Vec::new(); + for artifact in hash_manifest.artifacts.values().flatten() { + match &artifact.source { + Source::File(file) => hashes.push(file.hash.clone()), + Source::CompositeRot { archive_a, archive_b } => hashes + .extend([ + archive_a.hash.clone(), + archive_b.hash.clone(), + ]), + } + } + + let path_manifest: Manifest = + hash_manifest.into(); + anyhow::Ok((path_manifest, hashes)) + } + }) + .collect::>() + .try_collect::<(Vec<_>, Vec<_>)>() + .await?; + + let mut output_manifest = + File::create(output_dir.join("manifest.toml")).await?; + for manifest in manifests { + output_manifest + .write_all(toml::to_string_pretty(&manifest)?.as_bytes()) + .await?; + } + + hashes + .into_iter() + .flatten() + .map(|hash| { + let client = client.clone(); + let output_dir = output_dir.clone(); + async move { + let data = fetch_one(base_url, client, &hash).await?; + fs::write(output_dir.join(hash).with_extension("zip"), data) + .await?; + anyhow::Ok(()) + } + }) + .collect::>() + .try_collect::<()>() + .await?; + + output_manifest.sync_data().await?; + Ok(()) +} + +#[derive(Serialize, Deserialize)] +struct Manifest { + #[serde(rename = "artifact")] + artifacts: HashMap>, +} + +#[derive(Deserialize)] +struct Artifact { + name: String, + version: Version, + source: Source, +} + +#[derive(Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case")] +enum Source { + File(FileSource), + CompositeRot { archive_a: FileSource, archive_b: FileSource }, +} + +#[derive(Deserialize)] +struct FileSource { + hash: String, +} + +impl From> for Manifest { + fn from( + manifest: Manifest, + ) -> Manifest { + fn zip(hash: String) -> Utf8PathBuf { + Utf8PathBuf::from(hash).with_extension("zip") + } + + let mut artifacts = HashMap::new(); + for (kind, old_data) in manifest.artifacts { + let mut new_data = Vec::new(); + for artifact in old_data { + let source = match artifact.source { + Source::File(file) => DeserializedArtifactSource::File { + path: zip(file.hash), + }, + Source::CompositeRot { archive_a, archive_b } => { + DeserializedArtifactSource::CompositeRot { + archive_a: DeserializedFileArtifactSource::File { + path: zip(archive_a.hash), + }, + archive_b: DeserializedFileArtifactSource::File { + path: zip(archive_b.hash), + }, + } + } + }; + new_data.push(DeserializedArtifactData { + name: artifact.name, + version: SemverVersion(artifact.version), + source, + }); + } + artifacts.insert(kind, new_data); + } + + Manifest { artifacts } + } +} diff --git a/dev-tools/releng/src/job.rs b/dev-tools/releng/src/job.rs index acbac3dd8d6..aa00c316f48 100644 --- a/dev-tools/releng/src/job.rs +++ b/dev-tools/releng/src/job.rs @@ -6,6 +6,7 @@ use std::collections::HashMap; use std::future::Future; use std::pin::Pin; use std::process::Stdio; +use std::sync::Arc; use std::time::Instant; use anyhow::anyhow; @@ -26,11 +27,13 @@ use tokio::io::BufReader; use tokio::process::Command; use tokio::sync::oneshot; use tokio::sync::oneshot::error::RecvError; +use tokio::sync::Semaphore; use crate::cmd::CommandExt; pub(crate) struct Jobs { logger: Logger, + permits: Arc, log_dir: Utf8PathBuf, map: HashMap, } @@ -47,9 +50,14 @@ pub(crate) struct Selector<'a> { } impl Jobs { - pub(crate) fn new(logger: &Logger, log_dir: &Utf8Path) -> Jobs { + pub(crate) fn new( + logger: &Logger, + permits: Arc, + log_dir: &Utf8Path, + ) -> Jobs { Jobs { logger: logger.clone(), + permits, log_dir: log_dir.to_owned(), map: HashMap::new(), } @@ -70,6 +78,7 @@ impl Jobs { Job { future: Box::pin(run_job( self.logger.clone(), + self.permits.clone(), name.clone(), future, )), @@ -95,6 +104,7 @@ impl Jobs { // returning &mut std::mem::replace(command, Command::new("false")), self.logger.clone(), + self.permits.clone(), name.clone(), self.log_dir.join(&name).with_extension("log"), )), @@ -167,10 +177,17 @@ macro_rules! info_or_error { }; } -async fn run_job(logger: Logger, name: String, future: F) -> Result<()> +async fn run_job( + logger: Logger, + permits: Arc, + name: String, + future: F, +) -> Result<()> where F: Future> + 'static, { + let _ = permits.acquire_owned().await?; + info!(logger, "[{}] running task", name); let start = Instant::now(); let result = future.await; @@ -189,9 +206,12 @@ where async fn spawn_with_output( mut command: Command, logger: Logger, + permits: Arc, name: String, log_path: Utf8PathBuf, ) -> Result<()> { + let _ = permits.acquire_owned().await?; + let log_file_1 = File::create(log_path).await?; let log_file_2 = log_file_1.try_clone().await?; diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index b33498ef366..7e66e581ddc 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -3,9 +3,11 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. mod cmd; +mod hubris; mod job; use std::sync::Arc; +use std::time::Duration; use std::time::Instant; use anyhow::bail; @@ -26,6 +28,7 @@ use slog::Logger; use slog_term::FullFormat; use slog_term::TermDecorator; use tokio::process::Command; +use tokio::sync::Semaphore; use crate::cmd::CommandExt; use crate::job::Jobs; @@ -65,9 +68,22 @@ const RECOVERY_IMAGE_PACKAGES: [(&str, InstallMethod); 2] = [ ("installinator", InstallMethod::Install), ("mg-ddm-gz", InstallMethod::Install), ]; +/// Packages to ship with the TUF repo. +const TUF_PACKAGES: [&str; 11] = [ + "clickhouse_keeper", + "clickhouse", + "cockroachdb", + "crucible-pantry-zone", + "crucible-zone", + "external-dns", + "internal-dns", + "nexus", + "ntp", + "oximeter", + "probe", +]; const HELIOS_REPO: &str = "https://pkg.oxide.computer/helios/2/dev/"; -const OPTE_VERSION: &str = include_str!("../../../tools/opte_version"); static WORKSPACE_DIR: Lazy = Lazy::new(|| { // $CARGO_MANIFEST_DIR is at `.../omicron/dev-tools/releng` @@ -163,6 +179,8 @@ fn main() -> Result<()> { #[tokio::main] async fn do_run(logger: Logger, args: Args) -> Result<()> { + let permits = Arc::new(Semaphore::new(num_cpus::get())); + let commit = Command::new("git") .args(["rev-parse", "HEAD"]) .ensure_stdout(&logger) @@ -186,6 +204,14 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { &fs::read_to_string(WORKSPACE_DIR.join("package-manifest.toml")) .await?, )?); + let opte_version = + fs::read_to_string(WORKSPACE_DIR.join("tools/opte_version")).await?; + + let client = reqwest::ClientBuilder::new() + .connect_timeout(Duration::from_secs(15)) + .timeout(Duration::from_secs(15)) + .build() + .context("failed to build reqwest client")?; // PREFLIGHT ============================================================== let mut preflight_ok = true; @@ -289,7 +315,7 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { // DEFINE JOBS ============================================================ let tempdir = camino_tempfile::tempdir() .context("failed to create temporary directory")?; - let mut jobs = Jobs::new(&logger, &args.output_dir); + let mut jobs = Jobs::new(&logger, permits.clone(), &args.output_dir); jobs.push_command( "helios-setup", @@ -346,6 +372,20 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { ) .after(concat!($target_name, "-target")); + jobs.push( + concat!($target_name, "-stamp"), + stamp_packages( + logger.clone(), + permits.clone(), + args.output_dir.clone(), + omicron_package.clone(), + $target_name, + version.clone(), + $proto_packages.iter().map(|(name, _)| *name), + ), + ) + .after(concat!($target_name, "-package")); + let proto_dir = tempdir.path().join("proto").join($target_name); jobs.push( concat!($target_name, "-proto"), @@ -356,7 +396,7 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { manifest.clone(), ), ) - .after(concat!($target_name, "-package")); + .after(concat!($target_name, "-stamp")); // The ${os_short_commit} token will be expanded by `helios-build` let image_name = format!( @@ -373,11 +413,11 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { .arg(args.helios_dir.join("helios-build")) .arg("experiment-image") .arg("-o") // output directory for image - .arg(args.output_dir.join($target_name)) + .arg(args.output_dir.join(concat!("os-", $target_name))) .arg("-p") // use an external package repository .arg(format!("helios-dev={}", HELIOS_REPO)) .arg("-F") // pass extra image builder features - .arg(format!("optever={}", OPTE_VERSION.trim())) + .arg(format!("optever={}", opte_version.trim())) .arg("-P") // include all files from extra proto area .arg(proto_dir.join("root")) .arg("-N") // image name @@ -418,7 +458,6 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { image_build_args: ["-R"], image_dataset: args.recovery_dataset, } - // Build the recovery target after we build the host target. Only one // of these will build at a time since Cargo locks its target directory; // since host-package and host-image both take longer than their recovery @@ -429,6 +468,39 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { jobs.select("recovery-image").after("host-image"); } + jobs.push( + "tuf-stamp", + stamp_packages( + logger.clone(), + permits.clone(), + args.output_dir.clone(), + omicron_package.clone(), + "host", + version.clone(), + TUF_PACKAGES.into_iter(), + ), + ) + .after("host-proto"); + + jobs.push( + "hubris-staging", + hubris::fetch_hubris_artifacts( + "https://permslip-staging.corp.oxide.computer", + client.clone(), + WORKSPACE_DIR.join("tools/permslip_staging"), + args.output_dir.join("hubris-staging"), + ), + ); + jobs.push( + "hubris-production", + hubris::fetch_hubris_artifacts( + "https://signer-us-west.corp.oxide.computer", + client.clone(), + WORKSPACE_DIR.join("tools/permslip_production"), + args.output_dir.join("hubris-production"), + ), + ); + // RUN JOBS =============================================================== let start = Instant::now(); jobs.run_all().await?; @@ -448,6 +520,31 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { Ok(()) } +async fn stamp_packages( + logger: Logger, + permits: Arc, + output_dir: Utf8PathBuf, + omicron_package: Utf8PathBuf, + target_name: &'static str, + version: Version, + packages: impl Iterator, +) -> Result<()> { + let version = version.to_string(); + let mut jobs = Jobs::new(&logger, permits, &output_dir); + for package in packages { + jobs.push_command( + format!("stamp-{}", package), + Command::new(&omicron_package) + .arg("--target") + .arg(target_name) + .arg("stamp") + .arg(package) + .arg(&version), + ); + } + jobs.run_all().await +} + async fn build_proto_area( package_dir: Utf8PathBuf, proto_dir: Utf8PathBuf,