From e100c7f9bd9f6ad2896e7ea16acfc773237e9412 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Sat, 11 May 2024 19:45:06 +0000 Subject: [PATCH] wip --- Cargo.lock | 24 ++ Cargo.toml | 2 + dev-tools/releng/Cargo.toml | 26 ++ dev-tools/releng/src/cmd.rs | 115 +++++++++ dev-tools/releng/src/job.rs | 248 +++++++++++++++++++ dev-tools/releng/src/main.rs | 450 +++++++++++++++++++++++++++++++++++ package-manifest.toml | 2 +- workspace-hack/Cargo.toml | 2 + 8 files changed, 868 insertions(+), 1 deletion(-) create mode 100644 dev-tools/releng/Cargo.toml create mode 100644 dev-tools/releng/src/cmd.rs create mode 100644 dev-tools/releng/src/job.rs create mode 100644 dev-tools/releng/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index a362c64eefb..3dc9e82a811 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2547,6 +2547,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88a41f105fe1d5b6b34b2055e3dc59bb79b46b48b2040b9e6c7b4b5de097aa41" dependencies = [ "autocfg", + "tokio", ] [[package]] @@ -5580,6 +5581,28 @@ dependencies = [ "thiserror", ] +[[package]] +name = "omicron-releng" +version = "0.1.0" +dependencies = [ + "anyhow", + "camino", + "camino-tempfile", + "chrono", + "clap", + "fs-err", + "futures", + "omicron-workspace-hack", + "omicron-zone-package", + "semver 1.0.22", + "shell-words", + "slog", + "slog-async", + "slog-term", + "tar", + "tokio", +] + [[package]] name = "omicron-rpaths" version = "0.1.0" @@ -5767,6 +5790,7 @@ dependencies = [ "elliptic-curve", "ff", "flate2", + "fs-err", "futures", "futures-channel", "futures-core", diff --git a/Cargo.toml b/Cargo.toml index db5060184ba..9cc8244bfeb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = [ "dev-tools/omicron-dev", "dev-tools/oxlog", "dev-tools/reconfigurator-cli", + "dev-tools/releng", "dev-tools/xtask", "dns-server", "end-to-end-tests", @@ -103,6 +104,7 @@ default-members = [ "dev-tools/omicron-dev", "dev-tools/oxlog", "dev-tools/reconfigurator-cli", + "dev-tools/releng", # Do not include xtask in the list of default members, because this causes # hakari to not work as well and build times to be longer. # See omicron#4392. diff --git a/dev-tools/releng/Cargo.toml b/dev-tools/releng/Cargo.toml new file mode 100644 index 00000000000..95aa3b4bb41 --- /dev/null +++ b/dev-tools/releng/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "omicron-releng" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" + +[dependencies] +anyhow.workspace = true +camino-tempfile.workspace = true +camino.workspace = true +fs-err = { workspace = true, features = ["tokio"] } +omicron-workspace-hack.workspace = true +omicron-zone-package.workspace = true +semver.workspace = true +shell-words.workspace = true +slog-async.workspace = true +slog-term.workspace = true +slog.workspace = true +tar.workspace = true +clap.workspace = true +tokio = { workspace = true, features = ["full"] } +chrono.workspace = true +futures.workspace = true + +[lints] +workspace = true diff --git a/dev-tools/releng/src/cmd.rs b/dev-tools/releng/src/cmd.rs new file mode 100644 index 00000000000..7fa9d2a7134 --- /dev/null +++ b/dev-tools/releng/src/cmd.rs @@ -0,0 +1,115 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::ffi::OsStr; +use std::fmt::Write; +use std::process::ExitStatus; +use std::process::Output; +use std::process::Stdio; +use std::time::Instant; + +use anyhow::ensure; +use anyhow::Context; +use anyhow::Result; +use slog::debug; +use slog::Logger; +use tokio::process::Command; + +pub(crate) trait CommandExt { + fn check_status(&self, status: ExitStatus) -> Result<()>; + fn to_string(&self) -> String; + + async fn is_success(&mut self, logger: &Logger) -> Result; + async fn ensure_success(&mut self, logger: &Logger) -> Result<()>; + async fn ensure_stdout(&mut self, logger: &Logger) -> Result; +} + +impl CommandExt for Command { + fn check_status(&self, status: ExitStatus) -> Result<()> { + ensure!( + status.success(), + "command `{}` exited with {}", + self.to_string(), + status + ); + Ok(()) + } + + fn to_string(&self) -> String { + let command = self.as_std(); + let mut command_str = String::new(); + for (name, value) in command.get_envs() { + if let Some(value) = value { + write!( + command_str, + "{}={} ", + shell_words::quote(&name.to_string_lossy()), + shell_words::quote(&value.to_string_lossy()) + ) + .unwrap(); + } + } + write!( + command_str, + "{}", + shell_words::join( + std::iter::once(command.get_program()) + .chain(command.get_args()) + .map(OsStr::to_string_lossy) + ) + ) + .unwrap(); + command_str + } + + async fn is_success(&mut self, logger: &Logger) -> Result { + let output = run( + self.stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()), + logger, + ) + .await?; + Ok(output.status.success()) + } + + async fn ensure_success(&mut self, logger: &Logger) -> Result<()> { + let output = run( + self.stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()), + logger, + ) + .await?; + self.check_status(output.status) + } + + async fn ensure_stdout(&mut self, logger: &Logger) -> Result { + let output = run( + self.stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()), + logger, + ) + .await?; + self.check_status(output.status)?; + String::from_utf8(output.stdout).context("command stdout was not UTF-8") + } +} + +async fn run(command: &mut Command, logger: &Logger) -> Result { + debug!(logger, "running: {}", command.to_string()); + let start = Instant::now(); + let output = + command.kill_on_drop(true).output().await.with_context(|| { + format!("failed to exec `{}`", command.to_string()) + })?; + debug!( + logger, + "process exited with {} ({:?})", + output.status, + Instant::now().saturating_duration_since(start) + ); + Ok(output) +} diff --git a/dev-tools/releng/src/job.rs b/dev-tools/releng/src/job.rs new file mode 100644 index 00000000000..bfe42b59739 --- /dev/null +++ b/dev-tools/releng/src/job.rs @@ -0,0 +1,248 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; +use std::process::Stdio; +use std::time::Instant; + +use anyhow::anyhow; +use anyhow::Context; +use anyhow::Result; +use camino::Utf8Path; +use camino::Utf8PathBuf; +use fs_err::tokio::File; +use futures::stream::FuturesUnordered; +use futures::stream::TryStreamExt; +use slog::debug; +use slog::error; +use slog::Logger; +use tokio::io::AsyncBufReadExt; +use tokio::io::AsyncRead; +use tokio::io::AsyncWrite; +use tokio::io::AsyncWriteExt; +use tokio::io::BufReader; +use tokio::process::Command; +use tokio::sync::oneshot; +use tokio::sync::oneshot::error::RecvError; + +use crate::cmd::CommandExt; + +pub(crate) struct Jobs { + logger: Logger, + log_dir: Utf8PathBuf, + map: HashMap, +} + +struct Job { + future: Pin>>>, + wait_for: Vec>, + notify: Vec>, +} + +pub(crate) struct Selector<'a> { + jobs: &'a mut Jobs, + name: String, +} + +impl Jobs { + pub(crate) fn new(logger: &Logger, log_dir: &Utf8Path) -> Jobs { + Jobs { + logger: logger.clone(), + log_dir: log_dir.to_owned(), + map: HashMap::new(), + } + } + + pub(crate) fn push( + &mut self, + name: impl AsRef, + future: F, + ) -> Selector<'_> + where + F: Future> + 'static, + { + let name = name.as_ref().to_owned(); + assert!(!self.map.contains_key(&name), "duplicate job name {}", name); + self.map.insert( + name.clone(), + Job { + future: Box::pin(run_job( + self.logger.clone(), + name.clone(), + future, + )), + wait_for: Vec::new(), + notify: Vec::new(), + }, + ); + Selector { jobs: self, name } + } + + pub(crate) fn push_command( + &mut self, + name: impl AsRef, + command: &mut Command, + ) -> Selector<'_> { + let name = name.as_ref().to_owned(); + assert!(!self.map.contains_key(&name), "duplicate job name {}", name); + self.map.insert( + name.clone(), + Job { + future: Box::pin(spawn_with_output( + // terrible hack to deal with the `Command` builder + // returning &mut + std::mem::replace(command, Command::new("false")), + self.logger.clone(), + name.clone(), + self.log_dir.join(&name).with_extension("log"), + )), + wait_for: Vec::new(), + notify: Vec::new(), + }, + ); + Selector { jobs: self, name } + } + + pub(crate) fn select(&mut self, name: impl AsRef) -> Selector<'_> { + Selector { jobs: self, name: name.as_ref().to_owned() } + } + + pub(crate) async fn run_all(self) -> Result<()> { + self.map + .into_values() + .map(Job::run) + .collect::>() + .try_collect::<()>() + .await + } +} + +impl Job { + async fn run(self) -> Result<()> { + let result: Result<(), RecvError> = self + .wait_for + .into_iter() + .collect::>() + .try_collect::<()>() + .await; + result.map_err(|_| anyhow!("dependency failed"))?; + + self.future.await?; + for sender in self.notify { + sender.send(()).ok(); + } + Ok(()) + } +} + +impl<'a> Selector<'a> { + #[track_caller] + pub(crate) fn after(self, other: impl AsRef) -> Self { + let (sender, receiver) = oneshot::channel(); + self.jobs + .map + .get_mut(&self.name) + .expect("invalid job name") + .wait_for + .push(receiver); + self.jobs + .map + .get_mut(other.as_ref()) + .expect("invalid job name") + .notify + .push(sender); + self + } +} + +async fn run_job(logger: Logger, name: String, future: F) -> Result<()> +where + F: Future> + 'static, +{ + debug!(logger, "[{}] running task", name); + let start = Instant::now(); + let result = future.await; + let duration = Instant::now().saturating_duration_since(start); + match result { + Ok(()) => { + debug!(logger, "[{}] task succeeded ({:?})", name, duration); + Ok(()) + } + Err(err) => { + error!(logger, "[{}] task failed ({:?})", name, duration); + Err(err) + } + } +} + +async fn spawn_with_output( + mut command: Command, + logger: Logger, + name: String, + log_path: Utf8PathBuf, +) -> Result<()> { + let log_file_1 = File::create(log_path).await?; + let log_file_2 = log_file_1.try_clone().await?; + + debug!(logger, "[{}] running: {}", name, command.to_string()); + let start = Instant::now(); + let mut child = command + .kill_on_drop(true) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("failed to exec `{}`", command.to_string()))?; + + let stdout = reader( + &name, + child.stdout.take().unwrap(), + tokio::io::stdout(), + log_file_1, + ); + let stderr = reader( + &name, + child.stderr.take().unwrap(), + tokio::io::stderr(), + log_file_2, + ); + match tokio::try_join!(child.wait(), stdout, stderr) { + Ok((status, (), ())) => { + debug!( + logger, + "[{}] process exited with {} ({:?})", + name, + status, + Instant::now().saturating_duration_since(start) + ); + command.check_status(status) + } + Err(err) => Err(err).with_context(|| { + format!("I/O error while waiting for job {:?} to complete", name) + }), + } +} + +async fn reader( + name: &str, + reader: impl AsyncRead + Unpin, + mut terminal_writer: impl AsyncWrite + Unpin, + logfile_writer: File, +) -> std::io::Result<()> { + let mut reader = BufReader::new(reader); + let mut logfile_writer = tokio::fs::File::from(logfile_writer); + let mut buf = format!("[{:>16}] ", name).into_bytes(); + let prefix_len = buf.len(); + loop { + buf.truncate(prefix_len); + let size = reader.read_until(b'\n', &mut buf).await?; + if size == 0 { + return Ok(()); + } + terminal_writer.write_all(&buf).await?; + logfile_writer.write_all(&buf[prefix_len..]).await?; + } +} diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs new file mode 100644 index 00000000000..a9811550143 --- /dev/null +++ b/dev-tools/releng/src/main.rs @@ -0,0 +1,450 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +mod cmd; +mod job; + +use std::sync::Arc; + +use anyhow::ensure; +use anyhow::Context; +use anyhow::Result; +use camino::Utf8PathBuf; +use chrono::Utc; +use clap::Parser; +use fs_err::tokio as fs; +use omicron_zone_package::config::Config; +use semver::Version; +use slog::debug; +use slog::info; +use slog::Drain; +use slog::Logger; +use slog_term::FullFormat; +use slog_term::TermDecorator; +use tokio::process::Command; + +use crate::cmd::CommandExt; +use crate::job::Jobs; + +/// The base version we're currently building. Build information is appended to +/// this later on. +/// +/// Under current policy, each new release is a major version bump, and +/// generally referred to only by the major version (e.g. 8.0.0 is referred +/// to as "v8", "version 8", or "release 8" to customers). The use of semantic +/// versioning is mostly to hedge for perhaps wanting something more granular in +/// the future. +const BASE_VERSION: Version = Version::new(8, 0, 0); + +#[derive(Debug, Clone, Copy)] +enum InstallMethod { + /// Unpack the tarball to `/opt/oxide/`, and install + /// `pkg/manifest.xml` (if it exists) to + /// `/lib/svc/manifest/site/.xml`. + Install, + /// Copy the tarball to `/opt/oxide/.tar.gz`. + Bundle, +} + +/// Packages to install or bundle in the host OS image. +const HOST_IMAGE_PACKAGES: [(&str, InstallMethod); 7] = [ + ("mg-ddm-gz", InstallMethod::Install), + ("omicron-sled-agent", InstallMethod::Install), + ("overlay", InstallMethod::Bundle), + ("oxlog", InstallMethod::Install), + ("propolis-server", InstallMethod::Bundle), + ("pumpkind-gz", InstallMethod::Install), + ("switch-asic", InstallMethod::Bundle), +]; +/// Packages to install or bundle in the recovery (trampoline) OS image. +const RECOVERY_IMAGE_PACKAGES: [(&str, InstallMethod); 2] = [ + ("installinator", InstallMethod::Install), + ("mg-ddm-gz", InstallMethod::Install), +]; + +const HELIOS_REPO: &str = "https://pkg.oxide.computer/helios/2/dev/"; +const OPTE_VERSION: &str = include_str!("../../../tools/opte_version"); + +#[derive(Parser)] +struct Args { + /// Path to a Helios repository checkout (default: "helios" in the same + /// directory as "omicron") + #[clap(long)] + helios_path: Option, + + /// ZFS dataset to use for `helios-build` (default: "rpool/images/$LOGNAME") + #[clap(long)] + helios_image_dataset: Option, + + /// Ignore the current HEAD of the Helios repository checkout + #[clap(long)] + ignore_helios_origin: bool, + + /// Output dir for TUF repo and log files (default: "out/releng" in the + /// "omicron" directory) + #[clap(long)] + output_dir: Option, +} + +fn main() -> Result<()> { + let decorator = TermDecorator::new().build(); + let drain = FullFormat::new(decorator).build().fuse(); + let drain = slog_async::Async::new(drain).build().fuse(); + let logger = Logger::root(drain, slog::o!()); + + // Change the working directory to the workspace root. + let workspace_dir = + Utf8PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").context( + "$CARGO_MANIFEST_DIR is not set; run this via `cargo xtask releng`", + )?) + // $CARGO_MANIFEST_DIR is `.../omicron/dev-tools/releng` + .join("../..") + .canonicalize_utf8() + .context("failed to canonicalize workspace dir")?; + info!(logger, "changing working directory to {}", workspace_dir); + std::env::set_current_dir(&workspace_dir) + .context("failed to change working directory to workspace root")?; + + // Unset `CARGO*` and `RUSTUP_TOOLCHAIN`, which will interfere with various + // tools we're about to run. + for (name, _) in std::env::vars_os() { + if name + .to_str() + .map(|s| s.starts_with("CARGO") || s == "RUSTUP_TOOLCHAIN") + .unwrap_or(false) + { + debug!(logger, "unsetting {:?}", name); + std::env::remove_var(name); + } + } + + // Now that we're done mucking about with our environment (something that's + // not necessarily safe in multi-threaded programs), create a Tokio runtime + // and call `do_run`. + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap() + .block_on(do_run(logger, workspace_dir)) +} + +async fn do_run(logger: Logger, workspace_dir: Utf8PathBuf) -> Result<()> { + let args = Args::parse(); + + let helios_dir = args.helios_path.unwrap_or_else(|| { + workspace_dir + .parent() + .expect("omicron repo is not the root directory") + .join("helios") + }); + let output_dir = args + .output_dir + .unwrap_or_else(|| workspace_dir.join("out").join("releng")); + let tempdir = camino_tempfile::tempdir() + .context("failed to create temporary directory")?; + + let commit = Command::new("git") + .args(["rev-parse", "HEAD"]) + .ensure_stdout(&logger) + .await? + .trim() + .to_owned(); + + let mut version = BASE_VERSION.clone(); + // Differentiate between CI and local builds. + version.pre = + if std::env::var_os("CI").is_some() { "0.ci" } else { "0.local" } + .parse()?; + // Set the build metadata to the current commit hash. + let mut build = String::with_capacity(14); + build.push_str("git"); + build.extend(commit.chars().take(11)); + version.build = build.parse()?; + info!(logger, "version: {}", version); + + let manifest = Arc::new(omicron_zone_package::config::parse_manifest( + &fs::read_to_string(workspace_dir.join("package-manifest.toml")) + .await?, + )?); + + // PREFLIGHT ============================================================== + for (package, _) in HOST_IMAGE_PACKAGES + .into_iter() + .chain(RECOVERY_IMAGE_PACKAGES.into_iter()) + { + ensure!( + manifest.packages.contains_key(package), + "package {} to be installed in the OS image \ + is not listed in the package manifest", + package + ); + } + + // Ensure the Helios checkout exists + if helios_dir.exists() { + if !args.ignore_helios_origin { + // check that our helios clone is up to date + Command::new("git") + .arg("-C") + .arg(&helios_dir) + .args(["fetch", "--no-write-fetch-head", "origin", "master"]) + .ensure_success(&logger) + .await?; + let stdout = Command::new("git") + .arg("-C") + .arg(&helios_dir) + .args(["rev-parse", "HEAD", "origin/master"]) + .ensure_stdout(&logger) + .await?; + let mut lines = stdout.lines(); + let first = + lines.next().context("git-rev-parse output was empty")?; + ensure!( + lines.all(|line| line == first), + "helios checkout at {0} is out-of-date; run \ + `git pull -C {0}`, or run omicron-releng with \ + --ignore-helios-origin or --helios-path", + shell_words::quote(helios_dir.as_str()) + ); + } + } else { + info!(logger, "cloning helios to {}", helios_dir); + Command::new("git") + .args(["clone", "https://github.com/oxidecomputer/helios.git"]) + .arg(&helios_dir) + .ensure_success(&logger) + .await?; + } + + // Check that the omicron1 brand is installed + ensure!( + Command::new("pkg") + .args(["verify", "-q", "/system/zones/brand/omicron1/tools"]) + .is_success(&logger) + .await?, + "the omicron1 brand is not installed; install it with \ + `pfexec pkg install /system/zones/brand/omicron1/tools`" + ); + + // Check that the dataset for helios-image to use exists + let helios_image_dataset = match args.helios_image_dataset { + Some(s) => s, + None => format!( + "rpool/images/{}", + std::env::var("LOGNAME") + .context("$LOGNAME is not present in environment")? + ), + }; + ensure!( + Command::new("zfs") + .arg("list") + .arg(&helios_image_dataset) + .is_success(&logger) + .await?, + "the dataset {0} does not exist, which is required for helios-build; \ + run `pfexec zfs create -p {0}`, or run omicron-releng with \ + --helios-image-dataset to specify a different one", + shell_words::quote(&helios_image_dataset) + ); + + fs::create_dir_all(&output_dir).await?; + + // DEFINE JOBS ============================================================ + let mut jobs = Jobs::new(&logger, &output_dir); + + jobs.push_command( + "helios-setup", + Command::new("ptime") + .args(["-m", "gmake", "setup"]) + .current_dir(&helios_dir) + // ?! + .env("PWD", &helios_dir) + // Setting `BUILD_OS` to no makes setup skip repositories we don't need + // for building the OS itself (we are just building an image from an + // already-built OS). + .env("BUILD_OS", "no"), + ); + + jobs.push_command( + "omicron-package", + Command::new("ptime").args([ + "-m", + "cargo", + "build", + "--locked", + "--release", + "--bin", + "omicron-package", + ]), + ); + let omicron_package = workspace_dir.join("target/release/omicron-package"); + + macro_rules! os_image_jobs { + ( + target_name: $target_name:literal, + target_args: $target_args:expr, + proto_packages: $proto_packages:expr, + image_prefix: $image_prefix:literal, + image_build_args: $image_build_args:expr, + ) => { + jobs.push_command( + concat!($target_name, "-target"), + Command::new(&omicron_package) + .args(["--target", $target_name, "target", "create"]) + .args($target_args), + ) + .after("omicron-package"); + + jobs.push_command( + concat!($target_name, "-package"), + Command::new(&omicron_package).args([ + "--target", + $target_name, + "package", + ]), + ) + .after(concat!($target_name, "-target")); + + let proto_dir = tempdir.path().join("proto").join($target_name); + jobs.push( + concat!($target_name, "-proto"), + build_proto_area( + workspace_dir.join("out"), + proto_dir.clone(), + &$proto_packages, + manifest.clone(), + ), + ) + .after(concat!($target_name, "-package")); + + // The ${os_short_commit} token will be expanded by `helios-build` + let image_name = format!( + "{} {}/${{os_short_commit}} {}", + $image_prefix, + commit.chars().take(7).collect::(), + Utc::now().format("%Y-%m-%d %H:%M") + ); + + jobs.push_command( + concat!($target_name, "-image"), + Command::new("ptime") + .arg("-m") + .arg(helios_dir.join("helios-build")) + .arg("experiment-image") + .arg("-o") + .arg(output_dir.join($target_name)) + .arg("-p") + .arg(format!("helios-dev={}", HELIOS_REPO)) + .arg("-F") + .arg(format!("optever={}", OPTE_VERSION.trim())) + .arg("-P") + .arg(proto_dir.join("root")) + .arg("-N") + .arg(image_name) + .args($image_build_args) + .current_dir(&helios_dir), + ) + .after("helios-setup") + .after(concat!($target_name, "-proto")); + }; + } + + os_image_jobs! { + target_name: "recovery", + target_args: ["--image", "trampoline"], + proto_packages: RECOVERY_IMAGE_PACKAGES, + image_prefix: "recovery", + image_build_args: ["-R"], + } + os_image_jobs! { + target_name: "host", + target_args: [ + "--image", + "standard", + "--machine", + "gimlet", + "--switch", + "asic", + "--rack-topology", + "multi-sled" + ], + proto_packages: HOST_IMAGE_PACKAGES, + image_prefix: "ci", + image_build_args: ["-B"], + } + // avoid fighting for the target dir lock + jobs.select("host-package").after("recovery-package"); + // only one helios-build job can run at once + jobs.select("host-image").after("recovery-image"); + + // RUN JOBS =============================================================== + jobs.run_all().await?; + + // fs::create_dir_all(host_proto.path().join("root/root"))?; + // fs::write( + // host_proto.path().join("root/root/.profile"), + // "# Add opteadm, ddadm, oxlog to PATH\n\ + // export PATH=$PATH:/opt/oxide/opte/bin:/opt/oxide/mg-ddm:/opt/oxide/oxlog\n" + // )?; + + Ok(()) +} + +async fn build_proto_area( + package_dir: Utf8PathBuf, + proto_dir: Utf8PathBuf, + packages: &'static [(&'static str, InstallMethod)], + manifest: Arc, +) -> Result<()> { + let opt_oxide = proto_dir.join("root/opt/oxide"); + let manifest_site = proto_dir.join("root/lib/svc/manifest/site"); + fs::create_dir_all(&opt_oxide).await?; + + for &(package_name, method) in packages { + let package = + manifest.packages.get(package_name).expect("checked in preflight"); + match method { + InstallMethod::Install => { + let path = opt_oxide.join(&package.service_name); + fs::create_dir(&path).await?; + + let cloned_path = path.clone(); + let cloned_package_dir = package_dir.to_owned(); + tokio::task::spawn_blocking(move || -> Result<()> { + let mut archive = tar::Archive::new(std::fs::File::open( + cloned_package_dir + .join(package_name) + .with_extension("tar"), + )?); + archive.unpack(cloned_path).with_context(|| { + format!("failed to extract {}.tar.gz", package_name) + })?; + Ok(()) + }) + .await??; + + let smf_manifest = path.join("pkg").join("manifest.xml"); + if smf_manifest.exists() { + fs::create_dir_all(&manifest_site).await?; + fs::rename( + smf_manifest, + manifest_site + .join(&package.service_name) + .with_extension("xml"), + ) + .await?; + } + } + InstallMethod::Bundle => { + fs::copy( + package_dir.join(format!("{}.tar.gz", package_name)), + opt_oxide.join(format!("{}.tar.gz", package.service_name)), + ) + .await?; + } + } + } + + Ok(()) +} diff --git a/package-manifest.toml b/package-manifest.toml index 55e6f78221a..98cd5035d0f 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -592,7 +592,7 @@ only_for_targets.image = "standard" only_for_targets.switch = "asic" [package.pumpkind-gz] -service_name = "pumpkind-gz" +service_name = "pumpkind" source.type = "prebuilt" source.repo = "pumpkind" source.commit = "3fe9c306590fb2f28f54ace7fd18b3c126323683" diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index d15a2242be4..dc761a07761 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -47,6 +47,7 @@ either = { version = "1.11.0" } elliptic-curve = { version = "0.13.8", features = ["ecdh", "hazmat", "pem", "std"] } ff = { version = "0.13.0", default-features = false, features = ["alloc"] } flate2 = { version = "1.0.28" } +fs-err = { version = "2.11.0", default-features = false, features = ["tokio"] } futures = { version = "0.3.30" } futures-channel = { version = "0.3.30", features = ["sink"] } futures-core = { version = "0.3.30" } @@ -155,6 +156,7 @@ either = { version = "1.11.0" } elliptic-curve = { version = "0.13.8", features = ["ecdh", "hazmat", "pem", "std"] } ff = { version = "0.13.0", default-features = false, features = ["alloc"] } flate2 = { version = "1.0.28" } +fs-err = { version = "2.11.0", default-features = false, features = ["tokio"] } futures = { version = "0.3.30" } futures-channel = { version = "0.3.30", features = ["sink"] } futures-core = { version = "0.3.30" }