Restructure package format, add Composite Packages

Cargo.toml

@@ -9,6 +9,7 @@ description = "Packaging tools for Oxide's control plane software"
 
 [dependencies]
 anyhow = "1.0"
+async-trait = "0.1.57"
 chrono = "0.4.19"
 filetime = "0.2.16"
 flate2 = "1.0.24"

src/blob.rs

@@ -0,0 +1,98 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Tools for downloading blobs
+
+use anyhow::{anyhow, bail, Result};
+use chrono::{DateTime, FixedOffset, Utc};
+use reqwest::header::{CONTENT_LENGTH, LAST_MODIFIED};
+use std::path::Path;
+use std::str::FromStr;
+use tokio::io::AsyncWriteExt;
+
+// Path to the blob S3 Bucket.
+const S3_BUCKET: &str = "https://oxide-omicron-build.s3.amazonaws.com";
+// Name for the directory component where downloaded blobs are stored.
+pub(crate) const BLOB: &str = "blob";
+
+// Downloads "source" from S3_BUCKET to "destination".
+pub async fn download(source: &str, destination: &Path) -> Result<()> {
+    let url = format!("{}/{}", S3_BUCKET, source);
+    let client = reqwest::Client::new();
+
+    if destination.exists() {
+        // If destination exists, check against size and last modified time. If
+        // both are the same, then return Ok
+        let head_response = client.head(&url).send().await?;
+        if !head_response.status().is_success() {
+            bail!("head failed! {:?}", head_response);
+        }
+
+        let headers = head_response.headers();
+
+        // From S3, header looks like:
+        //
+        //    "Content-Length: 49283072"
+        let content_length = headers
+            .get(CONTENT_LENGTH)
+            .ok_or_else(|| anyhow!("no content length on {} HEAD response!", url))?;
+        let content_length: u64 = u64::from_str(content_length.to_str()?)?;
+
+        // From S3, header looks like:
+        //
+        //    "Last-Modified: Fri, 27 May 2022 20:50:17 GMT"
+        let last_modified = headers
+            .get(LAST_MODIFIED)
+            .ok_or_else(|| anyhow!("no last modified on {} HEAD response!", url))?;
+        let last_modified: DateTime<FixedOffset> =
+            chrono::DateTime::parse_from_rfc2822(last_modified.to_str()?)?;
+        let metadata = tokio::fs::metadata(&destination).await?;
+        let metadata_modified: DateTime<Utc> = metadata.modified()?.into();
+
+        if metadata.len() == content_length && metadata_modified == last_modified {
+            return Ok(());
+        }
+    }
+
+    println!(
+        "Downloading {} to {}",
+        source,
+        destination.to_string_lossy()
+    );
+
+    let response = client.get(url).send().await?;
+
+    // Store modified time from HTTPS response
+    let last_modified = response
+        .headers()
+        .get(LAST_MODIFIED)
+        .ok_or_else(|| anyhow!("no last modified on GET response!"))?;
+    let last_modified: DateTime<FixedOffset> =
+        chrono::DateTime::parse_from_rfc2822(last_modified.to_str()?)?;
+
+    // Write file bytes to destination
+    let mut file = tokio::fs::File::create(destination).await?;
+    file.write_all(&response.bytes().await?).await?;
+    drop(file);
+
+    // Set destination file's modified time based on HTTPS response
+    filetime::set_file_mtime(
+        destination,
+        filetime::FileTime::from_system_time(last_modified.into()),
+    )?;
+
+    Ok(())
+}
+
+#[test]
+fn test_converts() {
+    let content_length = "1966080";
+    let last_modified = "Fri, 30 Apr 2021 22:37:39 GMT";
+
+    let content_length: u64 = u64::from_str(content_length).unwrap();
+    assert_eq!(1966080, content_length);
+
+    let _last_modified: DateTime<FixedOffset> =
+        chrono::DateTime::parse_from_rfc2822(last_modified).unwrap();
+}

src/config.rs

@@ -4,49 +4,42 @@
 
 //! Configuration for a package.
 
-use crate::package::Package;
+use crate::package::{Package, PackageOutput};
+use crate::target::Target;
 use serde_derive::Deserialize;
 use std::collections::BTreeMap;
 use std::path::Path;
 use thiserror::Error;
 
-/// Describes the origin of an externally-built package.
-#[derive(Deserialize, Debug)]
-#[serde(tag = "type", rename_all = "lowercase")]
-pub enum ExternalPackageSource {
-    /// Downloads the package from the following URL:
-    ///
-    /// <https://buildomat.eng.oxide.computer/public/file/oxidecomputer/REPO/image/COMMIT/PACKAGE>
-    Prebuilt {
-        repo: String,
-        commit: String,
-        sha256: String,
-    },
-    /// Expects that a package will be manually built and placed into the output
-    /// directory.
-    Manual,
-}
-
-/// Describes a package which originates from outside this repo.
-#[derive(Deserialize, Debug)]
-pub struct ExternalPackage {
-    #[serde(flatten)]
-    pub package: Package,
-
-    pub source: ExternalPackageSource,
-}
-
 /// Describes the configuration for a set of packages.
 #[derive(Deserialize, Debug)]
 pub struct Config {
     /// Packages to be built and installed.
     #[serde(default, rename = "package")]
     pub packages: BTreeMap<String, Package>,
+}
 
-    /// Packages to be installed, but which have been created outside this
-    /// repository.
-    #[serde(default, rename = "external_package")]
-    pub external_packages: BTreeMap<String, ExternalPackage>,
+impl Config {
+    /// Returns target packages to be assembled on the builder machine.
+    pub fn packages_to_build(&self, target: &Target) -> BTreeMap<&String, &Package> {
+        self.packages
+            .iter()
+            .filter(|(_, pkg)| target.includes_package(&pkg))
+            .map(|(name, pkg)| (name, pkg))
+            .collect()
+    }
+
+    /// Returns target packages which should execute on the deployment machine.
+    pub fn packages_to_deploy(&self, target: &Target) -> BTreeMap<&String, &Package> {
+        let all_packages = self.packages_to_build(target);
+        all_packages
+            .into_iter()
+            .filter(|(_, pkg)| match pkg.output {
+                PackageOutput::Zone { intermediate_only } => !intermediate_only,
+                PackageOutput::Tarball => true,
+            })
+            .collect()
+    }
 }
 
 /// Errors which may be returned when parsing the server configuration.

src/lib.rs

@@ -2,5 +2,8 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
+pub mod blob;
 pub mod config;
 pub mod package;
+pub mod progress;
+pub mod target;