Scaffold S3 remote persistence interface

Cargo.toml

@@ -87,6 +87,10 @@ nix = "0.26.2"
 
 regex = "1.7.3"
 
+# Enabled with the "s3" feature.
+aws-config = "0.55.0"
+aws-sdk-s3 = "0.25.0"
+
 [workspace.dependencies.uuid]
 version = "1.3.0"
 features = [

bigtest/benchmark_harness.js

@@ -44,7 +44,7 @@ const VERSION_FILE = `${STATEDIR}/version`;
 
 const BATCH_SIZE = 14;
 const EXPECTED_NUM_TESTS = 500;
-const MAX_WORKER_OVERHEAD_PCT = 10; // 10%
+const MAX_WORKER_OVERHEAD_PCT = 16; // 16%
 
 function runBenchmark(
   {

crates/abq_cli/Cargo.toml

@@ -64,3 +64,4 @@ regex.workspace = true
 
 [features]
 test-abq-jest = []
+s3 = [ "abq_queue/s3" ]

crates/abq_queue/Cargo.toml

@@ -25,6 +25,15 @@ async-trait.workspace = true
 
 anyhow.workspace = true
 
+aws-config = { workspace = true, optional = true }
+aws-sdk-s3 = { workspace = true, optional = true }
+
+[features]
+s3 = [
+  "dep:aws-config",
+  "dep:aws-sdk-s3",
+]
+
 [dev-dependencies]
 abq_utils = { path = "../abq_utils", features = ["expose-native-protocols"] }
 abq_test_utils = { path = "../abq_test_support/abq_test_utils" }

crates/abq_queue/src/persistence.rs

@@ -1,2 +1,4 @@
 pub mod manifest;
 pub mod results;
+
+pub mod remote;

crates/abq_queue/src/persistence/remote.rs

@@ -0,0 +1,40 @@
+//! Utilities for remote persistence of [results] and [manifest]s.
+//!
+//! [results]: super::results
+//! [manifest]: super::manifest
+
+use std::path::Path;
+
+use abq_utils::{error::OpaqueResult, net_protocol::workers::RunId};
+use async_trait::async_trait;
+
+#[cfg(feature = "s3")]
+mod s3;
+
+#[cfg(feature = "s3")]
+pub use s3::{S3Client, S3Persister};
+
+pub enum PersistenceKind {
+    Manifest,
+    Results,
+}
+
+#[async_trait]
+pub trait RemotePersistence {
+    /// Stores a file from the local filesystem to the remote persistence.
+    async fn store(
+        &self,
+        kind: PersistenceKind,
+        run_id: RunId,
+        from_local_path: &Path,
+    ) -> OpaqueResult<()>;
+
+    /// Loads a file from the remote persistence to the local filesystem.
+    /// The given local path must have all intermediate directories already created.
+    async fn load(
+        &self,
+        kind: PersistenceKind,
+        run_id: RunId,
+        into_local_path: &Path,
+    ) -> OpaqueResult<()>;
+}