From df827c3af43422caa60ddd065de6f8046f896460 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Tue, 2 Jun 2026 22:04:12 -0700 Subject: [PATCH 1/5] feat(federation): add multistore-federation crate for outbound OIDC->backend STS New `multistore-federation` crate: the symmetric outbound half of multistore's existing inbound federation (multistore-sts / multistore-oidc-provider). Lets the proxy present its own OIDC identity to a backend cloud and assume a role there, so it can serve a private bucket without the operator holding long-lived keys. - aws: runtime-agnostic AssumeRoleWithWebIdentity request builder + XML response parser (caller owns HTTP), surfacing AWS as a typed error. - FederatedCredentials::apply_to(&mut BucketConfig): inject temp creds (access_key_id/secret_access_key/token), clear skip_signature, disable anonymous_access. Debug redacts secret + session token. No core changes; additive crate only. Tests cover parse, error path, redaction, request shape, and BucketConfig integration. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 12 ++ Cargo.toml | 2 + crates/federation/Cargo.toml | 14 ++ crates/federation/src/aws.rs | 248 +++++++++++++++++++++++++++ crates/federation/src/credentials.rs | 58 +++++++ crates/federation/src/error.rs | 26 +++ crates/federation/src/lib.rs | 100 +++++++++++ 7 files changed, 460 insertions(+) create mode 100644 crates/federation/Cargo.toml create mode 100644 crates/federation/src/aws.rs create mode 100644 crates/federation/src/credentials.rs create mode 100644 crates/federation/src/error.rs create mode 100644 crates/federation/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index a83ea68..8b61478 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1191,6 +1191,18 @@ dependencies = [ "worker", ] +[[package]] +name = "multistore-federation" +version = "0.4.0" +dependencies = [ + "chrono", + "multistore", + "quick-xml 0.37.5", + "serde", + "thiserror", + "url", +] + [[package]] name = "multistore-lambda" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index beeed22..897001b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/static-config", "crates/sts", "crates/oidc-provider", + "crates/federation", "examples/server", "examples/lambda", "examples/cf-workers", @@ -18,6 +19,7 @@ default-members = [ "crates/static-config", "crates/sts", "crates/oidc-provider", + "crates/federation", "examples/server", "examples/lambda", ] diff --git a/crates/federation/Cargo.toml b/crates/federation/Cargo.toml new file mode 100644 index 0000000..0451196 --- /dev/null +++ b/crates/federation/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "multistore-federation" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Outbound credential federation (OIDC identity -> backend cloud STS) for the multistore S3 proxy gateway" + +[dependencies] +multistore.workspace = true +chrono.workspace = true +serde.workspace = true +quick-xml.workspace = true +url.workspace = true +thiserror.workspace = true diff --git a/crates/federation/src/aws.rs b/crates/federation/src/aws.rs new file mode 100644 index 0000000..3934ed1 --- /dev/null +++ b/crates/federation/src/aws.rs @@ -0,0 +1,248 @@ +//! AWS STS `AssumeRoleWithWebIdentity` federation. +//! +//! This module is **runtime-agnostic**: it builds the request (URL + form body) +//! and parses the XML response, but does not perform the HTTP call itself. +//! Multistore deployments differ in their HTTP stack (reqwest on native, +//! `web_sys::fetch` on Cloudflare Workers), so the caller owns the transport. +//! +//! ```no_run +//! use multistore_federation::aws::AssumeRoleWithWebIdentity; +//! +//! let req = AssumeRoleWithWebIdentity { +//! role_arn: "arn:aws:iam::123456789012:role/my-role", +//! web_identity_token: "", +//! role_session_name: "multistore", +//! duration_seconds: 3600, +//! session_policy: None, +//! }; +//! let url = AssumeRoleWithWebIdentity::endpoint("us-east-1"); +//! let body = req.body(); +//! // POST `body` to `url` with content-type application/x-www-form-urlencoded, +//! // then: let creds = multistore_federation::aws::parse_response(&response_text)?; +//! # Ok::<(), multistore_federation::FederationError>(()) +//! ``` + +use crate::credentials::FederatedCredentials; +use crate::error::FederationError; +use chrono::{DateTime, Utc}; +use serde::Deserialize; + +/// Parameters for an `AssumeRoleWithWebIdentity` request. +/// +/// The web identity token is the OIDC assertion minted by the proxy (e.g. via +/// `multistore-oidc-provider`); the role's trust policy must trust the proxy's +/// issuer and may condition on the token's `aud`/`sub`. +#[derive(Debug, Clone)] +pub struct AssumeRoleWithWebIdentity<'a> { + /// ARN of the role to assume. + pub role_arn: &'a str, + /// The OIDC token presented as the web identity. + pub web_identity_token: &'a str, + /// Session name recorded in CloudTrail for this assumption. + pub role_session_name: &'a str, + /// Requested credential lifetime, in seconds (AWS clamps to the role's max). + pub duration_seconds: u32, + /// Optional inline session policy (further restricts the session, e.g. to a + /// key prefix) — `None` to use the role's permissions as-is. + pub session_policy: Option<&'a str>, +} + +impl<'a> AssumeRoleWithWebIdentity<'a> { + /// The regional STS endpoint URL to POST to. + /// + /// Regional endpoints are preferred over the global one; for non-standard + /// partitions (GovCloud, China) build the URL yourself. + pub fn endpoint(region: &str) -> String { + format!("https://sts.{region}.amazonaws.com/") + } + + /// The `application/x-www-form-urlencoded` request body. + pub fn body(&self) -> String { + let mut s = url::form_urlencoded::Serializer::new(String::new()); + s.append_pair("Action", "AssumeRoleWithWebIdentity"); + s.append_pair("Version", "2011-06-15"); + s.append_pair("RoleArn", self.role_arn); + s.append_pair("RoleSessionName", self.role_session_name); + s.append_pair("WebIdentityToken", self.web_identity_token); + s.append_pair("DurationSeconds", &self.duration_seconds.to_string()); + if let Some(policy) = self.session_policy { + s.append_pair("Policy", policy); + } + s.finish() + } +} + +/// Parse the body of an `AssumeRoleWithWebIdentity` response into credentials. +/// +/// AWS returns the same XML shape regardless of HTTP status on the error path +/// (an `` document), so this inspects the body rather than +/// relying on the status code: an error document becomes +/// [`FederationError::Sts`] carrying the provider's code and message. +pub fn parse_response(xml: &str) -> Result { + if xml.contains(", +} + +#[derive(Deserialize)] +struct ErrorResponse { + #[serde(rename = "Error")] + error: ErrorDetail, +} + +#[derive(Deserialize)] +struct ErrorDetail { + #[serde(rename = "Code")] + code: String, + #[serde(rename = "Message")] + message: String, +} + +#[cfg(test)] +mod tests { + use super::*; + + const SUCCESS: &str = r#" + + + scv1:conn:test + source-coop-data-proxy + + arn:aws:sts::123456789012:assumed-role/my-role/multistore + AROAEXAMPLE:multistore + + + ASIAEXAMPLE + secret/+key + sess+tok/en== + 2026-06-03T04:13:40Z + + https://data.source.coop + + + 11111111-2222-3333-4444-555555555555 + +"#; + + const ERROR: &str = r#" + + + Sender + InvalidIdentityToken + No OpenIDConnect provider found in your account for https://data.source.coop + + aaaa +"#; + + #[test] + fn parses_credentials() { + let creds = parse_response(SUCCESS).expect("should parse"); + assert_eq!(creds.access_key_id, "ASIAEXAMPLE"); + assert_eq!(creds.secret_access_key, "secret/+key"); + assert_eq!(creds.session_token, "sess+tok/en=="); + assert_eq!(creds.expiration.to_rfc3339(), "2026-06-03T04:13:40+00:00"); + } + + #[test] + fn surfaces_sts_error_as_typed_error() { + match parse_response(ERROR) { + Err(FederationError::Sts { code, message }) => { + assert_eq!(code, "InvalidIdentityToken"); + assert!(message.contains("No OpenIDConnect provider")); + } + other => panic!("expected Sts error, got {other:?}"), + } + } + + #[test] + fn debug_redacts_secrets() { + let creds = parse_response(SUCCESS).unwrap(); + let dbg = format!("{creds:?}"); + assert!(dbg.contains("ASIAEXAMPLE")); + assert!(!dbg.contains("secret/+key")); + assert!(!dbg.contains("sess+tok/en")); + assert!(dbg.contains("[REDACTED]")); + } + + #[test] + fn body_contains_expected_params() { + let req = AssumeRoleWithWebIdentity { + role_arn: "arn:aws:iam::123456789012:role/my-role", + web_identity_token: "tok.tok.tok", + role_session_name: "multistore", + duration_seconds: 3600, + session_policy: None, + }; + let body = req.body(); + assert!(body.contains("Action=AssumeRoleWithWebIdentity")); + assert!(body.contains("Version=2011-06-15")); + assert!(body.contains("DurationSeconds=3600")); + // RoleArn is percent-encoded (`:` and `/`). + assert!(body.contains("RoleArn=arn%3Aaws%3Aiam%3A%3A123456789012%3Arole%2Fmy-role")); + assert!(body.contains("WebIdentityToken=tok.tok.tok")); + assert!(!body.contains("Policy=")); + } + + #[test] + fn body_includes_session_policy_when_present() { + let req = AssumeRoleWithWebIdentity { + role_arn: "arn:aws:iam::1:role/r", + web_identity_token: "t", + role_session_name: "s", + duration_seconds: 900, + session_policy: Some("{\"Version\":\"2012-10-17\"}"), + }; + assert!(req.body().contains("Policy=")); + } + + #[test] + fn endpoint_is_regional() { + assert_eq!( + AssumeRoleWithWebIdentity::endpoint("us-west-2"), + "https://sts.us-west-2.amazonaws.com/" + ); + } +} diff --git a/crates/federation/src/credentials.rs b/crates/federation/src/credentials.rs new file mode 100644 index 0000000..bd5cb55 --- /dev/null +++ b/crates/federation/src/credentials.rs @@ -0,0 +1,58 @@ +//! Short-lived credentials obtained by federating into a backend cloud, and +//! the glue that injects them into a [`BucketConfig`]. + +use chrono::{DateTime, Utc}; +use multistore::types::BucketConfig; +use std::fmt; + +/// Temporary credentials for a backend object store, obtained by exchanging an +/// OIDC assertion at the backend cloud's STS (e.g. AWS +/// `AssumeRoleWithWebIdentity`). +/// +/// These are *backend* credentials — distinct from +/// [`multistore::types::TemporaryCredentials`], which are minted by the proxy's +/// own STS for callers. They carry only what an object-store client needs to +/// sign requests, plus the expiry so a caller can cache and refresh them. +#[derive(Clone)] +pub struct FederatedCredentials { + /// Temporary access key id (AWS `ASIA…`). + pub access_key_id: String, + /// Temporary secret access key. + pub secret_access_key: String, + /// Session token that must accompany requests using these credentials. + pub session_token: String, + /// When these credentials expire. + pub expiration: DateTime, +} + +impl FederatedCredentials { + /// Inject these credentials into a [`BucketConfig`] so the multistore + /// backend signs requests with them instead of going anonymous. + /// + /// Sets the canonical S3 option keys (`access_key_id`, `secret_access_key`, + /// and `token` — the alias object_store maps to the session token and that + /// multistore redacts in logs), clears `skip_signature`, and disables + /// `anonymous_access`. + pub fn apply_to(&self, config: &mut BucketConfig) { + let opts = &mut config.backend_options; + opts.insert("access_key_id".to_string(), self.access_key_id.clone()); + opts.insert( + "secret_access_key".to_string(), + self.secret_access_key.clone(), + ); + opts.insert("token".to_string(), self.session_token.clone()); + opts.remove("skip_signature"); + config.anonymous_access = false; + } +} + +impl fmt::Debug for FederatedCredentials { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FederatedCredentials") + .field("access_key_id", &self.access_key_id) + .field("secret_access_key", &"[REDACTED]") + .field("session_token", &"[REDACTED]") + .field("expiration", &self.expiration) + .finish() + } +} diff --git a/crates/federation/src/error.rs b/crates/federation/src/error.rs new file mode 100644 index 0000000..426e170 --- /dev/null +++ b/crates/federation/src/error.rs @@ -0,0 +1,26 @@ +//! Errors produced while federating into a backend cloud's STS. + +use thiserror::Error; + +/// Failure modes of an outbound federation exchange. +#[derive(Debug, Error)] +pub enum FederationError { + /// The STS endpoint returned an error document instead of credentials. + /// + /// The `code`/`message` come straight from the provider (e.g. AWS + /// `InvalidIdentityToken` / "No OpenIDConnect provider found…") and are + /// the most useful signal when diagnosing a trust-policy or issuer + /// misconfiguration. + #[error("STS returned an error: {code}: {message}")] + Sts { + /// Provider error code (e.g. `InvalidIdentityToken`). + code: String, + /// Human-readable provider message. + message: String, + }, + + /// The response could not be parsed as either a success or an error + /// document. + #[error("failed to parse STS response: {0}")] + Parse(String), +} diff --git a/crates/federation/src/lib.rs b/crates/federation/src/lib.rs new file mode 100644 index 0000000..b4d16ef --- /dev/null +++ b/crates/federation/src/lib.rs @@ -0,0 +1,100 @@ +//! Outbound credential federation for the multistore S3 proxy gateway. +//! +//! Multistore already federates *inbound* identity — callers exchange an OIDC +//! token for proxy credentials via [`multistore-sts`], and the proxy can act as +//! an OIDC provider via [`multistore-oidc-provider`]. This crate is the +//! symmetric *outbound* half: letting the proxy present its own identity to a +//! **backend cloud** and assume a role there, so it can serve data from a +//! private bucket the operator doesn't hold long-lived keys for. +//! +//! The flow, per backend, at bucket-resolution time: +//! +//! 1. Mint a short-lived OIDC assertion with the proxy's signing key +//! (`multistore-oidc-provider`), scoped via its `aud`/`sub` claims. +//! 2. Exchange it at the backend cloud's STS — for AWS, [`aws`]'s +//! `AssumeRoleWithWebIdentity` — for temporary [`FederatedCredentials`]. +//! 3. [`FederatedCredentials::apply_to`] those onto the [`BucketConfig`] so the +//! multistore backend signs requests with them instead of going anonymous. +//! +//! No long-lived backend secret is stored anywhere: the bucket config only +//! needs a role ARN, and the assumed credentials are short-lived and refreshed +//! before expiry. Trust and blast radius are governed by the backend role's +//! trust and permission policies, which the bucket owner controls. +//! +//! This crate is **runtime-agnostic** — it builds requests and parses +//! responses but does not perform HTTP, leaving transport to the caller (the +//! same split multistore uses elsewhere for native vs. Cloudflare Workers). +//! +//! [`multistore-sts`]: https://docs.rs/multistore-sts +//! [`multistore-oidc-provider`]: https://docs.rs/multistore-oidc-provider +//! [`BucketConfig`]: multistore::types::BucketConfig + +pub mod aws; +mod credentials; +mod error; + +pub use credentials::FederatedCredentials; +pub use error::FederationError; + +#[cfg(test)] +mod tests { + use super::*; + use chrono::{TimeZone, Utc}; + use multistore::types::BucketConfig; + use std::collections::HashMap; + + fn anon_s3_bucket() -> BucketConfig { + let mut backend_options = HashMap::new(); + backend_options.insert("bucket_name".to_string(), "my-bucket".to_string()); + backend_options.insert("region".to_string(), "us-west-2".to_string()); + backend_options.insert("skip_signature".to_string(), "true".to_string()); + BucketConfig { + name: "acct:product".to_string(), + backend_type: "s3".to_string(), + backend_prefix: None, + anonymous_access: true, + allowed_roles: vec![], + backend_options, + } + } + + #[test] + fn apply_to_signs_the_bucket() { + let creds = FederatedCredentials { + access_key_id: "ASIA123".to_string(), + secret_access_key: "secret".to_string(), + session_token: "session".to_string(), + expiration: Utc.with_ymd_and_hms(2026, 6, 3, 4, 13, 40).unwrap(), + }; + + let mut config = anon_s3_bucket(); + creds.apply_to(&mut config); + + assert_eq!(config.option("access_key_id"), Some("ASIA123")); + assert_eq!(config.option("secret_access_key"), Some("secret")); + // `token` is the alias object_store maps to the session token and that + // multistore redacts in `BucketConfig`'s Debug impl. + assert_eq!(config.option("token"), Some("session")); + // Anonymous/unsigned access must be turned off so the backend signs. + assert_eq!(config.option("skip_signature"), None); + assert!(!config.anonymous_access); + // Untouched options remain. + assert_eq!(config.option("bucket_name"), Some("my-bucket")); + } + + #[test] + fn bucket_debug_redacts_applied_secrets() { + let creds = FederatedCredentials { + access_key_id: "ASIA123".to_string(), + secret_access_key: "super-secret".to_string(), + session_token: "super-session".to_string(), + expiration: Utc.with_ymd_and_hms(2026, 6, 3, 4, 13, 40).unwrap(), + }; + let mut config = anon_s3_bucket(); + creds.apply_to(&mut config); + + let dbg = format!("{config:?}"); + assert!(!dbg.contains("super-secret")); + assert!(!dbg.contains("super-session")); + } +} From 39bf6ed0d2dddbf40a19cf92d4a212d591f75116 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Tue, 2 Jun 2026 23:30:52 -0700 Subject: [PATCH 2/5] refactor(federation): rename to backend-federation and dedupe via oidc-provider delegation Resolves the duplication between this crate and multistore-oidc-provider's existing (wired-in) outbound exchange. - Rename multistore-federation -> multistore-backend-federation: the outbound AssumeRoleWithWebIdentity *client*, conceptual peer to multistore-sts's inbound *server*. - Make multistore-oidc-provider delegate its AWS STS exchange to this crate (request build via form_pairs() + parse_response), deleting its hand-rolled extract_xml_value parser. Wired-in get_credentials + AwsBackendAuth middleware behavior is preserved (all existing tests pass). - Carry the primitive's improvements through delegation: DurationSeconds (now Option, omitted when None), inline session policy, typed StsError {code,message}, and secret-redacting Debug on CloudCredentials. - Fix apply_to to leave inbound anonymous_access untouched (it governs only outbound backend signing). - Add an env-gated live functional test against real AWS STS + S3 (tests/live_sts.rs) plus a CI job to run it, a documented federation bucket in the cf-workers wrangler.toml, and a crate README. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 24 +++ Cargo.lock | 30 ++-- Cargo.toml | 5 +- crates/backend-federation/Cargo.toml | 23 +++ crates/backend-federation/README.md | 47 ++++++ .../src/aws.rs | 116 ++++++++++--- .../src/credentials.rs | 11 +- .../src/error.rs | 4 +- .../src/lib.rs | 33 ++-- crates/backend-federation/tests/live_sts.rs | 152 ++++++++++++++++++ crates/federation/Cargo.toml | 14 -- crates/oidc-provider/Cargo.toml | 1 + crates/oidc-provider/src/exchange/aws.rs | 127 ++++++--------- crates/oidc-provider/src/lib.rs | 38 ++++- examples/cf-workers/wrangler.toml | 27 ++++ 15 files changed, 500 insertions(+), 152 deletions(-) create mode 100644 crates/backend-federation/Cargo.toml create mode 100644 crates/backend-federation/README.md rename crates/{federation => backend-federation}/src/aws.rs (66%) rename crates/{federation => backend-federation}/src/credentials.rs (82%) rename crates/{federation => backend-federation}/src/error.rs (90%) rename crates/{federation => backend-federation}/src/lib.rs (70%) create mode 100644 crates/backend-federation/tests/live_sts.rs delete mode 100644 crates/federation/Cargo.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e78159a..40f120b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -156,3 +156,27 @@ jobs: - name: Run integration tests run: uvx --with pytest,boto3,requests pytest tests/integration/ -v + + live-federation: + name: Live Backend Federation (real AWS STS) + runs-on: ubuntu-latest + # Self-skips when MULTISTORE_TEST_ROLE_ARN is unset, so it is safe to run on + # every push. To activate, set repository Actions variables + # MULTISTORE_TEST_ROLE_ARN / MULTISTORE_TEST_BUCKET / MULTISTORE_TEST_REGION + # and grant the role's trust policy this repo's GitHub OIDC subject. + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1 + with: + toolchain: stable + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + - name: Run live federation test + env: + MULTISTORE_TEST_ROLE_ARN: ${{ vars.MULTISTORE_TEST_ROLE_ARN }} + MULTISTORE_TEST_BUCKET: ${{ vars.MULTISTORE_TEST_BUCKET }} + MULTISTORE_TEST_REGION: ${{ vars.MULTISTORE_TEST_REGION }} + MULTISTORE_TEST_KEY: ${{ vars.MULTISTORE_TEST_KEY }} + run: cargo test -p multistore-backend-federation --test live_sts -- --nocapture diff --git a/Cargo.lock b/Cargo.lock index 8b61478..8e35be5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1144,6 +1144,23 @@ dependencies = [ "uuid", ] +[[package]] +name = "multistore-backend-federation" +version = "0.4.0" +dependencies = [ + "chrono", + "futures", + "multistore", + "object_store", + "quick-xml 0.37.5", + "reqwest", + "serde", + "serde_json", + "thiserror", + "tokio", + "url", +] + [[package]] name = "multistore-cf-workers" version = "0.4.0" @@ -1191,18 +1208,6 @@ dependencies = [ "worker", ] -[[package]] -name = "multistore-federation" -version = "0.4.0" -dependencies = [ - "chrono", - "multistore", - "quick-xml 0.37.5", - "serde", - "thiserror", - "url", -] - [[package]] name = "multistore-lambda" version = "0.4.0" @@ -1243,6 +1248,7 @@ dependencies = [ "base64", "chrono", "multistore", + "multistore-backend-federation", "rand 0.8.5", "rsa", "serde", diff --git a/Cargo.toml b/Cargo.toml index 897001b..17f0ade 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = [ "crates/static-config", "crates/sts", "crates/oidc-provider", - "crates/federation", + "crates/backend-federation", "examples/server", "examples/lambda", "examples/cf-workers", @@ -19,7 +19,7 @@ default-members = [ "crates/static-config", "crates/sts", "crates/oidc-provider", - "crates/federation", + "crates/backend-federation", "examples/server", "examples/lambda", ] @@ -110,3 +110,4 @@ multistore-metering = { path = "crates/metering", version = "0.4.0" } multistore-cf-workers = { path = "crates/cf-workers", version = "0.4.0" } multistore-oidc-provider = { path = "crates/oidc-provider", version = "0.4.0" } multistore-path-mapping = { path = "crates/path-mapping", version = "0.4.0" } +multistore-backend-federation = { path = "crates/backend-federation", version = "0.4.0" } diff --git a/crates/backend-federation/Cargo.toml b/crates/backend-federation/Cargo.toml new file mode 100644 index 0000000..8dbb9eb --- /dev/null +++ b/crates/backend-federation/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "multistore-backend-federation" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Outbound credential federation (OIDC identity -> backend cloud STS) for the multistore S3 proxy gateway" + +[dependencies] +multistore.workspace = true +chrono.workspace = true +serde.workspace = true +quick-xml.workspace = true +url.workspace = true +thiserror.workspace = true + +[dev-dependencies] +# Live, network-touching functional test (`tests/live_sts.rs`) — gated on env +# vars, so it self-skips during ordinary `cargo test`. +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +reqwest.workspace = true +object_store.workspace = true +serde_json.workspace = true +futures.workspace = true diff --git a/crates/backend-federation/README.md b/crates/backend-federation/README.md new file mode 100644 index 0000000..8d57cce --- /dev/null +++ b/crates/backend-federation/README.md @@ -0,0 +1,47 @@ +# multistore-backend-federation + +Outbound credential federation for the [`multistore`](https://crates.io/crates/multistore) S3 proxy gateway. The runtime-agnostic *client* side of AWS STS `AssumeRoleWithWebIdentity`: the proxy presents its own OIDC identity to a **backend cloud**, assumes a role there, and signs backend requests with the temporary credentials — so the operator never holds long-lived backend keys. + +It is the symmetric counterpart to [`multistore-sts`](../sts), which is the inbound `AssumeRoleWithWebIdentity` **server** (minting proxy credentials for callers). + +``` + inbound outbound + caller ──OIDC──▶ multistore-sts multistore-backend-federation ──OIDC──▶ backend cloud STS + ◀─proxy creds─ (server: mint) (client: build req / parse resp) ◀─backend creds─ +``` + +## How It Works + +``` +proxy's OIDC identity (multistore-oidc-provider mints + signs the JWT) + │ + │ self-signed JWT (web identity token) + ▼ +┌──────────────────────────────────────┐ +│ multistore-backend-federation │ +│ │ +│ 1. build AssumeRoleWithWebIdentity │ ← request URL + form body / pairs +│ request (this crate) │ +│ 2. caller POSTs it to backend STS │ ← transport owned by the caller +│ 3. parse_response(xml) │ ← typed creds or typed FederationError::Sts +│ 4. FederatedCredentials::apply_to │ ← inject into BucketConfig.backend_options +└──────────────────────────────────────┘ + │ + │ temporary backend AccessKeyId + SecretAccessKey + SessionToken + ▼ +multistore S3 backend signs requests to the private bucket +``` + +This crate is **mechanism only**: it owns the STS request/response shapes and the `BucketConfig` injection. It does *not* mint the JWT, perform HTTP, cache, or wire middleware — that orchestration lives in [`multistore-oidc-provider`](../oidc-provider), which delegates its AWS exchange to this crate. + +## Relationship to the other auth crates + +| crate | direction | role | +|---|---|---| +| `multistore-sts` | inbound | server: validate caller OIDC token, mint `TemporaryCredentials` | +| `multistore-oidc-provider` | outbound | mint the proxy's own JWT (sign/JWKS/discovery) + cache + middleware | +| `multistore-backend-federation` | outbound | client: build/parse backend STS exchange, inject `FederatedCredentials` | + +## Bring your own token + +Because the crate only depends on `multistore` (core) and a few wire libraries — no RSA/JWKS machinery — a caller that already holds a web-identity token (an external IdP, a workload-identity assertion, a pre-minted JWT) can use it standalone to exchange that token for backend credentials, without pulling in the full OIDC provider. diff --git a/crates/federation/src/aws.rs b/crates/backend-federation/src/aws.rs similarity index 66% rename from crates/federation/src/aws.rs rename to crates/backend-federation/src/aws.rs index 3934ed1..67ab034 100644 --- a/crates/federation/src/aws.rs +++ b/crates/backend-federation/src/aws.rs @@ -5,27 +5,40 @@ //! Multistore deployments differ in their HTTP stack (reqwest on native, //! `web_sys::fetch` on Cloudflare Workers), so the caller owns the transport. //! -//! ```no_run -//! use multistore_federation::aws::AssumeRoleWithWebIdentity; +//! ``` +//! use multistore_backend_federation::aws::{AssumeRoleWithWebIdentity, parse_response}; //! //! let req = AssumeRoleWithWebIdentity { //! role_arn: "arn:aws:iam::123456789012:role/my-role", //! web_identity_token: "", //! role_session_name: "multistore", -//! duration_seconds: 3600, +//! duration_seconds: Some(3600), //! session_policy: None, //! }; +//! assert!(req.body().contains("Action=AssumeRoleWithWebIdentity")); +//! //! let url = AssumeRoleWithWebIdentity::endpoint("us-east-1"); -//! let body = req.body(); -//! // POST `body` to `url` with content-type application/x-www-form-urlencoded, -//! // then: let creds = multistore_federation::aws::parse_response(&response_text)?; -//! # Ok::<(), multistore_federation::FederationError>(()) +//! assert_eq!(url, "https://sts.us-east-1.amazonaws.com/"); +//! +//! // POST `req.body()` (or `req.form_pairs()` if your HTTP client urlencodes +//! // for you) to `url` as application/x-www-form-urlencoded, then parse the reply: +//! let response_xml = r#" +//! +//! ASIAEXAMPLE +//! secret +//! token +//! 2030-01-01T00:00:00Z +//! "#; +//! let creds = parse_response(response_xml)?; +//! assert_eq!(creds.access_key_id, "ASIAEXAMPLE"); +//! # Ok::<(), multistore_backend_federation::FederationError>(()) //! ``` use crate::credentials::FederatedCredentials; use crate::error::FederationError; use chrono::{DateTime, Utc}; use serde::Deserialize; +use std::borrow::Cow; /// Parameters for an `AssumeRoleWithWebIdentity` request. /// @@ -40,8 +53,10 @@ pub struct AssumeRoleWithWebIdentity<'a> { pub web_identity_token: &'a str, /// Session name recorded in CloudTrail for this assumption. pub role_session_name: &'a str, - /// Requested credential lifetime, in seconds (AWS clamps to the role's max). - pub duration_seconds: u32, + /// Requested credential lifetime, in seconds. `None` omits `DurationSeconds` + /// so AWS applies the role's default (3600s); when set, AWS clamps to the + /// role's maximum and rejects values below 900. + pub duration_seconds: Option, /// Optional inline session policy (further restricts the session, e.g. to a /// key prefix) — `None` to use the role's permissions as-is. pub session_policy: Option<&'a str>, @@ -56,19 +71,39 @@ impl<'a> AssumeRoleWithWebIdentity<'a> { format!("https://sts.{region}.amazonaws.com/") } + /// The request as form key/value pairs, with values **unencoded**. + /// + /// Use this when the HTTP layer performs its own form-urlencoding (e.g. + /// reqwest's `.form(...)`); feeding it a pre-encoded [`body`](Self::body) + /// instead would double-encode the values. + pub fn form_pairs(&self) -> Vec<(&'static str, Cow<'a, str>)> { + let mut pairs = vec![ + ("Action", Cow::Borrowed("AssumeRoleWithWebIdentity")), + ("Version", Cow::Borrowed("2011-06-15")), + ("RoleArn", Cow::Borrowed(self.role_arn)), + ("RoleSessionName", Cow::Borrowed(self.role_session_name)), + ("WebIdentityToken", Cow::Borrowed(self.web_identity_token)), + ]; + if let Some(duration) = self.duration_seconds { + pairs.push(("DurationSeconds", Cow::Owned(duration.to_string()))); + } + if let Some(policy) = self.session_policy { + pairs.push(("Policy", Cow::Borrowed(policy))); + } + pairs + } + /// The `application/x-www-form-urlencoded` request body. + /// + /// Use this when the HTTP layer sends a raw body; for a layer that encodes + /// form pairs itself, use [`form_pairs`](Self::form_pairs) to avoid + /// double-encoding. pub fn body(&self) -> String { - let mut s = url::form_urlencoded::Serializer::new(String::new()); - s.append_pair("Action", "AssumeRoleWithWebIdentity"); - s.append_pair("Version", "2011-06-15"); - s.append_pair("RoleArn", self.role_arn); - s.append_pair("RoleSessionName", self.role_session_name); - s.append_pair("WebIdentityToken", self.web_identity_token); - s.append_pair("DurationSeconds", &self.duration_seconds.to_string()); - if let Some(policy) = self.session_policy { - s.append_pair("Policy", policy); + let mut form = url::form_urlencoded::Serializer::new(String::new()); + for (key, value) in self.form_pairs() { + form.append_pair(key, &value); } - s.finish() + form.finish() } } @@ -80,16 +115,14 @@ impl<'a> AssumeRoleWithWebIdentity<'a> { /// [`FederationError::Sts`] carrying the provider's code and message. pub fn parse_response(xml: &str) -> Result { if xml.contains(" Option { + std::env::var(name).ok().filter(|v| !v.is_empty()) +} + +/// Obtain an OIDC token: an explicit one if provided, else a GitHub Actions +/// token. Returns `None` when neither source is configured. +async fn web_identity_token() -> Option { + if let Some(token) = env("MULTISTORE_TEST_WEB_IDENTITY_TOKEN") { + return Some(token); + } + + let req_token = env("ACTIONS_ID_TOKEN_REQUEST_TOKEN")?; + let req_url = env("ACTIONS_ID_TOKEN_REQUEST_URL")?; + let client = reqwest::Client::new(); + let resp = client + .get(format!("{req_url}&audience=sts.amazonaws.com")) + .header("Authorization", format!("bearer {req_token}")) + .send() + .await + .expect("fetch GitHub Actions OIDC token") + .error_for_status() + .expect("GitHub Actions OIDC token request failed"); + let body: serde_json::Value = resp.json().await.expect("parse OIDC token JSON"); + Some( + body.get("value") + .and_then(|v| v.as_str()) + .expect("OIDC token response missing `value`") + .to_string(), + ) +} + +#[tokio::test] +async fn assume_role_and_read_private_bucket() { + let Some(role_arn) = env("MULTISTORE_TEST_ROLE_ARN") else { + eprintln!("skipping live_sts: MULTISTORE_TEST_ROLE_ARN not set"); + return; + }; + let bucket = env("MULTISTORE_TEST_BUCKET") + .expect("MULTISTORE_TEST_BUCKET must be set when MULTISTORE_TEST_ROLE_ARN is"); + let region = env("MULTISTORE_TEST_REGION").unwrap_or_else(|| "us-east-1".to_string()); + + let Some(token) = web_identity_token().await else { + panic!( + "MULTISTORE_TEST_ROLE_ARN is set but no web identity token source is available \ + (set MULTISTORE_TEST_WEB_IDENTITY_TOKEN, or run under GitHub Actions with \ + id-token: write)" + ); + }; + + // ── 1. Build the request with the crate under test and exchange it at + // real AWS STS. The caller owns the HTTP; reqwest urlencodes the + // unencoded `form_pairs()`. + let request = AssumeRoleWithWebIdentity { + role_arn: &role_arn, + web_identity_token: &token, + role_session_name: "multistore-itest", + duration_seconds: Some(900), + session_policy: None, + }; + let pairs = request.form_pairs(); + let form: Vec<(&str, &str)> = pairs.iter().map(|(k, v)| (*k, v.as_ref())).collect(); + + let endpoint = AssumeRoleWithWebIdentity::endpoint(®ion); + let body = reqwest::Client::new() + .post(&endpoint) + .form(&form) + .send() + .await + .expect("POST to AWS STS") + .text() + .await + .expect("read STS response body"); + + // ── 2. Parse with the crate under test. A trust/permission misconfig + // surfaces here as a typed `FederationError::Sts`. + let creds = parse_response(&body) + .unwrap_or_else(|e| panic!("STS exchange failed: {e}\n--- raw response ---\n{body}")); + assert!( + creds.access_key_id.starts_with("ASIA"), + "expected temporary (ASIA…) access key, got {:?}", + creds.access_key_id + ); + assert!(!creds.secret_access_key.is_empty()); + assert!(!creds.session_token.is_empty()); + + // ── 3. Prove the credentials actually authenticate against the private + // bucket, the same way multistore signs backend requests. + use object_store::aws::AmazonS3Builder; + use object_store::{ObjectStore, ObjectStoreExt}; + + let store = AmazonS3Builder::new() + .with_region(®ion) + .with_bucket_name(&bucket) + .with_access_key_id(&creds.access_key_id) + .with_secret_access_key(&creds.secret_access_key) + .with_token(&creds.session_token) + .build() + .expect("build S3 store from federated credentials"); + + if let Some(key) = env("MULTISTORE_TEST_KEY") { + let path = object_store::path::Path::from(key.as_str()); + let got = store + .get(&path) + .await + .unwrap_or_else(|e| panic!("GET {key} with federated creds failed: {e}")); + let bytes = got.bytes().await.expect("read object body"); + assert!(!bytes.is_empty(), "object {key} was empty"); + } else { + // Listing alone proves the credentials authenticate — an auth failure + // errors on the first poll; an empty bucket simply yields no items. + use futures::StreamExt; + let mut stream = store.list(None); + if let Some(item) = stream.next().await { + item.unwrap_or_else(|e| panic!("LIST {bucket} with federated creds failed: {e}")); + } + } +} diff --git a/crates/federation/Cargo.toml b/crates/federation/Cargo.toml deleted file mode 100644 index 0451196..0000000 --- a/crates/federation/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "multistore-federation" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "Outbound credential federation (OIDC identity -> backend cloud STS) for the multistore S3 proxy gateway" - -[dependencies] -multistore.workspace = true -chrono.workspace = true -serde.workspace = true -quick-xml.workspace = true -url.workspace = true -thiserror.workspace = true diff --git a/crates/oidc-provider/Cargo.toml b/crates/oidc-provider/Cargo.toml index 992bcb3..500b44d 100644 --- a/crates/oidc-provider/Cargo.toml +++ b/crates/oidc-provider/Cargo.toml @@ -12,6 +12,7 @@ gcp = [] [dependencies] multistore.workspace = true +multistore-backend-federation.workspace = true async-trait.workspace = true thiserror.workspace = true serde.workspace = true diff --git a/crates/oidc-provider/src/exchange/aws.rs b/crates/oidc-provider/src/exchange/aws.rs index c764745..7571554 100644 --- a/crates/oidc-provider/src/exchange/aws.rs +++ b/crates/oidc-provider/src/exchange/aws.rs @@ -3,6 +3,8 @@ use crate::{CloudCredentials, HttpExchange, OidcProviderError}; use super::CredentialExchange; +use multistore_backend_federation::aws::{parse_response, AssumeRoleWithWebIdentity}; +use multistore_backend_federation::FederatedCredentials; /// Configuration for exchanging a JWT for AWS credentials. #[derive(Debug, Clone)] @@ -15,6 +17,13 @@ pub struct AwsExchange { /// Session name included in the assumed role credentials. pub session_name: String, + + /// Requested credential lifetime, in seconds. `None` lets AWS apply the + /// role's default (3600s); otherwise AWS clamps to the role's maximum. + pub duration_seconds: Option, + + /// Optional inline session policy (JSON) that further restricts the session. + pub session_policy: Option, } impl Default for AwsExchange { @@ -23,6 +32,8 @@ impl Default for AwsExchange { role_arn: String::new(), sts_endpoint: "https://sts.amazonaws.com".into(), session_name: "s3-proxy".into(), + duration_seconds: None, + session_policy: None, } } } @@ -47,97 +58,49 @@ impl AwsExchange { self.session_name = name; self } + + /// Request a specific credential lifetime (seconds); AWS clamps to the role's max. + pub fn with_duration(mut self, seconds: u32) -> Self { + self.duration_seconds = Some(seconds); + self + } + + /// Attach an inline session policy (JSON) that further restricts the session. + pub fn with_session_policy(mut self, policy: String) -> Self { + self.session_policy = Some(policy); + self + } } impl CredentialExchange for AwsExchange { async fn exchange(&self, http: &H, jwt: &str) -> Result { - let form = [ - ("Action", "AssumeRoleWithWebIdentity"), - ("Version", "2011-06-15"), - ("RoleArn", &self.role_arn), - ("RoleSessionName", &self.session_name), - ("WebIdentityToken", jwt), - ]; + // Build the request with the canonical `multistore-backend-federation` + // primitive, hand its (unencoded) pairs to the runtime's HTTP client — + // which form-urlencodes them — then parse the reply with the same crate. + let request = AssumeRoleWithWebIdentity { + role_arn: &self.role_arn, + web_identity_token: jwt, + role_session_name: &self.session_name, + duration_seconds: self.duration_seconds, + session_policy: self.session_policy.as_deref(), + }; + + let pairs = request.form_pairs(); + let form: Vec<(&str, &str)> = pairs.iter().map(|(k, v)| (*k, v.as_ref())).collect(); let body = http.post_form(&self.sts_endpoint, &form).await?; - parse_assume_role_response(&body) + Ok(parse_response(&body)?.into()) } } -/// Parse the XML response from AWS STS `AssumeRoleWithWebIdentity`. -fn parse_assume_role_response(xml: &str) -> Result { - // Extract fields from the STS XML response. - // The response structure is: - // - // - // - // ... - // ... - // ... - // ... - // - // - // - let access_key_id = extract_xml_value(xml, "AccessKeyId")?; - let secret_access_key = extract_xml_value(xml, "SecretAccessKey")?; - let session_token = extract_xml_value(xml, "SessionToken")?; - let expiration_str = extract_xml_value(xml, "Expiration")?; - - let expires_at = chrono::DateTime::parse_from_rfc3339(&expiration_str) - .map_err(|e| OidcProviderError::ExchangeError(format!("invalid Expiration: {e}")))? - .with_timezone(&chrono::Utc); - - Ok(CloudCredentials { - access_key_id, - secret_access_key, - session_token, - expires_at, - }) -} - -/// Simple XML tag value extraction (avoids pulling in a full XML parser). -fn extract_xml_value(xml: &str, tag: &str) -> Result { - let open = format!("<{tag}>"); - let close = format!(""); - let start = xml.find(&open).ok_or_else(|| { - OidcProviderError::ExchangeError(format!("missing <{tag}> in STS response")) - })? + open.len(); - let end = xml[start..].find(&close).ok_or_else(|| { - OidcProviderError::ExchangeError(format!("missing in STS response")) - })? + start; - Ok(xml[start..end].to_string()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_sts_response() { - let xml = r#" - - - - ASIATESTKEYID - testsecretkey - testsessiontoken - 2025-01-15T12:00:00Z - - -"#; - - let creds = parse_assume_role_response(xml).unwrap(); - assert_eq!(creds.access_key_id, "ASIATESTKEYID"); - assert_eq!(creds.secret_access_key, "testsecretkey"); - assert_eq!(creds.session_token, "testsessiontoken"); - assert_eq!(creds.expires_at.to_rfc3339(), "2025-01-15T12:00:00+00:00"); - } - - #[test] - fn parse_sts_response_missing_field() { - let xml = "AK"; - let err = parse_assume_role_response(xml).unwrap_err(); - assert!(err.to_string().contains("SecretAccessKey")); +impl From for CloudCredentials { + fn from(c: FederatedCredentials) -> Self { + Self { + access_key_id: c.access_key_id, + secret_access_key: c.secret_access_key, + session_token: c.session_token, + expires_at: c.expiration, + } } } diff --git a/crates/oidc-provider/src/lib.rs b/crates/oidc-provider/src/lib.rs index 3a20149..76dfb8b 100644 --- a/crates/oidc-provider/src/lib.rs +++ b/crates/oidc-provider/src/lib.rs @@ -29,7 +29,10 @@ use exchange::CredentialExchange; use jwt::JwtSigner; /// Temporary cloud credentials obtained via token exchange. -#[derive(Debug, Clone)] +/// +/// `Debug` redacts the secret access key and session token so credentials are +/// never leaked into logs. +#[derive(Clone)] pub struct CloudCredentials { /// AWS access key ID. Empty string for Azure/GCP (bearer-token-only providers). pub access_key_id: String, @@ -41,6 +44,17 @@ pub struct CloudCredentials { pub expires_at: chrono::DateTime, } +impl std::fmt::Debug for CloudCredentials { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CloudCredentials") + .field("access_key_id", &self.access_key_id) + .field("secret_access_key", &"[REDACTED]") + .field("session_token", &"[REDACTED]") + .field("expires_at", &self.expires_at) + .finish() + } +} + /// HTTP client abstraction for outbound requests (STS token exchange). /// /// Each runtime provides its own implementation — `reqwest` on native, @@ -133,10 +147,32 @@ pub enum OidcProviderError { #[error("credential exchange failed: {0}")] ExchangeError(String), + /// The backend cloud's STS returned an error document (e.g. AWS + /// `InvalidIdentityToken`). The `code`/`message` come from the provider and + /// usually point at a trust-policy or issuer misconfiguration; they are not + /// sanitized, so do not echo them verbatim to untrusted callers. + #[error("STS returned an error: {code}: {message}")] + StsError { + /// Provider error code (e.g. `InvalidIdentityToken`). + code: String, + /// Human-readable provider message. + message: String, + }, + #[error("HTTP error: {0}")] HttpError(String), } +impl From for OidcProviderError { + fn from(e: multistore_backend_federation::FederationError) -> Self { + use multistore_backend_federation::FederationError as F; + match e { + F::Sts { code, message } => OidcProviderError::StsError { code, message }, + F::Parse(e) => OidcProviderError::ExchangeError(e.to_string()), + } + } +} + impl From for multistore::error::ProxyError { fn from(e: OidcProviderError) -> Self { multistore::error::ProxyError::Internal(e.to_string()) diff --git a/examples/cf-workers/wrangler.toml b/examples/cf-workers/wrangler.toml index 5c5cf0b..21533cd 100644 --- a/examples/cf-workers/wrangler.toml +++ b/examples/cf-workers/wrangler.toml @@ -38,6 +38,33 @@ bucket_name = "private-uploads" endpoint = "http://localhost:9000" secret_access_key = "minioadmin" +# Outbound backend federation (multistore-backend-federation via +# multistore-oidc-provider): instead of long-lived backend keys, the proxy mints +# its own OIDC assertion and assumes an AWS IAM role at request time +# (AssumeRoleWithWebIdentity), then signs backend requests with the resulting +# temporary credentials. +# +# Requires: +# - OIDC_PROVIDER_KEY (secret, RSA PEM) and OIDC_PROVIDER_ISSUER (var) set, so +# the worker serves JWKS/discovery and signs assertions. +# - An AWS IAM OIDC identity provider registered for OIDC_PROVIDER_ISSUER +# (AWS fetches its JWKS over HTTPS — the issuer must be publicly reachable). +# - An IAM role whose trust policy trusts that provider (optionally +# conditioning on the `sub` set below) and grants read on the bucket. +[[vars.PROXY_CONFIG.buckets]] +allowed_roles = [] +anonymous_access = true +backend_type = "s3" +name = "federated-private" + +[vars.PROXY_CONFIG.buckets.backend_options] +auth_type = "oidc" +oidc_role_arn = "arn:aws:iam::123456789012:role/multistore-backend-federation" +oidc_subject = "multistore" +bucket_name = "my-private-bucket" +endpoint = "https://s3.us-east-1.amazonaws.com" +region = "us-east-1" + [[vars.PROXY_CONFIG.buckets]] allowed_roles = [] anonymous_access = true From 5399ac42b7ca792df441efeb1be79872cb8aa9b7 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Wed, 3 Jun 2026 09:18:38 -0700 Subject: [PATCH 3/5] ci(backend-federation): make the unconfigured live test honest (skip, not green) The live-federation job was reporting success while the test self-skipped (MULTISTORE_TEST_ROLE_ARN unset), giving a misleading green check that implied the real AWS STS path was exercised when it wasn't. - Gate the live-federation job with `if: vars.MULTISTORE_TEST_ROLE_ARN != ''` so it shows as *skipped* (neutral), not *passed*, when unconfigured. - Mark the test `#[ignore]` so the ordinary unit-test job reports it as *ignored* rather than *passed*; the gated job runs it with `-- --ignored`. The crate's encoding/parsing/redaction logic remains covered by the unit tests; only the real network round-trip is gated on a configured role + bucket. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 11 ++++++++--- crates/backend-federation/tests/live_sts.rs | 7 +++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40f120b..588c549 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,10 +160,13 @@ jobs: live-federation: name: Live Backend Federation (real AWS STS) runs-on: ubuntu-latest - # Self-skips when MULTISTORE_TEST_ROLE_ARN is unset, so it is safe to run on - # every push. To activate, set repository Actions variables + # Gated: this job only runs when MULTISTORE_TEST_ROLE_ARN is configured. When + # it isn't, the job shows as *skipped* (neutral) rather than a misleading + # green "pass", so a passing checks list never implies the live AWS path was + # actually exercised. To activate, set repository Actions variables # MULTISTORE_TEST_ROLE_ARN / MULTISTORE_TEST_BUCKET / MULTISTORE_TEST_REGION # and grant the role's trust policy this repo's GitHub OIDC subject. + if: ${{ vars.MULTISTORE_TEST_ROLE_ARN != '' }} permissions: id-token: write contents: read @@ -179,4 +182,6 @@ jobs: MULTISTORE_TEST_BUCKET: ${{ vars.MULTISTORE_TEST_BUCKET }} MULTISTORE_TEST_REGION: ${{ vars.MULTISTORE_TEST_REGION }} MULTISTORE_TEST_KEY: ${{ vars.MULTISTORE_TEST_KEY }} - run: cargo test -p multistore-backend-federation --test live_sts -- --nocapture + # `--ignored` runs the otherwise-ignored live test; `--include-ignored` + # is not used so nothing else changes. + run: cargo test -p multistore-backend-federation --test live_sts -- --ignored --nocapture diff --git a/crates/backend-federation/tests/live_sts.rs b/crates/backend-federation/tests/live_sts.rs index 6d13824..5b1a98c 100644 --- a/crates/backend-federation/tests/live_sts.rs +++ b/crates/backend-federation/tests/live_sts.rs @@ -64,7 +64,14 @@ async fn web_identity_token() -> Option { ) } +// `#[ignore]` so the ordinary unit-test suite reports this as *ignored*, never +// as a misleading green "passed", when no AWS target is configured. The gated +// `live-federation` CI job runs it with `-- --ignored` only when the repo +// variables are present. Run locally with: +// MULTISTORE_TEST_ROLE_ARN=… MULTISTORE_TEST_BUCKET=… \ +// cargo test -p multistore-backend-federation --test live_sts -- --ignored --nocapture #[tokio::test] +#[ignore = "live AWS test; set MULTISTORE_TEST_ROLE_ARN/BUCKET and run with --ignored"] async fn assume_role_and_read_private_bucket() { let Some(role_arn) = env("MULTISTORE_TEST_ROLE_ARN") else { eprintln!("skipping live_sts: MULTISTORE_TEST_ROLE_ARN not set"); From 3f4d1a8fb8a2f6b5bd2d41eda7c18ff6f2f981d1 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Wed, 3 Jun 2026 09:21:33 -0700 Subject: [PATCH 4/5] ci(backend-federation): fail the live test when the AWS target is unconfigured MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per request: the live-federation job must fail — not skip — when MULTISTORE_TEST_ROLE_ARN is unset, so a green checks list genuinely means the real AWS STS path was exercised. - Drop the job's `if:` guard so it always runs. - The test now panics (instead of returning early) on missing MULTISTORE_TEST_ROLE_ARN, failing the `-- --ignored` run. - Kept `#[ignore]` so ordinary `cargo test` (and the unit-test CI job, and local dev) does not run it and is unaffected. CI's live-federation check will be RED until the repo Actions variables MULTISTORE_TEST_ROLE_ARN / MULTISTORE_TEST_BUCKET / MULTISTORE_TEST_REGION are set and the IAM role trusts this repo's GitHub OIDC subject. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 17 +++++++------- crates/backend-federation/tests/live_sts.rs | 25 +++++++++++---------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 588c549..9661383 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,13 +160,12 @@ jobs: live-federation: name: Live Backend Federation (real AWS STS) runs-on: ubuntu-latest - # Gated: this job only runs when MULTISTORE_TEST_ROLE_ARN is configured. When - # it isn't, the job shows as *skipped* (neutral) rather than a misleading - # green "pass", so a passing checks list never implies the live AWS path was - # actually exercised. To activate, set repository Actions variables - # MULTISTORE_TEST_ROLE_ARN / MULTISTORE_TEST_BUCKET / MULTISTORE_TEST_REGION - # and grant the role's trust policy this repo's GitHub OIDC subject. - if: ${{ vars.MULTISTORE_TEST_ROLE_ARN != '' }} + # Always runs and **fails** when the live AWS target is unconfigured (the + # test panics on missing env), so green CI genuinely means the real AWS STS + # path was exercised — it can't be silently skipped. To make it pass, set + # repository Actions variables MULTISTORE_TEST_ROLE_ARN / MULTISTORE_TEST_BUCKET + # / MULTISTORE_TEST_REGION and grant the role's trust policy this repo's + # GitHub OIDC subject (audience sts.amazonaws.com). permissions: id-token: write contents: read @@ -182,6 +181,6 @@ jobs: MULTISTORE_TEST_BUCKET: ${{ vars.MULTISTORE_TEST_BUCKET }} MULTISTORE_TEST_REGION: ${{ vars.MULTISTORE_TEST_REGION }} MULTISTORE_TEST_KEY: ${{ vars.MULTISTORE_TEST_KEY }} - # `--ignored` runs the otherwise-ignored live test; `--include-ignored` - # is not used so nothing else changes. + # `--ignored` runs the otherwise-ignored live test. It fails the job if + # the required variables above are not set. run: cargo test -p multistore-backend-federation --test live_sts -- --ignored --nocapture diff --git a/crates/backend-federation/tests/live_sts.rs b/crates/backend-federation/tests/live_sts.rs index 5b1a98c..c0f5359 100644 --- a/crates/backend-federation/tests/live_sts.rs +++ b/crates/backend-federation/tests/live_sts.rs @@ -6,14 +6,16 @@ //! credentials actually work by reading the private S3 bucket with them (via //! `object_store`, exactly how multistore uses them). //! -//! It is **gated on environment variables** and self-skips when they are -//! absent, so ordinary `cargo test` (and the unit-test CI job) runs it as a -//! no-op. It only does real work when pointed at a configured role + bucket. +//! It is `#[ignore]`d so it stays out of the ordinary unit-test suite (and +//! local `cargo test`), but the dedicated `live-federation` CI job runs it with +//! `--ignored` on every push and it **panics (fails) when the required +//! environment is absent** — so a configured role + bucket is enforced rather +//! than silently skipped. //! //! ## Required environment //! //! - `MULTISTORE_TEST_ROLE_ARN` — IAM role to assume. **If unset, the test -//! skips.** +//! fails.** //! - `MULTISTORE_TEST_BUCKET` — private S3 bucket the role can read. //! - `MULTISTORE_TEST_REGION` — bucket/STS region (default `us-east-1`). //! - `MULTISTORE_TEST_KEY` — optional object key to `GET`; if unset the test @@ -64,19 +66,18 @@ async fn web_identity_token() -> Option { ) } -// `#[ignore]` so the ordinary unit-test suite reports this as *ignored*, never -// as a misleading green "passed", when no AWS target is configured. The gated -// `live-federation` CI job runs it with `-- --ignored` only when the repo -// variables are present. Run locally with: +// `#[ignore]` keeps this out of the ordinary unit-test suite (it needs real AWS +// and would otherwise fail local `cargo test`). The `live-federation` CI job +// runs it with `-- --ignored` on every push and **fails if the required env +// vars are unset** — a configured role + bucket is enforced, never silently +// skipped. Run locally with: // MULTISTORE_TEST_ROLE_ARN=… MULTISTORE_TEST_BUCKET=… \ // cargo test -p multistore-backend-federation --test live_sts -- --ignored --nocapture #[tokio::test] #[ignore = "live AWS test; set MULTISTORE_TEST_ROLE_ARN/BUCKET and run with --ignored"] async fn assume_role_and_read_private_bucket() { - let Some(role_arn) = env("MULTISTORE_TEST_ROLE_ARN") else { - eprintln!("skipping live_sts: MULTISTORE_TEST_ROLE_ARN not set"); - return; - }; + let role_arn = env("MULTISTORE_TEST_ROLE_ARN") + .expect("MULTISTORE_TEST_ROLE_ARN must be set to run the live federation test"); let bucket = env("MULTISTORE_TEST_BUCKET") .expect("MULTISTORE_TEST_BUCKET must be set when MULTISTORE_TEST_ROLE_ARN is"); let region = env("MULTISTORE_TEST_REGION").unwrap_or_else(|| "us-east-1".to_string()); From b03d12e19b0f92ebd6992b0c4534e664333046fa Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Wed, 3 Jun 2026 10:19:04 -0700 Subject: [PATCH 5/5] test(backend-federation): full proxy-path federation smoke test on PRs and staging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an end-to-end test of the outbound federation path (proxy mints its own OIDC assertion -> AssumeRoleWithWebIdentity -> SigV4 read of a private bucket), run by the shared smoke-test job on every preview (PR) and staging deploy. To avoid creating/tearing down a per-PR AWS identity provider + role, PR previews reuse the STABLE staging OIDC issuer: all deployments share OIDC_PROVIDER_KEY, so a preview mints tokens whose `iss` is the staging URL and AWS validates them against the single, already-registered staging IAM OIDC provider. preview.yml and staging.yml both set oidc_issuer_override to ${{ vars.STAGING_OIDC_ISSUER }}; the rationale is commented in both workflows. - tests/smoke/test_federation.py: anonymous GET of the `federated-test` bucket must return the private object (no silent skip — fails if federation is unconfigured). - wrangler.deploy.toml: `federated-test` bucket (auth_type=oidc) with placeholder role ARN / bucket to replace with real test resources. - deploy.yml: pass FEDERATION_TEST_KEY to the smoke-test job. Requires (one-time, no per-PR IAM): set the STAGING_OIDC_ISSUER repo variable, register an AWS IAM OIDC provider for it, and create a role trusting it (sub=multistore, aud=sts.amazonaws.com) with read on the private bucket. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/deploy.yml | 5 +++ .github/workflows/preview.yml | 8 +++- .github/workflows/staging.yml | 4 ++ examples/cf-workers/wrangler.deploy.toml | 22 ++++++++++ tests/smoke/test_federation.py | 54 ++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 tests/smoke/test_federation.py diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 1b65cf5..145feba 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -107,6 +107,11 @@ jobs: id-token: write env: DEPLOY_URL: ${{ needs.deploy.outputs.deploy_url }} + # Object key the backend-federation smoke test GETs from the private + # bucket (defaults to hello.txt). The full federation path is validated + # here on both preview (PR) and staging deploys, against the stable + # staging OIDC issuer — see preview.yml / staging.yml. + FEDERATION_TEST_KEY: ${{ vars.FEDERATION_TEST_KEY }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index d8ff1c9..3d2a6f5 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -21,7 +21,13 @@ jobs: worker_name: multistore-proxy-pr-${{ github.event.pull_request.number }} wrangler_config: wrangler.deploy.toml environment: preview - oidc_issuer_override: "https://multistore-proxy-pr-${{ github.event.pull_request.number }}.${{ vars.CLOUDFLARE_WORKERS_SUBDOMAIN }}.workers.dev" + # Mint tokens with the STABLE staging issuer rather than this PR's own URL. + # All deployments share OIDC_PROVIDER_KEY, so a preview can sign assertions + # whose `iss` is the staging URL; AWS then validates them against the + # single, already-registered staging IAM OIDC provider. This lets the full + # backend-federation path (tests/smoke/test_federation.py) run on every PR + # WITHOUT creating/tearing down a per-PR identity provider + role. + oidc_issuer_override: ${{ vars.STAGING_OIDC_ISSUER }} secrets: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 4adf7fb..4daa96a 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -20,6 +20,10 @@ jobs: worker_name: multistore-staging wrangler_config: wrangler.deploy.toml environment: staging + # The canonical issuer the AWS IAM OIDC provider is registered against. + # Staging serves JWKS at this URL; PR previews reuse the same value (see + # preview.yml) so one provider + role covers both — no per-PR setup. + oidc_issuer_override: ${{ vars.STAGING_OIDC_ISSUER }} secrets: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} diff --git a/examples/cf-workers/wrangler.deploy.toml b/examples/cf-workers/wrangler.deploy.toml index 72bf244..09abc0b 100644 --- a/examples/cf-workers/wrangler.deploy.toml +++ b/examples/cf-workers/wrangler.deploy.toml @@ -39,6 +39,28 @@ endpoint = "https://s3.us-west-2.amazonaws.com" region = "us-east-1" skip_signature = "true" +# Backend-federation test bucket. The proxy mints its own OIDC assertion (iss = +# OIDC_PROVIDER_ISSUER, the stable staging issuer) and assumes an AWS IAM role +# via AssumeRoleWithWebIdentity to serve this private bucket — no long-lived +# backend keys. Exercised by tests/smoke/test_federation.py on every preview and +# staging deploy. Replace oidc_role_arn + bucket_name (+ region/endpoint) with +# your test resources; these are not secrets. The role's trust policy must trust +# the STAGING_OIDC_ISSUER IAM OIDC provider with `sub` = oidc_subject and +# audience sts.amazonaws.com, and grant s3:GetObject/s3:ListBucket on the bucket. +[[vars.PROXY_CONFIG.buckets]] +allowed_roles = [] +anonymous_access = true +backend_type = "s3" +name = "federated-test" + +[vars.PROXY_CONFIG.buckets.backend_options] +auth_type = "oidc" +oidc_role_arn = "arn:aws:iam::123456789012:role/multistore-backend-federation-test" +oidc_subject = "multistore" +bucket_name = "REPLACE_WITH_PRIVATE_TEST_BUCKET" +endpoint = "https://s3.us-east-1.amazonaws.com" +region = "us-east-1" + [[vars.PROXY_CONFIG.credentials]] access_key_id = "AKPROXY00000EXAMPLE" secret_access_key = "EXAMPLE000000000000" diff --git a/tests/smoke/test_federation.py b/tests/smoke/test_federation.py new file mode 100644 index 0000000..ad42378 --- /dev/null +++ b/tests/smoke/test_federation.py @@ -0,0 +1,54 @@ +"""Full backend-federation smoke test against the deployed proxy. + +The `federated-test` bucket (see examples/cf-workers/wrangler.deploy.toml) is +configured with `auth_type=oidc`: at request time the proxy mints its own OIDC +assertion (`iss` = OIDC_PROVIDER_ISSUER, the *stable staging* issuer) and +assumes an AWS IAM role via `AssumeRoleWithWebIdentity`, then signs the backend +read with the temporary credentials. A successful anonymous GET through the +proxy therefore exercises the whole outbound path end to end: + + mint JWT -> AssumeRoleWithWebIdentity at real AWS STS -> SigV4 GET of a + private bucket -> stream back to the caller. + +It runs on every preview (PR) and staging deploy via the shared smoke-test job. +PR previews reuse the staging issuer (see preview.yml / staging.yml), so AWS +validates against one already-registered IAM OIDC provider — no per-PR provider +or role is created. + +There is intentionally no skip path: if federation is not wired up (issuer +unset, provider/role/trust misconfigured, or the placeholder bucket config not +replaced), the GET fails and so does this test — the real path is never silently +reported as green. + +Env: + DEPLOY_URL: deployed proxy URL (set by the smoke-test job) + FEDERATION_TEST_KEY: object key in the private bucket to GET (default hello.txt) +""" + +import os + +import requests + +DEPLOY_URL = os.environ.get("DEPLOY_URL", "http://localhost:8787").rstrip("/") +FEDERATION_BUCKET = "federated-test" +FEDERATION_TEST_KEY = os.environ.get("FEDERATION_TEST_KEY", "hello.txt") + + +def test_federation_serves_private_object(): + """Anonymous GET of the federated bucket must return the private object, + proving the proxy assumed the backend role to read it.""" + url = f"{DEPLOY_URL}/{FEDERATION_BUCKET}/{FEDERATION_TEST_KEY}" + resp = requests.get(url, timeout=30) + + assert resp.status_code == 200, ( + f"expected 200 from federated GET {url}, got {resp.status_code}: " + f"{resp.text[:500]}\n" + "The proxy failed to assume the backend role. Check that:\n" + " - the STAGING_OIDC_ISSUER repo variable is set and the AWS IAM OIDC " + "provider is registered for it (audience sts.amazonaws.com),\n" + " - the role's trust policy allows sub=`multistore` and grants " + "s3:GetObject on the bucket, and\n" + " - the federated-test bucket in wrangler.deploy.toml points at your " + "real role ARN + private bucket." + ) + assert resp.content, "federated object was empty"