From 686f5e67f261e6dc48fb9f17cb91bc9bd6d2f41e Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Thu, 23 Oct 2025 16:07:33 -0700 Subject: [PATCH 01/16] wstd::http::client: derive Clone --- src/http/client.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/http/client.rs b/src/http/client.rs index a3f9718..de7ff11 100644 --- a/src/http/client.rs +++ b/src/http/client.rs @@ -6,7 +6,7 @@ use crate::time::Duration; use wasip2::http::types::RequestOptions as WasiRequestOptions; /// An HTTP client. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Client { options: Option, } @@ -85,7 +85,7 @@ impl Client { } } -#[derive(Default, Debug)] +#[derive(Default, Debug, Clone)] struct RequestOptions { connect_timeout: Option, first_byte_timeout: Option, From fe8b12b2292d0773447aeba68ef48a0c9340e362 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Thu, 23 Oct 2025 16:08:02 -0700 Subject: [PATCH 02/16] add wstd-aws crate, for using the aws sdk on top of wstd --- Cargo.toml | 6 ++- aws/Cargo.toml | 20 ++++++++++ aws/src/lib.rs | 101 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 aws/Cargo.toml create mode 100644 aws/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 9417066..605aa0a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,7 +44,7 @@ serde = { workspace = true, features = ["derive"] } serde_json.workspace = true [workspace] -members = [ +members = ["aws", "axum", "axum/macro", "macro", @@ -70,6 +70,9 @@ authors = [ [workspace.dependencies] anyhow = "1" async-task = "4.7" +aws-smithy-async = { version = "1.2.6", default-features = false } +aws-smithy-types = { version = "1.3.3", default-features = false } +aws-smithy-runtime-api = { version = "1.9.1", default-features = false } axum = { version = "0.8.6", default-features = false } bytes = "1.10.1" cargo_metadata = "0.22" @@ -88,6 +91,7 @@ quote = "1.0" serde= "1" serde_json = "1" serde_qs = "0.15" +sync_wrapper = "1" slab = "0.4.9" syn = "2.0" test-log = { version = "0.2", features = ["trace"] } diff --git a/aws/Cargo.toml b/aws/Cargo.toml new file mode 100644 index 0000000..7f0bc12 --- /dev/null +++ b/aws/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "wstd-aws" +description = "" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true +rust-version.workspace = true +authors.workspace = true + +[dependencies] +anyhow.workspace = true +aws-smithy-async = { workspace = true } +aws-smithy-types = { workspace = true, features = ["http-body-1-x"] } +aws-smithy-runtime-api = { workspace = true, features = ["client", "http-1x"] } +http-body-util.workspace = true +sync_wrapper = { workspace = true, features = ["futures"] } +wstd.workspace = true diff --git a/aws/src/lib.rs b/aws/src/lib.rs new file mode 100644 index 0000000..0a03ac0 --- /dev/null +++ b/aws/src/lib.rs @@ -0,0 +1,101 @@ +use anyhow::anyhow; +use aws_smithy_async::rt::sleep::{AsyncSleep, Sleep}; +use aws_smithy_runtime_api::client::http::{ + HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector, +}; +use aws_smithy_runtime_api::client::orchestrator::HttpRequest; +use aws_smithy_runtime_api::client::result::ConnectorError; +use aws_smithy_runtime_api::client::retries::ErrorKind; +use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; +use aws_smithy_runtime_api::http::Response; +use aws_smithy_types::body::SdkBody; +use http_body_util::{BodyStream, StreamBody}; +use std::time::Duration; +use sync_wrapper::SyncStream; +use wstd::http::{Body as WstdBody, BodyExt, Client}; + +pub fn sleep_impl() -> impl AsyncSleep + 'static { + WstdSleep +} + +#[derive(Debug)] +struct WstdSleep; +impl AsyncSleep for WstdSleep { + fn sleep(&self, duration: Duration) -> Sleep { + Sleep::new(async move { + wstd::task::sleep(wstd::time::Duration::from(duration)).await; + }) + } +} + +pub fn http_client() -> impl HttpClient + 'static { + WstdHttpClient +} + +#[derive(Debug)] +struct WstdHttpClient; + +impl HttpClient for WstdHttpClient { + fn http_connector( + &self, + settings: &HttpConnectorSettings, + // afaict, none of these components are relevant to this + // implementation. + _components: &RuntimeComponents, + ) -> SharedHttpConnector { + let mut client = Client::new(); + if let Some(timeout) = settings.connect_timeout() { + client.set_connect_timeout(timeout); + } + if let Some(timeout) = settings.read_timeout() { + client.set_first_byte_timeout(timeout); + } + SharedHttpConnector::new(WstdHttpConnector(client)) + } +} + +#[derive(Debug)] +struct WstdHttpConnector(Client); + +impl HttpConnector for WstdHttpConnector { + fn call(&self, request: HttpRequest) -> HttpConnectorFuture { + let client = self.0.clone(); + HttpConnectorFuture::new(async move { + let request = request + .try_into_http1x() + // This can only fail if the Extensions fail to convert + .map_err(|e| ConnectorError::other(Box::new(e), None))?; + // smithy's SdkBody Error is a non-'static boxed dyn stderror. + // Anyhow can't represent that, so convert it to the debug impl. + let request = + request.map(|body| WstdBody::from_http_body(body.map_err(|e| anyhow!("{e:?}")))); + // Any error given by send is considered a "ClientError" kind + // which should prevent smithy from retrying like it would for a + // throttling error + let response = client + .send(request) + .await + .map_err(|e| ConnectorError::other(e.into(), Some(ErrorKind::ClientError)))?; + + Response::try_from(response.map(|wstd_body| { + // You'd think that an SdkBody would just be an impl Body with + // the usual error type dance. + let nonsync_body = wstd_body + .into_boxed_body() + .map_err(|e| e.into_boxed_dyn_error()); + // But we have to do this weird dance: because Axum insists + // bodies are not Sync, wstd settled on non-Sync bodies. + // Smithy insists on Sync bodies. The SyncStream type exists + // to assert, because all Stream operations are on &mut self, + // all Streams are Sync. So, turn the Body into a Stream, make + // it sync, then back to a Body. + let nonsync_stream = BodyStream::new(nonsync_body); + let sync_stream = SyncStream::new(nonsync_stream); + let sync_body = StreamBody::new(sync_stream); + SdkBody::from_body_1_x(sync_body) + })) + // This can only fail if the Extensions fail to convert + .map_err(|e| ConnectorError::other(Box::new(e), None)) + }) + } +} From fb836a775ad6a6e9ab08fde071b3fea45c16a832 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Fri, 24 Oct 2025 14:42:42 -0700 Subject: [PATCH 03/16] wstd-sdk: add s3 bucket listing and fetch as an example derived from https://github.com/awsdocs/aws-doc-sdk-examples/blob/main/rustv1/examples/s3/src/bin/s3-helloworld.rs#L35 --- .cargo/config.toml | 2 +- Cargo.toml | 2 + aws/Cargo.toml | 5 +++ aws/examples/s3.rs | 94 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 aws/examples/s3.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index abcd047..ef7ad24 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,2 @@ [target.wasm32-wasip2] -runner = "wasmtime -Shttp" +runner = "wasmtime run -Shttp --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --dir .::." diff --git a/Cargo.toml b/Cargo.toml index 605aa0a..ae12bc1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,6 +70,8 @@ authors = [ [workspace.dependencies] anyhow = "1" async-task = "4.7" +aws-config = { version = "1.8.8", default-features = false } +aws-sdk-s3 = { version = "1.108.0", default-features = false } aws-smithy-async = { version = "1.2.6", default-features = false } aws-smithy-types = { version = "1.3.3", default-features = false } aws-smithy-runtime-api = { version = "1.9.1", default-features = false } diff --git a/aws/Cargo.toml b/aws/Cargo.toml index 7f0bc12..b4038e4 100644 --- a/aws/Cargo.toml +++ b/aws/Cargo.toml @@ -18,3 +18,8 @@ aws-smithy-runtime-api = { workspace = true, features = ["client", "http-1x"] } http-body-util.workspace = true sync_wrapper = { workspace = true, features = ["futures"] } wstd.workspace = true + +[dev-dependencies] +aws-config.workspace = true +aws-sdk-s3.workspace = true +clap.workspace = true diff --git a/aws/examples/s3.rs b/aws/examples/s3.rs new file mode 100644 index 0000000..9b496e5 --- /dev/null +++ b/aws/examples/s3.rs @@ -0,0 +1,94 @@ +use anyhow::Result; +use clap::{Parser, Subcommand}; + +use aws_config::{BehaviorVersion, Region}; +use aws_sdk_s3::Client; + +#[derive(Debug, Parser)] +#[command(version, about, long_about = None)] +struct Opts { + /// The AWS Region. + #[arg(short, long)] + region: String, + /// The name of the bucket. + #[arg(short, long)] + bucket: String, + + #[command(subcommand)] + command: Option, +} + +#[derive(Subcommand, Debug)] +enum Command { + List, + Get { + key: String, + #[arg(short, long)] + out: Option, + }, +} + +#[wstd::main] +async fn main() -> Result<()> { + let opts = Opts::parse(); + let config = aws_config::defaults(BehaviorVersion::latest()) + .region(Region::new(opts.region.clone())) + .sleep_impl(wstd_aws::sleep_impl()) + .http_client(wstd_aws::http_client()) + .load() + .await; + + let client = Client::new(&config); + + match opts.command.as_ref().unwrap_or(&Command::List) { + Command::List => list(&opts, &client).await, + Command::Get { key, out } => { + let contents = get(&opts, &client, &key).await?; + let output: &str = if let Some(out) = out { + out.as_str() + } else { + key.as_str() + }; + std::fs::write(output, contents)?; + Ok(()) + } + } +} + +async fn list(opts: &Opts, client: &Client) -> Result<()> { + let mut listing = client + .list_objects_v2() + .bucket(opts.bucket.clone()) + .into_paginator() + .send(); + + println!("key\tetag\tlast_modified\tstorage_class"); + while let Some(res) = listing.next().await { + let object = res?; + for item in object.contents() { + println!( + "{}\t{}\t{}\t{}", + item.key().unwrap_or_default(), + item.e_tag().unwrap_or_default(), + item.last_modified() + .map(|lm| format!("{lm}")) + .unwrap_or_default(), + item.storage_class() + .map(|sc| format!("{sc}")) + .unwrap_or_default(), + ); + } + } + Ok(()) +} + +async fn get(opts: &Opts, client: &Client, key: &str) -> Result> { + let object = client + .get_object() + .bucket(opts.bucket.clone()) + .key(key) + .send() + .await?; + let data = object.body.collect().await?; + Ok(data.to_vec()) +} From 6423a8ed181752b0f942fc6e72d59ea3378b1635 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Mon, 27 Oct 2025 11:53:59 -0700 Subject: [PATCH 04/16] ci: assume wstd-aws-ci-role --- .github/workflows/ci.yaml | 17 ++++++++++++++++- aws/.gitignore | 1 + 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 aws/.gitignore diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a468cb9..992a907 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -38,12 +38,27 @@ jobs: command: test args: -p wstd --target wasm32-wasip2 -- --nocapture + # pchickey made a role and bucket on his personal aws account + # it only provides ListBucket and GetObject for a single bucket + - name: Configure AWS Credentials + id: creds + uses: aws-actions/configure-aws-credentials@v5.1.0 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::313377415443:role/wstd-aws-ci-role + role-session-name: github-ci + - name: example tests uses: actions-rs/cargo@v1 with: command: test args: -p test-programs -- --nocapture - + env: + AWS_REGION: us-west-2 + AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.creds.outputs.aws-session-token }} + WSTD_EXAMPLE_BUCKET: wstd-example-bucket check_fmt_and_docs: name: Checking fmt and docs diff --git a/aws/.gitignore b/aws/.gitignore new file mode 100644 index 0000000..7d219f9 --- /dev/null +++ b/aws/.gitignore @@ -0,0 +1 @@ +.environment From e961c41efe823310d1878f402e104605b886d648 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Mon, 27 Oct 2025 12:28:58 -0700 Subject: [PATCH 05/16] test runs aws s3 example --- .github/workflows/ci.yaml | 29 ++++------ aws/examples/s3.rs | 2 +- test-programs/build.rs | 104 +++++++++++++++++----------------- test-programs/tests/aws_s3.rs | 59 +++++++++++++++++++ 4 files changed, 123 insertions(+), 71 deletions(-) create mode 100644 test-programs/tests/aws_s3.rs diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 992a907..0164198 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,16 +27,10 @@ jobs: uses: bytecodealliance/actions/wasmtime/setup@v1 - name: check - uses: actions-rs/cargo@v1 - with: - command: check - args: --workspace --all --bins --examples + run: cargo check --workspace --all --bins --examples - name: wstd tests - uses: actions-rs/cargo@v1 - with: - command: test - args: -p wstd --target wasm32-wasip2 -- --nocapture + run: cargo test -p wstd -p wstd-axum -p wstd-aws --target wasm32-wasip2 -- --nocapture # pchickey made a role and bucket on his personal aws account # it only provides ListBucket and GetObject for a single bucket @@ -48,17 +42,14 @@ jobs: role-to-assume: arn:aws:iam::313377415443:role/wstd-aws-ci-role role-session-name: github-ci - - name: example tests - uses: actions-rs/cargo@v1 - with: - command: test - args: -p test-programs -- --nocapture - env: - AWS_REGION: us-west-2 - AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.creds.outputs.aws-session-token }} - WSTD_EXAMPLE_BUCKET: wstd-example-bucket + - name: test-programs tests + run: cargo test -p test-programs -- --nocapture + env: + AWS_REGION: us-west-2 + AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.creds.outputs.aws-session-token }} + WSTD_EXAMPLE_BUCKET: wstd-example-bucket check_fmt_and_docs: name: Checking fmt and docs diff --git a/aws/examples/s3.rs b/aws/examples/s3.rs index 9b496e5..77bf507 100644 --- a/aws/examples/s3.rs +++ b/aws/examples/s3.rs @@ -43,7 +43,7 @@ async fn main() -> Result<()> { match opts.command.as_ref().unwrap_or(&Command::List) { Command::List => list(&opts, &client).await, Command::Get { key, out } => { - let contents = get(&opts, &client, &key).await?; + let contents = get(&opts, &client, key).await?; let output: &str = if let Some(out) = out { out.as_str() } else { diff --git a/test-programs/build.rs b/test-programs/build.rs index 12574e6..36a4484 100644 --- a/test-programs/build.rs +++ b/test-programs/build.rs @@ -1,36 +1,25 @@ -use cargo_metadata::TargetKind; +use cargo_metadata::{MetadataCommand, Package, TargetKind}; use heck::ToShoutySnakeCase; use std::env::var_os; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::Command; fn main() { let out_dir = PathBuf::from(var_os("OUT_DIR").expect("OUT_DIR env var exists")); - let meta = cargo_metadata::MetadataCommand::new() - .exec() - .expect("cargo metadata"); - let wstd_meta = meta - .packages - .iter() - .find(|p| *p.name == "wstd") - .expect("wstd is in cargo metadata"); - let wstd_axum_meta = meta - .packages - .iter() - .find(|p| *p.name == "wstd-axum") - .expect("wstd is in cargo metadata"); + let meta = MetadataCommand::new().exec().expect("cargo metadata"); - let wstd_root = wstd_meta.manifest_path.parent().unwrap(); println!( "cargo:rerun-if-changed={}", - wstd_root.as_os_str().to_str().unwrap() + meta.workspace_root.as_os_str().to_str().unwrap() ); fn build_examples(pkg: &str, out_dir: &PathBuf) { + // release build is required for aws sdk to not overflow wasm locals let status = Command::new("cargo") .arg("build") .arg("--examples") + .arg("--release") .arg("--target=wasm32-wasip2") .arg(format!("--package={pkg}")) .env("CARGO_TARGET_DIR", out_dir) @@ -43,46 +32,59 @@ fn main() { } build_examples("wstd", &out_dir); build_examples("wstd-axum", &out_dir); + build_examples("wstd-aws", &out_dir); let mut generated_code = "// THIS FILE IS GENERATED CODE\n".to_string(); - for binary in wstd_meta - .targets - .iter() - .filter(|t| t.kind == [TargetKind::Example]) - { - let component_path = out_dir - .join("wasm32-wasip2") - .join("debug") - .join("examples") - .join(format!("{}.wasm", binary.name)); + fn module_for(name: &str, out_dir: &Path, meta: &Package) -> String { + let mut generated_code = String::new(); + generated_code += &format!("pub mod {name} {{"); + for binary in meta + .targets + .iter() + .filter(|t| t.kind == [TargetKind::Example]) + { + let component_path = out_dir + .join("wasm32-wasip2") + .join("release") + .join("examples") + .join(format!("{}.wasm", binary.name)); - let const_name = binary.name.to_shouty_snake_case(); - generated_code += &format!( - "pub const {const_name}: &str = {:?};\n", - component_path.as_os_str().to_str().expect("path is str") - ); + let const_name = binary.name.to_shouty_snake_case(); + generated_code += &format!( + "pub const {const_name}: &str = {:?};\n", + component_path.as_os_str().to_str().expect("path is str") + ); + } + generated_code += "}\n\n"; // end `pub mod {name}` + generated_code } - generated_code += "pub mod axum {"; - for binary in wstd_axum_meta - .targets - .iter() - .filter(|t| t.kind == [TargetKind::Example]) - { - let component_path = out_dir - .join("wasm32-wasip2") - .join("debug") - .join("examples") - .join(format!("{}.wasm", binary.name)); - - let const_name = binary.name.to_shouty_snake_case(); - generated_code += &format!( - "pub const {const_name}: &str = {:?};\n", - component_path.as_os_str().to_str().expect("path is str") - ); - } - generated_code += "}"; // end `pub mod axum` + generated_code += &module_for( + "_wstd", + &out_dir, + meta.packages + .iter() + .find(|p| *p.name == "wstd") + .expect("wstd is in cargo metadata"), + ); + generated_code += "pub use _wstd::*;\n\n"; + generated_code += &module_for( + "axum", + &out_dir, + meta.packages + .iter() + .find(|p| *p.name == "wstd-axum") + .expect("wstd-axum is in cargo metadata"), + ); + generated_code += &module_for( + "aws", + &out_dir, + meta.packages + .iter() + .find(|p| *p.name == "wstd-aws") + .expect("wstd-aws is in cargo metadata"), + ); std::fs::write(out_dir.join("gen.rs"), generated_code).unwrap(); } diff --git a/test-programs/tests/aws_s3.rs b/test-programs/tests/aws_s3.rs new file mode 100644 index 0000000..1cd05f8 --- /dev/null +++ b/test-programs/tests/aws_s3.rs @@ -0,0 +1,59 @@ +use anyhow::Result; +use std::path::Path; +use std::process::Command; + +#[test_log::test] +fn aws_s3() -> Result<()> { + // bucket list command + let output = Command::new("wasmtime") + .arg("run") + .arg("-Shttp") + .args(["--env", "AWS_ACCESS_KEY_ID"]) + .args(["--env", "AWS_SECRET_ACCESS_KEY"]) + .args(["--env", "AWS_ACCESS_KEY_ID"]) + .arg(test_programs::aws::S3) + .arg(format!( + "--region={}", + std::env::var("AWS_REGION").unwrap_or_else(|_| "us-west-2".to_owned()) + )) + .arg(format!( + "--bucket={}", + std::env::var("WSTD_EXAMPLE_BUCKET") + .unwrap_or_else(|_| "wstd-example-bucket".to_owned()) + )) + .arg("list") + .output()?; + println!("{:?}", output); + assert!(output.status.success()); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("fluff.jpg")); + assert!(stdout.contains("shoug.jpg")); + + // bucket get command + let output = Command::new("wasmtime") + .arg("run") + .arg("-Shttp") + .args(["--env", "AWS_ACCESS_KEY_ID"]) + .args(["--env", "AWS_SECRET_ACCESS_KEY"]) + .args(["--env", "AWS_ACCESS_KEY_ID"]) + .args(["--dir", ".::."]) + .arg(test_programs::aws::S3) + .arg(format!( + "--region={}", + std::env::var("AWS_REGION").unwrap_or_else(|_| "us-west-2".to_owned()) + )) + .arg(format!( + "--bucket={}", + std::env::var("WSTD_EXAMPLE_BUCKET") + .unwrap_or_else(|_| "wstd-example-bucket".to_owned()) + )) + .arg("get") + .arg("shoug.jpg") + .output()?; + println!("{:?}", output); + assert!(output.status.success()); + + assert!(Path::new("shoug.jpg").exists()); + + Ok(()) +} From a8949cc72d39577657ebf3c21d9e6e6e7fadeb20 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Mon, 27 Oct 2025 13:01:55 -0700 Subject: [PATCH 06/16] add wstd-aws to publish --- ci/publish.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ci/publish.rs b/ci/publish.rs index bd20833..cff7b58 100644 --- a/ci/publish.rs +++ b/ci/publish.rs @@ -16,7 +16,13 @@ use std::thread; use std::time::Duration; // note that this list must be topologically sorted by dependencies -const CRATES_TO_PUBLISH: &[&str] = &["wstd-macro", "wstd", "wstd-axum-macro", "wstd-axum"]; +const CRATES_TO_PUBLISH: &[&str] = &[ + "wstd-macro", + "wstd", + "wstd-axum-macro", + "wstd-axum", + "wstd-aws", +]; #[derive(Debug)] struct Workspace { @@ -53,11 +59,13 @@ fn main() { bump_version(&krate, &crates, name == "bump-patch"); } // update the lock file - assert!(Command::new("cargo") - .arg("fetch") - .status() - .unwrap() - .success()); + assert!( + Command::new("cargo") + .arg("fetch") + .status() + .unwrap() + .success() + ); } "publish" => { From dcda5d0de0c2e827fa136a066136adf8b5a59f80 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Mon, 27 Oct 2025 15:12:09 -0700 Subject: [PATCH 07/16] ci: id-token write --- .github/workflows/ci.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0164198..fa6782a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -10,6 +10,10 @@ on: env: RUSTFLAGS: -Dwarnings +# required for AWS oidc +permissions: + id-token: write + jobs: build_and_test: name: Build and test From 8827eade46e1d5dede5338d45fb4e6ee11389246 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Mon, 27 Oct 2025 15:19:33 -0700 Subject: [PATCH 08/16] typo --- test-programs/tests/aws_s3.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-programs/tests/aws_s3.rs b/test-programs/tests/aws_s3.rs index 1cd05f8..3fa1ebb 100644 --- a/test-programs/tests/aws_s3.rs +++ b/test-programs/tests/aws_s3.rs @@ -10,7 +10,7 @@ fn aws_s3() -> Result<()> { .arg("-Shttp") .args(["--env", "AWS_ACCESS_KEY_ID"]) .args(["--env", "AWS_SECRET_ACCESS_KEY"]) - .args(["--env", "AWS_ACCESS_KEY_ID"]) + .args(["--env", "AWS_SESSION_TOKEN"]) .arg(test_programs::aws::S3) .arg(format!( "--region={}", From 4360d164ee0814796b03af10fee5b4cdb1bd54ff Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Tue, 28 Oct 2025 10:19:38 -0700 Subject: [PATCH 09/16] runner: forward session token as well --- .cargo/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index ef7ad24..8b62960 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,2 @@ [target.wasm32-wasip2] -runner = "wasmtime run -Shttp --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --dir .::." +runner = "wasmtime run -Shttp --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env AWS_SESSION_TOKEN --dir .::." From f9d0ac1e21103908570916ef15d8155e6111d357 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Tue, 28 Oct 2025 10:29:09 -0700 Subject: [PATCH 10/16] example: bugfix, clean up output --- aws/examples/s3.rs | 18 +++++++++++------- test-programs/tests/aws_s3.rs | 29 ++++++++++++++--------------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/aws/examples/s3.rs b/aws/examples/s3.rs index 77bf507..1e8256c 100644 --- a/aws/examples/s3.rs +++ b/aws/examples/s3.rs @@ -41,7 +41,10 @@ async fn main() -> Result<()> { let client = Client::new(&config); match opts.command.as_ref().unwrap_or(&Command::List) { - Command::List => list(&opts, &client).await, + Command::List => { + let output = list(&opts, &client).await?; + print!("{}", output); + } Command::Get { key, out } => { let contents = get(&opts, &client, key).await?; let output: &str = if let Some(out) = out { @@ -50,24 +53,25 @@ async fn main() -> Result<()> { key.as_str() }; std::fs::write(output, contents)?; - Ok(()) } } + Ok(()) } -async fn list(opts: &Opts, client: &Client) -> Result<()> { +async fn list(opts: &Opts, client: &Client) -> Result { let mut listing = client .list_objects_v2() .bucket(opts.bucket.clone()) .into_paginator() .send(); - println!("key\tetag\tlast_modified\tstorage_class"); + let mut output = String::new(); + output += "key\tetag\tlast_modified\tstorage_class\n"; while let Some(res) = listing.next().await { let object = res?; for item in object.contents() { - println!( - "{}\t{}\t{}\t{}", + output += &format!( + "{}\t{}\t{}\t{}\n", item.key().unwrap_or_default(), item.e_tag().unwrap_or_default(), item.last_modified() @@ -79,7 +83,7 @@ async fn list(opts: &Opts, client: &Client) -> Result<()> { ); } } - Ok(()) + Ok(output) } async fn get(opts: &Opts, client: &Client, key: &str) -> Result> { diff --git a/test-programs/tests/aws_s3.rs b/test-programs/tests/aws_s3.rs index 3fa1ebb..ea0d723 100644 --- a/test-programs/tests/aws_s3.rs +++ b/test-programs/tests/aws_s3.rs @@ -2,16 +2,22 @@ use anyhow::Result; use std::path::Path; use std::process::Command; +fn run_s3_example() -> Command { + let mut command = Command::new("wasmtime"); + command.arg("run"); + command.arg("-Shttp"); + command.args(["--env", "AWS_ACCESS_KEY_ID"]); + command.args(["--env", "AWS_SECRET_ACCESS_KEY"]); + command.args(["--env", "AWS_SESSION_TOKEN"]); + command.args(["--dir", ".::."]); + command.arg(test_programs::aws::S3); + command +} + #[test_log::test] fn aws_s3() -> Result<()> { // bucket list command - let output = Command::new("wasmtime") - .arg("run") - .arg("-Shttp") - .args(["--env", "AWS_ACCESS_KEY_ID"]) - .args(["--env", "AWS_SECRET_ACCESS_KEY"]) - .args(["--env", "AWS_SESSION_TOKEN"]) - .arg(test_programs::aws::S3) + let output = run_s3_example() .arg(format!( "--region={}", std::env::var("AWS_REGION").unwrap_or_else(|_| "us-west-2".to_owned()) @@ -30,14 +36,7 @@ fn aws_s3() -> Result<()> { assert!(stdout.contains("shoug.jpg")); // bucket get command - let output = Command::new("wasmtime") - .arg("run") - .arg("-Shttp") - .args(["--env", "AWS_ACCESS_KEY_ID"]) - .args(["--env", "AWS_SECRET_ACCESS_KEY"]) - .args(["--env", "AWS_ACCESS_KEY_ID"]) - .args(["--dir", ".::."]) - .arg(test_programs::aws::S3) + let output = run_s3_example() .arg(format!( "--region={}", std::env::var("AWS_REGION").unwrap_or_else(|_| "us-west-2".to_owned()) From fb75995abe11395c25e5592f78097cc8f38abc67 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Tue, 28 Oct 2025 11:20:31 -0700 Subject: [PATCH 11/16] need to debug ci --- aws/examples/s3.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/aws/examples/s3.rs b/aws/examples/s3.rs index 1e8256c..eb2a1c2 100644 --- a/aws/examples/s3.rs +++ b/aws/examples/s3.rs @@ -30,6 +30,19 @@ enum Command { #[wstd::main] async fn main() -> Result<()> { + println!( + "debug AWS_ACCESS_KEY_ID={:?}", + std::env::var("AWS_ACCESS_KEY_ID") + ); + println!( + "debug AWS_SECRET_ACCESS_KEY={:?}", + std::env::var("AWS_SECRET_ACCESS_KEY") + ); + println!( + "debug AWS_SESSION_TOKEN={:?}", + std::env::var("AWS_SESSION_TOKEN") + ); + let opts = Opts::parse(); let config = aws_config::defaults(BehaviorVersion::latest()) .region(Region::new(opts.region.clone())) From 2b6e97a05f5608a6c7476c095fc954cbc6812536 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Tue, 28 Oct 2025 11:26:13 -0700 Subject: [PATCH 12/16] does action put them into the env properly?? --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fa6782a..c6b36b4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -50,9 +50,9 @@ jobs: run: cargo test -p test-programs -- --nocapture env: AWS_REGION: us-west-2 - AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.creds.outputs.aws-session-token }} + #AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.aws-access-key-id }} + #AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.aws-secret-access-key }} + #AWS_SESSION_TOKEN: ${{ steps.creds.outputs.aws-session-token }} WSTD_EXAMPLE_BUCKET: wstd-example-bucket check_fmt_and_docs: From 8bf30bd3f892be76616f6f538ca2ee5ebb3b9e46 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Tue, 28 Oct 2025 11:49:22 -0700 Subject: [PATCH 13/16] remove debugging, add comments, ci can skip aws if auth fails --- .github/workflows/ci.yaml | 35 ++++++++++++++++++++--------------- aws/examples/s3.rs | 13 ------------- test-programs/Cargo.toml | 4 ++++ test-programs/tests/aws_s3.rs | 1 + 4 files changed, 25 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c6b36b4..726a7cf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -30,30 +30,35 @@ jobs: - name: Install wasmtime uses: bytecodealliance/actions/wasmtime/setup@v1 - - name: check - run: cargo check --workspace --all --bins --examples - - - name: wstd tests - run: cargo test -p wstd -p wstd-axum -p wstd-aws --target wasm32-wasip2 -- --nocapture - - # pchickey made a role and bucket on his personal aws account - # it only provides ListBucket and GetObject for a single bucket - - name: Configure AWS Credentials + # pchickey made a role `wstd-aws-ci-role` and bucket `wstd-example-bucket` + # on his personal aws account 313377415443. The role only provides + # ListBucket and GetObject for the example bucket, which is enough to pass + # the single integration test. The role is configured to trust GitHub + # actions for the bytecodealliance/wstd repo. This action will set the + # AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, and AWS_SESSION_TOKEN + # environment variables. + - name: get aws credentials id: creds uses: aws-actions/configure-aws-credentials@v5.1.0 + continue-on-error: true with: aws-region: us-west-2 role-to-assume: arn:aws:iam::313377415443:role/wstd-aws-ci-role role-session-name: github-ci + - name: check + run: cargo check --workspace --all --bins --examples + + - name: wstd tests + run: cargo test -p wstd -p wstd-axum -p wstd-aws --target wasm32-wasip2 -- --nocapture + - name: test-programs tests run: cargo test -p test-programs -- --nocapture - env: - AWS_REGION: us-west-2 - #AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.aws-access-key-id }} - #AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.aws-secret-access-key }} - #AWS_SESSION_TOKEN: ${{ steps.creds.outputs.aws-session-token }} - WSTD_EXAMPLE_BUCKET: wstd-example-bucket + if: steps.creds.outcome == 'success' + + - name: test-programs tests (no aws) + run: cargo test -p test-programs --features no-aws -- --nocapture + if: steps.creds.outcome != 'success' check_fmt_and_docs: name: Checking fmt and docs diff --git a/aws/examples/s3.rs b/aws/examples/s3.rs index eb2a1c2..1e8256c 100644 --- a/aws/examples/s3.rs +++ b/aws/examples/s3.rs @@ -30,19 +30,6 @@ enum Command { #[wstd::main] async fn main() -> Result<()> { - println!( - "debug AWS_ACCESS_KEY_ID={:?}", - std::env::var("AWS_ACCESS_KEY_ID") - ); - println!( - "debug AWS_SECRET_ACCESS_KEY={:?}", - std::env::var("AWS_SECRET_ACCESS_KEY") - ); - println!( - "debug AWS_SESSION_TOKEN={:?}", - std::env::var("AWS_SESSION_TOKEN") - ); - let opts = Opts::parse(); let config = aws_config::defaults(BehaviorVersion::latest()) .region(Region::new(opts.region.clone())) diff --git a/test-programs/Cargo.toml b/test-programs/Cargo.toml index 7f6191e..ba431b4 100644 --- a/test-programs/Cargo.toml +++ b/test-programs/Cargo.toml @@ -15,3 +15,7 @@ ureq.workspace = true [build-dependencies] cargo_metadata.workspace = true heck.workspace = true + +[features] +default = [] +no-aws = [] diff --git a/test-programs/tests/aws_s3.rs b/test-programs/tests/aws_s3.rs index ea0d723..c411d27 100644 --- a/test-programs/tests/aws_s3.rs +++ b/test-programs/tests/aws_s3.rs @@ -15,6 +15,7 @@ fn run_s3_example() -> Command { } #[test_log::test] +#[cfg_attr(feature = "no-aws", ignore)] fn aws_s3() -> Result<()> { // bucket list command let output = run_s3_example() From 00066be3e8bfce440b7d7eee3da50253d5d612d4 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Tue, 28 Oct 2025 12:24:02 -0700 Subject: [PATCH 14/16] readmes --- aws/README.md | 75 ++++++++++++++++++++++++++++++++++++++++++++++ aws/examples/s3.rs | 73 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 137 insertions(+), 11 deletions(-) create mode 100644 aws/README.md diff --git a/aws/README.md b/aws/README.md new file mode 100644 index 0000000..792a4e4 --- /dev/null +++ b/aws/README.md @@ -0,0 +1,75 @@ + +# wstd-aws: wstd support for the AWS Rust SDK + +This crate provides support for using the AWS Rust SDK for the `wasm32-wasip2` +target using the [`wstd`] crate. + +In many wasi settings, its necessary or desirable to use the wasi-http +interface to make http requests. Wasi-http interfaces provide an http +implementation, including the sockets layer and TLS, outside of the user's +component. `wstd` provides user-friendly async Rust interfaces to all of the +standardized wasi interfaces, including wasi-http. + +The AWS Rust SDK, by default, depends on `tokio`, `hyper`, and either `rustls` +or `s2n_tls`, and makes http requests over sockets (which can be provided as +wasi-sockets). Those dependencies may not work correctly under `wasm32-wasip2`, +and if they do, they will not use the wasi-http interfaces. To avoid using +http over sockets, make sure to set the `default-features = false` setting +when depending on any `aws-*` crates in your project. + +To configure `wstd`'s wasi-http client for the AWS Rust SDK, provide +`wstd_aws::sleep_impl()` and `wstd_aws::http_client()` to your +[`aws_config::ConfigLoader`]: + +``` + let config = aws_config::defaults(BehaviorVersion::latest()) + .sleep_impl(wstd_aws::sleep_impl()) + .http_client(wstd_aws::http_client()) + ...; +``` + +[`wstd`]: https://docs.rs/wstd/latest/wstd +[`aws_config::ConfigLoader`]: https://docs.rs/aws-config/1.8.8/aws_config/struct.ConfigLoader.html + +## Example + +An example s3 client is provided as a wasi cli command. It accepts command +line arguments with the subcommand `list` to list a bucket's contents, and +`get ` to get an object from a bucket and write it to the filesystem. + +This example *must be compiled in release mode* - in debug mode, the aws +sdk's generated code will overflow the maximum permitted wasm locals in +a single function. + +Compile it with: + +```sh +cargo build -p wstd-aws --target wasm32-wasip2 --release --examples +``` + +When running this example, you will need AWS credentials provided in environment +variables. + +Run it with: +```sh +wasmtime run -Shttp \ + --env AWS_ACCESS_KEY_ID \ + --env AWS_SECRET_ACCESS_KEY \ + --env AWS_SESSION_TOKEN \ + --dir .::. \ + target/wasm22-wasip2/release/examples/s3.wasm +``` + +or alternatively run it with: +```sh +cargo run --target wasm32-wasip2 -p wstd-aws --example s3 +``` + +which uses the wasmtime cli, as above, via configiration found in this +workspace's `.cargo/config`. + +By default, this script accesses the `wstd-example-bucket` in `us-west-2`. +To change the bucket or region, use the `--bucket` and `--region` cli +flags before the subcommand. + + diff --git a/aws/examples/s3.rs b/aws/examples/s3.rs index 1e8256c..3fcf4ae 100644 --- a/aws/examples/s3.rs +++ b/aws/examples/s3.rs @@ -1,3 +1,44 @@ +//! Example s3 client running on `wstd` via `wstd_aws` +//! +//! This example is a wasi cli command. It accepts command line arguments +//! with the subcommand `list` to list a bucket's contents, and `get ` +//! to get an object from a bucket and write it to the filesystem. +//! +//! This example *must be compiled in release mode* - in debug mode, the aws +//! sdk's generated code will overflow the maximum permitted wasm locals in +//! a single function. +//! +//! Compile it with: +//! +//! ```sh +//! cargo build -p wstd-aws --target wasm32-wasip2 --release --examples +//! ``` +//! +//! When running this example, you will need AWS credentials provided in environment +//! variables. +//! +//! Run it with: +//! ```sh +//! wasmtime run -Shttp \ +//! --env AWS_ACCESS_KEY_ID \ +//! --env AWS_SECRET_ACCESS_KEY \ +//! --env AWS_SESSION_TOKEN \ +//! --dir .::. \ +//! target/wasm22-wasip2/release/examples/s3.wasm +//! ``` +//! +//! or alternatively run it with: +//! ```sh +//! cargo run --target wasm32-wasip2 -p wstd-aws --example s3 +//! ``` +//! +//! which uses the wasmtime cli, as above, via configiration found in this +//! workspace's `.cargo/config`. +//! +//! By default, this script accesses the `wstd-example-bucket` in `us-west-2`. +//! To change the bucket or region, use the `--bucket` and `--region` cli +//! flags before the subcommand. + use anyhow::Result; use clap::{Parser, Subcommand}; @@ -7,12 +48,13 @@ use aws_sdk_s3::Client; #[derive(Debug, Parser)] #[command(version, about, long_about = None)] struct Opts { - /// The AWS Region. + /// The AWS Region. Defaults to us-west-2 if not provided. #[arg(short, long)] - region: String, - /// The name of the bucket. + region: Option, + /// The name of the bucket. Defaults to wstd-example-bucket if not + /// provided. #[arg(short, long)] - bucket: String, + bucket: Option, #[command(subcommand)] command: Option, @@ -31,8 +73,17 @@ enum Command { #[wstd::main] async fn main() -> Result<()> { let opts = Opts::parse(); + let region = opts + .region + .clone() + .unwrap_or_else(|| "us-west-2".to_owned()); + let bucket = opts + .bucket + .clone() + .unwrap_or_else(|| "wstd-example-bucket".to_owned()); + let config = aws_config::defaults(BehaviorVersion::latest()) - .region(Region::new(opts.region.clone())) + .region(Region::new(region)) .sleep_impl(wstd_aws::sleep_impl()) .http_client(wstd_aws::http_client()) .load() @@ -42,11 +93,11 @@ async fn main() -> Result<()> { match opts.command.as_ref().unwrap_or(&Command::List) { Command::List => { - let output = list(&opts, &client).await?; + let output = list(&bucket, &client).await?; print!("{}", output); } Command::Get { key, out } => { - let contents = get(&opts, &client, key).await?; + let contents = get(&bucket, &client, key).await?; let output: &str = if let Some(out) = out { out.as_str() } else { @@ -58,10 +109,10 @@ async fn main() -> Result<()> { Ok(()) } -async fn list(opts: &Opts, client: &Client) -> Result { +async fn list(bucket: &str, client: &Client) -> Result { let mut listing = client .list_objects_v2() - .bucket(opts.bucket.clone()) + .bucket(bucket.to_owned()) .into_paginator() .send(); @@ -86,10 +137,10 @@ async fn list(opts: &Opts, client: &Client) -> Result { Ok(output) } -async fn get(opts: &Opts, client: &Client, key: &str) -> Result> { +async fn get(bucket: &str, client: &Client, key: &str) -> Result> { let object = client .get_object() - .bucket(opts.bucket.clone()) + .bucket(bucket.to_owned()) .key(key) .send() .await?; From f20045641f58b59b0af32428cbb8d4bd767a285f Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Tue, 28 Oct 2025 12:31:45 -0700 Subject: [PATCH 15/16] typo --- aws/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/README.md b/aws/README.md index 792a4e4..d404bcd 100644 --- a/aws/README.md +++ b/aws/README.md @@ -57,7 +57,7 @@ wasmtime run -Shttp \ --env AWS_SECRET_ACCESS_KEY \ --env AWS_SESSION_TOKEN \ --dir .::. \ - target/wasm22-wasip2/release/examples/s3.wasm + target/wasm32-wasip2/release/examples/s3.wasm ``` or alternatively run it with: From 9d98c4d973cef400a074e9f329dfd4a61139e1f1 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Mon, 3 Nov 2025 09:17:06 -0800 Subject: [PATCH 16/16] comments runner capabilities --- .cargo/config.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.cargo/config.toml b/.cargo/config.toml index 8b62960..e9a242c 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,5 @@ [target.wasm32-wasip2] +# wasmtime is given: +# * AWS auth environment variables, for running the wstd-aws integration tests. +# * . directory is available at . runner = "wasmtime run -Shttp --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env AWS_SESSION_TOKEN --dir .::."