add wstd-aws crate, for integration with aws-sdk

.cargo/config.toml

@@ -1,2 +1,5 @@
 [target.wasm32-wasip2]
-runner = "wasmtime -Shttp"
+# wasmtime is given:
+# * AWS auth environment variables, for running the wstd-aws integration tests.
+# * . directory is available at .
+runner = "wasmtime run -Shttp --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env AWS_SESSION_TOKEN --dir .::."

.github/workflows/ci.yaml

@@ -10,6 +10,10 @@ on:
 env:
   RUSTFLAGS: -Dwarnings
 
+# required for AWS oidc
+permissions:
+  id-token: write
+
 jobs:
   build_and_test:
     name: Build and test
@@ -26,24 +30,35 @@ jobs:
     - name: Install wasmtime
       uses: bytecodealliance/actions/wasmtime/setup@v1
 
-    - name: check
-      uses: actions-rs/cargo@v1
+    # pchickey made a role `wstd-aws-ci-role` and bucket `wstd-example-bucket`
+    # on his personal aws account 313377415443. The role only provides
+    # ListBucket and GetObject for the example bucket, which is enough to pass
+    # the single integration test. The role is configured to trust GitHub
+    # actions for the bytecodealliance/wstd repo. This action will set the
+    # AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, and AWS_SESSION_TOKEN
+    # environment variables.
+    - name: get aws credentials
+      id: creds
+      uses: aws-actions/configure-aws-credentials@v5.1.0
+      continue-on-error: true
       with:
-        command: check
-        args: --workspace --all --bins --examples
+        aws-region: us-west-2
+        role-to-assume: arn:aws:iam::313377415443:role/wstd-aws-ci-role
+        role-session-name: github-ci
+
+    - name: check
+      run: cargo check --workspace --all --bins --examples
 
     - name: wstd tests
-      uses: actions-rs/cargo@v1
-      with:
-        command: test
-        args: -p wstd --target wasm32-wasip2 -- --nocapture
+      run: cargo test -p wstd -p wstd-axum -p wstd-aws --target wasm32-wasip2 -- --nocapture
 
-    - name: example tests
-      uses: actions-rs/cargo@v1
-      with:
-        command: test
-        args: -p test-programs -- --nocapture
+    - name: test-programs tests
+      run: cargo test -p test-programs -- --nocapture
+      if: steps.creds.outcome == 'success'
 
+    - name: test-programs tests (no aws)
+      run: cargo test -p test-programs --features no-aws -- --nocapture
+      if: steps.creds.outcome != 'success'
 
   check_fmt_and_docs:
     name: Checking fmt and docs

Cargo.toml

@@ -44,7 +44,7 @@ serde = { workspace = true, features = ["derive"] }
 serde_json.workspace = true
 
 [workspace]
-members = [
+members = ["aws",
     "axum",
     "axum/macro",
     "macro",
@@ -70,6 +70,11 @@ authors = [
 [workspace.dependencies]
 anyhow = "1"
 async-task = "4.7"
+aws-config = { version = "1.8.8", default-features = false }
+aws-sdk-s3 = { version = "1.108.0", default-features = false }
+aws-smithy-async = { version = "1.2.6", default-features = false }
+aws-smithy-types = { version = "1.3.3", default-features = false }
+aws-smithy-runtime-api = { version = "1.9.1", default-features = false }
 axum = { version = "0.8.6", default-features = false }
 bytes = "1.10.1"
 cargo_metadata = "0.22"
@@ -88,6 +93,7 @@ quote = "1.0"
 serde= "1"
 serde_json = "1"
 serde_qs = "0.15"
+sync_wrapper = "1"
 slab = "0.4.9"
 syn = "2.0"
 test-log = { version = "0.2", features = ["trace"] }

aws/.gitignore

@@ -0,0 +1 @@
+.environment

aws/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "wstd-aws"
+description = ""
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+keywords.workspace = true
+categories.workspace = true
+rust-version.workspace = true
+authors.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+aws-smithy-async = { workspace = true }
+aws-smithy-types = { workspace = true, features = ["http-body-1-x"] }
+aws-smithy-runtime-api = { workspace = true, features = ["client", "http-1x"] }
+http-body-util.workspace = true
+sync_wrapper = { workspace = true, features = ["futures"] }
+wstd.workspace = true
+
+[dev-dependencies]
+aws-config.workspace = true
+aws-sdk-s3.workspace = true
+clap.workspace = true

aws/README.md

@@ -0,0 +1,75 @@
+
+# wstd-aws: wstd support for the AWS Rust SDK
+
+This crate provides support for using the AWS Rust SDK for the `wasm32-wasip2`
+target using the [`wstd`] crate.
+
+In many wasi settings, its necessary or desirable to use the wasi-http
+interface to make http requests. Wasi-http interfaces provide an http
+implementation, including the sockets layer and TLS, outside of the user's
+component. `wstd` provides user-friendly async Rust interfaces to all of the
+standardized wasi interfaces, including wasi-http.
+
+The AWS Rust SDK, by default, depends on `tokio`, `hyper`, and either `rustls`
+or `s2n_tls`, and makes http requests over sockets (which can be provided as
+wasi-sockets). Those dependencies may not work correctly under `wasm32-wasip2`,
+and if they do, they will not use the wasi-http interfaces. To avoid using
+http over sockets, make sure to set the `default-features = false` setting
+when depending on any `aws-*` crates in your project.
+
+To configure `wstd`'s wasi-http client for the AWS Rust SDK, provide
+`wstd_aws::sleep_impl()` and `wstd_aws::http_client()` to your
+[`aws_config::ConfigLoader`]:
+
+```
+    let config = aws_config::defaults(BehaviorVersion::latest())
+        .sleep_impl(wstd_aws::sleep_impl())
+        .http_client(wstd_aws::http_client())
+        ...;
+```
+
+[`wstd`]: https://docs.rs/wstd/latest/wstd
+[`aws_config::ConfigLoader`]: https://docs.rs/aws-config/1.8.8/aws_config/struct.ConfigLoader.html
+
+## Example
+
+An example s3 client is provided as a wasi cli command. It accepts command
+line arguments with the subcommand `list` to list a bucket's contents, and
+`get <key>` to get an object from a bucket and write it to the filesystem.
+
+This example *must be compiled in release mode* - in debug mode, the aws
+sdk's generated code will overflow the maximum permitted wasm locals in
+a single function.
+
+Compile it with:
+
+```sh
+cargo build -p wstd-aws --target wasm32-wasip2 --release --examples
+```
+
+When running this example, you will need AWS credentials provided in environment
+variables.
+
+Run it with:
+```sh
+wasmtime run -Shttp \
+    --env AWS_ACCESS_KEY_ID \
+    --env AWS_SECRET_ACCESS_KEY \
+    --env AWS_SESSION_TOKEN \
+    --dir .::. \
+    target/wasm32-wasip2/release/examples/s3.wasm
+```
+
+or alternatively run it with:
+```sh
+cargo run --target wasm32-wasip2 -p wstd-aws --example s3
+```
+
+which uses the wasmtime cli, as above, via configiration found in this
+workspace's `.cargo/config`.
+
+By default, this script accesses the `wstd-example-bucket` in `us-west-2`.
+To change the bucket or region, use the `--bucket` and `--region` cli
+flags before the subcommand.
+
+

aws/examples/s3.rs

@@ -0,0 +1,149 @@
+//! Example s3 client running on `wstd` via `wstd_aws`
+//!
+//! This example is a wasi cli command. It accepts command line arguments
+//! with the subcommand `list` to list a bucket's contents, and `get <key>`
+//! to get an object from a bucket and write it to the filesystem.
+//!
+//! This example *must be compiled in release mode* - in debug mode, the aws
+//! sdk's generated code will overflow the maximum permitted wasm locals in
+//! a single function.
+//!
+//! Compile it with:
+//!
+//! ```sh
+//! cargo build -p wstd-aws --target wasm32-wasip2 --release --examples
+//! ```
+//!
+//! When running this example, you will need AWS credentials provided in environment
+//! variables.
+//!
+//! Run it with:
+//! ```sh
+//! wasmtime run -Shttp \
+//!     --env AWS_ACCESS_KEY_ID \
+//!     --env AWS_SECRET_ACCESS_KEY \
+//!     --env AWS_SESSION_TOKEN \
+//!     --dir .::. \
+//!     target/wasm22-wasip2/release/examples/s3.wasm
+//! ```
+//!
+//! or alternatively run it with:
+//! ```sh
+//! cargo run --target wasm32-wasip2 -p wstd-aws --example s3
+//! ```
+//!
+//! which uses the wasmtime cli, as above, via configiration found in this
+//! workspace's `.cargo/config`.
+//!
+//! By default, this script accesses the `wstd-example-bucket` in `us-west-2`.
+//! To change the bucket or region, use the `--bucket` and `--region` cli
+//! flags before the subcommand.
+
+use anyhow::Result;
+use clap::{Parser, Subcommand};
+
+use aws_config::{BehaviorVersion, Region};
+use aws_sdk_s3::Client;
+
+#[derive(Debug, Parser)]
+#[command(version, about, long_about = None)]
+struct Opts {
+    /// The AWS Region. Defaults to us-west-2 if not provided.
+    #[arg(short, long)]
+    region: Option<String>,
+    /// The name of the bucket. Defaults to wstd-example-bucket if not
+    /// provided.
+    #[arg(short, long)]
+    bucket: Option<String>,
+
+    #[command(subcommand)]
+    command: Option<Command>,
+}
+
+#[derive(Subcommand, Debug)]
+enum Command {
+    List,
+    Get {
+        key: String,
+        #[arg(short, long)]
+        out: Option<String>,
+    },
+}
+
+#[wstd::main]
+async fn main() -> Result<()> {
+    let opts = Opts::parse();
+    let region = opts
+        .region
+        .clone()
+        .unwrap_or_else(|| "us-west-2".to_owned());
+    let bucket = opts
+        .bucket
+        .clone()
+        .unwrap_or_else(|| "wstd-example-bucket".to_owned());
+
+    let config = aws_config::defaults(BehaviorVersion::latest())
+        .region(Region::new(region))
+        .sleep_impl(wstd_aws::sleep_impl())
+        .http_client(wstd_aws::http_client())
+        .load()
+        .await;
+
+    let client = Client::new(&config);
+
+    match opts.command.as_ref().unwrap_or(&Command::List) {
+        Command::List => {
+            let output = list(&bucket, &client).await?;
+            print!("{}", output);
+        }
+        Command::Get { key, out } => {
+            let contents = get(&bucket, &client, key).await?;
+            let output: &str = if let Some(out) = out {
+                out.as_str()
+            } else {
+                key.as_str()
+            };
+            std::fs::write(output, contents)?;
+        }
+    }
+    Ok(())
+}
+
+async fn list(bucket: &str, client: &Client) -> Result<String> {
+    let mut listing = client
+        .list_objects_v2()
+        .bucket(bucket.to_owned())
+        .into_paginator()
+        .send();
+
+    let mut output = String::new();
+    output += "key\tetag\tlast_modified\tstorage_class\n";
+    while let Some(res) = listing.next().await {
+        let object = res?;
+        for item in object.contents() {
+            output += &format!(
+                "{}\t{}\t{}\t{}\n",
+                item.key().unwrap_or_default(),
+                item.e_tag().unwrap_or_default(),
+                item.last_modified()
+                    .map(|lm| format!("{lm}"))
+                    .unwrap_or_default(),
+                item.storage_class()
+                    .map(|sc| format!("{sc}"))
+                    .unwrap_or_default(),
+            );
+        }
+    }
+    Ok(output)
+}
+
+async fn get(bucket: &str, client: &Client, key: &str) -> Result<Vec<u8>> {
+    let object = client
+        .get_object()
+        .bucket(bucket.to_owned())
+        .key(key)
+        .send()
+        .await?;
+    let data = object.body.collect().await?;
+    Ok(data.to_vec())
+}