From a6feb8d64a024792e574c339923847c8a13bbf3f Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Fri, 26 Apr 2024 00:19:31 +0200 Subject: [PATCH 1/7] chore: Use from path (to support fork via git) --- s3/Cargo.toml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/s3/Cargo.toml b/s3/Cargo.toml index 873120053d..3e9d150dda 100644 --- a/s3/Cargo.toml +++ b/s3/Cargo.toml @@ -22,10 +22,8 @@ path = "src/lib.rs" async-std = { version = "1", optional = true } async-trait = "0.1" attohttpc = { version = "0.19", optional = true, default-features = false } -aws-creds = { version = "0.30.0", default-features = false } -# aws-creds = { path = "../aws-creds", default-features = false } -aws-region = "0.25.1" -# aws-region = {path = "../aws-region"} +aws-creds = { path = "../aws-creds" } +aws-region = { path = "../aws-region" } base64 = "0.13" cfg-if = "1" time = { version = "^0.3.6", features = ["formatting", "macros"] } From 4e37f989ed08f0f151fb6db59841a44e8afe90d9 Mon Sep 17 00:00:00 2001 From: Janrupf Date: Thu, 11 Aug 2022 21:50:05 +0200 Subject: [PATCH 2/7] Fix date and time parsing (#287) --- aws-creds/src/credentials.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws-creds/src/credentials.rs b/aws-creds/src/credentials.rs index 340fcbb134..1e01d92dff 100644 --- a/aws-creds/src/credentials.rs +++ b/aws-creds/src/credentials.rs @@ -397,5 +397,5 @@ fn test_instance_metadata_creds_deserialization() { } "#, ) - .unwrap(); + .unwrap(); } From be20af58a314d48e3df36224bde9435af017e3c9 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Fri, 26 Apr 2024 00:23:36 +0200 Subject: [PATCH 3/7] chore: Use rust-s3 path for git protocol --- Cargo.toml | 6 +----- {s3 => rust-s3}/Cargo.toml | 3 +-- {s3 => rust-s3}/Makefile | 0 {s3 => rust-s3}/README.md | 0 {s3 => rust-s3}/bin/simple_crud.rs | 0 {s3 => rust-s3}/src/blocking.rs | 0 {s3 => rust-s3}/src/bucket.rs | 0 {s3 => rust-s3}/src/bucket_ops.rs | 0 {s3 => rust-s3}/src/command.rs | 0 {s3 => rust-s3}/src/deserializer.rs | 0 {s3 => rust-s3}/src/error.rs | 0 {s3 => rust-s3}/src/lib.rs | 0 {s3 => rust-s3}/src/request.rs | 0 {s3 => rust-s3}/src/request_trait.rs | 0 {s3 => rust-s3}/src/serde_types.rs | 0 {s3 => rust-s3}/src/signing.rs | 0 {s3 => rust-s3}/src/surf_request.rs | 0 {s3 => rust-s3}/src/utils.rs | 0 18 files changed, 2 insertions(+), 7 deletions(-) rename {s3 => rust-s3}/Cargo.toml (95%) rename {s3 => rust-s3}/Makefile (100%) rename {s3 => rust-s3}/README.md (100%) rename {s3 => rust-s3}/bin/simple_crud.rs (100%) rename {s3 => rust-s3}/src/blocking.rs (100%) rename {s3 => rust-s3}/src/bucket.rs (100%) rename {s3 => rust-s3}/src/bucket_ops.rs (100%) rename {s3 => rust-s3}/src/command.rs (100%) rename {s3 => rust-s3}/src/deserializer.rs (100%) rename {s3 => rust-s3}/src/error.rs (100%) rename {s3 => rust-s3}/src/lib.rs (100%) rename {s3 => rust-s3}/src/request.rs (100%) rename {s3 => rust-s3}/src/request_trait.rs (100%) rename {s3 => rust-s3}/src/serde_types.rs (100%) rename {s3 => rust-s3}/src/signing.rs (100%) rename {s3 => rust-s3}/src/surf_request.rs (100%) rename {s3 => rust-s3}/src/utils.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 126c82bcd6..989398ddb0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,2 @@ [workspace] -members = [ - "s3", - "aws-region", - "aws-creds" -] \ No newline at end of file +members = ["rust-s3", "aws-region", "aws-creds"] diff --git a/s3/Cargo.toml b/rust-s3/Cargo.toml similarity index 95% rename from s3/Cargo.toml rename to rust-s3/Cargo.toml index 3e9d150dda..5996f455be 100644 --- a/s3/Cargo.toml +++ b/rust-s3/Cargo.toml @@ -76,6 +76,5 @@ tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "fs"] } async-std = { version = "1", features = ["attributes"] } uuid = { version = "1", features = ["v4"] } env_logger = "0.9" -aws-creds = { version = "0.30", default-features = false } -# aws-creds = { path = "../aws-creds", features = ["http-credentials"] } +aws-creds = { path = "../aws-creds", features = ["http-credentials"] } anyhow = "1" diff --git a/s3/Makefile b/rust-s3/Makefile similarity index 100% rename from s3/Makefile rename to rust-s3/Makefile diff --git a/s3/README.md b/rust-s3/README.md similarity index 100% rename from s3/README.md rename to rust-s3/README.md diff --git a/s3/bin/simple_crud.rs b/rust-s3/bin/simple_crud.rs similarity index 100% rename from s3/bin/simple_crud.rs rename to rust-s3/bin/simple_crud.rs diff --git a/s3/src/blocking.rs b/rust-s3/src/blocking.rs similarity index 100% rename from s3/src/blocking.rs rename to rust-s3/src/blocking.rs diff --git a/s3/src/bucket.rs b/rust-s3/src/bucket.rs similarity index 100% rename from s3/src/bucket.rs rename to rust-s3/src/bucket.rs diff --git a/s3/src/bucket_ops.rs b/rust-s3/src/bucket_ops.rs similarity index 100% rename from s3/src/bucket_ops.rs rename to rust-s3/src/bucket_ops.rs diff --git a/s3/src/command.rs b/rust-s3/src/command.rs similarity index 100% rename from s3/src/command.rs rename to rust-s3/src/command.rs diff --git a/s3/src/deserializer.rs b/rust-s3/src/deserializer.rs similarity index 100% rename from s3/src/deserializer.rs rename to rust-s3/src/deserializer.rs diff --git a/s3/src/error.rs b/rust-s3/src/error.rs similarity index 100% rename from s3/src/error.rs rename to rust-s3/src/error.rs diff --git a/s3/src/lib.rs b/rust-s3/src/lib.rs similarity index 100% rename from s3/src/lib.rs rename to rust-s3/src/lib.rs diff --git a/s3/src/request.rs b/rust-s3/src/request.rs similarity index 100% rename from s3/src/request.rs rename to rust-s3/src/request.rs diff --git a/s3/src/request_trait.rs b/rust-s3/src/request_trait.rs similarity index 100% rename from s3/src/request_trait.rs rename to rust-s3/src/request_trait.rs diff --git a/s3/src/serde_types.rs b/rust-s3/src/serde_types.rs similarity index 100% rename from s3/src/serde_types.rs rename to rust-s3/src/serde_types.rs diff --git a/s3/src/signing.rs b/rust-s3/src/signing.rs similarity index 100% rename from s3/src/signing.rs rename to rust-s3/src/signing.rs diff --git a/s3/src/surf_request.rs b/rust-s3/src/surf_request.rs similarity index 100% rename from s3/src/surf_request.rs rename to rust-s3/src/surf_request.rs diff --git a/s3/src/utils.rs b/rust-s3/src/utils.rs similarity index 100% rename from s3/src/utils.rs rename to rust-s3/src/utils.rs From 4ba2d8b4ea85bdf0081d6577e721675e2e41507f Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Fri, 26 Apr 2024 00:45:55 +0200 Subject: [PATCH 4/7] chore: Use aws-region from upstream as crate --- rust-s3/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-s3/Cargo.toml b/rust-s3/Cargo.toml index 5996f455be..d765dec8f1 100644 --- a/rust-s3/Cargo.toml +++ b/rust-s3/Cargo.toml @@ -23,7 +23,7 @@ async-std = { version = "1", optional = true } async-trait = "0.1" attohttpc = { version = "0.19", optional = true, default-features = false } aws-creds = { path = "../aws-creds" } -aws-region = { path = "../aws-region" } +aws-region = { version = "0.25.4" } base64 = "0.13" cfg-if = "1" time = { version = "^0.3.6", features = ["formatting", "macros"] } From 3fd5a88a571cdc5b6ceb75b8020ccd52a9065450 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Wed, 15 May 2024 14:42:29 +0200 Subject: [PATCH 5/7] chore: Use aws-creds from upstream --- Cargo.toml | 2 +- rust-s3/Cargo.toml | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 989398ddb0..49c9d043cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,2 +1,2 @@ [workspace] -members = ["rust-s3", "aws-region", "aws-creds"] +members = ["rust-s3"] diff --git a/rust-s3/Cargo.toml b/rust-s3/Cargo.toml index d765dec8f1..f7e44c78da 100644 --- a/rust-s3/Cargo.toml +++ b/rust-s3/Cargo.toml @@ -14,15 +14,11 @@ edition = "2018" name = "s3" path = "src/lib.rs" -# [[bin]] -# name = "simple_crud" -# path = "bin/simple_crud.rs" - [dependencies] async-std = { version = "1", optional = true } async-trait = "0.1" attohttpc = { version = "0.19", optional = true, default-features = false } -aws-creds = { path = "../aws-creds" } +aws-creds = { version = "0.36.0" } aws-region = { version = "0.25.4" } base64 = "0.13" cfg-if = "1" @@ -76,5 +72,5 @@ tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "fs"] } async-std = { version = "1", features = ["attributes"] } uuid = { version = "1", features = ["v4"] } env_logger = "0.9" -aws-creds = { path = "../aws-creds", features = ["http-credentials"] } +aws-creds = { version = "0.36.0", features = ["http-credentials"] } anyhow = "1" From 371118db869d59cccc5c84be7b65a6e663052387 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Wed, 15 May 2024 14:42:59 +0200 Subject: [PATCH 6/7] chore: Remove aws-creds, aws-region because we moved to upstream --- aws-creds/Cargo.toml | 37 ---- aws-creds/Makefile | 13 -- aws-creds/README.md | 35 --- aws-creds/src/credentials.rs | 401 ----------------------------------- aws-creds/src/error.rs | 30 --- aws-creds/src/lib.rs | 9 - aws-region/Cargo.toml | 19 -- aws-region/Makefile | 13 -- aws-region/README.md | 19 -- aws-region/src/error.rs | 4 - aws-region/src/lib.rs | 6 - aws-region/src/region.rs | 266 ----------------------- 12 files changed, 852 deletions(-) delete mode 100644 aws-creds/Cargo.toml delete mode 100644 aws-creds/Makefile delete mode 100644 aws-creds/README.md delete mode 100644 aws-creds/src/credentials.rs delete mode 100644 aws-creds/src/error.rs delete mode 100644 aws-creds/src/lib.rs delete mode 100644 aws-region/Cargo.toml delete mode 100644 aws-region/Makefile delete mode 100644 aws-region/README.md delete mode 100644 aws-region/src/error.rs delete mode 100644 aws-region/src/lib.rs delete mode 100644 aws-region/src/region.rs diff --git a/aws-creds/Cargo.toml b/aws-creds/Cargo.toml deleted file mode 100644 index e4934b7d5d..0000000000 --- a/aws-creds/Cargo.toml +++ /dev/null @@ -1,37 +0,0 @@ -[package] -name = "aws-creds" -version = "0.30.0" -authors = ["Drazen Urch"] -description = "Tiny Rust library for working with Amazon IAM credential,s, supports `s3` crate" -repository = "https://github.com/durch/rust-s3" -readme = "README.md" -keywords = ["AWS", "S3", "Wasabi", "Minio", "Yandex"] -license = "MIT" -documentation = "https://durch.github.io/rust-s3/" -edition = "2018" - -[lib] -name = "awscreds" -path = "src/lib.rs" - -[dependencies] -thiserror = "1" -dirs = "4" -rust-ini = "0.18" -attohttpc = { version = "0.19", default-features = false, features = [ - "json", -], optional = true } -url = "2" -serde-xml-rs = "0.5" -serde = { version = "1", features = ["derive"] } -time = { version = "^0.3.6", features = ["serde", "serde-well-known"] } - -[features] -default = ["native-tls"] -http-credentials = ["attohttpc"] -native-tls = ["http-credentials", "attohttpc/tls"] -rustls-tls = ["http-credentials", "attohttpc/tls-rustls"] - -[dev-dependencies] -env_logger = "0.9" -serde_json = "1" diff --git a/aws-creds/Makefile b/aws-creds/Makefile deleted file mode 100644 index b38dcfaa6d..0000000000 --- a/aws-creds/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -ci: fmt-check clippy test - -fmt-check: - cargo fmt --all -- --check - -clippy: - cargo clippy --all-features -- -D warnings - -test: - cargo test --all-features - - - diff --git a/aws-creds/README.md b/aws-creds/README.md deleted file mode 100644 index f138889aa7..0000000000 --- a/aws-creds/README.md +++ /dev/null @@ -1,35 +0,0 @@ - - -# Example - -```rust -// AWS access credentials: access key, secret key, and optional token. -# Example -// Loads from the standard AWS credentials file with the given profile name, -// defaults to "default". -use awscreds::Credentials; - -// Load credentials from `[default]` profile -let credentials = Credentials::default(); -// Also loads credentials from `[default]` profile -let credentials = Credentials::new(None, None, None, None); -// Load credentials from `[my-profile]` profile -let credentials = Credentials::new(None, None, None, Some("my-profile".into())); -// Credentials may also be initialized directly or by the following environment variables: -// - `AWS_ACCESS_KEY_ID`, -// - `AWS_SECRET_ACCESS_KEY` -// - `AWS_SESSION_TOKEN` -// The order of preference is arguments, then environment, and finally AWS -// credentials file. - -use s3::credentials::Credentials; -// Load credentials directly -let access_key = String::from("AKIAIOSFODNN7EXAMPLE"); -let secret_key = String::from("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); -let credentials = Credentials::new(Some(access_key), Some(secret_key), None, None); -// Load credentials from the environment -use std::env; -env::set_var("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE"); -env::set_var("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); -let credentials = Credentials::new(None, None, None, None); -``` \ No newline at end of file diff --git a/aws-creds/src/credentials.rs b/aws-creds/src/credentials.rs deleted file mode 100644 index 1e01d92dff..0000000000 --- a/aws-creds/src/credentials.rs +++ /dev/null @@ -1,401 +0,0 @@ -#![allow(dead_code)] -use crate::error::CredentialsError; -use ini::Ini; -use serde::{Deserialize, Serialize}; -use serde_xml_rs as serde_xml; -use std::collections::HashMap; -use std::env; -use std::sync::atomic::AtomicU32; -use std::sync::atomic::Ordering; -use std::time::Duration; -use url::Url; - -/// AWS access credentials: access key, secret key, and optional token. -/// -/// # Example -/// -/// Loads from the standard AWS credentials file with the given profile name, -/// defaults to "default". -/// -/// ```no_run -/// # // Do not execute this as it would cause unit tests to attempt to access -/// # // real user credentials. -/// use awscreds::Credentials; -/// -/// // Load credentials from `[default]` profile -/// #[cfg(feature="http-credentials")] -/// let credentials = Credentials::default(); -/// -/// // Also loads credentials from `[default]` profile -/// #[cfg(feature="http-credentials")] -/// let credentials = Credentials::new(None, None, None, None, None); -/// -/// // Load credentials from `[my-profile]` profile -/// #[cfg(feature="http-credentials")] -/// let credentials = Credentials::new(None, None, None, None, Some("my-profile".into())); -/// ``` -/// // Use anonymous credentials for public objects -/// let credentials = Credentials::anonymous(); -/// -/// Credentials may also be initialized directly or by the following environment variables: -/// -/// - `AWS_ACCESS_KEY_ID`, -/// - `AWS_SECRET_ACCESS_KEY` -/// - `AWS_SESSION_TOKEN` -/// -/// The order of preference is arguments, then environment, and finally AWS -/// credentials file. -/// -/// ``` -/// use awscreds::Credentials; -/// -/// // Load credentials directly -/// let access_key = "AKIAIOSFODNN7EXAMPLE"; -/// let secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; -/// #[cfg(feature="http-credentials")] -/// let credentials = Credentials::new(Some(access_key), Some(secret_key), None, None, None); -/// -/// // Load credentials from the environment -/// use std::env; -/// env::set_var("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE"); -/// env::set_var("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); -/// #[cfg(feature="http-credentials")] -/// let credentials = Credentials::new(None, None, None, None, None); -/// ``` -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub struct Credentials { - /// AWS public access key. - pub access_key: Option, - /// AWS secret key. - pub secret_key: Option, - /// Temporary token issued by AWS service. - pub security_token: Option, - pub session_token: Option, - pub expiration: Option, -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "PascalCase")] -pub struct AssumeRoleWithWebIdentityResponse { - pub assume_role_with_web_identity_result: AssumeRoleWithWebIdentityResult, - pub response_metadata: ResponseMetadata, -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "PascalCase")] -pub struct AssumeRoleWithWebIdentityResult { - pub subject_from_web_identity_token: String, - pub audience: String, - pub assumed_role_user: AssumedRoleUser, - pub credentials: StsResponseCredentials, - pub provider: String, -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "PascalCase")] -pub struct StsResponseCredentials { - pub session_token: String, - pub secret_access_key: String, - pub expiration: time::OffsetDateTime, - pub access_key_id: String, -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "PascalCase")] -pub struct AssumedRoleUser { - pub arn: String, - pub assumed_role_id: String, -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "PascalCase")] -pub struct ResponseMetadata { - pub request_id: String, -} - -/// The global request timeout in milliseconds. 0 means no timeout. -/// -/// Defaults to 30 seconds. -static REQUEST_TIMEOUT_MS: AtomicU32 = AtomicU32::new(30_000); - -/// Sets the timeout for all credentials HTTP requests and returns the -/// old timeout value, if any; this timeout applies after a 30-second -/// connection timeout. -/// -/// Short durations are bumped to one millisecond, and durations -/// greater than 4 billion milliseconds (49 days) are rounded up to -/// infinity (no timeout). -/// The global default value is 30 seconds. -#[cfg(feature = "http-credentials")] -pub fn set_request_timeout(timeout: Option) -> Option { - use std::convert::TryInto; - let duration_ms = timeout - .as_ref() - .map(Duration::as_millis) - .unwrap_or(u128::MAX) - .max(1); // A 0 duration means infinity. - - // Store that non-zero u128 value in an AtomicU32 by mapping large - // values to 0: `http_get` maps that to no (infinite) timeout. - let prev = REQUEST_TIMEOUT_MS.swap(duration_ms.try_into().unwrap_or(0), Ordering::Relaxed); - - if prev == 0 { - None - } else { - Some(Duration::from_millis(prev as u64)) - } -} - -/// Sends a GET request to `url` with a request timeout if one was set. -#[cfg(feature = "http-credentials")] -fn http_get(url: &str) -> attohttpc::Result { - let mut builder = attohttpc::get(url); - - let timeout_ms = REQUEST_TIMEOUT_MS.load(Ordering::Relaxed); - if timeout_ms > 0 { - builder = builder.timeout(Duration::from_millis(timeout_ms as u64)); - } - - builder.send() -} - -impl Credentials { - #[cfg(feature = "http-credentials")] - pub fn from_sts_env(session_name: &str) -> Result { - let role_arn = env::var("AWS_ROLE_ARN")?; - let web_identity_token_file = env::var("AWS_WEB_IDENTITY_TOKEN_FILE")?; - let web_identity_token = std::fs::read_to_string(web_identity_token_file)?; - Credentials::from_sts(&role_arn, session_name, &web_identity_token) - } - - #[cfg(feature = "http-credentials")] - pub fn from_sts( - role_arn: &str, - session_name: &str, - web_identity_token: &str, - ) -> Result { - let url = Url::parse_with_params( - "https://sts.amazonaws.com/", - &[ - ("Action", "AssumeRoleWithWebIdentity"), - ("RoleSessionName", session_name), - ("RoleArn", role_arn), - ("WebIdentityToken", web_identity_token), - ("Version", "2011-06-15"), - ], - )?; - let response = http_get(url.as_str())?; - let serde_response = - serde_xml::from_str::(&response.text()?)?; - // assert!(serde_xml::from_str::(&response.text()?).unwrap()); - - Ok(Credentials { - access_key: Some( - serde_response - .assume_role_with_web_identity_result - .credentials - .access_key_id, - ), - secret_key: Some( - serde_response - .assume_role_with_web_identity_result - .credentials - .secret_access_key, - ), - security_token: None, - session_token: Some( - serde_response - .assume_role_with_web_identity_result - .credentials - .session_token, - ), - expiration: Some( - serde_response - .assume_role_with_web_identity_result - .credentials - .expiration, - ), - }) - } - - #[cfg(feature = "http-credentials")] - pub fn default() -> Result { - Credentials::new(None, None, None, None, None) - } - - pub fn anonymous() -> Result { - Ok(Credentials { - access_key: None, - secret_key: None, - security_token: None, - session_token: None, - expiration: None, - }) - } - - /// Initialize Credentials directly with key ID, secret key, and optional - /// token. - #[cfg(feature = "http-credentials")] - pub fn new( - access_key: Option<&str>, - secret_key: Option<&str>, - security_token: Option<&str>, - session_token: Option<&str>, - profile: Option<&str>, - ) -> Result { - if access_key.is_some() { - return Ok(Credentials { - access_key: access_key.map(|s| s.to_string()), - secret_key: secret_key.map(|s| s.to_string()), - security_token: security_token.map(|s| s.to_string()), - session_token: session_token.map(|s| s.to_string()), - expiration: None, - }); - } - - Credentials::from_sts_env("aws-creds") - .or_else(|_| Credentials::from_env()) - .or_else(|_| Credentials::from_profile(profile)) - .or_else(|_| Credentials::from_instance_metadata()) - } - - pub fn from_env_specific( - access_key_var: Option<&str>, - secret_key_var: Option<&str>, - security_token_var: Option<&str>, - session_token_var: Option<&str>, - ) -> Result { - let access_key = from_env_with_default(access_key_var, "AWS_ACCESS_KEY_ID")?; - let secret_key = from_env_with_default(secret_key_var, "AWS_SECRET_ACCESS_KEY")?; - - let security_token = from_env_with_default(security_token_var, "AWS_SECURITY_TOKEN").ok(); - let session_token = from_env_with_default(session_token_var, "AWS_SESSION_TOKEN").ok(); - Ok(Credentials { - access_key: Some(access_key), - secret_key: Some(secret_key), - security_token, - session_token, - expiration: None, - }) - } - - pub fn from_env() -> Result { - Credentials::from_env_specific(None, None, None, None) - } - - #[cfg(feature = "http-credentials")] - pub fn from_instance_metadata() -> Result { - let resp: CredentialsFromInstanceMetadata = - match env::var("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") { - Ok(credentials_path) => { - // We are on ECS - attohttpc::get(&format!("http://169.254.170.2{}", credentials_path)) - .send()? - .json()? - } - Err(_) => { - if !is_ec2() { - return Err(CredentialsError::NotEc2); - } - - let role = attohttpc::get( - "http://169.254.169.254/latest/meta-data/iam/security-credentials", - ) - .send()? - .text()?; - - attohttpc::get(&format!( - "http://169.254.169.254/latest/meta-data/iam/security-credentials/{}", - role - )) - .send()? - .json()? - } - }; - - Ok(Credentials { - access_key: Some(resp.access_key_id), - secret_key: Some(resp.secret_access_key), - security_token: Some(resp.token), - expiration: Some(resp.expiration), - session_token: None, - }) - } - - pub fn from_profile(section: Option<&str>) -> Result { - let home_dir = dirs::home_dir().ok_or(CredentialsError::HomeDir)?; - let profile = format!("{}/.aws/credentials", home_dir.display()); - let conf = Ini::load_from_file(&profile)?; - let section = section.unwrap_or("default"); - let data = conf - .section(Some(section)) - .ok_or(CredentialsError::ConfigNotFound)?; - let access_key = data - .get("aws_access_key_id") - .map(|s| s.to_string()) - .ok_or(CredentialsError::ConfigMissingAccessKeyId)?; - let secret_key = data - .get("aws_secret_access_key") - .map(|s| s.to_string()) - .ok_or(CredentialsError::ConfigMissingSecretKey)?; - let credentials = Credentials { - access_key: Some(access_key), - secret_key: Some(secret_key), - security_token: data.get("aws_security_token").map(|s| s.to_string()), - session_token: data.get("aws_session_token").map(|s| s.to_string()), - expiration: None, - }; - Ok(credentials) - } -} - -fn from_env_with_default(var: Option<&str>, default: &str) -> Result { - let val = var.unwrap_or(default); - env::var(val) - .or_else(|_e| env::var(val)) - .map_err(|_| CredentialsError::MissingEnvVar(val.to_string(), default.to_string())) -} - -fn is_ec2() -> bool { - if let Ok(uuid) = std::fs::read_to_string("/sys/hypervisor/uuid") { - if uuid.starts_with("ec2") { - return true; - } - } - if let Ok(vendor) = std::fs::read_to_string("/sys/class/dmi/id/board_vendor") { - if vendor.starts_with("Amazon EC2") { - return true; - } - } - false -} - -#[derive(Deserialize)] -#[serde(rename_all = "PascalCase")] -struct CredentialsFromInstanceMetadata { - access_key_id: String, - secret_access_key: String, - token: String, - #[serde(with = "time::serde::rfc3339")] - expiration: time::OffsetDateTime, // TODO fix #163 -} -#[cfg(test)] -#[test] -fn test_instance_metadata_creds_deserialization() { - // As documented here: - // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials - serde_json::from_str::( - r#" - { - "Code" : "Success", - "LastUpdated" : "2012-04-26T16:39:16Z", - "Type" : "AWS-HMAC", - "AccessKeyId" : "ASIAIOSFODNN7EXAMPLE", - "SecretAccessKey" : "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - "Token" : "token", - "Expiration" : "2017-05-17T15:09:54Z" - } - "#, - ) - .unwrap(); -} diff --git a/aws-creds/src/error.rs b/aws-creds/src/error.rs deleted file mode 100644 index 4e123576f0..0000000000 --- a/aws-creds/src/error.rs +++ /dev/null @@ -1,30 +0,0 @@ -use thiserror::Error; - -#[derive(Error, Debug)] -pub enum CredentialsError { - #[error("Not an AWS instance")] - NotEc2, - #[error("Config not found")] - ConfigNotFound, - #[error("Missing aws_access_key_id section in config")] - ConfigMissingAccessKeyId, - #[error("Missing aws_access_key_id section in config")] - ConfigMissingSecretKey, - #[error("Neither {0}, nor {1} exists in the environment")] - MissingEnvVar(String, String), - #[cfg(feature = "http-credentials")] - #[error("attohttpc: {0}")] - Atto(#[from] attohttpc::Error), - #[error("ini: {0}")] - Ini(#[from] ini::Error), - #[error("serde_xml: {0}")] - SerdeXml(#[from] serde_xml_rs::Error), - #[error("url parse: {0}")] - UrlParse(#[from] url::ParseError), - #[error("io: {0}")] - Io(#[from] std::io::Error), - #[error("env var: {0}")] - Env(#[from] std::env::VarError), - #[error("Invalid home dir")] - HomeDir, -} diff --git a/aws-creds/src/lib.rs b/aws-creds/src/lib.rs deleted file mode 100644 index 4fe4ffa184..0000000000 --- a/aws-creds/src/lib.rs +++ /dev/null @@ -1,9 +0,0 @@ -#![allow(unused_imports)] -#![forbid(unsafe_code)] - -mod credentials; -pub use credentials::*; -pub mod error; - -// Reexport for e.g. users who need to build Credentials -pub use time; diff --git a/aws-region/Cargo.toml b/aws-region/Cargo.toml deleted file mode 100644 index 55978e6286..0000000000 --- a/aws-region/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "aws-region" -version = "0.25.1" -authors = ["Drazen Urch"] -description = "Tiny Rust library for working with Amazon AWS regions, supports `s3` crate" -repository = "https://github.com/durch/rust-s3" -readme = "README.md" -keywords = ["Amazon", "AWS", "S3", "Wasabi", "Minio"] -license = "MIT" -documentation = "https://durch.github.io/rust-s3/" -edition = "2018" - -[lib] -name = "awsregion" -path = "src/lib.rs" - -[dependencies] -thiserror = "1" -serde = { version = "1", features = ["derive"], optional = true } diff --git a/aws-region/Makefile b/aws-region/Makefile deleted file mode 100644 index b38dcfaa6d..0000000000 --- a/aws-region/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -ci: fmt-check clippy test - -fmt-check: - cargo fmt --all -- --check - -clippy: - cargo clippy --all-features -- -D warnings - -test: - cargo test --all-features - - - diff --git a/aws-region/README.md b/aws-region/README.md deleted file mode 100644 index 9cc45364c1..0000000000 --- a/aws-region/README.md +++ /dev/null @@ -1,19 +0,0 @@ -AWS S3 [region identifier](https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region), -passing in custom values is also possible, in that case it is up to you to pass a valid endpoint, -otherwise boom will happen :) - -# Example -```rust -use std::str::FromStr; -use awsregion::Region; - -// Parse from a string -let region: Region = "us-east-1".parse().unwrap(); -// Choose region directly -let region = Region::EuWest2; - -// Custom region requires valid region name and endpoint -let region_name = "nl-ams".to_string(); -let endpoint = "https://s3.nl-ams.scw.cloud".to_string(); -let region = Region::Custom { region: region_name, endpoint }; -``` \ No newline at end of file diff --git a/aws-region/src/error.rs b/aws-region/src/error.rs deleted file mode 100644 index f3eb7bbc22..0000000000 --- a/aws-region/src/error.rs +++ /dev/null @@ -1,4 +0,0 @@ -use thiserror::Error; - -#[derive(Error, Debug)] -pub enum RegionError {} diff --git a/aws-region/src/lib.rs b/aws-region/src/lib.rs deleted file mode 100644 index df7ae79e7c..0000000000 --- a/aws-region/src/lib.rs +++ /dev/null @@ -1,6 +0,0 @@ -#![allow(unused_imports)] -#![forbid(unsafe_code)] - -mod region; -pub use region::*; -pub mod error; diff --git a/aws-region/src/region.rs b/aws-region/src/region.rs deleted file mode 100644 index 29d8b21a4c..0000000000 --- a/aws-region/src/region.rs +++ /dev/null @@ -1,266 +0,0 @@ -#[cfg(feature = "serde")] -use serde::{Deserialize, Serialize}; -use std::fmt; -use std::str::{self, FromStr}; - -/// AWS S3 [region identifier](https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region), -/// passing in custom values is also possible, in that case it is up to you to pass a valid endpoint, -/// otherwise boom will happen :) -/// -/// Serde support available with the `serde` feature -/// -/// # Example -/// ``` -/// use std::str::FromStr; -/// use awsregion::Region; -/// -/// // Parse from a string -/// let region: Region = "us-east-1".parse().unwrap(); -/// -/// // Choose region directly -/// let region = Region::EuWest2; -/// -/// // Custom region requires valid region name and endpoint -/// let region_name = "nl-ams".to_string(); -/// let endpoint = "https://s3.nl-ams.scw.cloud".to_string(); -/// let region = Region::Custom { region: region_name, endpoint }; -/// -/// ``` -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum Region { - /// us-east-1 - UsEast1, - /// us-east-2 - UsEast2, - /// us-west-1 - UsWest1, - /// us-west-2 - UsWest2, - /// ca-central-1 - CaCentral1, - /// af-south-1 - AfSouth1, - /// ap-east-1 - ApEast1, - /// ap-south-1 - ApSouth1, - /// ap-northeast-1 - ApNortheast1, - /// ap-northeast-2 - ApNortheast2, - /// ap-northeast-3 - ApNortheast3, - /// ap-southeast-1 - ApSoutheast1, - /// ap-southeast-2 - ApSoutheast2, - /// cn-north-1 - CnNorth1, - /// cn-northwest-1 - CnNorthwest1, - /// eu-north-1 - EuNorth1, - /// eu-central-1 - EuCentral1, - /// eu-west-1 - EuWest1, - /// eu-west-2 - EuWest2, - /// eu-west-3 - EuWest3, - /// me-south-1 - MeSouth1, - /// sa-east-1 - SaEast1, - /// Digital Ocean nyc3 - DoNyc3, - /// Digital Ocean ams3 - DoAms3, - /// Digital Ocean sgp1 - DoSgp1, - /// Digiral Ocean fra1 - DoFra1, - /// Yandex Object Storage - Yandex, - /// Wasabi us-east-1 - WaUsEast1, - /// Wasabi us-east-2 - WaUsEast2, - /// Wasabi us-west-1 - WaUsWest1, - /// Wasabi eu-central-1 - WaEuCentral1, - /// Custom region - R2 { - account_id: String, - }, - Custom { - region: String, - endpoint: String, - }, -} - -impl fmt::Display for Region { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - use self::Region::*; - match *self { - UsEast1 => write!(f, "us-east-1"), - UsEast2 => write!(f, "us-east-2"), - UsWest1 => write!(f, "us-west-1"), - UsWest2 => write!(f, "us-west-2"), - AfSouth1 => write!(f, "af-south-1"), - CaCentral1 => write!(f, "ca-central-1"), - ApEast1 => write!(f, "ap-east-1"), - ApSouth1 => write!(f, "ap-south-1"), - ApNortheast1 => write!(f, "ap-northeast-1"), - ApNortheast2 => write!(f, "ap-northeast-2"), - ApNortheast3 => write!(f, "ap-northeast-3"), - ApSoutheast1 => write!(f, "ap-southeast-1"), - ApSoutheast2 => write!(f, "ap-southeast-2"), - CnNorth1 => write!(f, "cn-north-1"), - CnNorthwest1 => write!(f, "cn-northwest-1"), - EuNorth1 => write!(f, "eu-north-1"), - EuCentral1 => write!(f, "eu-central-1"), - EuWest1 => write!(f, "eu-west-1"), - EuWest2 => write!(f, "eu-west-2"), - EuWest3 => write!(f, "eu-west-3"), - SaEast1 => write!(f, "sa-east-1"), - MeSouth1 => write!(f, "me-south-1"), - DoNyc3 => write!(f, "nyc3"), - DoAms3 => write!(f, "ams3"), - DoSgp1 => write!(f, "sgp1"), - DoFra1 => write!(f, "fra1"), - Yandex => write!(f, "ru-central1"), - WaUsEast1 => write!(f, "us-east-1"), - WaUsEast2 => write!(f, "us-east-2"), - WaUsWest1 => write!(f, "us-west-1"), - WaEuCentral1 => write!(f, "eu-central-1"), - R2 { .. } => write!(f, "auto"), - Custom { ref region, .. } => write!(f, "{}", region), - } - } -} - -impl FromStr for Region { - type Err = std::str::Utf8Error; - - fn from_str(s: &str) -> Result { - use self::Region::*; - match s { - "us-east-1" => Ok(UsEast1), - "us-east-2" => Ok(UsEast2), - "us-west-1" => Ok(UsWest1), - "us-west-2" => Ok(UsWest2), - "ca-central-1" => Ok(CaCentral1), - "af-south-1" => Ok(AfSouth1), - "ap-east-1" => Ok(ApEast1), - "ap-south-1" => Ok(ApSouth1), - "ap-northeast-1" => Ok(ApNortheast1), - "ap-northeast-2" => Ok(ApNortheast2), - "ap-northeast-3" => Ok(ApNortheast3), - "ap-southeast-1" => Ok(ApSoutheast1), - "ap-southeast-2" => Ok(ApSoutheast2), - "cn-north-1" => Ok(CnNorth1), - "cn-northwest-1" => Ok(CnNorthwest1), - "eu-north-1" => Ok(EuNorth1), - "eu-central-1" => Ok(EuCentral1), - "eu-west-1" => Ok(EuWest1), - "eu-west-2" => Ok(EuWest2), - "eu-west-3" => Ok(EuWest3), - "sa-east-1" => Ok(SaEast1), - "me-south-1" => Ok(MeSouth1), - "nyc3" => Ok(DoNyc3), - "ams3" => Ok(DoAms3), - "sgp1" => Ok(DoSgp1), - "fra1" => Ok(DoFra1), - "yandex" => Ok(Yandex), - "ru-central1" => Ok(Yandex), - "wa-us-east-1" => Ok(WaUsEast1), - "wa-us-east-2" => Ok(WaUsEast2), - "wa-us-west-1" => Ok(WaUsWest1), - "wa-eu-central-1" => Ok(WaEuCentral1), - x => Ok(Custom { - region: x.to_string(), - endpoint: x.to_string(), - }), - } - } -} - -impl Region { - pub fn endpoint(&self) -> String { - use self::Region::*; - match *self { - // Surprisingly, us-east-1 does not have a - // s3-us-east-1.amazonaws.com DNS record - UsEast1 => String::from("s3.amazonaws.com"), - UsEast2 => String::from("s3-us-east-2.amazonaws.com"), - UsWest1 => String::from("s3-us-west-1.amazonaws.com"), - UsWest2 => String::from("s3-us-west-2.amazonaws.com"), - CaCentral1 => String::from("s3-ca-central-1.amazonaws.com"), - AfSouth1 => String::from("s3-af-south-1.amazonaws.com"), - ApEast1 => String::from("s3-ap-east-1.amazonaws.com"), - ApSouth1 => String::from("s3-ap-south-1.amazonaws.com"), - ApNortheast1 => String::from("s3-ap-northeast-1.amazonaws.com"), - ApNortheast2 => String::from("s3-ap-northeast-2.amazonaws.com"), - ApNortheast3 => String::from("s3-ap-northeast-3.amazonaws.com"), - ApSoutheast1 => String::from("s3-ap-southeast-1.amazonaws.com"), - ApSoutheast2 => String::from("s3-ap-southeast-2.amazonaws.com"), - CnNorth1 => String::from("s3.cn-north-1.amazonaws.com.cn"), - CnNorthwest1 => String::from("s3.cn-northwest-1.amazonaws.com.cn"), - EuNorth1 => String::from("s3-eu-north-1.amazonaws.com"), - EuCentral1 => String::from("s3.eu-central-1.amazonaws.com"), - EuWest1 => String::from("s3-eu-west-1.amazonaws.com"), - EuWest2 => String::from("s3-eu-west-2.amazonaws.com"), - EuWest3 => String::from("s3-eu-west-3.amazonaws.com"), - SaEast1 => String::from("s3-sa-east-1.amazonaws.com"), - MeSouth1 => String::from("s3-me-south-1.amazonaws.com"), - DoNyc3 => String::from("nyc3.digitaloceanspaces.com"), - DoAms3 => String::from("ams3.digitaloceanspaces.com"), - DoSgp1 => String::from("sgp1.digitaloceanspaces.com"), - DoFra1 => String::from("fra1.digitaloceanspaces.com"), - Yandex => String::from("storage.yandexcloud.net"), - WaUsEast1 => String::from("s3.us-east-1.wasabisys.com"), - WaUsEast2 => String::from("s3.us-east-2.wasabisys.com"), - WaUsWest1 => String::from("s3.us-west-1.wasabisys.com"), - WaEuCentral1 => String::from("s3.eu-central-1.wasabisys.com"), - R2 { ref account_id } => format!("{}.r2.cloudflarestorage.com", account_id), - Custom { ref endpoint, .. } => endpoint.to_string(), - } - } - - pub fn scheme(&self) -> String { - match *self { - Region::Custom { ref endpoint, .. } => match endpoint.find("://") { - Some(pos) => endpoint[..pos].to_string(), - None => "https".to_string(), - }, - _ => "https".to_string(), - } - } - - pub fn host(&self) -> String { - match *self { - Region::Custom { ref endpoint, .. } => match endpoint.find("://") { - Some(pos) => endpoint[pos + 3..].to_string(), - None => endpoint.to_string(), - }, - _ => self.endpoint(), - } - } -} - -#[test] -fn yandex_object_storage() { - let yandex = Region::Custom { - endpoint: "storage.yandexcloud.net".to_string(), - region: "ru-central1".to_string(), - }; - - let yandex_region = "ru-central1".parse::().unwrap(); - - assert_eq!(yandex.endpoint(), yandex_region.endpoint()); - - assert_eq!(yandex.to_string(), yandex_region.to_string()); -} From c662b9c66c2929da185c46084fc5f455030ad75f Mon Sep 17 00:00:00 2001 From: Maxim Leonovich Date: Mon, 24 Feb 2025 16:31:00 -0800 Subject: [PATCH 7/7] Bring back aws-creds; Support using AWS_STS_REGIONAL_ENDPOINTS --- Cargo.toml | 2 +- aws-creds/Cargo.toml | 39 +++ aws-creds/Makefile | 13 + aws-creds/README.md | 35 +++ aws-creds/src/credentials.rs | 504 +++++++++++++++++++++++++++++++++++ aws-creds/src/error.rs | 34 +++ aws-creds/src/lib.rs | 9 + rust-s3/Cargo.toml | 4 +- 8 files changed, 637 insertions(+), 3 deletions(-) create mode 100644 aws-creds/Cargo.toml create mode 100644 aws-creds/Makefile create mode 100644 aws-creds/README.md create mode 100644 aws-creds/src/credentials.rs create mode 100644 aws-creds/src/error.rs create mode 100644 aws-creds/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 49c9d043cd..6a30631165 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,2 +1,2 @@ [workspace] -members = ["rust-s3"] +members = ["rust-s3", "aws-creds"] diff --git a/aws-creds/Cargo.toml b/aws-creds/Cargo.toml new file mode 100644 index 0000000000..2d88d3d4ba --- /dev/null +++ b/aws-creds/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "aws-creds" +version = "0.38.0" +authors = ["Drazen Urch"] +description = "Rust library for working with Amazon IAM credential,s, supports `s3` crate" +repository = "https://github.com/durch/rust-s3" +readme = "README.md" +keywords = ["AWS", "S3", "Wasabi", "Minio", "R2"] +license = "MIT" +documentation = "https://docs.rs/aws-creds/latest/awscreds/" +edition = "2021" + +[lib] +name = "awscreds" +path = "src/lib.rs" + +[dependencies] +thiserror = "1" +home = "0.5" +rust-ini = "0.21" +attohttpc = { version = "0.28", default-features = false, features = [ + "json", +], optional = true } +url = "2" +quick-xml = { version = "0.32", features = ["serialize"] } +serde = { version = "1", features = ["derive"] } +time = { version = "^0.3.6", features = ["serde", "serde-well-known"] } +log = "0.4" + +[features] +default = ["native-tls"] +http-credentials = ["attohttpc"] +native-tls = ["http-credentials", "attohttpc/tls"] +native-tls-vendored = ["http-credentials", "attohttpc/tls-vendored"] +rustls-tls = ["http-credentials", "attohttpc/tls-rustls"] + +[dev-dependencies] +env_logger = "0.11" +serde_json = "1" diff --git a/aws-creds/Makefile b/aws-creds/Makefile new file mode 100644 index 0000000000..b38dcfaa6d --- /dev/null +++ b/aws-creds/Makefile @@ -0,0 +1,13 @@ +ci: fmt-check clippy test + +fmt-check: + cargo fmt --all -- --check + +clippy: + cargo clippy --all-features -- -D warnings + +test: + cargo test --all-features + + + diff --git a/aws-creds/README.md b/aws-creds/README.md new file mode 100644 index 0000000000..f138889aa7 --- /dev/null +++ b/aws-creds/README.md @@ -0,0 +1,35 @@ + + +# Example + +```rust +// AWS access credentials: access key, secret key, and optional token. +# Example +// Loads from the standard AWS credentials file with the given profile name, +// defaults to "default". +use awscreds::Credentials; + +// Load credentials from `[default]` profile +let credentials = Credentials::default(); +// Also loads credentials from `[default]` profile +let credentials = Credentials::new(None, None, None, None); +// Load credentials from `[my-profile]` profile +let credentials = Credentials::new(None, None, None, Some("my-profile".into())); +// Credentials may also be initialized directly or by the following environment variables: +// - `AWS_ACCESS_KEY_ID`, +// - `AWS_SECRET_ACCESS_KEY` +// - `AWS_SESSION_TOKEN` +// The order of preference is arguments, then environment, and finally AWS +// credentials file. + +use s3::credentials::Credentials; +// Load credentials directly +let access_key = String::from("AKIAIOSFODNN7EXAMPLE"); +let secret_key = String::from("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); +let credentials = Credentials::new(Some(access_key), Some(secret_key), None, None); +// Load credentials from the environment +use std::env; +env::set_var("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE"); +env::set_var("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); +let credentials = Credentials::new(None, None, None, None); +``` \ No newline at end of file diff --git a/aws-creds/src/credentials.rs b/aws-creds/src/credentials.rs new file mode 100644 index 0000000000..0b2741a140 --- /dev/null +++ b/aws-creds/src/credentials.rs @@ -0,0 +1,504 @@ +#![allow(dead_code)] + +use crate::error::CredentialsError; +use ini::Ini; +use log::debug; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::env; +use std::ops::Deref; +use std::sync::atomic::AtomicU32; +use std::sync::atomic::Ordering; +use std::time::Duration; +use time::OffsetDateTime; +use url::Url; + +/// AWS access credentials: access key, secret key, and optional token. +/// +/// # Example +/// +/// Loads from the standard AWS credentials file with the given profile name, +/// defaults to "default". +/// +/// ```no_run +/// # // Do not execute this as it would cause unit tests to attempt to access +/// # // real user credentials. +/// use awscreds::Credentials; +/// +/// // Load credentials from `[default]` profile +/// let credentials = Credentials::default(); +/// +/// // Also loads credentials from `[default]` profile +/// let credentials = Credentials::new(None, None, None, None, None); +/// +/// // Load credentials from `[my-profile]` profile +/// let credentials = Credentials::new(None, None, None, None, Some("my-profile".into())); +/// +/// // Use anonymous credentials for public objects +/// let credentials = Credentials::anonymous(); +/// ``` +/// +/// Credentials may also be initialized directly or by the following environment variables: +/// +/// - `AWS_ACCESS_KEY_ID`, +/// - `AWS_SECRET_ACCESS_KEY` +/// - `AWS_SESSION_TOKEN` +/// +/// The order of preference is arguments, then environment, and finally AWS +/// credentials file. +/// +/// ``` +/// use awscreds::Credentials; +/// +/// // Load credentials directly +/// let access_key = "AKIAIOSFODNN7EXAMPLE"; +/// let secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; +/// let credentials = Credentials::new(Some(access_key), Some(secret_key), None, None, None); +/// +/// // Load credentials from the environment +/// use std::env; +/// env::set_var("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE"); +/// env::set_var("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); +/// let credentials = Credentials::new(None, None, None, None, None); +/// ``` +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct Credentials { + /// AWS public access key. + pub access_key: Option, + /// AWS secret key. + pub secret_key: Option, + /// Temporary token issued by AWS service. + pub security_token: Option, + pub session_token: Option, + pub expiration: Option, +} + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize)] +#[repr(transparent)] +pub struct Rfc3339OffsetDateTime(#[serde(with = "time::serde::rfc3339")] pub time::OffsetDateTime); + +impl From for Rfc3339OffsetDateTime { + fn from(v: time::OffsetDateTime) -> Self { + Self(v) + } +} + +impl From for time::OffsetDateTime { + fn from(v: Rfc3339OffsetDateTime) -> Self { + v.0 + } +} + +impl Deref for Rfc3339OffsetDateTime { + type Target = time::OffsetDateTime; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +pub struct AssumeRoleWithWebIdentityResponse { + pub assume_role_with_web_identity_result: AssumeRoleWithWebIdentityResult, + pub response_metadata: ResponseMetadata, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +pub struct AssumeRoleWithWebIdentityResult { + pub subject_from_web_identity_token: String, + pub audience: String, + pub assumed_role_user: AssumedRoleUser, + pub credentials: StsResponseCredentials, + pub provider: String, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +pub struct StsResponseCredentials { + pub session_token: String, + pub secret_access_key: String, + pub expiration: Rfc3339OffsetDateTime, + pub access_key_id: String, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +pub struct AssumedRoleUser { + pub arn: String, + pub assumed_role_id: String, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +pub struct ResponseMetadata { + pub request_id: String, +} + +/// The global request timeout in milliseconds. 0 means no timeout. +/// +/// Defaults to 30 seconds. +static REQUEST_TIMEOUT_MS: AtomicU32 = AtomicU32::new(30_000); + +/// Sets the timeout for all credentials HTTP requests and returns the +/// old timeout value, if any; this timeout applies after a 30-second +/// connection timeout. +/// +/// Short durations are bumped to one millisecond, and durations +/// greater than 4 billion milliseconds (49 days) are rounded up to +/// infinity (no timeout). +/// The global default value is 30 seconds. +#[cfg(feature = "http-credentials")] +pub fn set_request_timeout(timeout: Option) -> Option { + use std::convert::TryInto; + let duration_ms = timeout + .as_ref() + .map(Duration::as_millis) + .unwrap_or(u128::MAX) + .max(1); // A 0 duration means infinity. + + // Store that non-zero u128 value in an AtomicU32 by mapping large + // values to 0: `http_get` maps that to no (infinite) timeout. + let prev = REQUEST_TIMEOUT_MS.swap(duration_ms.try_into().unwrap_or(0), Ordering::Relaxed); + + if prev == 0 { + None + } else { + Some(Duration::from_millis(prev as u64)) + } +} + +#[cfg(feature = "http-credentials")] +fn apply_timeout(builder: attohttpc::RequestBuilder) -> attohttpc::RequestBuilder { + let timeout_ms = REQUEST_TIMEOUT_MS.load(Ordering::Relaxed); + if timeout_ms > 0 { + return builder.timeout(Duration::from_millis(timeout_ms as u64)); + } + builder +} + +/// Sends a GET request to `url` with a request timeout if one was set. +#[cfg(feature = "http-credentials")] +fn http_get(url: &str) -> attohttpc::Result { + let builder = apply_timeout(attohttpc::get(url)); + + builder.send() +} + +impl Credentials { + pub fn refresh(&mut self) -> Result<(), CredentialsError> { + if let Some(expiration) = self.expiration { + if expiration.0 <= OffsetDateTime::now_utc() { + debug!("Refreshing credentials!"); + let refreshed = Credentials::default()?; + *self = refreshed + } + } + Ok(()) + } + + #[cfg(feature = "http-credentials")] + pub fn from_sts_env(session_name: &str) -> Result { + let role_arn = env::var("AWS_ROLE_ARN")?; + let web_identity_token_file = env::var("AWS_WEB_IDENTITY_TOKEN_FILE")?; + let web_identity_token = std::fs::read_to_string(web_identity_token_file)?; + Credentials::from_sts(&role_arn, session_name, &web_identity_token) + } + + #[cfg(feature = "http-credentials")] + pub fn from_sts( + role_arn: &str, + session_name: &str, + web_identity_token: &str, + ) -> Result { + let use_regional_sts = env::var("AWS_STS_REGIONAL_ENDPOINTS") == Ok("regional".to_string()); + let sts_url = if use_regional_sts { + format!("https://sts.{}.amazonaws.com/", env::var("AWS_REGION")?) + } else { + "https://sts.amazonaws.com/".to_string() + }; + let url = Url::parse_with_params( + sts_url.as_str(), + &[ + ("Action", "AssumeRoleWithWebIdentity"), + ("RoleSessionName", session_name), + ("RoleArn", role_arn), + ("WebIdentityToken", web_identity_token), + ("Version", "2011-06-15"), + ], + )?; + let response = http_get(url.as_str())?; + let serde_response = + quick_xml::de::from_str::(&response.text()?)?; + // assert!(quick_xml::de::from_str::(&response.text()?).unwrap()); + + Ok(Credentials { + access_key: Some( + serde_response + .assume_role_with_web_identity_result + .credentials + .access_key_id, + ), + secret_key: Some( + serde_response + .assume_role_with_web_identity_result + .credentials + .secret_access_key, + ), + security_token: None, + session_token: Some( + serde_response + .assume_role_with_web_identity_result + .credentials + .session_token, + ), + expiration: Some( + serde_response + .assume_role_with_web_identity_result + .credentials + .expiration, + ), + }) + } + + #[allow(clippy::should_implement_trait)] + pub fn default() -> Result { + Credentials::new(None, None, None, None, None) + } + + pub fn anonymous() -> Result { + Ok(Credentials { + access_key: None, + secret_key: None, + security_token: None, + session_token: None, + expiration: None, + }) + } + + /// Initialize Credentials directly with key ID, secret key, and optional + /// token. + pub fn new( + access_key: Option<&str>, + secret_key: Option<&str>, + security_token: Option<&str>, + session_token: Option<&str>, + profile: Option<&str>, + ) -> Result { + if access_key.is_some() { + return Ok(Credentials { + access_key: access_key.map(|s| s.to_string()), + secret_key: secret_key.map(|s| s.to_string()), + security_token: security_token.map(|s| s.to_string()), + session_token: session_token.map(|s| s.to_string()), + expiration: None, + }); + } + + let credentials = Credentials::from_env().or_else(|_| Credentials::from_profile(profile)); + + #[cfg(feature = "http-credentials")] + let credentials = credentials + .or_else(|_| Credentials::from_sts_env("aws-creds")) + .or_else(|_| Credentials::from_instance_metadata_v2(false)) + .or_else(|_| Credentials::from_instance_metadata(false)); + + credentials.map_err(|_| CredentialsError::NoCredentials) + } + + pub fn from_env_specific( + access_key_var: Option<&str>, + secret_key_var: Option<&str>, + security_token_var: Option<&str>, + session_token_var: Option<&str>, + ) -> Result { + let access_key = from_env_with_default(access_key_var, "AWS_ACCESS_KEY_ID")?; + let secret_key = from_env_with_default(secret_key_var, "AWS_SECRET_ACCESS_KEY")?; + + let security_token = from_env_with_default(security_token_var, "AWS_SECURITY_TOKEN").ok(); + let session_token = from_env_with_default(session_token_var, "AWS_SESSION_TOKEN").ok(); + Ok(Credentials { + access_key: Some(access_key), + secret_key: Some(secret_key), + security_token, + session_token, + expiration: None, + }) + } + + pub fn from_env() -> Result { + Credentials::from_env_specific(None, None, None, None) + } + + #[cfg(feature = "http-credentials")] + pub fn from_instance_metadata(not_ec2: bool) -> Result { + let resp: CredentialsFromInstanceMetadata = + match env::var("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") { + Ok(credentials_path) => { + // We are on ECS + apply_timeout(attohttpc::get(format!( + "http://169.254.170.2{}", + credentials_path + ))) + .send()? + .json()? + } + Err(_) => { + if !not_ec2 && !is_ec2() { + return Err(CredentialsError::NotEc2); + } + + let role = apply_timeout(attohttpc::get( + "http://169.254.169.254/latest/meta-data/iam/security-credentials", + )) + .send()? + .text()?; + + apply_timeout(attohttpc::get(format!( + "http://169.254.169.254/latest/meta-data/iam/security-credentials/{}", + role + ))) + .send()? + .json()? + } + }; + + Ok(Credentials { + access_key: Some(resp.access_key_id), + secret_key: Some(resp.secret_access_key), + security_token: Some(resp.token), + expiration: Some(resp.expiration), + session_token: None, + }) + } + + #[cfg(feature = "http-credentials")] + pub fn from_instance_metadata_v2(not_ec2: bool) -> Result { + if !not_ec2 && !is_ec2() { + return Err(CredentialsError::NotEc2); + } + + let token = apply_timeout(attohttpc::put("http://169.254.169.254/latest/api/token")) + .header("X-aws-ec2-metadata-token-ttl-seconds", "21600") + .send()?; + if !token.is_success() { + return Err(CredentialsError::UnexpectedStatusCode( + token.status().as_u16(), + )); + } + let token = token.text()?; + + let role = apply_timeout(attohttpc::get( + "http://169.254.169.254/latest/meta-data/iam/security-credentials", + )) + .header("X-aws-ec2-metadata-token", &token) + .send()? + .text()?; + + let resp: CredentialsFromInstanceMetadata = apply_timeout(attohttpc::get(format!( + "http://169.254.169.254/latest/meta-data/iam/security-credentials/{}", + role + ))) + .header("X-aws-ec2-metadata-token", &token) + .send()? + .json()?; + + Ok(Credentials { + access_key: Some(resp.access_key_id), + secret_key: Some(resp.secret_access_key), + security_token: Some(resp.token), + expiration: Some(resp.expiration), + session_token: None, + }) + } + + pub fn from_profile(section: Option<&str>) -> Result { + let home_dir = home::home_dir().ok_or(CredentialsError::HomeDir)?; + let profile = format!("{}/.aws/credentials", home_dir.display()); + let conf = Ini::load_from_file(profile)?; + let section = section.unwrap_or("default"); + let data = conf + .section(Some(section)) + .ok_or(CredentialsError::ConfigNotFound)?; + let access_key = data + .get("aws_access_key_id") + .map(|s| s.to_string()) + .ok_or(CredentialsError::ConfigMissingAccessKeyId)?; + let secret_key = data + .get("aws_secret_access_key") + .map(|s| s.to_string()) + .ok_or(CredentialsError::ConfigMissingSecretKey)?; + let credentials = Credentials { + access_key: Some(access_key), + secret_key: Some(secret_key), + security_token: data.get("aws_security_token").map(|s| s.to_string()), + session_token: data.get("aws_session_token").map(|s| s.to_string()), + expiration: None, + }; + Ok(credentials) + } +} + +fn from_env_with_default(var: Option<&str>, default: &str) -> Result { + let val = var.unwrap_or(default); + env::var(val) + .or_else(|_e| env::var(val)) + .map_err(|_| CredentialsError::MissingEnvVar(val.to_string(), default.to_string())) +} + +fn is_ec2() -> bool { + if let Ok(uuid) = std::fs::read_to_string("/sys/hypervisor/uuid") { + if uuid.starts_with("ec2") { + return true; + } + } + if let Ok(vendor) = std::fs::read_to_string("/sys/class/dmi/id/board_vendor") { + if vendor.starts_with("Amazon EC2") { + return true; + } + } + false +} + +#[derive(Deserialize)] +#[serde(rename_all = "PascalCase")] +struct CredentialsFromInstanceMetadata { + access_key_id: String, + secret_access_key: String, + token: String, + expiration: Rfc3339OffsetDateTime, // TODO fix #163 +} + +#[cfg(test)] +#[test] +fn test_instance_metadata_creds_deserialization() { + // As documented here: + // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials + serde_json::from_str::( + r#" + { + "Code" : "Success", + "LastUpdated" : "2012-04-26T16:39:16Z", + "Type" : "AWS-HMAC", + "AccessKeyId" : "ASIAIOSFODNN7EXAMPLE", + "SecretAccessKey" : "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "Token" : "token", + "Expiration" : "2017-05-17T15:09:54Z" + } + "#, + ) + .unwrap(); +} + +#[cfg(test)] +#[ignore] +#[test] +fn test_credentials_refresh() { + let mut c = Credentials::default().expect("Could not generate credentials"); + let e = Rfc3339OffsetDateTime(OffsetDateTime::now_utc()); + c.expiration = Some(e); + std::thread::sleep(std::time::Duration::from_secs(3)); + c.refresh().expect("Could not refresh"); + assert!(c.expiration.is_none()) +} diff --git a/aws-creds/src/error.rs b/aws-creds/src/error.rs new file mode 100644 index 0000000000..a90a240b74 --- /dev/null +++ b/aws-creds/src/error.rs @@ -0,0 +1,34 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum CredentialsError { + #[error("Not an AWS instance")] + NotEc2, + #[error("Config not found")] + ConfigNotFound, + #[error("Missing aws_access_key_id section in config")] + ConfigMissingAccessKeyId, + #[error("Missing aws_access_key_id section in config")] + ConfigMissingSecretKey, + #[error("Neither {0}, nor {1} exists in the environment")] + MissingEnvVar(String, String), + #[cfg(feature = "http-credentials")] + #[error("attohttpc: {0}")] + Atto(#[from] attohttpc::Error), + #[error("ini: {0}")] + Ini(#[from] ini::Error), + #[error("serde_xml: {0}")] + SerdeXml(#[from] quick_xml::de::DeError), + #[error("url parse: {0}")] + UrlParse(#[from] url::ParseError), + #[error("io: {0}")] + Io(#[from] std::io::Error), + #[error("env var: {0}")] + Env(#[from] std::env::VarError), + #[error("Invalid home dir")] + HomeDir, + #[error("Could not get valid credentials from STS, ENV, Profile or Instance metadata")] + NoCredentials, + #[error("unexpected status code: {0}")] + UnexpectedStatusCode(u16), +} diff --git a/aws-creds/src/lib.rs b/aws-creds/src/lib.rs new file mode 100644 index 0000000000..4fe4ffa184 --- /dev/null +++ b/aws-creds/src/lib.rs @@ -0,0 +1,9 @@ +#![allow(unused_imports)] +#![forbid(unsafe_code)] + +mod credentials; +pub use credentials::*; +pub mod error; + +// Reexport for e.g. users who need to build Credentials +pub use time; diff --git a/rust-s3/Cargo.toml b/rust-s3/Cargo.toml index f7e44c78da..edbf33da53 100644 --- a/rust-s3/Cargo.toml +++ b/rust-s3/Cargo.toml @@ -18,7 +18,7 @@ path = "src/lib.rs" async-std = { version = "1", optional = true } async-trait = "0.1" attohttpc = { version = "0.19", optional = true, default-features = false } -aws-creds = { version = "0.36.0" } +aws-creds = { path = "../aws-creds" } aws-region = { version = "0.25.4" } base64 = "0.13" cfg-if = "1" @@ -72,5 +72,5 @@ tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "fs"] } async-std = { version = "1", features = ["attributes"] } uuid = { version = "1", features = ["v4"] } env_logger = "0.9" -aws-creds = { version = "0.36.0", features = ["http-credentials"] } +aws-creds = { path = "../aws-creds", features = ["http-credentials"] } anyhow = "1"