diff --git a/Cargo.lock b/Cargo.lock index 4ef2c70..cc750cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -169,12 +169,53 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "attohttpc" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a13149d0cf3f7f9b9261fad4ec63b2efbf9a80665f52def86282d26255e6331" +dependencies = [ + "http 1.1.0", + "log", + "rustls 0.22.4", + "serde", + "serde_json", + "url", + "webpki-roots", +] + [[package]] name = "autocfg" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-creds" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f84143206b9c72b3c5cb65415de60c7539c79cd1559290fddec657939131be0" +dependencies = [ + "attohttpc", + "home", + "log", + "quick-xml", + "rust-ini", + "serde", + "thiserror", + "time", + "url", +] + +[[package]] +name = "aws-region" +version = "0.25.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9aed3f9c7eac9be28662fdb3b0f4d1951e812f7c64fed4f0327ba702f459b3b" +dependencies = [ + "thiserror", +] + [[package]] name = "backoff" version = "0.4.0" @@ -229,6 +270,7 @@ dependencies = [ "k8s-openapi", "kube", "pretty_assertions", + "rust-s3", "serde", "serde_yml", "simplelog", @@ -266,9 +308,9 @@ dependencies = [ "futures-core", "futures-util", "hex", - "http", + "http 1.1.0", "http-body-util", - "hyper", + "hyper 1.5.0", "hyper-named-pipe", "hyper-util", "hyperlocal-next", @@ -449,6 +491,26 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom", + "once_cell", + "tiny-keccak", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -499,6 +561,12 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-common" version = "0.1.6" @@ -585,6 +653,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -598,6 +667,15 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + [[package]] name = "dyn-clone" version = "1.0.17" @@ -714,6 +792,7 @@ checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", + "futures-executor", "futures-io", "futures-sink", "futures-task", @@ -736,6 +815,17 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.31" @@ -880,6 +970,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "home" version = "0.5.9" @@ -889,6 +988,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.1.0" @@ -900,6 +1010,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -907,7 +1028,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.1.0", ] [[package]] @@ -918,8 +1039,8 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -929,6 +1050,12 @@ version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humansize" version = "2.1.3" @@ -938,6 +1065,29 @@ dependencies = [ "libm", ] +[[package]] +name = "hyper" +version = "0.14.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.5.0" @@ -947,8 +1097,8 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.1", "httparse", "itoa", "pin-project-lite", @@ -964,7 +1114,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" dependencies = [ "hex", - "hyper", + "hyper 1.5.0", "hyper-util", "pin-project-lite", "tokio", @@ -972,6 +1122,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.31", + "rustls 0.21.12", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.27.3" @@ -979,15 +1143,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", - "http", - "hyper", + "http 1.1.0", + "hyper 1.5.0", "hyper-util", "log", - "rustls", - "rustls-native-certs", + "rustls 0.23.16", + "rustls-native-certs 0.8.0", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.0", "tower-service", ] @@ -997,7 +1161,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper", + "hyper 1.5.0", "hyper-util", "pin-project-lite", "tokio", @@ -1013,9 +1177,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.1.0", + "http-body 1.0.1", + "hyper 1.5.0", "pin-project-lite", "socket2", "tokio", @@ -1031,7 +1195,7 @@ checksum = "acf569d43fa9848e510358c07b80f4adf34084ddc28c6a4a651ee8474c070dcc" dependencies = [ "hex", "http-body-util", - "hyper", + "hyper 1.5.0", "hyper-util", "pin-project-lite", "tokio", @@ -1353,19 +1517,19 @@ dependencies = [ "either", "futures", "home", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.1", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.5.0", + "hyper-rustls 0.27.3", "hyper-timeout", "hyper-util", "jsonpath-rust", "k8s-openapi", "kube-core", "pem", - "rustls", - "rustls-pemfile", + "rustls 0.23.16", + "rustls-pemfile 2.2.0", "secrecy", "serde", "serde_json", @@ -1386,7 +1550,7 @@ checksum = "2797d3044a238825432129cd9537e12c2a6dacbbb5352381af5ea55e1505ed4f" dependencies = [ "chrono", "form_urlencoded", - "http", + "http 1.1.0", "json-patch", "k8s-openapi", "schemars", @@ -1503,6 +1667,23 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "maybe-async" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.7.4" @@ -1590,6 +1771,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.5", +] + [[package]] name = "paris" version = "1.5.15" @@ -1835,6 +2026,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "quick-xml" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d3a6e5838b60e0e8fa7a43f22ade549a37d61f8bdbe636d0d7816191de969c2" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quote" version = "1.0.37" @@ -1927,6 +2128,54 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rust-ini" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e310ef0e1b6eeb79169a1171daf9abcb87a2e17c03bee2c4bb100b55c75409f" +dependencies = [ + "cfg-if", + "ordered-multimap", + "trim-in-place", +] + +[[package]] +name = "rust-s3" +version = "0.35.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3df3f353b1f4209dcf437d777cda90279c397ab15a0cd6fd06bd32c88591533" +dependencies = [ + "async-trait", + "aws-creds", + "aws-region", + "base64 0.22.1", + "bytes", + "cfg-if", + "futures", + "hex", + "hmac", + "http 0.2.12", + "hyper 0.14.31", + "hyper-rustls 0.24.2", + "log", + "maybe-async", + "md5", + "percent-encoding", + "quick-xml", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", + "serde", + "serde_derive", + "serde_json", + "sha2", + "thiserror", + "time", + "tokio", + "tokio-rustls 0.24.1", + "tokio-stream", + "url", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -1946,6 +2195,32 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + +[[package]] +name = "rustls" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +dependencies = [ + "log", + "ring", + "rustls-pki-types", + "rustls-webpki 0.102.8", + "subtle", + "zeroize", +] + [[package]] name = "rustls" version = "0.23.16" @@ -1956,11 +2231,23 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.102.8", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.4", + "schannel", + "security-framework", +] + [[package]] name = "rustls-native-certs" version = "0.8.0" @@ -1968,12 +2255,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 2.2.0", "rustls-pki-types", "schannel", "security-framework", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -1989,6 +2285,16 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.102.8" @@ -2054,6 +2360,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "secrecy" version = "0.8.0" @@ -2452,6 +2768,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.7.6" @@ -2490,17 +2815,38 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls", + "rustls 0.23.16", "rustls-pki-types", "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.12" @@ -2541,8 +2887,8 @@ dependencies = [ "base64 0.21.7", "bitflags", "bytes", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -2595,6 +2941,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "trim-in-place" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343e926fc669bc8cde4fa3129ab681c63671bae288b1f1081ceee6d9d37904fc" + [[package]] name = "try-lock" version = "0.2.5" @@ -2823,6 +3175,15 @@ version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +[[package]] +name = "webpki-roots" +version = "0.26.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 47db42d..d9bb0a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,11 @@ bollard = "0.16.1" tar = "0.4.42" tempfile = "3.13.0" figment = { version = "0.10.19", features = ["env", "yaml", "test"] } +rust-s3 = { version = "0.35.1", default-features = false, features = [ + "fail-on-err", + "tokio-rustls-tls", +] } + [dev-dependencies] pretty_assertions = "1.4.1" diff --git a/src/access_handlers/s3.rs b/src/access_handlers/s3.rs index 5976ce9..f1ed323 100644 --- a/src/access_handlers/s3.rs +++ b/src/access_handlers/s3.rs @@ -1,9 +1,96 @@ -use anyhow::{Error, Result}; +use anyhow::{anyhow, bail, Context, Error, Result}; +use s3; +use simplelog::*; +use tokio; -use crate::configparser::{get_config, get_profile_config}; +use crate::configparser::{ + config::{ProfileConfig, S3Config}, + get_config, get_profile_config, +}; /// s3 bucket access checks -pub fn check(profile_name: &str) -> Result<()> { +#[tokio::main(flavor = "current_thread")] // make this a sync function +pub async fn check(profile_name: &str) -> Result<()> { let profile = get_profile_config(profile_name)?; + + let bucket = bucket_client(&profile.s3)?; + + if !bucket.exists().await? { + bail!("bucket {} does not exist!", profile.s3.bucket_name); + } + + // try uploading file to bucket + debug!("uploading test file to bucket"); + let test_file = ("/beavercds-test-file", "access test file!"); + bucket + .put_object_with_content_type(test_file.0, test_file.1.as_bytes(), "text/plain") + .await + .with_context(|| { + format!( + "could not upload to asset bucket {:?}", + profile.s3.bucket_name + ) + })?; + + // download it to check + debug!("downloading test file"); + let from_bucket = bucket.get_object(test_file.0).await?; + if from_bucket.bytes() != test_file.1 { + bail!("uploaded test file contents do not match, somehow!?"); + } + + // download as anonymous to check public access + debug!("downloading test file as public user"); + let public_bucket = bucket_client_anonymous(&profile.s3)?; + let from_public = public_bucket + .get_object(test_file.0) + .await + .with_context(|| { + anyhow!( + "public download from asset bucket {:?} failed", + profile.s3.bucket_name + ) + })?; + if from_public.bytes() != test_file.1 { + bail!("contents of public bucket do not match uploaded file"); + } + + // clean up test file after checks + bucket.delete_object(test_file.0).await?; + Ok(()) } + +/// create bucket client for passed profile config +pub fn bucket_client(config: &S3Config) -> Result> { + trace!("creating bucket client"); + // TODO: once_cell this so it reuses the same bucket? + let region = s3::Region::Custom { + region: config.region.clone(), + endpoint: config.endpoint.clone(), + }; + let creds = s3::creds::Credentials::new( + Some(&config.access_key), + Some(&config.secret_key), + None, + None, + None, + )?; + let bucket = s3::Bucket::new(&config.bucket_name, region, creds)?.with_path_style(); + + Ok(bucket) +} + +/// create public/anonymous bucket client for passed profile config +pub fn bucket_client_anonymous(config: &S3Config) -> Result> { + trace!("creating anon bucket client"); + // TODO: once_cell this so it reuses the same bucket? + let region = s3::Region::Custom { + region: config.region.clone(), + endpoint: config.endpoint.clone(), + }; + let creds = s3::creds::Credentials::anonymous()?; + let bucket = s3::Bucket::new(&config.bucket_name, region, creds)?.with_path_style(); + + Ok(bucket) +} diff --git a/src/cli.rs b/src/cli.rs index a232e13..7f6a04a 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -68,5 +68,8 @@ pub enum Commands { /// Check container registry access and permissions #[arg(short, long)] registry: bool, + + #[arg(short, long, help = "Check S3 asset bucket access and permissions")] + bucket: bool, }, } diff --git a/src/commands/check_access.rs b/src/commands/check_access.rs index 2f2f64c..dee5226 100644 --- a/src/commands/check_access.rs +++ b/src/commands/check_access.rs @@ -7,35 +7,50 @@ use std::process::exit; use crate::access_handlers as access; use crate::configparser::{get_config, get_profile_config}; -pub fn run(profile: &str, kubernetes: &bool, frontend: &bool, registry: &bool) { +pub fn run(profile: &str, kubernetes: &bool, frontend: &bool, registry: &bool, bucket: &bool) { // if user did not give a specific check, check all of them - let check_all = !kubernetes && !frontend && !registry; + let check_all = !kubernetes && !frontend && !registry && !bucket; let config = get_config().unwrap(); - let to_check: Vec<_> = match profile { + let profiles_to_check: Vec<_> = match profile { "all" => config.profiles.keys().cloned().collect(), p => vec![String::from(p)], }; - let results: Result<(), Vec<_>> = to_check.into_iter().try_for_each(|p| { - check_profile( - &p, - *kubernetes || check_all, - *frontend || check_all, - *registry || check_all, - ) - }); + let results: Vec<_> = profiles_to_check + .iter() + .map(|profile_name| { + ( + profile_name, // associate profile name to results + check_profile( + profile_name, + *kubernetes || check_all, + *frontend || check_all, + *registry || check_all, + *bucket || check_all, + ), + ) + }) + .collect(); + + debug!("access results: {results:?}"); // die if there were any errors - match results { - Ok(_) => info!(" all good!"), - Err(errs) => { - error!("Error checking profile {profile}:"); - errs.iter().for_each(|e| error!("{e:?}\n")); - exit(1) + let mut should_exit = false; + for (profile, result) in results.iter() { + match result { + Ok(_) => info!(" all good!"), + Err(errs) => { + error!("{} errors checking profile {profile}:", errs.len()); + errs.iter().for_each(|e| error!("{e:?}\n")); + should_exit = true + } } } + if should_exit { + exit(1); + } } /// checks a single profile (`profile`) for the given accesses @@ -44,22 +59,36 @@ fn check_profile( kubernetes: bool, frontend: bool, registry: bool, + bucket: bool, ) -> Result<(), Vec> { info!("checking profile {name}..."); - let mut results = vec![]; + let mut errs = vec![]; if kubernetes { - results.push(access::kube::check(name).context("could not access kubernetes cluster")); + match access::kube::check(name).context("could not access kubernetes cluster") { + Err(e) => errs.push(e), + Ok(_) => info!(" kubernetes ok!"), + }; } if frontend { - results.push(access::frontend::check(name).context("could not access frontend")); + match access::frontend::check(name).context("could not access frontend") { + Err(e) => errs.push(e), + Ok(_) => info!(" frontend ok!"), + }; } if registry { - results.push(access::docker::check(name).context("could not access container registry")); + match access::docker::check(name).context("could not access container registry") { + Err(e) => errs.push(e), + Ok(_) => info!(" registry ok!"), + }; + } + if bucket { + match access::s3::check(name).context("could not access asset bucket") { + Err(e) => errs.push(e), + Ok(_) => info!(" bucket ok!"), + }; } - - let (ok, errs): (Vec<_>, Vec<_>) = results.into_iter().partition_result(); if !errs.is_empty() { Err(errs) diff --git a/src/configparser/config.rs b/src/configparser/config.rs index 314596b..a47a514 100644 --- a/src/configparser/config.rs +++ b/src/configparser/config.rs @@ -13,14 +13,15 @@ pub fn parse() -> Result { debug!("trying to parse rcds.yaml"); let env_overrides = Env::prefixed("BEAVERCDS_").split("_").map(|var| { - // Using "_" as the split character works for almost all of our keys. - // but some of the profile settings keys have underscores as part of the - // key. This handles those few keys by undoing the s/_/./ that the - // Figment split() did. + // Using "_" as the split character works for almost all of our keys, + // but some profile settings have underscores. This handles those few + // keys by undoing the s/_/./ that the figment::split() did. var.to_string() .to_lowercase() .replace("frontend.", "frontend_") .replace("challenges.", "challenges_") + .replace("s3.access.", "s3.access_") + .replace("s3.secret.", "s3.secret_") .into() }); trace!( @@ -101,6 +102,7 @@ struct ProfileConfig { challenges_domain: String, kubeconfig: Option, kubecontext: String, + s3: S3Config, } #[derive(Debug, PartialEq, Serialize, Deserialize)] @@ -110,3 +112,13 @@ struct ChallengePoints { min: i64, max: i64, } + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[fully_pub] +struct S3Config { + bucket_name: String, + endpoint: String, + region: String, + access_key: String, + secret_key: String, +} diff --git a/src/main.rs b/src/main.rs index ab1910f..a84c4ad 100644 --- a/src/main.rs +++ b/src/main.rs @@ -30,9 +30,10 @@ fn main() { kubernetes, frontend, registry, + bucket, } => { commands::validate::run(); - commands::check_access::run(profile, kubernetes, frontend, registry) + commands::check_access::run(profile, kubernetes, frontend, registry, bucket) } #[allow(unused_variables)] diff --git a/src/tests/parsing/config.rs b/src/tests/parsing/config.rs index 6260702..1732a96 100644 --- a/src/tests/parsing/config.rs +++ b/src/tests/parsing/config.rs @@ -104,13 +104,13 @@ fn all_yaml() { challenges_domain: "chals.frontend.example".to_string(), kubeconfig: None, kubecontext: "testcluster".to_string(), - // s3: S3Config { - // bucket_name: "asset_testing".to_string(), - // endpoint: "s3.example".to_string(), - // region: "us-fake-1".to_string(), - // access_key: "accesskey".to_string(), - // secret_key: "secretkey".to_string(), - // } + s3: S3Config { + bucket_name: "asset_testing".to_string(), + endpoint: "s3.example".to_string(), + region: "us-fake-1".to_string(), + access_key: "accesskey".to_string(), + secret_key: "secretkey".to_string(), + }, }, )]), }; @@ -193,8 +193,8 @@ fn yaml_with_env_overrides() { let profile = config.profiles.get("testing").unwrap(); assert_eq!(profile.frontend_token, "envtoken"); - // assert_eq!(profile.s3.access_key, "envkey"); - // assert_eq!(profile.s3.secret_key, "envsecret"); + assert_eq!(profile.s3.access_key, "envkey"); + assert_eq!(profile.s3.secret_key, "envsecret"); Ok(()) }); @@ -263,8 +263,8 @@ fn partial_yaml_with_env() { let profile = config.profiles.get("testing").unwrap(); assert_eq!(profile.frontend_token, "envtoken"); - // assert_eq!(profile.s3.access_key, "envkey"); - // assert_eq!(profile.s3.secret_key, "envsecret"); + assert_eq!(profile.s3.access_key, "envkey"); + assert_eq!(profile.s3.secret_key, "envsecret"); Ok(()) }); diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..a76af28 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,30 @@ +# Running Tests + +Since this needs to interact with a container registry, S3 storage, and K8S, +there is some extra setup needed before running `cargo test` or running against +the test chals repo. + +## `setup.sh` + +Main setup script. Run or source this file to set up infrastructure. +Recommended to source this file to set the config override environment +environment variables for test tokens and addresses. + +Spins up a local Minikube K8S cluster and other test environment components via +Docker Compose. + +```sh +source tests/setup.sh up +source tests/setup.sh down +``` + +## `services.compose.yaml` + +Non-K8S resources required to run tests against: + - Container registry + - S3 buckets (via Minio) + +## `repo/` + +Example challenges repo to test against. Contains a variety of challenge types: +static file only (garf), HTTP web (bar), and TCP pwn (notsh). diff --git a/tests/docker-compose.testregistry.yaml b/tests/docker-compose.testregistry.yaml deleted file mode 100644 index 7e99d7d..0000000 --- a/tests/docker-compose.testregistry.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# compose to create registry container and ui to see if images pushed ok -services: - registry-server: - image: registry - ports: - - 5000:5000 - container_name: registry-server - - registry-ui: - image: joxit/docker-registry-ui - ports: - - 8000:80 - user: root - environment: - - SINGLE_REGISTRY=true - - NGINX_PROXY_PASS_URL=http://registry-server:5000 - container_name: registry-ui diff --git a/tests/repo/rcds.yaml b/tests/repo/rcds.yaml index 15cfb11..a229fd7 100644 --- a/tests/repo/rcds.yaml +++ b/tests/repo/rcds.yaml @@ -34,8 +34,8 @@ profiles: challenges_domain: chals.frontend.example kubecontext: testcluster s3: - # local minio + bucket_name: testbucket endpoint: localhost:9000 region: x - accessKey: accesskey - secretAccessKey: secretkey + access_key: somekey + secret_key: somesecret diff --git a/tests/services.compose.yaml b/tests/services.compose.yaml new file mode 100644 index 0000000..161a607 --- /dev/null +++ b/tests/services.compose.yaml @@ -0,0 +1,40 @@ +# compose to create registry container and ui to see if images pushed ok +services: + registry-server: + image: registry + ports: + - 5000:5000 + + registry-ui: + image: joxit/docker-registry-ui + ports: + - 8000:80 + user: root + environment: + - SINGLE_REGISTRY=true + - NGINX_PROXY_PASS_URL=http://registry-server:5000 + + minio: + image: quay.io/minio/minio + command: server /data --console-address ':9001' + ports: + - 9000:9000 + - 9001:9001 + environment: + MINIO_ROOT_USER: testuser + MINIO_ROOT_PASSWORD: notsecure + + # minio image does not set up default buckets or permissions from envvars, so + # use sidecar image to set up test bucket and allow public downloads + createbuckets: + image: quay.io/minio/minio + depends_on: + - minio + entrypoint: > + /bin/sh -xec " + while ! curl --silent http://minio:9001 > /dev/null ; do sleep 1 ; done; + /usr/bin/mc alias set myminio http://minio:9000 testuser notsecure; + /usr/bin/mc mb myminio/testbucket; + /usr/bin/mc anonymous set download myminio/testbucket; + exit 0; + " diff --git a/tests/setup.sh b/tests/setup.sh new file mode 100755 index 0000000..456af93 --- /dev/null +++ b/tests/setup.sh @@ -0,0 +1,51 @@ +# #!/bin/bash + +exit_cmd (){ + [ "$BASH_SOURCE" = "$0" ] && echo exit || echo return +} + +dockpod (){ + command -v podman || echo docker +} + +export MINIKUBE_PROFILE=beavercds +COMPOSE_FILE="$(git rev-parse --show-toplevel)/tests/services.compose.yaml" + +start_stuff (){ + # start cluster + minikube start --container-runtime=cri-o + + # start registry + $(dockpod) compose -f $COMPOSE_FILE up -d + + # export variables if sourced or echo them if run + export BEAVERCDS_REGISTRY_DOMAIN="host.minikube.internal:5000/testing" + export BEAVERCDS_PROFILES_TESTING_KUBECONTEXT="$MINIKUBE_PROFILE" + export BEAVERCDS_PROFILES_TESTING_S3_ENDPOINT="http://localhost:9000" + export BEAVERCDS_PROFILES_TESTING_S3_REGION="" + export BEAVERCDS_PROFILES_TESTING_S3_ACCESS_KEY=$(cat $COMPOSE_FILE | yq -r .services.minio.environment.MINIO_ROOT_USER) + export BEAVERCDS_PROFILES_TESTING_S3_SECRET_KEY=$(cat $COMPOSE_FILE | yq -r .services.minio.environment.MINIO_ROOT_PASSWORD) + + if [ $(exit_cmd) = "exit" ] ; then + echo + echo "export these vars manually, or source this script to export" + env | grep BEAVERCDS | sort + fi +} + +stop_stuff (){ + minikube delete + $(dockpod) compose -f $(git rev-parse --show-toplevel)/tests/services.compose.yaml down --volumes +} + + +case "${1:-}" in + start | up) start_stuff ;; + stop | down | rm) stop_stuff ;; + *) + echo "usage:" 1>&2 + echo " $0 up" 1>&2 + echo " $0 down" 1>&2 + $(exit_cmd) 2 + ;; +esac