diff --git a/Justfile b/Justfile index 93e599c8..818c18e2 100644 --- a/Justfile +++ b/Justfile @@ -224,6 +224,10 @@ _wick-db-tests: {{wick}} test ./examples/db/tests/postgres-numeric-tests.wick {{wick}} test ./examples/db/tests/postgres-null-tests.wick {{wick}} test ./examples/db/tests/postgres-date-tests.wick + {{wick}} test ./examples/db/postgres-component.wick + {{wick}} test ./examples/db/azuresql-component.wick + {{wick}} test ./examples/db/sqlite-component.wick + {{wick}} test ./examples/db/sqlite-inmemory-component.wick {{wick}} test ./tests/cli-tests/tests/cmd/db/azuresql-tx-test.wick # Run `wick` tests for http components diff --git a/crates/components/wick-sql/src/component.rs b/crates/components/wick-sql/src/component.rs index e2b85cb7..3f8c471f 100644 --- a/crates/components/wick-sql/src/component.rs +++ b/crates/components/wick-sql/src/component.rs @@ -46,15 +46,11 @@ impl Client { normalize_operations(config.operations_mut(), DbKind::Postgres); Arc::new(crate::sqlx::SqlXComponent::new(config.clone(), resolver).await?) } - "sqlite" => { + "file" | "sqlite" => { normalize_operations(config.operations_mut(), DbKind::Sqlite); - Arc::new(crate::sqlx::SqlXComponent::new(config.clone(), resolver).await?) } - - _ => { - return Err(Error::InvalidScheme(url.scheme().to_owned())); - } + _ => return Err(Error::InvalidScheme(url.scheme().to_owned())), }; Ok(Self { inner: client }) @@ -440,7 +436,7 @@ fn normalize_inline_ids(orig_query: &str, mut orig_args: Vec) -> (Cow Result<()> { let pg = init_sqlite_component().await?; let input = packet_stream!(("input", 1_i32)); - let inv = Invocation::test("postgres", "wick://__local__/test", input, None)?; + let inv = Invocation::test("sqlite", "wick://__local__/test", input, None)?; let response = pg.handle(inv, Default::default(), panic_callback()).await.unwrap(); let packets: Vec<_> = response.collect().await; diff --git a/crates/components/wick-sql/src/error.rs b/crates/components/wick-sql/src/error.rs index 04e788ff..27f6b4c5 100644 --- a/crates/components/wick-sql/src/error.rs +++ b/crates/components/wick-sql/src/error.rs @@ -17,6 +17,11 @@ pub enum Error { #[error("Unknown database scheme '{0}'")] InvalidScheme(String), + #[error( + "To use in-memory SQLite databases, use the URL 'sqlite://memory'; to use a SQLite DB file, use a 'file://' URL" + )] + SqliteScheme, + #[error("Failed to prepare arguments: {0}")] Prepare(String), diff --git a/crates/components/wick-sql/src/sqlx/component.rs b/crates/components/wick-sql/src/sqlx/component.rs index b8954862..4c93af6e 100644 --- a/crates/components/wick-sql/src/sqlx/component.rs +++ b/crates/components/wick-sql/src/sqlx/component.rs @@ -183,8 +183,26 @@ fn validate(config: &SqlComponentConfig, _resolver: &Resolver) -> Result<(), Err async fn init_client(config: &SqlComponentConfig, addr: &Url) -> Result { let pool = match addr.scheme() { - "sqlite" => CtxPool::SqlLite(sqlite::connect(config, addr).await?), + "file" => CtxPool::SqlLite( + sqlite::connect( + config, + Some( + addr + .to_file_path() + .map_err(|_e| Error::SqliteConnect(format!("could not convert url {} to filepath", addr)))? + .to_str() + .unwrap(), + ), + ) + .await?, + ), "postgres" => CtxPool::Postgres(postgres::connect(config, addr).await?), + "sqlite" => { + if addr.host() != Some(url::Host::Domain("memory")) { + return Err(Error::SqliteScheme); + } + CtxPool::SqlLite(sqlite::connect(config, None).await?) + } "mysql" => unimplemented!("MySql is not supported yet"), "mssql" => unreachable!(), s => return Err(Error::InvalidScheme(s.to_owned())), diff --git a/crates/components/wick-sql/src/sqlx/sqlite.rs b/crates/components/wick-sql/src/sqlx/sqlite.rs index 541c6726..2e354ef0 100644 --- a/crates/components/wick-sql/src/sqlx/sqlite.rs +++ b/crates/components/wick-sql/src/sqlx/sqlite.rs @@ -1,19 +1,19 @@ mod serialize; use sqlx::sqlite::SqlitePoolOptions; use sqlx::{Sqlite, SqlitePool}; -use url::Url; use wick_config::config::components::SqlComponentConfig; pub(crate) use self::serialize::*; use crate::common::sql_wrapper::ConvertedType; use crate::Error; -pub(crate) async fn connect(_config: &SqlComponentConfig, addr: &Url) -> Result { +pub(crate) async fn connect(_config: &SqlComponentConfig, addr: Option<&str>) -> Result { + let addr = addr.unwrap_or(":memory:"); debug!(%addr, "connecting to sqlite"); let pool = SqlitePoolOptions::new() .max_connections(5) - .connect(addr.as_ref()) + .connect(addr) .await .map_err(|e| Error::SqliteConnect(e.to_string()))?; Ok(pool) diff --git a/crates/components/wick-sql/src/sqlx/sqlite/serialize.rs b/crates/components/wick-sql/src/sqlx/sqlite/serialize.rs index b03553a5..ac08fefc 100644 --- a/crates/components/wick-sql/src/sqlx/sqlite/serialize.rs +++ b/crates/components/wick-sql/src/sqlx/sqlite/serialize.rs @@ -138,7 +138,7 @@ mod integration_test { async fn connect() -> SqliteConnection { let db = std::env::var("SQLITE_DB").unwrap(); - let conn_string = format!("sqlite://{}", db); + let conn_string = format!("file://{}", db); SqliteConnection::connect(&conn_string).await.unwrap() } diff --git a/crates/wick/wick-config/src/lockdown.rs b/crates/wick/wick-config/src/lockdown.rs index 796a4e6a..4e472c76 100644 --- a/crates/wick/wick-config/src/lockdown.rs +++ b/crates/wick/wick-config/src/lockdown.rs @@ -5,7 +5,7 @@ mod volume; pub use error::*; -use crate::audit::{Audit, AuditedResource, AuditedResourceBinding}; +use crate::audit::{Audit, AuditedResource, AuditedResourceBinding, AuditedVolume}; use crate::config::{ConfigOrDefinition, LockdownConfiguration, ResourceRestriction}; pub(crate) fn validate_resource( @@ -33,15 +33,40 @@ pub(crate) fn validate_resource( _ => None, }), ), - AuditedResource::Url(v) => self::url::validate( - component_id, - &resource.name, - v, - resource_restrictions.iter().filter_map(|r| match r { - ResourceRestriction::Url(v) => Some(v), - _ => None, - }), - ), + AuditedResource::Url(v) => { + if v.url.scheme() == "file" { + let Ok(path) = v.url.to_file_path() else { + return Err(LockdownError::new(vec![FailureKind::FileUrlInvalid(v.url.clone())])); + }; + if !path.exists() { + return Err(LockdownError::new(vec![FailureKind::FileUrlNotFound(v.url.clone())])); + } + let dir = if path.is_file() { + path.parent().map(std::path::Path::to_path_buf).unwrap() + } else { + path + }; + self::volume::validate( + component_id, + &resource.name, + &AuditedVolume { path: dir }, + resource_restrictions.iter().filter_map(|r| match r { + ResourceRestriction::Volume(v) => Some(v), + _ => None, + }), + ) + } else { + self::url::validate( + component_id, + &resource.name, + v, + resource_restrictions.iter().filter_map(|r| match r { + ResourceRestriction::Url(v) => Some(v), + _ => None, + }), + ) + } + } AuditedResource::Volume(v) => self::volume::validate( component_id, &resource.name, @@ -78,10 +103,12 @@ pub trait Lockdown { mod test { use std::path::PathBuf; + use ::url::Url; use anyhow::Result; use normpath::PathExt; use super::*; + use crate::audit::AuditedUrl; use crate::config::{ components, AppConfigurationBuilder, @@ -92,6 +119,7 @@ mod test { ImportDefinition, LockdownConfigurationBuilder, ResourceRestriction, + UrlRestriction, VolumeRestriction, }; use crate::WickConfiguration; @@ -124,6 +152,90 @@ mod test { ) } + fn pwdify(s: impl Into) -> String { + s.into().replace("$CRATE", env!("CARGO_MANIFEST_DIR")) + } + + fn path(path: impl Into) -> PathBuf { + PathBuf::from(pwdify(path)) + } + + fn url(path: impl Into) -> Url { + pwdify(path).parse().unwrap() + } + + fn mktmpfile(file: impl Into) -> String { + let file = std::env::temp_dir().join(file.into()); + std::fs::write(&file, "test contents").unwrap(); + file.to_string_lossy().to_string() + } + + #[rstest::rstest] + #[case("$CRATE", "file:///$CRATE/DOES_NOT_EXIST")] + #[case("$CRATE", "file://$CRATE/DOES_NOT_EXIST")] + #[case("$CRATE", "file:///$CRATE/../../../README.md")] + #[case("$CRATE", "file://$CRATE/../../../README.md")] + #[case("$CRATE", "file:/$CRATE/../../../README.md")] + #[case("$CRATE", format!("file:///{}",mktmpfile("TEST.md")))] + fn test_file_url_volume_restriction_fails( + #[case] allowed_path: impl Into, + #[case] resource_url: impl Into, + ) -> Result<()> { + let allowed_path = path(allowed_path); + let resource_url = url(resource_url); + let file_url = AuditedResourceBinding { + name: "url".to_owned(), + resource: AuditedResource::Url(AuditedUrl { url: resource_url }), + }; + + // Allow all URLs for "test_component" but have a volume restriction that should fail. + let lockdown = new_lockdown_config(vec![ + ResourceRestriction::Url(UrlRestriction::new_from_template(vec!["test_component".into()], "*")), + ResourceRestriction::Volume(VolumeRestriction::new_from_template( + vec!["test_component".into()], + allowed_path.to_string_lossy(), + )), + ]); + + let result = validate_resource("test_component", &file_url, &lockdown); + if let Err(e) = result { + println!("{}", e); + } else { + panic!("Expected an error, got {:?}", result); + } + + Ok(()) + } + + #[rstest::rstest] + #[case("$CRATE", "file:///$CRATE/README.md")] + #[case("$CRATE", "https://google.com")] + #[case("$CRATE", "postgres://pg:pg@127.0.0.1:5432")] + fn test_file_url_volume_restriction_passes( + #[case] allowed_path: impl Into, + #[case] resource_url: impl Into, + ) -> Result<()> { + let allowed_path = path(allowed_path); + let resource_url = url(resource_url); + let file_url = AuditedResourceBinding { + name: "url".to_owned(), + resource: AuditedResource::Url(AuditedUrl { url: resource_url }), + }; + + // Allow all URLs for "test_component" but have a volume restriction that should fail. + let lockdown = new_lockdown_config(vec![ + ResourceRestriction::Url(UrlRestriction::new_from_template(vec!["test_component".into()], "*")), + ResourceRestriction::Volume(VolumeRestriction::new_from_template( + vec!["test_component".into()], + allowed_path.to_string_lossy(), + )), + ]); + + validate_resource("test_component", &file_url, &lockdown)?; + + Ok(()) + } + #[test_logger::test(tokio::test)] async fn test_tree_walker() -> Result<()> { let mut config = AppConfigurationBuilder::default(); diff --git a/crates/wick/wick-config/src/lockdown/error.rs b/crates/wick/wick-config/src/lockdown/error.rs index 2f454717..3a39129d 100644 --- a/crates/wick/wick-config/src/lockdown/error.rs +++ b/crates/wick/wick-config/src/lockdown/error.rs @@ -58,6 +58,10 @@ pub enum FailureKind { Address(String, String), /// A component is not allowed to access given url. Url(String, String), + /// A file:// URL could not be turned into a filepath. + FileUrlInvalid(url::Url), + /// A file:// URL does not point to a concrete file. + FileUrlNotFound(url::Url), } impl std::fmt::Display for FailureKind { @@ -80,6 +84,8 @@ impl std::fmt::Display for FailureKind { FailureKind::Port(id, port) => write!(f, "component {} is not allowed to access {}", id, port), FailureKind::Address(id,address) => write!(f, "component {} is not allowed to access {}", id, address), FailureKind::Url(id, url) => write!(f, "component {} is not allowed to access {}", id, url), + FailureKind::FileUrlInvalid(url) => write!(f, "could not create a file path out of {}", url), + FailureKind::FileUrlNotFound(url) => write!(f, "file URL '{}' does not point to a valid file", url), } } } diff --git a/crates/wick/wick-config/src/lockdown/url.rs b/crates/wick/wick-config/src/lockdown/url.rs index 92973ee1..fb870bcf 100644 --- a/crates/wick/wick-config/src/lockdown/url.rs +++ b/crates/wick/wick-config/src/lockdown/url.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] use std::collections::HashSet; use wildmatch::WildMatch; @@ -15,6 +14,7 @@ pub(crate) fn validate<'a>( restrictions: impl Iterator, ) -> Result<(), LockdownError> { let mut failures = HashSet::new(); + for restriction in restrictions { match is_allowed(component_id, resource_id, resource, restriction) { Ok(_) => return Ok(()), diff --git a/examples/db/azuresql-component.wick b/examples/db/azuresql-component.wick index bb924d01..355177f1 100644 --- a/examples/db/azuresql-component.wick +++ b/examples/db/azuresql-component.wick @@ -30,10 +30,7 @@ component: type: string - name: email type: string - query: INSERT INTO users(name, email) OUTPUT INSERTED.* VALUES ($1, $2) - arguments: - - name - - email + query: INSERT INTO users(name, email) OUTPUT INSERTED.* VALUES (${name}, ${email}) - name: set_user_with_id inputs: - name: id @@ -42,10 +39,7 @@ component: type: string - name: email type: string - query: INSERT INTO users(id, name, email) OUTPUT INSERTED.* VALUES ($1, $2, $3) - arguments: - - name - - email + query: INSERT INTO users(id, name, email) OUTPUT INSERTED.* VALUES (${id}, ${name}, ${email}) - name: set_user_with_columns inputs: - name: input @@ -53,3 +47,22 @@ component: query: INSERT INTO users(name, email) OUTPUT INSERTED.* VALUES ($1, $2) arguments: - input... # This special `spread` syntax expands the input array into individual positional arguments +tests: + - with: + password: '{{ctx.env.TEST_PASSWORD}}' + host: '{{ctx.env.TEST_HOST}}' + port: '{{ctx.env.MSSQL_PORT}}' + cases: + - operation: set_user + inputs: + - name: name + value: TEST_NAME + - name: email + value: TEST_EMAIL@example.com + outputs: + - name: output + assertions: + - operator: Contains + value: + email: TEST_EMAIL@example.com + name: TEST_NAME diff --git a/examples/db/postgres-component.wick b/examples/db/postgres-component.wick index 083e8877..b5dd8f32 100644 --- a/examples/db/postgres-component.wick +++ b/examples/db/postgres-component.wick @@ -21,19 +21,14 @@ component: inputs: - name: id type: i32 - query: SELECT * FROM users WHERE id = $1 - arguments: - - id + query: SELECT * FROM users WHERE id = ${id} - name: set_user inputs: - name: name type: string - name: email type: string - query: INSERT INTO users(name, email) VALUES ($1, $2) RETURNING * - arguments: - - name - - email + query: INSERT INTO users(name, email) VALUES (${name}, ${email}) RETURNING * - name: set_user_with_columns inputs: - name: input @@ -41,3 +36,22 @@ component: query: INSERT INTO users(name, email) VALUES ($1, $2) RETURNING * arguments: - input... # This is special "spread" syntax that expands the input array into individual positional arguments +tests: + - with: + password: '{{ctx.env.TEST_PASSWORD}}' + host: '{{ctx.env.TEST_HOST}}' + port: '{{ctx.env.POSTGRES_PORT}}' + cases: + - operation: set_user + inputs: + - name: name + value: TEST_NAME + - name: email + value: TEST_EMAIL@example.com + outputs: + - name: output + assertions: + - operator: Contains + value: + email: TEST_EMAIL@example.com + name: TEST_NAME diff --git a/examples/db/sqlite-component.wick b/examples/db/sqlite-component.wick new file mode 100644 index 00000000..22382502 --- /dev/null +++ b/examples/db/sqlite-component.wick @@ -0,0 +1,51 @@ +name: my_component +kind: wick/component@v1 +resources: + - name: DBADDR + resource: + kind: wick/resource/url@v1 + url: file://{{ ctx.root_config.db_file }} +component: + kind: wick/component/sql@v1 + resource: DBADDR + tls: false + with: + - name: db_file + type: string + operations: + - name: get_user + inputs: + - name: id + type: i32 + query: SELECT * FROM users WHERE id = ${id} + - name: set_user + inputs: + - name: name + type: string + - name: email + type: string + query: INSERT INTO users(name, email) VALUES (${name}, ${email}) RETURNING * + - name: set_user_with_columns + inputs: + - name: input + type: string[] + query: INSERT INTO users(name, email) VALUES ($1, $2) RETURNING * + arguments: + - input... # This is special "spread" syntax that expands the input array into individual positional arguments +tests: + - with: + db_file: '{{ctx.env.SQLITE_DB}}' + cases: + - operation: set_user + inputs: + - name: name + value: TEST_NAME + - name: email + value: TEST_EMAIL@example.com + outputs: + - name: output + assertions: + - operator: Contains + value: + email: TEST_EMAIL@example.com + name: TEST_NAME diff --git a/examples/db/sqlite-inmemory-component.wick b/examples/db/sqlite-inmemory-component.wick new file mode 100644 index 00000000..e441ea24 --- /dev/null +++ b/examples/db/sqlite-inmemory-component.wick @@ -0,0 +1,63 @@ +name: my_component +kind: wick/component@v1 +resources: + - name: DBADDR + resource: + kind: wick/resource/url@v1 + url: sqlite://memory +component: + kind: wick/component/sql@v1 + resource: DBADDR + tls: false + with: + - name: db_file + type: string + operations: + - name: init + inputs: [] + exec: | + CREATE TABLE users ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + email TEXT NOT NULL + ); + - name: set_user + inputs: + - name: name + type: string + - name: email + type: string + query: INSERT INTO users(name, email) VALUES (${name}, ${email}) RETURNING * +tests: + - with: + db_file: '{{ctx.env.SQLITE_DB}}' + cases: + - operation: init + inputs: [] + outputs: + - name: output + value: 0 + - operation: set_user + inputs: + - name: name + value: TEST_NAME + - name: email + value: TEST_EMAIL@example.com + outputs: + - name: output + value: + email: TEST_EMAIL@example.com + name: TEST_NAME + id: 1 + - operation: set_user + inputs: + - name: name + value: TEST_NAME2 + - name: email + value: TEST_EMAIL2@example.com + outputs: + - name: output + value: + email: TEST_EMAIL2@example.com + name: TEST_NAME2 + id: 2 diff --git a/tests/cli-tests/tests/cmd/db/sqlite-component.toml b/tests/cli-tests/tests/cmd/db/sqlite-component.toml new file mode 100644 index 00000000..48e2a357 --- /dev/null +++ b/tests/cli-tests/tests/cmd/db/sqlite-component.toml @@ -0,0 +1,16 @@ +#:schema https://raw.githubusercontent.com/assert-rs/trycmd/main/schema.json +bin.name = "wick" +args = [ + "invoke", + "examples/db/postgres-component.wick", + '--with', + "{\"password\":\"{{ctx.env.TEST_PASSWORD}}\",\"port\":\"{{ctx.env.POSTGRES_PORT}}\",\"host\":\"{{ctx.env.TEST_HOST}}\"}", + "set_user", + "--", + "--name", + "TEST_NAME", + "--email", + "TEST_EMAIL@example.com" +] +stdout = """{"payload":{"value":{"email":"TEST_EMAIL@example.com","id":[..],"name":"TEST_NAME"}},"port":"output"} +"""