From 485c5f4ffaf96ba66b2255f76c299cd0d01e9c0b Mon Sep 17 00:00:00 2001 From: Raminder Singh Date: Tue, 7 Oct 2025 16:36:50 +0530 Subject: [PATCH] feat: encrypt s3_access_key_id as well --- etl-api/src/configs/destination.rs | 45 +++++++++++++------ ...g_supabase_destination_can_be_updated.snap | 4 +- ...stination_and_pipeline_can_be_updated.snap | 4 +- ...stination_and_pipeline_can_be_created.snap | 4 +- etl-api/tests/support/mocks.rs | 4 +- etl-config/src/shared/destination.rs | 2 +- etl-replicator/src/core.rs | 10 ++--- 7 files changed, 47 insertions(+), 26 deletions(-) diff --git a/etl-api/src/configs/destination.rs b/etl-api/src/configs/destination.rs index 2b942fbdf..25a73d7d3 100644 --- a/etl-api/src/configs/destination.rs +++ b/etl-api/src/configs/destination.rs @@ -252,6 +252,8 @@ impl Encrypt for StoredDestinationConfig { } => { let encrypted_catalog_token = encrypt_text(catalog_token.expose_secret().to_owned(), encryption_key)?; + let encrypted_s3_access_key_id = + encrypt_text(s3_access_key_id.expose_secret().to_owned(), encryption_key)?; let encrypted_s3_secret_access_key = encrypt_text( s3_secret_access_key.expose_secret().to_owned(), encryption_key, @@ -262,7 +264,7 @@ impl Encrypt for StoredDestinationConfig { warehouse_name, namespace, catalog_token: encrypted_catalog_token, - s3_access_key_id, + s3_access_key_id: encrypted_s3_access_key_id, s3_secret_access_key: encrypted_s3_secret_access_key, s3_region, }, @@ -336,7 +338,7 @@ impl Decrypt for EncryptedStoredDestinationConfig { warehouse_name, namespace, catalog_token: encrypted_catalog_token, - s3_access_key_id, + s3_access_key_id: encrypted_s3_access_key_id, s3_secret_access_key: encrypted_s3_secret_access_key, s3_region, } => { @@ -345,6 +347,11 @@ impl Decrypt for EncryptedStoredDestinationConfig { encryption_key, )?); + let s3_access_key_id = SerializableSecretString::from(decrypt_text( + encrypted_s3_access_key_id, + encryption_key, + )?); + let s3_secret_access_key = SerializableSecretString::from(decrypt_text( encrypted_s3_secret_access_key, encryption_key, @@ -386,7 +393,7 @@ pub enum StoredIcebergConfig { warehouse_name: String, namespace: String, catalog_token: SerializableSecretString, - s3_access_key_id: String, + s3_access_key_id: SerializableSecretString, s3_secret_access_key: SerializableSecretString, s3_region: String, }, @@ -412,7 +419,7 @@ pub enum FullApiIcebergConfig { )] catalog_token: SerializableSecretString, #[schema(example = "9156667efc2c70d89af6588da86d2924")] - s3_access_key_id: String, + s3_access_key_id: SerializableSecretString, #[schema(example = "ca833e890916d848c69135924bcd75e5909184814a0ebc6c988937ee094120d4")] s3_secret_access_key: SerializableSecretString, #[schema(example = "ap-southeast-1")] @@ -436,7 +443,7 @@ pub enum EncryptedStoredIcebergConfig { warehouse_name: String, namespace: String, catalog_token: EncryptedValue, - s3_access_key_id: String, + s3_access_key_id: EncryptedValue, s3_secret_access_key: EncryptedValue, s3_region: String, }, @@ -504,7 +511,7 @@ mod tests { warehouse_name: "my-warehouse".to_string(), namespace: "my-namespace".to_string(), catalog_token: SerializableSecretString::from("eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IjFkNzFjMGEyNmIxMDFjODQ5ZTkxZmQ1NjdjYjA5NTJmIn0.eyJleHAiOjIwNzA3MTcxNjAsImlhdCI6MTc1NjE0NTE1MCwiaXNzIjoic3VwYWJhc2UiLCJyZWYiOiJhYmNkZWZnaGlqbGttbm9wcXJzdCIsInJvbGUiOiJzZXJ2aWNlX3JvbGUifQ.YdTWkkIvwjSkXot3NC07xyjPjGWQMNzLq5EPzumzrdLzuHrj-zuzI-nlyQtQ5V7gZauysm-wGwmpztRXfPc3AQ".to_string()), - s3_access_key_id: "9156667efc2c70d89af6588da86d2924".to_string(), + s3_access_key_id: SerializableSecretString::from("9156667efc2c70d89af6588da86d2924".to_string()), s3_secret_access_key: SerializableSecretString::from("ca833e890916d848c69135924bcd75e5909184814a0ebc6c988937ee094120d4".to_string()), s3_region: "ap-southeast-1".to_string(), }, @@ -547,7 +554,10 @@ mod tests { p1_catalog_token.expose_secret(), p2_catalog_token.expose_secret() ); - assert_eq!(p1_s3_access_key_id, p2_s3_access_key_id); + assert_eq!( + p1_s3_access_key_id.expose_secret(), + p2_s3_access_key_id.expose_secret() + ); assert_eq!( p1_s3_secret_access_key.expose_secret(), p2_s3_secret_access_key.expose_secret() @@ -653,7 +663,7 @@ mod tests { warehouse_name: "my-warehouse".to_string(), namespace: "my-namespace".to_string(), catalog_token: SerializableSecretString::from("eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IjFkNzFjMGEyNmIxMDFjODQ5ZTkxZmQ1NjdjYjA5NTJmIn0.eyJleHAiOjIwNzA3MTcxNjAsImlhdCI6MTc1NjE0NTE1MCwiaXNzIjoic3VwYWJhc2UiLCJyZWYiOiJhYmNkZWZnaGlqbGttbm9wcXJzdCIsInJvbGUiOiJzZXJ2aWNlX3JvbGUifQ.YdTWkkIvwjSkXot3NC07xyjPjGWQMNzLq5EPzumzrdLzuHrj-zuzI-nlyQtQ5V7gZauysm-wGwmpztRXfPc3AQ".to_string()), - s3_access_key_id: "9156667efc2c70d89af6588da86d2924".to_string(), + s3_access_key_id: SerializableSecretString::from("9156667efc2c70d89af6588da86d2924".to_string()), s3_secret_access_key: SerializableSecretString::from("ca833e890916d848c69135924bcd75e5909184814a0ebc6c988937ee094120d4".to_string()), s3_region: "ap-southeast-1".to_string(), }, @@ -697,7 +707,10 @@ mod tests { assert_eq!(p1_project_ref, p2_project_ref); assert_eq!(p1_warehouse_name, p2_warehouse_name); assert_eq!(p1_namespace, p2_namespace); - assert_eq!(p1_s3_access_key_id, p2_s3_access_key_id); + assert_eq!( + p1_s3_access_key_id.expose_secret(), + p2_s3_access_key_id.expose_secret() + ); assert_eq!(p1_s3_region, p2_s3_region); // Assert that secret fields were encrypted and decrypted correctly assert_eq!( @@ -815,7 +828,7 @@ mod tests { warehouse_name: "my-warehouse".to_string(), namespace: "my-namespace".to_string(), catalog_token: SerializableSecretString::from("eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IjFkNzFjMGEyNmIxMDFjODQ5ZTkxZmQ1NjdjYjA5NTJmIn0.eyJleHAiOjIwNzA3MTcxNjAsImlhdCI6MTc1NjE0NTE1MCwiaXNzIjoic3VwYWJhc2UiLCJyZWYiOiJhYmNkZWZnaGlqbGttbm9wcXJzdCIsInJvbGUiOiJzZXJ2aWNlX3JvbGUifQ.YdTWkkIvwjSkXot3NC07xyjPjGWQMNzLq5EPzumzrdLzuHrj-zuzI-nlyQtQ5V7gZauysm-wGwmpztRXfPc3AQ".to_string()), - s3_access_key_id: "9156667efc2c70d89af6588da86d2924".to_string(), + s3_access_key_id: SerializableSecretString::from("9156667efc2c70d89af6588da86d2924".to_string()), s3_secret_access_key: SerializableSecretString::from("ca833e890916d848c69135924bcd75e5909184814a0ebc6c988937ee094120d4".to_string()), s3_region: "ap-southeast-1".to_string(), }, @@ -858,7 +871,10 @@ mod tests { p1_catalog_token.expose_secret(), p2_catalog_token.expose_secret() ); - assert_eq!(p1_s3_access_key_id, p2_s3_access_key_id); + assert_eq!( + p1_s3_access_key_id.expose_secret(), + p2_s3_access_key_id.expose_secret() + ); assert_eq!( p1_s3_secret_access_key.expose_secret(), p2_s3_secret_access_key.expose_secret() @@ -918,7 +934,7 @@ mod tests { warehouse_name: "my-warehouse".to_string(), namespace: "my-namespace".to_string(), catalog_token: SerializableSecretString::from("token123".to_string()), - s3_access_key_id: "access_key_123".to_string(), + s3_access_key_id: SerializableSecretString::from("access_key_123".to_string()), s3_secret_access_key: SerializableSecretString::from("secret123".to_string()), s3_region: "us-west-2".to_string(), }, @@ -964,7 +980,10 @@ mod tests { orig_catalog_token.expose_secret(), deser_catalog_token.expose_secret() ); - assert_eq!(orig_s3_access_key_id, &deser_s3_access_key_id); + assert_eq!( + orig_s3_access_key_id.expose_secret(), + deser_s3_access_key_id.expose_secret() + ); assert_eq!( orig_s3_secret_access_key.expose_secret(), deser_s3_secret_access_key.expose_secret() diff --git a/etl-api/tests/snapshots/destinations__an_existing_iceberg_supabase_destination_can_be_updated.snap b/etl-api/tests/snapshots/destinations__an_existing_iceberg_supabase_destination_can_be_updated.snap index 7419c82a2..ac2585910 100644 --- a/etl-api/tests/snapshots/destinations__an_existing_iceberg_supabase_destination_can_be_updated.snap +++ b/etl-api/tests/snapshots/destinations__an_existing_iceberg_supabase_destination_can_be_updated.snap @@ -10,7 +10,9 @@ Iceberg { catalog_token: SerializableSecretString( SecretBox([REDACTED]), ), - s3_access_key_id: "updated9156667efc2c70d89af6588da86d2924", + s3_access_key_id: SerializableSecretString( + SecretBox([REDACTED]), + ), s3_secret_access_key: SerializableSecretString( SecretBox([REDACTED]), ), diff --git a/etl-api/tests/snapshots/destinations_pipelines__an_existing_iceberg_supabase_destination_and_pipeline_can_be_updated.snap b/etl-api/tests/snapshots/destinations_pipelines__an_existing_iceberg_supabase_destination_and_pipeline_can_be_updated.snap index a866b98a9..e08093226 100644 --- a/etl-api/tests/snapshots/destinations_pipelines__an_existing_iceberg_supabase_destination_and_pipeline_can_be_updated.snap +++ b/etl-api/tests/snapshots/destinations_pipelines__an_existing_iceberg_supabase_destination_and_pipeline_can_be_updated.snap @@ -10,7 +10,9 @@ Iceberg { catalog_token: SerializableSecretString( SecretBox([REDACTED]), ), - s3_access_key_id: "updated9156667efc2c70d89af6588da86d2924", + s3_access_key_id: SerializableSecretString( + SecretBox([REDACTED]), + ), s3_secret_access_key: SerializableSecretString( SecretBox([REDACTED]), ), diff --git a/etl-api/tests/snapshots/destinations_pipelines__iceberg_supabase_destination_and_pipeline_can_be_created.snap b/etl-api/tests/snapshots/destinations_pipelines__iceberg_supabase_destination_and_pipeline_can_be_created.snap index 41a3914e1..cbd4ab915 100644 --- a/etl-api/tests/snapshots/destinations_pipelines__iceberg_supabase_destination_and_pipeline_can_be_created.snap +++ b/etl-api/tests/snapshots/destinations_pipelines__iceberg_supabase_destination_and_pipeline_can_be_created.snap @@ -10,7 +10,9 @@ Iceberg { catalog_token: SerializableSecretString( SecretBox([REDACTED]), ), - s3_access_key_id: "9156667efc2c70d89af6588da86d2924", + s3_access_key_id: SerializableSecretString( + SecretBox([REDACTED]), + ), s3_secret_access_key: SerializableSecretString( SecretBox([REDACTED]), ), diff --git a/etl-api/tests/support/mocks.rs b/etl-api/tests/support/mocks.rs index fdbc0065a..3f9ebed0b 100644 --- a/etl-api/tests/support/mocks.rs +++ b/etl-api/tests/support/mocks.rs @@ -78,7 +78,7 @@ pub mod destinations { catalog_token: SerializableSecretString::from( "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IjFkNzFjMGEyNmIxMDFjODQ5ZTkxZmQ1NjdjYjA5NTJmIn0.eyJleHAiOjIwNzA3MTcxNjAsImlhdCI6MTc1NjE0NTE1MCwiaXNzIjoic3VwYWJhc2UiLCJyZWYiOiJhYmNkZWZnaGlqbGttbm9wcXJzdCIsInJvbGUiOiJzZXJ2aWNlX3JvbGUifQ.YdTWkkIvwjSkXot3NC07xyjPjGWQMNzLq5EPzumzrdLzuHrj-zuzI-nlyQtQ5V7gZauysm-wGwmpztRXfPc3AQ".to_string() ), - s3_access_key_id: "9156667efc2c70d89af6588da86d2924".to_string(), + s3_access_key_id: SerializableSecretString::from("9156667efc2c70d89af6588da86d2924".to_string()), s3_secret_access_key: SerializableSecretString::from( "ca833e890916d848c69135924bcd75e5909184814a0ebc6c988937ee094120d4".to_string() ), @@ -99,7 +99,7 @@ pub mod destinations { catalog_token: SerializableSecretString::from( "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IjJlOGQxZDNjN2MyMTJkOTU4ZmEyOGU2ZDhjZDEwYTMzIn0.eyJleHAiOjIwNzA3MTcxNjAsImlhdCI6MTc1NjE0NTE1MCwiaXNzIjoic3VwYWJhc2UiLCJyZWYiOiJ0c3JxcG9ubWxramloZ2ZlZGNiYSIsInJvbGUiOiJzZXJ2aWNlX3JvbGUifQ.UpdatedTokenSignatureForTesting".to_string() ), - s3_access_key_id: "updated9156667efc2c70d89af6588da86d2924".to_string(), + s3_access_key_id: SerializableSecretString::from("updated9156667efc2c70d89af6588da86d2924".to_string()), s3_secret_access_key: SerializableSecretString::from( "updatedca833e890916d848c69135924bcd75e5909184814a0ebc6c988937ee094120d4".to_string() ), diff --git a/etl-config/src/shared/destination.rs b/etl-config/src/shared/destination.rs index ba1b18194..394ac8698 100644 --- a/etl-config/src/shared/destination.rs +++ b/etl-config/src/shared/destination.rs @@ -59,7 +59,7 @@ pub enum IcebergConfig { /// Catalog authentication token catalog_token: SerializableSecretString, /// The S3 access key id - s3_access_key_id: String, + s3_access_key_id: SerializableSecretString, /// The S3 secret access key s3_secret_access_key: SerializableSecretString, /// The S3 region diff --git a/etl-replicator/src/core.rs b/etl-replicator/src/core.rs index 8b38f8089..cf60ac485 100644 --- a/etl-replicator/src/core.rs +++ b/etl-replicator/src/core.rs @@ -92,7 +92,7 @@ pub async fn start_replicator_with_config( supabase_domain, catalog_token.expose_secret().to_string(), warehouse_name.clone(), - s3_access_key_id.clone(), + s3_access_key_id.expose_secret().to_string(), s3_secret_access_key.expose_secret().to_string(), s3_region.clone(), ); @@ -160,18 +160,14 @@ fn log_destination_config(config: &DestinationConfig) { project_ref, catalog_token: _, warehouse_name, - s3_access_key_id, + s3_access_key_id: _, s3_secret_access_key: _, s3_region, }, } => { debug!( namespace, - project_ref, - warehouse_name, - s3_access_key_id, - s3_region, - "using Supabase iceberg destination config" + project_ref, warehouse_name, s3_region, "using Supabase iceberg destination config" ) } DestinationConfig::Iceberg {