Skip to content

feat: encrypt s3_access_key_id as well #382

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 7, 2025
Merged

feat: encrypt s3_access_key_id as well #382

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 32 additions & 13 deletions etl-api/src/configs/destination.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,8 @@ impl Encrypt<EncryptedStoredDestinationConfig> for StoredDestinationConfig {
} => {
let encrypted_catalog_token =
encrypt_text(catalog_token.expose_secret().to_owned(), encryption_key)?;
let encrypted_s3_access_key_id =
encrypt_text(s3_access_key_id.expose_secret().to_owned(), encryption_key)?;
let encrypted_s3_secret_access_key = encrypt_text(
s3_secret_access_key.expose_secret().to_owned(),
encryption_key,
Expand All @@ -262,7 +264,7 @@ impl Encrypt<EncryptedStoredDestinationConfig> for StoredDestinationConfig {
warehouse_name,
namespace,
catalog_token: encrypted_catalog_token,
s3_access_key_id,
s3_access_key_id: encrypted_s3_access_key_id,
s3_secret_access_key: encrypted_s3_secret_access_key,
s3_region,
},
Expand Down Expand Up @@ -336,7 +338,7 @@ impl Decrypt<StoredDestinationConfig> for EncryptedStoredDestinationConfig {
warehouse_name,
namespace,
catalog_token: encrypted_catalog_token,
s3_access_key_id,
s3_access_key_id: encrypted_s3_access_key_id,
s3_secret_access_key: encrypted_s3_secret_access_key,
s3_region,
} => {
Expand All @@ -345,6 +347,11 @@ impl Decrypt<StoredDestinationConfig> for EncryptedStoredDestinationConfig {
encryption_key,
)?);

let s3_access_key_id = SerializableSecretString::from(decrypt_text(
encrypted_s3_access_key_id,
encryption_key,
)?);

let s3_secret_access_key = SerializableSecretString::from(decrypt_text(
encrypted_s3_secret_access_key,
encryption_key,
Expand Down Expand Up @@ -386,7 +393,7 @@ pub enum StoredIcebergConfig {
warehouse_name: String,
namespace: String,
catalog_token: SerializableSecretString,
s3_access_key_id: String,
s3_access_key_id: SerializableSecretString,
s3_secret_access_key: SerializableSecretString,
s3_region: String,
},
Expand All @@ -412,7 +419,7 @@ pub enum FullApiIcebergConfig {
)]
catalog_token: SerializableSecretString,
#[schema(example = "9156667efc2c70d89af6588da86d2924")]
s3_access_key_id: String,
s3_access_key_id: SerializableSecretString,
#[schema(example = "ca833e890916d848c69135924bcd75e5909184814a0ebc6c988937ee094120d4")]
s3_secret_access_key: SerializableSecretString,
#[schema(example = "ap-southeast-1")]
Expand All @@ -436,7 +443,7 @@ pub enum EncryptedStoredIcebergConfig {
warehouse_name: String,
namespace: String,
catalog_token: EncryptedValue,
s3_access_key_id: String,
s3_access_key_id: EncryptedValue,
s3_secret_access_key: EncryptedValue,
s3_region: String,
},
Expand Down Expand Up @@ -504,7 +511,7 @@ mod tests {
warehouse_name: "my-warehouse".to_string(),
namespace: "my-namespace".to_string(),
catalog_token: SerializableSecretString::from("eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IjFkNzFjMGEyNmIxMDFjODQ5ZTkxZmQ1NjdjYjA5NTJmIn0.eyJleHAiOjIwNzA3MTcxNjAsImlhdCI6MTc1NjE0NTE1MCwiaXNzIjoic3VwYWJhc2UiLCJyZWYiOiJhYmNkZWZnaGlqbGttbm9wcXJzdCIsInJvbGUiOiJzZXJ2aWNlX3JvbGUifQ.YdTWkkIvwjSkXot3NC07xyjPjGWQMNzLq5EPzumzrdLzuHrj-zuzI-nlyQtQ5V7gZauysm-wGwmpztRXfPc3AQ".to_string()),
s3_access_key_id: "9156667efc2c70d89af6588da86d2924".to_string(),
s3_access_key_id: SerializableSecretString::from("9156667efc2c70d89af6588da86d2924".to_string()),
s3_secret_access_key: SerializableSecretString::from("ca833e890916d848c69135924bcd75e5909184814a0ebc6c988937ee094120d4".to_string()),
s3_region: "ap-southeast-1".to_string(),
},
Expand Down Expand Up @@ -547,7 +554,10 @@ mod tests {
p1_catalog_token.expose_secret(),
p2_catalog_token.expose_secret()
);
assert_eq!(p1_s3_access_key_id, p2_s3_access_key_id);
assert_eq!(
p1_s3_access_key_id.expose_secret(),
p2_s3_access_key_id.expose_secret()
);
assert_eq!(
p1_s3_secret_access_key.expose_secret(),
p2_s3_secret_access_key.expose_secret()
Expand Down Expand Up @@ -653,7 +663,7 @@ mod tests {
warehouse_name: "my-warehouse".to_string(),
namespace: "my-namespace".to_string(),
catalog_token: SerializableSecretString::from("eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IjFkNzFjMGEyNmIxMDFjODQ5ZTkxZmQ1NjdjYjA5NTJmIn0.eyJleHAiOjIwNzA3MTcxNjAsImlhdCI6MTc1NjE0NTE1MCwiaXNzIjoic3VwYWJhc2UiLCJyZWYiOiJhYmNkZWZnaGlqbGttbm9wcXJzdCIsInJvbGUiOiJzZXJ2aWNlX3JvbGUifQ.YdTWkkIvwjSkXot3NC07xyjPjGWQMNzLq5EPzumzrdLzuHrj-zuzI-nlyQtQ5V7gZauysm-wGwmpztRXfPc3AQ".to_string()),
s3_access_key_id: "9156667efc2c70d89af6588da86d2924".to_string(),
s3_access_key_id: SerializableSecretString::from("9156667efc2c70d89af6588da86d2924".to_string()),
s3_secret_access_key: SerializableSecretString::from("ca833e890916d848c69135924bcd75e5909184814a0ebc6c988937ee094120d4".to_string()),
s3_region: "ap-southeast-1".to_string(),
},
Expand Down Expand Up @@ -697,7 +707,10 @@ mod tests {
assert_eq!(p1_project_ref, p2_project_ref);
assert_eq!(p1_warehouse_name, p2_warehouse_name);
assert_eq!(p1_namespace, p2_namespace);
assert_eq!(p1_s3_access_key_id, p2_s3_access_key_id);
assert_eq!(
p1_s3_access_key_id.expose_secret(),
p2_s3_access_key_id.expose_secret()
);
assert_eq!(p1_s3_region, p2_s3_region);
// Assert that secret fields were encrypted and decrypted correctly
assert_eq!(
Expand Down Expand Up @@ -815,7 +828,7 @@ mod tests {
warehouse_name: "my-warehouse".to_string(),
namespace: "my-namespace".to_string(),
catalog_token: SerializableSecretString::from("eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IjFkNzFjMGEyNmIxMDFjODQ5ZTkxZmQ1NjdjYjA5NTJmIn0.eyJleHAiOjIwNzA3MTcxNjAsImlhdCI6MTc1NjE0NTE1MCwiaXNzIjoic3VwYWJhc2UiLCJyZWYiOiJhYmNkZWZnaGlqbGttbm9wcXJzdCIsInJvbGUiOiJzZXJ2aWNlX3JvbGUifQ.YdTWkkIvwjSkXot3NC07xyjPjGWQMNzLq5EPzumzrdLzuHrj-zuzI-nlyQtQ5V7gZauysm-wGwmpztRXfPc3AQ".to_string()),
s3_access_key_id: "9156667efc2c70d89af6588da86d2924".to_string(),
s3_access_key_id: SerializableSecretString::from("9156667efc2c70d89af6588da86d2924".to_string()),
s3_secret_access_key: SerializableSecretString::from("ca833e890916d848c69135924bcd75e5909184814a0ebc6c988937ee094120d4".to_string()),
s3_region: "ap-southeast-1".to_string(),
},
Expand Down Expand Up @@ -858,7 +871,10 @@ mod tests {
p1_catalog_token.expose_secret(),
p2_catalog_token.expose_secret()
);
assert_eq!(p1_s3_access_key_id, p2_s3_access_key_id);
assert_eq!(
p1_s3_access_key_id.expose_secret(),
p2_s3_access_key_id.expose_secret()
);
assert_eq!(
p1_s3_secret_access_key.expose_secret(),
p2_s3_secret_access_key.expose_secret()
Expand Down Expand Up @@ -918,7 +934,7 @@ mod tests {
warehouse_name: "my-warehouse".to_string(),
namespace: "my-namespace".to_string(),
catalog_token: SerializableSecretString::from("token123".to_string()),
s3_access_key_id: "access_key_123".to_string(),
s3_access_key_id: SerializableSecretString::from("access_key_123".to_string()),
s3_secret_access_key: SerializableSecretString::from("secret123".to_string()),
s3_region: "us-west-2".to_string(),
},
Expand Down Expand Up @@ -964,7 +980,10 @@ mod tests {
orig_catalog_token.expose_secret(),
deser_catalog_token.expose_secret()
);
assert_eq!(orig_s3_access_key_id, &deser_s3_access_key_id);
assert_eq!(
orig_s3_access_key_id.expose_secret(),
deser_s3_access_key_id.expose_secret()
);
assert_eq!(
orig_s3_secret_access_key.expose_secret(),
deser_s3_secret_access_key.expose_secret()
Expand Down
4 changes: 3 additions & 1 deletion ...ests/snapshots/destinations__an_existing_iceberg_supabase_destination_can_be_updated.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ Iceberg {
catalog_token: SerializableSecretString(
SecretBox<str>([REDACTED]),
),
s3_access_key_id: "updated9156667efc2c70d89af6588da86d2924",
s3_access_key_id: SerializableSecretString(
SecretBox<str>([REDACTED]),
),
s3_secret_access_key: SerializableSecretString(
SecretBox<str>([REDACTED]),
),
Expand Down
4 changes: 3 additions & 1 deletion ...ions_pipelines__an_existing_iceberg_supabase_destination_and_pipeline_can_be_updated.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ Iceberg {
catalog_token: SerializableSecretString(
SecretBox<str>([REDACTED]),
),
s3_access_key_id: "updated9156667efc2c70d89af6588da86d2924",
s3_access_key_id: SerializableSecretString(
SecretBox<str>([REDACTED]),
),
s3_secret_access_key: SerializableSecretString(
SecretBox<str>([REDACTED]),
),
Expand Down
4 changes: 3 additions & 1 deletion ...ots/destinations_pipelines__iceberg_supabase_destination_and_pipeline_can_be_created.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ Iceberg {
catalog_token: SerializableSecretString(
SecretBox<str>([REDACTED]),
),
s3_access_key_id: "9156667efc2c70d89af6588da86d2924",
s3_access_key_id: SerializableSecretString(
SecretBox<str>([REDACTED]),
),
s3_secret_access_key: SerializableSecretString(
SecretBox<str>([REDACTED]),
),
Expand Down
4 changes: 2 additions & 2 deletions etl-api/tests/support/mocks.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ pub mod destinations {
catalog_token: SerializableSecretString::from(
"eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IjFkNzFjMGEyNmIxMDFjODQ5ZTkxZmQ1NjdjYjA5NTJmIn0.eyJleHAiOjIwNzA3MTcxNjAsImlhdCI6MTc1NjE0NTE1MCwiaXNzIjoic3VwYWJhc2UiLCJyZWYiOiJhYmNkZWZnaGlqbGttbm9wcXJzdCIsInJvbGUiOiJzZXJ2aWNlX3JvbGUifQ.YdTWkkIvwjSkXot3NC07xyjPjGWQMNzLq5EPzumzrdLzuHrj-zuzI-nlyQtQ5V7gZauysm-wGwmpztRXfPc3AQ".to_string()
),
s3_access_key_id: "9156667efc2c70d89af6588da86d2924".to_string(),
s3_access_key_id: SerializableSecretString::from("9156667efc2c70d89af6588da86d2924".to_string()),
s3_secret_access_key: SerializableSecretString::from(
"ca833e890916d848c69135924bcd75e5909184814a0ebc6c988937ee094120d4".to_string()
),
Expand All @@ -99,7 +99,7 @@ pub mod destinations {
catalog_token: SerializableSecretString::from(
"eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IjJlOGQxZDNjN2MyMTJkOTU4ZmEyOGU2ZDhjZDEwYTMzIn0.eyJleHAiOjIwNzA3MTcxNjAsImlhdCI6MTc1NjE0NTE1MCwiaXNzIjoic3VwYWJhc2UiLCJyZWYiOiJ0c3JxcG9ubWxramloZ2ZlZGNiYSIsInJvbGUiOiJzZXJ2aWNlX3JvbGUifQ.UpdatedTokenSignatureForTesting".to_string()
),
s3_access_key_id: "updated9156667efc2c70d89af6588da86d2924".to_string(),
s3_access_key_id: SerializableSecretString::from("updated9156667efc2c70d89af6588da86d2924".to_string()),
s3_secret_access_key: SerializableSecretString::from(
"updatedca833e890916d848c69135924bcd75e5909184814a0ebc6c988937ee094120d4".to_string()
),
Expand Down
2 changes: 1 addition & 1 deletion etl-config/src/shared/destination.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ pub enum IcebergConfig {
/// Catalog authentication token
catalog_token: SerializableSecretString,
/// The S3 access key id
s3_access_key_id: String,
s3_access_key_id: SerializableSecretString,
/// The S3 secret access key
s3_secret_access_key: SerializableSecretString,
/// The S3 region
Expand Down
10 changes: 3 additions & 7 deletions etl-replicator/src/core.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ pub async fn start_replicator_with_config(
supabase_domain,
catalog_token.expose_secret().to_string(),
warehouse_name.clone(),
s3_access_key_id.clone(),
s3_access_key_id.expose_secret().to_string(),
s3_secret_access_key.expose_secret().to_string(),
s3_region.clone(),
);
Expand Down Expand Up @@ -160,18 +160,14 @@ fn log_destination_config(config: &DestinationConfig) {
project_ref,
catalog_token: _,
warehouse_name,
s3_access_key_id,
s3_access_key_id: _,
s3_secret_access_key: _,
s3_region,
},
} => {
debug!(
namespace,
project_ref,
warehouse_name,
s3_access_key_id,
s3_region,
"using Supabase iceberg destination config"
project_ref, warehouse_name, s3_region, "using Supabase iceberg destination config"
)
}
DestinationConfig::Iceberg {
Expand Down