diff --git a/keystorage/aws/src/main/java/tech/pegasys/signers/aws/AwsSecretsManager.java b/keystorage/aws/src/main/java/tech/pegasys/signers/aws/AwsSecretsManager.java index 92f6d754..7afe4a06 100644 --- a/keystorage/aws/src/main/java/tech/pegasys/signers/aws/AwsSecretsManager.java +++ b/keystorage/aws/src/main/java/tech/pegasys/signers/aws/AwsSecretsManager.java @@ -13,19 +13,31 @@ package tech.pegasys.signers.aws; import java.io.Closeable; +import java.util.Collection; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.BiFunction; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.model.Filter; +import software.amazon.awssdk.services.secretsmanager.model.FilterNameStringType; import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse; +import software.amazon.awssdk.services.secretsmanager.model.ListSecretsRequest; import software.amazon.awssdk.services.secretsmanager.model.ResourceNotFoundException; import software.amazon.awssdk.services.secretsmanager.model.SecretsManagerException; +import software.amazon.awssdk.services.secretsmanager.paginators.ListSecretsIterable; public class AwsSecretsManager implements Closeable { + private static final Logger LOG = LogManager.getLogger(); + private final SecretsManagerClient secretsManagerClient; private AwsSecretsManager(final SecretsManagerClient secretsManagerClient) { @@ -68,6 +80,60 @@ public Optional fetchSecret(final String secretName) { } } + private ListSecretsIterable listSecrets( + final Collection tagKeys, final Collection tagValues) { + final ListSecretsRequest.Builder listSecretsRequestBuilder = ListSecretsRequest.builder(); + if (!tagKeys.isEmpty()) { + listSecretsRequestBuilder.filters( + Filter.builder().key(FilterNameStringType.TAG_KEY).values(tagKeys).build()); + } + if (!tagValues.isEmpty()) { + listSecretsRequestBuilder.filters( + Filter.builder().key(FilterNameStringType.TAG_VALUE).values(tagValues).build()); + } + return secretsManagerClient.listSecretsPaginator(listSecretsRequestBuilder.build()); + } + + public Collection mapSecrets( + final Collection tagKeys, + final Collection tagValues, + final BiFunction mapper) { + final Set result = ConcurrentHashMap.newKeySet(); + listSecrets(tagKeys, tagValues) + .iterator() + .forEachRemaining( + listSecretsResponse -> { + listSecretsResponse + .secretList() + .parallelStream() + .forEach( + secretEntry -> { + try { + final Optional secretValue = fetchSecret(secretEntry.name()); + if (secretValue.isEmpty()) { + LOG.warn( + "Failed to fetch secret value '{}', and was discarded", + secretEntry.name()); + } else { + final R obj = mapper.apply(secretEntry.name(), secretValue.get()); + if (obj == null) { + LOG.warn( + "Mapped '{}' to a null object, and was discarded", + secretEntry.name()); + } else { + result.add(obj); + } + } + } catch (final Exception e) { + LOG.warn( + "Failed to map secret '{}' to requested object type.", + secretEntry.name()); + } + }); + }); + return result; + } + @Override public void close() { this.secretsManagerClient.close(); diff --git a/keystorage/aws/src/test/java/tech/pegasys/signers/aws/AwsSecretsManagerTest.java b/keystorage/aws/src/test/java/tech/pegasys/signers/aws/AwsSecretsManagerTest.java index d84dc3bd..35129ff7 100644 --- a/keystorage/aws/src/test/java/tech/pegasys/signers/aws/AwsSecretsManagerTest.java +++ b/keystorage/aws/src/test/java/tech/pegasys/signers/aws/AwsSecretsManagerTest.java @@ -15,10 +15,17 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -29,6 +36,7 @@ import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; import software.amazon.awssdk.services.secretsmanager.model.CreateSecretRequest; import software.amazon.awssdk.services.secretsmanager.model.DeleteSecretRequest; +import software.amazon.awssdk.services.secretsmanager.model.Tag; @TestInstance(TestInstance.Lifecycle.PER_CLASS) class AwsSecretsManagerTest { @@ -45,6 +53,9 @@ class AwsSecretsManagerTest { private AwsSecretsManager awsSecretsManagerInvalidCredentials; private SecretsManagerClient secretsManagerClient; private String secretName; + private String secretNamePrefix; + private List secretNames; + private AbstractMap secretTags; private static final String SECRET_VALUE = "{\"crypto\": {\"kdf\": {\"function\": \"scrypt\", \"params\": {\"dklen\": 32, \"n\": 262144, \"r\": 8, \"p\": 1, \"salt\": \"3d9b30b612f4f5e9423dc43c0490396798a179d35dd58d48dc1f5d6d42b07ab6\"}, \"message\": \"\"}, \"checksum\": {\"function\": \"sha256\", \"params\": {}, \"message\": \"c762b7453eab3332cda31d9dee1894cf541373617e591a8e7ab8f14f5830f723\"}, \"cipher\": {\"function\": \"aes-128-ctr\", \"params\": {\"iv\": \"095f79f6bb5daab60355ab6aa894b3c8\"}, \"message\": \"4ca342a769ec1c00d6a6d69e18cdf821f42849d4431da7df827b01ba162ed763\"}}, \"description\": \"\", \"pubkey\": \"8fb7c68f3291b8db46ef86a8b9544cad7052dd7cf817862063d1f151f3c443cd3907830b09a86fe0513f0e863beccf25\", \"path\": \"m/12381/3600/0/0/0\", \"uuid\": \"88fc9701-8670-4378-a3ba-00be25c1330c\", \"version\": 4}"; @@ -81,17 +92,9 @@ private void setupSecretsManagerClient() { .build(); } - private void createSecret() { - secretName = "signers-aws-integration/" + UUID.randomUUID(); - final CreateSecretRequest secretRequest = - CreateSecretRequest.builder().name(secretName).secretString(SECRET_VALUE).build(); - secretsManagerClient.createSecret(secretRequest); - } - - private void deleteSecret() { - final DeleteSecretRequest secretRequest = - DeleteSecretRequest.builder().secretId(secretName).build(); - secretsManagerClient.deleteSecret(secretRequest); + private void initializeVariables() { + secretNames = new ArrayList<>(); + secretTags = new HashMap(); } private void closeClients() { @@ -101,12 +104,70 @@ private void closeClients() { secretsManagerClient.close(); } + void createSecret( + final boolean multipleSecrets, final boolean multipleTags, final boolean sharedTag) { + secretNamePrefix = "signers-aws-integration/"; + secretName = secretNamePrefix + UUID.randomUUID(); + secretNames.add(secretName); + + final List tags = new ArrayList<>(); + + if (sharedTag) { + secretTags + .entrySet() + .forEach( + entry -> tags.add(Tag.builder().key(entry.getKey()).value(entry.getValue()).build())); + } else if (multipleTags) { + tags.add(Tag.builder().key(secretNamePrefix + UUID.randomUUID()).value(secretName).build()); + } + + tags.add(Tag.builder().key(secretNamePrefix + UUID.randomUUID()).value(secretName).build()); + + tags.forEach( + tag -> { + secretTags.put(tag.key(), tag.value()); + }); + + final CreateSecretRequest secretRequest = + CreateSecretRequest.builder() + .name(secretName) + .secretString(SECRET_VALUE) + .tags(tags) + .build(); + secretsManagerClient.createSecret(secretRequest); + + if (multipleSecrets) { + createSecret(false, multipleTags, sharedTag); + } + } + + @AfterEach + void deleteSecrets() { + secretNames.forEach( + name -> { + final DeleteSecretRequest secretRequest = + DeleteSecretRequest.builder().secretId(name).build(); + secretsManagerClient.deleteSecret(secretRequest); + }); + secretNames.clear(); + secretTags.clear(); + } + + private void validateMappedSecret( + final Collection> secretEntries, + final String secretName) { + final Optional> secretEntry = + secretEntries.stream().filter(e -> e.getKey().equals(secretName)).findAny(); + assertThat(secretEntry).isPresent(); + assertThat(secretEntry.get().getValue()).isEqualTo(SECRET_VALUE); + } + @BeforeAll void setup() { verifyEnvironmentVariables(); setupSecretsManagers(); setupSecretsManagerClient(); - createSecret(); + initializeVariables(); } @AfterAll @@ -114,25 +175,28 @@ void teardown() { if (awsSecretsManagerDefault != null || awsSecretsManagerExplicit != null || secretsManagerClient != null) { - deleteSecret(); + deleteSecrets(); closeClients(); } } @Test void fetchSecretWithDefaultManager() { + createSecret(false, false, false); Optional secret = awsSecretsManagerDefault.fetchSecret(secretName); assertThat(secret).hasValue(SECRET_VALUE); } @Test void fetchSecretWithExplicitManager() { + createSecret(false, false, false); Optional secret = awsSecretsManagerExplicit.fetchSecret(secretName); assertThat(secret).hasValue(SECRET_VALUE); } @Test void fetchSecretWithInvalidCredentialsReturnsEmpty() { + createSecret(false, false, false); assertThatExceptionOfType(RuntimeException.class) .isThrownBy(() -> awsSecretsManagerInvalidCredentials.fetchSecret(secretName)) .withMessageContaining("Failed to fetch secret from AWS Secrets Manager."); @@ -140,7 +204,122 @@ void fetchSecretWithInvalidCredentialsReturnsEmpty() { @Test void fetchingNonExistentSecretReturnsEmpty() { + createSecret(false, false, false); Optional secret = awsSecretsManagerDefault.fetchSecret("signers-aws-integration/empty"); assertThat(secret).isEmpty(); } + + @Test + void listAndMapSingleSecretWithSingleTag() { + createSecret(false, false, false); + + final Collection> secretEntries = + awsSecretsManagerExplicit.mapSecrets( + secretTags.keySet().stream().collect(Collectors.toList()), + secretTags.values().stream().collect(Collectors.toList()), + AbstractMap.SimpleEntry::new); + + secretNames.forEach(secretName -> validateMappedSecret(secretEntries, secretName)); + } + + @Test + void listAndMapSingleSecretWithMultipleTags() { + createSecret(false, true, false); + + final Collection> secretEntries = + awsSecretsManagerExplicit.mapSecrets( + secretTags.keySet().stream().collect(Collectors.toList()), + secretTags.values().stream().collect(Collectors.toList()), + AbstractMap.SimpleEntry::new); + + secretNames.forEach(secretName -> validateMappedSecret(secretEntries, secretName)); + } + + @Test + void listAndMapMultipleSecretsWithMultipleTags() { + createSecret(true, true, false); + + final Collection> secretEntries = + awsSecretsManagerExplicit.mapSecrets( + secretTags.keySet().stream().collect(Collectors.toList()), + secretTags.values().stream().collect(Collectors.toList()), + AbstractMap.SimpleEntry::new); + + secretNames.forEach(secretName -> validateMappedSecret(secretEntries, secretName)); + } + + @Test + void listAndMapMultipleSecretsWithSharedTags() { + createSecret(true, false, true); + + final Collection> secretEntries = + awsSecretsManagerExplicit.mapSecrets( + secretTags.keySet().stream().collect(Collectors.toList()), + secretTags.values().stream().collect(Collectors.toList()), + AbstractMap.SimpleEntry::new); + + secretNames.forEach(secretName -> validateMappedSecret(secretEntries, secretName)); + } + + @Test + void listAndMapMultipleSecretsWithMultipleAndSharedTags() { + createSecret(true, false, true); + createSecret(true, true, false); + + final Collection> secretEntries = + awsSecretsManagerExplicit.mapSecrets( + secretTags.keySet().stream().collect(Collectors.toList()), + secretTags.values().stream().collect(Collectors.toList()), + AbstractMap.SimpleEntry::new); + + secretNames.forEach(secretName -> validateMappedSecret(secretEntries, secretName)); + } + + @Test + void throwsAwayObjectsThatFailMapper() { + createSecret(true, false, false); + + final String failEntryName = secretNames.get(1); + + Collection> secretEntries = + awsSecretsManagerExplicit.mapSecrets( + secretTags.keySet().stream().collect(Collectors.toList()), + secretTags.values().stream().collect(Collectors.toList()), + (name, value) -> { + if (name.equals(failEntryName)) { + throw new RuntimeException("Arbitrary Failure"); + } + return new AbstractMap.SimpleEntry<>(name, value); + }); + + validateMappedSecret(secretEntries, secretNames.get(0)); + + final Optional> failEntry = + secretEntries.stream().filter(e -> e.getKey().equals(failEntryName)).findAny(); + assertThat(failEntry).isEmpty(); + } + + @Test + void throwsAwayObjectsWhichMapToNull() { + createSecret(true, false, false); + + final String nullEntryName = secretNames.get(1); + + Collection> secretEntries = + awsSecretsManagerExplicit.mapSecrets( + secretTags.keySet().stream().collect(Collectors.toList()), + secretTags.values().stream().collect(Collectors.toList()), + (name, value) -> { + if (name.equals(nullEntryName)) { + return null; + } + return new AbstractMap.SimpleEntry<>(name, value); + }); + + validateMappedSecret(secretEntries, secretNames.get(0)); + + final Optional> nullEntry = + secretEntries.stream().filter(e -> e.getKey().equals("MyBls")).findAny(); + assertThat(nullEntry).isEmpty(); + } }