Skip to content

Commit

Permalink
Support for AWS bulk loading (#141)
Browse files Browse the repository at this point in the history 
Implemented mapper function similar to Azure to support bulk loading of secrets from AWS Secrets Manager.

Supports issue Consensys/web3signer#499
  • Loading branch information
@georgep9
georgep9 committed May 4, 2022
1 parent de176bc commit f80804a
Show file tree
Hide file tree
Showing 2 changed files with 258 additions and 13 deletions.
66 changes: 66 additions & 0 deletions keystorage/aws/src/main/java/tech/pegasys/signers/aws/AwsSecretsManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,19 +13,31 @@
package tech.pegasys.signers.aws;

import java.io.Closeable;
import java.util.Collection;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.BiFunction;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;
import software.amazon.awssdk.services.secretsmanager.model.Filter;
import software.amazon.awssdk.services.secretsmanager.model.FilterNameStringType;
import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest;
import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse;
import software.amazon.awssdk.services.secretsmanager.model.ListSecretsRequest;
import software.amazon.awssdk.services.secretsmanager.model.ResourceNotFoundException;
import software.amazon.awssdk.services.secretsmanager.model.SecretsManagerException;
import software.amazon.awssdk.services.secretsmanager.paginators.ListSecretsIterable;

public class AwsSecretsManager implements Closeable {

private static final Logger LOG = LogManager.getLogger();

private final SecretsManagerClient secretsManagerClient;

private AwsSecretsManager(final SecretsManagerClient secretsManagerClient) {
Expand Down Expand Up @@ -68,6 +80,60 @@ public Optional<String> fetchSecret(final String secretName) {
}
}

private ListSecretsIterable listSecrets(
final Collection<String> tagKeys, final Collection<String> tagValues) {
final ListSecretsRequest.Builder listSecretsRequestBuilder = ListSecretsRequest.builder();
if (!tagKeys.isEmpty()) {
listSecretsRequestBuilder.filters(
Filter.builder().key(FilterNameStringType.TAG_KEY).values(tagKeys).build());
}
if (!tagValues.isEmpty()) {
listSecretsRequestBuilder.filters(
Filter.builder().key(FilterNameStringType.TAG_VALUE).values(tagValues).build());
}
return secretsManagerClient.listSecretsPaginator(listSecretsRequestBuilder.build());
}

public <R> Collection<R> mapSecrets(
final Collection<String> tagKeys,
final Collection<String> tagValues,
final BiFunction<String, String, R> mapper) {
final Set<R> result = ConcurrentHashMap.newKeySet();
listSecrets(tagKeys, tagValues)
.iterator()
.forEachRemaining(
listSecretsResponse -> {
listSecretsResponse
.secretList()
.parallelStream()
.forEach(
secretEntry -> {
try {
final Optional<String> secretValue = fetchSecret(secretEntry.name());
if (secretValue.isEmpty()) {
LOG.warn(
"Failed to fetch secret value '{}', and was discarded",
secretEntry.name());
} else {
final R obj = mapper.apply(secretEntry.name(), secretValue.get());
if (obj == null) {
LOG.warn(
"Mapped '{}' to a null object, and was discarded",
secretEntry.name());
} else {
result.add(obj);
}
}
} catch (final Exception e) {
LOG.warn(
"Failed to map secret '{}' to requested object type.",
secretEntry.name());
}
});
});
return result;
}

@Override
public void close() {
this.secretsManagerClient.close();
Expand Down
205 changes: 192 additions & 13 deletions keystorage/aws/src/test/java/tech/pegasys/signers/aws/AwsSecretsManagerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,17 @@
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;

import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;

import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
Expand All @@ -29,6 +36,7 @@
import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;
import software.amazon.awssdk.services.secretsmanager.model.CreateSecretRequest;
import software.amazon.awssdk.services.secretsmanager.model.DeleteSecretRequest;
import software.amazon.awssdk.services.secretsmanager.model.Tag;

@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class AwsSecretsManagerTest {
Expand All @@ -45,6 +53,9 @@ class AwsSecretsManagerTest {
private AwsSecretsManager awsSecretsManagerInvalidCredentials;
private SecretsManagerClient secretsManagerClient;
private String secretName;
private String secretNamePrefix;
private List<String> secretNames;
private AbstractMap<String, String> secretTags;

private static final String SECRET_VALUE =
"{\"crypto\": {\"kdf\": {\"function\": \"scrypt\", \"params\": {\"dklen\": 32, \"n\": 262144, \"r\": 8, \"p\": 1, \"salt\": \"3d9b30b612f4f5e9423dc43c0490396798a179d35dd58d48dc1f5d6d42b07ab6\"}, \"message\": \"\"}, \"checksum\": {\"function\": \"sha256\", \"params\": {}, \"message\": \"c762b7453eab3332cda31d9dee1894cf541373617e591a8e7ab8f14f5830f723\"}, \"cipher\": {\"function\": \"aes-128-ctr\", \"params\": {\"iv\": \"095f79f6bb5daab60355ab6aa894b3c8\"}, \"message\": \"4ca342a769ec1c00d6a6d69e18cdf821f42849d4431da7df827b01ba162ed763\"}}, \"description\": \"\", \"pubkey\": \"8fb7c68f3291b8db46ef86a8b9544cad7052dd7cf817862063d1f151f3c443cd3907830b09a86fe0513f0e863beccf25\", \"path\": \"m/12381/3600/0/0/0\", \"uuid\": \"88fc9701-8670-4378-a3ba-00be25c1330c\", \"version\": 4}";
Expand Down Expand Up @@ -81,17 +92,9 @@ private void setupSecretsManagerClient() {
.build();
}

private void createSecret() {
secretName = "signers-aws-integration/" + UUID.randomUUID();
final CreateSecretRequest secretRequest =
CreateSecretRequest.builder().name(secretName).secretString(SECRET_VALUE).build();
secretsManagerClient.createSecret(secretRequest);
}

private void deleteSecret() {
final DeleteSecretRequest secretRequest =
DeleteSecretRequest.builder().secretId(secretName).build();
secretsManagerClient.deleteSecret(secretRequest);
private void initializeVariables() {
secretNames = new ArrayList<>();
secretTags = new HashMap<String, String>();
}

private void closeClients() {
Expand All @@ -101,46 +104,222 @@ private void closeClients() {
secretsManagerClient.close();
}

void createSecret(
final boolean multipleSecrets, final boolean multipleTags, final boolean sharedTag) {
secretNamePrefix = "signers-aws-integration/";
secretName = secretNamePrefix + UUID.randomUUID();
secretNames.add(secretName);

final List<Tag> tags = new ArrayList<>();

if (sharedTag) {
secretTags
.entrySet()
.forEach(
entry -> tags.add(Tag.builder().key(entry.getKey()).value(entry.getValue()).build()));
} else if (multipleTags) {
tags.add(Tag.builder().key(secretNamePrefix + UUID.randomUUID()).value(secretName).build());
}

tags.add(Tag.builder().key(secretNamePrefix + UUID.randomUUID()).value(secretName).build());

tags.forEach(
tag -> {
secretTags.put(tag.key(), tag.value());
});

final CreateSecretRequest secretRequest =
CreateSecretRequest.builder()
.name(secretName)
.secretString(SECRET_VALUE)
.tags(tags)
.build();
secretsManagerClient.createSecret(secretRequest);

if (multipleSecrets) {
createSecret(false, multipleTags, sharedTag);
}
}

@AfterEach
void deleteSecrets() {
secretNames.forEach(
name -> {
final DeleteSecretRequest secretRequest =
DeleteSecretRequest.builder().secretId(name).build();
secretsManagerClient.deleteSecret(secretRequest);
});
secretNames.clear();
secretTags.clear();
}

private void validateMappedSecret(
final Collection<AbstractMap.SimpleEntry<String, String>> secretEntries,
final String secretName) {
final Optional<AbstractMap.SimpleEntry<String, String>> secretEntry =
secretEntries.stream().filter(e -> e.getKey().equals(secretName)).findAny();
assertThat(secretEntry).isPresent();
assertThat(secretEntry.get().getValue()).isEqualTo(SECRET_VALUE);
}

@BeforeAll
void setup() {
verifyEnvironmentVariables();
setupSecretsManagers();
setupSecretsManagerClient();
createSecret();
initializeVariables();
}

@AfterAll
void teardown() {
if (awsSecretsManagerDefault != null
|| awsSecretsManagerExplicit != null
|| secretsManagerClient != null) {
deleteSecret();
deleteSecrets();
closeClients();
}
}

@Test
void fetchSecretWithDefaultManager() {
createSecret(false, false, false);
Optional<String> secret = awsSecretsManagerDefault.fetchSecret(secretName);
assertThat(secret).hasValue(SECRET_VALUE);
}

@Test
void fetchSecretWithExplicitManager() {
createSecret(false, false, false);
Optional<String> secret = awsSecretsManagerExplicit.fetchSecret(secretName);
assertThat(secret).hasValue(SECRET_VALUE);
}

@Test
void fetchSecretWithInvalidCredentialsReturnsEmpty() {
createSecret(false, false, false);
assertThatExceptionOfType(RuntimeException.class)
.isThrownBy(() -> awsSecretsManagerInvalidCredentials.fetchSecret(secretName))
.withMessageContaining("Failed to fetch secret from AWS Secrets Manager.");
}

@Test
void fetchingNonExistentSecretReturnsEmpty() {
createSecret(false, false, false);
Optional<String> secret = awsSecretsManagerDefault.fetchSecret("signers-aws-integration/empty");
assertThat(secret).isEmpty();
}

@Test
void listAndMapSingleSecretWithSingleTag() {
createSecret(false, false, false);

final Collection<AbstractMap.SimpleEntry<String, String>> secretEntries =
awsSecretsManagerExplicit.mapSecrets(
secretTags.keySet().stream().collect(Collectors.toList()),
secretTags.values().stream().collect(Collectors.toList()),
AbstractMap.SimpleEntry::new);

secretNames.forEach(secretName -> validateMappedSecret(secretEntries, secretName));
}

@Test
void listAndMapSingleSecretWithMultipleTags() {
createSecret(false, true, false);

final Collection<AbstractMap.SimpleEntry<String, String>> secretEntries =
awsSecretsManagerExplicit.mapSecrets(
secretTags.keySet().stream().collect(Collectors.toList()),
secretTags.values().stream().collect(Collectors.toList()),
AbstractMap.SimpleEntry::new);

secretNames.forEach(secretName -> validateMappedSecret(secretEntries, secretName));
}

@Test
void listAndMapMultipleSecretsWithMultipleTags() {
createSecret(true, true, false);

final Collection<AbstractMap.SimpleEntry<String, String>> secretEntries =
awsSecretsManagerExplicit.mapSecrets(
secretTags.keySet().stream().collect(Collectors.toList()),
secretTags.values().stream().collect(Collectors.toList()),
AbstractMap.SimpleEntry::new);

secretNames.forEach(secretName -> validateMappedSecret(secretEntries, secretName));
}

@Test
void listAndMapMultipleSecretsWithSharedTags() {
createSecret(true, false, true);

final Collection<AbstractMap.SimpleEntry<String, String>> secretEntries =
awsSecretsManagerExplicit.mapSecrets(
secretTags.keySet().stream().collect(Collectors.toList()),
secretTags.values().stream().collect(Collectors.toList()),
AbstractMap.SimpleEntry::new);

secretNames.forEach(secretName -> validateMappedSecret(secretEntries, secretName));
}

@Test
void listAndMapMultipleSecretsWithMultipleAndSharedTags() {
createSecret(true, false, true);
createSecret(true, true, false);

final Collection<AbstractMap.SimpleEntry<String, String>> secretEntries =
awsSecretsManagerExplicit.mapSecrets(
secretTags.keySet().stream().collect(Collectors.toList()),
secretTags.values().stream().collect(Collectors.toList()),
AbstractMap.SimpleEntry::new);

secretNames.forEach(secretName -> validateMappedSecret(secretEntries, secretName));
}

@Test
void throwsAwayObjectsThatFailMapper() {
createSecret(true, false, false);

final String failEntryName = secretNames.get(1);

Collection<AbstractMap.SimpleEntry<String, String>> secretEntries =
awsSecretsManagerExplicit.mapSecrets(
secretTags.keySet().stream().collect(Collectors.toList()),
secretTags.values().stream().collect(Collectors.toList()),
(name, value) -> {
if (name.equals(failEntryName)) {
throw new RuntimeException("Arbitrary Failure");
}
return new AbstractMap.SimpleEntry<>(name, value);
});

validateMappedSecret(secretEntries, secretNames.get(0));

final Optional<AbstractMap.SimpleEntry<String, String>> failEntry =
secretEntries.stream().filter(e -> e.getKey().equals(failEntryName)).findAny();
assertThat(failEntry).isEmpty();
}

@Test
void throwsAwayObjectsWhichMapToNull() {
createSecret(true, false, false);

final String nullEntryName = secretNames.get(1);

Collection<AbstractMap.SimpleEntry<String, String>> secretEntries =
awsSecretsManagerExplicit.mapSecrets(
secretTags.keySet().stream().collect(Collectors.toList()),
secretTags.values().stream().collect(Collectors.toList()),
(name, value) -> {
if (name.equals(nullEntryName)) {
return null;
}
return new AbstractMap.SimpleEntry<>(name, value);
});

validateMappedSecret(secretEntries, secretNames.get(0));

final Optional<AbstractMap.SimpleEntry<String, String>> nullEntry =
secretEntries.stream().filter(e -> e.getKey().equals("MyBls")).findAny();
assertThat(nullEntry).isEmpty();
}
}

0 comments on commit f80804a

Please sign in to comment.