Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(aws): cache regions when parsing aws accounts #5532

Merged
merged 6 commits into from Sep 30, 2021
Merged

feat(aws): cache regions when parsing aws accounts #5532

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
10e4fb1
feat(aws): cache regions while loading aws accounts
apoorvmahajan Sep 15, 2021
da276a6
feat(aws): make aws account egionsCache concurrent. Also adds logic t…
apoorvmahajan Sep 20, 2021
36d699b
refactor(aws): rework how default regions are handled when aws accoun…
apoorvmahajan Sep 23, 2021
cddc464
refactor(aws): address review comments
apoorvmahajan Sep 24, 2021
0aa9a9e
feat(aws): increase default number of threads to use while loading aw…
apoorvmahajan Sep 24, 2021
c033676
Merge branch 'master' into aws_cache_regions
mergify[bot] Sep 30, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
241 changes: 151 additions & 90 deletions ...n/java/com/netflix/spinnaker/clouddriver/aws/security/config/AmazonCredentialsParser.java
View file
Open in desktop
Expand Up @@ -18,16 +18,18 @@
package com.netflix.spinnaker.clouddriver.aws.security.config;

import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.util.CollectionUtils;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Maps;
import com.netflix.spinnaker.clouddriver.aws.security.*;
import com.netflix.spinnaker.clouddriver.aws.security.config.AccountsConfiguration.Account;
import com.netflix.spinnaker.clouddriver.aws.security.config.CredentialsConfig.Region;
import com.netflix.spinnaker.credentials.definition.CredentialsParser;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.ConcurrentMap;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
Expand All @@ -46,8 +48,15 @@ public class AmazonCredentialsParser<
private final CredentialTranslator<V> credentialTranslator;
private final ObjectMapper objectMapper;
private final CredentialsConfig credentialsConfig;
private Lazy<List<Region>> defaultRegions;
private final AccountsConfiguration accountsConfig;
// this is used to cache all the regions found while parsing the accounts. This helps in
// reducing the number of API calls made since known regions are already cached.
private final ConcurrentMap<String, Region> regionCache;
private List<String> defaultRegionNames;

// this is a key used in the regions cache to indicate that default regions have been
// processed
private static final String DEFAULT_REGIONS_PROCESSED_KEY = "default_regions_processed";

public AmazonCredentialsParser(
AWSCredentialsProvider credentialsProvider,
Expand Down Expand Up @@ -75,88 +84,62 @@ public AmazonCredentialsParser(
this.objectMapper = new ObjectMapper();
this.credentialTranslator = findTranslator(credentialsType, this.objectMapper);
this.credentialsConfig = credentialsConfig;
this.defaultRegions = createDefaults(credentialsConfig.getDefaultRegions());
this.accountsConfig = accountsConfig;
this.regionCache = Maps.newConcurrentMap();
this.defaultRegionNames = new ArrayList<>();

// look in the credentials config to find default region names
if (!CollectionUtils.isNullOrEmpty(credentialsConfig.getDefaultRegions())) {
this.defaultRegionNames =
credentialsConfig.getDefaultRegions().stream()
.map(Region::getName)
.collect(Collectors.toList());
}
}

private Lazy<List<Region>> createDefaults(final List<Region> defaults) {
return new Lazy<>(
new Lazy.Loader<List<Region>>() {
@Override
public List<Region> get() {
if (defaults == null) {
return toRegion(awsAccountInfoLookup.listRegions());
} else {
List<Region> result = new ArrayList<>(defaults.size());
List<String> toLookup = new ArrayList<>();
for (Region def : defaults) {
if (def.getAvailabilityZones() == null || def.getAvailabilityZones().isEmpty()) {
toLookup.add(def.getName());
} else {
result.add(def);
}
}
if (!toLookup.isEmpty()) {
List<Region> resolved = toRegion(awsAccountInfoLookup.listRegions(toLookup));
for (Region region : resolved) {
Region fromDefault = find(defaults, region.getName());
if (fromDefault != null) {
region.setPreferredZones(fromDefault.getPreferredZones());
region.setDeprecated(fromDefault.getDeprecated());
}
}
result.addAll(resolved);
}
return result;
}
}
});
}

private List<Region> initRegions(Lazy<List<Region>> defaults, List<Region> toInit) {
if (toInit == null) {
// when accounts are parsed in a multi-threaded manner, the defaults.get() result could be the
// same for multiple
// accounts which do not have any regions set. Thus, when the account.setRegion() call is
// made, it could end
// up throwing a ConcurrentModificationException because at that point, the same object (i.e.
// default.get) will be
// attempted to be sorted for multiple accounts at a time. Hence, to get around it, we
// initialize it in a new
// list
return new ArrayList<>(defaults.get());
/**
* method to initialize the regions specified in an AWS account in the configuration.
*
* <p>Live call to get regions from the AWS API will be made if:
*
* <p>- An account's region does not have availability zones defined and that region doesn't exist
* in the region cache.
*/
private List<Region> initRegions(List<Region> toInit) {
// initialize regions cache if it hasn't been done already. We do this here and not in
// toInit.isNullOrEmpty() because we need the default region values if a region in toInit list
// has no availability zones specified.
initializeRegionsCacheWithDefaultRegions();

if (CollectionUtils.isNullOrEmpty(toInit)) {
return getRegionsFromCache(this.defaultRegionNames);
}

Map<String, Region> toInitByName =
toInit.stream().collect(Collectors.toMap(Region::getName, Function.identity()));

List<Region> result = new ArrayList<>(toInit.size());
List<String> toLookup = new ArrayList<>();
for (Region r : toInit) {
if (r.getAvailabilityZones() == null || r.getAvailabilityZones().isEmpty()) {
toLookup.add(r.getName());
for (Region region : toInit) {
// only attempt to lookup regions that don't have any availability zones set in the config
if (CollectionUtils.isNullOrEmpty(region.getAvailabilityZones())) {
Region fromCache = regionCache.get(region.getName());
// no need to lookup the region if it already exists in the cache
if (fromCache != null) {
result.add(fromCache);
} else {
toLookup.add(region.getName());
}
} else {
result.add(r);
result.add(region);
}
}

for (Iterator<String> lookups = toLookup.iterator(); lookups.hasNext(); ) {
List<Region> r = defaults.get();
String a = lookups.next();
Region fromDefault = find(r, a);
if (fromDefault != null) {
lookups.remove();
result.add(fromDefault);
}
}
// toLookup now contains the list of regions that we need to fetch from the cache and/or AWS API
if (!toLookup.isEmpty()) {
List<Region> resolved = toRegion(awsAccountInfoLookup.listRegions(toLookup));
List<Region> resolved = getRegionsFromCache(toLookup);
for (Region region : resolved) {
Region src = find(toInit, region.getName());
if (src == null || src.getPreferredZones() == null) {
src = find(defaults.get(), region.getName());
}

if (src != null) {
region.setPreferredZones(src.getPreferredZones());
}
Expand All @@ -178,6 +161,71 @@ private List<Region> initRegions(Lazy<List<Region>> defaults, List<Region> toIni
return result;
}

/**
* method to initialize the regions cache by processing the default regions which may have been
* specified in the configuration.
*
* <p>Live call to get regions from the AWS API will be made if:
*
* <p>1. no default regions exist in the config - in this case, it will fetch all AWS regions
*
* <p>2. default regions exist in the config but they don't have availability zones defined
*/
private void initializeRegionsCacheWithDefaultRegions() {
// synchronized block is added here to handle the multi-threading case where multiple threads
// may attempt to initialize the regions cache at the same time when it is empty in the
// beginning. This block will reduce the number of api calls made to look up regions
// by only allowing one of the threads to do that.
synchronized (this) {
if (!regionCache.containsKey(DEFAULT_REGIONS_PROCESSED_KEY)) {
// if there are no default regions specified, then fetch all the AWS regions.
if (defaultRegionNames.isEmpty()) {
log.info("No default regions specified in the configuration. Retrieving all the regions");
// save all the newly found regions in the cache
toRegion(awsAccountInfoLookup.listRegions())
.forEach(
region -> {
log.info("adding region: {} to regions cache", region.getName());
regionCache.putIfAbsent(region.getName(), region);
});
} else {
List<String> toLookup = new ArrayList<>();
for (Region region : credentialsConfig.getDefaultRegions()) {
log.info("Found default region: {} in the configuration", region.getName());
if (region.getAvailabilityZones() != null && !region.getAvailabilityZones().isEmpty()) {
log.info("Adding default region: {} to the regions cache", region.getName());
regionCache.put(region.getName(), region);
} else {
toLookup.add(region.getName());
}
}

if (!toLookup.isEmpty()) {
log.info("Fetching default regions: {}", toLookup);
List<AmazonCredentials.AWSRegion> newRegions =
awsAccountInfoLookup.listRegions(toLookup);

// save all the newly found regions in the cache
toRegion(newRegions)
.forEach(
region -> {
log.info("adding default region: {} to the regions cache", region.getName());
Region fromDefault =
find(credentialsConfig.getDefaultRegions(), region.getName());
if (fromDefault != null) {
region.setPreferredZones(fromDefault.getPreferredZones());
region.setDeprecated(fromDefault.getDeprecated());
}
regionCache.put(region.getName(), region);
});
}
}
// this helps us know that we have processed default regions. The value here doesn't matter.
regionCache.put(DEFAULT_REGIONS_PROCESSED_KEY, new Region());
}
}
}

private static Region find(List<Region> src, String name) {
if (src != null) {
for (Region r : src) {
Expand All @@ -189,6 +237,39 @@ private static Region find(List<Region> src, String name) {
return null;
}

private List<Region> getRegionsFromCache(final List<String> regionNames) {
// if no region names are provided, return everything from the cache except the
// DEFAULT_REGIONS_PROCESSED_KEY
if (regionNames.isEmpty()) {
return regionCache.entrySet().stream()
.filter(entry -> !entry.getKey().equals(DEFAULT_REGIONS_PROCESSED_KEY))
.map(Map.Entry::getValue)
.collect(Collectors.toList());
}

// determine if any regions are missing from the cache
List<String> cacheMisses = new ArrayList<>();
for (String region : regionNames) {
if (!regionCache.containsKey(region)) {
cacheMisses.add(region);
}
}

if (!cacheMisses.isEmpty()) {
List<AmazonCredentials.AWSRegion> newRegions;
log.info("Regions: {} do not exist in the regions cache", cacheMisses);
newRegions = awsAccountInfoLookup.listRegions(cacheMisses);
// save all the newly found regions in the cache
toRegion(newRegions)
.forEach(
region -> {
log.info("adding region: {} to regions cache", region.getName());
regionCache.putIfAbsent(region.getName(), region);
});
}
return regionNames.stream().map(regionCache::get).collect(Collectors.toList());
}

private static List<Region> toRegion(List<AmazonCredentials.AWSRegion> src) {
List<Region> result = new ArrayList<>(src.size());
for (AmazonCredentials.AWSRegion r : src) {
Expand Down Expand Up @@ -251,8 +332,8 @@ private V parseAccount(CredentialsConfig config, Account account) throws Throwab
account.setAccountType(account.getName());
}

log.info("Initializing regions for aws account: {}", account.getName());
account.setRegions(initRegions(defaultRegions, account.getRegions()));
log.info("Setting regions for aws account: {}", account.getName());
account.setRegions(initRegions(account.getRegions()));
account.setDefaultSecurityGroups(
account.getDefaultSecurityGroups() != null
? account.getDefaultSecurityGroups()
Expand Down Expand Up @@ -307,26 +388,6 @@ private V parseAccount(CredentialsConfig config, Account account) throws Throwab
return credentialTranslator.translate(credentialsProvider, account);
}

private static class Lazy<T> {
public static interface Loader<T> {
T get();
}

private final Loader<T> loader;
private final AtomicReference<T> ref = new AtomicReference<>();

public Lazy(Loader<T> loader) {
this.loader = loader;
}

public T get() {
if (ref.get() == null) {
ref.set(loader.get());
}
return ref.get();
}
}

private static String templateFirstNonNull(Map<String, String> substitutions, String... values) {
for (String value : values) {
if (value != null) {
Expand All @@ -341,7 +402,7 @@ static <T extends AmazonCredentials> CredentialTranslator<T> findTranslator(
return new CopyConstructorTranslator<>(objectMapper, credentialsType);
}

static interface CredentialTranslator<T extends AmazonCredentials> {
interface CredentialTranslator<T extends AmazonCredentials> {
Class<T> getCredentialType();

boolean resolveAccountId();
Expand Down
9 changes: 6 additions & 3 deletions ...rc/main/java/com/netflix/spinnaker/clouddriver/aws/security/config/CredentialsConfig.java
View file
Open in desktop
Expand Up @@ -149,10 +149,13 @@ public static class LoadAccounts {
* Only applicable when multiThreadingEnabled: true. This specifies the number of threads that
* should be used to load the aws accounts.
*
* <p>Adjust this number appropriately based on: - number of aws many accounts, - number of
* clouddriver pods (so that aws api calls are not rate-limited)
* <p>Adjust this number appropriately based on:
*
* <p>- number of aws many accounts
*
* <p>- number of clouddriver pods (so that aws api calls are not rate-limited)
*/
private int numberOfThreads = 5;
private int numberOfThreads = 15;

/**
* Only applicable when multiThreadingEnabled: true. This specifies the max amount of time for
Expand Down