spinnaker-plugins · nimakaviani · Oct 22, 2020 · Oct 20, 2020 · Oct 22, 2020
diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml
@@ -20,10 +20,10 @@ jobs:
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
           restore-keys: |
             ${{ runner.os }}-gradle-
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - name: Grant execute permission for gradlew
         run: chmod +x gradlew
       - name: Run unit tests

diff --git a/README.md b/README.md
@@ -54,6 +54,10 @@ accountProvision:
   syncAgentFrequencyInMilliSeconds: 10000 # How often agent scheduler should run.
   iamAuth: false # Enable IAM authentication for API Gateway.
   iamAuthRegion: 'us-west-2' # Specify which region API Gateway is deployed. Required if `iamAuth` is enabled.
+  connectionTimeout: 2000 # How long to wait before initial connection timeouts
+  readTimeout: 6000 # How long to wait for remote server to return results.
+  maxBackoffTime: 3600000 # How long, in milli seconds, maximum backoff time should be.
+
 ```
 
 

diff --git a/account-registration/account-registration.gradle b/account-registration/account-registration.gradle
@@ -39,13 +39,14 @@ dependencies {
   compileOnly (group: 'com.netflix.spinnaker.kork', name: 'kork-plugins-spring-api', version: "${korkVersion}")
   compileOnly (group: 'org.springframework', name: 'spring-web', version: '5.2.4.RELEASE')
   compileOnly (group: 'com.netflix.spinnaker.orca', name: 'orca-core', version: "${orcaVersion}")
-  compileOnly 'com.netflix.spinnaker.clouddriver:clouddriver-api:5.67.0'
-  compileOnly 'com.netflix.spinnaker.clouddriver:clouddriver-aws:5.67.0'
-  compileOnly 'com.netflix.spinnaker.clouddriver:clouddriver-ecs:5.67.0'
-  compileOnly 'com.netflix.spinnaker.clouddriver:cats-core:5.67.0'
-  compileOnly 'com.netflix.spinnaker.clouddriver:clouddriver-security:5.67.0'
-  compileOnly 'com.netflix.spinnaker.clouddriver:clouddriver-eureka:5.67.0'
-  compileOnly 'com.netflix.spinnaker.clouddriver:clouddriver-core:5.67.0'
+  compileOnly "com.netflix.spinnaker.clouddriver:clouddriver-api:${clouddriverVersion}"
+  compileOnly "com.netflix.spinnaker.clouddriver:clouddriver-aws:${clouddriverVersion}"
+  compileOnly "com.netflix.spinnaker.clouddriver:clouddriver-ecs:${clouddriverVersion}"
+  compileOnly "com.netflix.spinnaker.clouddriver:clouddriver-lambda:${clouddriverVersion}"
+  compileOnly "com.netflix.spinnaker.clouddriver:cats-core:${clouddriverVersion}"
+  compileOnly "com.netflix.spinnaker.clouddriver:clouddriver-security:${clouddriverVersion}"
+  compileOnly "com.netflix.spinnaker.clouddriver:clouddriver-eureka:${clouddriverVersion}"
+  compileOnly "com.netflix.spinnaker.clouddriver:clouddriver-core:${clouddriverVersion}"
   compileOnly 'com.netflix.spinnaker.fiat:fiat-core:1.22.0'
   compileOnly 'com.amazonaws:aws-java-sdk:1.11.802'
   compileOnly 'org.projectlombok:lombok:+'
@@ -54,13 +55,13 @@ dependencies {
   annotationProcessor("org.pf4j:pf4j:$pf4jVersion")
 
   testImplementation (group: 'com.netflix.spinnaker.orca', name: 'orca-core', version: "${orcaVersion}")
-  testImplementation 'com.netflix.spinnaker.clouddriver:clouddriver-api:5.67.0'
-  testImplementation 'com.netflix.spinnaker.clouddriver:clouddriver-aws:5.67.0'
-  testImplementation 'com.netflix.spinnaker.clouddriver:clouddriver-ecs:5.67.0'
-  testImplementation 'com.netflix.spinnaker.clouddriver:cats-core:5.67.0'
-  testImplementation 'com.netflix.spinnaker.clouddriver:clouddriver-security:5.67.0'
-  testImplementation 'com.netflix.spinnaker.clouddriver:clouddriver-eureka:5.67.0'
-  testImplementation 'com.netflix.spinnaker.clouddriver:clouddriver-core:5.67.0'
+  testImplementation "com.netflix.spinnaker.clouddriver:clouddriver-api:${clouddriverVersion}"
+  testImplementation "com.netflix.spinnaker.clouddriver:clouddriver-aws:${clouddriverVersion}"
+  testImplementation "com.netflix.spinnaker.clouddriver:clouddriver-ecs:${clouddriverVersion}"
+  testImplementation "com.netflix.spinnaker.clouddriver:cats-core:${clouddriverVersion}"
+  testImplementation "com.netflix.spinnaker.clouddriver:clouddriver-security:${clouddriverVersion}"
+  testImplementation "com.netflix.spinnaker.clouddriver:clouddriver-eureka:${clouddriverVersion}"
+  testImplementation "com.netflix.spinnaker.clouddriver:clouddriver-core:${clouddriverVersion}"
   testImplementation 'com.netflix.spinnaker.fiat:fiat-core:1.22.0'
   testImplementation "org.junit.jupiter:junit-jupiter-api:5.3.2"
   testImplementation group: 'io.strikt', name: 'strikt-core', version: '0.22.1'

diff --git a/...tration/src/main/java/com/amazon/aws/spinnaker/plugin/registration/AccountPagination.java b/...tration/src/main/java/com/amazon/aws/spinnaker/plugin/registration/AccountPagination.java
@@ -23,5 +23,5 @@
 @Data
 public class AccountPagination {
     @JsonProperty("NextUrl")
-    private String nextUrl;
+    private String nextUrl = "";
 }
diff --git a/...gistration/src/main/java/com/amazon/aws/spinnaker/plugin/registration/AccountsStatus.java b/...gistration/src/main/java/com/amazon/aws/spinnaker/plugin/registration/AccountsStatus.java
@@ -37,11 +37,13 @@
 import org.springframework.web.client.RestTemplate;
 import org.springframework.web.util.UriComponentsBuilder;
 
+import java.time.Duration;
 import java.time.Instant;
 import java.time.OffsetDateTime;
 import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeParseException;
 import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
 
 @Slf4j
 @Data
@@ -56,6 +58,10 @@ public class AccountsStatus {
     private boolean iamAuth;
     @Value("${accountProvision.iamAuthRegion:us-west-2}")
     private String region;
+    @Value("${accountProvision.maxBackoffTime:3600000}")
+    private long maxBackoffTime;
+    private AtomicInteger retryCount = new AtomicInteger(0);
+    private Instant nextTry;
     private RestTemplate restTemplate;
     private final CredentialsConfig credentialsConfig;
     private ECSCredentialsConfig ecsCredentialsConfig;
@@ -64,11 +70,17 @@ public class AccountsStatus {
     @Autowired
     AccountsStatus(
             CredentialsConfig credentialsConfig,
-            @Value("${accountProvision.url:http://localhost:8080}") String url
+            @Value("${accountProvision.url:http://localhost:8080}") String url,
+            @Value("${accountProvision.connectionTimeout:2000}") Long connectionTimeout,
+            @Value("${accountProvision.readTimeout:6000}") Long readTimeout
     ) {
         this.credentialsConfig = credentialsConfig;
         this.remoteHostUrl = url;
-        this.restTemplate = new RestTemplateBuilder().interceptors(new PlusEncoderInterceptor()).build();
+        this.restTemplate = new RestTemplateBuilder()
+                .interceptors(new PlusEncoderInterceptor())
+                .setConnectTimeout(Duration.ofMillis(connectionTimeout))
+                .setReadTimeout(Duration.ofMillis(connectionTimeout))
+                .build();
     }
 
     @Autowired(required = false)
@@ -85,17 +97,29 @@ public List<ECSCredentialsConfig.Account> getECSAccountsAsList() {
     }
 
     public boolean getDesiredAccounts() {
+        if (nextTry != null && Instant.now().isBefore(nextTry)) {
+            log.debug("In backoff time. Will not attempt to retrieve accounts.");
+            return false;
+        }
         if (lastSyncTime != null) {
             log.info("Last time synced with remote host is: {}", lastSyncTime);
         } else {
             log.info("Last sync time is not set. Will perform a full sync.");
         }
-        Response response = getResourceFromRemoteHost(remoteHostUrl);
+        Response response = null;
+        try {
+            response = getResourceFromRemoteHost(remoteHostUrl);
+        } catch (Exception e) {
+            log.error("Could not get account information from remote host.", e);
+            setBackoffTime();
+            return false;
+        }
         if (response == null) {
+            setBackoffTime();
             return false;
         }
-        String nextUrl = response.getPagination().getNextUrl();
-        if (!"".equals(nextUrl)) {
+        if (response.getPagination() != null && !"".equals(response.getPagination().getNextUrl())) {
+            String nextUrl = response.getPagination().getNextUrl();
             List<Account> accounts = response.getAccounts();
             while (nextUrl != null && !"".equals(nextUrl)) {
                 log.info("Calling next URL, {}", nextUrl);
@@ -194,13 +218,16 @@ private Response getResourceFromRemoteHost(String url) {
         }
         if (response.getAccounts() == null || response.getAccounts().isEmpty()) {
             log.info("No accounts returned from remote host.");
-            return null;
+            response.setAccounts(Collections.emptyList());
+            return response;
         }
         log.info("Received a valid response from remote host.");
         return response;
     }
 
     public void markSynced() {
+        this.retryCount.set(0);
+        this.nextTry = null;
         this.lastSyncTime = this.lastAttemptedTIme;
     }
 
@@ -212,21 +239,7 @@ private Response getResourceFromApiGateway(String url) {
                 return null;
             }
         }
-        try {
-            return callApiGateway(url);
-        } catch (HttpClientErrorException e) {
-            if (HttpStatus.FORBIDDEN == e.getStatusCode()) {
-                log.error(e.getMessage());
-                log.info("Received 403 from API Gateway. Retrying..");
-                makeHeaderGenerator(url);
-                if (this.headerGenerator == null) {
-                    log.error("Failed to generate resources required for AWS Signature V4 to authenticate with API Gateway.");
-                    return null;
-                }
-                return callApiGateway(url);
-            }
-            throw e;
-        }
+        return callApiGateway(url);
     }
 
     private void makeHeaderGenerator(String url) {
@@ -254,6 +267,30 @@ private Response getResources(String url) {
     }
 
     private Response callApiGateway(String url) {
+        int retry = 0;
+        while (retry <= 1) {
+            try{
+                return doCallApiGateway(url);
+            } catch (Exception e) {
+                if (e instanceof HttpClientErrorException) {
+                    HttpClientErrorException ex = (HttpClientErrorException) e;
+                    if (HttpStatus.FORBIDDEN == ex.getStatusCode()) {
+                        log.error(e.getMessage());
+                        log.info("Received 403 from API Gateway.");
+                        makeHeaderGenerator(url);
+                        if (this.headerGenerator == null) {
+                            log.error("Failed to generate resources required for AWS Signature V4 to authenticate with API Gateway.");
+                        }
+                    }
+                }
+                log.error("Error encountered while calling remote host: {}", e.getMessage());
+            }
+            retry += 1;
+        }
+        return null;
+    }
+
+    private Response doCallApiGateway(String url) {
         UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(url);
         HashMap<String, List<String>> queryStrings = new HashMap<>();
         for (Map.Entry<String, List<String>> entry : builder.build().getQueryParams().entrySet()) {
@@ -283,7 +320,7 @@ private Response callApiGateway(String url) {
     }
 
     private String findMostRecentTime(Response response) {
-        List<Instant> instants = new ArrayList();
+        List<Instant> instants = new ArrayList<>();
         HashMap<Instant, String> map = new HashMap<>();
         DateTimeFormatter timeFormatter = DateTimeFormatter.ISO_DATE_TIME;
         for (Account account : response.getAccounts()) {
@@ -307,4 +344,16 @@ private String findMostRecentTime(Response response) {
         log.debug("Most recent timestamp is {}", oldest.toString());
         return map.get(oldest);
     }
+
+    private void setBackoffTime() {
+        int count = retryCount.getAndIncrement();
+        long waitTime = (long) Math.pow(2, count) * 1000L;
+        if (waitTime > maxBackoffTime) {
+            waitTime = maxBackoffTime;
+        }
+        Random random = new Random();
+        long randWait = random.nextInt(10) * 100L;
+        nextTry = Instant.now().plusMillis(waitTime - randWait);
+        log.info("Next try: {}", nextTry.toString());
+    }
 }
diff --git a/...ain/java/com/amazon/aws/spinnaker/plugin/registration/AmazonAWSCachingAgentScheduler.java b/...ain/java/com/amazon/aws/spinnaker/plugin/registration/AmazonAWSCachingAgentScheduler.java
@@ -32,7 +32,9 @@
 import org.springframework.scheduling.annotation.Scheduled;
 
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
 
 public class AmazonAWSCachingAgentScheduler {
@@ -49,6 +51,7 @@ public class AmazonAWSCachingAgentScheduler {
     private final Optional<ExecutorService> reservationReportPool;
     private final Collection<AgentProvider> agentProviders;
     private final DynamicConfigService dynamicConfigService;
+    private final Set<String> publicRegions = new HashSet<>();
 
     @Autowired
     private AmazonAWSCachingAgentScheduler(LazyLoadCredentialsRepository lazyLoadCredentialsRepository,
@@ -76,8 +79,10 @@ private AmazonAWSCachingAgentScheduler(LazyLoadCredentialsRepository lazyLoadCre
 
     @Scheduled(fixedDelayString = "${accountProvision.syncAgentFrequencyInMilliSeconds:10000}")
     public void synchronizeAwsProvider() {
-        AmazonProviderUtils.synchronizeAwsProvider(awsProvider, amazonCloudProvider, amazonClientProvider,
+        Set<String> regions =  AmazonProviderUtils.synchronizeAwsProvider(awsProvider, amazonCloudProvider, amazonClientProvider,
                 amazonS3DataProvider, lazyLoadCredentialsRepository, objectMapper, eddaApiFactory, ctx,
-                registry, reservationReportPool, agentProviders, eddaTimeoutConfig, dynamicConfigService);
+                registry, reservationReportPool, agentProviders, eddaTimeoutConfig, dynamicConfigService,
+                publicRegions);
+        publicRegions.addAll(regions);
     }
 }
diff --git a/...src/main/java/com/amazon/aws/spinnaker/plugin/registration/AmazonPollingSynchronizer.java b/...src/main/java/com/amazon/aws/spinnaker/plugin/registration/AmazonPollingSynchronizer.java
@@ -83,7 +83,7 @@ void schedule() throws Throwable {
         sync();
     }
 
-    void sync() throws Throwable {
+    synchronized void sync() throws Throwable {
         log.debug("Checking remote host for account updates.");
         if (!accountsStatus.getDesiredAccounts()) {
             log.debug("Nothing to do.");