feat(cfn/changeset): Introduce CFN change sets (#3731)

* feat(cfn/changeset): Introduce CFN change sets

This patch is a step towards fully supporting CloudFormation change
sets in Spinnaker. Specifically, this patch brings:
- Extend the deployCloudFormation operation with a flag indicating
wether it's a regular stack or a change set.
- If it's a change set, the create change set API call in AWS is
called.
- The on-demand cahing agent now only updates a single stack
if the stack is provided in the data map. This makes the
force cache refresh operation much more efficient by only
updating the relevant stack (instead of iterating over all
stacks in a given account/region).

* Include some tests for deploy change sets

clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/deploy/description/DeployCloudFormationDescription.java

@@ -15,6 +15,7 @@
  */
 package com.netflix.spinnaker.clouddriver.aws.deploy.description;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -32,4 +33,9 @@ public class DeployCloudFormationDescription extends AbstractAmazonCredentialsDe
   private Map<String, String> tags = new HashMap<>();
   private String region;
   private List<String> capabilities = new ArrayList<>();
+
+  @JsonProperty("isChangeSet")
+  private boolean isChangeSet;
+
+  private String changeSetName;
 }

clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/deploy/ops/DeployCloudFormationAtomicOperation.java

@@ -76,17 +76,35 @@ public Map operate(List priorOutputs) {
         description.getTags().entrySet().stream()
             .map(entry -> new Tag().withKey(entry.getKey()).withValue(entry.getValue()))
             .collect(Collectors.toList());
-    try {
-      String stackId =
-          createStack(
-              amazonCloudFormation, template, parameters, tags, description.getCapabilities());
-      return Collections.singletonMap("stackId", stackId);
-    } catch (AlreadyExistsException e) {
-      String stackId =
-          updateStack(
-              amazonCloudFormation, template, parameters, tags, description.getCapabilities());
-      return Collections.singletonMap("stackId", stackId);
+
+    boolean stackExists = stackExists(amazonCloudFormation);
+
+    String stackId;
+    if (description.isChangeSet()) {
+      ChangeSetType changeSetType = stackExists ? ChangeSetType.UPDATE : ChangeSetType.CREATE;
+      log.info("{} change set for stack: {}", changeSetType, description);
+      stackId =
+          createChangeSet(
+              amazonCloudFormation,
+              template,
+              parameters,
+              tags,
+              description.getCapabilities(),
+              changeSetType);
+    } else {
+      if (stackExists) {
+        log.info("Updating existing stack {}", description);
+        stackId =
+            updateStack(
+                amazonCloudFormation, template, parameters, tags, description.getCapabilities());
+      } else {
+        log.info("Creating new stack: {}", description);
+        stackId =
+            createStack(
+                amazonCloudFormation, template, parameters, tags, description.getCapabilities());
+      }
     }
+    return Collections.singletonMap("stackId", stackId);
   }
 
   private String createStack(
@@ -130,16 +148,57 @@ private String updateStack(
       return updateStackResult.getStackId();
     } catch (AmazonCloudFormationException e) {
       // No changes on the stack, ignore failure
-      return amazonCloudFormation
-          .describeStacks(new DescribeStacksRequest().withStackName(description.getStackName()))
-          .getStacks().stream()
-          .findFirst()
-          .orElseThrow(
-              () ->
-                  new IllegalArgumentException(
-                      "No CloudFormation Stack found with stack name "
-                          + description.getStackName()))
-          .getStackId();
+      return getStackId(amazonCloudFormation);
     }
   }
+
+  private String createChangeSet(
+      AmazonCloudFormation amazonCloudFormation,
+      String template,
+      List<Parameter> parameters,
+      List<Tag> tags,
+      List<String> capabilities,
+      ChangeSetType changeSetType) {
+    Task task = TaskRepository.threadLocalTask.get();
+    task.updateStatus(BASE_PHASE, "CloudFormation Stack exists. Creating a change set");
+    CreateChangeSetRequest createChangeSetRequest =
+        new CreateChangeSetRequest()
+            .withStackName(description.getStackName())
+            .withChangeSetName(description.getChangeSetName())
+            .withParameters(parameters)
+            .withTags(tags)
+            .withTemplateBody(template)
+            .withCapabilities(capabilities)
+            .withChangeSetType(changeSetType);
+    task.updateStatus(BASE_PHASE, "Uploading CloudFormation ChangeSet");
+    try {
+      CreateChangeSetResult createChangeSetResult =
+          amazonCloudFormation.createChangeSet(createChangeSetRequest);
+      return createChangeSetResult.getStackId();
+    } catch (AmazonCloudFormationException e) {
+      log.error("Error creating change set", e);
+      throw e;
+    }
+  }
+
+  private boolean stackExists(AmazonCloudFormation amazonCloudFormation) {
+    try {
+      getStackId(amazonCloudFormation);
+      return true;
+    } catch (Exception e) {
+      return false;
+    }
+  }
+
+  private String getStackId(AmazonCloudFormation amazonCloudFormation) {
+    return amazonCloudFormation
+        .describeStacks(new DescribeStacksRequest().withStackName(description.getStackName()))
+        .getStacks().stream()
+        .findFirst()
+        .orElseThrow(
+            () ->
+                new IllegalArgumentException(
+                    "No CloudFormation Stack found with stack name " + description.getStackName()))
+        .getStackId();
+  }
 }

clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/model/AmazonCloudFormationChangeSet.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2019 Adevinta.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.netflix.spinnaker.clouddriver.aws.model;
+
+import com.amazonaws.services.cloudformation.model.Change;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import java.util.List;
+
+@JsonInclude(JsonInclude.Include.NON_EMPTY)
+public class AmazonCloudFormationChangeSet {
+
+  private String name;
+  private String status;
+  private String statusReason;
+  private List<Change> changes;
+
+  public String getName() {
+    return name;
+  }
+
+  public String getStatus() {
+    return status;
+  }
+
+  public String getStatusReason() {
+    return statusReason;
+  }
+
+  public List<Change> getChanges() {
+    return changes;
+  }
+}

clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/model/AmazonCloudFormationStack.java

@@ -16,7 +16,9 @@
 package com.netflix.spinnaker.clouddriver.aws.model;
 
 import com.fasterxml.jackson.annotation.JsonInclude;
+import java.util.ArrayList;
 import java.util.Date;
+import java.util.List;
 import java.util.Map;
 
 @JsonInclude(JsonInclude.Include.NON_EMPTY)
@@ -31,6 +33,7 @@ public class AmazonCloudFormationStack {
   private String stackStatusReason;
   private String accountName;
   private String accountId;
+  private List<AmazonCloudFormationChangeSet> changeSets;
   private Date creationTime;
 
   public String getStackId() {
@@ -69,6 +72,10 @@ public String getStackStatusReason() {
     return stackStatusReason;
   }
 
+  public List<AmazonCloudFormationChangeSet> getChangeSets() {
+    return (changeSets == null) ? null : new ArrayList<>(changeSets);
+  }
+
   public Date getCreationTime() {
     return creationTime;
   }

clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/provider/agent/AmazonCloudFormationCachingAgent.java

@@ -22,11 +22,7 @@
 import com.amazonaws.services.cloudformation.model.*;
 import com.amazonaws.services.cloudformation.model.Stack;
 import com.netflix.spectator.api.Registry;
-import com.netflix.spinnaker.cats.agent.AccountAware;
-import com.netflix.spinnaker.cats.agent.AgentDataType;
-import com.netflix.spinnaker.cats.agent.CacheResult;
-import com.netflix.spinnaker.cats.agent.CachingAgent;
-import com.netflix.spinnaker.cats.agent.DefaultCacheResult;
+import com.netflix.spinnaker.cats.agent.*;
 import com.netflix.spinnaker.cats.cache.CacheData;
 import com.netflix.spinnaker.cats.cache.DefaultCacheData;
 import com.netflix.spinnaker.cats.provider.ProviderCache;
@@ -42,7 +38,8 @@
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
-public class AmazonCloudFormationCachingAgent implements CachingAgent, OnDemandAgent, AccountAware {
+public class AmazonCloudFormationCachingAgent
+    implements CachingAgent, OnDemandAgent, AccountAware, AgentIntervalAware {
   private final AmazonClientProvider amazonClientProvider;
   private final NetflixAmazonCredentials account;
   private final String region;
@@ -93,8 +90,14 @@ public OnDemandResult handle(ProviderCache providerCache, Map<String, ?> data) {
           "Updating CloudFormation cache for account: {} and region: {}",
           account.getName(),
           this.region);
+      DescribeStacksRequest describeStacksRequest =
+          Optional.ofNullable((String) data.get("stackName"))
+              .map(stackName -> new DescribeStacksRequest().withStackName(stackName))
+              .orElse(new DescribeStacksRequest());
       return new OnDemandResult(
-          getOnDemandAgentType(), loadData(providerCache), Collections.emptyMap());
+          getOnDemandAgentType(),
+          queryStacks(providerCache, describeStacksRequest, true),
+          Collections.emptyMap());
     } else {
       return null;
     }
@@ -133,40 +136,24 @@ public Collection<AgentDataType> getProvidedDataTypes() {
 
   @Override
   public CacheResult loadData(ProviderCache providerCache) {
-    log.info("Describing items in {}", getAgentType());
+    return queryStacks(providerCache, new DescribeStacksRequest(), false);
+  }
+
+  public CacheResult queryStacks(
+      ProviderCache providerCache,
+      DescribeStacksRequest describeStacksRequest,
+      boolean isPartialResult) {
+    log.info("Describing items in {}, partial result: {}", getAgentType(), isPartialResult);
     AmazonCloudFormation cloudformation =
         amazonClientProvider.getAmazonCloudFormation(account, region);
 
-    Collection<CacheData> stackCacheData = new ArrayList<>();
+    ArrayList<CacheData> stackCacheData = new ArrayList<>();
 
     try {
-      List<Stack> stacks = cloudformation.describeStacks().getStacks();
+      List<Stack> stacks = cloudformation.describeStacks(describeStacksRequest).getStacks();
 
       for (Stack stack : stacks) {
-        Map<String, Object> stackAttributes = new HashMap<>();
-        stackAttributes.put("stackId", stack.getStackId());
-        stackAttributes.put(
-            "tags", stack.getTags().stream().collect(Collectors.toMap(Tag::getKey, Tag::getValue)));
-        stackAttributes.put(
-            "outputs",
-            stack.getOutputs().stream()
-                .collect(Collectors.toMap(Output::getOutputKey, Output::getOutputValue)));
-        stackAttributes.put("stackName", stack.getStackName());
-        stackAttributes.put("region", region);
-        stackAttributes.put("accountName", account.getName());
-        stackAttributes.put("accountId", account.getAccountId());
-        stackAttributes.put("stackStatus", stack.getStackStatus());
-        stackAttributes.put("creationTime", stack.getCreationTime());
-
-        if (stack.getStackStatus().endsWith("ROLLBACK_COMPLETE")) {
-          DescribeStackEventsRequest request =
-              new DescribeStackEventsRequest().withStackName(stack.getStackName());
-          cloudformation.describeStackEvents(request).getStackEvents().stream()
-              .filter(e -> e.getResourceStatus().endsWith("FAILED"))
-              .findFirst()
-              .map(StackEvent::getResourceStatusReason)
-              .map(statusReason -> stackAttributes.put("stackStatusReason", statusReason));
-        }
+        Map<String, Object> stackAttributes = getStackAttributes(stack, cloudformation);
         String stackCacheKey =
             Keys.getCloudFormationKey(stack.getStackId(), region, account.getName());
         Map<String, Collection<String>> relationships = new HashMap<>();
@@ -180,6 +167,74 @@ public CacheResult loadData(ProviderCache providerCache) {
     log.info("Caching {} items in {}", stackCacheData.size(), getAgentType());
     HashMap<String, Collection<CacheData>> result = new HashMap<>();
     result.put(STACKS.getNs(), stackCacheData);
-    return new DefaultCacheResult(result);
+    return new DefaultCacheResult(result, isPartialResult);
+  }
+
+  private Map<String, Object> getStackAttributes(Stack stack, AmazonCloudFormation cloudformation) {
+    Map<String, Object> stackAttributes = new HashMap<>();
+    stackAttributes.put("stackId", stack.getStackId());
+    stackAttributes.put(
+        "tags", stack.getTags().stream().collect(Collectors.toMap(Tag::getKey, Tag::getValue)));
+    stackAttributes.put(
+        "outputs",
+        stack.getOutputs().stream()
+            .collect(Collectors.toMap(Output::getOutputKey, Output::getOutputValue)));
+    stackAttributes.put("stackName", stack.getStackName());
+    stackAttributes.put("region", region);
+    stackAttributes.put("accountName", account.getName());
+    stackAttributes.put("accountId", account.getAccountId());
+    stackAttributes.put("stackStatus", stack.getStackStatus());
+    stackAttributes.put("creationTime", stack.getCreationTime());
+    stackAttributes.put("changeSets", getChangeSets(stack, cloudformation));
+    getStackStatusReason(stack, cloudformation)
+        .map(statusReason -> stackAttributes.put("stackStatusReason", statusReason));
+    return stackAttributes;
+  }
+
+  private List<Map<String, Object>> getChangeSets(
+      Stack stack, AmazonCloudFormation cloudformation) {
+    ListChangeSetsRequest listChangeSetsRequest =
+        new ListChangeSetsRequest().withStackName(stack.getStackName());
+    ListChangeSetsResult listChangeSetsResult =
+        cloudformation.listChangeSets(listChangeSetsRequest);
+    return listChangeSetsResult.getSummaries().stream()
+        .map(
+            summary -> {
+              Map<String, Object> changeSetAttributes = new HashMap<>();
+              changeSetAttributes.put("name", summary.getChangeSetName());
+              changeSetAttributes.put("status", summary.getStatus());
+              changeSetAttributes.put("statusReason", summary.getStatusReason());
+              DescribeChangeSetRequest describeChangeSetRequest =
+                  new DescribeChangeSetRequest()
+                      .withChangeSetName(summary.getChangeSetName())
+                      .withStackName(stack.getStackName());
+              DescribeChangeSetResult describeChangeSetResult =
+                  cloudformation.describeChangeSet(describeChangeSetRequest);
+              changeSetAttributes.put("changes", describeChangeSetResult.getChanges());
+              log.debug(
+                  "Adding change set attributes for stack {}: {}",
+                  stack.getStackName(),
+                  changeSetAttributes);
+              return changeSetAttributes;
+            })
+        .collect(Collectors.toList());
+  }
+
+  private Optional<String> getStackStatusReason(Stack stack, AmazonCloudFormation cloudformation) {
+    if (stack.getStackStatus().endsWith("ROLLBACK_COMPLETE")) {
+      DescribeStackEventsRequest request =
+          new DescribeStackEventsRequest().withStackName(stack.getStackName());
+      return cloudformation.describeStackEvents(request).getStackEvents().stream()
+          .filter(e -> e.getResourceStatus().endsWith("FAILED"))
+          .findFirst()
+          .map(StackEvent::getResourceStatusReason);
+    } else {
+      return Optional.empty();
+    }
+  }
+
+  @Override
+  public Long getAgentInterval() {
+    return 60000L;
   }
 }

clouddriver-aws/src/test/groovy/com/netflix/spinnaker/clouddriver/aws/deploy/converters/DeployCloudFormationAtomicOperationConverterSpec.groovy

@@ -51,7 +51,9 @@ class DeployCloudFormationAtomicOperationConverterSpec extends Specification {
                  tags           : [ tag1: "tag1" ],
                  capabilities   : [ "cap1", "cap2" ],
                  region         : "eu-west_1",
-                 credentials    : "credentials"]
+                 credentials    : "credentials",
+                 isChangeSet    : true,
+                 changeSetName  : "changeSetName"]
 
     when:
     def description = converter.convertDescription(input)