feat(amazon): allow preferSourceCapacity as fallback on deploy when u…

…seSourceCapacity does not find a source (#1545)

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/pipeline/providers/aws/CaptureSourceServerGroupCapacityTask.groovy

@@ -40,28 +40,35 @@ class CaptureSourceServerGroupCapacityTask implements Task {
     def stageOutputs = [:]
     StageData stageData = stage.mapTo(StageData)
     if (stageData.useSourceCapacity) {
-      def sourceServerGroup = oortHelper.getTargetServerGroup(
-        stageData.source.account,
-        stageData.source.asgName,
-        stageData.source.region,
-        stageData.cloudProvider ?: stageData.providerType
-      ).orElse(null)
-
-      if (sourceServerGroup) {
-        // capture the source server group's capacity AND specify an explicit capacity to use when deploying the next
-        // server group (ie. no longer use source capacity)
+      if (!stageData.source && stageData.preferSourceCapacity) {
         stageData.setUseSourceCapacity(false)
-        stageData.source.useSourceCapacity = false
         stageOutputs = [
-          useSourceCapacity                : false,
-          source                           : stageData.source,
-          sourceServerGroupCapacitySnapshot: sourceServerGroup.capacity,
-          capacity                         : [
-            min    : sourceServerGroup.capacity.desired,
-            desired: sourceServerGroup.capacity.desired,
-            max    : sourceServerGroup.capacity.max
-          ]
+          useSourceCapacity: false
         ]
+      } else {
+        def sourceServerGroup = oortHelper.getTargetServerGroup(
+          stageData.source.account,
+          stageData.source.asgName,
+          stageData.source.region,
+          stageData.cloudProvider ?: stageData.providerType
+        ).orElse(null)
+
+        if (sourceServerGroup) {
+          // capture the source server group's capacity AND specify an explicit capacity to use when deploying the next
+          // server group (ie. no longer use source capacity)
+          stageData.setUseSourceCapacity(false)
+          stageData.source.useSourceCapacity = false
+          stageOutputs = [
+            useSourceCapacity                : false,
+            source                           : stageData.source,
+            sourceServerGroupCapacitySnapshot: sourceServerGroup.capacity,
+            capacity                         : [
+              min    : sourceServerGroup.capacity.desired,
+              desired: sourceServerGroup.capacity.desired,
+              max    : sourceServerGroup.capacity.max
+            ]
+          ]
+        }
       }
     }
 

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/kato/pipeline/strategy/DetermineSourceServerGroupTask.groovy

@@ -47,6 +47,7 @@ class DetermineSourceServerGroupTask implements RetryableTask {
   @Override
   TaskResult execute(Stage stage) {
     def stageData = stage.mapTo(StageData)
+    Boolean isNotFound = false
     if (!stageData.source && !stageData.region && !stageData.availabilityZones) {
       throw new IllegalStateException("No 'source' or 'region' or 'availabilityZones' in stage context")
     }
@@ -55,6 +56,7 @@ class DetermineSourceServerGroupTask implements RetryableTask {
       def source = sourceResolver.getSource(stage)
       Boolean useSourceCapacity = useSourceCapacity(stage, source)
       if (useSourceCapacity && !source) {
+        isNotFound = true
         throw new IllegalStateException( "Cluster is configured to copy capacity from the current server group, " +
           "but no server group was found for the cluster '${stageData.cluster}' in " +
           "${stageData.account}/${stageData.availabilityZones?.keySet()?.getAt(0)}. If this is a new cluster, you must " +
@@ -76,9 +78,6 @@ class DetermineSourceServerGroupTask implements RetryableTask {
       lastException = ex
     }
 
-    Boolean useSourceCapacity = useSourceCapacity(stage, null)
-    boolean isNotFound = (lastException instanceof RetrofitError && lastException.kind == RetrofitError.Kind.HTTP && lastException.response.status == 404)
-
     StringWriter sw = new StringWriter()
     sw.append(lastException.message).append("\n")
     new PrintWriter(sw).withWriter { lastException.printStackTrace(it as PrintWriter) }
@@ -89,21 +88,27 @@ class DetermineSourceServerGroupTask implements RetryableTask {
       consecutiveNotFound: isNotFound ? (stage.context.consecutiveNotFound ?: 0) + 1 : 0
     ]
 
-    if (ctx.consecutiveNotFound >= MIN_CONSECUTIVE_404 && !useSourceCapacity) {
-      //we aren't asking to use the source capacity for sizing and have got some repeated 404s on the cluster - assume that is a legit
+    if (ctx.consecutiveNotFound >= MIN_CONSECUTIVE_404 && preferSourceCapacity(stage)) {
+      if (!stage.context.capacity) {
+        throw new IllegalStateException("Could not find source server group to copy capacity from, and no capacity specified.")
+      }
       return new TaskResult(ExecutionStatus.SUCCEEDED, ctx)
     }
 
-    if (ctx.attempt <= MAX_ATTEMPTS) {
-      return new TaskResult(ExecutionStatus.RUNNING, ctx)
+    if (ctx.consecutiveNotFound >= MIN_CONSECUTIVE_404 && useSourceCapacity(stage, null) || ctx.attempt > MAX_ATTEMPTS) {
+      throw new IllegalStateException(lastException.getMessage(), lastException)
     }
 
-    throw new IllegalStateException(lastException.getMessage(), lastException)
+    return new TaskResult(ExecutionStatus.RUNNING, ctx)
   }
 
   Boolean useSourceCapacity(Stage stage, StageData.Source source) {
     if (source?.useSourceCapacity != null) return source.useSourceCapacity
     if (stage.context.useSourceCapacity != null) return (stage.context.useSourceCapacity as Boolean)
     return null
   }
+
+  Boolean preferSourceCapacity(Stage stage) {
+    return stage.context.getOrDefault("preferSourceCapacity", false)
+  }
 }

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/kato/pipeline/support/StageData.groovy

@@ -33,6 +33,7 @@ class StageData {
   Map<String, List<String>> availabilityZones
   int maxRemainingAsgs
   Boolean useSourceCapacity
+  Boolean preferSourceCapacity
   Source source
 
   String getCluster() {
@@ -67,12 +68,21 @@ class StageData {
     return useSourceCapacity ?: false
   }
 
+  Boolean getPreferSourceCapacity() {
+    if (source?.preferSourceCapacity != null) {
+      return source.preferSourceCapacity
+    }
+
+    return preferSourceCapacity ?: false
+  }
+
   static class Source {
     String account
     String region
     String asgName
     String serverGroupName
     Boolean useSourceCapacity
+    Boolean preferSourceCapacity
   }
 
 }

orca-clouddriver/src/test/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/servergroup/CaptureSourceServerGroupCapacityTaskSpec.groovy

@@ -52,6 +52,17 @@ class CaptureSourceServerGroupCapacityTaskSpec extends Specification {
     [source: [useSourceCapacity: false]] || _
   }
 
+  void "should set useSourceCapacity to false if no source found and preferSourceCapacity is true"() {
+    given:
+    def stage = new Stage<>(new Pipeline(), "", [useSourceCapacity: true, preferSourceCapacity: true])
+
+    when:
+    def result = task.execute(stage)
+
+    then:
+    result.context == [useSourceCapacity: false]
+  }
+
   void "should capture source server group capacity and update target capacity"() {
     given:
     def stage = new Stage<>(new Pipeline(), "", [

orca-clouddriver/src/test/groovy/com/netflix/spinnaker/orca/kato/pipeline/strategy/DetermineSourceServerGroupTaskSpec.groovy

@@ -26,6 +26,7 @@ import retrofit.client.Response
 import retrofit.converter.Converter
 import retrofit.mime.TypedString
 import spock.lang.Specification
+import spock.lang.Unroll
 
 class DetermineSourceServerGroupTaskSpec extends Specification {
 
@@ -90,7 +91,7 @@ class DetermineSourceServerGroupTaskSpec extends Specification {
     given:
     Stage stage = new Stage<>(new Pipeline(), 'deploy', 'deploy', [
       account    : 'test',
-      region    : 'us-east-1',
+      region     : 'us-east-1',
       application: 'foo'])
 
     def resolver = Mock(SourceResolver)
@@ -106,7 +107,7 @@ class DetermineSourceServerGroupTaskSpec extends Specification {
     given:
     Stage stage = new Stage<>(new Pipeline(), 'deploy', 'deploy', [
       account    : 'test',
-      source: [ region    : 'us-east-1', account: 'test', asgName: 'foo-test-v000' ],
+      source     : [region: 'us-east-1', account: 'test', asgName: 'foo-test-v000'],
       application: 'foo'])
 
     def resolver = Mock(SourceResolver)
@@ -199,9 +200,8 @@ class DetermineSourceServerGroupTaskSpec extends Specification {
     ex.cause.is expected
   }
 
-  void "should #status after #attempt consecutive 404s with useSourceCapacity #useSourceCapacity"() {
-    Response response = new Response("http://oort.com", 404, "NOT_FOUND", [], new TypedString(""))
-    Exception expected = RetrofitError.httpError("http://oort.com", response, Stub(Converter), Response)
+  @Unroll
+  void "should be #status after #attempt consecutive missing source with useSourceCapacity #useSourceCapacity"() {
     Stage stage = new Stage<>(new Pipeline(), 'deploy', 'deploy', [
       account            : 'test',
       application        : 'foo',
@@ -216,27 +216,92 @@ class DetermineSourceServerGroupTaskSpec extends Specification {
     def taskResult = new DetermineSourceServerGroupTask(sourceResolver: resolver).execute(stage)
 
     then:
-    1 * resolver.getSource(_) >> { throw expected }
+    1 * resolver.getSource(_) >> null
 
     taskResult.status == status
 
     where:
     status                    | useSourceCapacity | attempt
-    ExecutionStatus.SUCCEEDED | false             | DetermineSourceServerGroupTask.MIN_CONSECUTIVE_404
-    ExecutionStatus.RUNNING   | true              | DetermineSourceServerGroupTask.MIN_CONSECUTIVE_404
-    ExecutionStatus.RUNNING   | false             | 1
+    ExecutionStatus.SUCCEEDED | false             | 1
     ExecutionStatus.RUNNING   | true              | 1
   }
 
-  def 'should fail if no source resolved and useSourceCapacity requested'() {
+  def 'should throw exception if no source resolved and useSourceCapacity requested after attempts limit is reached'() {
     Stage stage = new Stage<>(new Pipeline(), 'deploy', 'deploy', [
       account            : 'test',
       application        : 'foo',
       useSourceCapacity  : true,
+      attempt            : DetermineSourceServerGroupTask.MIN_CONSECUTIVE_404,
+      consecutiveNotFound: DetermineSourceServerGroupTask.MIN_CONSECUTIVE_404,
       availabilityZones  : ['us-east-1': []]])
 
     def resolver = Mock(SourceResolver)
 
+    when:
+    new DetermineSourceServerGroupTask(sourceResolver: resolver).execute(stage)
+
+    then:
+    1 * resolver.getSource(_) >> null
+    thrown IllegalStateException
+  }
+
+  @Unroll
+  def 'should be #status after #attempt consecutive missing source with useSourceCapacity and preferSourceCapacity and capacity context #capacity'() {
+    Stage stage = new Stage<>(new Pipeline(), 'deploy', 'deploy', [
+      account             : 'test',
+      application         : 'foo',
+      useSourceCapacity   : true,
+      preferSourceCapacity: true,
+      attempt             : attempt,
+      consecutiveNotFound : attempt,
+      capacity            : capacity,
+      availabilityZones   : ['us-east-1': []]])
+
+    def resolver = Mock(SourceResolver)
+
+    when:
+    def taskResult = new DetermineSourceServerGroupTask(sourceResolver: resolver).execute(stage)
+
+    then:
+    1 * resolver.getSource(_) >> null
+
+    taskResult.status == status
+
+    where:
+    status                    | capacity | attempt
+    ExecutionStatus.RUNNING   | null     | 1
+    ExecutionStatus.SUCCEEDED | [min: 0] | DetermineSourceServerGroupTask.MIN_CONSECUTIVE_404 - 1
+  }
+
+  def 'should throw exception if useSourceCapacity and preferSourceCapacity set, but source not found and no capacity specified'() {
+    Stage stage = new Stage<>(new Pipeline(), 'deploy', 'deploy', [
+      account             : 'test',
+      application         : 'foo',
+      useSourceCapacity   : true,
+      preferSourceCapacity: true,
+      attempt             : DetermineSourceServerGroupTask.MIN_CONSECUTIVE_404,
+      consecutiveNotFound : DetermineSourceServerGroupTask.MIN_CONSECUTIVE_404,
+      availabilityZones   : ['us-east-1': []]])
+
+    def resolver = Mock(SourceResolver)
+
+    when:
+    new DetermineSourceServerGroupTask(sourceResolver: resolver).execute(stage)
+
+    then:
+    1 * resolver.getSource(_) >> null
+    thrown IllegalStateException
+  }
+
+  def 'should fail if no source resolved and useSourceCapacity requested'() {
+    Stage stage = new Stage<>(new Pipeline(), 'deploy', 'deploy', [
+      account          : 'test',
+      application      : 'foo',
+      useSourceCapacity: true,
+      availabilityZones: ['us-east-1': []]])
+
+    def resolver = Mock(SourceResolver)
+
     when:
     def result = new DetermineSourceServerGroupTask(sourceResolver: resolver).execute(stage)