fix(rollback): capture the source capacity before anything else

Does not entirely address this issue but makes it less likely to happen: spinnaker/spinnaker#4895 The "danger zone" is when a resize of the source server group happens between the start of the rollback and the moment we take the capacity snapshot. This change should reduce that "danger zone" from minutes to seconds.
spinnaker · Sep 17, 2019 · 8a409a6 · 8a409a6
1 parent 66c7b4c
commit 8a409a6
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 10 deletions.
diff --git a/.../netflix/spinnaker/orca/clouddriver/pipeline/servergroup/rollback/ExplicitRollback.groovy b/.../netflix/spinnaker/orca/clouddriver/pipeline/servergroup/rollback/ExplicitRollback.groovy
@@ -92,12 +92,15 @@ class ExplicitRollback implements Rollback {
       ]
     }
 
-    def stages = [enableServerGroupStage]
+    // https://github.com/spinnaker/spinnaker/issues/4895 - capture the source capacity as early as possible
+    def stages = []
     if (!parentStage.getContext().containsKey("sourceServerGroupCapacitySnapshot")) {
       // capacity has been previously captured (likely as part of a failed deploy), no need to do again!
       stages << buildCaptureSourceServerGroupCapacityStage(parentStage, parentStage.mapTo(ResizeStrategy.Source))
     }
 
+    stages << enableServerGroupStage
+
     Map resizeServerGroupContext = new HashMap(parentStage.context) + [
       action                       : ResizeStrategy.ResizeAction.scale_to_server_group.toString(),
       source                       : {

diff --git a/...flix/spinnaker/orca/clouddriver/pipeline/servergroup/rollback/ExplicitRollbackSpec.groovy b/...flix/spinnaker/orca/clouddriver/pipeline/servergroup/rollback/ExplicitRollbackSpec.groovy
@@ -83,17 +83,13 @@ class ExplicitRollbackSpec extends Specification {
     then:
     beforeStages.isEmpty()
     afterStages*.type == [
+      "captureSourceServerGroupCapacity",  // spinnaker/issues/4895, capture capacity snapshot first
       "enableServerGroup",
-      "captureSourceServerGroupCapacity",
       "resizeServerGroup",
       "disableServerGroup",
       "applySourceServerGroupCapacity"
     ]
-    afterStages[0].context == stage.context + [
-      serverGroupName: "servergroup-v001",
-      targetHealthyDeployPercentage: 95
-    ]
-    afterStages[1].context == [
+    afterStages[0].context == [
       source           : [
         asgName        : "servergroup-v002",
         serverGroupName: "servergroup-v002",
@@ -103,6 +99,10 @@ class ExplicitRollbackSpec extends Specification {
       ],
       useSourceCapacity: true
     ]
+    afterStages[1].context == stage.context + [
+      serverGroupName: "servergroup-v001",
+      targetHealthyDeployPercentage: 95
+    ]
     afterStages[2].context == stage.context + [
       action            : "scale_to_server_group",
       source            : new ResizeStrategy.Source(null, null, "us-west-1", null, "servergroup-v002", "test", "aws"),
@@ -152,8 +152,8 @@ class ExplicitRollbackSpec extends Specification {
 
     where:
     delayBeforeDisableSeconds || expectedAfterStageTypes
-    null                      || ["enableServerGroup", "captureSourceServerGroupCapacity", "resizeServerGroup", "disableServerGroup", "applySourceServerGroupCapacity"]
-    0                         || ["enableServerGroup", "captureSourceServerGroupCapacity", "resizeServerGroup", "disableServerGroup", "applySourceServerGroupCapacity"]
-    1                         || ["enableServerGroup", "captureSourceServerGroupCapacity", "resizeServerGroup", "wait", "disableServerGroup", "applySourceServerGroupCapacity"]
+    null                      || ["captureSourceServerGroupCapacity", "enableServerGroup", "resizeServerGroup", "disableServerGroup", "applySourceServerGroupCapacity"]
+    0                         || ["captureSourceServerGroupCapacity", "enableServerGroup", "resizeServerGroup", "disableServerGroup", "applySourceServerGroupCapacity"]
+    1                         || ["captureSourceServerGroupCapacity", "enableServerGroup", "resizeServerGroup", "wait", "disableServerGroup", "applySourceServerGroupCapacity"]
   }
 }