fix(provider/google): Retry firewall rule deletes and accept 404. (#1624

)
spinnaker · May 11, 2017 · 3214067 · 3214067
1 parent 057a751
commit 3214067
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 2 deletions.
diff --git a/...x/spinnaker/clouddriver/google/deploy/ops/DeleteGoogleSecurityGroupAtomicOperation.groovy b/...x/spinnaker/clouddriver/google/deploy/ops/DeleteGoogleSecurityGroupAtomicOperation.groovy
@@ -18,7 +18,9 @@ package com.netflix.spinnaker.clouddriver.google.deploy.ops
 
 import com.netflix.spinnaker.clouddriver.data.task.Task
 import com.netflix.spinnaker.clouddriver.data.task.TaskRepository
+import com.netflix.spinnaker.clouddriver.google.deploy.SafeRetry
 import com.netflix.spinnaker.clouddriver.google.deploy.description.DeleteGoogleSecurityGroupDescription
+import org.springframework.beans.factory.annotation.Autowired
 
 /**
  * Delete a firewall rule from the specified project.
@@ -28,6 +30,9 @@ import com.netflix.spinnaker.clouddriver.google.deploy.description.DeleteGoogleS
 class DeleteGoogleSecurityGroupAtomicOperation extends GoogleAtomicOperation<Void> {
   private static final String BASE_PHASE = "DELETE_SECURITY_GROUP"
 
+  @Autowired
+  SafeRetry safeRetry
+
   private static Task getTask() {
     TaskRepository.threadLocalTask.get()
   }
@@ -49,8 +54,20 @@ class DeleteGoogleSecurityGroupAtomicOperation extends GoogleAtomicOperation<Voi
     def project = description.credentials.project
     def firewallRuleName = description.securityGroupName
 
-    timeExecute(compute.firewalls().delete(project, firewallRuleName),
-                "compute.firewalls.delete", TAG_SCOPE, SCOPE_GLOBAL)
+    safeRetry.doRetry(
+        {
+          timeExecute(
+              compute.firewalls().delete(project, firewallRuleName),
+              "compute.firewalls.delete",
+              TAG_SCOPE, SCOPE_GLOBAL)
+        },
+        "Firewall rule ${firewallRuleName}",
+        task,
+        [400, 403, 412],
+        [404],
+        [action: "delete", phase: BASE_PHASE, operation: "compute.firewalls.delete", (TAG_SCOPE): SCOPE_GLOBAL],
+        registry
+    )
 
     task.updateStatus BASE_PHASE, "Done deleting security group $firewallRuleName."
     null

diff --git a/...ker/clouddriver/google/deploy/ops/DeleteGoogleSecurityGroupAtomicOperationUnitSpec.groovy b/...ker/clouddriver/google/deploy/ops/DeleteGoogleSecurityGroupAtomicOperationUnitSpec.groovy
@@ -20,18 +20,23 @@ import com.google.api.services.compute.Compute
 import com.netflix.spectator.api.DefaultRegistry
 import com.netflix.spinnaker.clouddriver.data.task.Task
 import com.netflix.spinnaker.clouddriver.data.task.TaskRepository
+import com.netflix.spinnaker.clouddriver.google.deploy.SafeRetry
 import com.netflix.spinnaker.clouddriver.google.deploy.description.DeleteGoogleSecurityGroupDescription
 import com.netflix.spinnaker.clouddriver.google.security.GoogleNamedAccountCredentials
+import spock.lang.Shared
 import spock.lang.Specification
 import spock.lang.Subject
 
 class DeleteGoogleSecurityGroupAtomicOperationUnitSpec extends Specification {
   private static final SECURITY_GROUP_NAME = "spinnaker-test-sg"
   private static final ACCOUNT_NAME = "prod"
   private static final PROJECT_NAME = "my-project"
+  @Shared
+  SafeRetry safeRetry
 
   def setupSpec() {
     TaskRepository.threadLocalTask.set(Mock(Task))
+    safeRetry = new SafeRetry(maxRetries: 10, maxWaitInterval: 60000, retryIntervalBase: 0, jitterMultiplier: 0)
   }
 
   void "should delete firewall rule"() {
@@ -46,6 +51,7 @@ class DeleteGoogleSecurityGroupAtomicOperationUnitSpec extends Specification {
                                                                  credentials: credentials)
       @Subject def operation = new DeleteGoogleSecurityGroupAtomicOperation(description)
       operation.registry = registry
+      operation.safeRetry = safeRetry
 
     when:
       operation.operate([])