feat(bakery): Clean up cached data created by Rosco. (#912) (#922)

* feat(bakery): Clean up cached data created by Rosco.

* fix: typo.

* feat: Add unit test.

Co-authored-by: Fernando Freire <dogonthehorizon@gmail.com>
(cherry picked from commit 96cd6d2)

Co-authored-by: armory-abedonik <106548537+armory-abedonik@users.noreply.github.com>

rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/api/DeleteBakesRequest.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2022 Armory, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.rosco.api;
+
+import java.util.ArrayList;
+import java.util.List;
+import lombok.Data;
+
+@Data
+public class DeleteBakesRequest {
+
+  private List<String> pipelineExecutionIds = new ArrayList<>();
+}

rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/persistence/BakeStore.groovy

@@ -107,6 +107,11 @@ interface BakeStore {
    */
   public String deleteBakeByKeyPreserveDetails(String bakeKey)
 
+  /**
+   * Delete the bake entities associated with the given pipeline execution id.
+   */
+  void deleteBakeByPipelineExecutionId(String pipelineExecutionId);
+
   /**
    * Cancel the incomplete bake associated with the bake id and delete the completed bake details associated with the
    * bake id. If the bake is still incomplete, remove the bake id from the set of incomplete bakes.

rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/persistence/RedisBackedBakeStore.groovy

@@ -70,12 +70,16 @@ class RedisBackedBakeStore implements BakeStore {
             return false
           end
         """)
-        // Expected key list: "allBakes", bake id, bake key, this instance incomplete bakes key, lock key
+        // Expected key list: "allBakes", bake id, bake key, this instance incomplete bakes key, lock key, pipeline execution key
         // Expected arg list: createdTimestampMilliseconds, region, bake request json, bake status json, bake logs json, command, rosco instance id
         storeNewBakeStatusSHA = jedis.scriptLoad("""\
           -- Delete the bake id key.
           redis.call('DEL', KEYS[2])
 
+          -- Add bake id and bake key to set identified by pipeline execution key.
+          redis.call('SADD', KEYS[6], KEYS[2])
+          redis.call('SADD', KEYS[6], KEYS[3])
+
           -- Add bake key to set of bakes.
           redis.call('ZADD', KEYS[1], ARGV[1], KEYS[3])
 
@@ -283,6 +287,19 @@ class RedisBackedBakeStore implements BakeStore {
 
           return ret
         """)
+        // Expected key list: pipeline execution key
+        // Expected arg list:
+        deleteBakeByPipelineExecutionKeySHA = jedis.scriptLoad("""
+          -- Get all bake entities' keys by pipeline execution key.
+          local bake_keys = redis.call('SMEMBERS', KEYS[1])
+
+          for _,key in ipairs(bake_keys)
+          do
+            redis.call('DEL', key)
+          end
+
+          redis.call('DEL', KEYS[1])
+        """)
       }
     }
   }
@@ -308,12 +325,14 @@ class RedisBackedBakeStore implements BakeStore {
   @Override
   public BakeStatus storeNewBakeStatus(String bakeKey, String region, BakeRecipe bakeRecipe, BakeRequest bakeRequest, BakeStatus bakeStatus, String command) {
     def lockKey = "lock:$bakeKey"
+    def pipelineExecutionId = getPipelineExecutionId(bakeRequest.spinnaker_execution_id)
+    def pipelineExecutionKey = getBakePipelineExecutionKey(pipelineExecutionId)
     def bakeRecipeJson = mapper.writeValueAsString(bakeRecipe)
     def bakeRequestJson = mapper.writeValueAsString(bakeRequest)
     def bakeStatusJson = mapper.writeValueAsString(bakeStatus)
     def bakeLogsJson = mapper.writeValueAsString(bakeStatus.logsContent ? [logsContent: bakeStatus.logsContent] : [:])
     def createdTimestampMilliseconds = timeInMilliseconds
-    def keyList = ["allBakes", bakeStatus.id, bakeKey, thisInstanceIncompleteBakesKey, lockKey.toString()]
+    def keyList = ["allBakes", bakeStatus.id, bakeKey, thisInstanceIncompleteBakesKey, lockKey.toString(), pipelineExecutionKey.toString()]
     def argList = [createdTimestampMilliseconds as String, region, bakeRecipeJson, bakeRequestJson, bakeStatusJson, bakeLogsJson, command, roscoInstanceId]
     def result = evalSHA("storeNewBakeStatusSHA", keyList, argList)
 
@@ -473,6 +492,13 @@ class RedisBackedBakeStore implements BakeStore {
     return evalSHA("deleteBakeByKeyPreserveDetailsSHA", keyList, argList)
   }
 
+  @Override
+  public void deleteBakeByPipelineExecutionId(String pipelineExecutionId) {
+    def keyList = [getBakePipelineExecutionKey(pipelineExecutionId)]
+
+    evalSHA("deleteBakeByPipelineExecutionKeySHA", keyList, [])
+  }
+
   @Override
   public boolean cancelBakeById(String bakeId) {
     def bakeStatus = new BakeStatus(id: bakeId,
@@ -592,4 +618,12 @@ class RedisBackedBakeStore implements BakeStore {
       }
     }
   }
+
+  private static String getBakePipelineExecutionKey(String pipelineExecutionId) {
+    return "bake:pipeline_execution:$pipelineExecutionId"
+  }
+
+  private static String getPipelineExecutionId(String spinnakerExecutionId) {
+    return spinnakerExecutionId.split(":")[0]
+  }
 }

rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/controllers/BakeryController.groovy

@@ -21,6 +21,7 @@ import com.netflix.spinnaker.rosco.api.Bake
 import com.netflix.spinnaker.rosco.api.BakeOptions
 import com.netflix.spinnaker.rosco.api.BakeRequest
 import com.netflix.spinnaker.rosco.api.BakeStatus
+import com.netflix.spinnaker.rosco.api.DeleteBakesRequest
 import com.netflix.spinnaker.rosco.jobs.BakeRecipe
 import com.netflix.spinnaker.rosco.jobs.JobExecutor
 import com.netflix.spinnaker.rosco.jobs.JobRequest
@@ -308,6 +309,13 @@ class BakeryController {
     }
   }
 
+  @RequestMapping(value = '/api/v1/bakes/delete-requests', method = RequestMethod.POST)
+  void createDeleteBakesRequest(@RequestBody DeleteBakesRequest deleteBakesRequest) {
+    deleteBakesRequest.getPipelineExecutionIds().forEach({ pipelineExecutionId ->
+      bakeStore.deleteBakeByPipelineExecutionId(pipelineExecutionId)
+    })
+  }
+
   // TODO(duftler): Synchronize this with existing bakery api.
   @ApiOperation(value = "Cancel bake request")
   @RequestMapping(value = "/api/v1/{region}/cancel/{statusId}", method = RequestMethod.GET)

rosco-web/src/test/groovy/com/netflix/spinnaker/rosco/controllers/BakeryControllerSpec.groovy

@@ -21,6 +21,7 @@ import com.netflix.spinnaker.rosco.api.Bake
 import com.netflix.spinnaker.rosco.api.BakeOptions
 import com.netflix.spinnaker.rosco.api.BakeRequest
 import com.netflix.spinnaker.rosco.api.BakeStatus
+import com.netflix.spinnaker.rosco.api.DeleteBakesRequest
 import com.netflix.spinnaker.rosco.jobs.BakeRecipe
 import com.netflix.spinnaker.rosco.persistence.RedisBackedBakeStore
 import com.netflix.spinnaker.rosco.providers.CloudProviderBakeHandler
@@ -782,4 +783,20 @@ class BakeryControllerSpec extends Specification {
       thrown BakeOptions.Exception
   }
 
+  void 'delete bakes by pipeline execution ids'() {
+    setup:
+    def bakeStoreMock = Mock(RedisBackedBakeStore)
+    def pipelineExecutionId = UUID.randomUUID().toString()
+    def deleteBakesRequest = new DeleteBakesRequest()
+    deleteBakesRequest.pipelineExecutionIds.add(pipelineExecutionId)
+
+    @Subject
+    def bakeryController = new BakeryController(bakeStore: bakeStoreMock)
+
+    when:
+    bakeryController.createDeleteBakesRequest(deleteBakesRequest)
+
+    then:
+    1 * bakeStoreMock.deleteBakeByPipelineExecutionId(pipelineExecutionId)
+  }
 }