From 91f6247bfe262da42e76a69bea8b5e49bb6ed41e Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 23 Nov 2022 15:54:32 +0000 Subject: [PATCH] feat(bakery): Clean up cached data created by Rosco. (#4323) (#4349) Co-authored-by: Fernando Freire (cherry picked from commit e4973b0efa33557eb4d5b781345458047b989cf4) Co-authored-by: armory-abedonik <106548537+armory-abedonik@users.noreply.github.com> --- .../orca/bakery/api/BakeryService.groovy | 3 + .../orca/bakery/api/DeleteBakesRequest.java | 29 +++++++++ ...keryPipelineDependencyCleanupOperator.java | 41 +++++++++++++ .../orca/bakery/BakerySelectorSpec.groovy | 6 ++ ...pelineCleanupPollingNotificationAgent.java | 32 +++++++--- .../PipelineDependencyCleanupOperator.java | 25 ++++++++ ...cutionCleanupPollingNotificationAgent.java | 60 ++++++++++++------- ...CleanupPollingNotificationAgentSpec.groovy | 11 +++- ...CleanupPollingNotificationAgentSpec.groovy | 11 +++- ...PipelineCleanupPollingNotificationAgent.kt | 6 +- ...xecutionCleanupPollingNotificationAgent.kt | 6 +- ...CleanupPollingNotificationAgentSpec.groovy | 3 +- ...CleanupPollingNotificationAgentSpec.groovy | 3 +- 13 files changed, 194 insertions(+), 42 deletions(-) create mode 100644 orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/api/DeleteBakesRequest.java create mode 100644 orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/pipeline/BakeryPipelineDependencyCleanupOperator.java create mode 100644 orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/scheduling/PipelineDependencyCleanupOperator.java diff --git a/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/api/BakeryService.groovy b/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/api/BakeryService.groovy index e25ccbe0b8..13a3e8cbab 100644 --- a/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/api/BakeryService.groovy +++ b/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/api/BakeryService.groovy @@ -38,6 +38,9 @@ interface BakeryService { @GET("/api/v1/{region}/bake/{bakeId}") Bake lookupBake(@Path("region") String region, @Path("bakeId") String bakeId) + @POST("/api/v1/bakes/delete-requests") + Void createDeleteBakesRequest(@Body DeleteBakesRequest deleteBakesRequest) + // // Methods below this line are not supported by the Netflix Bakery, and are only available // iff bakery.roscoApisEnabled is true. diff --git a/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/api/DeleteBakesRequest.java b/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/api/DeleteBakesRequest.java new file mode 100644 index 0000000000..673a3da9df --- /dev/null +++ b/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/api/DeleteBakesRequest.java @@ -0,0 +1,29 @@ +/* + * Copyright 2022 Armory, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.bakery.api; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.ArrayList; +import java.util.List; +import lombok.Data; + +@Data +public class DeleteBakesRequest { + + @JsonProperty("pipelineExecutionIds") + private List pipelineExecutionIds = new ArrayList<>(); +} diff --git a/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/pipeline/BakeryPipelineDependencyCleanupOperator.java b/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/pipeline/BakeryPipelineDependencyCleanupOperator.java new file mode 100644 index 0000000000..7c15ae5529 --- /dev/null +++ b/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/pipeline/BakeryPipelineDependencyCleanupOperator.java @@ -0,0 +1,41 @@ +/* + * Copyright 2022 Armory, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.bakery.pipeline; + +import com.netflix.spinnaker.orca.bakery.api.BakeryService; +import com.netflix.spinnaker.orca.bakery.api.DeleteBakesRequest; +import com.netflix.spinnaker.orca.notifications.scheduling.PipelineDependencyCleanupOperator; +import java.util.List; +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +@Component +@ConditionalOnExpression("${bakery-pipeline-dependency-cleanup.enabled:false}") +@RequiredArgsConstructor +public class BakeryPipelineDependencyCleanupOperator implements PipelineDependencyCleanupOperator { + + private final BakeryService bakeryService; + + @Override + public void cleanup(List pipelineExecutionIds) { + DeleteBakesRequest deleteBakesRequest = new DeleteBakesRequest(); + deleteBakesRequest.setPipelineExecutionIds(pipelineExecutionIds); + + bakeryService.createDeleteBakesRequest(deleteBakesRequest); + } +} diff --git a/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/BakerySelectorSpec.groovy b/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/BakerySelectorSpec.groovy index 35afc423f4..8a425082b4 100644 --- a/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/BakerySelectorSpec.groovy +++ b/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/BakerySelectorSpec.groovy @@ -22,6 +22,7 @@ import com.netflix.spinnaker.orca.bakery.api.Bake import com.netflix.spinnaker.orca.bakery.api.BakeRequest import com.netflix.spinnaker.orca.bakery.api.BakeStatus import com.netflix.spinnaker.orca.bakery.api.BakeryService +import com.netflix.spinnaker.orca.bakery.api.DeleteBakesRequest import com.netflix.spinnaker.orca.bakery.api.BaseImage import com.netflix.spinnaker.orca.bakery.api.manifests.BakeManifestRequest import com.netflix.spinnaker.orca.bakery.config.BakeryConfigurationProperties @@ -145,6 +146,11 @@ class BakerySelectorSpec extends Specification { return null } + @Override + Void createDeleteBakesRequest(@Body DeleteBakesRequest deleteBakesRequest) { + return null + } + @Override BaseImage getBaseImage(@Path("cloudProvider") String cloudProvider, @Path("imageId") String imageId) { return null diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/scheduling/OldPipelineCleanupPollingNotificationAgent.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/scheduling/OldPipelineCleanupPollingNotificationAgent.java index 8b3c50b57e..c79a175605 100644 --- a/orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/scheduling/OldPipelineCleanupPollingNotificationAgent.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/scheduling/OldPipelineCleanupPollingNotificationAgent.java @@ -102,6 +102,7 @@ public Boolean call(PipelineExecution execution) { private final long pollingIntervalMs; private final int thresholdDays; private final int minimumPipelineExecutions; + private final List pipelineDependencyCleanupOperators; private final Id deletedId; private final Id timerId; @@ -115,7 +116,8 @@ public OldPipelineCleanupPollingNotificationAgent( @Value("${pollers.old-pipeline-cleanup.interval-ms:3600000}") long pollingIntervalMs, @Value("${pollers.old-pipeline-cleanup.threshold-days:30}") int thresholdDays, @Value("${pollers.old-pipeline-cleanup.minimum-pipeline-executions:5}") - int minimumPipelineExecutions) { + int minimumPipelineExecutions, + List pipelineDependencyCleanupOperators) { super(clusterLock); this.executionRepository = executionRepository; this.clock = clock; @@ -123,6 +125,7 @@ public OldPipelineCleanupPollingNotificationAgent( this.pollingIntervalMs = pollingIntervalMs; this.thresholdDays = thresholdDays; this.minimumPipelineExecutions = minimumPipelineExecutions; + this.pipelineDependencyCleanupOperators = pipelineDependencyCleanupOperators; deletedId = registry.createId("pollers.oldPipelineCleanup.deleted"); timerId = registry.createId("pollers.oldPipelineCleanup.timing"); @@ -179,14 +182,25 @@ private void cleanup(List executions) { } executions.sort(sorter); - executions - .subList(0, (executions.size() - minimumPipelineExecutions)) - .forEach( - p -> { - log.info("Deleting pipeline execution " + p.id + ": " + p.toString()); - executionRepository.delete(PIPELINE, p.id); - registry.counter(deletedId.withTag("application", p.application)).increment(); - }); + + List removingPipelineExecutions = + executions.subList(0, (executions.size() - minimumPipelineExecutions)); + + List removingPipelineExecutionIds = + removingPipelineExecutions.stream() + .map(pipelineExecutionDetails -> pipelineExecutionDetails.id) + .collect(Collectors.toList()); + + pipelineDependencyCleanupOperators.forEach( + pipelineDependencyCleanupOperator -> + pipelineDependencyCleanupOperator.cleanup(removingPipelineExecutionIds)); + + removingPipelineExecutions.forEach( + p -> { + log.info("Deleting pipeline execution " + p.id + ": " + p.toString()); + executionRepository.delete(PIPELINE, p.id); + registry.counter(deletedId.withTag("application", p.application)).increment(); + }); } private static class PipelineExecutionDetails { diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/scheduling/PipelineDependencyCleanupOperator.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/scheduling/PipelineDependencyCleanupOperator.java new file mode 100644 index 0000000000..ca4aa91b0b --- /dev/null +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/scheduling/PipelineDependencyCleanupOperator.java @@ -0,0 +1,25 @@ +/* + * Copyright 2022 Armory, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.notifications.scheduling; + +import java.util.List; + +/** This interface is responsible for the pipeline execution dependencies cleanup. */ +public interface PipelineDependencyCleanupOperator { + + void cleanup(List pipelineExecutionIds); +} diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/scheduling/TopApplicationExecutionCleanupPollingNotificationAgent.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/scheduling/TopApplicationExecutionCleanupPollingNotificationAgent.java index 237d221909..57147aa774 100644 --- a/orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/scheduling/TopApplicationExecutionCleanupPollingNotificationAgent.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/notifications/scheduling/TopApplicationExecutionCleanupPollingNotificationAgent.java @@ -35,6 +35,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -71,6 +72,7 @@ public class TopApplicationExecutionCleanupPollingNotificationAgent private final Registry registry; private final long pollingIntervalMs; private final int threshold; + private final List pipelineDependencyCleanupOperators; private final Id deleteCountId; private final Id timerId; @@ -82,12 +84,14 @@ public TopApplicationExecutionCleanupPollingNotificationAgent( Registry registry, @Value("${pollers.top-application-execution-cleanup.interval-ms:3600000}") long pollingIntervalMs, - @Value("${pollers.top-application-execution-cleanup.threshold:2500}") int threshold) { + @Value("${pollers.top-application-execution-cleanup.threshold:2500}") int threshold, + List pipelineDependencyCleanupOperators) { super(clusterLock); this.executionRepository = executionRepository; this.registry = registry; this.pollingIntervalMs = pollingIntervalMs; this.threshold = threshold; + this.pipelineDependencyCleanupOperators = pipelineDependencyCleanupOperators; deleteCountId = registry.createId("pollers.topApplicationExecutionCleanup.deleted"); timerId = registry.createId("pollers.topApplicationExecutionCleanup.timing"); @@ -139,28 +143,38 @@ private void cleanup(Observable observable, String applicatio observable.filter(filter).map(mapper).toList().toBlocking().single(); executions.sort(comparing(a -> (Long) Optional.ofNullable(a.get("startTime")).orElse(0L))); if (executions.size() > threshold) { - executions - .subList(0, (executions.size() - threshold)) - .forEach( - it -> { - Long startTime = - Optional.ofNullable((Long) it.get("startTime")) - .orElseGet(() -> (Long) it.get("buildTime")); - log.info( - "Deleting {} execution {} (startTime: {}, application: {}, pipelineConfigId: {}, status: {})", - type, - it.get("id"), - startTime != null ? Instant.ofEpochMilli(startTime) : null, - application, - it.get("pipelineConfigId"), - it.get("status")); - if (type.equals("orchestration")) { - executionRepository.delete(ORCHESTRATION, (String) it.get("id")); - registry.counter(deleteCountId.withTag("application", application)).increment(); - } else { - throw new IllegalArgumentException(format("Unsupported type '%s'", type)); - } - }); + List removingPipelineExecutions = + executions.subList(0, (executions.size() - threshold)); + + List removingPipelineExecutionIds = + removingPipelineExecutions.stream() + .map(pipelineExecution -> (String) pipelineExecution.get("id")) + .collect(Collectors.toList()); + + pipelineDependencyCleanupOperators.forEach( + pipelineDependencyCleanupOperator -> + pipelineDependencyCleanupOperator.cleanup(removingPipelineExecutionIds)); + + removingPipelineExecutions.forEach( + it -> { + Long startTime = + Optional.ofNullable((Long) it.get("startTime")) + .orElseGet(() -> (Long) it.get("buildTime")); + log.info( + "Deleting {} execution {} (startTime: {}, application: {}, pipelineConfigId: {}, status: {})", + type, + it.get("id"), + startTime != null ? Instant.ofEpochMilli(startTime) : null, + application, + it.get("pipelineConfigId"), + it.get("status")); + if (type.equals("orchestration")) { + executionRepository.delete(ORCHESTRATION, (String) it.get("id")); + registry.counter(deleteCountId.withTag("application", application)).increment(); + } else { + throw new IllegalArgumentException(format("Unsupported type '%s'", type)); + } + }); } } } diff --git a/orca-core/src/test/groovy/com/netflix/spinnaker/orca/notifications/scheduling/OldPipelineCleanupPollingNotificationAgentSpec.groovy b/orca-core/src/test/groovy/com/netflix/spinnaker/orca/notifications/scheduling/OldPipelineCleanupPollingNotificationAgentSpec.groovy index 2a93460a27..21186a3c20 100644 --- a/orca-core/src/test/groovy/com/netflix/spinnaker/orca/notifications/scheduling/OldPipelineCleanupPollingNotificationAgentSpec.groovy +++ b/orca-core/src/test/groovy/com/netflix/spinnaker/orca/notifications/scheduling/OldPipelineCleanupPollingNotificationAgentSpec.groovy @@ -45,7 +45,8 @@ class OldPipelineCleanupPollingNotificationAgentSpec extends Specification { new NoopRegistry(), 5000, 1, - 5 + 5, + [] ).filter expect: @@ -82,7 +83,8 @@ class OldPipelineCleanupPollingNotificationAgentSpec extends Specification { new NoopRegistry(), 5000, 1, - 5 + 5, + [] ).mapper expect: @@ -122,6 +124,7 @@ class OldPipelineCleanupPollingNotificationAgentSpec extends Specification { 1 * retrieveAllApplicationNames(PIPELINE) >> ["orca"] 1 * retrievePipelinesForApplication("orca") >> rx.Observable.from(pipelines) } + def pipelineDependencyCleanupOperator = Mock(PipelineDependencyCleanupOperator) def agent = new OldPipelineCleanupPollingNotificationAgent( Mock(NotificationClusterLock), executionRepository, @@ -129,7 +132,8 @@ class OldPipelineCleanupPollingNotificationAgentSpec extends Specification { new NoopRegistry(), 5000, thresholdDays, - retain + retain, + [pipelineDependencyCleanupOperator] ) when: @@ -140,6 +144,7 @@ class OldPipelineCleanupPollingNotificationAgentSpec extends Specification { // expect D1-5 to be too old, but for the most recent 3 to be retained 1 * executionRepository.delete(PIPELINE, '1') 1 * executionRepository.delete(PIPELINE, '2') + 1 * pipelineDependencyCleanupOperator.cleanup(['1', '2']) } private diff --git a/orca-core/src/test/groovy/com/netflix/spinnaker/orca/notifications/scheduling/TopApplicationPipelineExecutionCleanupPollingNotificationAgentSpec.groovy b/orca-core/src/test/groovy/com/netflix/spinnaker/orca/notifications/scheduling/TopApplicationPipelineExecutionCleanupPollingNotificationAgentSpec.groovy index 3545f1e3ab..f9f23e4bf9 100644 --- a/orca-core/src/test/groovy/com/netflix/spinnaker/orca/notifications/scheduling/TopApplicationPipelineExecutionCleanupPollingNotificationAgentSpec.groovy +++ b/orca-core/src/test/groovy/com/netflix/spinnaker/orca/notifications/scheduling/TopApplicationPipelineExecutionCleanupPollingNotificationAgentSpec.groovy @@ -40,7 +40,8 @@ class TopApplicationPipelineExecutionCleanupPollingNotificationAgentSpec extends Mock(ExecutionRepository), new NoopRegistry(), 5000, - 2500 + 2500, + [] ).filter expect: @@ -70,7 +71,8 @@ class TopApplicationPipelineExecutionCleanupPollingNotificationAgentSpec extends Mock(ExecutionRepository), new NoopRegistry(), 5000, - 2500 + 2500, + [] ).mapper expect: @@ -92,12 +94,14 @@ class TopApplicationPipelineExecutionCleanupPollingNotificationAgentSpec extends 1 * retrieveAllApplicationNames(_, _) >> ["app1"] 1 * retrieveOrchestrationsForApplication("app1", _) >> rx.Observable.from(orchestrations) } + def pipelineDependencyCleanupOperator = Mock(PipelineDependencyCleanupOperator) def agent = new TopApplicationExecutionCleanupPollingNotificationAgent( Mock(NotificationClusterLock), executionRepository, new NoopRegistry(), 5000, - 2 + 2, + [pipelineDependencyCleanupOperator] ) when: @@ -105,6 +109,7 @@ class TopApplicationPipelineExecutionCleanupPollingNotificationAgentSpec extends then: 1 * executionRepository.delete(ORCHESTRATION, orchestrations[0].id) + 1 * pipelineDependencyCleanupOperator.cleanup([orchestrations[0].id]) } private static Collection buildExecutions(AtomicInteger stageStartTime, diff --git a/orca-sql/src/main/kotlin/com/netflix/spinnaker/orca/sql/cleanup/OldPipelineCleanupPollingNotificationAgent.kt b/orca-sql/src/main/kotlin/com/netflix/spinnaker/orca/sql/cleanup/OldPipelineCleanupPollingNotificationAgent.kt index 160dddc196..8c39eff70a 100644 --- a/orca-sql/src/main/kotlin/com/netflix/spinnaker/orca/sql/cleanup/OldPipelineCleanupPollingNotificationAgent.kt +++ b/orca-sql/src/main/kotlin/com/netflix/spinnaker/orca/sql/cleanup/OldPipelineCleanupPollingNotificationAgent.kt @@ -21,6 +21,7 @@ import com.netflix.spinnaker.config.OldPipelineCleanupAgentConfigurationProperti import com.netflix.spinnaker.config.OrcaSqlProperties import com.netflix.spinnaker.orca.api.pipeline.models.ExecutionType import com.netflix.spinnaker.orca.notifications.NotificationClusterLock +import com.netflix.spinnaker.orca.notifications.scheduling.PipelineDependencyCleanupOperator import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository import java.time.Clock import java.time.Instant @@ -45,7 +46,8 @@ class OldPipelineCleanupPollingNotificationAgent( registry: Registry, private val executionRepository: ExecutionRepository, private val configurationProperties: OldPipelineCleanupAgentConfigurationProperties, - private val orcaSqlProperties: OrcaSqlProperties + private val orcaSqlProperties: OrcaSqlProperties, + private val pipelineDependencyCleanupOperators: List ) : AbstractCleanupPollingAgent( clusterLock, configurationProperties.intervalMs, @@ -161,6 +163,8 @@ class OldPipelineCleanupPollingNotificationAgent( .limit(configurationProperties.minimumPipelineExecutions, Int.MAX_VALUE) .fetch(field("id"), String::class.java) + pipelineDependencyCleanupOperators.forEach { it.cleanup(executionsToRemove) } + executionsToRemove.chunked(configurationProperties.chunkSize).forEach { ids -> deletedExecutionCount.addAndGet(ids.size) executionRepository.delete(ExecutionType.PIPELINE, ids) diff --git a/orca-sql/src/main/kotlin/com/netflix/spinnaker/orca/sql/cleanup/TopApplicationExecutionCleanupPollingNotificationAgent.kt b/orca-sql/src/main/kotlin/com/netflix/spinnaker/orca/sql/cleanup/TopApplicationExecutionCleanupPollingNotificationAgent.kt index d67b07cfe0..9927fa4aca 100644 --- a/orca-sql/src/main/kotlin/com/netflix/spinnaker/orca/sql/cleanup/TopApplicationExecutionCleanupPollingNotificationAgent.kt +++ b/orca-sql/src/main/kotlin/com/netflix/spinnaker/orca/sql/cleanup/TopApplicationExecutionCleanupPollingNotificationAgent.kt @@ -21,6 +21,7 @@ import com.netflix.spinnaker.config.OrcaSqlProperties import com.netflix.spinnaker.config.TopApplicationExecutionCleanupAgentConfigurationProperties import com.netflix.spinnaker.orca.api.pipeline.models.ExecutionType import com.netflix.spinnaker.orca.notifications.NotificationClusterLock +import com.netflix.spinnaker.orca.notifications.scheduling.PipelineDependencyCleanupOperator import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository import java.util.concurrent.atomic.AtomicInteger import org.jooq.DSLContext @@ -39,7 +40,8 @@ class TopApplicationExecutionCleanupPollingNotificationAgent( registry: Registry, private val executionRepository: ExecutionRepository, private val configurationProperties: TopApplicationExecutionCleanupAgentConfigurationProperties, - private val orcaSqlProperties: OrcaSqlProperties + private val orcaSqlProperties: OrcaSqlProperties, + private val pipelineDependencyCleanupOperators: List ) : AbstractCleanupPollingAgent( clusterLock, configurationProperties.intervalMs, @@ -119,6 +121,8 @@ class TopApplicationExecutionCleanupPollingNotificationAgent( log.debug("Found {} old orchestrations for {}", executionsToRemove.size, application) + pipelineDependencyCleanupOperators.forEach { it.cleanup(executionsToRemove) } + executionsToRemove.chunked(configurationProperties.chunkSize).forEach { ids -> deletedExecutionCount.addAndGet(ids.size) diff --git a/orca-sql/src/test/groovy/com/netflix/spinnaker/orca/sql/cleanup/OldPipelineCleanupPollingNotificationAgentSpec.groovy b/orca-sql/src/test/groovy/com/netflix/spinnaker/orca/sql/cleanup/OldPipelineCleanupPollingNotificationAgentSpec.groovy index 3b2841fe50..fb4c0ff93b 100644 --- a/orca-sql/src/test/groovy/com/netflix/spinnaker/orca/sql/cleanup/OldPipelineCleanupPollingNotificationAgentSpec.groovy +++ b/orca-sql/src/test/groovy/com/netflix/spinnaker/orca/sql/cleanup/OldPipelineCleanupPollingNotificationAgentSpec.groovy @@ -73,7 +73,8 @@ abstract class OldPipelineCleanupPollingNotificationAgentSpec extends Specificat ["exceptionalApp"], 50 ), - new OrcaSqlProperties() + new OrcaSqlProperties(), + [] ) def setupSpec() { diff --git a/orca-sql/src/test/groovy/com/netflix/spinnaker/orca/sql/cleanup/TopApplicationExecutionCleanupPollingNotificationAgentSpec.groovy b/orca-sql/src/test/groovy/com/netflix/spinnaker/orca/sql/cleanup/TopApplicationExecutionCleanupPollingNotificationAgentSpec.groovy index 9983f7677a..b6530ce265 100644 --- a/orca-sql/src/test/groovy/com/netflix/spinnaker/orca/sql/cleanup/TopApplicationExecutionCleanupPollingNotificationAgentSpec.groovy +++ b/orca-sql/src/test/groovy/com/netflix/spinnaker/orca/sql/cleanup/TopApplicationExecutionCleanupPollingNotificationAgentSpec.groovy @@ -74,7 +74,8 @@ abstract class TopApplicationExecutionCleanupPollingNotificationAgentSpec extend exceptionApp2 : 5 ] ), - new OrcaSqlProperties() + new OrcaSqlProperties(), + [] ) def setupSpec() {