From c35ae94f9ae87ac2ea13a8c657ee6b1bd06f2554 Mon Sep 17 00:00:00 2001 From: Shiwei Tang Date: Sun, 7 Apr 2024 01:05:51 +0800 Subject: [PATCH] fix: workflows that are retrying should not be deleted (Fixes #12636) Co-authored-by: Anton Gilgur <4970083+agilgur5@users.noreply.github.com> Signed-off-by: Shiwei Tang --- workflow/gccontroller/gc_controller.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/workflow/gccontroller/gc_controller.go b/workflow/gccontroller/gc_controller.go index 05da97603671..32751df976dc 100644 --- a/workflow/gccontroller/gc_controller.go +++ b/workflow/gccontroller/gc_controller.go @@ -26,7 +26,7 @@ import ( "github.com/argoproj/argo-workflows/v3/workflow/util" ) -var ticker *time.Ticker = time.NewTicker(50 * time.Millisecond) +var ticker = time.NewTicker(50 * time.Millisecond) type Controller struct { wfclientset wfclientset.Interface @@ -211,9 +211,22 @@ func (c *Controller) deleteWorkflow(ctx context.Context, key string) error { // It should be impossible for a workflow to have been queue without a valid key. namespace, name, _ := cache.SplitMetaNamespaceKey(key) + // Double check that this workflow is still completed. If it were retried, it may be running again (c.f. https://github.com/argoproj/argo-workflows/issues/12636) + obj, exists, err := c.wfInformer.GetStore().GetByKey(key) + if err != nil { + return nil + } + if exists { + un, ok := obj.(*unstructured.Unstructured) + if ok && !common.IsDone(un) { + log.Infof("Workflow '%s' is not completed due to a retry operation, ignore deletion", key) + return nil + } + } + // Any workflow that was queued must need deleting, therefore we do not check the expiry again. log.Infof("Deleting garbage collected workflow '%s'", key) - err := c.wfclientset.ArgoprojV1alpha1().Workflows(namespace).Delete(ctx, name, metav1.DeleteOptions{PropagationPolicy: commonutil.GetDeletePropagation()}) + err = c.wfclientset.ArgoprojV1alpha1().Workflows(namespace).Delete(ctx, name, metav1.DeleteOptions{PropagationPolicy: commonutil.GetDeletePropagation()}) if err != nil { if apierr.IsNotFound(err) { log.Infof("Workflow already deleted '%s'", key)