CLOUDP-82115: Implement delete retry loop

pkg/controller/atlas/api_error.go

@@ -13,4 +13,7 @@ const (
 
 	// Error indicates that the database user doesn't exist
 	UsernameNotFound = "USERNAME_NOT_FOUND"
+
+	// Error indicates that the cluster doesn't exist
+	ClusterNotFound = "CLUSTER_NOT_FOUND"
 )

pkg/controller/atlascluster/atlascluster_controller.go

@@ -20,7 +20,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"time"
 
+	"go.mongodb.org/atlas/mongodbatlas"
 	"go.uber.org/zap"
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -150,6 +152,8 @@ func (r *AtlasClusterReconciler) Delete(e event.DeleteEvent) error {
 		return errors.New("cannot read project resource")
 	}
 
+	log = log.With("projectID", project.Status.ID, "clusterName", cluster.Spec.Name)
+
 	connection, err := atlas.ReadConnection(log, r.Client, r.OperatorPod, project.ConnectionSecretObjectKey())
 	if err != nil {
 		return err
@@ -160,12 +164,28 @@ func (r *AtlasClusterReconciler) Delete(e event.DeleteEvent) error {
 		return fmt.Errorf("cannot build Atlas client: %w", err)
 	}
 
-	_, err = atlasClient.Clusters.Delete(context.Background(), project.Status.ID, cluster.Spec.Name)
-	if err != nil {
-		return fmt.Errorf("cannot delete Atlas cluster: %w", err)
-	}
-
-	log.Infow("Started Atlas cluster deletion process", "projectID", project.Status.ID, "clusterName", cluster.Spec.Name)
-
+	go func() {
+		timeout := time.Now().Add(workflow.DefaultTimeout)
+
+		for time.Now().Before(timeout) {
+			_, err = atlasClient.Clusters.Delete(context.Background(), project.Status.ID, cluster.Spec.Name)
+			var apiError *mongodbatlas.ErrorResponse
+			if errors.As(err, &apiError) && apiError.ErrorCode == atlas.ClusterNotFound {
+				log.Info("Cluster doesn't exist or is already deleted")
+				return
+			}
+
+			if err != nil {
+				log.Errorw("cannot delete Atlas cluster", "error", err)
+				time.Sleep(workflow.DefaultRetry)
+				continue
+			}
+
+			log.Info("Started Atlas cluster deletion process")
+			return
+		}
+
+		log.Error("Failed to delete Atlas cluster in time")
+	}()
 	return nil
 }

pkg/controller/atlasproject/atlasproject_controller.go

@@ -18,8 +18,11 @@ package atlasproject
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"time"
 
+	"go.mongodb.org/atlas/mongodbatlas"
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -134,12 +137,29 @@ func (r *AtlasProjectReconciler) Delete(e event.DeleteEvent) error {
 		return fmt.Errorf("cannot build Atlas client: %w", err)
 	}
 
-	_, err = atlasClient.Projects.Delete(context.Background(), project.Status.ID)
-	if err != nil {
-		return fmt.Errorf("cannot delete Atlas project: %w", err)
-	}
-
-	log.Infow("Successfully deleted Atlas project", "projectID", project.Status.ID)
+	go func() {
+		timeout := time.Now().Add(workflow.DefaultTimeout)
+
+		for time.Now().Before(timeout) {
+			_, err = atlasClient.Projects.Delete(context.Background(), project.Status.ID)
+			var apiError *mongodbatlas.ErrorResponse
+			if errors.As(err, &apiError) && apiError.ErrorCode == atlas.NotInGroup {
+				log.Infow("Project doesn't exist or is already deleted", "projectID", project.Status.ID)
+				return
+			}
+
+			if err != nil {
+				log.Errorw("cannot delete Atlas project", "error", err)
+				time.Sleep(workflow.DefaultRetry)
+				continue
+			}
+
+			log.Infow("Successfully deleted Atlas project", "projectID", project.Status.ID)
+			return
+		}
+
+		log.Errorw("Failed to delete Atlas project in time", "projectID", project.Status.ID)
+	}()
 
 	return nil
 }

pkg/controller/workflow/result.go

@@ -6,7 +6,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
-const DefaultRetry = time.Second * 10
+const (
+	DefaultRetry   = time.Second * 10
+	DefaultTimeout = time.Minute * 20
+)
 
 type Result struct {
 	terminated   bool

test/int/cluster_test.go

@@ -58,19 +58,12 @@ var _ = Describe("AtlasCluster", func() {
 			if createdCluster != nil {
 				By("Removing Atlas Cluster " + createdCluster.Name)
 				Expect(k8sClient.Delete(context.Background(), createdCluster)).To(Succeed())
-				Eventually(checkAtlasClusterRemoved(createdProject.Status.ID, createdCluster.Name), 600, interval).Should(BeTrue())
+				Eventually(checkAtlasClusterRemoved(createdProject.Status.ID, createdCluster.Spec.Name), 600, interval).Should(BeTrue())
 			}
 
-			// TODO: CLOUDP-82115
-			// By("Removing Atlas Project " + createdProject.Status.ID)
-			// Expect(k8sClient.Delete(context.Background(), createdProject)).To(Succeed())
-			// Eventually(checkAtlasProjectRemoved(createdProject.Status.ID), 20, interval).Should(BeTrue())
-
 			By("Removing Atlas Project " + createdProject.Status.ID)
-			// This is a bit strange but the delete request right after the cluster is removed may fail with "Still active cluster" error
-			// UI shows the cluster being deleted though. Seems to be the issue only if removal is done using API,
-			// if the cluster is terminated using UI - it stays in "Deleting" state
-			Eventually(removeAtlasProject(createdProject.Status.ID), 600, interval).Should(BeTrue())
+			Expect(k8sClient.Delete(context.Background(), createdProject)).To(Succeed())
+			Eventually(checkAtlasProjectRemoved(createdProject.Status.ID), 60, interval).Should(BeTrue())
 		}
 		removeControllersAndNamespace()
 	})

test/int/dbuser_test.go

@@ -28,12 +28,13 @@ import (
 	"github.com/mongodb/mongodb-atlas-kubernetes/pkg/util/testutil"
 )
 
-const DevMode = false
-
-const UserPasswordSecret = "user-password-secret"
-const DBUserPassword = "Passw0rd!"
-const UserPasswordSecret2 = "second-user-password-secret"
-const DBUserPassword2 = "H@lla#!"
+const (
+	DevMode             = false
+	UserPasswordSecret  = "user-password-secret"
+	DBUserPassword      = "Passw0rd!"
+	UserPasswordSecret2 = "second-user-password-secret"
+	DBUserPassword2     = "H@lla#!"
+)
 
 var _ = Describe("AtlasDatabaseUser", func() {
 	const interval = time.Second * 1
@@ -101,20 +102,23 @@ var _ = Describe("AtlasDatabaseUser", func() {
 
 			return
 		}
+
 		if createdProject != nil && createdProject.ID() != "" {
 			if createdClusterGCP != nil {
 				By("Removing Atlas Cluster " + createdClusterGCP.Name)
 				Expect(k8sClient.Delete(context.Background(), createdClusterGCP)).To(Succeed())
-				Eventually(checkAtlasClusterRemoved(createdProject.Status.ID, createdClusterGCP.Name), 600, interval).Should(BeTrue())
+				Eventually(checkAtlasClusterRemoved(createdProject.Status.ID, createdClusterGCP.Spec.Name), 600, interval).Should(BeTrue())
 			}
+
 			if createdClusterAWS != nil {
 				By("Removing Atlas Cluster " + createdClusterAWS.Name)
 				Expect(k8sClient.Delete(context.Background(), createdClusterAWS)).To(Succeed())
-				Eventually(checkAtlasClusterRemoved(createdProject.Status.ID, createdClusterAWS.Name), 600, interval).Should(BeTrue())
+				Eventually(checkAtlasClusterRemoved(createdProject.Status.ID, createdClusterAWS.Spec.Name), 600, interval).Should(BeTrue())
 			}
 
 			By("Removing Atlas Project " + createdProject.Status.ID)
-			Eventually(removeAtlasProject(createdProject.Status.ID), 600, interval).Should(BeTrue())
+			Expect(k8sClient.Delete(context.Background(), createdProject)).To(Succeed())
+			Eventually(checkAtlasProjectRemoved(createdProject.Status.ID), 60, interval).Should(BeTrue())
 		}
 		removeControllersAndNamespace()
 	})
@@ -289,12 +293,14 @@ func normalize(user mongodbatlas.DatabaseUser, projectID string) mongodbatlas.Da
 	user.Password = ""
 	return user
 }
+
 func tryConnect(projectID string, cluster mdbv1.AtlasCluster, user mdbv1.AtlasDatabaseUser) func() error {
 	return func() error {
 		_, err := mongoClient(projectID, cluster, user)
 		return err
 	}
 }
+
 func mongoClient(projectID string, cluster mdbv1.AtlasCluster, user mdbv1.AtlasDatabaseUser) (*mongo.Client, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()

test/int/project_test.go

@@ -373,10 +373,8 @@ var _ = Describe("AtlasProject", func() {
 				Eventually(testutil.WaitFor(k8sClient, createdProject, status.TrueCondition(status.ReadyType)),
 					20, interval).Should(BeTrue())
 			})
-
 		})
 	})
-
 })
 
 func buildConnectionSecret(name string) corev1.Secret {
@@ -389,19 +387,6 @@ func buildConnectionSecret(name string) corev1.Secret {
 	}
 }
 
-func removeAtlasProject(projectID string) func() bool {
-	return func() bool {
-		_, err := atlasClient.Projects.Delete(context.Background(), projectID)
-		if err != nil {
-			var apiError *mongodbatlas.ErrorResponse
-			Expect(errors.As(err, &apiError)).To(BeTrue())
-			Expect(apiError.ErrorCode).To(Equal(atlas.CannotCloseGroupActiveAtlasCluster))
-			return false
-		}
-		return true
-	}
-}
-
 // checkAtlasProjectRemoved returns true if the Atlas Project is removed from Atlas.
 func checkAtlasProjectRemoved(projectID string) func() bool {
 	return func() bool {