diff --git a/pkg/controller/atlas/api_error.go b/pkg/controller/atlas/api_error.go index 803b06f4f4..e151600239 100644 --- a/pkg/controller/atlas/api_error.go +++ b/pkg/controller/atlas/api_error.go @@ -13,4 +13,7 @@ const ( // Error indicates that the database user doesn't exist UsernameNotFound = "USERNAME_NOT_FOUND" + + // Error indicates that the cluster doesn't exist + ClusterNotFound = "CLUSTER_NOT_FOUND" ) diff --git a/pkg/controller/atlascluster/atlascluster_controller.go b/pkg/controller/atlascluster/atlascluster_controller.go index 95e2f1a508..465201f277 100644 --- a/pkg/controller/atlascluster/atlascluster_controller.go +++ b/pkg/controller/atlascluster/atlascluster_controller.go @@ -20,7 +20,9 @@ import ( "context" "errors" "fmt" + "time" + "go.mongodb.org/atlas/mongodbatlas" "go.uber.org/zap" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" @@ -150,6 +152,8 @@ func (r *AtlasClusterReconciler) Delete(e event.DeleteEvent) error { return errors.New("cannot read project resource") } + log = log.With("projectID", project.Status.ID, "clusterName", cluster.Spec.Name) + connection, err := atlas.ReadConnection(log, r.Client, r.OperatorPod, project.ConnectionSecretObjectKey()) if err != nil { return err @@ -160,12 +164,28 @@ func (r *AtlasClusterReconciler) Delete(e event.DeleteEvent) error { return fmt.Errorf("cannot build Atlas client: %w", err) } - _, err = atlasClient.Clusters.Delete(context.Background(), project.Status.ID, cluster.Spec.Name) - if err != nil { - return fmt.Errorf("cannot delete Atlas cluster: %w", err) - } - - log.Infow("Started Atlas cluster deletion process", "projectID", project.Status.ID, "clusterName", cluster.Spec.Name) - + go func() { + timeout := time.Now().Add(workflow.DefaultTimeout) + + for time.Now().Before(timeout) { + _, err = atlasClient.Clusters.Delete(context.Background(), project.Status.ID, cluster.Spec.Name) + var apiError *mongodbatlas.ErrorResponse + if errors.As(err, &apiError) && apiError.ErrorCode == atlas.ClusterNotFound { + log.Info("Cluster doesn't exist or is already deleted") + return + } + + if err != nil { + log.Errorw("cannot delete Atlas cluster", "error", err) + time.Sleep(workflow.DefaultRetry) + continue + } + + log.Info("Started Atlas cluster deletion process") + return + } + + log.Error("Failed to delete Atlas cluster in time") + }() return nil } diff --git a/pkg/controller/atlasproject/atlasproject_controller.go b/pkg/controller/atlasproject/atlasproject_controller.go index 61ade1ba1e..f7cc19900e 100644 --- a/pkg/controller/atlasproject/atlasproject_controller.go +++ b/pkg/controller/atlasproject/atlasproject_controller.go @@ -18,8 +18,11 @@ package atlasproject import ( "context" + "errors" "fmt" + "time" + "go.mongodb.org/atlas/mongodbatlas" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -134,12 +137,29 @@ func (r *AtlasProjectReconciler) Delete(e event.DeleteEvent) error { return fmt.Errorf("cannot build Atlas client: %w", err) } - _, err = atlasClient.Projects.Delete(context.Background(), project.Status.ID) - if err != nil { - return fmt.Errorf("cannot delete Atlas project: %w", err) - } - - log.Infow("Successfully deleted Atlas project", "projectID", project.Status.ID) + go func() { + timeout := time.Now().Add(workflow.DefaultTimeout) + + for time.Now().Before(timeout) { + _, err = atlasClient.Projects.Delete(context.Background(), project.Status.ID) + var apiError *mongodbatlas.ErrorResponse + if errors.As(err, &apiError) && apiError.ErrorCode == atlas.NotInGroup { + log.Infow("Project doesn't exist or is already deleted", "projectID", project.Status.ID) + return + } + + if err != nil { + log.Errorw("cannot delete Atlas project", "error", err) + time.Sleep(workflow.DefaultRetry) + continue + } + + log.Infow("Successfully deleted Atlas project", "projectID", project.Status.ID) + return + } + + log.Errorw("Failed to delete Atlas project in time", "projectID", project.Status.ID) + }() return nil } diff --git a/pkg/controller/workflow/result.go b/pkg/controller/workflow/result.go index 43536583a4..0016f4bdbe 100644 --- a/pkg/controller/workflow/result.go +++ b/pkg/controller/workflow/result.go @@ -6,7 +6,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -const DefaultRetry = time.Second * 10 +const ( + DefaultRetry = time.Second * 10 + DefaultTimeout = time.Minute * 20 +) type Result struct { terminated bool diff --git a/test/int/cluster_test.go b/test/int/cluster_test.go index c9b720a593..9c727f1a45 100644 --- a/test/int/cluster_test.go +++ b/test/int/cluster_test.go @@ -58,19 +58,12 @@ var _ = Describe("AtlasCluster", func() { if createdCluster != nil { By("Removing Atlas Cluster " + createdCluster.Name) Expect(k8sClient.Delete(context.Background(), createdCluster)).To(Succeed()) - Eventually(checkAtlasClusterRemoved(createdProject.Status.ID, createdCluster.Name), 600, interval).Should(BeTrue()) + Eventually(checkAtlasClusterRemoved(createdProject.Status.ID, createdCluster.Spec.Name), 600, interval).Should(BeTrue()) } - // TODO: CLOUDP-82115 - // By("Removing Atlas Project " + createdProject.Status.ID) - // Expect(k8sClient.Delete(context.Background(), createdProject)).To(Succeed()) - // Eventually(checkAtlasProjectRemoved(createdProject.Status.ID), 20, interval).Should(BeTrue()) - By("Removing Atlas Project " + createdProject.Status.ID) - // This is a bit strange but the delete request right after the cluster is removed may fail with "Still active cluster" error - // UI shows the cluster being deleted though. Seems to be the issue only if removal is done using API, - // if the cluster is terminated using UI - it stays in "Deleting" state - Eventually(removeAtlasProject(createdProject.Status.ID), 600, interval).Should(BeTrue()) + Expect(k8sClient.Delete(context.Background(), createdProject)).To(Succeed()) + Eventually(checkAtlasProjectRemoved(createdProject.Status.ID), 60, interval).Should(BeTrue()) } removeControllersAndNamespace() }) diff --git a/test/int/dbuser_test.go b/test/int/dbuser_test.go index c1ad868e99..df19c06d59 100644 --- a/test/int/dbuser_test.go +++ b/test/int/dbuser_test.go @@ -28,12 +28,13 @@ import ( "github.com/mongodb/mongodb-atlas-kubernetes/pkg/util/testutil" ) -const DevMode = false - -const UserPasswordSecret = "user-password-secret" -const DBUserPassword = "Passw0rd!" -const UserPasswordSecret2 = "second-user-password-secret" -const DBUserPassword2 = "H@lla#!" +const ( + DevMode = false + UserPasswordSecret = "user-password-secret" + DBUserPassword = "Passw0rd!" + UserPasswordSecret2 = "second-user-password-secret" + DBUserPassword2 = "H@lla#!" +) var _ = Describe("AtlasDatabaseUser", func() { const interval = time.Second * 1 @@ -101,20 +102,23 @@ var _ = Describe("AtlasDatabaseUser", func() { return } + if createdProject != nil && createdProject.ID() != "" { if createdClusterGCP != nil { By("Removing Atlas Cluster " + createdClusterGCP.Name) Expect(k8sClient.Delete(context.Background(), createdClusterGCP)).To(Succeed()) - Eventually(checkAtlasClusterRemoved(createdProject.Status.ID, createdClusterGCP.Name), 600, interval).Should(BeTrue()) + Eventually(checkAtlasClusterRemoved(createdProject.Status.ID, createdClusterGCP.Spec.Name), 600, interval).Should(BeTrue()) } + if createdClusterAWS != nil { By("Removing Atlas Cluster " + createdClusterAWS.Name) Expect(k8sClient.Delete(context.Background(), createdClusterAWS)).To(Succeed()) - Eventually(checkAtlasClusterRemoved(createdProject.Status.ID, createdClusterAWS.Name), 600, interval).Should(BeTrue()) + Eventually(checkAtlasClusterRemoved(createdProject.Status.ID, createdClusterAWS.Spec.Name), 600, interval).Should(BeTrue()) } By("Removing Atlas Project " + createdProject.Status.ID) - Eventually(removeAtlasProject(createdProject.Status.ID), 600, interval).Should(BeTrue()) + Expect(k8sClient.Delete(context.Background(), createdProject)).To(Succeed()) + Eventually(checkAtlasProjectRemoved(createdProject.Status.ID), 60, interval).Should(BeTrue()) } removeControllersAndNamespace() }) @@ -289,12 +293,14 @@ func normalize(user mongodbatlas.DatabaseUser, projectID string) mongodbatlas.Da user.Password = "" return user } + func tryConnect(projectID string, cluster mdbv1.AtlasCluster, user mdbv1.AtlasDatabaseUser) func() error { return func() error { _, err := mongoClient(projectID, cluster, user) return err } } + func mongoClient(projectID string, cluster mdbv1.AtlasCluster, user mdbv1.AtlasDatabaseUser) (*mongo.Client, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() diff --git a/test/int/project_test.go b/test/int/project_test.go index 2c6f768653..5d0d6de5e3 100644 --- a/test/int/project_test.go +++ b/test/int/project_test.go @@ -373,10 +373,8 @@ var _ = Describe("AtlasProject", func() { Eventually(testutil.WaitFor(k8sClient, createdProject, status.TrueCondition(status.ReadyType)), 20, interval).Should(BeTrue()) }) - }) }) - }) func buildConnectionSecret(name string) corev1.Secret { @@ -389,19 +387,6 @@ func buildConnectionSecret(name string) corev1.Secret { } } -func removeAtlasProject(projectID string) func() bool { - return func() bool { - _, err := atlasClient.Projects.Delete(context.Background(), projectID) - if err != nil { - var apiError *mongodbatlas.ErrorResponse - Expect(errors.As(err, &apiError)).To(BeTrue()) - Expect(apiError.ErrorCode).To(Equal(atlas.CannotCloseGroupActiveAtlasCluster)) - return false - } - return true - } -} - // checkAtlasProjectRemoved returns true if the Atlas Project is removed from Atlas. func checkAtlasProjectRemoved(projectID string) func() bool { return func() bool {