From 7aecda9139366b7f516f7f01af30ac09bef29a37 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 5 Mar 2021 12:51:16 +0300 Subject: [PATCH 01/11] Implement delete retry loop --- .../atlascluster/atlascluster_controller.go | 15 ++++++++++++--- .../atlasproject/atlasproject_controller.go | 15 ++++++++++++--- pkg/controller/workflow/result.go | 5 ++++- test/int/cluster_test.go | 10 ++-------- test/int/dbuser_test.go | 14 ++++++++++---- 5 files changed, 40 insertions(+), 19 deletions(-) diff --git a/pkg/controller/atlascluster/atlascluster_controller.go b/pkg/controller/atlascluster/atlascluster_controller.go index e5f0cd85dd..c9d8dab10b 100644 --- a/pkg/controller/atlascluster/atlascluster_controller.go +++ b/pkg/controller/atlascluster/atlascluster_controller.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "time" "go.uber.org/zap" "k8s.io/apimachinery/pkg/runtime" @@ -160,9 +161,17 @@ func (r *AtlasClusterReconciler) Delete(e event.DeleteEvent) error { return fmt.Errorf("cannot build Atlas client: %w", err) } - _, err = atlasClient.Clusters.Delete(context.Background(), project.Status.ID, cluster.Spec.Name) - if err != nil { - return fmt.Errorf("cannot delete Atlas cluster: %w", err) + timeout := time.Now().Add(workflow.DefaultTimeout) + + for time.Now().Before(timeout) { + _, err = atlasClient.Clusters.Delete(context.Background(), project.Status.ID, cluster.Spec.Name) + if err != nil { + log.Errorw("cannot delete Atlas cluster", "error", err) + time.Sleep(workflow.DefaultRetry) + continue + } + + break } log.Infow("Started Atlas cluster deletion process", "projectID", project.Status.ID, "clusterName", cluster.Name) diff --git a/pkg/controller/atlasproject/atlasproject_controller.go b/pkg/controller/atlasproject/atlasproject_controller.go index 61ade1ba1e..1d89811128 100644 --- a/pkg/controller/atlasproject/atlasproject_controller.go +++ b/pkg/controller/atlasproject/atlasproject_controller.go @@ -19,6 +19,7 @@ package atlasproject import ( "context" "fmt" + "time" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" @@ -134,9 +135,17 @@ func (r *AtlasProjectReconciler) Delete(e event.DeleteEvent) error { return fmt.Errorf("cannot build Atlas client: %w", err) } - _, err = atlasClient.Projects.Delete(context.Background(), project.Status.ID) - if err != nil { - return fmt.Errorf("cannot delete Atlas project: %w", err) + timeout := time.Now().Add(workflow.DefaultTimeout) + + for time.Now().Before(timeout) { + _, err = atlasClient.Projects.Delete(context.Background(), project.Status.ID) + if err != nil { + log.Errorw("cannot delete Atlas project", "error", err) + time.Sleep(workflow.DefaultRetry) + continue + } + + break } log.Infow("Successfully deleted Atlas project", "projectID", project.Status.ID) diff --git a/pkg/controller/workflow/result.go b/pkg/controller/workflow/result.go index 43536583a4..0016f4bdbe 100644 --- a/pkg/controller/workflow/result.go +++ b/pkg/controller/workflow/result.go @@ -6,7 +6,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -const DefaultRetry = time.Second * 10 +const ( + DefaultRetry = time.Second * 10 + DefaultTimeout = time.Minute * 20 +) type Result struct { terminated bool diff --git a/test/int/cluster_test.go b/test/int/cluster_test.go index 4429576823..e0b83b67c2 100644 --- a/test/int/cluster_test.go +++ b/test/int/cluster_test.go @@ -62,15 +62,9 @@ var _ = Describe("AtlasCluster", func() { } // TODO: CLOUDP-82115 - // By("Removing Atlas Project " + createdProject.Status.ID) - // Expect(k8sClient.Delete(context.Background(), createdProject)).To(Succeed()) - // Eventually(checkAtlasProjectRemoved(createdProject.Status.ID), 20, interval).Should(BeTrue()) - By("Removing Atlas Project " + createdProject.Status.ID) - // This is a bit strange but the delete request right after the cluster is removed may fail with "Still active cluster" error - // UI shows the cluster being deleted though. Seems to be the issue only if removal is done using API, - // if the cluster is terminated using UI - it stays in "Deleting" state - Eventually(removeAtlasProject(createdProject.Status.ID), 600, interval).Should(BeTrue()) + Expect(k8sClient.Delete(context.Background(), createdProject)).To(Succeed()) + Eventually(checkAtlasProjectRemoved(createdProject.Status.ID), 20, interval).Should(BeTrue()) } removeControllersAndNamespace() }) diff --git a/test/int/dbuser_test.go b/test/int/dbuser_test.go index 25acf02844..9271d9c9f7 100644 --- a/test/int/dbuser_test.go +++ b/test/int/dbuser_test.go @@ -23,9 +23,11 @@ import ( "github.com/mongodb/mongodb-atlas-kubernetes/pkg/util/testutil" ) -const UserPasswordSecret = "user-password-secret" -const DevMode = false -const DBUserPassword = "Passw0rd!" +const ( + UserPasswordSecret = "user-password-secret" + DevMode = false + DBUserPassword = "Passw0rd!" +) var _ = Describe("AtlasDatabaseUser", func() { const interval = time.Second * 1 @@ -91,6 +93,7 @@ var _ = Describe("AtlasDatabaseUser", func() { return } + if createdProject != nil && createdProject.ID() != "" { if createdClusterGCP != nil { By("Removing Atlas Cluster " + createdClusterGCP.Name) @@ -104,7 +107,8 @@ var _ = Describe("AtlasDatabaseUser", func() { } By("Removing Atlas Project " + createdProject.Status.ID) - Eventually(removeAtlasProject(createdProject.Status.ID), 600, interval).Should(BeTrue()) + Expect(k8sClient.Delete(context.Background(), createdProject)).To(Succeed()) + Eventually(checkAtlasProjectRemoved(createdProject.Status.ID), 20, interval).Should(BeTrue()) } removeControllersAndNamespace() }) @@ -199,12 +203,14 @@ func normalize(user mongodbatlas.DatabaseUser, projectID string) mongodbatlas.Da user.Password = "" return user } + func tryConnect(projectID string, cluster mdbv1.AtlasCluster, user mdbv1.AtlasDatabaseUser) func() error { return func() error { _, err := mongoClient(projectID, cluster, user) return err } } + func mongoClient(projectID string, cluster mdbv1.AtlasCluster, user mdbv1.AtlasDatabaseUser) (*mongo.Client, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() From a66ffddb889049abcba8ba50b741098ce0a1ec11 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 5 Mar 2021 14:14:27 +0300 Subject: [PATCH 02/11] test --- test/int/cluster_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/int/cluster_test.go b/test/int/cluster_test.go index e0b83b67c2..d4bc31013f 100644 --- a/test/int/cluster_test.go +++ b/test/int/cluster_test.go @@ -61,10 +61,9 @@ var _ = Describe("AtlasCluster", func() { Eventually(checkAtlasClusterRemoved(createdProject.Status.ID, createdCluster.Name), 600, interval).Should(BeTrue()) } - // TODO: CLOUDP-82115 By("Removing Atlas Project " + createdProject.Status.ID) Expect(k8sClient.Delete(context.Background(), createdProject)).To(Succeed()) - Eventually(checkAtlasProjectRemoved(createdProject.Status.ID), 20, interval).Should(BeTrue()) + Eventually(checkAtlasProjectRemoved(createdProject.Status.ID), 600, interval).Should(BeTrue()) } removeControllersAndNamespace() }) @@ -350,10 +349,12 @@ func checkAtlasClusterRemoved(projectID string, clusterName string) func() bool _, r, err := atlasClient.Clusters.Get(context.Background(), projectID, clusterName) if err != nil { if r != nil && r.StatusCode == http.StatusNotFound { + fmt.Println("cluster removed!", time.Now()) return true } } + fmt.Println("cluster exists", time.Now()) return false } } From 1832036cae9e12dec1848eaa2c64a9b75bd86eed Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 5 Mar 2021 14:57:22 +0300 Subject: [PATCH 03/11] test --- test/int/cluster_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/int/cluster_test.go b/test/int/cluster_test.go index d4bc31013f..3ea4ae20a6 100644 --- a/test/int/cluster_test.go +++ b/test/int/cluster_test.go @@ -349,7 +349,7 @@ func checkAtlasClusterRemoved(projectID string, clusterName string) func() bool _, r, err := atlasClient.Clusters.Get(context.Background(), projectID, clusterName) if err != nil { if r != nil && r.StatusCode == http.StatusNotFound { - fmt.Println("cluster removed!", time.Now()) + fmt.Println("cluster removed!", time.Now(), projectID, clusterName) return true } } From 2cc8a5395d85ccaf01fb3192a2e1495691e0b225 Mon Sep 17 00:00:00 2001 From: Pavel Vasilev Date: Fri, 5 Mar 2021 15:47:56 +0300 Subject: [PATCH 04/11] Increase project delete timeout --- test/int/dbuser_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/int/dbuser_test.go b/test/int/dbuser_test.go index 9271d9c9f7..7db48ef05a 100644 --- a/test/int/dbuser_test.go +++ b/test/int/dbuser_test.go @@ -108,7 +108,7 @@ var _ = Describe("AtlasDatabaseUser", func() { By("Removing Atlas Project " + createdProject.Status.ID) Expect(k8sClient.Delete(context.Background(), createdProject)).To(Succeed()) - Eventually(checkAtlasProjectRemoved(createdProject.Status.ID), 20, interval).Should(BeTrue()) + Eventually(checkAtlasProjectRemoved(createdProject.Status.ID), 600, interval).Should(BeTrue()) } removeControllersAndNamespace() }) From 34455f247bcd5f2975b49908fd1520ccb2e5e666 Mon Sep 17 00:00:00 2001 From: Pavel Vasilev Date: Fri, 5 Mar 2021 15:50:14 +0300 Subject: [PATCH 05/11] Remove unused method --- test/int/project_test.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/test/int/project_test.go b/test/int/project_test.go index 2c6f768653..5d0d6de5e3 100644 --- a/test/int/project_test.go +++ b/test/int/project_test.go @@ -373,10 +373,8 @@ var _ = Describe("AtlasProject", func() { Eventually(testutil.WaitFor(k8sClient, createdProject, status.TrueCondition(status.ReadyType)), 20, interval).Should(BeTrue()) }) - }) }) - }) func buildConnectionSecret(name string) corev1.Secret { @@ -389,19 +387,6 @@ func buildConnectionSecret(name string) corev1.Secret { } } -func removeAtlasProject(projectID string) func() bool { - return func() bool { - _, err := atlasClient.Projects.Delete(context.Background(), projectID) - if err != nil { - var apiError *mongodbatlas.ErrorResponse - Expect(errors.As(err, &apiError)).To(BeTrue()) - Expect(apiError.ErrorCode).To(Equal(atlas.CannotCloseGroupActiveAtlasCluster)) - return false - } - return true - } -} - // checkAtlasProjectRemoved returns true if the Atlas Project is removed from Atlas. func checkAtlasProjectRemoved(projectID string) func() bool { return func() bool { From 3dd25faf8d0c69304c1cf0b5476dfac1fad1aacf Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 5 Mar 2021 17:07:55 +0300 Subject: [PATCH 06/11] Improve logging --- pkg/controller/atlascluster/atlascluster_controller.go | 6 +++--- pkg/controller/atlasproject/atlasproject_controller.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/controller/atlascluster/atlascluster_controller.go b/pkg/controller/atlascluster/atlascluster_controller.go index c9d8dab10b..b7ec970cab 100644 --- a/pkg/controller/atlascluster/atlascluster_controller.go +++ b/pkg/controller/atlascluster/atlascluster_controller.go @@ -171,10 +171,10 @@ func (r *AtlasClusterReconciler) Delete(e event.DeleteEvent) error { continue } - break + log.Infow("Started Atlas cluster deletion process", "projectID", project.Status.ID, "clusterName", cluster.Name) + return nil } - log.Infow("Started Atlas cluster deletion process", "projectID", project.Status.ID, "clusterName", cluster.Name) - + log.Errorw("Failed to delete Atlas cluster in time", "projectID", project.Status.ID, "clusterName", cluster.Name) return nil } diff --git a/pkg/controller/atlasproject/atlasproject_controller.go b/pkg/controller/atlasproject/atlasproject_controller.go index 1d89811128..2d244150ed 100644 --- a/pkg/controller/atlasproject/atlasproject_controller.go +++ b/pkg/controller/atlasproject/atlasproject_controller.go @@ -145,11 +145,11 @@ func (r *AtlasProjectReconciler) Delete(e event.DeleteEvent) error { continue } - break + log.Infow("Successfully deleted Atlas project", "projectID", project.Status.ID) + return nil } - log.Infow("Successfully deleted Atlas project", "projectID", project.Status.ID) - + log.Errorw("Failed to delete Atlas project in time", "projectID", project.Status.ID) return nil } From d91f97227edf91f49044e54390aaaaf025365f5b Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 5 Mar 2021 17:10:22 +0300 Subject: [PATCH 07/11] Remove debug outputs --- test/int/cluster_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/int/cluster_test.go b/test/int/cluster_test.go index 3ea4ae20a6..4e4480262c 100644 --- a/test/int/cluster_test.go +++ b/test/int/cluster_test.go @@ -349,12 +349,10 @@ func checkAtlasClusterRemoved(projectID string, clusterName string) func() bool _, r, err := atlasClient.Clusters.Get(context.Background(), projectID, clusterName) if err != nil { if r != nil && r.StatusCode == http.StatusNotFound { - fmt.Println("cluster removed!", time.Now(), projectID, clusterName) return true } } - fmt.Println("cluster exists", time.Now()) return false } } From d93ca2264b2bf6260a963769e4658242ec65a861 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 5 Mar 2021 17:58:30 +0300 Subject: [PATCH 08/11] Try to make delete-loops non-blocking --- .../atlascluster/atlascluster_controller.go | 28 ++++++++++--------- .../atlasproject/atlasproject_controller.go | 27 ++++++++++-------- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/pkg/controller/atlascluster/atlascluster_controller.go b/pkg/controller/atlascluster/atlascluster_controller.go index b7ec970cab..67685f2224 100644 --- a/pkg/controller/atlascluster/atlascluster_controller.go +++ b/pkg/controller/atlascluster/atlascluster_controller.go @@ -161,20 +161,22 @@ func (r *AtlasClusterReconciler) Delete(e event.DeleteEvent) error { return fmt.Errorf("cannot build Atlas client: %w", err) } - timeout := time.Now().Add(workflow.DefaultTimeout) - - for time.Now().Before(timeout) { - _, err = atlasClient.Clusters.Delete(context.Background(), project.Status.ID, cluster.Spec.Name) - if err != nil { - log.Errorw("cannot delete Atlas cluster", "error", err) - time.Sleep(workflow.DefaultRetry) - continue + go func() { + timeout := time.Now().Add(workflow.DefaultTimeout) + + for time.Now().Before(timeout) { + _, err = atlasClient.Clusters.Delete(context.Background(), project.Status.ID, cluster.Spec.Name) + if err != nil { + log.Errorw("cannot delete Atlas cluster", "error", err) + time.Sleep(workflow.DefaultRetry) + continue + } + + log.Infow("Started Atlas cluster deletion process", "projectID", project.Status.ID, "clusterName", cluster.Name) + return } - log.Infow("Started Atlas cluster deletion process", "projectID", project.Status.ID, "clusterName", cluster.Name) - return nil - } - - log.Errorw("Failed to delete Atlas cluster in time", "projectID", project.Status.ID, "clusterName", cluster.Name) + log.Errorw("Failed to delete Atlas cluster in time", "projectID", project.Status.ID, "clusterName", cluster.Name) + }() return nil } diff --git a/pkg/controller/atlasproject/atlasproject_controller.go b/pkg/controller/atlasproject/atlasproject_controller.go index 2d244150ed..df2e6472c5 100644 --- a/pkg/controller/atlasproject/atlasproject_controller.go +++ b/pkg/controller/atlasproject/atlasproject_controller.go @@ -135,21 +135,24 @@ func (r *AtlasProjectReconciler) Delete(e event.DeleteEvent) error { return fmt.Errorf("cannot build Atlas client: %w", err) } - timeout := time.Now().Add(workflow.DefaultTimeout) - - for time.Now().Before(timeout) { - _, err = atlasClient.Projects.Delete(context.Background(), project.Status.ID) - if err != nil { - log.Errorw("cannot delete Atlas project", "error", err) - time.Sleep(workflow.DefaultRetry) - continue + go func() { + timeout := time.Now().Add(workflow.DefaultTimeout) + + for time.Now().Before(timeout) { + _, err = atlasClient.Projects.Delete(context.Background(), project.Status.ID) + if err != nil { + log.Errorw("cannot delete Atlas project", "error", err) + time.Sleep(workflow.DefaultRetry) + continue + } + + log.Infow("Successfully deleted Atlas project", "projectID", project.Status.ID) + return } - log.Infow("Successfully deleted Atlas project", "projectID", project.Status.ID) - return nil - } + log.Errorw("Failed to delete Atlas project in time", "projectID", project.Status.ID) + }() - log.Errorw("Failed to delete Atlas project in time", "projectID", project.Status.ID) return nil } From 0e2e4a06c334357b66c7564b72699708d7282810 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 5 Mar 2021 18:15:30 +0300 Subject: [PATCH 09/11] Fix cluster deletion in tests using wrong Name --- test/int/cluster_test.go | 4 ++-- test/int/dbuser_test.go | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/test/int/cluster_test.go b/test/int/cluster_test.go index 4e4480262c..d9d055720c 100644 --- a/test/int/cluster_test.go +++ b/test/int/cluster_test.go @@ -58,12 +58,12 @@ var _ = Describe("AtlasCluster", func() { if createdCluster != nil { By("Removing Atlas Cluster " + createdCluster.Name) Expect(k8sClient.Delete(context.Background(), createdCluster)).To(Succeed()) - Eventually(checkAtlasClusterRemoved(createdProject.Status.ID, createdCluster.Name), 600, interval).Should(BeTrue()) + Eventually(checkAtlasClusterRemoved(createdProject.Status.ID, createdCluster.Spec.Name), 600, interval).Should(BeTrue()) } By("Removing Atlas Project " + createdProject.Status.ID) Expect(k8sClient.Delete(context.Background(), createdProject)).To(Succeed()) - Eventually(checkAtlasProjectRemoved(createdProject.Status.ID), 600, interval).Should(BeTrue()) + Eventually(checkAtlasProjectRemoved(createdProject.Status.ID), 60, interval).Should(BeTrue()) } removeControllersAndNamespace() }) diff --git a/test/int/dbuser_test.go b/test/int/dbuser_test.go index 7db48ef05a..ce25277215 100644 --- a/test/int/dbuser_test.go +++ b/test/int/dbuser_test.go @@ -98,17 +98,18 @@ var _ = Describe("AtlasDatabaseUser", func() { if createdClusterGCP != nil { By("Removing Atlas Cluster " + createdClusterGCP.Name) Expect(k8sClient.Delete(context.Background(), createdClusterGCP)).To(Succeed()) - Eventually(checkAtlasClusterRemoved(createdProject.Status.ID, createdClusterGCP.Name), 600, interval).Should(BeTrue()) + Eventually(checkAtlasClusterRemoved(createdProject.Status.ID, createdClusterGCP.Spec.Name), 600, interval).Should(BeTrue()) } + if createdClusterAWS != nil { By("Removing Atlas Cluster " + createdClusterAWS.Name) Expect(k8sClient.Delete(context.Background(), createdClusterAWS)).To(Succeed()) - Eventually(checkAtlasClusterRemoved(createdProject.Status.ID, createdClusterAWS.Name), 600, interval).Should(BeTrue()) + Eventually(checkAtlasClusterRemoved(createdProject.Status.ID, createdClusterAWS.Spec.Name), 600, interval).Should(BeTrue()) } By("Removing Atlas Project " + createdProject.Status.ID) Expect(k8sClient.Delete(context.Background(), createdProject)).To(Succeed()) - Eventually(checkAtlasProjectRemoved(createdProject.Status.ID), 600, interval).Should(BeTrue()) + Eventually(checkAtlasProjectRemoved(createdProject.Status.ID), 60, interval).Should(BeTrue()) } removeControllersAndNamespace() }) From 525a420c8b86c0eec3f685a20130474d51cac3c4 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 5 Mar 2021 19:33:27 +0300 Subject: [PATCH 10/11] Add early return in case resource already deleted --- pkg/controller/atlas/api_error.go | 3 +++ pkg/controller/atlascluster/atlascluster_controller.go | 7 +++++++ pkg/controller/atlasproject/atlasproject_controller.go | 8 ++++++++ 3 files changed, 18 insertions(+) diff --git a/pkg/controller/atlas/api_error.go b/pkg/controller/atlas/api_error.go index 803b06f4f4..e151600239 100644 --- a/pkg/controller/atlas/api_error.go +++ b/pkg/controller/atlas/api_error.go @@ -13,4 +13,7 @@ const ( // Error indicates that the database user doesn't exist UsernameNotFound = "USERNAME_NOT_FOUND" + + // Error indicates that the cluster doesn't exist + ClusterNotFound = "CLUSTER_NOT_FOUND" ) diff --git a/pkg/controller/atlascluster/atlascluster_controller.go b/pkg/controller/atlascluster/atlascluster_controller.go index 67685f2224..7463e1665a 100644 --- a/pkg/controller/atlascluster/atlascluster_controller.go +++ b/pkg/controller/atlascluster/atlascluster_controller.go @@ -22,6 +22,7 @@ import ( "fmt" "time" + "go.mongodb.org/atlas/mongodbatlas" "go.uber.org/zap" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" @@ -166,6 +167,12 @@ func (r *AtlasClusterReconciler) Delete(e event.DeleteEvent) error { for time.Now().Before(timeout) { _, err = atlasClient.Clusters.Delete(context.Background(), project.Status.ID, cluster.Spec.Name) + var apiError *mongodbatlas.ErrorResponse + if errors.As(err, &apiError) && apiError.ErrorCode == atlas.ClusterNotFound { + log.Infow("Cluster doesn't exist or is already deleted", "projectID", project.Status.ID, "clusterName", cluster.Name) + return + } + if err != nil { log.Errorw("cannot delete Atlas cluster", "error", err) time.Sleep(workflow.DefaultRetry) diff --git a/pkg/controller/atlasproject/atlasproject_controller.go b/pkg/controller/atlasproject/atlasproject_controller.go index df2e6472c5..95e8188d51 100644 --- a/pkg/controller/atlasproject/atlasproject_controller.go +++ b/pkg/controller/atlasproject/atlasproject_controller.go @@ -18,9 +18,11 @@ package atlasproject import ( "context" + "errors" "fmt" "time" + "go.mongodb.org/atlas/mongodbatlas" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -140,6 +142,12 @@ func (r *AtlasProjectReconciler) Delete(e event.DeleteEvent) error { for time.Now().Before(timeout) { _, err = atlasClient.Projects.Delete(context.Background(), project.Status.ID) + var apiError *mongodbatlas.ErrorResponse + if errors.As(err, &apiError) && apiError.ErrorCode == atlas.ClusterNotFound { + log.Infow("Project doesn't exist or is already deleted", "projectID", project.Status.ID) + return + } + if err != nil { log.Errorw("cannot delete Atlas project", "error", err) time.Sleep(workflow.DefaultRetry) From 23edcd45ec3331c3d26230b448078d484e7fba68 Mon Sep 17 00:00:00 2001 From: Pavel Date: Tue, 9 Mar 2021 12:59:12 +0300 Subject: [PATCH 11/11] Fix project error code --- pkg/controller/atlasproject/atlasproject_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/atlasproject/atlasproject_controller.go b/pkg/controller/atlasproject/atlasproject_controller.go index 95e8188d51..f7cc19900e 100644 --- a/pkg/controller/atlasproject/atlasproject_controller.go +++ b/pkg/controller/atlasproject/atlasproject_controller.go @@ -143,7 +143,7 @@ func (r *AtlasProjectReconciler) Delete(e event.DeleteEvent) error { for time.Now().Before(timeout) { _, err = atlasClient.Projects.Delete(context.Background(), project.Status.ID) var apiError *mongodbatlas.ErrorResponse - if errors.As(err, &apiError) && apiError.ErrorCode == atlas.ClusterNotFound { + if errors.As(err, &apiError) && apiError.ErrorCode == atlas.NotInGroup { log.Infow("Project doesn't exist or is already deleted", "projectID", project.Status.ID) return }