Merge pull request #2090 from timuthy/feature.cleanup-err-code

Add error code for cluster cleanup

pkg/apis/core/v1beta1/helper/errors.go

@@ -52,16 +52,36 @@ var (
 	dependenciesRegexp           = regexp.MustCompile(`(?i)(PendingVerification|Access Not Configured|accessNotConfigured|DependencyViolation|OptInRequired|DeleteConflict|Conflict|inactive billing state|ReadOnlyDisabledSubscription|is already being used|not available in the current hardware cluster)`)
 )
 
-// DetermineError determines the Garden error code for the given error message.
-func DetermineError(message string) error {
-	code := determineErrorCode(message)
-	if code == "" {
+// DetermineError determines the Garden error code for the given error and creates a new error with the given message.
+func DetermineError(err error, message string) error {
+	if err == nil {
 		return errors.New(message)
 	}
-	return &errorWithCode{code, message}
+
+	errMsg := message
+	if errMsg == "" {
+		errMsg = err.Error()
+	}
+
+	code := determineErrorCode(err)
+	if code == "" {
+		return errors.New(errMsg)
+	}
+	return &errorWithCode{code, errMsg}
 }
 
-func determineErrorCode(message string) gardencorev1beta1.ErrorCode {
+func determineErrorCode(err error) gardencorev1beta1.ErrorCode {
+	var coder Coder
+
+	// first try to re-use code from error
+	if errors.As(err, &coder) {
+		switch coder.Code() {
+		case gardencorev1beta1.ErrorInfraUnauthorized, gardencorev1beta1.ErrorInfraQuotaExceeded, gardencorev1beta1.ErrorInfraInsufficientPrivileges, gardencorev1beta1.ErrorInfraDependencies:
+			return coder.Code()
+		}
+	}
+
+	message := err.Error()
 	switch {
 	case unauthorizedRegexp.MatchString(message):
 		return gardencorev1beta1.ErrorInfraUnauthorized
@@ -86,7 +106,8 @@ type Coder interface {
 func ExtractErrorCodes(err error) []gardencorev1beta1.ErrorCode {
 	var codes []gardencorev1beta1.ErrorCode
 	for _, err := range utilerrors.Errors(err) {
-		if coder, ok := err.(Coder); ok {
+		var coder Coder
+		if errors.As(err, &coder) {
 			codes = append(codes, coder.Code())
 		}
 	}

pkg/apis/core/v1beta1/helper/errors_test.go

@@ -16,29 +16,48 @@ package helper_test
 
 import (
 	"errors"
+	"fmt"
 
 	gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	. "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/ginkgo/extensions/table"
 	. "github.com/onsi/gomega"
+	. "github.com/onsi/gomega/types"
 )
 
 var _ = Describe("helper", func() {
 	Describe("errors", func() {
 		Describe("#DetermineError", func() {
 			DescribeTable("appropriate error should be determined",
-				func(msg string, expectedErr error) {
-					Expect(DetermineError(msg)).To(Equal(expectedErr))
+				func(err error, msg string, expectedErr error) {
+					Expect(DetermineError(err, msg)).To(Equal(expectedErr))
 				},
 
-				Entry("no code to extract", "foo", errors.New("foo")),
-				Entry("unauthorized", "unauthorized", NewErrorWithCode(gardencorev1beta1.ErrorInfraUnauthorized, "unauthorized")),
-				Entry("quota exceeded", "limitexceeded", NewErrorWithCode(gardencorev1beta1.ErrorInfraQuotaExceeded, "limitexceeded")),
-				Entry("insufficient privileges", "accessdenied", NewErrorWithCode(gardencorev1beta1.ErrorInfraInsufficientPrivileges, "accessdenied")),
-				Entry("infrastructure dependencies", "pendingverification", NewErrorWithCode(gardencorev1beta1.ErrorInfraDependencies, "pendingverification")),
-				Entry("infrastructure dependencies", "not available in the current hardware cluster", NewErrorWithCode(gardencorev1beta1.ErrorInfraDependencies, "not available in the current hardware cluster")),
+				Entry("no error", nil, "foo", errors.New("foo")),
+				Entry("no code to extract", errors.New("foo"), "", errors.New("foo")),
+				Entry("unauthorized", errors.New("unauthorized"), "", NewErrorWithCode(gardencorev1beta1.ErrorInfraUnauthorized, "unauthorized")),
+				Entry("unauthorized with coder", NewErrorWithCode(gardencorev1beta1.ErrorInfraUnauthorized, ""), "", NewErrorWithCode(gardencorev1beta1.ErrorInfraUnauthorized, "")),
+				Entry("quota exceeded", errors.New("limitexceeded"), "", NewErrorWithCode(gardencorev1beta1.ErrorInfraQuotaExceeded, "limitexceeded")),
+				Entry("quota exceeded with coder", NewErrorWithCode(gardencorev1beta1.ErrorInfraQuotaExceeded, "limitexceeded"), "", NewErrorWithCode(gardencorev1beta1.ErrorInfraQuotaExceeded, "limitexceeded")),
+				Entry("insufficient privileges", errors.New("accessdenied"), "", NewErrorWithCode(gardencorev1beta1.ErrorInfraInsufficientPrivileges, "accessdenied")),
+				Entry("insufficient privileges with coder", NewErrorWithCode(gardencorev1beta1.ErrorInfraInsufficientPrivileges, "accessdenied"), "", NewErrorWithCode(gardencorev1beta1.ErrorInfraInsufficientPrivileges, "accessdenied")),
+				Entry("infrastructure dependencies", errors.New("pendingverification"), "", NewErrorWithCode(gardencorev1beta1.ErrorInfraDependencies, "pendingverification")),
+				Entry("infrastructure dependencies", errors.New("not available in the current hardware cluster"), "error occurred: not available in the current hardware cluster", NewErrorWithCode(gardencorev1beta1.ErrorInfraDependencies, "error occurred: not available in the current hardware cluster")),
+				Entry("infrastructure dependencies with coder", NewErrorWithCode(gardencorev1beta1.ErrorInfraDependencies, "not available in the current hardware cluster"), "error occurred: not available in the current hardware cluster", NewErrorWithCode(gardencorev1beta1.ErrorInfraDependencies, "error occurred: not available in the current hardware cluster")),
+			)
+		})
+		Describe("#ExtractErrorCodes", func() {
+			DescribeTable("appropriate error code should be extracted",
+				func(err error, matcher GomegaMatcher) {
+					Expect(ExtractErrorCodes(err)).To(matcher)
+				},
+
+				Entry("no error given", nil, BeEmpty()),
+				Entry("no code error given", errors.New("error"), BeEmpty()),
+				Entry("code error given", NewErrorWithCode(gardencorev1beta1.ErrorInfraUnauthorized, ""), ConsistOf(Equal(gardencorev1beta1.ErrorInfraUnauthorized))),
+				Entry("wrapped code error", fmt.Errorf("error %w", NewErrorWithCode(gardencorev1beta1.ErrorInfraUnauthorized, "")), ConsistOf(Equal(gardencorev1beta1.ErrorInfraUnauthorized))),
 			)
 		})
 	})

pkg/apis/core/v1beta1/types_common.go

@@ -28,6 +28,8 @@ const (
 	ErrorInfraQuotaExceeded ErrorCode = "ERR_INFRA_QUOTA_EXCEEDED"
 	// ErrorInfraDependencies indicates that the last error occurred due to dependent objects on the cloud provider level.
 	ErrorInfraDependencies ErrorCode = "ERR_INFRA_DEPENDENCIES"
+	// ErrorCleanupClusterResources indicates that the last error occurred due to resources in the cluster are stuck in deletion.
+	ErrorCleanupClusterResources ErrorCode = "ERR_CLEANUP_CLUSTER_RESOURCES"
 )
 
 // LastError indicates the last occurred error for an operation on a resource.

pkg/gardenlet/controller/backupbucket/backup_bucket_actuator.go

@@ -16,6 +16,7 @@ package backupbucket
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -207,7 +208,7 @@ func (a *actuator) waitUntilBackupBucketExtensionReconciled(ctx context.Context)
 		backupBucket = bb
 		return retry.Ok()
 	}); err != nil {
-		return gardencorev1beta1helper.DetermineError(fmt.Sprintf("Error while waiting for backupBucket object to become ready: %v", err))
+		return gardencorev1beta1helper.DetermineError(err, fmt.Sprintf("Error while waiting for backupBucket object to become ready: %v", err))
 	}
 
 	var (
@@ -307,9 +308,9 @@ func (a *actuator) waitUntilBackupBucketExtensionDeleted(ctx context.Context) er
 	}); err != nil {
 		message := fmt.Sprintf("Error while waiting for backupBucket object to be deleted")
 		if lastError != nil {
-			return gardencorev1beta1helper.DetermineError(fmt.Sprintf("%s: %s", message, lastError.Description))
+			return gardencorev1beta1helper.DetermineError(errors.New(lastError.Description), fmt.Sprintf("%s: %s", message, lastError.Description))
 		}
-		return gardencorev1beta1helper.DetermineError(fmt.Sprintf("%s: %s", message, err.Error()))
+		return gardencorev1beta1helper.DetermineError(err, fmt.Sprintf("%s: %s", message, err.Error()))
 	}
 
 	return nil

pkg/gardenlet/controller/backupentry/backup_entry_actuator.go

@@ -151,7 +151,7 @@ func (a *actuator) waitUntilCoreBackupBucketReconciled(ctx context.Context, bb *
 		}
 		return retry.Ok()
 	}); err != nil {
-		return gardencorev1beta1helper.DetermineError(fmt.Sprintf("Error while waiting for BackupBucket object to become ready: %v", err))
+		return gardencorev1beta1helper.DetermineError(err, fmt.Sprintf("Error while waiting for BackupBucket object to become ready: %v", err))
 	}
 	return nil
 }
@@ -243,7 +243,7 @@ func (a *actuator) waitUntilBackupEntryExtensionReconciled(ctx context.Context)
 		}
 		return retry.Ok()
 	}); err != nil {
-		return gardencorev1beta1helper.DetermineError(fmt.Sprintf("Error while waiting for backupEntry object to become ready: %v", err))
+		return gardencorev1beta1helper.DetermineError(err, fmt.Sprintf("Error while waiting for backupEntry object to become ready: %v", err))
 	}
 	return nil
 }
@@ -287,9 +287,9 @@ func (a *actuator) waitUntilBackupEntryExtensionDeleted(ctx context.Context) err
 	}); err != nil {
 		message := fmt.Sprintf("Error while waiting for backupEntry object to be deleted")
 		if lastError != nil {
-			return gardencorev1beta1helper.DetermineError(fmt.Sprintf("%s: %s", message, lastError.Description))
+			return gardencorev1beta1helper.DetermineError(errors.New(lastError.Description), fmt.Sprintf("%s: %s", message, lastError.Description))
 		}
-		return gardencorev1beta1helper.DetermineError(fmt.Sprintf("%s: %s", message, err.Error()))
+		return gardencorev1beta1helper.DetermineError(err, fmt.Sprintf("%s: %s", message, err.Error()))
 	}
 	return nil
 }

pkg/operation/botanist/cleanup.go

@@ -18,7 +18,9 @@ import (
 	"context"
 	"time"
 
+	gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
+	"github.com/gardener/gardener/pkg/apis/core/v1beta1/helper"
 	"github.com/gardener/gardener/pkg/operation/common"
 	"github.com/gardener/gardener/pkg/utils/flow"
 	utilclient "github.com/gardener/gardener/pkg/utils/kubernetes/client"
@@ -170,7 +172,7 @@ func cleanResourceFn(cleanOps utilclient.CleanOps, c client.Client, list runtime
 		return retry.Until(ctx, DefaultInterval, func(ctx context.Context) (done bool, err error) {
 			if err := cleanOps.CleanAndEnsureGone(ctx, c, list, opts...); err != nil {
 				if utilclient.AreObjectsRemaining(err) {
-					return retry.MinorError(err)
+					return retry.MinorError(helper.NewErrorWithCode(gardencorev1beta1.ErrorCleanupClusterResources, err.Error()))
 				}
 				return retry.SevereError(err)
 			}

pkg/operation/botanist/controlplane.go

@@ -16,6 +16,7 @@ package botanist
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"hash/crc32"
 	"path/filepath"
@@ -425,7 +426,7 @@ func (b *Botanist) waitUntilControlPlaneReady(ctx context.Context, name string)
 		}
 		return retry.Ok()
 	}); err != nil {
-		return gardencorev1beta1helper.DetermineError(fmt.Sprintf("failed to create control plane: %v", err))
+		return gardencorev1beta1helper.DetermineError(err, fmt.Sprintf("failed to create control plane: %v", err))
 	}
 	return nil
 }
@@ -463,9 +464,9 @@ func (b *Botanist) waitUntilControlPlaneDeleted(ctx context.Context, name string
 	}); err != nil {
 		message := fmt.Sprintf("Failed to delete control plane")
 		if lastError != nil {
-			return gardencorev1beta1helper.DetermineError(fmt.Sprintf("%s: %s", message, lastError.Description))
+			return gardencorev1beta1helper.DetermineError(errors.New(lastError.Description), fmt.Sprintf("%s: %s", message, lastError.Description))
 		}
-		return gardencorev1beta1helper.DetermineError(fmt.Sprintf("%s: %s", message, err.Error()))
+		return gardencorev1beta1helper.DetermineError(err, fmt.Sprintf("%s: %s", message, err.Error()))
 	}
 	return nil
 }

pkg/operation/botanist/dns.go

@@ -225,7 +225,7 @@ func (b *Botanist) waitUntilDNSProviderReady(ctx context.Context, name string) e
 		b.Logger.Infof("Waiting for %q DNS provider to be ready... (status=%s, message=%s)", name, status, message)
 		return retry.MinorError(fmt.Errorf("DNS provider %q is not ready (status=%s, message=%s)", name, status, message))
 	}); err != nil {
-		return gardencorev1beta1helper.DetermineError(fmt.Sprintf("Failed to create DNS provider for %q DNS record: %q (status=%s, message=%s)", name, err.Error(), status, message))
+		return gardencorev1beta1helper.DetermineError(err, fmt.Sprintf("Failed to create DNS provider for %q DNS record: %q (status=%s, message=%s)", name, err.Error(), status, message))
 	}
 
 	return nil
@@ -280,7 +280,7 @@ func (b *Botanist) waitUntilDNSEntryReady(ctx context.Context, name string) erro
 		b.Logger.Infof("Waiting for %q DNS record to be ready... (status=%s, message=%s)", name, status, message)
 		return retry.MinorError(fmt.Errorf("DNS record %q is not ready (status=%s, message=%s)", name, status, message))
 	}); err != nil {
-		return gardencorev1beta1helper.DetermineError(fmt.Sprintf("Failed to create %q DNS record: %q (status=%s, message=%s)", name, err.Error(), status, message))
+		return gardencorev1beta1helper.DetermineError(err, fmt.Sprintf("Failed to create %q DNS record: %q (status=%s, message=%s)", name, err.Error(), status, message))
 	}
 
 	return nil

pkg/operation/botanist/extension.go

@@ -16,6 +16,7 @@ package botanist
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
@@ -122,7 +123,7 @@ func (b *Botanist) WaitUntilExtensionResourcesReady(ctx context.Context) error {
 
 				return retry.Ok()
 			}); err != nil {
-				return gardencorev1beta1helper.DetermineError(fmt.Sprintf("failed waiting for extension %s to be ready: %v", name, err))
+				return gardencorev1beta1helper.DetermineError(err, fmt.Sprintf("failed waiting for extension %s to be ready: %v", name, err))
 			}
 			return nil
 		})
@@ -177,9 +178,9 @@ func (b *Botanist) WaitUntilExtensionResourcesDeleted(ctx context.Context) error
 			}); err != nil {
 				message := fmt.Sprintf("Failed waiting for extension delete")
 				if lastError != nil {
-					return gardencorev1beta1helper.DetermineError(fmt.Sprintf("%s: %s", message, lastError.Description))
+					return gardencorev1beta1helper.DetermineError(errors.New(lastError.Description), fmt.Sprintf("%s: %s", message, lastError.Description))
 				}
-				return gardencorev1beta1helper.DetermineError(fmt.Sprintf("%s: %s", message, err.Error()))
+				return gardencorev1beta1helper.DetermineError(err, fmt.Sprintf("%s: %s", message, err.Error()))
 			}
 			return nil
 		})

pkg/operation/botanist/infrastructure.go

@@ -16,6 +16,7 @@ package botanist
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"time"
 
@@ -124,7 +125,7 @@ func (b *Botanist) WaitUntilInfrastructureReady(ctx context.Context) error {
 
 		return retry.Ok()
 	}); err != nil {
-		return gardencorev1beta1helper.DetermineError(fmt.Sprintf("failed to create infrastructure: %v", err))
+		return gardencorev1beta1helper.DetermineError(err, fmt.Sprintf("failed to create infrastructure: %v", err))
 	}
 	return nil
 }
@@ -152,9 +153,9 @@ func (b *Botanist) WaitUntilInfrastructureDeleted(ctx context.Context) error {
 	}); err != nil {
 		message := fmt.Sprintf("Failed to delete infrastructure")
 		if lastError != nil {
-			return gardencorev1beta1helper.DetermineError(fmt.Sprintf("%s: %s", message, lastError.Description))
+			return gardencorev1beta1helper.DetermineError(errors.New(lastError.Description), fmt.Sprintf("%s: %s", message, lastError.Description))
 		}
-		return gardencorev1beta1helper.DetermineError(fmt.Sprintf("%s: %s", message, err.Error()))
+		return gardencorev1beta1helper.DetermineError(err, fmt.Sprintf("%s: %s", message, err.Error()))
 	}
 
 	return nil

pkg/operation/botanist/network.go

@@ -16,6 +16,7 @@ package botanist
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"time"
 
@@ -84,7 +85,7 @@ func (b *Botanist) WaitUntilNetworkIsReady(ctx context.Context) error {
 		}
 		return retry.Ok()
 	}); err != nil {
-		return gardencorev1beta1helper.DetermineError(fmt.Sprintf("failed to create network: %v", err))
+		return gardencorev1beta1helper.DetermineError(err, fmt.Sprintf("failed to create network: %v", err))
 	}
 	return nil
 }
@@ -112,9 +113,9 @@ func (b *Botanist) WaitUntilNetworkIsDeleted(ctx context.Context) error {
 	}); err != nil {
 		message := fmt.Sprintf("Failed to delete Network")
 		if lastError != nil {
-			return gardencorev1beta1helper.DetermineError(fmt.Sprintf("%s: %s", message, lastError.Description))
+			return gardencorev1beta1helper.DetermineError(errors.New(lastError.Description), fmt.Sprintf("%s: %s", message, lastError.Description))
 		}
-		return gardencorev1beta1helper.DetermineError(fmt.Sprintf("%s: %s", message, err.Error()))
+		return gardencorev1beta1helper.DetermineError(err, fmt.Sprintf("%s: %s", message, err.Error()))
 	}
 
 	return nil

pkg/operation/botanist/worker.go

@@ -16,6 +16,7 @@ package botanist
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"time"
 
@@ -158,7 +159,7 @@ func (b *Botanist) WaitUntilWorkerReady(ctx context.Context) error {
 		b.Shoot.MachineDeployments = worker.Status.MachineDeployments
 		return retry.Ok()
 	}); err != nil {
-		return gardencorev1beta1helper.DetermineError(fmt.Sprintf("Error while waiting for worker object to become ready: %v", err))
+		return gardencorev1beta1helper.DetermineError(err, fmt.Sprintf("Error while waiting for worker object to become ready: %v", err))
 	}
 	return nil
 }
@@ -186,9 +187,9 @@ func (b *Botanist) WaitUntilWorkerDeleted(ctx context.Context) error {
 	}); err != nil {
 		message := fmt.Sprintf("Error while waiting for worker object to be deleted")
 		if lastError != nil {
-			return gardencorev1beta1helper.DetermineError(fmt.Sprintf("%s: %s", message, lastError.Description))
+			return gardencorev1beta1helper.DetermineError(errors.New(lastError.Description), fmt.Sprintf("%s: %s", message, lastError.Description))
 		}
-		return gardencorev1beta1helper.DetermineError(fmt.Sprintf("%s: %s", message, err.Error()))
+		return gardencorev1beta1helper.DetermineError(err, fmt.Sprintf("%s: %s", message, err.Error()))
 	}
 
 	return nil

pkg/utils/errors/errors.go

@@ -104,7 +104,7 @@ func (t *reconciliationError) Cause() error {
 }
 
 // GetID returns the ID of the error if possible.
-// If err does not implement ErrorID or is nill an empty string will be returned.
+// If err does not implement ErrorID or is nil an empty string will be returned.
 func GetID(err error) string {
 	type errorIDer interface {
 		ErrorID() string

pkg/utils/retry/retry.go

@@ -111,6 +111,12 @@ func (r *retryError) Cause() error {
 	return r.ctxError
 }
 
+// Unwrap implements the Unwrap function
+// https://golang.org/pkg/errors/#Unwrap
+func (r *retryError) Unwrap() error {
+	return r.err
+}
+
 // Error implements error.
 func (r *retryError) Error() string {
 	if r.err != nil {