Recreate managed resource secrets that have deletion timestamp (#8812)

* Recreate deleted MR secrets

* Fix label copying

* Add test

* Format

* Change remove version, tmp name, nil finalizers

* Do not recreate secrets without del timestamp

* Remove some obsolete code

* Run tasks in parallel

cmd/gardenlet/app/app.go

@@ -21,12 +21,14 @@ import (
 	"net/http"
 	"os"
 	goruntime "runtime"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/go-logr/logr"
 	"github.com/spf13/cobra"
+	"golang.org/x/time/rate"
 	certificatesv1 "k8s.io/api/certificates/v1"
 	coordinationv1 "k8s.io/api/coordination/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -35,6 +37,8 @@ import (
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/rest"
@@ -69,6 +73,7 @@ import (
 	"github.com/gardener/gardener/pkg/gardenlet/bootstrap/certificate"
 	"github.com/gardener/gardener/pkg/gardenlet/controller"
 	gardenerhealthz "github.com/gardener/gardener/pkg/healthz"
+	"github.com/gardener/gardener/pkg/resourcemanager/controller/garbagecollector/references"
 	"github.com/gardener/gardener/pkg/utils"
 	"github.com/gardener/gardener/pkg/utils/flow"
 	gardenerutils "github.com/gardener/gardener/pkg/utils/gardener"
@@ -382,6 +387,11 @@ func (g *garden) Start(ctx context.Context) error {
 		return err
 	}
 
+	log.Info("Recreating wrongly deleted managed resource secrets")
+	if err := recreateDeletedManagedResourceSecrets(ctx, g.mgr.GetClient()); err != nil {
+		return err
+	}
+
 	log.Info("Setting up shoot client map")
 	shootClientMap, err := clientmapbuilder.
 		NewShootClientMapBuilder().
@@ -480,6 +490,166 @@ func (g *garden) cleanupOrphanedExtensionsServiceAccounts(ctx context.Context, g
 	return flow.Parallel(taskFns...)(ctx)
 }
 
+const (
+	grmFinalizer           = "resources.gardener.cloud/gardener-resource-manager"
+	tempSecretLabel        = "resources.gardener.cloud/temp-secret"
+	tempSecretOldNameLabel = "resources.gardener.cloud/temp-secret-old-name"
+)
+
+// TODO(dimityrmirchev): Remove this code after v1.87 has been released.
+func recreateDeletedManagedResourceSecrets(ctx context.Context, c client.Client) error {
+	// check for already existing temp secrets
+	// these can occur in case the process is killed during cleanup phase
+	tempSecretList := &corev1.SecretList{}
+	if err := c.List(ctx, tempSecretList, client.MatchingLabels{tempSecretLabel: "true"}); err != nil {
+		return err
+	}
+
+	var (
+		tasks   []flow.TaskFn
+		limiter = rate.NewLimiter(rate.Limit(20), 20)
+	)
+	for _, temp := range tempSecretList.Items {
+		temp := temp
+		tasks = append(tasks, func(ctx context.Context) error {
+			originalName := temp.Labels[tempSecretOldNameLabel]
+			original := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: originalName, Namespace: temp.Namespace}}
+
+			if err := limiter.Wait(ctx); err != nil {
+				return err
+			}
+
+			if err := c.Get(ctx, client.ObjectKeyFromObject(original), original); err != nil {
+				if apierrors.IsNotFound(err) {
+					// original secret is not found so we recreate it
+					original := temp.DeepCopy()
+					delete(original.Labels, tempSecretLabel)
+					delete(original.Labels, tempSecretOldNameLabel)
+					original.ResourceVersion = ""
+					original.Name = originalName
+
+					if err := c.Create(ctx, original); err != nil {
+						return fmt.Errorf("failed to recreate the original secret %w", err)
+					}
+					if err := c.Delete(ctx, &temp); client.IgnoreNotFound(err) != nil {
+						return err
+					}
+					return nil
+				}
+
+				return err
+			}
+
+			// the original secret exists. check if the finalizer and deletion timestamp are there
+			if original.DeletionTimestamp != nil && slices.Contains(original.Finalizers, grmFinalizer) {
+				if err := removeFinalizersAndWait(ctx, c, original.DeepCopy()); err != nil {
+					return err
+				}
+
+				// zero meta info
+				original.DeletionTimestamp = nil
+				original.ResourceVersion = ""
+				original.Finalizers = nil
+
+				if err := c.Create(ctx, original); err != nil {
+					return fmt.Errorf("failed to recreate the original secret %w", err)
+				}
+			}
+
+			// secret was already recreated. just delete the temporary one
+			if err := c.Delete(ctx, &temp); client.IgnoreNotFound(err) != nil {
+				return err
+			}
+			return nil
+		})
+	}
+
+	if err := flow.Parallel(tasks...)(ctx); err != nil {
+		return err
+	}
+
+	secretsToRecreate, err := getSecretsToRecreate(ctx, c)
+	if err != nil {
+		return fmt.Errorf("failed listing secrets for recreation %w", err)
+	}
+
+	tasks = []flow.TaskFn{}
+	for _, original := range secretsToRecreate {
+		original := original
+		tasks = append(tasks, func(ctx context.Context) error {
+			tempSecret := original.DeepCopy()
+			tempSecret.Name = "tmp-" + original.Name
+			metav1.SetMetaDataLabel(&tempSecret.ObjectMeta, tempSecretLabel, "true")
+			metav1.SetMetaDataLabel(&tempSecret.ObjectMeta, tempSecretOldNameLabel, original.Name)
+			tempSecret.DeletionTimestamp = nil
+			tempSecret.ResourceVersion = ""
+			tempSecret.Finalizers = nil
+
+			if err := limiter.Wait(ctx); err != nil {
+				return err
+			}
+
+			if err := c.Create(ctx, tempSecret); err != nil {
+				return fmt.Errorf("failed to create a temporary secret %w", err)
+			}
+
+			if err := removeFinalizersAndWait(ctx, c, original.DeepCopy()); err != nil {
+				return err
+			}
+
+			// zero meta info
+			original.DeletionTimestamp = nil
+			original.ResourceVersion = ""
+			original.Finalizers = nil
+
+			// recreate the original and delete the temporary one
+			if err := c.Create(ctx, &original); err != nil {
+				return fmt.Errorf("failed to recreate the original secret %w", err)
+			}
+			if err := c.Delete(ctx, tempSecret); client.IgnoreNotFound(err) != nil {
+				return err
+			}
+			return nil
+		})
+	}
+	return flow.Parallel(tasks...)(ctx)
+}
+
+// TODO(dimityrmirchev): Remove this code after v1.87 has been released.
+func removeFinalizersAndWait(ctx context.Context, c client.Client, secret *corev1.Secret) error {
+	patch := client.StrategicMergeFrom(secret.DeepCopy())
+	secret.Finalizers = []string{}
+	if err := c.Patch(ctx, secret, patch); err != nil {
+		return fmt.Errorf("failed to patch the original secret %w", err)
+	}
+
+	cancelCtx, cancel := context.WithTimeout(ctx, time.Second*10)
+	defer cancel()
+	return kubernetesutils.WaitUntilResourceDeleted(cancelCtx, c, secret, 1*time.Second)
+}
+
+// TODO(dimityrmirchev): Remove this code after v1.87 has been released.
+func getSecretsToRecreate(ctx context.Context, c client.Client) ([]corev1.Secret, error) {
+	selector := labels.NewSelector()
+	isGC, err := labels.NewRequirement(references.LabelKeyGarbageCollectable, selection.Equals, []string{"true"})
+	if err != nil {
+		return nil, err
+	}
+	notTemp, err := labels.NewRequirement(tempSecretLabel, selection.DoesNotExist, nil)
+	if err != nil {
+		return nil, err
+	}
+	selector.Add(*isGC, *notTemp)
+	secretList := &corev1.SecretList{}
+	if err := c.List(ctx, secretList, client.MatchingLabelsSelector{Selector: selector}); err != nil {
+		return nil, err
+	}
+	secretsToRecreate := slices.DeleteFunc(secretList.Items, func(s corev1.Secret) bool {
+		return !slices.Contains(s.Finalizers, grmFinalizer) || s.DeletionTimestamp == nil
+	})
+	return secretsToRecreate, nil
+}
+
 func (g *garden) registerSeed(ctx context.Context, gardenClient client.Client) error {
 	seed := &gardencorev1beta1.Seed{
 		ObjectMeta: metav1.ObjectMeta{