Backup{Bucket,Entry} resources are only reconciled once per 24h

```improvement operator
The `BackupBucket` and `BackupEntry` resources in the garden cluster are now only reconciled once per 24h by the gardenlet.
```

pkg/controllerutils/miscellaneous.go

@@ -16,9 +16,13 @@ package controllerutils
 
 import (
 	"strings"
+	"time"
 
+	gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	"github.com/gardener/gardener/pkg/operation/common"
 	"github.com/gardener/gardener/pkg/utils"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 const separator = ","
@@ -83,3 +87,39 @@ func setTaskAnnotations(annotations map[string]string, tasks []string) {
 
 	annotations[common.ShootTasks] = strings.Join(tasks, separator)
 }
+
+var (
+	// Now is a function for returning the current time.
+	Now = time.Now
+	// RandomDuration is a function for returning a random duration.
+	RandomDuration = utils.RandomDuration
+)
+
+// ReconcileOncePer24hDuration returns the duration until the next reconciliation should happen while respecting that
+// only one reconciliation should happen per 24h. If the deletion timestamp is set or the generation has changed or the
+// last operation does not indicate success or indicates that the last reconciliation happened more than 24h ago then 0
+// will be returned.
+func ReconcileOncePer24hDuration(objectMeta metav1.ObjectMeta, observedGeneration int64, lastOperation *gardencorev1beta1.LastOperation) time.Duration {
+	if objectMeta.DeletionTimestamp != nil {
+		return 0
+	}
+
+	if objectMeta.Generation != observedGeneration {
+		return 0
+	}
+
+	if lastOperation == nil ||
+		lastOperation.State != gardencorev1beta1.LastOperationStateSucceeded ||
+		(lastOperation.Type != gardencorev1beta1.LastOperationTypeCreate && lastOperation.Type != gardencorev1beta1.LastOperationTypeReconcile) {
+		return 0
+	}
+
+	// If last reconciliation happened more than 24h ago then we want to reconcile immediately, so let's only compute
+	// a delay if the last reconciliation was within the last 24h.
+	if lastReconciliation := lastOperation.LastUpdateTime.Time; Now().UTC().Before(lastReconciliation.UTC().Add(24 * time.Hour)) {
+		durationUntilLastReconciliationWas24hAgo := lastReconciliation.UTC().Add(24 * time.Hour).Sub(Now().UTC())
+		return RandomDuration(durationUntilLastReconciliationWas24hAgo)
+	}
+
+	return 0
+}

pkg/controllerutils/miscellaneous_test.go

@@ -16,20 +16,23 @@ package controllerutils_test
 
 import (
 	"strings"
+	"time"
 
-	"github.com/gardener/gardener/pkg/controllerutils"
+	gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
+	. "github.com/gardener/gardener/pkg/controllerutils"
 	"github.com/gardener/gardener/pkg/operation/common"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/ginkgo/extensions/table"
 	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 var _ = Describe("controller", func() {
 	Describe("utils", func() {
 		DescribeTable("#GetTasks",
 			func(existingTasks map[string]string, expectedResult []string) {
-				result := controllerutils.GetTasks(existingTasks)
+				result := GetTasks(existingTasks)
 				Expect(result).To(Equal(expectedResult))
 			},
 
@@ -41,7 +44,7 @@ var _ = Describe("controller", func() {
 
 		DescribeTable("#AddTasks",
 			func(existingTasks map[string]string, tasks []string, expectedTasks []string) {
-				controllerutils.AddTasks(existingTasks, tasks...)
+				AddTasks(existingTasks, tasks...)
 
 				if expectedTasks == nil {
 					Expect(existingTasks[common.ShootTasks]).To(BeEmpty())
@@ -70,7 +73,7 @@ var _ = Describe("controller", func() {
 
 		DescribeTable("#RemoveTasks",
 			func(existingTasks map[string]string, tasks []string, expectedTasks []string) {
-				controllerutils.RemoveTasks(existingTasks, tasks...)
+				RemoveTasks(existingTasks, tasks...)
 
 				if expectedTasks == nil {
 					Expect(existingTasks[common.ShootTasks]).To(BeEmpty())
@@ -101,7 +104,7 @@ var _ = Describe("controller", func() {
 
 		DescribeTable("#HasTask",
 			func(existingTasks map[string]string, task string, expectedResult bool) {
-				result := controllerutils.HasTask(existingTasks, task)
+				result := HasTask(existingTasks, task)
 				Expect(result).To(Equal(expectedResult))
 			},
 
@@ -111,4 +114,27 @@ var _ = Describe("controller", func() {
 			Entry("task in list", map[string]string{common.ShootTasks: "some-task" + "," + common.ShootTaskDeployInfrastructure}, "some-task", true),
 		)
 	})
+
+	var deletionTimestamp = metav1.Now()
+	DescribeTable("#ReconcileOncePer24hDuration",
+		func(objectMeta metav1.ObjectMeta, observedGeneration int64, lastOperation *gardencorev1beta1.LastOperation, expectedDuration time.Duration) {
+			oldNow := Now
+			defer func() { Now = oldNow }()
+			Now = func() time.Time { return time.Date(1, 1, 2, 1, 0, 0, 0, time.UTC) }
+
+			oldRandomDuration := RandomDuration
+			defer func() { RandomDuration = oldRandomDuration }()
+			RandomDuration = func(time.Duration) time.Duration { return time.Minute }
+
+			Expect(ReconcileOncePer24hDuration(objectMeta, observedGeneration, lastOperation)).To(Equal(expectedDuration))
+		},
+
+		Entry("deletion timestamp set", metav1.ObjectMeta{DeletionTimestamp: &deletionTimestamp}, int64(0), nil, time.Duration(0)),
+		Entry("generation not equal observed generation", metav1.ObjectMeta{Generation: int64(1)}, int64(0), nil, time.Duration(0)),
+		Entry("last operation is nil", metav1.ObjectMeta{}, int64(0), nil, time.Duration(0)),
+		Entry("last operation state is succeeded", metav1.ObjectMeta{}, int64(0), &gardencorev1beta1.LastOperation{State: gardencorev1beta1.LastOperationStateSucceeded}, time.Duration(0)),
+		Entry("last operation type is not create or reconcile", metav1.ObjectMeta{}, int64(0), &gardencorev1beta1.LastOperation{State: gardencorev1beta1.LastOperationStateSucceeded, Type: gardencorev1beta1.LastOperationTypeRestore}, time.Duration(0)),
+		Entry("last reconciliation was more than 24h ago", metav1.ObjectMeta{}, int64(0), &gardencorev1beta1.LastOperation{State: gardencorev1beta1.LastOperationStateSucceeded, Type: gardencorev1beta1.LastOperationTypeReconcile, LastUpdateTime: metav1.Time{Time: time.Date(1, 1, 1, 0, 30, 0, 0, time.UTC)}}, time.Duration(0)),
+		Entry("last reconciliation was not more than 24h ago", metav1.ObjectMeta{}, int64(0), &gardencorev1beta1.LastOperation{State: gardencorev1beta1.LastOperationStateSucceeded, Type: gardencorev1beta1.LastOperationTypeReconcile, LastUpdateTime: metav1.Time{Time: time.Date(1, 1, 1, 1, 30, 0, 0, time.UTC)}}, time.Minute),
+	)
 })

pkg/gardenlet/controller/backupbucket/backup_bucket_controller.go

@@ -15,22 +15,37 @@
 package backupbucket
 
 import (
-	gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
-	"github.com/gardener/gardener/pkg/logger"
+	"time"
 
 	"k8s.io/client-go/tools/cache"
+
+	gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
+	"github.com/gardener/gardener/pkg/controllerutils"
+	"github.com/gardener/gardener/pkg/logger"
 )
 
 func (c *Controller) backupBucketAdd(obj interface{}) {
+	backupBucket, ok := obj.(*gardencorev1beta1.BackupBucket)
+	if !ok {
+		logger.Logger.Errorf("Couldn't convert object: %T", obj)
+		return
+	}
+
 	key, err := cache.MetaNamespaceKeyFunc(obj)
 	if err != nil {
 		logger.Logger.Errorf("Couldn't get key for object %+v: %v", obj, err)
 		return
 	}
-	c.backupBucketQueue.Add(key)
+
+	addAfter := controllerutils.ReconcileOncePer24hDuration(backupBucket.ObjectMeta, backupBucket.Status.ObservedGeneration, backupBucket.Status.LastOperation)
+	if addAfter > 0 {
+		logger.Logger.Infof("Scheduled next reconciliation for BackupBucket %q in %s (%s)", key, addAfter, time.Now().Add(addAfter))
+	}
+
+	c.backupBucketQueue.AddAfter(key, addAfter)
 }
 
-func (c *Controller) backupBucketUpdate(oldObj, newObj interface{}) {
+func (c *Controller) backupBucketUpdate(_, newObj interface{}) {
 	var (
 		newBackupBucket    = newObj.(*gardencorev1beta1.BackupBucket)
 		backupBucketLogger = logger.NewFieldLogger(logger.Logger, "backupbucket", newBackupBucket.Name)
@@ -52,7 +67,6 @@ func (c *Controller) backupBucketUpdate(oldObj, newObj interface{}) {
 	// }
 
 	c.backupBucketAdd(newObj)
-
 }
 
 func (c *Controller) backupBucketDelete(obj interface{}) {

pkg/gardenlet/controller/backupentry/backup_entry_controller.go

@@ -19,36 +19,34 @@ import (
 	"time"
 
 	gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
+	"github.com/gardener/gardener/pkg/controllerutils"
 	"github.com/gardener/gardener/pkg/logger"
-	"github.com/gardener/gardener/pkg/utils"
 
 	"k8s.io/client-go/tools/cache"
 )
 
 func (c *Controller) backupEntryAdd(obj interface{}) {
-	key, err := cache.MetaNamespaceKeyFunc(obj)
-	if err != nil {
-		logger.Logger.Errorf("Couldn't get key for object %+v: %v", obj, err)
-		return
-	}
-
 	backupEntry, ok := obj.(*gardencorev1beta1.BackupEntry)
 	if !ok {
+		logger.Logger.Errorf("Couldn't convert object: %T", obj)
 		return
 	}
 
-	if backupEntry.Generation == backupEntry.Status.ObservedGeneration {
-		// spread BackupEntry reconciliation across one minute to avoid reconciling all BackupEntries roughly at the
-		// same time after startup of the gardenlet
-		c.backupEntryQueue.AddAfter(key, utils.RandomDuration(time.Minute))
+	key, err := cache.MetaNamespaceKeyFunc(obj)
+	if err != nil {
+		logger.Logger.Errorf("Couldn't get key for object %+v: %v", obj, err)
 		return
 	}
 
-	// don't add random duration for enqueueing new BackupBuckets, that have never been reconciled
-	c.backupEntryQueue.Add(key)
+	addAfter := controllerutils.ReconcileOncePer24hDuration(backupEntry.ObjectMeta, backupEntry.Status.ObservedGeneration, backupEntry.Status.LastOperation)
+	if addAfter > 0 {
+		logger.Logger.Infof("Scheduled next reconciliation for BackupEntry %q in %s (%s)", key, addAfter, time.Now().Add(addAfter))
+	}
+
+	c.backupEntryQueue.AddAfter(key, addAfter)
 }
 
-func (c *Controller) backupEntryUpdate(oldObj, newObj interface{}) {
+func (c *Controller) backupEntryUpdate(_, newObj interface{}) {
 	var (
 		newBackupEntry    = newObj.(*gardencorev1beta1.BackupEntry)
 		backupEntryLogger = logger.NewFieldLogger(logger.Logger, "backupentry", fmt.Sprintf("%s/%s", newBackupEntry.Namespace, newBackupEntry.Name))