[GEP-17] Add force restoring of shoots and backup entries (gardener#5123

)

* Add force restoring of shoots and backup entries

* Disable force restoring if owner checks are disabled

* Add new controllers config to example gardenlet config

* Address code review comments.

charts/gardener/gardenlet/charts/runtime/templates/configmap-componentconfig.yaml

@@ -144,6 +144,16 @@ data:
         syncJitterPeriod: {{ .Values.global.gardenlet.config.controllers.managedSeed.syncJitterPeriod }}
         {{- end }}
       {{- end }}
+      shootMigration:
+        concurrentSyncs: {{ required ".Values.global.gardenlet.config.controllers.shootMigration.concurrentSyncs is required" .Values.global.gardenlet.config.controllers.shootMigration.concurrentSyncs }}
+        syncPeriod: {{ required ".Values.global.gardenlet.config.controllers.shootMigration.syncPeriod is required" .Values.global.gardenlet.config.controllers.shootMigration.syncPeriod }}
+        gracePeriod: {{ required ".Values.global.gardenlet.config.controllers.shootMigration.gracePeriod is required" .Values.global.gardenlet.config.controllers.shootMigration.gracePeriod }}
+        lastOperationStaleDuration: {{ required ".Values.global.gardenlet.config.controllers.shootMigration.lastOperationStaleDuration is required" .Values.global.gardenlet.config.controllers.shootMigration.lastOperationStaleDuration }}
+      backupEntryMigration:
+        concurrentSyncs: {{ required ".Values.global.gardenlet.config.controllers.backupEntryMigration.concurrentSyncs is required" .Values.global.gardenlet.config.controllers.backupEntryMigration.concurrentSyncs }}
+        syncPeriod: {{ required ".Values.global.gardenlet.config.controllers.backupEntryMigration.syncPeriod is required" .Values.global.gardenlet.config.controllers.backupEntryMigration.syncPeriod }}
+        gracePeriod: {{ required ".Values.global.gardenlet.config.controllers.backupEntryMigration.gracePeriod is required" .Values.global.gardenlet.config.controllers.backupEntryMigration.gracePeriod }}
+        lastOperationStaleDuration: {{ required ".Values.global.gardenlet.config.controllers.backupEntryMigration.lastOperationStaleDuration is required" .Values.global.gardenlet.config.controllers.backupEntryMigration.lastOperationStaleDuration }}
     resources:
       capacity:
         shoots: {{ required ".Values.global.gardenlet.config.resources.capacity.shoots is required" .Values.global.gardenlet.config.resources.capacity.shoots }}

charts/gardener/gardenlet/values.yaml

@@ -109,6 +109,16 @@ global:
           syncPeriod: 1h
           waitSyncPeriod: 15s
           syncJitterPeriod: 5m
+        shootMigration:
+          concurrentSyncs: 5
+          syncPeriod: 1m
+          gracePeriod: 2h
+          lastOperationStaleDuration: 10m
+        backupEntryMigration:
+          concurrentSyncs: 5
+          syncPeriod: 1m
+          gracePeriod: 10m
+          lastOperationStaleDuration: 2m
       resources:
         capacity:
           shoots: 250

docs/api-reference/core.md

@@ -2322,6 +2322,20 @@ string
 at the beginning of a create/reconcile operation. It is used when moving the BackupEntry between seeds.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>migrationStartTime</code></br>
+<em>
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#time-v1-meta">
+Kubernetes meta/v1.Time
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>MigrationStartTime is the time when a migration to a different seed was initiated.</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="core.gardener.cloud/v1beta1.CRI">CRI
@@ -9248,6 +9262,20 @@ string
 <p>List of addresses on which the Kube API server can be reached.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>migrationStartTime</code></br>
+<em>
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#time-v1-meta">
+Kubernetes meta/v1.Time
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>MigrationStartTime is the time when a migration to a different seed was initiated.</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="core.gardener.cloud/v1beta1.ShootTemplate">ShootTemplate

docs/deployment/feature_gates.md

@@ -39,6 +39,7 @@ The following tables are a summary of the feature gates that you can set on diff
 | WorkerPoolKubernetesVersion                  | `false` | `Alpha` | `1.35` |        |
 | CopyEtcdBackupsDuringControlPlaneMigration   | `false` | `Alpha` | `1.37` |        |
 | SecretBindingProviderValidation              | `false` | `Alpha` | `1.38` |        |
+| ForceRestore                                 | `false` | `Alpha` | `1.38` |        |
 
 ## Feature gates for graduated or deprecated features
 
@@ -114,3 +115,4 @@ A *General Availability* (GA) feature is also referred to as a *stable* feature.
 	- requires the provider type of a SecretBinding to be set (on SecretBinding creation)
 	- requires the SecretBinding provider type to match the Shoot provider type (on Shoot creation)
 	- enforces immutability on the provider type of a SecretBinding
+* `ForceRestore` enables forcing the shoot's restoration to the destination seed during control plane migration if the preparation for migration in the source seed is not finished after a certain grace period and is considered unlikely to succeed (falling back to the [control plane migration "bad case" scenario](../proposals/17-shoot-control-plane-migration-bad-case.md)). If you enable this feature gate, make sure to also enable `UseDNSRecords` and `CopyEtcdBackupsDuringControlPlaneMigration`.

example/20-componentconfig-gardenlet.yaml

@@ -70,6 +70,16 @@ controllers:
     syncPeriod: 1h
     waitSyncPeriod: 15s
     syncJitterPeriod: 5m
+  shootMigration:
+    concurrentSyncs: 5
+    syncPeriod: 1m
+    gracePeriod: 2h
+    lastOperationStaleDuration: 10m
+  backupEntryMigration:
+    concurrentSyncs: 5
+    syncPeriod: 1m
+    gracePeriod: 10m
+    lastOperationStaleDuration: 2m
 resources:
   capacity:
     shoots: 200
@@ -107,6 +117,7 @@ featureGates:
   UseDNSRecords: true
   DenyInvalidExtensionResources: true
   CopyEtcdBackupsDuringControlPlaneMigration: false
+  ForceRestore: false
 # seedConfig:
 #   metadata:
 #     name: my-seed

landscaper/pkg/gardenlet/chart/charttest/charttest.go

@@ -647,6 +647,18 @@ func ComputeExpectedGardenletConfiguration(
 				ConcurrentSyncs:          &twenty,
 				DeletionGracePeriodHours: &zero,
 			},
+			BackupEntryMigration: &gardenletconfigv1alpha1.BackupEntryMigrationControllerConfiguration{
+				ConcurrentSyncs: &five,
+				SyncPeriod: &metav1.Duration{
+					Duration: time.Minute,
+				},
+				GracePeriod: &metav1.Duration{
+					Duration: 10 * time.Minute,
+				},
+				LastOperationStaleDuration: &metav1.Duration{
+					Duration: 2 * time.Minute,
+				},
+			},
 			Bastion: &gardenletconfigv1alpha1.BastionControllerConfiguration{
 				ConcurrentSyncs: &twenty,
 			},
@@ -718,6 +730,18 @@ func ComputeExpectedGardenletConfiguration(
 					},
 				},
 			},
+			ShootMigration: &gardenletconfigv1alpha1.ShootMigrationControllerConfiguration{
+				ConcurrentSyncs: &five,
+				SyncPeriod: &metav1.Duration{
+					Duration: time.Minute,
+				},
+				GracePeriod: &metav1.Duration{
+					Duration: 2 * time.Hour,
+				},
+				LastOperationStaleDuration: &metav1.Duration{
+					Duration: 10 * time.Minute,
+				},
+			},
 			ShootStateSync: &gardenletconfigv1alpha1.ShootStateSyncControllerConfiguration{
 				ConcurrentSyncs: &five,
 				SyncPeriod: &metav1.Duration{

pkg/apis/core/helper/helpers.go

@@ -221,6 +221,11 @@ func SeedSettingShootDNSEnabled(settings *core.SeedSettings) bool {
 	return settings == nil || settings.ShootDNS == nil || settings.ShootDNS.Enabled
 }
 
+// SeedSettingOwnerChecksEnabled returns true if the 'ownerChecks' setting is enabled.
+func SeedSettingOwnerChecksEnabled(settings *core.SeedSettings) bool {
+	return settings == nil || settings.OwnerChecks == nil || settings.OwnerChecks.Enabled
+}
+
 // ShootUsesUnmanagedDNS returns true if the shoot's DNS section is marked as 'unmanaged'.
 func ShootUsesUnmanagedDNS(shoot *core.Shoot) bool {
 	if shoot.Spec.DNS == nil {

pkg/apis/core/helper/helpers_test.go

@@ -445,6 +445,17 @@ var _ = Describe("helper", func() {
 		Entry("shoot dns 'enabled' is true", &core.SeedSettings{ShootDNS: &core.SeedSettingShootDNS{Enabled: true}}, true),
 	)
 
+	DescribeTable("#SeedSettingOwnerChecksEnabled",
+		func(settings *core.SeedSettings, expected bool) {
+			Expect(SeedSettingOwnerChecksEnabled(settings)).To(Equal(expected))
+		},
+
+		Entry("no settings", nil, true),
+		Entry("no owner checks setting", &core.SeedSettings{}, true),
+		Entry("owner checks enabled", &core.SeedSettings{OwnerChecks: &core.SeedSettingOwnerChecks{Enabled: true}}, true),
+		Entry("owner checks disabled", &core.SeedSettings{OwnerChecks: &core.SeedSettingOwnerChecks{Enabled: false}}, false),
+	)
+
 	classificationPreview := core.ClassificationPreview
 	classificationDeprecated := core.ClassificationDeprecated
 	classificationSupported := core.ClassificationSupported

pkg/apis/core/types_backupentry.go

@@ -68,4 +68,6 @@ type BackupEntryStatus struct {
 	// SeedName is the name of the seed to which this BackupEntry is currently scheduled. This field is populated
 	// at the beginning of a create/reconcile operation. It is used when moving the BackupEntry between seeds.
 	SeedName *string
+	// MigrationStartTime is the time when a migration to a different seed was initiated.
+	MigrationStartTime *metav1.Time
 }

pkg/apis/core/types_shoot.go

@@ -139,6 +139,8 @@ type ShootStatus struct {
 	ClusterIdentity *string
 	// List of addresses on which the Kube API server can be reached.
 	AdvertisedAddresses []ShootAdvertisedAddress
+	// MigrationStartTime is the time when a migration to a different seed was initiated.
+	MigrationStartTime *metav1.Time
 }
 
 // ShootAdvertisedAddress contains information for the shoot's Kube API server.