Restore etcd with 1 replica and scale it up after kapi deployment for…

… HA shoots
gardener · Mar 21, 2024 · b5fce79 · b5fce79
1 parent 3636a01
commit b5fce79
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 43 deletions.
diff --git a/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go b/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go
@@ -312,6 +312,18 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope
 			SkipIf:       o.Shoot.HibernationEnabled || skipReadiness,
 			Dependencies: flow.NewTaskIDs(deployKubeAPIServer),
 		})
+		scaleEtcdAfterRestore = g.Add(flow.Task{
+			Name:         "Scaling main and events etcd after restoration from backup",
+			Fn:           flow.TaskFn(botanist.ScaleUpETCD).RetryUntilTimeout(defaultInterval, helper.GetEtcdDeployTimeout(o.Shoot, defaultTimeout)),
+			SkipIf:       !botanist.IsRestorePhase() || o.Shoot.HibernationEnabled || skipReadiness,
+			Dependencies: flow.NewTaskIDs(waitUntilKubeAPIServerIsReady),
+		})
+		_ = g.Add(flow.Task{
+			Name:         "Waiting until main and events etcd scaled up after restoration from backup",
+			Fn:           flow.TaskFn(botanist.WaitUntilEtcdsReady),
+			SkipIf:       !botanist.IsRestorePhase() || o.Shoot.HibernationEnabled || skipReadiness,
+			Dependencies: flow.NewTaskIDs(scaleEtcdAfterRestore),
+		})
 		deployGardenerResourceManager = g.Add(flow.Task{
 			Name:         "Deploying gardener-resource-manager",
 			Fn:           flow.TaskFn(botanist.DeployGardenerResourceManager).RetryUntilTimeout(defaultInterval, defaultTimeout),

diff --git a/pkg/gardenlet/operation/botanist/etcd.go b/pkg/gardenlet/operation/botanist/etcd.go
@@ -140,10 +140,7 @@ func (b *Botanist) DeployEtcd(ctx context.Context) error {
 		}
 	}
 
-	return flow.Parallel(
-		b.deployOrRestoreMainEtcd,
-		b.Shoot.Components.ControlPlane.EtcdEvents.Deploy,
-	)(ctx)
+	return b.deployOrRestoreEtcd(ctx)
 }
 
 // WaitUntilEtcdsReady waits until both etcd-main and etcd-events are ready.
@@ -192,17 +189,20 @@ func (b *Botanist) scaleETCD(ctx context.Context, replicas int32) error {
 	return b.Shoot.Components.ControlPlane.EtcdEvents.Scale(ctx, replicas)
 }
 
-func (b *Botanist) deployOrRestoreMainEtcd(ctx context.Context) error {
+func (b *Botanist) deployOrRestoreEtcd(ctx context.Context) error {
 	isRestoreRequired, err := b.isRestorationOfMultiNodeMainEtcdRequired(ctx)
 	if err != nil {
 		return err
 	}
 
 	if isRestoreRequired {
-		return b.restoreMultiNodeMainEtcd(ctx)
+		return b.restoreMultiNodeEtcd(ctx)
 	}
 
-	return b.Shoot.Components.ControlPlane.EtcdMain.Deploy(ctx)
+	return flow.Parallel(
+		b.Shoot.Components.ControlPlane.EtcdMain.Deploy,
+		b.Shoot.Components.ControlPlane.EtcdEvents.Deploy,
+	)(ctx)
 }
 
 func (b *Botanist) isRestorationOfMultiNodeMainEtcdRequired(ctx context.Context) (bool, error) {
@@ -233,22 +233,21 @@ func (b *Botanist) isRestorationOfMultiNodeMainEtcdRequired(ctx context.Context)
 	return true, nil
 }
 
-func (b *Botanist) restoreMultiNodeMainEtcd(ctx context.Context) error {
-	originalReplicas := b.Shoot.Components.ControlPlane.EtcdMain.GetReplicas()
-	defer func() {
-		// Revert the original replica count for the etcd. This is done in case a step
-		// is added to the reconciliation flow that depends on the etcd's replica count.
-		b.Shoot.Components.ControlPlane.EtcdMain.SetReplicas(originalReplicas)
-	}()
-
-	b.Shoot.Components.ControlPlane.EtcdMain.SetReplicas(ptr.To(int32(1)))
-	if err := b.Shoot.Components.ControlPlane.EtcdMain.Deploy(ctx); err != nil {
-		return err
-	}
-	if err := b.Shoot.Components.ControlPlane.EtcdMain.Wait(ctx); err != nil {
-		return err
+func (b *Botanist) restoreMultiNodeEtcd(ctx context.Context) error {
+	for _, component := range []etcd.Interface{b.Shoot.Components.ControlPlane.EtcdMain, b.Shoot.Components.ControlPlane.EtcdEvents} {
+		originalReplicas := component.GetReplicas()
+		defer func() {
+			// Revert the original replica count for the etcd. This is done in case a step
+			// is added to the reconciliation flow that depends on the etcd's replica count.
+			component.SetReplicas(originalReplicas)
+		}()
+		component.SetReplicas(ptr.To(int32(1)))
 	}
-	return b.Shoot.Components.ControlPlane.EtcdMain.Scale(ctx, getEtcdReplicas(b.Shoot.GetInfo()))
+
+	return flow.Parallel(
+		b.Shoot.Components.ControlPlane.EtcdMain.Deploy,
+		b.Shoot.Components.ControlPlane.EtcdEvents.Deploy,
+	)(ctx)
 }
 
 func determineBackupSchedule(shoot *gardencorev1beta1.Shoot) (string, error) {

diff --git a/pkg/gardenlet/operation/botanist/etcd_test.go b/pkg/gardenlet/operation/botanist/etcd_test.go
@@ -422,39 +422,39 @@ var _ = Describe("Etcd", func() {
 					expectGetBackupSecret()
 				})
 
-				It("should properly restore multi-node main etcd from backup if etcd does not exist", func() {
-					gomock.InOrder(
-						etcdMain.EXPECT().Get(ctx).Return(nil, apierrors.NewNotFound(schema.GroupResource{}, "")),
-						etcdMain.EXPECT().GetReplicas().Return(ptr.To(int32(3))),
-						etcdMain.EXPECT().SetReplicas(ptr.To(int32(1))),
-						etcdMain.EXPECT().Deploy(ctx),
-						etcdMain.EXPECT().Wait(ctx),
-						etcdMain.EXPECT().Scale(ctx, int32(3)),
-						etcdMain.EXPECT().SetReplicas(ptr.To(int32(3))),
-					)
+				It("should properly restore multi-node etcd from backup if etcd main does not exist yet", func() {
+					// Expect the checks for whether multi-node etcd has to be restored
+					etcdMain.EXPECT().Get(ctx).Return(nil, apierrors.NewNotFound(schema.GroupResource{}, ""))
+
+					for _, etcd := range []*mocketcd.MockInterface{etcdMain, etcdEvents} {
+						gomock.InOrder(
+							etcd.EXPECT().GetReplicas().Return(ptr.To(int32(3))),
+							etcd.EXPECT().SetReplicas(ptr.To(int32(1))),
+							etcd.EXPECT().Deploy(ctx),
+							etcd.EXPECT().SetReplicas(ptr.To(int32(3))),
+						)
+					}
 
-					etcdEvents.EXPECT().Deploy(ctx)
 					Expect(botanist.DeployEtcd(ctx)).To(Succeed())
 				})
 
-				It("should properly restore multi-node main etcd from backup if it is deployed with 1 replica", func() {
+				It("should properly restore multi-node etcd from backup if it is deployed with 1 replica", func() {
 					etcdMain.EXPECT().Get(ctx).DoAndReturn(func(_ context.Context) (*druidv1alpha1.Etcd, error) {
 						return &druidv1alpha1.Etcd{
 							Spec: druidv1alpha1.EtcdSpec{
 								Replicas: 1,
 							},
 						}, nil
 					})
-					gomock.InOrder(
-						etcdMain.EXPECT().GetReplicas().Return(ptr.To(int32(3))),
-						etcdMain.EXPECT().SetReplicas(ptr.To(int32(1))),
-						etcdMain.EXPECT().Deploy(ctx),
-						etcdMain.EXPECT().Wait(ctx),
-						etcdMain.EXPECT().Scale(ctx, int32(3)),
-						etcdMain.EXPECT().SetReplicas(ptr.To(int32(3))),
-					)
 
-					etcdEvents.EXPECT().Deploy(ctx)
+					for _, etcd := range []*mocketcd.MockInterface{etcdMain, etcdEvents} {
+						gomock.InOrder(
+							etcd.EXPECT().GetReplicas().Return(ptr.To(int32(3))),
+							etcd.EXPECT().SetReplicas(ptr.To(int32(1))),
+							etcd.EXPECT().Deploy(ctx),
+							etcd.EXPECT().SetReplicas(ptr.To(int32(3))),
+						)
+					}
 
 					Expect(botanist.DeployEtcd(ctx)).To(Succeed())
 				})