fix: tear down control plane static pods when etcd is stopped

When `etcd` is stopped, control plane can't function anymore as API server connects to the local etcd instance. Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>
siderolabs · Sep 15, 2021 · a394d1e · a394d1e
1 parent 1c05089
commit a394d1e
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 8 deletions.
diff --git a/internal/app/machined/pkg/controllers/k8s/control_plane_static_pod.go b/internal/app/machined/pkg/controllers/k8s/control_plane_static_pod.go
@@ -82,6 +82,10 @@ func (ctrl *ControlPlaneStaticPodController) Run(ctx context.Context, r controll
 		etcdResource, err := r.Get(ctx, resource.NewMetadata(v1alpha1.NamespaceName, v1alpha1.ServiceType, "etcd", resource.VersionUndefined))
 		if err != nil {
 			if state.IsNotFoundError(err) {
+				if err = ctrl.teardownAll(ctx, r); err != nil {
+					return fmt.Errorf("error tearing down: %w", err)
+				}
+
 				continue
 			}
 

diff --git a/internal/app/machined/pkg/controllers/k8s/control_plane_static_pod_test.go b/internal/app/machined/pkg/controllers/k8s/control_plane_static_pod_test.go
@@ -116,6 +116,24 @@ func (suite *ControlPlaneStaticPodSuite) TestReconcileDefaults() {
 			)
 		},
 	))
+
+	// tear down etcd service
+	suite.Require().NoError(suite.state.Destroy(suite.ctx, v1alpha1.NewService("etcd").Metadata()))
+
+	suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
+		func() error {
+			list, err := suite.state.List(suite.ctx, resource.NewMetadata(k8s.ControlPlaneNamespaceName, k8s.StaticPodType, "", resource.VersionUndefined))
+			if err != nil {
+				return err
+			}
+
+			if len(list.Items) > 0 {
+				return retry.ExpectedErrorf("expected no pods, got %d", len(list.Items))
+			}
+
+			return nil
+		},
+	))
 }
 
 func (suite *ControlPlaneStaticPodSuite) TestReconcileExtraMounts() {

diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod_controller.go b/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod_controller.go
@@ -176,7 +176,7 @@ func (ctrl *KubeletStaticPodController) Run(ctx context.Context, r controller.Ru
 		for _, staticPod := range staticPods.Items {
 			switch staticPod.Metadata().Phase() {
 			case resource.PhaseRunning:
-				if err = ctrl.writePod(ctx, r, logger, staticPod); err != nil {
+				if err = ctrl.writePod(logger, staticPod); err != nil {
 					return fmt.Errorf("error running pod: %w", err)
 				}
 			case resource.PhaseTearingDown:
@@ -207,13 +207,7 @@ func (ctrl *KubeletStaticPodController) podFilename(staticPod resource.Resource)
 	return fmt.Sprintf("%s%s.yaml", constants.TalosManifestPrefix, staticPod.Metadata().ID())
 }
 
-func (ctrl *KubeletStaticPodController) writePod(ctx context.Context, r controller.Runtime, logger *zap.Logger, staticPod resource.Resource) error {
-	staticPodStatus := k8s.NewStaticPodStatus(staticPod.Metadata().Namespace(), staticPod.Metadata().ID())
-
-	if err := r.AddFinalizer(ctx, staticPod.Metadata(), staticPodStatus.String()); err != nil {
-		return err
-	}
-
+func (ctrl *KubeletStaticPodController) writePod(logger *zap.Logger, staticPod resource.Resource) error {
 	renderedPod, err := yaml.Marshal(staticPod.Spec())
 	if err != nil {
 		return err