Fix controlplane running as DaemonSet on single node clusters

This commit fixes passed 'control_plane_replicas' value to Kubernetes
Helm chart which caused kube-scheduler and kube-controller-manager to
run as DaemonSet on single controlplane node clusters, which breaks the
ability to update it gracefully.

It also adds tests that controlplane is using right resource type on
different controlplane sizes and that both can be gracefully updated
without breaking cluster functionality.

Closes #1097
Closes #90

Signed-off-by: Mateusz Gozdek <mateusz@kinvolk.io>

assets/terraform-modules/bootkube/assets.tf

@@ -69,7 +69,7 @@ resource "local_file" "kubernetes" {
     kube_scheduler_image          = var.container_images["kube_scheduler"]
     kube_proxy_image              = var.container_images["kube_proxy"]
     coredns_image                 = var.container_images["coredns"]
-    control_plane_replicas        = max(2, length(var.etcd_servers))
+    control_plane_replicas        = length(var.etcd_servers)
     cloud_provider                = var.cloud_provider
     pod_cidr                      = var.pod_cidr
     service_cidr                  = var.service_cidr

test/components/kubernetes/controlplane-multi-node_disruptive_test.go

@@ -0,0 +1,59 @@
+// Copyright 2020 The Lokomotive Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build aws aws_edge
+// +build disruptivee2e
+
+package kubernetes_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	testutil "github.com/kinvolk/lokomotive/test/components/util"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestControlplaneComponentsDaemonSetsCanBeGracefullyUpdated(t *testing.T) {
+	client := testutil.CreateKubeClient(t)
+	dsClient := client.AppsV1().DaemonSets(namespace)
+
+	components := components()
+	components["kube-proxy"] = testutil.RetryInterval
+
+	for c, waitTime := range components {
+		c := c
+		waitTime := waitTime
+
+		t.Run(c, func(t *testing.T) {
+			ds, err := dsClient.Get(context.TODO(), c, metav1.GetOptions{})
+			if err != nil {
+				t.Fatalf("Getting DaemonSet %q: %v", c, err)
+			}
+
+			// Use current time to have different value on each test run.
+			ds.Spec.Template.Annotations["update-test"] = time.Now().String()
+
+			if _, err := dsClient.Update(context.TODO(), ds, metav1.UpdateOptions{}); err != nil {
+				t.Fatalf("Updating DaemonSet %q: %v", c, err)
+			}
+
+			// Wait a bit to let Kubernetes trigger pod updates.
+			time.Sleep(waitTime)
+
+			testutil.WaitForDaemonSet(t, client, namespace, c, testutil.RetryInterval, testutil.Timeout)
+		})
+	}
+}

test/components/coredns/coredns_test.go → test/components/kubernetes/controlplane-multi-node_test.go

@@ -12,24 +12,27 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build aws aws_edge packet
+// +build aws aws_edge
 // +build e2e
 
-package coredns
+package kubernetes_test
 
 import (
 	"testing"
 
 	testutil "github.com/kinvolk/lokomotive/test/components/util"
 )
 
-func TestCoreDNSDaemonSet(t *testing.T) {
-	t.Parallel()
+func TestControlplaneComponentsAreDaemonSetsOnMultiControllerNodeCluster(t *testing.T) {
+	client := testutil.CreateKubeClient(t)
 
-	namespace := "kube-system"
-	daemonset := "coredns"
+	for c := range components() {
+		c := c
 
-	client := testutil.CreateKubeClient(t)
+		t.Run(c, func(t *testing.T) {
+			t.Parallel()
 
-	testutil.WaitForDaemonSet(t, client, namespace, daemonset, testutil.RetryInterval, testutil.Timeout)
+			testutil.WaitForDaemonSet(t, client, namespace, c, testutil.RetryInterval, testutil.Timeout)
+		})
+	}
 }

test/components/kubernetes/controlplane-single-node_disruptive_test.go

@@ -0,0 +1,57 @@
+// Copyright 2020 The Lokomotive Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build packet baremetal
+// +build disruptivee2e
+
+package kubernetes_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	testutil "github.com/kinvolk/lokomotive/test/components/util"
+)
+
+func TestControlplaneComponentsDeploymentsCanBeGracefullyUpdated(t *testing.T) {
+	client := testutil.CreateKubeClient(t)
+	deployClient := client.AppsV1().Deployments(namespace)
+
+	for c, waitTime := range components() {
+		c := c
+		waitTime := waitTime
+
+		t.Run(c, func(t *testing.T) {
+			deploy, err := deployClient.Get(context.TODO(), c, metav1.GetOptions{})
+			if err != nil {
+				t.Fatalf("Getting Deployment %q: %v", c, err)
+			}
+
+			// Use current time to have different value on each test run.
+			deploy.Spec.Template.Annotations["update-test"] = time.Now().String()
+
+			if _, err := deployClient.Update(context.TODO(), deploy, metav1.UpdateOptions{}); err != nil {
+				t.Fatalf("Updating Deployment %q: %v", c, err)
+			}
+
+			// Wait a bit to let Kubernetes trigger pod updates.
+			time.Sleep(waitTime)
+
+			testutil.WaitForDeployment(t, client, namespace, c, testutil.RetryInterval, testutil.Timeout)
+		})
+	}
+}

test/components/kubernetes/controller-manager_test.go → test/components/kubernetes/controlplane-single-node_test.go

@@ -12,24 +12,27 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build aws aws_edge packet
+// +build packet baremetal
 // +build e2e
 
-package kubernetes
+package kubernetes_test
 
 import (
 	"testing"
 
 	testutil "github.com/kinvolk/lokomotive/test/components/util"
 )
 
-func TestControllerManagerDaemonSet(t *testing.T) {
-	t.Parallel()
+func TestControlplaneComponentsAreDeploymentsOnSingleControllerNodeCluster(t *testing.T) {
+	client := testutil.CreateKubeClient(t)
 
-	namespace := "kube-system"
-	daemonset := "kube-controller-manager"
+	for c := range components() {
+		c := c
 
-	client := testutil.CreateKubeClient(t)
+		t.Run(c, func(t *testing.T) {
+			t.Parallel()
 
-	testutil.WaitForDaemonSet(t, client, namespace, daemonset, testutil.RetryInterval, testutil.Timeout)
+			testutil.WaitForDeployment(t, client, namespace, c, testutil.RetryInterval, testutil.Timeout)
+		})
+	}
 }

test/components/kubernetes/controlplane_test.go

@@ -0,0 +1,22 @@
+package kubernetes_test
+
+import (
+	"time"
+
+	testutil "github.com/kinvolk/lokomotive/test/components/util"
+)
+
+const (
+	namespace = "kube-system"
+)
+
+func components() map[string]time.Duration {
+	return map[string]time.Duration{
+		// If we kill active kube-controller-manager, it takes time for new one to kick in
+		// so we need to wait longer.
+		"kube-controller-manager": 1 * time.Minute,
+		"kube-scheduler":          testutil.RetryInterval,
+		"kube-apiserver":          testutil.RetryInterval,
+		"coredns":                 testutil.RetryInterval,
+	}
+}