Skip to content
This repository has been archived by the owner on Jun 29, 2022. It is now read-only.

Commit

Permalink
Fix controlplane running as DaemonSet on single node clusters
Browse files Browse the repository at this point in the history
This commit fixes passed 'control_plane_replicas' value to Kubernetes
Helm chart which caused kube-scheduler and kube-controller-manager to
run as DaemonSet on single controlplane node clusters, which breaks the
ability to update it gracefully.

It also adds tests that controlplane is using right resource type on
different controlplane sizes and that both can be gracefully updated
without breaking cluster functionality.

Closes #1097
Closes #90

Signed-off-by: Mateusz Gozdek <mateusz@kinvolk.io>
  • Loading branch information
invidian committed Nov 27, 2020
1 parent e597112 commit 7b385c4
Show file tree
Hide file tree
Showing 7 changed files with 183 additions and 39 deletions.
42 changes: 21 additions & 21 deletions assets/terraform-modules/bootkube/assets.tf
Original file line number Diff line number Diff line change
Expand Up @@ -31,26 +31,26 @@ resource "local_file" "bootstrap-scheduler" {
resource "local_file" "kube-apiserver" {
filename = "${var.asset_dir}/charts/kube-system/kube-apiserver.yaml"
content = templatefile("${path.module}/resources/charts/kube-apiserver.yaml", {
kube_apiserver_image = var.container_images["kube_apiserver"]
etcd_servers = join(",", formatlist("https://%s:2379", var.etcd_servers))
cloud_provider = var.cloud_provider
service_cidr = var.service_cidr
trusted_certs_dir = var.trusted_certs_dir
ca_cert = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
apiserver_key = base64encode(tls_private_key.apiserver.private_key_pem)
apiserver_cert = base64encode(tls_locally_signed_cert.apiserver.cert_pem)
serviceaccount_pub = base64encode(tls_private_key.service-account.public_key_pem)
etcd_ca_cert = base64encode(tls_self_signed_cert.etcd-ca.cert_pem)
etcd_client_cert = base64encode(tls_locally_signed_cert.client.cert_pem)
etcd_client_key = base64encode(tls_private_key.client.private_key_pem)
enable_aggregation = var.enable_aggregation
aggregation_ca_cert = var.enable_aggregation == true ? base64encode(join(" ", tls_self_signed_cert.aggregation-ca.*.cert_pem)) : ""
aggregation_client_cert = var.enable_aggregation == true ? base64encode(join(" ", tls_locally_signed_cert.aggregation-client.*.cert_pem)) : ""
aggregation_client_key = var.enable_aggregation == true ? base64encode(join(" ", tls_private_key.aggregation-client.*.private_key_pem)) : ""
replicas = length(var.etcd_servers)
extra_flags = var.kube_apiserver_extra_flags
enable_tls_bootstrap = var.enable_tls_bootstrap
ignore_x509_cn_check = var.ignore_x509_cn_check
kube_apiserver_image = var.container_images["kube_apiserver"]
etcd_servers = join(",", formatlist("https://%s:2379", var.etcd_servers))
cloud_provider = var.cloud_provider
service_cidr = var.service_cidr
trusted_certs_dir = var.trusted_certs_dir
ca_cert = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
apiserver_key = base64encode(tls_private_key.apiserver.private_key_pem)
apiserver_cert = base64encode(tls_locally_signed_cert.apiserver.cert_pem)
serviceaccount_pub = base64encode(tls_private_key.service-account.public_key_pem)
etcd_ca_cert = base64encode(tls_self_signed_cert.etcd-ca.cert_pem)
etcd_client_cert = base64encode(tls_locally_signed_cert.client.cert_pem)
etcd_client_key = base64encode(tls_private_key.client.private_key_pem)
enable_aggregation = var.enable_aggregation
aggregation_ca_cert = var.enable_aggregation == true ? base64encode(join(" ", tls_self_signed_cert.aggregation-ca.*.cert_pem)) : ""
aggregation_client_cert = var.enable_aggregation == true ? base64encode(join(" ", tls_locally_signed_cert.aggregation-client.*.cert_pem)) : ""
aggregation_client_key = var.enable_aggregation == true ? base64encode(join(" ", tls_private_key.aggregation-client.*.private_key_pem)) : ""
replicas = length(var.etcd_servers)
extra_flags = var.kube_apiserver_extra_flags
enable_tls_bootstrap = var.enable_tls_bootstrap
ignore_x509_cn_check = var.ignore_x509_cn_check
})
}

Expand All @@ -69,7 +69,7 @@ resource "local_file" "kubernetes" {
kube_scheduler_image = var.container_images["kube_scheduler"]
kube_proxy_image = var.container_images["kube_proxy"]
coredns_image = var.container_images["coredns"]
control_plane_replicas = max(2, length(var.etcd_servers))
control_plane_replicas = length(var.etcd_servers)
cloud_provider = var.cloud_provider
pod_cidr = var.pod_cidr
service_cidr = var.service_cidr
Expand Down
4 changes: 2 additions & 2 deletions pkg/assets/generated_assets.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// Copyright 2020 The Lokomotive Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build aws aws_edge
// +build disruptivee2e

package kubernetes_test

import (
"context"
"testing"
"time"

testutil "github.com/kinvolk/lokomotive/test/components/util"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func TestControlplaneComponentsDaemonSetsCanBeGracefullyUpdated(t *testing.T) {
client := testutil.CreateKubeClient(t)
dsClient := client.AppsV1().DaemonSets(namespace)

components := components()
components["kube-proxy"] = testutil.RetryInterval

for c, waitTime := range components {
c := c
waitTime := waitTime

t.Run(c, func(t *testing.T) {
ds, err := dsClient.Get(context.TODO(), c, metav1.GetOptions{})
if err != nil {
t.Fatalf("Getting DaemonSet %q: %v", c, err)
}

// Use current time to have different value on each test run.
ds.Spec.Template.Annotations["update-test"] = time.Now().String()

if _, err := dsClient.Update(context.TODO(), ds, metav1.UpdateOptions{}); err != nil {
t.Fatalf("Updating DaemonSet %q: %v", c, err)
}

// Wait a bit to let Kubernetes trigger pod updates.
time.Sleep(waitTime)

testutil.WaitForDaemonSet(t, client, namespace, c, testutil.RetryInterval, testutil.Timeout)
})
}
}
Loading

0 comments on commit 7b385c4

Please sign in to comment.