From a9d9f89f72a5ca8c99acae29eb49d1b0f0639822 Mon Sep 17 00:00:00 2001 From: Bharath Vedartham Date: Tue, 12 Jan 2021 20:17:03 +0530 Subject: [PATCH] Default cgroup driver to systemd from k8s 1.20 Currently, kOps uses cgroupfs cgroup driver for the kubelet and CRIs. This PR defaults the cgroup driver to systemd for clusters created with k8s versions >= 1.20. Using systemd as the cgroup-driver is the recommended way as per https://kubernetes.io/docs/setup/production-environment/container-runtimes/ --- docs/cluster_spec.md | 28 +++++++++++++++++++ nodeup/pkg/model/docker_test.go | 4 +++ nodeup/pkg/model/kubelet.go | 5 ++++ pkg/model/components/containerd.go | 2 +- pkg/model/components/docker.go | 18 ++++++++++++ pkg/model/components/kubelet.go | 5 ++++ .../cloudformation.json.extracted.yaml | 14 ++++++++++ .../cloudformation.json.extracted.yaml | 22 +++++++++++++++ 8 files changed, 97 insertions(+), 1 deletion(-) diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index e73a444b00abd..9ebf156ab2f94 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -932,6 +932,7 @@ spec: ### Configuration It is possible to override the [containerd](https://github.com/containerd/containerd/blob/master/README.md) daemon options for all the nodes in the cluster. See the [API docs](https://pkg.go.dev/k8s.io/kops/pkg/apis/kops#ContainerdConfig) for the full list of options. +Overriding the configuration of containerd has to be done with care as the default config may change with new releases and can lead to incompatibilities. ```yaml spec: @@ -1178,3 +1179,30 @@ spec: ``` which would end up in a drop-in file on all masters and nodes of the cluster. + +## cgroupDriver + +As of Kubernetes 1.20, kOps will default the cgroup driver of the kubelet and the container runtime to use systemd as the default cgroup driver +as opposed to cgroup fs. + +It is important to ensure that the kubelet and the container runtime are using the same cgroup driver. Below are examples showing +how to set the cgroup driver for kubelet and the container runtime. + + +Setting kubelet to use cgroupfs +```yaml +spec: + kubelet: + cgroupDriver: cgroupfs +``` + +Setting Docker to use cgroupfs +```yaml +spec: + docker: + execOpt: + - native.cgroupdriver=cgroupfs +``` + +In the case of containerd, the cgroup-driver is dependant on the cgroup driver of kubelet. To use cgroupfs, just update the +cgroupDriver of kubelet to use cgroupfs. diff --git a/nodeup/pkg/model/docker_test.go b/nodeup/pkg/model/docker_test.go index 8f0c8d225196e..b8b2f248e407b 100644 --- a/nodeup/pkg/model/docker_test.go +++ b/nodeup/pkg/model/docker_test.go @@ -97,6 +97,10 @@ func TestDockerBuilder_BuildFlags(t *testing.T) { kops.DockerConfig{Bridge: fi.String("br0")}, "--bridge=br0", }, + { + kops.DockerConfig{ExecOpt: []string{"native.cgroupdriver=systemd"}}, + "--exec-opt=native.cgroupdriver=systemd", + }, } for _, g := range grid { diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go index e9f52d7a504ba..0fb5ff9debf0b 100644 --- a/nodeup/pkg/model/kubelet.go +++ b/nodeup/pkg/model/kubelet.go @@ -176,6 +176,11 @@ func (b *KubeletBuilder) buildManifestDirectory(kubeletConfig *kops.KubeletConfi // buildSystemdEnvironmentFile renders the environment file for the kubelet func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.KubeletConfigSpec) (*nodetasks.File, error) { + // Use systemd as the default cgroup driver from k8s 1.20 + if b.IsKubernetesGTE("1.20") && kubeletConfig.CgroupDriver == "" { + kubeletConfig.CgroupDriver = "systemd" + } + // @step: ensure the masters do not get a bootstrap configuration if b.UseBootstrapTokens() && b.IsMaster { kubeletConfig.BootstrapKubeconfig = "" diff --git a/pkg/model/components/containerd.go b/pkg/model/components/containerd.go index 4ca7de907b6f1..08f46c7cfd514 100644 --- a/pkg/model/components/containerd.go +++ b/pkg/model/components/containerd.go @@ -61,9 +61,9 @@ func (b *ContainerdOptionsBuilder) BuildOptions(o interface{}) error { for name, endpoints := range containerd.RegistryMirrors { config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "registry", "mirrors", name, "endpoint"}, endpoints) } + config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", "runc", "runtime_type"}, "io.containerd.runc.v2") containerd.ConfigOverride = fi.String(config.String()) } - } else if clusterSpec.ContainerRuntime == "docker" { // Docker version should always be available dockerVersion := fi.StringValue(clusterSpec.Docker.Version) diff --git a/pkg/model/components/docker.go b/pkg/model/components/docker.go index cb9101c35f4f0..adbca52dff918 100644 --- a/pkg/model/components/docker.go +++ b/pkg/model/components/docker.go @@ -73,5 +73,23 @@ func (b *DockerOptionsBuilder) BuildOptions(o interface{}) error { // and it is an error to specify the flag twice. docker.Storage = fi.String("overlay2,overlay,aufs") + // Set systemd as the default cgroup driver in docker from k8s 1.20. + if b.IsKubernetesGTE("1.20") && getDockerCgroupDriver(docker.ExecOpt) == "" { + docker.ExecOpt = append(docker.ExecOpt, "native.cgroupdriver=systemd") + } + return nil } + +// checks if cgroup-driver is configured or not for docker or not. +func getDockerCgroupDriver(execOpts []string) string { + for _, value := range execOpts { + if value == "native.cgroupdriver=systemd" { + return "systemd" + } else if value == "native.cgroupdriver=cgroupfs" { + return "cgroupfs" + } + } + + return "" +} diff --git a/pkg/model/components/kubelet.go b/pkg/model/components/kubelet.go index 2e7a12c43e00a..858373d6ac6ac 100644 --- a/pkg/model/components/kubelet.go +++ b/pkg/model/components/kubelet.go @@ -213,5 +213,10 @@ func (b *KubeletOptionsBuilder) BuildOptions(o interface{}) error { } } + // Set systemd as the default cgroup driver for kubelet from k8s 1.20 + if b.IsKubernetesGTE("1.20") && clusterSpec.Kubelet.CgroupDriver == "" { + clusterSpec.Kubelet.CgroupDriver = "systemd" + } + return nil } diff --git a/tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml index 20ef8319a8dc7..bf6a53a8cb3fc 100644 --- a/tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml @@ -144,6 +144,13 @@ Resources.AWSEC2LaunchTemplatemasterustest1amasterscontainerdexamplecom.Properti [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".registry] [plugins."io.containerd.grpc.v1.cri".registry.mirrors] @@ -479,6 +486,13 @@ Resources.AWSEC2LaunchTemplatenodescontainerdexamplecom.Properties.LaunchTemplat [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".registry] [plugins."io.containerd.grpc.v1.cri".registry.mirrors] diff --git a/tests/integration/update_cluster/containerd/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/containerd/cloudformation.json.extracted.yaml index 8abe5dbfa2906..a4d38b4d41bdb 100644 --- a/tests/integration/update_cluster/containerd/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/containerd/cloudformation.json.extracted.yaml @@ -139,6 +139,17 @@ Resources.AWSEC2LaunchTemplatemasterustest1amasterscontainerdexamplecom.Properti containerd: configOverride: | version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" logLevel: info version: 1.4.3 docker: @@ -452,6 +463,17 @@ Resources.AWSEC2LaunchTemplatenodescontainerdexamplecom.Properties.LaunchTemplat containerd: configOverride: | version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" logLevel: info version: 1.4.3 docker: