Default cgroup driver to systemd from k8s 1.20

Currently, kOps uses cgroupfs cgroup driver for the kubelet and CRIs. This PR defaults the cgroup driver to systemd for clusters created with k8s versions >= 1.20. Using systemd as the cgroup-driver is the recommended way as per https://kubernetes.io/docs/setup/production-environment/container-runtimes/
kubernetes · Jan 12, 2021 · ede3ec4 · ede3ec4
1 parent 065daaa
commit ede3ec4
Show file tree

Hide file tree

Showing 8 changed files with 97 additions and 0 deletions.
diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md
@@ -932,6 +932,7 @@ spec:
 ### Configuration
 
 It is possible to override the [containerd](https://github.com/containerd/containerd/blob/master/README.md) daemon options for all the nodes in the cluster. See the [API docs](https://pkg.go.dev/k8s.io/kops/pkg/apis/kops#ContainerdConfig) for the full list of options.
+Overriding the configuration of containerd has to be done with care as the default config may change with new releases and can lead to incompatibilities.
 
 ```yaml
 spec:
@@ -1178,3 +1179,30 @@ spec:
 ```
 
 which would end up in a drop-in file on all masters and nodes of the cluster.
+
+## cgroupDriver
+
+As of Kubernetes 1.20, kOps will default the cgroup driver of the kubelet and the container runtime to use systemd as the default cgroup driver
+as opposed to cgroup fs.
+
+It is important to ensure that the kubelet and the container runtime are using the same cgroup driver. Below are examples showing
+how to set the cgroup driver for kubelet and the container runtime. 
+
+
+Setting kubelet to use cgroupfs
+```yaml
+spec:
+  kubelet:
+    cgroupDriver: cgroupfs
+```
+
+Setting Docker to use cgroupfs
+```yaml
+spec:
+  docker:
+    execOpt:
+      - native.cgroupdriver=cgroupfs
+```
+
+In the case of containerd, the cgroup-driver is dependant on the cgroup driver of kubelet. To use cgroupfs, just update the 
+cgroupDriver of kubelet to use cgroupfs.
diff --git a/nodeup/pkg/model/docker_test.go b/nodeup/pkg/model/docker_test.go
@@ -97,6 +97,10 @@ func TestDockerBuilder_BuildFlags(t *testing.T) {
 			kops.DockerConfig{Bridge: fi.String("br0")},
 			"--bridge=br0",
 		},
+		{
+			kops.DockerConfig{ExecOpt: []string{"native.cgroupdriver=systemd"}},
+			"--exec-opt=native.cgroupdriver=systemd",
+		},
 	}
 
 	for _, g := range grid {

diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go
@@ -176,6 +176,11 @@ func (b *KubeletBuilder) buildManifestDirectory(kubeletConfig *kops.KubeletConfi
 
 // buildSystemdEnvironmentFile renders the environment file for the kubelet
 func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.KubeletConfigSpec) (*nodetasks.File, error) {
+	// Use systemd as the default cgroup driver from k8s 1.20
+	if b.IsKubernetesGTE("1.20") && kubeletConfig.CgroupDriver == "" {
+		kubeletConfig.CgroupDriver = "systemd"
+	}
+
 	// @step: ensure the masters do not get a bootstrap configuration
 	if b.UseBootstrapTokens() && b.IsMaster {
 		kubeletConfig.BootstrapKubeconfig = ""

diff --git a/pkg/model/components/containerd.go b/pkg/model/components/containerd.go
@@ -61,6 +61,7 @@ func (b *ContainerdOptionsBuilder) BuildOptions(o interface{}) error {
 			for name, endpoints := range containerd.RegistryMirrors {
 				config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "registry", "mirrors", name, "endpoint"}, endpoints)
 			}
+			config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", "runc", "runtime_type"}, "io.containerd.runc.v2")
 			containerd.ConfigOverride = fi.String(config.String())
 		}
 

diff --git a/pkg/model/components/docker.go b/pkg/model/components/docker.go
@@ -73,5 +73,23 @@ func (b *DockerOptionsBuilder) BuildOptions(o interface{}) error {
 	// and it is an error to specify the flag twice.
 	docker.Storage = fi.String("overlay2,overlay,aufs")
 
+	// Set systemd as the default cgroup driver in docker from k8s 1.20.
+	if b.IsKubernetesGTE("1.20") && getDockerCgroupDriver(docker.ExecOpt) == "" {
+		docker.ExecOpt = append(docker.ExecOpt, "native.cgroupdriver=systemd")
+	}
+
 	return nil
 }
+
+// checks if cgroup-driver is configured or not for docker or not.
+func getDockerCgroupDriver(execOpts []string) string {
+	for _, value := range execOpts {
+		if value == "native.cgroupdriver=systemd" {
+			return "systemd"
+		} else if value == "native.cgroupdriver=cgroupfs" {
+			return "cgroupfs"
+		}
+	}
+
+	return ""
+}
diff --git a/pkg/model/components/kubelet.go b/pkg/model/components/kubelet.go
@@ -213,5 +213,10 @@ func (b *KubeletOptionsBuilder) BuildOptions(o interface{}) error {
 		}
 	}
 
+	// Set systemd as the default cgroup driver for kubelet from k8s 1.20
+	if b.IsKubernetesGTE("1.20") && clusterSpec.Kubelet.CgroupDriver == "" {
+		clusterSpec.Kubelet.CgroupDriver = "systemd"
+	}
+
 	return nil
 }
diff --git a/tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml
@@ -144,6 +144,13 @@ Resources.AWSEC2LaunchTemplatemasterustest1amasterscontainerdexamplecom.Properti
 
         [plugins."io.containerd.grpc.v1.cri"]
 
+          [plugins."io.containerd.grpc.v1.cri".containerd]
+
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+                runtime_type = "io.containerd.runc.v2"
+
           [plugins."io.containerd.grpc.v1.cri".registry]
 
             [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
@@ -479,6 +486,13 @@ Resources.AWSEC2LaunchTemplatenodescontainerdexamplecom.Properties.LaunchTemplat
 
         [plugins."io.containerd.grpc.v1.cri"]
 
+          [plugins."io.containerd.grpc.v1.cri".containerd]
+
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+                runtime_type = "io.containerd.runc.v2"
+
           [plugins."io.containerd.grpc.v1.cri".registry]
 
             [plugins."io.containerd.grpc.v1.cri".registry.mirrors]

diff --git a/tests/integration/update_cluster/containerd/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/containerd/cloudformation.json.extracted.yaml
@@ -139,6 +139,17 @@ Resources.AWSEC2LaunchTemplatemasterustest1amasterscontainerdexamplecom.Properti
   containerd:
     configOverride: |
       version = 2
+
+      [plugins]
+
+        [plugins."io.containerd.grpc.v1.cri"]
+
+          [plugins."io.containerd.grpc.v1.cri".containerd]
+
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+                runtime_type = "io.containerd.runc.v2"
     logLevel: info
     version: 1.4.3
   docker:
@@ -452,6 +463,17 @@ Resources.AWSEC2LaunchTemplatenodescontainerdexamplecom.Properties.LaunchTemplat
   containerd:
     configOverride: |
       version = 2
+
+      [plugins]
+
+        [plugins."io.containerd.grpc.v1.cri"]
+
+          [plugins."io.containerd.grpc.v1.cri".containerd]
+
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+                runtime_type = "io.containerd.runc.v2"
     logLevel: info
     version: 1.4.3
   docker: