diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index e73a444b00abd..9ebf156ab2f94 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -932,6 +932,7 @@ spec: ### Configuration It is possible to override the [containerd](https://github.com/containerd/containerd/blob/master/README.md) daemon options for all the nodes in the cluster. See the [API docs](https://pkg.go.dev/k8s.io/kops/pkg/apis/kops#ContainerdConfig) for the full list of options. +Overriding the configuration of containerd has to be done with care as the default config may change with new releases and can lead to incompatibilities. ```yaml spec: @@ -1178,3 +1179,30 @@ spec: ``` which would end up in a drop-in file on all masters and nodes of the cluster. + +## cgroupDriver + +As of Kubernetes 1.20, kOps will default the cgroup driver of the kubelet and the container runtime to use systemd as the default cgroup driver +as opposed to cgroup fs. + +It is important to ensure that the kubelet and the container runtime are using the same cgroup driver. Below are examples showing +how to set the cgroup driver for kubelet and the container runtime. + + +Setting kubelet to use cgroupfs +```yaml +spec: + kubelet: + cgroupDriver: cgroupfs +``` + +Setting Docker to use cgroupfs +```yaml +spec: + docker: + execOpt: + - native.cgroupdriver=cgroupfs +``` + +In the case of containerd, the cgroup-driver is dependant on the cgroup driver of kubelet. To use cgroupfs, just update the +cgroupDriver of kubelet to use cgroupfs. diff --git a/nodeup/pkg/model/docker_test.go b/nodeup/pkg/model/docker_test.go index 8f0c8d225196e..b8b2f248e407b 100644 --- a/nodeup/pkg/model/docker_test.go +++ b/nodeup/pkg/model/docker_test.go @@ -97,6 +97,10 @@ func TestDockerBuilder_BuildFlags(t *testing.T) { kops.DockerConfig{Bridge: fi.String("br0")}, "--bridge=br0", }, + { + kops.DockerConfig{ExecOpt: []string{"native.cgroupdriver=systemd"}}, + "--exec-opt=native.cgroupdriver=systemd", + }, } for _, g := range grid { diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go index e9f52d7a504ba..0fb5ff9debf0b 100644 --- a/nodeup/pkg/model/kubelet.go +++ b/nodeup/pkg/model/kubelet.go @@ -176,6 +176,11 @@ func (b *KubeletBuilder) buildManifestDirectory(kubeletConfig *kops.KubeletConfi // buildSystemdEnvironmentFile renders the environment file for the kubelet func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.KubeletConfigSpec) (*nodetasks.File, error) { + // Use systemd as the default cgroup driver from k8s 1.20 + if b.IsKubernetesGTE("1.20") && kubeletConfig.CgroupDriver == "" { + kubeletConfig.CgroupDriver = "systemd" + } + // @step: ensure the masters do not get a bootstrap configuration if b.UseBootstrapTokens() && b.IsMaster { kubeletConfig.BootstrapKubeconfig = "" diff --git a/pkg/model/components/containerd.go b/pkg/model/components/containerd.go index 4ca7de907b6f1..08f46c7cfd514 100644 --- a/pkg/model/components/containerd.go +++ b/pkg/model/components/containerd.go @@ -61,9 +61,9 @@ func (b *ContainerdOptionsBuilder) BuildOptions(o interface{}) error { for name, endpoints := range containerd.RegistryMirrors { config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "registry", "mirrors", name, "endpoint"}, endpoints) } + config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", "runc", "runtime_type"}, "io.containerd.runc.v2") containerd.ConfigOverride = fi.String(config.String()) } - } else if clusterSpec.ContainerRuntime == "docker" { // Docker version should always be available dockerVersion := fi.StringValue(clusterSpec.Docker.Version) diff --git a/pkg/model/components/docker.go b/pkg/model/components/docker.go index cb9101c35f4f0..adbca52dff918 100644 --- a/pkg/model/components/docker.go +++ b/pkg/model/components/docker.go @@ -73,5 +73,23 @@ func (b *DockerOptionsBuilder) BuildOptions(o interface{}) error { // and it is an error to specify the flag twice. docker.Storage = fi.String("overlay2,overlay,aufs") + // Set systemd as the default cgroup driver in docker from k8s 1.20. + if b.IsKubernetesGTE("1.20") && getDockerCgroupDriver(docker.ExecOpt) == "" { + docker.ExecOpt = append(docker.ExecOpt, "native.cgroupdriver=systemd") + } + return nil } + +// checks if cgroup-driver is configured or not for docker or not. +func getDockerCgroupDriver(execOpts []string) string { + for _, value := range execOpts { + if value == "native.cgroupdriver=systemd" { + return "systemd" + } else if value == "native.cgroupdriver=cgroupfs" { + return "cgroupfs" + } + } + + return "" +} diff --git a/pkg/model/components/kubelet.go b/pkg/model/components/kubelet.go index 2e7a12c43e00a..858373d6ac6ac 100644 --- a/pkg/model/components/kubelet.go +++ b/pkg/model/components/kubelet.go @@ -213,5 +213,10 @@ func (b *KubeletOptionsBuilder) BuildOptions(o interface{}) error { } } + // Set systemd as the default cgroup driver for kubelet from k8s 1.20 + if b.IsKubernetesGTE("1.20") && clusterSpec.Kubelet.CgroupDriver == "" { + clusterSpec.Kubelet.CgroupDriver = "systemd" + } + return nil } diff --git a/tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml index 20ef8319a8dc7..bf6a53a8cb3fc 100644 --- a/tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml @@ -144,6 +144,13 @@ Resources.AWSEC2LaunchTemplatemasterustest1amasterscontainerdexamplecom.Properti [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".registry] [plugins."io.containerd.grpc.v1.cri".registry.mirrors] @@ -479,6 +486,13 @@ Resources.AWSEC2LaunchTemplatenodescontainerdexamplecom.Properties.LaunchTemplat [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".registry] [plugins."io.containerd.grpc.v1.cri".registry.mirrors] diff --git a/tests/integration/update_cluster/containerd/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/containerd/cloudformation.json.extracted.yaml index 8abe5dbfa2906..a4d38b4d41bdb 100644 --- a/tests/integration/update_cluster/containerd/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/containerd/cloudformation.json.extracted.yaml @@ -139,6 +139,17 @@ Resources.AWSEC2LaunchTemplatemasterustest1amasterscontainerdexamplecom.Properti containerd: configOverride: | version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" logLevel: info version: 1.4.3 docker: @@ -452,6 +463,17 @@ Resources.AWSEC2LaunchTemplatenodescontainerdexamplecom.Properties.LaunchTemplat containerd: configOverride: | version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" logLevel: info version: 1.4.3 docker: