From e4efd02fcfcf2d190870ef136be319ccf327f098 Mon Sep 17 00:00:00 2001 From: Matthew Ceroni Date: Sun, 1 Aug 2021 20:35:19 -0700 Subject: [PATCH 1/6] Add runtime group/slice and assign containerd and kubelet --- packages/containerd/containerd.service | 5 +++-- packages/kubernetes-1.17/kubelet.service | 1 + packages/kubernetes-1.18/kubelet.service | 1 + packages/kubernetes-1.19/kubelet.service | 1 + packages/kubernetes-1.20/kubelet.service | 1 + packages/kubernetes-1.21/kubelet.service | 1 + packages/release/release.spec | 6 +++++- packages/release/runtime.slice | 4 ++++ 8 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 packages/release/runtime.slice diff --git a/packages/containerd/containerd.service b/packages/containerd/containerd.service index 9f47867d19d..8179b961d0a 100644 --- a/packages/containerd/containerd.service +++ b/packages/containerd/containerd.service @@ -1,10 +1,11 @@ [Unit] Description=containerd container runtime Documentation=https://containerd.io -After=network-online.target configured.target -Wants=network-online.target configured.target +After=network-online.target configured.target runtime.slice +Wants=network-online.target configured.target runtime.slice [Service] +Slice=runtime.slice EnvironmentFile=/etc/network/proxy.env ExecStart=/usr/bin/containerd Type=notify diff --git a/packages/kubernetes-1.17/kubelet.service b/packages/kubernetes-1.17/kubelet.service index d9716943d78..8bbd2799ebf 100644 --- a/packages/kubernetes-1.17/kubelet.service +++ b/packages/kubernetes-1.17/kubelet.service @@ -6,6 +6,7 @@ Wants=configured.target BindsTo=containerd.service [Service] +Slice=runtime.slice Type=notify EnvironmentFile=/etc/network/proxy.env EnvironmentFile=/etc/kubernetes/kubelet/env diff --git a/packages/kubernetes-1.18/kubelet.service b/packages/kubernetes-1.18/kubelet.service index d9716943d78..8bbd2799ebf 100644 --- a/packages/kubernetes-1.18/kubelet.service +++ b/packages/kubernetes-1.18/kubelet.service @@ -6,6 +6,7 @@ Wants=configured.target BindsTo=containerd.service [Service] +Slice=runtime.slice Type=notify EnvironmentFile=/etc/network/proxy.env EnvironmentFile=/etc/kubernetes/kubelet/env diff --git a/packages/kubernetes-1.19/kubelet.service b/packages/kubernetes-1.19/kubelet.service index d9716943d78..8bbd2799ebf 100644 --- a/packages/kubernetes-1.19/kubelet.service +++ b/packages/kubernetes-1.19/kubelet.service @@ -6,6 +6,7 @@ Wants=configured.target BindsTo=containerd.service [Service] +Slice=runtime.slice Type=notify EnvironmentFile=/etc/network/proxy.env EnvironmentFile=/etc/kubernetes/kubelet/env diff --git a/packages/kubernetes-1.20/kubelet.service b/packages/kubernetes-1.20/kubelet.service index d9716943d78..8bbd2799ebf 100644 --- a/packages/kubernetes-1.20/kubelet.service +++ b/packages/kubernetes-1.20/kubelet.service @@ -6,6 +6,7 @@ Wants=configured.target BindsTo=containerd.service [Service] +Slice=runtime.slice Type=notify EnvironmentFile=/etc/network/proxy.env EnvironmentFile=/etc/kubernetes/kubelet/env diff --git a/packages/kubernetes-1.21/kubelet.service b/packages/kubernetes-1.21/kubelet.service index d9716943d78..8bbd2799ebf 100644 --- a/packages/kubernetes-1.21/kubelet.service +++ b/packages/kubernetes-1.21/kubelet.service @@ -6,6 +6,7 @@ Wants=configured.target BindsTo=containerd.service [Service] +Slice=runtime.slice Type=notify EnvironmentFile=/etc/network/proxy.env EnvironmentFile=/etc/kubernetes/kubelet/env diff --git a/packages/release/release.spec b/packages/release/release.spec index 89a7901bb80..e752faa01d4 100644 --- a/packages/release/release.spec +++ b/packages/release/release.spec @@ -49,6 +49,9 @@ Source1060: capture-kernel-dump.service Source1061: disable-kexec-load.service Source1062: load-crash-kernel.service +# systemd cgroups/slices +Source1080: runtime.slice + BuildArch: noarch Requires: %{_cross_os}acpid Requires: %{_cross_os}audit @@ -111,7 +114,7 @@ install -d %{buildroot}%{_cross_unitdir} install -p -m 0644 \ %{S:1001} %{S:1002} %{S:1003} %{S:1004} %{S:1005} \ %{S:1006} %{S:1007} %{S:1008} %{S:1009} %{S:1010} %{S:1011} \ - %{S:1015} %{S:1040} %{S:1041} %{S:1060} %{S:1061} %{S:1062} \ + %{S:1015} %{S:1040} %{S:1041} %{S:1060} %{S:1061} %{S:1062} %{S:1080} \ %{buildroot}%{_cross_unitdir} LOWERPATH=$(systemd-escape --path %{_cross_sharedstatedir}/kernel-devel/lower) @@ -166,6 +169,7 @@ ln -s %{_cross_unitdir}/preconfigured.target %{buildroot}%{_cross_unitdir}/defau %{_cross_unitdir}/*-kernels.mount %{_cross_unitdir}/*-licenses.mount %{_cross_unitdir}/var-lib-bottlerocket.mount +%{_cross_unitdir}/runtime.slice %{_cross_unitdir}/set-hostname.service %dir %{_cross_templatedir} %{_cross_templatedir}/motd diff --git a/packages/release/runtime.slice b/packages/release/runtime.slice new file mode 100644 index 00000000000..5e189639afe --- /dev/null +++ b/packages/release/runtime.slice @@ -0,0 +1,4 @@ +[Unit] +Description=Kubernetes and container runtime slice +Documentation=man:systemd.special(7) +Before=slices.target From 7b823c36359b9e7607e5df65cce9ececb8f003c8 Mon Sep 17 00:00:00 2001 From: Matthew Ceroni Date: Mon, 2 Aug 2021 12:11:56 -0700 Subject: [PATCH 2/6] Update kubelet-config setting kubeReservedCgroup --- packages/kubernetes-1.17/kubelet-config | 1 + packages/kubernetes-1.18/kubelet-config | 1 + packages/kubernetes-1.19/kubelet-config | 1 + packages/kubernetes-1.20/kubelet-config | 1 + packages/kubernetes-1.21/kubelet-config | 1 + 5 files changed, 5 insertions(+) diff --git a/packages/kubernetes-1.17/kubelet-config b/packages/kubernetes-1.17/kubelet-config index f72becb2acf..ae54b0a3793 100644 --- a/packages/kubernetes-1.17/kubelet-config +++ b/packages/kubernetes-1.17/kubelet-config @@ -68,6 +68,7 @@ kubeReserved: {{~/if}} {{~/if}} ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}" +kubeReservedCgroup: "/runtime" {{~#if settings.kubernetes.system-reserved}} systemReserved: {{~#each settings.kubernetes.system-reserved}} diff --git a/packages/kubernetes-1.18/kubelet-config b/packages/kubernetes-1.18/kubelet-config index f72becb2acf..ae54b0a3793 100644 --- a/packages/kubernetes-1.18/kubelet-config +++ b/packages/kubernetes-1.18/kubelet-config @@ -68,6 +68,7 @@ kubeReserved: {{~/if}} {{~/if}} ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}" +kubeReservedCgroup: "/runtime" {{~#if settings.kubernetes.system-reserved}} systemReserved: {{~#each settings.kubernetes.system-reserved}} diff --git a/packages/kubernetes-1.19/kubelet-config b/packages/kubernetes-1.19/kubelet-config index bd81a83c6e9..01d9b081963 100644 --- a/packages/kubernetes-1.19/kubelet-config +++ b/packages/kubernetes-1.19/kubelet-config @@ -68,6 +68,7 @@ kubeReserved: {{~/if}} {{~/if}} ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}" +kubeReservedCgroup: "/runtime" {{~#if settings.kubernetes.system-reserved}} systemReserved: {{~#each settings.kubernetes.system-reserved}} diff --git a/packages/kubernetes-1.20/kubelet-config b/packages/kubernetes-1.20/kubelet-config index bd81a83c6e9..01d9b081963 100644 --- a/packages/kubernetes-1.20/kubelet-config +++ b/packages/kubernetes-1.20/kubelet-config @@ -68,6 +68,7 @@ kubeReserved: {{~/if}} {{~/if}} ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}" +kubeReservedCgroup: "/runtime" {{~#if settings.kubernetes.system-reserved}} systemReserved: {{~#each settings.kubernetes.system-reserved}} diff --git a/packages/kubernetes-1.21/kubelet-config b/packages/kubernetes-1.21/kubelet-config index bd81a83c6e9..01d9b081963 100644 --- a/packages/kubernetes-1.21/kubelet-config +++ b/packages/kubernetes-1.21/kubelet-config @@ -68,6 +68,7 @@ kubeReserved: {{~/if}} {{~/if}} ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}" +kubeReservedCgroup: "/runtime" {{~#if settings.kubernetes.system-reserved}} systemReserved: {{~#each settings.kubernetes.system-reserved}} From 228a321a6925c0679bd6c65864638cde5824131e Mon Sep 17 00:00:00 2001 From: Matthew Ceroni Date: Tue, 3 Aug 2021 13:32:59 -0700 Subject: [PATCH 3/6] Remove slice from After and Wants for containerd service --- packages/containerd/containerd.service | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/containerd/containerd.service b/packages/containerd/containerd.service index 8179b961d0a..3278dd7f593 100644 --- a/packages/containerd/containerd.service +++ b/packages/containerd/containerd.service @@ -1,8 +1,8 @@ [Unit] Description=containerd container runtime Documentation=https://containerd.io -After=network-online.target configured.target runtime.slice -Wants=network-online.target configured.target runtime.slice +After=network-online.target configured.target +Wants=network-online.target configured.target [Service] Slice=runtime.slice From cfed16cd5904d8bc8481382c41171297ce8f21f8 Mon Sep 17 00:00:00 2001 From: Matthew Ceroni Date: Wed, 4 Aug 2021 08:28:03 -0700 Subject: [PATCH 4/6] Set config setting systemReservedCgroup to /system --- packages/kubernetes-1.17/kubelet-config | 1 + packages/kubernetes-1.18/kubelet-config | 1 + packages/kubernetes-1.19/kubelet-config | 1 + packages/kubernetes-1.20/kubelet-config | 1 + packages/kubernetes-1.21/kubelet-config | 1 + 5 files changed, 5 insertions(+) diff --git a/packages/kubernetes-1.17/kubelet-config b/packages/kubernetes-1.17/kubelet-config index ae54b0a3793..69b1453f4c9 100644 --- a/packages/kubernetes-1.17/kubelet-config +++ b/packages/kubernetes-1.17/kubelet-config @@ -74,6 +74,7 @@ systemReserved: {{~#each settings.kubernetes.system-reserved}} {{@key}}: "{{this}}" {{~/each}} +systemReservedCgroup: "/system" {{~/if}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} {{~#if settings.kubernetes.cpu-manager-reconcile-period}} diff --git a/packages/kubernetes-1.18/kubelet-config b/packages/kubernetes-1.18/kubelet-config index ae54b0a3793..69b1453f4c9 100644 --- a/packages/kubernetes-1.18/kubelet-config +++ b/packages/kubernetes-1.18/kubelet-config @@ -74,6 +74,7 @@ systemReserved: {{~#each settings.kubernetes.system-reserved}} {{@key}}: "{{this}}" {{~/each}} +systemReservedCgroup: "/system" {{~/if}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} {{~#if settings.kubernetes.cpu-manager-reconcile-period}} diff --git a/packages/kubernetes-1.19/kubelet-config b/packages/kubernetes-1.19/kubelet-config index 01d9b081963..6558113b4fd 100644 --- a/packages/kubernetes-1.19/kubelet-config +++ b/packages/kubernetes-1.19/kubelet-config @@ -74,6 +74,7 @@ systemReserved: {{~#each settings.kubernetes.system-reserved}} {{@key}}: "{{this}}" {{~/each}} +systemReservedCgroup: "/system" {{~/if}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} {{~#if settings.kubernetes.cpu-manager-reconcile-period}} diff --git a/packages/kubernetes-1.20/kubelet-config b/packages/kubernetes-1.20/kubelet-config index 01d9b081963..6558113b4fd 100644 --- a/packages/kubernetes-1.20/kubelet-config +++ b/packages/kubernetes-1.20/kubelet-config @@ -74,6 +74,7 @@ systemReserved: {{~#each settings.kubernetes.system-reserved}} {{@key}}: "{{this}}" {{~/each}} +systemReservedCgroup: "/system" {{~/if}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} {{~#if settings.kubernetes.cpu-manager-reconcile-period}} diff --git a/packages/kubernetes-1.21/kubelet-config b/packages/kubernetes-1.21/kubelet-config index 01d9b081963..6558113b4fd 100644 --- a/packages/kubernetes-1.21/kubelet-config +++ b/packages/kubernetes-1.21/kubelet-config @@ -74,6 +74,7 @@ systemReserved: {{~#each settings.kubernetes.system-reserved}} {{@key}}: "{{this}}" {{~/each}} +systemReservedCgroup: "/system" {{~/if}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} {{~#if settings.kubernetes.cpu-manager-reconcile-period}} From 34fed14cf98266573173c707efd1d3a9e274077f Mon Sep 17 00:00:00 2001 From: Matthew Ceroni Date: Fri, 6 Aug 2021 09:28:48 -0700 Subject: [PATCH 5/6] Create {cpuset,hugetlb}/runtime.slice directory tmpfiles --- packages/release/release-tmpfiles.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/release/release-tmpfiles.conf b/packages/release/release-tmpfiles.conf index cf0aec20c2b..fa0ea4ccf96 100644 --- a/packages/release/release-tmpfiles.conf +++ b/packages/release/release-tmpfiles.conf @@ -2,4 +2,6 @@ C /etc/hosts - - - - C /etc/nsswitch.conf - - - - C /etc/wicked/ifconfig/eth0.xml - - - - d /var/log/kdump 0700 root root - +d /sys/fs/cgroup/cpuset/runtime.slice 0755 root root - +d /sys/fs/cgroup/hugetlb/runtime.slice 0755 root root - T /var/log/kdump - - - - security.selinux=system_u:object_r:secret_t:s0 From 9f630a683044abb264b03673d71319b6771e2ad9 Mon Sep 17 00:00:00 2001 From: Matthew Ceroni Date: Fri, 6 Aug 2021 15:59:48 -0700 Subject: [PATCH 6/6] Create {cpuset,hugetlb}/system.slice directory tmpfiles --- packages/release/release-tmpfiles.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/release/release-tmpfiles.conf b/packages/release/release-tmpfiles.conf index fa0ea4ccf96..537a28b0ded 100644 --- a/packages/release/release-tmpfiles.conf +++ b/packages/release/release-tmpfiles.conf @@ -4,4 +4,6 @@ C /etc/wicked/ifconfig/eth0.xml - - - - d /var/log/kdump 0700 root root - d /sys/fs/cgroup/cpuset/runtime.slice 0755 root root - d /sys/fs/cgroup/hugetlb/runtime.slice 0755 root root - +d /sys/fs/cgroup/cpuset/system.slice 0755 root root - +d /sys/fs/cgroup/hugetlb/system.slice 0755 root root - T /var/log/kdump - - - - security.selinux=system_u:object_r:secret_t:s0