From e0ab1c161e8c61ae0f10ba954b2958f5054feee9 Mon Sep 17 00:00:00 2001 From: Ronny Baturov Date: Mon, 8 Apr 2024 13:47:58 +0300 Subject: [PATCH] Systemd processes not being moved to cpuset/systemd.slice fix (#1016) This is a manual backport for #992 * Systemd processes not being moved to cpuset/systemd.slice fix The script cpuset-configure.sh is responsible to move the systemd processes to the cpuset/systemd.slice cgroup and is executed in a form of a service (cpuset-configure.service). In the current implementation, the script is executed too early - some system processes are yet to be created. This in turn leads to them not being moved to the custom system slice. Moreover, in the current implementation, the script is executed before the network-online.target. The intention was to execute the script before kubelet and crio services are initialized (by the fact network-online.target is a common parent) in order to make sure that no workload pods are starting before we are making this transition. The fix I'm proposing consist of the following changes: 1. Adding an After statements - The script will start once crio service is initialized, due to the fact it's initialized in the very end of the boot process, just a bit before kubelet. Thereby we can ensure late starting processes do not fall between the cracks. 2. Narrowing down the Before statement to a more accurate one, reflecting its original intention. (Running the script before kubelet only would be enough guarantee no workload pods are started at that time). * Added a test to verify system processes are in the correct cgroup When we are using cgroups v1 we are counting on the cpuset-configure.service to move all the system services to the custom system.slice. This test ensures the service indeed moved them. It is also a good practice to check for similar errors on cgroup v2 systems. --------- Signed-off-by: Ronny Baturov --- .../components/machineconfig/machineconfig.go | 18 +++++++------- .../functests/1_performance/performance.go | 24 +++++++++++++++++++ ...nshift-bootstrap-master_machineconfig.yaml | 3 ++- ...nshift-bootstrap-worker_machineconfig.yaml | 3 ++- ...nshift-bootstrap-master_machineconfig.yaml | 3 ++- ...nshift-bootstrap-worker_machineconfig.yaml | 3 ++- .../default/manual_machineconfig.yaml | 3 ++- .../manual_machineconfig.yaml | 3 ++- .../no-ref/manual_machineconfig.yaml | 3 ++- 9 files changed, 48 insertions(+), 15 deletions(-) diff --git a/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go b/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go index b0f1c7b51..096ebe955 100644 --- a/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go +++ b/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go @@ -80,6 +80,7 @@ const ( systemdSectionInstall = "Install" systemdDescription = "Description" systemdBefore = "Before" + systemdAfter = "After" systemdEnvironment = "Environment" systemdType = "Type" systemdRemainAfterExit = "RemainAfterExit" @@ -88,13 +89,12 @@ const ( ) const ( - systemdServiceIRQBalance = "irqbalance.service" - systemdServiceKubelet = "kubelet.service" - systemdServiceCrio = "crio.service" - systemdServiceTypeOneshot = "oneshot" - systemdTargetMultiUser = "multi-user.target" - systemdTargetNetworkOnline = "network-online.target" - systemdTrue = "true" + systemdServiceIRQBalance = "irqbalance.service" + systemdServiceKubelet = "kubelet.service" + systemdServiceCrio = "crio.service" + systemdServiceTypeOneshot = "oneshot" + systemdTargetMultiUser = "multi-user.target" + systemdTrue = "true" ) const ( @@ -451,7 +451,9 @@ func getCpusetConfigureServiceOptions() []*unit.UnitOption { // Description unit.NewUnitOption(systemdSectionUnit, systemdDescription, "Move services to reserved cpuset"), // Before - unit.NewUnitOption(systemdSectionUnit, systemdBefore, systemdTargetNetworkOnline), + unit.NewUnitOption(systemdSectionUnit, systemdBefore, systemdServiceKubelet), + // After + unit.NewUnitOption(systemdSectionUnit, systemdAfter, systemdServiceCrio), // Type unit.NewUnitOption(systemdSectionService, systemdType, systemdServiceTypeOneshot), // ExecStart diff --git a/test/e2e/performanceprofile/functests/1_performance/performance.go b/test/e2e/performanceprofile/functests/1_performance/performance.go index fc79400f3..6c55951f4 100644 --- a/test/e2e/performanceprofile/functests/1_performance/performance.go +++ b/test/e2e/performanceprofile/functests/1_performance/performance.go @@ -32,6 +32,7 @@ import ( componentprofile "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/profile" profileutil "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/profile" testutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/cgroup" testclient "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/client" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/cluster" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/discovery" @@ -289,6 +290,29 @@ var _ = Describe("[rfe_id:27368][performance]", Ordered, func() { }) }) + Context("Using performance profile", func() { + It("[test_id: 73107] Should have system services running on the system.slice cgroup", func() { + for _, node := range workerRTNodes { + processesFound := make([]string, 0) + rootCgroupPath := "/rootfs/sys/fs/cgroup/cpuset/cgroup.procs" + isV2, err := cgroup.IsVersion2(context.TODO(), testclient.Client) + if err != nil { + Expect(err).ToNot(HaveOccurred()) + } + if isV2 { + rootCgroupPath = "/rootfs/sys/fs/cgroup/cgroup.procs" + } + // Getting the list of processes that are running on the root cgroup, filtering out the kernel threads (are presented in [square brackets]). + command := fmt.Sprintf("cat %s | xargs ps -o cmd | grep -v \"\\[\"", rootCgroupPath) + output, err := nodes.ExecCommandOnNode([]string{"/bin/bash", "-c", command}, &node) + Expect(err).ToNot(HaveOccurred()) + cmds := strings.Split(output, "\n") + processesFound = append(processesFound, cmds[1:]...) + Expect(processesFound).To(BeEmpty(), "The node %s has the following processes on the root cgroup: %v", node.Name, processesFound) + } + }) + }) + Context("Tuned kernel parameters", func() { It("[test_id:28466][crit:high][vendor:cnf-qe@redhat.com][level:acceptance] Should contain configuration injected through openshift-node-performance profile", func() { sysctlMap := map[string]string{ diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml index 4ef3fcd7e..0db397805 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml @@ -139,7 +139,8 @@ spec: - contents: | [Unit] Description=Move services to reserved cpuset - Before=network-online.target + Before=kubelet.service + After=crio.service [Service] Type=oneshot diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml index b7425bb0b..bf3523f46 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml @@ -139,7 +139,8 @@ spec: - contents: | [Unit] Description=Move services to reserved cpuset - Before=network-online.target + Before=kubelet.service + After=crio.service [Service] Type=oneshot diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml index 4ef3fcd7e..0db397805 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml @@ -139,7 +139,8 @@ spec: - contents: | [Unit] Description=Move services to reserved cpuset - Before=network-online.target + Before=kubelet.service + After=crio.service [Service] Type=oneshot diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml index b7425bb0b..bf3523f46 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml @@ -139,7 +139,8 @@ spec: - contents: | [Unit] Description=Move services to reserved cpuset - Before=network-online.target + Before=kubelet.service + After=crio.service [Service] Type=oneshot diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml index 4e8d3969a..3498c53ab 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml @@ -142,7 +142,8 @@ spec: - contents: | [Unit] Description=Move services to reserved cpuset - Before=network-online.target + Before=kubelet.service + After=crio.service [Service] Type=oneshot diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/manual_machineconfig.yaml index ff9296a42..86ac5c9d7 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/manual_machineconfig.yaml @@ -146,7 +146,8 @@ spec: - contents: | [Unit] Description=Move services to reserved cpuset - Before=network-online.target + Before=kubelet.service + After=crio.service [Service] Type=oneshot diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml index 4aca4ae1d..24addc3c5 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml @@ -141,7 +141,8 @@ spec: - contents: | [Unit] Description=Move services to reserved cpuset - Before=network-online.target + Before=kubelet.service + After=crio.service [Service] Type=oneshot