Systemd processes not being moved to cpuset/systemd.slice fix (#1016)

This is a manual backport for #992

* Systemd processes not being moved to cpuset/systemd.slice fix

The script cpuset-configure.sh is responsible to move the
systemd processes to the cpuset/systemd.slice cgroup and  is executed
in a form of a service (cpuset-configure.service).

In the current implementation, the script is executed too early - some
system processes are yet to be created.
This in turn leads to them not being moved to the custom system slice.

Moreover, in the current implementation, the script is executed before
the network-online.target. The intention was to execute the script before
kubelet and crio services are initialized (by the fact network-online.target
is a common parent) in order to make sure that no workload pods are starting
before we are making this transition.

The fix I'm proposing consist of the following changes:
1. Adding an After statements - The script will start once crio service is initialized,
due to the fact it's initialized in the very end of the boot process,
just a bit before kubelet.
Thereby we can ensure late starting processes do not fall between the cracks.
2. Narrowing down the Before statement to a more accurate
one, reflecting its original intention. (Running the script before kubelet
only would be enough guarantee no workload pods are started at that time).



* Added a test to verify system processes are in the correct cgroup

When we are using cgroups v1 we are counting on the cpuset-configure.service
to move all the system services to the custom system.slice.
This test ensures the service indeed moved them.

It is also a good practice to check for similar errors on cgroup v2 systems.



---------

Signed-off-by: Ronny Baturov <rbaturov@redhat.com>

pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go

@@ -80,6 +80,7 @@ const (
 	systemdSectionInstall  = "Install"
 	systemdDescription     = "Description"
 	systemdBefore          = "Before"
+	systemdAfter           = "After"
 	systemdEnvironment     = "Environment"
 	systemdType            = "Type"
 	systemdRemainAfterExit = "RemainAfterExit"
@@ -88,13 +89,12 @@ const (
 )
 
 const (
-	systemdServiceIRQBalance   = "irqbalance.service"
-	systemdServiceKubelet      = "kubelet.service"
-	systemdServiceCrio         = "crio.service"
-	systemdServiceTypeOneshot  = "oneshot"
-	systemdTargetMultiUser     = "multi-user.target"
-	systemdTargetNetworkOnline = "network-online.target"
-	systemdTrue                = "true"
+	systemdServiceIRQBalance  = "irqbalance.service"
+	systemdServiceKubelet     = "kubelet.service"
+	systemdServiceCrio        = "crio.service"
+	systemdServiceTypeOneshot = "oneshot"
+	systemdTargetMultiUser    = "multi-user.target"
+	systemdTrue               = "true"
 )
 
 const (
@@ -451,7 +451,9 @@ func getCpusetConfigureServiceOptions() []*unit.UnitOption {
 		// Description
 		unit.NewUnitOption(systemdSectionUnit, systemdDescription, "Move services to reserved cpuset"),
 		// Before
-		unit.NewUnitOption(systemdSectionUnit, systemdBefore, systemdTargetNetworkOnline),
+		unit.NewUnitOption(systemdSectionUnit, systemdBefore, systemdServiceKubelet),
+		// After
+		unit.NewUnitOption(systemdSectionUnit, systemdAfter, systemdServiceCrio),
 		// Type
 		unit.NewUnitOption(systemdSectionService, systemdType, systemdServiceTypeOneshot),
 		// ExecStart

test/e2e/performanceprofile/functests/1_performance/performance.go

@@ -32,6 +32,7 @@ import (
 	componentprofile "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/profile"
 	profileutil "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/profile"
 	testutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils"
+	"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/cgroup"
 	testclient "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/client"
 	"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/cluster"
 	"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/discovery"
@@ -289,6 +290,29 @@ var _ = Describe("[rfe_id:27368][performance]", Ordered, func() {
 		})
 	})
 
+	Context("Using performance profile", func() {
+		It("[test_id: 73107] Should have system services running on the system.slice cgroup", func() {
+			for _, node := range workerRTNodes {
+				processesFound := make([]string, 0)
+				rootCgroupPath := "/rootfs/sys/fs/cgroup/cpuset/cgroup.procs"
+				isV2, err := cgroup.IsVersion2(context.TODO(), testclient.Client)
+				if err != nil {
+					Expect(err).ToNot(HaveOccurred())
+				}
+				if isV2 {
+					rootCgroupPath = "/rootfs/sys/fs/cgroup/cgroup.procs"
+				}
+				// Getting the list of processes that are running on the root cgroup, filtering out the kernel threads (are presented in [square brackets]).
+				command := fmt.Sprintf("cat %s | xargs ps -o cmd | grep -v \"\\[\"", rootCgroupPath)
+				output, err := nodes.ExecCommandOnNode([]string{"/bin/bash", "-c", command}, &node)
+				Expect(err).ToNot(HaveOccurred())
+				cmds := strings.Split(output, "\n")
+				processesFound = append(processesFound, cmds[1:]...)
+				Expect(processesFound).To(BeEmpty(), "The node %s has the following processes on the root cgroup: %v", node.Name, processesFound)
+			}
+		})
+	})
+
 	Context("Tuned kernel parameters", func() {
 		It("[test_id:28466][crit:high][vendor:cnf-qe@redhat.com][level:acceptance] Should contain configuration injected through openshift-node-performance profile", func() {
 			sysctlMap := map[string]string{

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml

@@ -139,7 +139,8 @@ spec:
       - contents: |
           [Unit]
           Description=Move services to reserved cpuset
-          Before=network-online.target
+          Before=kubelet.service
+          After=crio.service
 
           [Service]
           Type=oneshot

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml

@@ -139,7 +139,8 @@ spec:
       - contents: |
           [Unit]
           Description=Move services to reserved cpuset
-          Before=network-online.target
+          Before=kubelet.service
+          After=crio.service
 
           [Service]
           Type=oneshot

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml

@@ -139,7 +139,8 @@ spec:
       - contents: |
           [Unit]
           Description=Move services to reserved cpuset
-          Before=network-online.target
+          Before=kubelet.service
+          After=crio.service
 
           [Service]
           Type=oneshot

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml

@@ -139,7 +139,8 @@ spec:
       - contents: |
           [Unit]
           Description=Move services to reserved cpuset
-          Before=network-online.target
+          Before=kubelet.service
+          After=crio.service
 
           [Service]
           Type=oneshot

test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml

@@ -142,7 +142,8 @@ spec:
       - contents: |
           [Unit]
           Description=Move services to reserved cpuset
-          Before=network-online.target
+          Before=kubelet.service
+          After=crio.service
 
           [Service]
           Type=oneshot

test/e2e/performanceprofile/testdata/render-expected-output/manual_machineconfig.yaml

@@ -146,7 +146,8 @@ spec:
       - contents: |
           [Unit]
           Description=Move services to reserved cpuset
-          Before=network-online.target
+          Before=kubelet.service
+          After=crio.service
 
           [Service]
           Type=oneshot

test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml

@@ -141,7 +141,8 @@ spec:
       - contents: |
           [Unit]
           Description=Move services to reserved cpuset
-          Before=network-online.target
+          Before=kubelet.service
+          After=crio.service
 
           [Service]
           Type=oneshot