set RPS for veth on host level only (#508)

RPS handling on pod container level using crio-hooks causes long delay times when running the low latency script to set the RPS mask (https://bugzilla.redhat.com/show_bug.cgi?id=2109965)

For RAN low latency solution it might be sufficient only to set the RPS on the host level and avoid setting it on the container level while utilizing RSS behavior.

In the past the low latency hook was added with RPS additional settings on virtual devices since there was an issue where the start and shutdown of big amount of pods will initiate the creation of the systemd service that should update the new interfaces rps_cpus mask and can create an additional CPU load under the cluster (openshift-kni/performance-addon-operators#659)
This might not be the case any more thus we need to examine how the revert of the aforementioned PR will behave now.

Signed-off-by: Talor Itzhak <titzhak@redhat.com>

Signed-off-by: Talor Itzhak <titzhak@redhat.com>

assets/performanceprofile/configs/99-netdev-rps.rules

@@ -1 +1,11 @@
-SUBSYSTEM=="net", ACTION=="add", ENV{DEVPATH}!="/devices/virtual/net/veth*", ENV{ID_BUS}!="pci", TAG+="systemd", ENV{SYSTEMD_WANTS}="update-rps@%k.service"
+# We should apply the rule on the virtual interfaces of the host, because of the RPS mask that would be consulted,
+# is the one on the RX side of the veth in the host.
+# Consider the following diagram:
+# Pod A <veth1 - veth2> host <veth3 - veth4> Pod B
+#  veth2's RPS affinity is the one determining the CPUs that are handling the packet processing when sending data from Pod A to pod B.
+# Additional common scenarios:
+# 1. Pod A = sender, host = receiver
+#  The RPS affinity of the host side should be consulted (because it’s the receiver) and it should be set to cpus not sensitive to preemption (reserved pool).
+# 2. Pod A = receiver, host = sender
+#  In case of no RPS mask on the receiver side, the sender needs to pay the price and do all the processing on its cores.
+SUBSYSTEM=="net", ACTION=="add", ENV{ID_BUS}!="pci", TAG+="systemd", ENV{SYSTEMD_WANTS}="update-rps@%k.service"

assets/performanceprofile/scripts/set-rps-mask.sh

@@ -33,4 +33,6 @@ function find_dev_dir {
 [ -d "${dev_dir}" ] || { sleep 5; find_dev_dir; }  # search failed, wait a little and try again
 [ -d "${dev_dir}" ] || { echo "${dev_dir}" directory not found >&2 ; exit 0; } # the interface disappeared, not an error
 
-find "${dev_dir}"/queues -type f -name rps_cpus -exec sh -c "echo ${mask} | cat > {}" \;
+for i in "${dev_dir}"/queues/rx-*/rps_cpus; do
+  echo "${mask}" > "${i}"
+done

pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go

@@ -44,15 +44,13 @@ const (
 	// OCIHooksConfigDir is the default directory for the OCI hooks
 	OCIHooksConfigDir = "/etc/containers/oci/hooks.d"
 	// OCIHooksConfig file contains the low latency hooks configuration
-	OCIHooksConfig       = "99-low-latency-hooks.json"
 	ociTemplateRPSMask   = "RPSMask"
 	udevRulesDir         = "/etc/udev/rules.d"
 	udevRpsRules         = "99-netdev-rps.rules"
 	udevPhysicalRpsRules = "99-netdev-physical-rps.rules"
 	// scripts
 	hugepagesAllocation       = "hugepages-allocation"
 	setCPUsOffline            = "set-cpus-offline"
-	ociHooks                  = "low-latency-hooks"
 	setRPSMask                = "set-rps-mask"
 	clearIRQBalanceBannedCPUs = "clear-irqbalance-banned-cpus"
 )
@@ -146,7 +144,7 @@ func getIgnitionConfig(profile *performancev2.PerformanceProfile) (*igntypes.Con
 
 	// add script files under the node /usr/local/bin directory
 	mode := 0700
-	for _, script := range []string{hugepagesAllocation, ociHooks, setRPSMask, setCPUsOffline, clearIRQBalanceBannedCPUs} {
+	for _, script := range []string{hugepagesAllocation, setRPSMask, setCPUsOffline, clearIRQBalanceBannedCPUs} {
 		dst := getBashScriptPath(script)
 		content, err := assets.Scripts.ReadFile(fmt.Sprintf("scripts/%s.sh", script))
 		if err != nil {
@@ -164,15 +162,6 @@ func getIgnitionConfig(profile *performancev2.PerformanceProfile) (*igntypes.Con
 	crioConfSnippetDst := filepath.Join(crioConfd, crioRuntimesConfig)
 	addContent(ignitionConfig, crioConfigSnippetContent, crioConfSnippetDst, &crioConfdRuntimesMode)
 
-	// add crio hooks config  under the node cri-o hook directory
-	crioHooksConfigsMode := 0644
-	ociHooksConfigContent, err := GetOCIHooksConfigContent(OCIHooksConfig, profile)
-	if err != nil {
-		return nil, err
-	}
-	ociHookConfigDst := filepath.Join(OCIHooksConfigDir, OCIHooksConfig)
-	addContent(ignitionConfig, ociHooksConfigContent, ociHookConfigDst, &crioHooksConfigsMode)
-
 	// add rps udev rule
 	rpsRulesMode := 0644
 	var rpsRulesContent []byte

test/e2e/performanceprofile/functests/1_performance/performance.go

@@ -2,7 +2,6 @@ package __performance
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"path/filepath"
 	"reflect"
@@ -299,31 +298,13 @@ var _ = Describe("[rfe_id:27368][performance]", func() {
 
 			expectedRPSCPUs, err := cpuset.Parse(string(*profile.Spec.CPU.Reserved))
 			Expect(err).ToNot(HaveOccurred())
-			ociHookPath := filepath.Join("/rootfs", machineconfig.OCIHooksConfigDir, machineconfig.OCIHooksConfig)
-			Expect(err).ToNot(HaveOccurred())
 			for _, node := range workerRTNodes {
-				// Verify the OCI RPS hook uses the correct RPS mask
-				hooksConfig, err := nodes.ExecCommandOnMachineConfigDaemon(&node, []string{"cat", ociHookPath})
-				Expect(err).ToNot(HaveOccurred())
-
-				var hooks map[string]interface{}
-				err = json.Unmarshal(hooksConfig, &hooks)
-				Expect(err).ToNot(HaveOccurred())
-				hook := hooks["hook"].(map[string]interface{})
-				Expect(hook).ToNot(BeNil())
-				args := hook["args"].([]interface{})
-				Expect(len(args)).To(Equal(2), "unexpected arguments: %v", args)
-
-				rpsCPUs, err := components.CPUMaskToCPUSet(args[1].(string))
-				Expect(err).ToNot(HaveOccurred())
-				Expect(rpsCPUs).To(Equal(expectedRPSCPUs), "the hook rps mask is different from the reserved CPUs")
-
 				// Verify the systemd RPS service uses the correct RPS mask
 				cmd := []string{"sed", "-n", "s/^ExecStart=.*echo \\([A-Fa-f0-9]*\\) .*/\\1/p", "/rootfs/etc/systemd/system/update-rps@.service"}
 				serviceRPSCPUs, err := nodes.ExecCommandOnNode(cmd, &node)
 				Expect(err).ToNot(HaveOccurred())
 
-				rpsCPUs, err = components.CPUMaskToCPUSet(serviceRPSCPUs)
+				rpsCPUs, err := components.CPUMaskToCPUSet(serviceRPSCPUs)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(rpsCPUs).To(Equal(expectedRPSCPUs), "the service rps mask is different from the reserved CPUs")