Skip to content

Commit

Permalink
set RPS for veth on host level only
Browse files Browse the repository at this point in the history
RPS handling on pod container level using crio-hooks causes long delay
times when running the low latency script to set the RPS mask
(https://bugzilla.redhat.com/show_bug.cgi?id=2109965)

For RAN low latency solution it might be sufficient
only to set the RPS on the host level and avoid
setting it on the container level while utilizing RSS behavior.

In the past the low latency hook was added with RPS additional
settings on virtual devices since there was an issue where the start
and shutdown of big amount of pods will initiate the creation
of the systemd service that should update the new interfaces
rps_cpus mask and can create an additional CPU load
under the cluster (openshift-kni#659)

This might not be the case any more thus we need to examine
how the revert of the aforementioned PR will behave now.

Co-authored-by: Yanir Quinn <yquinn@redhat.com>
Signed-off-by: Talor Itzhak <titzhak@redhat.com>
  • Loading branch information
Tal-or and yanirq committed Nov 23, 2022
1 parent a0b68a4 commit 12c8dba
Show file tree
Hide file tree
Showing 6 changed files with 58 additions and 159 deletions.
11 changes: 0 additions & 11 deletions build/assets/configs/99-low-latency-hooks.json

This file was deleted.

2 changes: 1 addition & 1 deletion build/assets/configs/99-netdev-rps.rules
@@ -1 +1 @@
SUBSYSTEM=="net", ACTION=="add", ENV{DEVPATH}!="/devices/virtual/net/veth*", ENV{ID_BUS}!="pci", TAG+="systemd", ENV{SYSTEMD_WANTS}="update-rps@%k.service"
SUBSYSTEM=="net", ACTION=="add", ENV{ID_BUS}!="pci", TAG+="systemd", ENV{SYSTEMD_WANTS}="update-rps@%k.service"
38 changes: 0 additions & 38 deletions build/assets/scripts/low-latency-hooks.sh

This file was deleted.

20 changes: 1 addition & 19 deletions functests/1_performance/performance.go
Expand Up @@ -2,7 +2,6 @@ package __performance

import (
"context"
"encoding/json"
"fmt"
"path/filepath"
"reflect"
Expand Down Expand Up @@ -360,31 +359,14 @@ var _ = Describe("[rfe_id:27368][performance]", func() {

expectedRPSCPUs, err := cpuset.Parse(string(*profile.Spec.CPU.Reserved))
Expect(err).ToNot(HaveOccurred())
ociHookPath := filepath.Join("/rootfs", machineconfig.OCIHooksConfigDir, machineconfig.OCIHooksConfig+".json")
Expect(err).ToNot(HaveOccurred())
for _, node := range workerRTNodes {
// Verify the OCI RPS hook uses the correct RPS mask
hooksConfig, err := nodes.ExecCommandOnMachineConfigDaemon(&node, []string{"cat", ociHookPath})
Expect(err).ToNot(HaveOccurred())

var hooks map[string]interface{}
err = json.Unmarshal(hooksConfig, &hooks)
Expect(err).ToNot(HaveOccurred())
hook := hooks["hook"].(map[string]interface{})
Expect(hook).ToNot(BeNil())
args := hook["args"].([]interface{})
Expect(len(args)).To(Equal(2), "unexpected arguments: %v", args)

rpsCPUs, err := components.CPUMaskToCPUSet(args[1].(string))
Expect(err).ToNot(HaveOccurred())
Expect(rpsCPUs).To(Equal(expectedRPSCPUs), "the hook rps mask is different from the reserved CPUs")

// Verify the systemd RPS service uses the correct RPS mask
cmd := []string{"sed", "-n", "s/^ExecStart=.*echo \\([A-Fa-f0-9]*\\) .*/\\1/p", "/rootfs/etc/systemd/system/update-rps@.service"}
serviceRPSCPUs, err := nodes.ExecCommandOnNode(cmd, &node)
Expect(err).ToNot(HaveOccurred())

rpsCPUs, err = components.CPUMaskToCPUSet(serviceRPSCPUs)
rpsCPUs, err := components.CPUMaskToCPUSet(serviceRPSCPUs)
Expect(err).ToNot(HaveOccurred())
Expect(rpsCPUs).To(Equal(expectedRPSCPUs), "the service rps mask is different from the reserved CPUs")

Expand Down
Expand Up @@ -39,11 +39,8 @@ const (
bashScriptsDir = "/usr/local/bin"
crioConfd = "/etc/crio/crio.conf.d"
crioRuntimesConfig = "99-runtimes.conf"
ociHooks = "low-latency-hooks"
// OCIHooksConfigDir is the default directory for the OCI hooks
OCIHooksConfigDir = "/etc/containers/oci/hooks.d"
// OCIHooksConfig file contains the low latency hooks configuration
OCIHooksConfig = "99-low-latency-hooks"
OCIHooksConfigDir = "/etc/containers/oci/hooks.d"
ociTemplateRPSMask = "RPSMask"
udevRulesDir = "/etc/udev/rules.d"
udevRpsRules = "99-netdev-rps"
Expand Down Expand Up @@ -138,7 +135,7 @@ func getIgnitionConfig(assetsDir string, profile *performancev2.PerformanceProfi

// add script files under the node /usr/local/bin directory
mode := 0700
for _, script := range []string{hugepagesAllocation, ociHooks, setRPSMask} {
for _, script := range []string{hugepagesAllocation, setRPSMask} {
src := filepath.Join(assetsDir, "scripts", fmt.Sprintf("%s.sh", script))
if err := addFile(ignitionConfig, src, getBashScriptPath(script), &mode); err != nil {
return nil, err
Expand All @@ -161,23 +158,6 @@ func getIgnitionConfig(assetsDir string, profile *performancev2.PerformanceProfi
return nil, err
}

// add crio hooks config under the node cri-o hook directory
crioHooksConfigsMode := 0644
config := fmt.Sprintf("%s.json", OCIHooksConfig)
ociHooksConfigContent, err := GetOCIHooksConfigContent(filepath.Join(assetsDir, "configs", config), profile)
if err != nil {
return nil, err
}

if err := addContent(
ignitionConfig,
ociHooksConfigContent,
filepath.Join(OCIHooksConfigDir, config),
&crioHooksConfigsMode,
); err != nil {
return nil, err
}

// add rps udev rule
rpsRuleMode := 0644
var rule string
Expand Down

0 comments on commit 12c8dba

Please sign in to comment.