Skip to content

Commit

Permalink
e2e:utils: introduce cgroup package
Browse files Browse the repository at this point in the history 
This package provides a ControllerGetter which is cgroup version agnostic.
It provides a unified api for testing the mixedcpus feature for both v1 and v2.

The package can be expand in the future for testing additional features
that requires cgroups inspection.

Also in the future this package should be runtime (crun/runc) agnostic.

Signed-off-by: Talor Itzhak <titzhak@redhat.com>
  • Loading branch information
@Tal-or
Tal-or committed Jan 28, 2024
1 parent 69a21cd commit 58b3575
Show file tree
Hide file tree
Showing 6 changed files with 313 additions and 2 deletions.
4 changes: 2 additions & 2 deletions test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go
View file
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components"
profileutil "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/profile"
testutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils"
"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/cgroup/runtime"
testclient "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/client"
testlog "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/log"
"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/mcps"
Expand Down Expand Up @@ -96,7 +97,7 @@ var _ = Describe("Mixedcpus", Ordered, func() {
"/rootfs",
"/bin/bash",
"-c",
fmt.Sprintf("/bin/awk -F '\"' '/shared_cpuset.*/ { print $2 }' %s", crioRuntimeConfigFile),
fmt.Sprintf("/bin/awk -F '\"' '/shared_cpuset.*/ { print $2 }' %s", runtime.CRIORuntimeConfigFile),
}
cpus, err := nodes.ExecCommandOnNode(ctx, cmd, worker)
Expect(err).ToNot(HaveOccurred(), "failed to execute command on node; cmd=%q node=%q", cmd, worker)
Expand Down Expand Up @@ -124,7 +125,6 @@ func setup(ctx context.Context, profile *performancev2.PerformanceProfile) func(
profile.Spec.CPU.Shared = cpuSetToPerformanceCPUSet(&sharedcpu)
profile.Spec.WorkloadHints.MixedCpus = pointer.Bool(true)
testprofiles.UpdateWithRetry(profile)

mcp, err := mcps.GetByProfile(profile)
Expect(err).ToNot(HaveOccurred())

Expand Down
53 changes: 53 additions & 0 deletions test/e2e/performanceprofile/functests/utils/cgroup/cgroup.go
View file
@@ -0,0 +1,53 @@
package cgroup

import (
"context"
"fmt"
corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/controller-runtime/pkg/client"

apiconfigv1 "github.com/openshift/api/config/v1"
cgroupv1 "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/cgroup/v1"
cgroupv2 "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/cgroup/v2"
)

type ControllersGetter interface {
// Pod is for getting controller config at the pod level
Pod(ctx context.Context, pod *corev1.Pod, controllerConfig interface{}) error
// Container is for getting controller config at the container level
Container(ctx context.Context, pod *corev1.Pod, containerName string, controllerConfig interface{}) error
// Child is for getting controller config at the container's child level
Child(ctx context.Context, pod *corev1.Pod, containerName, childName string, controllerConfig interface{}) error
}

func IsVersion2(ctx context.Context, c client.Client) (bool, error) {
nodecfg := &apiconfigv1.Node{}
key := client.ObjectKey{
Name: "cluster",
}
err := c.Get(ctx, key, nodecfg)
if err != nil {
return false, fmt.Errorf("failed to get configs.node object. name=%q; %w", key.Name, err)
}
if nodecfg.Spec.CgroupMode == apiconfigv1.CgroupModeV1 {
return false, nil
}
// in case `apiconfigv1.CgroupMode` not set, it's returned true since
// the platform's default is cgroupV2
return true, nil
}

// BuildGetter return a cgroup information getter that complies with
// the cgroup version that is currently used by the nodes on the cluster
func BuildGetter(ctx context.Context, c client.Client, k8sClient *kubernetes.Clientset) (ControllersGetter, error) {
isV2, versionErr := IsVersion2(ctx, c)
if versionErr != nil {
return nil, versionErr
}
if isV2 {
return cgroupv2.NewManager(c, k8sClient), nil
} else {
return cgroupv1.NewManager(c, k8sClient), nil
}
}
20 changes: 20 additions & 0 deletions test/e2e/performanceprofile/functests/utils/cgroup/controller/controller.go
View file
@@ -0,0 +1,20 @@
package controller

const CgroupMountPoint = "/sys/fs/cgroup"

type CpuSet struct {
Cpus string
Mems string
Effective string
// Partition only applicable for cgroupv2
Partition string
Exclusive string
// SchedLoadBalance true if enabled, only applicable for cgroupv1
SchedLoadBalance bool
}

type Cpu struct {
Quota string
Period string
cpuStat map[string]string
}
37 changes: 37 additions & 0 deletions test/e2e/performanceprofile/functests/utils/cgroup/runtime/runtime.go
View file
@@ -0,0 +1,37 @@
package runtime

import (
"context"
"fmt"
"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/nodes"
corev1 "k8s.io/api/core/v1"
"path/filepath"
"sigs.k8s.io/controller-runtime/pkg/client"
)

const (
Crun = "crun"
Runc = "runc"
CRIORuntimeConfigFile = "/etc/crio/crio.conf.d/99-runtimes.conf"
)

// GetContainerRuntimeTypeFor return the container runtime type that is being used
// in the node where the given pod is running
func GetContainerRuntimeTypeFor(ctx context.Context, c client.Client, pod *corev1.Pod) (string, error) {
node := &corev1.Node{}
if err := c.Get(ctx, client.ObjectKey{Name: pod.Spec.NodeName}, node); err != nil {
return "", err
}
cmd := []string{
"chroot",
"/rootfs",
"/bin/bash",
"-c",
fmt.Sprintf("/bin/awk -F '\"' '/runtime_path.*/ { print $2 }' %s", CRIORuntimeConfigFile),
}
out, err := nodes.ExecCommandOnNode(ctx, cmd, node)
if err != nil {
return "", fmt.Errorf("failed to execute command on node; cmd=%q node=%q", cmd, node.Name)
}
return filepath.Base(out), nil
}
101 changes: 101 additions & 0 deletions test/e2e/performanceprofile/functests/utils/cgroup/v1/v1.go
View file
@@ -0,0 +1,101 @@
package v1

import (
"context"
"fmt"
"path"
"strings"

corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/cgroup/controller"
"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/cgroup/runtime"
"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/pods"
)

type ControllersManager struct {
client client.Client
k8sClient *kubernetes.Clientset
}

func NewManager(c client.Client, k8sClient *kubernetes.Clientset) *ControllersManager {
return &ControllersManager{client: c, k8sClient: k8sClient}
}

func (cm *ControllersManager) CpuSet(ctx context.Context, pod *corev1.Pod, containerName, childName, runtimeType string) (*controller.CpuSet, error) {
cfg := &controller.CpuSet{}
dirPath := path.Join(controller.CgroupMountPoint, childName)
cmd := []string{
"/bin/cat",
dirPath + "/cpuset/cpuset.cpus",
dirPath + "/cpuset/cpuset.cpu_exclusive",
dirPath + "/cpuset/cpuset.effective_cpus",
dirPath + "/cpuset/cpuset.mems",
dirPath + "/cpuset/cpuset.sched_load_balance",
}
b, err := pods.ExecCommandOnPod(cm.k8sClient, pod, containerName, cmd)
if err != nil {
return nil, fmt.Errorf("failed to retrieve cgroup config for pod. pod=%q, container=%q; %w", client.ObjectKeyFromObject(pod).String(), containerName, err)
}
output := strings.Split(string(b), "\r\n")
cfg.Cpus = output[0]
cfg.Exclusive = output[1]
cfg.Effective = output[2]
cfg.Mems = output[3]
cfg.SchedLoadBalance = output[4] == "1"
return cfg, nil
}

func (cm *ControllersManager) Cpu(ctx context.Context, pod *corev1.Pod, containerName, childName, runtimeType string) (*controller.Cpu, error) {
cfg := &controller.Cpu{}
dirPath := path.Join(controller.CgroupMountPoint, childName)
cmd := []string{
"/bin/cat",
dirPath + "/cpu/cpu.cfs_quota_us",
dirPath + "/cpu/cpu.cfs_period_us",
}
b, err := pods.ExecCommandOnPod(cm.k8sClient, pod, containerName, cmd)
if err != nil {
return nil, fmt.Errorf("failed to retrieve cgroup config for pod. pod=%q, container=%q; %w", client.ObjectKeyFromObject(pod).String(), containerName, err)
}
output := strings.Split(string(b), "\r\n")
cfg.Quota = output[0]
cfg.Period = output[1]
return cfg, nil
}

func (cm *ControllersManager) Pod(ctx context.Context, pod *corev1.Pod, controllerConfig interface{}) error {
// TODO
return nil
}

func (cm *ControllersManager) Container(ctx context.Context, pod *corev1.Pod, containerName string, controllerConfig interface{}) error {
runtimeType, err := runtime.GetContainerRuntimeTypeFor(ctx, cm.client, pod)
if err != nil {
return err
}
switch cc := controllerConfig.(type) {
case *controller.CpuSet:
cfg, err := cm.CpuSet(ctx, pod, containerName, "", runtimeType)
if err != nil {
return err
}
*cc = *cfg
case *controller.Cpu:
cfg, err := cm.Cpu(ctx, pod, containerName, "", runtimeType)
if err != nil {
return err
}
*cc = *cfg
default:
return fmt.Errorf("failed to get the controller config type")
}
return err
}

func (cm *ControllersManager) Child(ctx context.Context, pod *corev1.Pod, containerName, childName string, controllerConfig interface{}) error {
// TODO
return nil
}
100 changes: 100 additions & 0 deletions test/e2e/performanceprofile/functests/utils/cgroup/v2/v2.go
View file
@@ -0,0 +1,100 @@
package v2

import (
"context"
"fmt"
"path"
"strings"

corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/cgroup/controller"
"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/cgroup/runtime"
"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/pods"
)

type ControllersManager struct {
client client.Client
k8sClient *kubernetes.Clientset
}

func NewManager(c client.Client, k8sClient *kubernetes.Clientset) *ControllersManager {
return &ControllersManager{client: c, k8sClient: k8sClient}
}

func (cm *ControllersManager) CpuSet(ctx context.Context, pod *corev1.Pod, containerName, childName, runtimeType string) (*controller.CpuSet, error) {
cfg := &controller.CpuSet{}
dirPath := path.Join(controller.CgroupMountPoint, childName)
cmd := []string{
"/bin/cat",
dirPath + "/cpuset.cpus",
dirPath + "/cpuset.cpus.exclusive",
dirPath + "/cpuset.cpus.effective",
dirPath + "/cpuset.cpus.partition",
dirPath + "/cpuset.mems",
}
b, err := pods.ExecCommandOnPod(cm.k8sClient, pod, containerName, cmd)
if err != nil {
return nil, fmt.Errorf("failed to retrieve cgroup config for pod. pod=%q, container=%q; %w", client.ObjectKeyFromObject(pod).String(), containerName, err)
}
output := strings.Split(string(b), "\r\n")
cfg.Cpus = output[0]
cfg.Exclusive = output[1]
cfg.Effective = output[2]
cfg.Partition = output[3]
cfg.Mems = output[4]
return cfg, nil
}

func (cm *ControllersManager) Cpu(ctx context.Context, pod *corev1.Pod, containerName, childName, runtimeType string) (*controller.Cpu, error) {
cfg := &controller.Cpu{}
dirPath := path.Join(controller.CgroupMountPoint, childName)
cmd := []string{
"/bin/cat",
dirPath + "/cpu.max",
}
b, err := pods.ExecCommandOnPod(cm.k8sClient, pod, containerName, cmd)
if err != nil {
return nil, fmt.Errorf("failed to retrieve cgroup config for pod. pod=%q, container=%q; %w", client.ObjectKeyFromObject(pod).String(), containerName, err)
}
output := strings.Split(string(b), "\r\n")
quotaAndPeriod := strings.Split(output[0], " ")
cfg.Quota = quotaAndPeriod[0]
cfg.Period = quotaAndPeriod[1]
return cfg, nil
}

func (cm *ControllersManager) Pod(ctx context.Context, pod *corev1.Pod, controllerConfig interface{}) error {
// TODO
return nil
}

func (cm *ControllersManager) Container(ctx context.Context, pod *corev1.Pod, containerName string, controllerConfig interface{}) error {
return cm.Child(ctx, pod, containerName, "", controllerConfig)
}

func (cm *ControllersManager) Child(ctx context.Context, pod *corev1.Pod, containerName, childName string, controllerConfig interface{}) error {
runtimeType, err := runtime.GetContainerRuntimeTypeFor(ctx, cm.client, pod)
if err != nil {
return err
}
switch cc := controllerConfig.(type) {
case *controller.CpuSet:
cfg, err := cm.CpuSet(ctx, pod, containerName, childName, runtimeType)
if err != nil {
return err
}
*cc = *cfg
case *controller.Cpu:
cfg, err := cm.Cpu(ctx, pod, containerName, childName, runtimeType)
if err != nil {
return err
}
*cc = *cfg
default:
return fmt.Errorf("failed to get the controller config type")
}
return nil
}

0 comments on commit 58b3575

Please sign in to comment.