CNF-9173: e2e: cgroups: introduce cgroup package #906
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| package cgroup | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| corev1 "k8s.io/api/core/v1" | ||
| "k8s.io/client-go/kubernetes" | ||
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
|
|
||
| apiconfigv1 "github.com/openshift/api/config/v1" | ||
| cgroupv1 "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/cgroup/v1" | ||
| cgroupv2 "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/cgroup/v2" | ||
| ) | ||
|
|
||
| type ControllersGetter interface { | ||
| // Pod is for getting controller config at the pod level | ||
| Pod(ctx context.Context, pod *corev1.Pod, controllerConfig interface{}) error | ||
| // Container is for getting controller config at the container level | ||
| Container(ctx context.Context, pod *corev1.Pod, containerName string, controllerConfig interface{}) error | ||
| // Child is for getting controller config at the container's child level | ||
| Child(ctx context.Context, pod *corev1.Pod, containerName, childName string, controllerConfig interface{}) error | ||
| } | ||
|
|
||
| func IsVersion2(ctx context.Context, c client.Client) (bool, error) { | ||
| nodecfg := &apiconfigv1.Node{} | ||
| key := client.ObjectKey{ | ||
| Name: "cluster", | ||
| } | ||
| err := c.Get(ctx, key, nodecfg) | ||
| if err != nil { | ||
| return false, fmt.Errorf("failed to get configs.node object. name=%q; %w", key.Name, err) | ||
| } | ||
| if nodecfg.Spec.CgroupMode == apiconfigv1.CgroupModeV1 { | ||
| return false, nil | ||
| } | ||
| // in case `apiconfigv1.CgroupMode` not set, it's returned true since | ||
| // the platform's default is cgroupV2 | ||
| return true, nil | ||
| } | ||
|
|
||
| // BuildGetter return a cgroup information getter that complies with | ||
| // the cgroup version that is currently used by the nodes on the cluster | ||
| func BuildGetter(ctx context.Context, c client.Client, k8sClient *kubernetes.Clientset) (ControllersGetter, error) { | ||
| isV2, versionErr := IsVersion2(ctx, c) | ||
| if versionErr != nil { | ||
| return nil, versionErr | ||
| } | ||
| if isV2 { | ||
| return cgroupv2.NewManager(c, k8sClient), nil | ||
| } else { | ||
| return cgroupv1.NewManager(c, k8sClient), nil | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| package controller | ||
|
|
||
| const CgroupMountPoint = "/sys/fs/cgroup" | ||
|
|
||
| type CpuSet struct { | ||
| Cpus string | ||
| Mems string | ||
| Effective string | ||
| // Partition only applicable for cgroupv2 | ||
| Partition string | ||
| Exclusive string | ||
| // SchedLoadBalance true if enabled, only applicable for cgroupv1 | ||
| SchedLoadBalance string | ||
| } | ||
|
|
||
| type Cpu struct { | ||
| Quota string | ||
| Period string | ||
| cpuStat map[string]string | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| package runtime | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/nodes" | ||
| corev1 "k8s.io/api/core/v1" | ||
| "path/filepath" | ||
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
| ) | ||
|
|
||
| const ( | ||
| Crun = "crun" | ||
| Runc = "runc" | ||
| CRIORuntimeConfigFile = "/etc/crio/crio.conf.d/99-runtimes.conf" | ||
| ) | ||
|
|
||
| // GetContainerRuntimeTypeFor return the container runtime type that is being used | ||
| // in the node where the given pod is running | ||
| func GetContainerRuntimeTypeFor(ctx context.Context, c client.Client, pod *corev1.Pod) (string, error) { | ||
| node := &corev1.Node{} | ||
| if err := c.Get(ctx, client.ObjectKey{Name: pod.Spec.NodeName}, node); err != nil { | ||
| return "", err | ||
| } | ||
| cmd := []string{ | ||
| "chroot", | ||
| "/rootfs", | ||
| "/bin/bash", | ||
| "-c", | ||
| fmt.Sprintf("/bin/awk -F '\"' '/runtime_path.*/ { print $2 }' %s", CRIORuntimeConfigFile), | ||
| } | ||
| out, err := nodes.ExecCommandOnNode(ctx, cmd, node) | ||
| if err != nil { | ||
| return "", fmt.Errorf("failed to execute command on node; cmd=%q node=%q", cmd, node.Name) | ||
| } | ||
| return filepath.Base(out), nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,117 @@ | ||
| package v1 | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "path" | ||
| "strings" | ||
|
|
||
| corev1 "k8s.io/api/core/v1" | ||
| "k8s.io/client-go/kubernetes" | ||
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
|
|
||
| "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/cgroup/controller" | ||
| "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/cgroup/runtime" | ||
| "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/log" | ||
| "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/pods" | ||
| ) | ||
|
|
||
| type ControllersManager struct { | ||
| client client.Client | ||
| k8sClient *kubernetes.Clientset | ||
| } | ||
|
|
||
| func NewManager(c client.Client, k8sClient *kubernetes.Clientset) *ControllersManager { | ||
| return &ControllersManager{client: c, k8sClient: k8sClient} | ||
| } | ||
|
|
||
| func (cm *ControllersManager) CpuSet(ctx context.Context, pod *corev1.Pod, containerName, childName, runtimeType string) (*controller.CpuSet, error) { | ||
| cfg := &controller.CpuSet{} | ||
| dirPath := path.Join(controller.CgroupMountPoint, "cpuset", childName) | ||
| store := map[string]*string{ | ||
| "cpuset.cpus": &cfg.Cpus, | ||
| "cpuset.cpus.exclusive": &cfg.Exclusive, | ||
| "cpuset.cpus.effective": &cfg.Effective, | ||
| "cpuset.sched_load_balance": &cfg.SchedLoadBalance, | ||
| "cpuset.mems": &cfg.Mems, | ||
| } | ||
| err := cm.execAndStore(pod, containerName, dirPath, store) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to retrieve cgroup config for pod. pod=%q, container=%q; %w", client.ObjectKeyFromObject(pod).String(), containerName, err) | ||
| } | ||
Tal-or marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| return cfg, nil | ||
| } | ||
|
|
||
| func (cm *ControllersManager) Cpu(ctx context.Context, pod *corev1.Pod, containerName, childName, runtimeType string) (*controller.Cpu, error) { | ||
| cfg := &controller.Cpu{} | ||
|
There was a problem hiding this comment. We are passing runtimeType but we are not using this variable in Cpu function There was a problem hiding this comment. Yes I need to learn the details, i just passed the argument for now |
||
| dirPath := path.Join(controller.CgroupMountPoint, "cpu", childName) | ||
| cmd := []string{ | ||
| "/bin/cat", | ||
| dirPath + "/cpu.cfs_quota_us", | ||
| dirPath + "/cpu.cfs_period_us", | ||
| } | ||
| b, err := pods.ExecCommandOnPod(cm.k8sClient, pod, containerName, cmd) | ||
| if err != nil { | ||
Tal-or marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| return nil, fmt.Errorf("failed to retrieve cgroup config for pod. pod=%q, container=%q; %w", client.ObjectKeyFromObject(pod).String(), containerName, err) | ||
| } | ||
| output := strings.Split(string(b), "\r\n") | ||
| cfg.Quota = output[0] | ||
| cfg.Period = output[1] | ||
| return cfg, nil | ||
| } | ||
|
|
||
| func (cm *ControllersManager) Pod(ctx context.Context, pod *corev1.Pod, controllerConfig interface{}) error { | ||
| // TODO | ||
| return nil | ||
| } | ||
|
|
||
| func (cm *ControllersManager) Container(ctx context.Context, pod *corev1.Pod, containerName string, controllerConfig interface{}) error { | ||
| runtimeType, err := runtime.GetContainerRuntimeTypeFor(ctx, cm.client, pod) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| switch cc := controllerConfig.(type) { | ||
| case *controller.CpuSet: | ||
| cfg, err := cm.CpuSet(ctx, pod, containerName, "", runtimeType) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| *cc = *cfg | ||
| case *controller.Cpu: | ||
| cfg, err := cm.Cpu(ctx, pod, containerName, "", runtimeType) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| *cc = *cfg | ||
| default: | ||
| return fmt.Errorf("failed to get the controller config type") | ||
| } | ||
| return err | ||
| } | ||
|
|
||
| func (cm *ControllersManager) Child(ctx context.Context, pod *corev1.Pod, containerName, childName string, controllerConfig interface{}) error { | ||
| // TODO | ||
| return nil | ||
| } | ||
|
|
||
| func (cm *ControllersManager) execAndStore(pod *corev1.Pod, containerName, dirPath string, store map[string]*string) error { | ||
|
There was a problem hiding this comment. not a big fan of mutating arguments (and of maps whose values are pointers) but I see why you are doing like this and I don't have compelling suggestions There was a problem hiding this comment. Yea, it was the only way to generalize the calls per each controller file |
||
| for k, v := range store { | ||
| fullPath := dirPath + "/" + k | ||
| cmd := []string{ | ||
| "/bin/cat", | ||
| fullPath, | ||
| } | ||
| b, err := pods.ExecCommandOnPod(cm.k8sClient, pod, containerName, cmd) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if len(b) == 0 { | ||
| log.Warningf("empty value in cgroupv1 controller file; pod=%q,container=%q,file=%q", pod.Name, containerName, fullPath) | ||
| *v = "" | ||
| continue | ||
| } | ||
| output := strings.Trim(string(b), "\r\n") | ||
| *v = output | ||
| } | ||
| return nil | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this works. An alternative option could have been to add a variant function which accepts a container name and reimplement the existing ExecCommandOnPod (with the current signature) on top of it. But we can keep this approach.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes this is an option as well.