Merge pull request #176 from openshift-cherrypick-robot/cherry-pick-1…

…75-to-release-4.6

[release-4.6] Bug 1896381: Add a weak dependency on kmod to tuned.

assets/tuned/patches/090-kmod-weak-dependency.diff

@@ -0,0 +1,16 @@
+Fixes issue #304 with modprobe being missed by the [modules] plugin.
+
+See: https://github.com/redhat-performance/tuned/pull/305
+
+diff --git a/tuned.spec b/tuned.spec
+index f28e7b58..2a3fc6bf 100644
+--- a/tuned.spec
++++ b/tuned.spec
+@@ -88,6 +88,7 @@ Requires: util-linux, dbus, polkit
+ Recommends: hdparm
+ Recommends: kernel-tools
+ Recommends: python-dmidecode
++Recommends: kmod
+ %endif
+ %if 0%{?rhel} > 7
+ Requires: python3-syspurpose

test/e2e/basic/modules.go

@@ -0,0 +1,79 @@
+package e2e
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+
+	coreapi "k8s.io/api/core/v1"
+
+	ntoconfig "github.com/openshift/cluster-node-tuning-operator/pkg/config"
+	util "github.com/openshift/cluster-node-tuning-operator/test/e2e/util"
+)
+
+// Test loading a kernel module by applying a custom profile via node labelling.
+var _ = ginkgo.Describe("[basic][modules] Node Tuning Operator load kernel module", func() {
+	const (
+		profileModules   = "../testing_manifests/tuned_modules_load.yaml"
+		nodeLabelModules = "tuned.openshift.io/module-load"
+		procModules      = "/proc/modules"
+		moduleName       = "joydev"
+	)
+
+	ginkgo.Context("module loading", func() {
+		var (
+			node *coreapi.Node
+		)
+
+		// Cleanup code to roll back cluster changes done by this test even if it fails in the middle of ginkgo.It()
+		ginkgo.AfterEach(func() {
+			ginkgo.By("cluster changes rollback")
+			if node != nil {
+				util.ExecAndLogCommand("oc", "label", "node", "--overwrite", node.Name, nodeLabelModules+"-")
+			}
+			util.ExecAndLogCommand("oc", "delete", "-n", ntoconfig.OperatorNamespace(), "-f", profileModules)
+		})
+
+		ginkgo.It(fmt.Sprintf("modules: %s loaded", moduleName), func() {
+			cmdGrepModule := []string{"grep", fmt.Sprintf("^%s ", moduleName), procModules}
+			ginkgo.By("getting a list of worker nodes")
+			nodes, err := util.GetNodesByRole(cs, "worker")
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+			gomega.Expect(len(nodes)).NotTo(gomega.BeZero(), "number of worker nodes is 0")
+
+			node = &nodes[0]
+			ginkgo.By(fmt.Sprintf("getting a tuned Pod running on node %s", node.Name))
+			pod, err := util.GetTunedForNode(cs, node)
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			ginkgo.By(fmt.Sprintf("labelling node %s with label %s", node.Name, nodeLabelModules))
+			_, _, err = util.ExecAndLogCommand("oc", "label", "node", "--overwrite", node.Name, nodeLabelModules+"=")
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			ginkgo.By(fmt.Sprintf("trying to remove the %s module if loaded", moduleName))
+			_, err = util.ExecCmdInPod(pod, "rmmod", moduleName)
+
+			ginkgo.By(fmt.Sprintf("ensuring the %s module is not loaded", moduleName))
+			_, err = util.ExecCmdInPod(pod, cmdGrepModule...)
+			gomega.Expect(err).To(gomega.HaveOccurred())
+
+			ginkgo.By(fmt.Sprintf("creating profile %s", profileModules))
+			_, _, err = util.ExecAndLogCommand("oc", "create", "-n", ntoconfig.OperatorNamespace(), "-f", profileModules)
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			ginkgo.By(fmt.Sprintf("ensuring the %s module is loaded", moduleName))
+			_, err = util.PollExecCmdInPod(5*time.Second, 5*time.Minute, pod, cmdGrepModule...)
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			ginkgo.By(fmt.Sprintf("deleting profile %s", profileModules))
+			_, _, err = util.ExecAndLogCommand("oc", "delete", "-n", ntoconfig.OperatorNamespace(), "-f", profileModules)
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			ginkgo.By(fmt.Sprintf("removing label %s from node %s", nodeLabelModules, node.Name))
+			_, _, err = util.ExecAndLogCommand("oc", "label", "node", "--overwrite", node.Name, nodeLabelModules+"-")
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		})
+	})
+})

test/e2e/reboots/stalld.go

@@ -18,7 +18,6 @@ var _ = ginkgo.Describe("[reboots][stalld] Node Tuning Operator installing syste
 		profileRealtime   = "../testing_manifests/stalld.yaml"
 		mcpRealtime       = "../../../examples/realtime-mcp.yaml"
 		nodeLabelRealtime = "node-role.kubernetes.io/worker-rt"
-		procCmdline       = "/proc/cmdline"
 	)
 
 	ginkgo.Context("stalld", func() {

test/e2e/testing_manifests/tuned_modules_load.yaml

@@ -0,0 +1,19 @@
+apiVersion: tuned.openshift.io/v1
+kind: Tuned
+metadata:
+  name: openshift-module-load
+  namespace: openshift-cluster-node-tuning-operator
+spec:
+  profile:
+  - data: |
+      [main]
+      summary=An OpenShift profile to load 'joydev' module
+      include=openshift-node
+      [modules]
+      joydev=+r
+    name: openshift-module-load
+  recommend:
+  - match:
+    - label: tuned.openshift.io/module-load
+    priority: 20
+    profile: openshift-module-load

test/e2e/util/util.go

@@ -25,7 +25,7 @@ func Logf(format string, args ...interface{}) {
 	fmt.Fprintf(ginkgo.GinkgoWriter, format+"\n", args...)
 }
 
-func ExecAndLogCommand(name string, arg ...string) (bytes.Buffer, bytes.Buffer, error) {
+func execCommand(log bool, name string, arg ...string) (bytes.Buffer, bytes.Buffer, error) {
 	var (
 		stdout bytes.Buffer
 		stderr bytes.Buffer
@@ -35,12 +35,18 @@ func ExecAndLogCommand(name string, arg ...string) (bytes.Buffer, bytes.Buffer,
 	cmd.Stdout = &stdout
 	cmd.Stderr = &stderr
 	err := cmd.Run()
-	Logf("run command '%s %v':\n  out=%s\n  err=%s\n  ret=%v",
-		name, arg, strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()), err)
+	if log {
+		Logf("run command '%s %v':\n  out=%s\n  err=%s\n  ret=%v",
+			name, arg, strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()), err)
+	}
 
 	return stdout, stderr, err
 }
 
+func ExecAndLogCommand(name string, arg ...string) (bytes.Buffer, bytes.Buffer, error) {
+	return execCommand(true, name, arg...)
+}
+
 // GetNodesByRole returns a list of nodes that match a given role.
 func GetNodesByRole(cs *framework.ClientSet, role string) ([]corev1.Node, error) {
 	listOptions := metav1.ListOptions{
@@ -77,22 +83,21 @@ func GetTunedForNode(cs *framework.ClientSet, node *corev1.Node) (*corev1.Pod, e
 // GetSysctl returns a sysctl value for sysctlVar from inside a (tuned) pod.
 func GetSysctl(sysctlVar string, pod *corev1.Pod) (string, error) {
 	var (
-		val, explain string
-		err          error
+		val          string
+		err, explain error
 	)
 	err = wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) {
-		var out []byte
-		out, err = exec.Command("oc", "rsh", "-n", ntoconfig.OperatorNamespace(), pod.Name,
-			"sysctl", "-n", sysctlVar).CombinedOutput()
+		var out string
+		out, err = ExecCmdInPod(pod, "sysctl", "-n", sysctlVar)
 		if err != nil {
-			explain = fmt.Sprintf("out=%s; err=%s", string(out), err.Error())
+			explain = fmt.Errorf("out=%s; err=%s", out, err.Error())
 			return false, nil
 		}
-		val = strings.TrimSpace(string(out))
+		val = strings.TrimSpace(out)
 		return true, nil
 	})
 	if err != nil {
-		return "", fmt.Errorf("failed to retrieve sysctl value %s in pod %s: %s", sysctlVar, pod.Name, explain)
+		return "", fmt.Errorf("failed to retrieve sysctl value %s in pod %s: %v", sysctlVar, pod.Name, explain)
 	}
 
 	return val, nil
@@ -104,40 +109,56 @@ func ExecCmdInPod(pod *corev1.Pod, args ...string) (string, error) {
 	cmd := []string{"rsh", "-n", ntoconfig.OperatorNamespace(), pod.Name}
 	cmd = append(cmd, args...)
 
-	b, err := exec.Command(entryPoint, cmd...).CombinedOutput()
+	stdout, stderr, err := execCommand(false, entryPoint, cmd...)
+
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("failed to run %q in Pod %s:\n  out=%s\n  err=%s\n  ret=%v", args, pod.Name, stdout.String(), stderr.String(), err.Error())
 	}
-	return string(b), nil
+
+	return stdout.String(), nil
 }
 
-// GetFileInPod returns content for file from inside a (tuned) pod.
-func GetFileInPod(pod *corev1.Pod, file string) (string, error) {
-	var val, explain string
-	err := wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) {
-		out, err := ExecCmdInPod(pod, "cat", file)
+// PollExecCmdInPod executes a command with arguments in a Pod
+// until the command succeeds or times out.
+func PollExecCmdInPod(interval, duration time.Duration, pod *corev1.Pod, args ...string) (string, error) {
+	var (
+		val     string
+		explain error
+	)
+	err := wait.PollImmediate(interval, duration, func() (bool, error) {
+		out, err := ExecCmdInPod(pod, args...)
 		if err != nil {
-			explain = fmt.Sprintf("failed to cat %s", file)
+			explain = fmt.Errorf("out=%s; err=%s", out, err.Error())
 			return false, nil
 		}
-		val = strings.TrimSpace(out)
+		val = out
 		return true, nil
 	})
 	if err != nil {
-		return "", fmt.Errorf("failed to retrieve %s content in pod %s: %s", file, pod.Name, explain)
+		return "", fmt.Errorf("failed to run %q in Pod %s: %v", args, pod.Name, explain)
 	}
 
 	return val, nil
 }
 
+// GetFileInPod returns content for file from inside a (tuned) pod.
+func GetFileInPod(pod *corev1.Pod, file string) (string, error) {
+	return PollExecCmdInPod(5*time.Second, 5*time.Minute, pod, "cat", file)
+}
+
 // EnsureSysctl makes sure a sysctl value for sysctlVar from inside a (tuned) pod
 // is equal to valExp.  Returns an error otherwise.
 func EnsureSysctl(pod *corev1.Pod, sysctlVar string, valExp string) error {
-	var val string
+	var (
+		val     string
+		explain error
+	)
 	wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) {
 		var err error
 		val, err = GetSysctl(sysctlVar, pod)
+
 		if err != nil {
+			explain = err
 			return false, nil
 		}
 
@@ -147,7 +168,7 @@ func EnsureSysctl(pod *corev1.Pod, sysctlVar string, valExp string) error {
 		return true, nil
 	})
 	if val != valExp {
-		return fmt.Errorf("sysctl %s=%s on %s, expected %s.", sysctlVar, val, pod.Name, valExp)
+		return fmt.Errorf("sysctl %s=%s on %s, expected %s: %v", sysctlVar, val, pod.Name, valExp, explain)
 	}
 
 	return nil