"OCPBUGS-63: e2e: performance: configure isolated cpus in the profile"

test/e2e/performanceprofile/functests/5_latency_testing/5_latency_testing_suite_test.go

@@ -3,6 +3,7 @@ package __latency_testing_test
 import (
 	"context"
 	"encoding/json"
+	"reflect"
 	"testing"
 	"time"
 
@@ -12,12 +13,19 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
 
+	performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2"
+	testutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils"
 	testclient "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/client"
 	"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/images"
 	"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/junit"
 	testlog "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/log"
 	"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/namespaces"
+	"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/nodes"
+	"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/profiles"
+	"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/profilesupdate"
+
 	ginkgo_reporters "kubevirt.io/qe-tools/pkg/ginkgo-reporters"
 )
 
@@ -26,32 +34,71 @@ var prePullNamespace = &corev1.Namespace{
 		Name: "testing-prepull",
 	},
 }
+var profile *performancev2.PerformanceProfile
 
-var _ = AfterSuite(func() {
-	prePullNamespaceName := prePullNamespace.Name
-	err := testclient.Client.Delete(context.TODO(), prePullNamespace)
-	testlog.Infof("deleted namespace %q err=%v", prePullNamespace.Name, err)
+var _ = BeforeSuite(func() {
+	Expect(testclient.ClientsEnabled).To(BeTrue())
+
+	// update PP isolated CPUs. the new cpu set for isolated should have an even number of CPUs to avoid failing the pod on SMTAlignment error,
+	// and should be greater than what is requested by the test cases in the suite so the test runs properly
+	var err error
+	profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels)
+	Expect(err).ToNot(HaveOccurred())
+	workerNodes, err := nodes.GetByLabels(testutils.NodeSelectorLabels)
 	Expect(err).ToNot(HaveOccurred())
-	err = namespaces.WaitForDeletion(prePullNamespaceName, 5*time.Minute)
-})
 
-func Test5LatencyTesting(t *testing.T) {
-	RegisterFailHandler(Fail)
+	initialIsolated := profile.Spec.CPU.Isolated
+	initialReserved := profile.Spec.CPU.Reserved
+	//updated both sets to ensure there is no overlap
+	latencyIsolatedSet := performancev2.CPUSet("1-9")
+	latencyReservedSet := performancev2.CPUSet("0")
 
-	if !testclient.ClientsEnabled {
-		t.Fatalf("client not enabled")
+	totalCpus := cpuset.MustParse(string(latencyIsolatedSet)).Size() + cpuset.MustParse(string(latencyReservedSet)).Size()
+	nodesWithSufficientCpu := nodes.GetByCpuAllocatable(workerNodes, totalCpus)
+	//before applying the changes verify that there are compute nodes with sufficient cpus
+	if len(nodesWithSufficientCpu) != 0 {
+		if *initialIsolated != latencyIsolatedSet || *initialReserved != latencyReservedSet {
+			testlog.Info("Update the isolated and reserved cpus sets of the profile")
+			err = profilesupdate.UpdateIsolatedReservedCpus(latencyIsolatedSet, latencyReservedSet)
+			if err != nil {
+				testlog.Error("could not update the profile with the desired CPUs sets")
+			}
+		}
 	}
 
 	if err := createNamespace(); err != nil {
-		t.Fatalf("cannot create the namespace: %v", err)
+		testlog.Errorf("cannot create the namespace: %v", err)
 	}
 
 	ds, err := images.PrePull(testclient.Client, images.Test(), prePullNamespace.Name, "cnf-tests")
 	if err != nil {
 		data, _ := json.Marshal(ds) // we can safely skip errors
 		testlog.Infof("DaemonSet %s/%s image=%q status:\n%s", ds.Namespace, ds.Name, images.Test(), string(data))
-		t.Fatalf("cannot prepull image %q: %v", images.Test(), err)
+		testlog.Errorf("cannot prepull image %q: %v", images.Test(), err)
 	}
+})
+
+var _ = AfterSuite(func() {
+	prePullNamespaceName := prePullNamespace.Name
+	err := testclient.Client.Delete(context.TODO(), prePullNamespace)
+	if err != nil {
+		testlog.Errorf("namespace %q could not be deleted err=%v", prePullNamespace.Name, err)
+	}
+	namespaces.WaitForDeletion(prePullNamespaceName, 5*time.Minute)
+
+	currentProfile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels)
+	Expect(err).ToNot(HaveOccurred())
+	if reflect.DeepEqual(currentProfile.Spec, profile.Spec) != true {
+		testlog.Info("Restore initial performance profile")
+		err = profilesupdate.ApplyProfile(profile)
+		if err != nil {
+			testlog.Errorf("could not restore the initial profile: %v", err)
+		}
+	}
+})
+
+func Test5LatencyTesting(t *testing.T) {
+	RegisterFailHandler(Fail)
 
 	rr := []Reporter{}
 	if ginkgo_reporters.Polarion.Run {

test/e2e/performanceprofile/functests/utils/profilesupdate/profile_update.go

@@ -0,0 +1,79 @@
+package profilesupdate
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+
+	mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
+
+	performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2"
+	"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components"
+	profilecontroller "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/profile"
+	testutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils"
+	testclient "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/client"
+	testlog "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/log"
+	"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/mcps"
+	"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/profiles"
+)
+
+//UpdateIsolatedReservedCpus Updates the current performance profile with new sets of isolated and reserved cpus, and returns true if the update was successfull and false otherwise
+func UpdateIsolatedReservedCpus(isolatedSet performancev2.CPUSet, reservedSet performancev2.CPUSet) error {
+	profile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels)
+	if err != nil {
+		return fmt.Errorf("could not get the performance profile: %v", err)
+	}
+	updatedProfile := profile.DeepCopy()
+	updatedProfile.Spec.CPU = &performancev2.CPU{
+		Isolated: &isolatedSet,
+		Reserved: &reservedSet,
+	}
+
+	err = ApplyProfile(updatedProfile)
+	if err == nil {
+		testlog.Infof("successfully updated performance profile %q with new isolated cpus set: %q and new reserved cpus set: %q", profile.Name, updatedProfile.Spec.CPU.Isolated, updatedProfile.Spec.CPU.Reserved)
+	}
+	return err
+}
+
+//ApplyProfile applies the new profile and returns true if the changes were applied indeed and false otherwise
+func ApplyProfile(profile *performancev2.PerformanceProfile) error {
+	testlog.Info("Getting MCP for profile")
+	mcpLabel := profilecontroller.GetMachineConfigLabel(profile)
+	key, value := components.GetFirstKeyAndValue(mcpLabel)
+	mcpsByLabel, err := mcps.GetByLabel(key, value)
+	if err != nil {
+		return fmt.Errorf("Failed getting MCP by label key %v value %v: %v", key, value, err)
+	}
+	if len(mcpsByLabel) != 1 {
+		return fmt.Errorf("Unexpected number of MCPs found: %v", len(mcpsByLabel))
+	}
+	performanceMCP := &mcpsByLabel[0]
+	testlog.Info("Verifying that mcp is ready for update")
+	mcps.WaitForCondition(performanceMCP.Name, mcv1.MachineConfigPoolUpdated, corev1.ConditionTrue)
+
+	testlog.Info("Applying changes in performance profile and waiting until mcp will start updating")
+	profiles.UpdateWithRetry(profile)
+	mcps.WaitForCondition(performanceMCP.Name, mcv1.MachineConfigPoolUpdating, corev1.ConditionTrue)
+
+	testlog.Info("Waiting when mcp finishes updates")
+	mcps.WaitForCondition(performanceMCP.Name, mcv1.MachineConfigPoolUpdated, corev1.ConditionTrue)
+
+	//check if the values were indeed updated
+	profilekey := types.NamespacedName{
+		Name:      profile.Name,
+		Namespace: profile.Namespace,
+	}
+	updatedProfile := &performancev2.PerformanceProfile{}
+	if err = testclient.Client.Get(context.TODO(), profilekey, updatedProfile); err != nil {
+		return fmt.Errorf("could not fetch the profile: %v", err)
+	}
+
+	if reflect.DeepEqual(updatedProfile.Spec, profile.Spec) != true {
+		return fmt.Errorf("the profile %q was not updated as expected", updatedProfile.Name)
+	}
+	return nil
+}