From 9d81f7a6d8c4023c1524c493373605cd61204a6f Mon Sep 17 00:00:00 2001 From: Marcel Apfelbaum Date: Mon, 21 Sep 2020 13:47:29 +0300 Subject: [PATCH 1/3] deploy: Force PAO installation on master nodes PAO is meant to run on "control plane". Use podAffinity to schedule PAO only on master nodes. (https://docs.openshift.com/container-platform/4.5/nodes/scheduling/nodes-scheduler-pod-affinity.html) Use taints tolerations to allow PAO scheduling on the master nodes. (https://docs.openshift.com/container-platform/4.5/nodes/scheduling/nodes-scheduler-taints-tolerations.html) Update the manifests according to the above. Signed-off-by: Marcel Apfelbaum --- ...ce-addon-operator.v4.6.0.clusterserviceversion.yaml | 10 ++++++++++ deploy/operator.yaml | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/deploy/olm-catalog/performance-addon-operator/4.6.0/performance-addon-operator.v4.6.0.clusterserviceversion.yaml b/deploy/olm-catalog/performance-addon-operator/4.6.0/performance-addon-operator.v4.6.0.clusterserviceversion.yaml index c0001735a..8599c04af 100644 --- a/deploy/olm-catalog/performance-addon-operator/4.6.0/performance-addon-operator.v4.6.0.clusterserviceversion.yaml +++ b/deploy/olm-catalog/performance-addon-operator/4.6.0/performance-addon-operator.v4.6.0.clusterserviceversion.yaml @@ -118,6 +118,13 @@ spec: labels: name: performance-operator spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists containers: - command: - performance-operator @@ -137,6 +144,9 @@ spec: name: performance-operator resources: {} serviceAccountName: performance-operator + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master permissions: - rules: - apiGroups: diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 0e8260303..8c50317ef 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -13,6 +13,16 @@ spec: name: performance-operator spec: serviceAccountName: performance-operator + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule containers: - name: performance-operator # Replace this with the built image name From b0550b785415ef4ddd7b6150661bdfa8c1768562 Mon Sep 17 00:00:00 2001 From: Marcel Apfelbaum Date: Mon, 21 Sep 2020 13:56:08 +0300 Subject: [PATCH 2/3] functests/config: Add a test verifying PAO runs in master nodes Straight forward test that checks the node PAO is running on is a master node. Signed-off-by: Marcel Apfelbaum --- functests/0_config/config.go | 23 +++++++++++++++++++++++ functests/utils/consts.go | 2 ++ functests/utils/pods/pods.go | 22 ++++++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/functests/0_config/config.go b/functests/0_config/config.go index 9876524b8..66868fb3a 100644 --- a/functests/0_config/config.go +++ b/functests/0_config/config.go @@ -5,10 +5,12 @@ import ( "fmt" "io/ioutil" "os" + "strings" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -27,6 +29,8 @@ import ( testclient "github.com/openshift-kni/performance-addon-operators/functests/utils/client" "github.com/openshift-kni/performance-addon-operators/functests/utils/discovery" "github.com/openshift-kni/performance-addon-operators/functests/utils/mcps" + "github.com/openshift-kni/performance-addon-operators/functests/utils/nodes" + "github.com/openshift-kni/performance-addon-operators/functests/utils/pods" "github.com/openshift-kni/performance-addon-operators/functests/utils/profiles" "github.com/openshift-kni/performance-addon-operators/pkg/apis" performancev1 "github.com/openshift-kni/performance-addon-operators/pkg/apis/performance/v1" @@ -37,6 +41,25 @@ import ( var _ = Describe("[performance][config] Performance configuration", func() { + It("Should run performance profile pod on a master node", func() { + pod, err := pods.GetPerformanceOperatorPod() + Expect(err).ToNot(HaveOccurred(), "Failed to find the Performance Addon Operator pod") + + Expect(strings.HasPrefix(pod.Name, "performance-operator")).To(BeTrue(), + "Performance Addon Operator pod name should start with performance-operator prefix") + + masterNodes, err := nodes.GetByRole(testutils.RoleMaster) + Expect(err).ToNot(HaveOccurred(), "Failed to query the master nodes") + for _, node := range masterNodes { + if node.Name == pod.Spec.NodeName { + return + } + } + + // Fail + Expect(true).To(Reject(), "Performance Addon Operator is not running in a master node") + }) + It("Should successfully deploy the performance profile", func() { performanceProfile := testProfile() diff --git a/functests/utils/consts.go b/functests/utils/consts.go index 7ae935051..73fedf865 100644 --- a/functests/utils/consts.go +++ b/functests/utils/consts.go @@ -67,6 +67,8 @@ func init() { const ( // RoleWorker contains the worker role RoleWorker = "worker" + // RoleMaster contains the master role + RoleMaster = "master" ) const ( diff --git a/functests/utils/pods/pods.go b/functests/utils/pods/pods.go index 98bf2d0fc..e69d86437 100644 --- a/functests/utils/pods/pods.go +++ b/functests/utils/pods/pods.go @@ -11,9 +11,11 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" testutils "github.com/openshift-kni/performance-addon-operators/functests/utils" testclient "github.com/openshift-kni/performance-addon-operators/functests/utils/client" @@ -143,3 +145,23 @@ func GetContainerIDByName(pod *corev1.Pod, containerName string) (string, error) } return "", fmt.Errorf("failed to find the container ID for the container %q under the pod %q", containerName, pod.Name) } + +// GetPerformanceOperatorPod returns the pod running the Performance Profile Operator +func GetPerformanceOperatorPod() (*corev1.Pod, error) { + selector, err := labels.Parse(fmt.Sprintf("%s=%s", "name", "performance-operator")) + if err != nil { + return nil, err + } + + pods := &corev1.PodList{} + + opts := &client.ListOptions{LabelSelector: selector, Namespace: testutils.PerformanceOperatorNamespace} + if err := testclient.Client.List(context.TODO(), pods, opts); err != nil { + return nil, err + } + if len(pods.Items) != 1 { + return nil, fmt.Errorf("incorrect performance operator pods count: %d", len(pods.Items)) + } + + return &pods.Items[0], nil +} From d3769d140b8d398e16dfffe4db2429cb0ae72e74 Mon Sep 17 00:00:00 2001 From: Francesco Romani Date: Fri, 25 Sep 2020 07:32:45 +0200 Subject: [PATCH 3/3] functests: move placement test from 0_config The performance-operator node placement test needs to run after the operator pod was created, hence we move it from the 0_config suite to a later suite. Signed-off-by: Francesco Romani --- functests/0_config/config.go | 23 ----------------------- functests/1_performance/performance.go | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/functests/0_config/config.go b/functests/0_config/config.go index 66868fb3a..9876524b8 100644 --- a/functests/0_config/config.go +++ b/functests/0_config/config.go @@ -5,12 +5,10 @@ import ( "fmt" "io/ioutil" "os" - "strings" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -29,8 +27,6 @@ import ( testclient "github.com/openshift-kni/performance-addon-operators/functests/utils/client" "github.com/openshift-kni/performance-addon-operators/functests/utils/discovery" "github.com/openshift-kni/performance-addon-operators/functests/utils/mcps" - "github.com/openshift-kni/performance-addon-operators/functests/utils/nodes" - "github.com/openshift-kni/performance-addon-operators/functests/utils/pods" "github.com/openshift-kni/performance-addon-operators/functests/utils/profiles" "github.com/openshift-kni/performance-addon-operators/pkg/apis" performancev1 "github.com/openshift-kni/performance-addon-operators/pkg/apis/performance/v1" @@ -41,25 +37,6 @@ import ( var _ = Describe("[performance][config] Performance configuration", func() { - It("Should run performance profile pod on a master node", func() { - pod, err := pods.GetPerformanceOperatorPod() - Expect(err).ToNot(HaveOccurred(), "Failed to find the Performance Addon Operator pod") - - Expect(strings.HasPrefix(pod.Name, "performance-operator")).To(BeTrue(), - "Performance Addon Operator pod name should start with performance-operator prefix") - - masterNodes, err := nodes.GetByRole(testutils.RoleMaster) - Expect(err).ToNot(HaveOccurred(), "Failed to query the master nodes") - for _, node := range masterNodes { - if node.Name == pod.Spec.NodeName { - return - } - } - - // Fail - Expect(true).To(Reject(), "Performance Addon Operator is not running in a master node") - }) - It("Should successfully deploy the performance profile", func() { performanceProfile := testProfile() diff --git a/functests/1_performance/performance.go b/functests/1_performance/performance.go index 7720eebab..bfc1b3716 100644 --- a/functests/1_performance/performance.go +++ b/functests/1_performance/performance.go @@ -10,6 +10,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" corev1 "k8s.io/api/core/v1" "k8s.io/api/node/v1beta1" @@ -61,6 +62,29 @@ var _ = Describe("[rfe_id:27368][performance]", func() { Expect(err).ToNot(HaveOccurred()) }) + // self-tests; these are only vaguely related to performance becase these are enablement conditions, not actual settings. + // For example, running on control plane means we leave more resources for the workload. + Context("Performance Operator", func() { + It("Should run on the control plane nodes", func() { + pod, err := pods.GetPerformanceOperatorPod() + Expect(err).ToNot(HaveOccurred(), "Failed to find the Performance Addon Operator pod") + + Expect(strings.HasPrefix(pod.Name, "performance-operator")).To(BeTrue(), + "Performance Addon Operator pod name should start with performance-operator prefix") + + masterNodes, err := nodes.GetByRole(testutils.RoleMaster) + Expect(err).ToNot(HaveOccurred(), "Failed to query the master nodes") + for _, node := range masterNodes { + if node.Name == pod.Spec.NodeName { + return + } + } + + // Fail + Expect(true).To(Reject(), "Performance Addon Operator is not running in a master node") + }) + }) + Context("Tuned CRs generated from profile", func() { It("[test_id:31748] Should have the expected name for tuned from the profile owner object", func() { tunedExpectedName := components.GetComponentName(testutils.PerformanceProfileName, components.ProfileNamePerformance)