From cc405aa957470eff7b78a6959d78f7b181363280 Mon Sep 17 00:00:00 2001 From: Xu Liu Date: Thu, 5 May 2022 08:40:21 +0000 Subject: [PATCH] Fix tolerations for Kubernetes >= 1.24 The taints for control-plane Nodes are changed for cluster version >= 1.24. Add a new toleration for Pods running on control-plane Nodes to make sure they can be scheduled. Signed-off-by: Xu Liu --- build/yamls/antrea-aks.yml | 2 ++ build/yamls/antrea-eks.yml | 2 ++ build/yamls/antrea-gke.yml | 2 ++ build/yamls/antrea-ipsec.yml | 2 ++ build/yamls/antrea.yml | 2 ++ build/yamls/base/controller.yml | 3 +++ test/e2e/framework.go | 28 +++++++++++++++------------- test/e2e/networkpolicy_test.go | 3 +-- test/e2e/performance_test.go | 4 +--- 9 files changed, 30 insertions(+), 18 deletions(-) diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index eee762e7e45..105cea3117f 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -3194,6 +3194,8 @@ spec: operator: Exists - effect: NoSchedule key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane volumes: - configMap: name: antrea-config-82h2mk24gg diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 53a6ef8c0a3..2fa3ce0415c 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -3194,6 +3194,8 @@ spec: operator: Exists - effect: NoSchedule key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane volumes: - configMap: name: antrea-config-82h2mk24gg diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 78552174cbd..7203ce07922 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -3194,6 +3194,8 @@ spec: operator: Exists - effect: NoSchedule key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane volumes: - configMap: name: antrea-config-c9ck44454h diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 9241703a4f8..5f41328725f 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -3208,6 +3208,8 @@ spec: operator: Exists - effect: NoSchedule key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane volumes: - configMap: name: antrea-config-tmhkc66d6c diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index d5a7d4db29f..3c37a74685e 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -3199,6 +3199,8 @@ spec: operator: Exists - effect: NoSchedule key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane volumes: - configMap: name: antrea-config-hkhbh5gf99 diff --git a/build/yamls/base/controller.yml b/build/yamls/base/controller.yml index 7ce4a28da36..652e2e1fd25 100644 --- a/build/yamls/base/controller.yml +++ b/build/yamls/base/controller.yml @@ -226,6 +226,9 @@ spec: # Allow it to schedule onto master nodes. - key: node-role.kubernetes.io/master effect: NoSchedule + # Control-plane taint for Kubernetes >= 1.24. + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule serviceAccountName: antrea-controller containers: - name: antrea-controller diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 217f11260e3..09c30a1efc0 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -371,12 +371,19 @@ func labelNodeRoleControlPlane() string { return labelNodeRoleControlPlane } -func controlPlaneNoScheduleToleration() corev1.Toleration { +func controlPlaneNoScheduleTolerations() []corev1.Toleration { // the Node taint still uses "master" in K8s v1.20 - return corev1.Toleration{ - Key: "node-role.kubernetes.io/master", - Operator: corev1.TolerationOpExists, - Effect: corev1.TaintEffectNoSchedule, + return []corev1.Toleration{ + { + Key: "node-role.kubernetes.io/master", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "node-role.kubernetes.io/control-plane", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, } } @@ -1042,8 +1049,7 @@ func (data *TestData) CreatePodOnNodeInNamespace(name, ns string, nodeName, ctrN } if nodeName == controlPlaneNodeName() { // tolerate NoSchedule taint if we want Pod to run on control-plane Node - noScheduleToleration := controlPlaneNoScheduleToleration() - podSpec.Tolerations = []corev1.Toleration{noScheduleToleration} + podSpec.Tolerations = controlPlaneNoScheduleTolerations() } pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -2312,9 +2318,7 @@ func (data *TestData) createAgnhostPodOnNodeWithAnnotations(name string, ns stri func (data *TestData) createDaemonSet(name string, ns string, ctrName string, image string, cmd []string, args []string) (*appsv1.DaemonSet, func() error, error) { podSpec := corev1.PodSpec{ - Tolerations: []corev1.Toleration{ - controlPlaneNoScheduleToleration(), - }, + Tolerations: controlPlaneNoScheduleTolerations(), Containers: []corev1.Container{ { Name: ctrName, @@ -2386,9 +2390,7 @@ func (data *TestData) waitForDaemonSetPods(timeout time.Duration, dsName string, func (data *TestData) createStatefulSet(name string, ns string, size int32, ctrName string, image string, cmd []string, args []string, mutateFunc func(*appsv1.StatefulSet)) (*appsv1.StatefulSet, func() error, error) { podSpec := corev1.PodSpec{ - Tolerations: []corev1.Toleration{ - controlPlaneNoScheduleToleration(), - }, + Tolerations: controlPlaneNoScheduleTolerations(), Containers: []corev1.Container{ { Name: ctrName, diff --git a/test/e2e/networkpolicy_test.go b/test/e2e/networkpolicy_test.go index 313ad31b145..c4d3939f418 100644 --- a/test/e2e/networkpolicy_test.go +++ b/test/e2e/networkpolicy_test.go @@ -816,8 +816,7 @@ func testIngressPolicyWithEndPort(t *testing.T, data *TestData) { } if nodeName == controlPlaneNodeName() { // tolerate NoSchedule taint if we want Pod to run on control-plane Node - noScheduleToleration := controlPlaneNoScheduleToleration() - podSpec.Tolerations = []corev1.Toleration{noScheduleToleration} + podSpec.Tolerations = controlPlaneNoScheduleTolerations() } pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/e2e/performance_test.go b/test/e2e/performance_test.go index 02b32f6664c..c8b2c61de32 100644 --- a/test/e2e/performance_test.go +++ b/test/e2e/performance_test.go @@ -50,8 +50,6 @@ var ( customizePolicyRules = flag.Int("perf.http.policy_rules", 0, "Number of CIDRs in the network policy") httpConcurrency = flag.Int("perf.http.concurrency", 1, "Number of multiple requests to make at a time") realizeTimeout = flag.Duration("perf.realize.timeout", 5*time.Minute, "Timeout of the realization of network policies") - // tolerate NoSchedule taint to let the Pod run on control-plane Node - noScheduleToleration = controlPlaneNoScheduleToleration() labelSelector = &metav1.LabelSelector{ MatchLabels: map[string]string{"app": perfTestAppLabel}, } @@ -118,7 +116,7 @@ func createPerfTestPodDefinition(name, containerName, image string) *corev1.Pod "kubernetes.io/hostname": controlPlaneNodeName(), } - podSpec.Tolerations = []corev1.Toleration{noScheduleToleration} + podSpec.Tolerations = controlPlaneNoScheduleTolerations() pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name,