From 7673d42af5e710a5073d5fd80378f6e96d35512a Mon Sep 17 00:00:00 2001 From: Xu Liu Date: Fri, 6 May 2022 21:01:07 +0800 Subject: [PATCH] Fix tolerations for Kubernetes >= 1.24 (#3731) The taints for control-plane Nodes are changed for cluster version >= 1.24. Add a new toleration for Pods running on control-plane Nodes to make sure they can be scheduled. Signed-off-by: Xu Liu --- build/charts/antrea/README.md | 2 +- build/charts/antrea/values.yaml | 3 +++ build/yamls/antrea-aks.yml | 2 ++ build/yamls/antrea-eks.yml | 2 ++ build/yamls/antrea-gke.yml | 2 ++ build/yamls/antrea-ipsec.yml | 2 ++ build/yamls/antrea.yml | 2 ++ test/e2e/framework.go | 28 +++++++++++++++------------- test/e2e/networkpolicy_test.go | 3 +-- test/e2e/performance_test.go | 4 +--- 10 files changed, 31 insertions(+), 19 deletions(-) diff --git a/build/charts/antrea/README.md b/build/charts/antrea/README.md index 9005e7c0b0d..2b14782d05f 100644 --- a/build/charts/antrea/README.md +++ b/build/charts/antrea/README.md @@ -61,7 +61,7 @@ Kubernetes: `>= 1.16.0-0` | controller.podLabels | object | `{}` | Labels to be added to antrea-controller Pod. | | controller.priorityClassName | string | `"system-cluster-critical"` | Prority class to use for the antrea-controller Pod. | | controller.selfSignedCert | bool | `true` | Indicates whether to use auto-generated self-signed TLS certificates. If false, a Secret named "antrea-controller-tls" must be provided with the following keys: ca.crt, tls.crt, tls.key. | -| controller.tolerations | list | `[{"key":"CriticalAddonsOnly","operator":"Exists"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"}]` | Tolerations for the antrea-controller Pod. | +| controller.tolerations | list | `[{"key":"CriticalAddonsOnly","operator":"Exists"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane"}]` | Tolerations for the antrea-controller Pod. | | defaultMTU | int | `0` | Default MTU to use for the host gateway interface and the network interface of each Pod. By default, antrea-agent will discover the MTU of the Node's primary interface and adjust it to accommodate for tunnel encapsulation overhead if applicable. | | egress.exceptCIDRs | list | `[]` | CIDR ranges to which outbound Pod traffic will not be SNAT'd by Egresses. | | enableBridgingMode | bool | `false` | Enable bridging mode of Pod network on Nodes, in which the Node's transport interface is connected to the OVS bridge. | diff --git a/build/charts/antrea/values.yaml b/build/charts/antrea/values.yaml index 8a587f202b3..45a1297ba26 100644 --- a/build/charts/antrea/values.yaml +++ b/build/charts/antrea/values.yaml @@ -203,6 +203,9 @@ controller: # Allow it to schedule onto master nodes. - key: node-role.kubernetes.io/master effect: NoSchedule + # Control-plane taint for Kubernetes >= 1.24. + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule # -- Node selector for the antrea-controller Pod. nodeSelector: kubernetes.io/os: linux diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index c5fb6bfbe7b..19b74ae9fd9 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -3723,6 +3723,8 @@ spec: operator: Exists - effect: NoSchedule key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane serviceAccountName: antrea-controller containers: - name: antrea-controller diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 80487df0538..ce89e9404b4 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -3725,6 +3725,8 @@ spec: operator: Exists - effect: NoSchedule key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane serviceAccountName: antrea-controller containers: - name: antrea-controller diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index a0ed8f18816..37e33fb6fb6 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -3723,6 +3723,8 @@ spec: operator: Exists - effect: NoSchedule key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane serviceAccountName: antrea-controller containers: - name: antrea-controller diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index d029f1a8e2f..639049eb71a 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -3772,6 +3772,8 @@ spec: operator: Exists - effect: NoSchedule key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane serviceAccountName: antrea-controller containers: - name: antrea-controller diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 1ceb4e17493..0890cf97e89 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -3723,6 +3723,8 @@ spec: operator: Exists - effect: NoSchedule key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane serviceAccountName: antrea-controller containers: - name: antrea-controller diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 7b3a80c52b0..9a812110357 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -379,12 +379,19 @@ func labelNodeRoleControlPlane() string { return labelNodeRoleControlPlane } -func controlPlaneNoScheduleToleration() corev1.Toleration { +func controlPlaneNoScheduleTolerations() []corev1.Toleration { // the Node taint still uses "master" in K8s v1.20 - return corev1.Toleration{ - Key: "node-role.kubernetes.io/master", - Operator: corev1.TolerationOpExists, - Effect: corev1.TaintEffectNoSchedule, + return []corev1.Toleration{ + { + Key: "node-role.kubernetes.io/master", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "node-role.kubernetes.io/control-plane", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, } } @@ -1119,8 +1126,7 @@ func (data *TestData) CreatePodOnNodeInNamespace(name, ns string, nodeName, ctrN } if nodeName == controlPlaneNodeName() { // tolerate NoSchedule taint if we want Pod to run on control-plane Node - noScheduleToleration := controlPlaneNoScheduleToleration() - podSpec.Tolerations = []corev1.Toleration{noScheduleToleration} + podSpec.Tolerations = controlPlaneNoScheduleTolerations() } pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -2390,9 +2396,7 @@ func (data *TestData) createAgnhostPodOnNodeWithAnnotations(name string, ns stri func (data *TestData) createDaemonSet(name string, ns string, ctrName string, image string, cmd []string, args []string) (*appsv1.DaemonSet, func() error, error) { podSpec := corev1.PodSpec{ - Tolerations: []corev1.Toleration{ - controlPlaneNoScheduleToleration(), - }, + Tolerations: controlPlaneNoScheduleTolerations(), Containers: []corev1.Container{ { Name: ctrName, @@ -2464,9 +2468,7 @@ func (data *TestData) waitForDaemonSetPods(timeout time.Duration, dsName string, func (data *TestData) createStatefulSet(name string, ns string, size int32, ctrName string, image string, cmd []string, args []string, mutateFunc func(*appsv1.StatefulSet)) (*appsv1.StatefulSet, func() error, error) { podSpec := corev1.PodSpec{ - Tolerations: []corev1.Toleration{ - controlPlaneNoScheduleToleration(), - }, + Tolerations: controlPlaneNoScheduleTolerations(), Containers: []corev1.Container{ { Name: ctrName, diff --git a/test/e2e/networkpolicy_test.go b/test/e2e/networkpolicy_test.go index 313ad31b145..c4d3939f418 100644 --- a/test/e2e/networkpolicy_test.go +++ b/test/e2e/networkpolicy_test.go @@ -816,8 +816,7 @@ func testIngressPolicyWithEndPort(t *testing.T, data *TestData) { } if nodeName == controlPlaneNodeName() { // tolerate NoSchedule taint if we want Pod to run on control-plane Node - noScheduleToleration := controlPlaneNoScheduleToleration() - podSpec.Tolerations = []corev1.Toleration{noScheduleToleration} + podSpec.Tolerations = controlPlaneNoScheduleTolerations() } pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/e2e/performance_test.go b/test/e2e/performance_test.go index 02b32f6664c..c8b2c61de32 100644 --- a/test/e2e/performance_test.go +++ b/test/e2e/performance_test.go @@ -50,8 +50,6 @@ var ( customizePolicyRules = flag.Int("perf.http.policy_rules", 0, "Number of CIDRs in the network policy") httpConcurrency = flag.Int("perf.http.concurrency", 1, "Number of multiple requests to make at a time") realizeTimeout = flag.Duration("perf.realize.timeout", 5*time.Minute, "Timeout of the realization of network policies") - // tolerate NoSchedule taint to let the Pod run on control-plane Node - noScheduleToleration = controlPlaneNoScheduleToleration() labelSelector = &metav1.LabelSelector{ MatchLabels: map[string]string{"app": perfTestAppLabel}, } @@ -118,7 +116,7 @@ func createPerfTestPodDefinition(name, containerName, image string) *corev1.Pod "kubernetes.io/hostname": controlPlaneNodeName(), } - podSpec.Tolerations = []corev1.Toleration{noScheduleToleration} + podSpec.Tolerations = controlPlaneNoScheduleTolerations() pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name,