helm: Add pod affinity for cilium-envoy

This commit is to avoid cilium-envoy running on the node without cilium
agent. Two main changes are:

- podAffinity with requiredDuringSchedulingIgnoredDuringExecution to
  cilium agent
- tolerations with same values as the agent itself, this is to cater
  node without cilium cases

Relates: cilium#25081, cilium#30034
Fixes: cilium#31149
Signed-off-by: Tam Mach <tam.mach@cilium.io>

install/kubernetes/cilium/templates/cilium-envoy/daemonset.yaml

@@ -229,8 +229,8 @@ spec:
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- with .Values.envoy.tolerations }}
       tolerations:
+      {{- with .Values.envoy.tolerations }}
         {{- toYaml . | trim | nindent 8 }}
       {{- end }}
       volumes:

install/kubernetes/cilium/values.yaml.tmpl

@@ -1936,17 +1936,39 @@ envoy:
           labelSelector:
             matchLabels:
               k8s-app: cilium-envoy
+    podAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+      - topologyKey: kubernetes.io/hostname
+        labelSelector:
+          matchLabels:
+            k8s-app: cilium
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+            - key: cilium.io/no-schedule
+              operator: NotIn
+              values:
+              - true
   # -- Node selector for cilium-envoy.
   nodeSelector:
     kubernetes.io/os: linux
   # -- Node tolerations for envoy scheduling to nodes with taints
+  # By default it follows the same tolerations as the agent itself
+  # to allow the Cilium agent on this node to connect to cilium-envoy pod.
   # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/
   tolerations:
-    - operator: Exists
-      # - key: "key"
-      #   operator: "Equal|Exists"
-      #   value: "value"
-      #   effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)"
+  - key: node.kubernetes.io/not-ready
+    effect: NoSchedule
+  - key: node-role.kubernetes.io/master
+    effect: NoSchedule
+  - key: node-role.kubernetes.io/control-plane
+    effect: NoSchedule
+  - key: node.cloudprovider.kubernetes.io/uninitialized
+    effect: NoSchedule
+    value: "true"
+  - key: CriticalAddonsOnly
+    operator: "Exists"
   # -- The priority class to use for cilium-envoy.
   priorityClassName: ~
   # -- DNS policy for Cilium envoy pods.