install: update helm templates to add HA capabilities for operator

[ upstream commit 930bde7 ] Signed-off-by: Deepesh Pathak <deepshpathak@gmail.com> Signed-off-by: André Martins <andre@cilium.io>
cilium · Aug 7, 2020 · 976337b · 976337b
1 parent 8fa829b
commit 976337b
Show file tree

Hide file tree

Showing 7 changed files with 80 additions and 2 deletions.
diff --git a/.github/workflows/helm-check.yaml b/.github/workflows/helm-check.yaml
@@ -50,6 +50,10 @@ jobs:
         run: |
           sed -i 's;pullPolicy: Always;pullPolicy: Never;g' install/kubernetes/cilium/values.yaml
 
+      - name: Run a single operator replica since it's a single node cluster
+        run: |
+          sed -i 's;  numReplicas: 2;  numReplicas: 1;g' install/kubernetes/cilium/values.yaml
+
       - name: Create kind cluster
         uses: helm/kind-action@v1.0.0-rc.1
         with:

diff --git a/install/kubernetes/cilium/charts/operator/templates/clusterrole.yaml b/install/kubernetes/cilium/charts/operator/templates/clusterrole.yaml
@@ -57,3 +57,22 @@ rules:
   - get
   - list
   - watch
+# For cilium-operator running in HA mode.
+#
+# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election
+# between mulitple running instances.
+# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less
+# common and fewer objects in the cluster watch "all Leases".
+# The support for leases was introduced in coordination.k8s.io/v1 during Kubernetes 1.14 release.
+# In Cilium we currently don't support HA mode for K8s version < 1.14. This condition make sure
+# that we only authorize access to leases resources in supported K8s versions.
+{{- if or (ge .Capabilities.KubeVersion.Minor "14") (gt .Capabilities.KubeVersion.Major "1") }}
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - update
+{{- end }}
diff --git a/install/kubernetes/cilium/charts/operator/templates/deployment.yaml b/install/kubernetes/cilium/charts/operator/templates/deployment.yaml
@@ -7,7 +7,14 @@ metadata:
   name: cilium-operator
   namespace: {{ .Release.Namespace }}
 spec:
+  # We support HA mode only for Kubernetes version > 1.14
+  # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go
+  # for more details.
+{{- if or (ge .Capabilities.KubeVersion.Minor "14") (gt .Capabilities.KubeVersion.Major "1") }}
+  replicas: {{ .Values.numReplicas }}
+{{- else }}
   replicas: 1
+{{- end }}
   selector:
     matchLabels:
       io.cilium/app: operator

diff --git a/install/kubernetes/cilium/charts/operator/values.yaml b/install/kubernetes/cilium/charts/operator/values.yaml
@@ -6,3 +6,8 @@ serviceAccount:
 
 # Specifies the resources for the operator container
 resources: {}
+
+
+# Number of replicas to run for cilium operator deployment.
+numReplicas: 2
+
diff --git a/install/kubernetes/cilium/values.yaml b/install/kubernetes/cilium/values.yaml
@@ -18,6 +18,9 @@ config:
 operator:
   enabled: true
 
+  # Number of replicas to run for cilium-operator deployment.
+  numReplicas: 2
+
 # Include the PreFlight DaemonSet
 preflight:
   enabled: false

diff --git a/install/kubernetes/experimental-install.yaml b/install/kubernetes/experimental-install.yaml
@@ -339,6 +339,23 @@ rules:
   - get
   - list
   - watch
+# For cilium-operator running in HA mode.
+#
+# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election
+# between mulitple running instances.
+# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less
+# common and fewer objects in the cluster watch "all Leases".
+# The support for leases was introduced in coordination.k8s.io/v1 during Kubernetes 1.14 release.
+# In Cilium we currently don't support HA mode for K8s version < 1.14. This condition make sure
+# that we only authorize access to leases resources in supported K8s versions.
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - update
 ---
 # Source: cilium/charts/agent/templates/clusterrolebinding.yaml
 apiVersion: rbac.authorization.k8s.io/v1
@@ -793,7 +810,10 @@ metadata:
   name: cilium-operator
   namespace: kube-system
 spec:
-  replicas: 1
+  # We support HA mode only for Kubernetes version > 1.14
+  # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go
+  # for more details.
+  replicas: 2
   selector:
     matchLabels:
       io.cilium/app: operator

diff --git a/install/kubernetes/quick-install.yaml b/install/kubernetes/quick-install.yaml
@@ -261,6 +261,23 @@ rules:
   - get
   - list
   - watch
+# For cilium-operator running in HA mode.
+#
+# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election
+# between mulitple running instances.
+# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less
+# common and fewer objects in the cluster watch "all Leases".
+# The support for leases was introduced in coordination.k8s.io/v1 during Kubernetes 1.14 release.
+# In Cilium we currently don't support HA mode for K8s version < 1.14. This condition make sure
+# that we only authorize access to leases resources in supported K8s versions.
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - update
 ---
 # Source: cilium/charts/agent/templates/clusterrolebinding.yaml
 apiVersion: rbac.authorization.k8s.io/v1
@@ -539,7 +556,10 @@ metadata:
   name: cilium-operator
   namespace: kube-system
 spec:
-  replicas: 1
+  # We support HA mode only for Kubernetes version > 1.14
+  # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go
+  # for more details.
+  replicas: 2
   selector:
     matchLabels:
       io.cilium/app: operator