ClusterMesh/helm: support multiple replicas

This commit makes changes to the helm templates for
clustermesh-apiserver to support deploying multiple replicas.

- Use a unique cluster id for etcd:

Each replica of the clustermesh-apiserver deploys its own discrete etcd
cluster. Utilize the K8s downward API to provide the Pod UUID to the
etcd cluster as an initial cluster token, so that each instance has a
unique cluster ID. This is necessary to distinguish connections to
multiple clustermesh-apiserver Pods using the same K8s Service.

- Use session affinity for the clustermesh-apiserver Service

Session affinity ensures that connections from a client are passed to
the same service backend each time. This will allow a Cilium Agent or
KVStoreMesh instance to maintain a connection to the same backend for
both long-living, streaming connections, such as watches on the kv
store, and short, single-response connections, such as checking the
status of a cluster. However, this can be unreliable if the l3/l4
loadbalancer used does not also implement sticky sessions to direct
connections from a particular client to the same cluster node.

Signed-off-by: Tim Horner <timothy.horner@isovalent.com>

install/kubernetes/cilium/templates/clustermesh-apiserver/deployment.yaml

@@ -59,7 +59,7 @@ spec:
         {{- end }}
         # These need to match the equivalent arguments to etcd in the main container.
         - --etcd-cluster-name=clustermesh-apiserver
-        - --etcd-initial-cluster-token=clustermesh-apiserver
+        - --etcd-initial-cluster-token=$(INITIAL_CLUSTER_TOKEN)
         - --etcd-data-dir=/var/run/etcd
         {{- with .Values.clustermesh.apiserver.etcd.init.extraArgs }}
         {{- toYaml . | trim | nindent 8 }}
@@ -76,6 +76,10 @@ spec:
             configMapKeyRef:
               name: cilium-config
               key: cluster-name
+        - name: INITIAL_CLUSTER_TOKEN
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.uid
         {{- with .Values.clustermesh.apiserver.etcd.init.extraEnv }}
         {{- toYaml . | trim | nindent 8 }}
         {{- end }}
@@ -108,7 +112,7 @@ spec:
         # uses net.SplitHostPort() internally and it accepts the that format.
         - --listen-client-urls=https://127.0.0.1:2379,https://[$(HOSTNAME_IP)]:2379
         - --advertise-client-urls=https://[$(HOSTNAME_IP)]:2379
-        - --initial-cluster-token=clustermesh-apiserver
+        - --initial-cluster-token=$(INITIAL_CLUSTER_TOKEN)
         - --auto-compaction-retention=1
         {{- if .Values.clustermesh.apiserver.metrics.etcd.enabled }}
         - --listen-metrics-urls=http://[$(HOSTNAME_IP)]:{{ .Values.clustermesh.apiserver.metrics.etcd.port }}
@@ -121,6 +125,10 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: status.podIP
+        - name: INITIAL_CLUSTER_TOKEN
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.uid
         ports:
         - name: etcd
           containerPort: 2379

install/kubernetes/cilium/templates/clustermesh-apiserver/service.yaml

@@ -35,4 +35,5 @@ spec:
   {{- if .Values.clustermesh.apiserver.service.internalTrafficPolicy }}
   internalTrafficPolicy: {{ .Values.clustermesh.apiserver.service.internalTrafficPolicy }}
   {{- end }}
+  sessionAffinity: ClientIP
 {{- end }}