kvstoremesh: add helm configuration

This commit extends the helm chart to allow configuring kvstoremesh. In particular, the clustermesh-apiserver deployment is enriched with the additional kvstoremesh sidecar container (when kvstoremesh is enabled), appropriately mounting the secret containing the remote kvstore configurations. Additionally, the configuration used by the agents is modified to connect to the local kvstore instance (through the corresponding service) instead of the remote ones. Signed-off-by: Marco Iorio <marco.iorio@isovalent.com>
cilium · Jun 16, 2023 · 2b7ada8 · 2b7ada8
1 parent 5eea35c
commit 2b7ada8
Show file tree

Hide file tree

Showing 15 changed files with 326 additions and 10 deletions.
diff --git a/Documentation/helm-values.rst b/Documentation/helm-values.rst
diff --git a/Documentation/observability/metrics.rst b/Documentation/observability/metrics.rst
@@ -192,20 +192,24 @@ Prometheus namespace. Etcd metrics are exported under the ``etcd_`` Prometheus n
 Installation
 ------------
 
-You can enable metrics for ``clustermesh-apiserver`` with the Helm value
-``clustermesh.apiserver.metrics.enabled=true``.
-To enable metrics for the sidecar etcd instance, use
-``clustermesh.apiserver.metrics.etcd.enabled=true``.
+You can enable the metrics for different Cluster Mesh API Server components by
+setting the following values:
+
+* clustermesh-apiserver: ``clustermesh.apiserver.metrics.enabled=true``
+* kvstoremesh: ``clustermesh.apiserver.metrics.kvstoremesh.enabled=true``
+* sidecar etcd instance: ``clustermesh.apiserver.metrics.etcd.enabled=true``
 
 .. parsed-literal::
 
    helm install cilium |CHART_RELEASE| \\
      --namespace kube-system \\
      --set clustermesh.useAPIServer=true \\
      --set clustermesh.apiserver.metrics.enabled=true \\
+     --set clustermesh.apiserver.metrics.kvstoremesh.enabled=true \\
      --set clustermesh.apiserver.metrics.etcd.enabled=true
 
-The ports can be configured via ``clustermesh.apiserver.metrics.port`` and
+You can figure the ports by way of ``clustermesh.apiserver.metrics.port``,
+``clustermesh.apiserver.metrics.kvstoremesh.port`` and
 ``clustermesh.apiserver.metrics.etcd.port`` respectively.
 
 You can automatically create a

diff --git a/install/kubernetes/Makefile.digests b/install/kubernetes/Makefile.digests
@@ -6,6 +6,7 @@ export CILIUM_DIGEST := ""
 export CLUSTERMESH_APISERVER_DIGEST := ""
 export DOCKER_PLUGIN_DIGEST := ""
 export HUBBLE_RELAY_DIGEST := ""
+export KVSTOREMESH_DIGEST := ""
 export OPERATOR_AWS_DIGEST := ""
 export OPERATOR_AZURE_DIGEST := ""
 export OPERATOR_ALIBABACLOUD_DIGEST := ""

diff --git a/install/kubernetes/Makefile.values b/install/kubernetes/Makefile.values
@@ -15,6 +15,7 @@ ifeq ($(RELEASE),yes)
     export CILIUM_OPERATOR_BASE_REPO:=${RELEASE_REGISTRY}/${RELEASE_ORG}/operator
     export CLUSTERMESH_APISERVER_REPO:=${RELEASE_REGISTRY}/${RELEASE_ORG}/clustermesh-apiserver
     export HUBBLE_RELAY_REPO:=${RELEASE_REGISTRY}/${RELEASE_ORG}/hubble-relay
+    export KVSTOREMESH_REPO:=${RELEASE_REGISTRY}/${RELEASE_ORG}/kvstoremesh
 else
     export CILIUM_BRANCH:=main
     export PULL_POLICY:=Always
@@ -24,6 +25,7 @@ else
     export CILIUM_VERSION:=latest
     export CLUSTERMESH_APISERVER_REPO:=${CI_REGISTRY}/${CI_ORG}/clustermesh-apiserver-ci
     export HUBBLE_RELAY_REPO:=${CI_REGISTRY}/${CI_ORG}/hubble-relay-ci
+    export KVSTOREMESH_REPO:=${CI_REGISTRY}/${CI_ORG}/kvstoremesh-ci
 endif
 
 ifndef CILIUM_BRANCH

diff --git a/install/kubernetes/cilium/README.md b/install/kubernetes/cilium/README.md
diff --git a/install/kubernetes/cilium/templates/cilium-agent/daemonset.yaml b/install/kubernetes/cilium/templates/cilium-agent/daemonset.yaml
@@ -716,7 +716,7 @@ spec:
       tolerations:
         {{- toYaml . | trim | nindent 8 }}
       {{- end }}
-      {{- if and .Values.clustermesh.useAPIServer .Values.clustermesh.config.enabled }}
+      {{- if and .Values.clustermesh.useAPIServer .Values.clustermesh.config.enabled (not .Values.clustermesh.apiserver.kvstoremesh.enabled) }}
       hostAliases:
       {{- range $cluster := .Values.clustermesh.config.clusters }}
       {{- range $ip := $cluster.ips }}

diff --git a/install/kubernetes/cilium/templates/clustermesh-apiserver/deployment.yaml b/install/kubernetes/cilium/templates/clustermesh-apiserver/deployment.yaml
@@ -218,6 +218,68 @@ spec:
         securityContext:
           {{- toYaml . | nindent 10 }}
         {{- end }}
+      {{- if .Values.clustermesh.apiserver.kvstoremesh.enabled }}
+      - name: kvstoremesh
+        image: {{ include "cilium.image" .Values.clustermesh.apiserver.kvstoremesh.image | quote }}
+        imagePullPolicy: {{ .Values.clustermesh.apiserver.kvstoremesh.image.pullPolicy }}
+        command:
+        - /usr/bin/kvstoremesh
+        args:
+        {{- if .Values.debug.enabled }}
+        - --debug
+        {{- end }}
+        - --cluster-name=$(CLUSTER_NAME)
+        - --cluster-id=$(CLUSTER_ID)
+        - --kvstore-opt=etcd.config=/var/lib/cilium/etcd-config.yaml
+        - --kvstore-opt=etcd.qps=100
+        - --kvstore-opt=etcd.maxInflight=10
+        - --clustermesh-config=/var/lib/cilium/clustermesh
+        {{- if .Values.clustermesh.apiserver.metrics.kvstoremesh.enabled }}
+        - --prometheus-serve-addr=:{{ .Values.clustermesh.apiserver.metrics.kvstoremesh.port }}
+        {{- end }}
+        {{- with .Values.clustermesh.apiserver.kvstoremesh.extraArgs }}
+        {{- toYaml . | trim | nindent 8 }}
+        {{- end }}
+        env:
+        - name: CLUSTER_NAME
+          valueFrom:
+            configMapKeyRef:
+              name: cilium-config
+              key: cluster-name
+        - name: CLUSTER_ID
+          valueFrom:
+            configMapKeyRef:
+              name: cilium-config
+              key: cluster-id
+        {{- with .Values.clustermesh.apiserver.kvstoremesh.extraEnv }}
+        {{- toYaml . | trim | nindent 8 }}
+        {{- end }}
+        {{- if .Values.clustermesh.apiserver.metrics.kvstoremesh.enabled }}
+        ports:
+        - name: kvmesh-metrics
+          containerPort: {{ .Values.clustermesh.apiserver.metrics.kvstoremesh.port }}
+          protocol: TCP
+        {{- end }}
+        {{- with .Values.clustermesh.apiserver.kvstoremesh.resources }}
+        resources:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        volumeMounts:
+        - name: etcd-admin-client
+          mountPath: /var/lib/cilium/etcd-secrets
+          readOnly: true
+        - name: kvstoremesh-secrets
+          mountPath: /var/lib/cilium/clustermesh
+          readOnly: true
+        {{- with .Values.clustermesh.apiserver.kvstoremesh.extraVolumeMounts }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        terminationMessagePolicy: FallbackToLogsOnError
+        {{- with .Values.clustermesh.apiserver.kvstoremesh.securityContext }}
+        securityContext:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+      {{- end }}
       volumes:
       - name: etcd-server-secrets
         projected:
@@ -272,6 +334,29 @@ spec:
       {{- end }}
       - name: etcd-data-dir
         emptyDir: {}
+      {{- if .Values.clustermesh.apiserver.kvstoremesh.enabled }}
+      - name: kvstoremesh-secrets
+        projected:
+          # note: the leading zero means this number is in octal representation: do not remove it
+          defaultMode: 0400
+          sources:
+          - secret:
+              name: cilium-kvstoremesh
+              # note: items are not explicitly listed here, since the entries of this secret
+              # depend on the peers configured, and that would cause a restart of this pod
+              # at every addition/removal. Leaving the field empty makes each secret entry
+              # to be automatically projected into the volume as a file whose name is the key.
+          - secret:
+              name: clustermesh-apiserver-remote-cert
+              optional: true
+              items:
+              - key: tls.key
+                path: common-etcd-client.key
+              - key: tls.crt
+                path: common-etcd-client.crt
+              - key: ca.crt
+                path: common-etcd-client-ca.crt
+      {{- end }}
       {{- with .Values.clustermesh.apiserver.extraVolumes }}
       {{- toYaml . | nindent 6 }}
       {{- end }}
@@ -303,4 +388,13 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- if and .Values.clustermesh.config.enabled .Values.clustermesh.apiserver.kvstoremesh.enabled }}
+      hostAliases:
+      {{- range $cluster := .Values.clustermesh.config.clusters }}
+      {{- range $ip := $cluster.ips }}
+      - ip: {{ $ip }}
+        hostnames: [ "{{ $cluster.name }}.{{ $.Values.clustermesh.config.domain }}" ]
+      {{- end }}
+      {{- end }}
+      {{- end }}
 {{- end }}
diff --git a/install/kubernetes/cilium/templates/clustermesh-apiserver/metrics-service.yaml b/install/kubernetes/cilium/templates/clustermesh-apiserver/metrics-service.yaml
@@ -1,6 +1,6 @@
 {{- if and
   (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer)
-  (or .Values.clustermesh.apiserver.metrics.enabled .Values.clustermesh.apiserver.metrics.etcd.enabled) }}
+  (or .Values.clustermesh.apiserver.metrics.enabled .Values.clustermesh.apiserver.metrics.kvstoremesh.enabled .Values.clustermesh.apiserver.metrics.etcd.enabled) }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -21,6 +21,12 @@ spec:
     protocol: TCP
     targetPort: apiserv-metrics
   {{- end }}
+  {{- if .Values.clustermesh.apiserver.metrics.kvstoremesh.enabled }}
+  - name: kvmesh-metrics
+    port: {{ .Values.clustermesh.apiserver.metrics.kvstoremesh.port }}
+    protocol: TCP
+    targetPort: kvmesh-metrics
+  {{- end }}
   {{- if .Values.clustermesh.apiserver.metrics.etcd.enabled }}
   - name: etcd-metrics
     port: {{ .Values.clustermesh.apiserver.metrics.etcd.port }}

diff --git a/install/kubernetes/cilium/templates/clustermesh-apiserver/servicemonitor.yaml b/install/kubernetes/cilium/templates/clustermesh-apiserver/servicemonitor.yaml
@@ -1,6 +1,6 @@
 {{- if and
   (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer)
-  (or .Values.clustermesh.apiserver.metrics.enabled .Values.clustermesh.apiserver.metrics.etcd.enabled)
+  (or .Values.clustermesh.apiserver.metrics.enabled .Values.clustermesh.apiserver.metrics.kvstoremesh.enabled .Values.clustermesh.apiserver.metrics.etcd.enabled)
   .Values.clustermesh.apiserver.metrics.serviceMonitor.enabled }}
 ---
 apiVersion: monitoring.coreos.com/v1
@@ -40,6 +40,20 @@ spec:
     {{- toYaml . | nindent 4 }}
     {{- end }}
   {{- end }}
+  {{- if .Values.clustermesh.apiserver.metrics.kvstoremesh.enabled }}
+  - port: kvmesh-metrics
+    interval: {{ .Values.clustermesh.apiserver.metrics.serviceMonitor.kvstoremesh.interval | quote }}
+    honorLabels: true
+    path: /metrics
+    {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.kvstoremesh.relabelings }}
+    relabelings:
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+    {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.kvstoremesh.metricRelabelings }}
+    metricRelabelings:
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- end }}
   {{- if .Values.clustermesh.apiserver.metrics.etcd.enabled }}
   - port: etcd-metrics
     interval: {{ .Values.clustermesh.apiserver.metrics.serviceMonitor.etcd.interval | quote }}

diff --git a/install/kubernetes/cilium/templates/clustermesh-apiserver/users-configmap.yaml b/install/kubernetes/cilium/templates/clustermesh-apiserver/users-configmap.yaml
@@ -10,6 +10,10 @@ metadata:
 data:
   users.yaml: |
     users:
+    {{- if .Values.clustermesh.apiserver.kvstoremesh.enabled }}
+    - name: remote-{{ .Values.cluster.name }}
+      role: remote
+    {{- end }}
     {{- range .Values.clustermesh.config.clusters }}
     - name: remote-{{ .name }}
       role: remote

diff --git a/install/kubernetes/cilium/templates/clustermesh-config/_helpers.tpl b/install/kubernetes/cilium/templates/clustermesh-config/_helpers.tpl
@@ -2,11 +2,14 @@
 {{- $cluster := index . 0 -}}
 {{- $domain := index . 1 -}}
 {{- $hasCustomCACert := index . 2 -}}
+{{- $override := index . 3 -}}
 {{- /* The parenthesis around $cluster.tls are required, since it can be null: https://stackoverflow.com/a/68807258 */}}
 {{- $prefix := ternary "common-" (printf "%s." $cluster.name) (or (empty ($cluster.tls).cert) (empty ($cluster.tls).key)) -}}
 
 endpoints:
-{{- if $cluster.ips }}
+{{- if $override }}
+- {{ $override }}
+{{- else if $cluster.ips }}
 - https://{{ $cluster.name }}.{{ $domain }}:{{ $cluster.port }}
 {{- else }}
 - https://{{ $cluster.address | required "missing clustermesh.apiserver.config.clusters.address" }}:{{ $cluster.port }}

diff --git a/install/kubernetes/cilium/templates/clustermesh-config/clustermesh-secret.yaml b/install/kubernetes/cilium/templates/clustermesh-config/clustermesh-secret.yaml
@@ -6,9 +6,10 @@ metadata:
   name: cilium-clustermesh
   namespace: {{ .Release.Namespace }}
 data:
+  {{- $override := ternary (printf "https://clustermesh-apiserver.%s.svc:2379" .Release.Namespace) "" $.Values.clustermesh.apiserver.kvstoremesh.enabled }}
   {{- range .Values.clustermesh.config.clusters }}
   {{- $hasCustomCACert := or (.tls).caCert $.Values.clustermesh.apiserver.tls.ca.cert }}
-  {{ .name }}: {{ include "clustermesh-config-generate-etcd-cfg" (list . $.Values.clustermesh.config.domain $hasCustomCACert) | b64enc }}
+  {{ .name }}: {{ include "clustermesh-config-generate-etcd-cfg" (list . $.Values.clustermesh.config.domain $hasCustomCACert $override) | b64enc }}
   {{- /* The parenthesis around .tls are required, since it can be null: https://stackoverflow.com/a/68807258 */}}
   {{- if and (.tls).cert (.tls).key }}
   {{- if $hasCustomCACert }}

diff --git a/install/kubernetes/cilium/templates/clustermesh-config/kvstoremesh-secret.yaml b/install/kubernetes/cilium/templates/clustermesh-config/kvstoremesh-secret.yaml
@@ -0,0 +1,19 @@
+{{- if and .Values.clustermesh.useAPIServer .Values.clustermesh.config.enabled .Values.clustermesh.apiserver.kvstoremesh.enabled }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cilium-kvstoremesh
+  namespace: {{ .Release.Namespace }}
+data:
+  {{- range .Values.clustermesh.config.clusters }}
+  {{- $hasCustomCACert := or (.tls).caCert $.Values.clustermesh.apiserver.tls.ca.cert }}
+  {{ .name }}: {{ include "clustermesh-config-generate-etcd-cfg" (list . $.Values.clustermesh.config.domain $hasCustomCACert "") | b64enc }}
+  {{- /* The parenthesis around .tls are required, since it can be null: https://stackoverflow.com/a/68807258 */}}
+  {{- if and (.tls).cert (.tls).key }}
+  {{ .name }}.etcd-client-ca.crt: {{ .tls.caCert | default $.Values.clustermesh.apiserver.tls.ca.cert }}
+  {{ .name }}.etcd-client.key: {{ .tls.key }}
+  {{ .name }}.etcd-client.crt: {{ .tls.cert }}
+  {{- end }}
+  {{- end }}
+{{- end }}