change alertmanager-svc-headless from http to grpc port + expose 9094 TCP and UDP

CHANGELOG.md

@@ -11,6 +11,7 @@
 ## 2.1.0 / 2023-03-17
 
 * [FEATURE] add autoscaler for the ruler #430
+* [CHANGE] change alertmanager-svc-headless from http to grpc port #420
 * [ENHANCEMENT] Add annotations and labels to memberlist service #433
 * [ENHANCEMENT] Add podTargetLabels to all Prometheus servicemonitor CRs #439
 * [DEPENDENCY] Update Helm release memcached to v6.3.12 #441

ci/test-deployment-values.yaml

@@ -128,7 +128,7 @@ runtimeconfigmap:
   annotations:
     foo: bar
 alertmanager:
-  replicas: 1
+  replicas: 3
   statefulSet:
     enabled: false
   extraVolumes:

ci/test-sts-values.yaml

@@ -119,7 +119,7 @@ runtimeconfigmap:
   annotations:
     foo: bar
 alertmanager:
-  replicas: 1
+  replicas: 3
   statefulSet:
     enabled: true
   extraVolumes:

templates/alertmanager/alertmanager-statefulset.yaml

@@ -1,3 +1,5 @@
+{{- $svcClusterAddress := ((.Values.config.alertmanager.cluster).listen_address) | default "0.0.0.0:9094" }}
+{{- $svcClusterPort := (split ":" $svcClusterAddress)._1 }}
 {{- if .Values.alertmanager.enabled -}}
 {{- if .Values.alertmanager.statefulSet.enabled -}}
 apiVersion: apps/v1
@@ -152,6 +154,15 @@ spec:
           args:
             - "-target=alertmanager"
             - "-config.file=/etc/cortex/cortex.yaml"
+            {{- if and (gt (int .Values.alertmanager.replicas) 1) (ne .Values.config.alertmanager.cluster.listen_address "") }}
+            {{- $fullName := include "cortex.alertmanagerFullname" . }}
+            {{- $peers := list }}
+            {{- range $i := until (int .Values.alertmanager.replicas) }}
+	      {{- $peer := printf "%s-%d.%s-headless.%s.svc.cluster.local:%s" $fullName $i $fullName $.Release.Namespace $svcClusterPort }}
+              {{- $peers = append $peers $peer }}
+            {{- end }}
+            - "-alertmanager.cluster.peers={{ join "," $peers }}"
+            {{- end }}
             {{- range $key, $value := .Values.alertmanager.extraArgs }}
             - "-{{ $key }}={{ $value }}"
             {{- end }}
@@ -175,6 +186,15 @@ spec:
             - name: gossip
               containerPort: {{ .Values.config.memberlist.bind_port }}
               protocol: TCP
+            - name: grpc
+              containerPort: {{ .Values.config.server.grpc_listen_port }}
+              protocol: TCP
+            - containerPort: {{ $svcClusterPort }}
+              name: alert-clu-tcp
+              protocol: TCP
+            - containerPort: {{ $svcClusterPort }}
+              name: alert-clu-udp
+              protocol: UDP
           startupProbe:
             {{- toYaml .Values.alertmanager.startupProbe | nindent 12 }}
           livenessProbe:

templates/alertmanager/alertmanager-svc-headless.yaml

@@ -1,5 +1,4 @@
 {{- if .Values.alertmanager.enabled -}}
-{{- if .Values.alertmanager.statefulSet.enabled -}}
 apiVersion: v1
 kind: Service
 metadata:
@@ -17,11 +16,18 @@ spec:
   clusterIP: None
   publishNotReadyAddresses: true
   ports:
-    - port: {{ .Values.config.server.http_listen_port }}
+    - port: {{ .Values.config.server.grpc_listen_port }}
       protocol: TCP
-      name: http-metrics
-      targetPort: http-metrics
+      name: grpc
+      targetPort: grpc
+    - port: 9094
+      protocol: UDP
+      name: alert-clu-udp
+      targetPort: alert-clu-udp
+    - port: 9094
+      protocol: TCP
+      name: alert-clu-tcp
+      targetPort: alert-clu-tcp
   selector:
     {{- include "cortex.alertmanagerSelectorLabels" . | nindent 4 }}
 {{- end -}}
-{{- end -}}

values.yaml

@@ -129,6 +129,10 @@ config:
   runtime_config:
     file: /etc/cortex-runtime-config/runtime_config.yaml
   alertmanager:
+    # -- Enable alertmanager gossip cluster
+    # -- Disable alertmanager gossip cluster by setting empty listen_address to empty string
+    cluster:
+      listen_address: '0.0.0.0:9094'
     # -- Enable the experimental alertmanager config api.
     enable_api: false
     external_url: '/api/prom/alertmanager'