k8s-prom: Slack alert

ruby-no-kai · Aug 31, 2022 · 42df8b0 · 42df8b0
1 parent 0cd9788
commit 42df8b0
Showing 1 changed file with 38 additions and 0 deletions.
diff --git a/tf/k8s-prom/kube-prometheus-stack.values.jsonnet b/tf/k8s-prom/kube-prometheus-stack.values.jsonnet
@@ -25,11 +25,48 @@ local volumeClaimTemplate(size) = {
   },
   alertmanager: {
     config: {
+      global: {
+        slack_api_url_file: '/etc/alertmanager/secrets/alertmanager-slack-webhook/url',
+      },
+      route: {
+        routes: [
+          {
+            matchers: [
+              'severity=~"warning|critical"',
+            ],
+            group_by: ['alertname', 'job'],
+            group_wait: '12s',
+            group_interval: '12s',
+            repeat_interval: '1h',
+            receiver: 'slack-default',
+          }
+        ],
+        receiver: "null",
+      },
+      receivers: [
+        {
+          name: 'null',
+        },
+        {
+          name: 'slack-default',
+          slack_configs: [
+            {
+              send_resolved: true,
+              title_link: "https://grafana.rubykaigi.net/alerting/list?view=state&dataSource=Prometheus&queryString=alertname%3D{{ .GroupLabels.alertname | urlquery }}",
+              title: "{{ .GroupLabels.alertname }}{{ with .GroupLabels.job }} - {{ . }}{{ end }}",
+              text: "{{ range .Alerts }}*{{ .Status }}* {{ with .Labels.instance }}{{ . }} - {{ end }}{{ .Annotations.summary }}\n{{ end }}",
+            },
+          ],
+        },
+      ],
     },
     alertmanagerSpec: {
       storage: {
         volumeClaimTemplate: volumeClaimTemplate('10Gi'),
       },
+      secrets: [
+        'alertmanager-slack-webhook'
+      ],
       retention: '720h',
     },
   },
@@ -39,6 +76,7 @@ local volumeClaimTemplate(size) = {
 
   defaultRules: {
     rules: {
+      general: false,
       kubeControllerManager: false,
       kubeScheduler: false,
     },