-
Notifications
You must be signed in to change notification settings - Fork 164
/
alertmanager.gotmpl
97 lines (97 loc) · 4.42 KB
/
alertmanager.gotmpl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
{{- $receivers := .instance | get "alerts.receivers" (.root | get "alerts.receivers" (list "slack")) }}
{{- $suffix := (true | ternary "" ".monitoring.svc.cluster.local") }}
{{- $homeReceivers := .root | get "home.receivers" (list "slack") }}
global:
{{- if or (has "slack" $receivers ) (and .root.otomi.isHomeMonitored (has "slack" $homeReceivers)) }}
slack_api_url: {{ .instance | get "alerts.slack.url" (.root | get "alerts.slack.url" (.root | get "home.slack.url" nil)) }}
{{- end }}
{{- if or (has "email" $receivers) (and .root.otomi.isHomeMonitored (has "email" $homeReceivers)) }}
smtp_smarthost: {{ .instance | get "alerts.email.smtp.smarthost" (.root | get "alerts.email.smtp.smarthost" (.root | get "home.email.smtp.smarthost" nil)) }}
smtp_hello: {{ .instance | get "alerts.email.smtp.hello" (.root | get "alerts.email.smtp.hello" .root.cluster.domain) }}
smtp_from: {{ .instance | get "alerts.email.smtp.from" (.root | get "alerts.email.smtp.from" (print "alerts@" .root.cluster.domain)) }}
smtp_auth_username: {{ .instance | get "alerts.email.smtp.auth_username" (.root | get "alerts.email.smtp.auth_username" (.root | get "home.email.smtp.auth_username" nil)) }}
smtp_auth_password: {{ .instance | get "alerts.email.smtp.auth_password" (.root | get "alerts.email.smtp.auth_password" (.root | get "home.email.smtp.auth_password" nil)) }}
smtp_auth_secret: {{ .instance | get "alerts.email.smtp.auth_secret" (.root | get "alerts.email.smtp.auth_secret" (.root | get "home.email.smtp.auth_secret" nil)) }}
smtp_auth_identity: {{ .instance | get "alerts.email.smtp.auth_identity" (.root | get "alerts.email.smtp.auth_identity" (.root | get "home.email.smtp.auth_identity" nil)) }}
{{- end }}
route:
receiver: default
group_by: [alertname]
group_interval: {{ .instance | get "alerts.groupInterval" (.root | get "alerts.groupInterval" "5m") }}
repeat_interval: {{ .instance | get "alerts.repeatInterval" (.root | get "alerts.repeatInterval" "3h") }}
routes:
- match:
alertname: Watchdog
receiver: "null"
- match:
alertname: CPUThrottlingHigh
receiver: "null"
{{- if eq .root.cluster.provider "azure" }}
- match:
alertname: KubeAPILatencyHigh
receiver: "null"
{{- end }}
- match:
severity: critical
receiver: critical
{{- if .root.otomi.isHomeMonitored }}
continue: true
- match:
severity: critical
receiver: critical-home
{{- end }}
receivers:
- name: "null"
{{- if has "slack" $receivers }}
- name: default
slack_configs:
- channel: "#{{ .instance | get "alerts.slack.channel" (.root | get "alerts.slack.channel" "mon-otomi") }}"
{{- .slackTpl | nindent 8 }}
- name: critical
slack_configs:
- channel: "#{{ .instance | get "alerts.slack.channelCrit" (.root | get "alerts.slack.channelCrit" "mon-otomi-crit") }}"
{{- .slackTpl | nindent 8 }}
{{- end }}
{{- if has "msteams" $receivers }}
- name: default
webhook_configs:
- url: "http://prometheus-msteams{{ $suffix }}:800/low_priority_channel"
send_resolved: true
- name: critical
webhook_configs:
- url: "http://prometheus-msteams{{ $suffix }}:800/high_priority_channel"
send_resolved: true
{{- end }}
{{- if has "email" $receivers }}
{{- $criticalTo := .instance | get "alerts.email.critical" (.root | get "alerts.email.critical") }}
{{- $nonCriticalTo := .instance | get "alerts.email.nonCritical" (.root | get "alerts.email.nonCritical") }}
- name: default
email_configs:
- to: {{ $nonCriticalTo }}
send_resolved: true
- name: critical
email_configs:
- to: {{ $criticalTo }}
send_resolved: true
{{- end }}
{{- if .root.otomi.isHomeMonitored }}
- name: critical-home
{{- $receivers := .root.home | get "receivers" }}
# sending criticals also to home to be aware of issues
{{- if has "slack" $receivers }}
slack_configs:
- channel: "#{{ .root | get "home.slack.channelCrit" "mon-otomi-crit" }}"
{{- .slackTpl | nindent 8 }}
{{- end }}
{{- if has "msteams" $receivers }}
webhook_configs:
- url: "http://prometheus-msteams.monitoring.svc.cluster.local:800/high_priority_channel"
send_resolved: true
{{- end }}
{{- if has "email" $receivers }}
email_configs:
- to: {{ .root | get "home.email.to" }}
from: {{ .root | get "home.email.from" (print "alerts@" .root.cluster.domain) }}
send_resolved: true
{{- end }}
{{- end }}