Merge pull request #49 from benclapp/custom-alert-annotations

Add optional alertLabels and alertAnnotations
metalmatze · Jun 11, 2021 · ba86443 · ba86443
2 parents 56f86bf + eaff717
commit ba86443
Show file tree

Hide file tree

Showing 7 changed files with 32 additions and 16 deletions.
diff --git a/examples/http-request-errors.yaml b/examples/http-request-errors.yaml
@@ -1,5 +1,6 @@
 alerts:
-- expr: |
+- annotations: {}
+  expr: |
     (
       sum(rate(promhttp_metric_handler_requests_total{namespace="default",job="fooapp",code=~"5.."}[5m]))
       /
@@ -11,7 +12,8 @@ alerts:
     job: fooapp
     namespace: default
     severity: warning
-- expr: |
+- annotations: {}
+  expr: |
     (
       sum(rate(promhttp_metric_handler_requests_total{namespace="default",job="fooapp",code=~"5.."}[5m]))
       /

diff --git a/examples/http-request-latency.yaml b/examples/http-request-latency.yaml
@@ -1,12 +1,14 @@
 alerts:
-- expr: |
+- annotations: {}
+  expr: |
     prometheus_http_request_duration_seconds:histogram_quantile > 0.500
   for: 5m
   labels:
     job: fooapp
     namespace: default
     severity: warning
-- expr: |
+- annotations: {}
+  expr: |
     prometheus_http_request_duration_seconds:histogram_quantile > 1.000
   for: 5m
   labels:

diff --git a/slo-libsonnet/error-burn.libsonnet b/slo-libsonnet/error-burn.libsonnet
@@ -5,6 +5,8 @@ local errors = import 'errors.libsonnet';
     local slo = {
       alertName: 'ErrorBudgetBurn',
       alertMessage: 'High error budget burn for %s (current value: {{ $value }})' % [std.strReplace(std.join(',', self.selectors), '"', '')],
+      alertLabels: {},
+      alertAnnotations: {},
       metric: error 'must set metric for error burn',
       recordingrule: '%s:burnrate%%s' % self.metric,  // double %% at the end as we template again further on
       selectors: [],
@@ -65,10 +67,10 @@ local errors = import 'errors.libsonnet';
           },
           labels: labels {
             severity: w.severity,
-          },
+          } + slo.alertLabels,
           annotations: {
             message: slo.alertMessage,
-          },
+          } + slo.alertAnnotations,
           'for': '%(for)s' % w,
         }
         for w in slo.windows

diff --git a/slo-libsonnet/errorlatency-burn.libsonnet b/slo-libsonnet/errorlatency-burn.libsonnet
@@ -5,6 +5,8 @@ local util = import '_util.libsonnet';
     local slo = {
       alertName: 'ErrorBudgetBurn',
       alertMessage: 'App is burning too much error budget',
+      alertLabels: {},
+      alertAnnotations: {},
       metric: error 'must set metric for error burn',  // This has to be a histogram metric without _bucket or _count
       recordingrule: '%s:burnrate%%s' % self.metric,  // double %% at the end as we template again further on
       selectors: error 'must set selectors for error burn',
@@ -71,10 +73,10 @@ local util = import '_util.libsonnet';
           ],
           labels: labels {
             severity: w.severity,
-          },
+          } + slo.alertLabels,
           annotations: {
             message: slo.alertMessage,
-          },
+          } + slo.alertAnnotations,
           'for': '%(for)s' % w,
         }
         for w in slo.windows

diff --git a/slo-libsonnet/errors.libsonnet b/slo-libsonnet/errors.libsonnet
@@ -4,6 +4,8 @@ local util = import '_util.libsonnet';
   errors(param):: {
     local slo = {
       metric: error 'must set metric for errors',
+      alertLabels: {},
+      alertAnnotations: {},
       selectors: [],
       errorSelectors: ['code=~"5.."'],
       rate: '5m',
@@ -30,7 +32,8 @@ local util = import '_util.libsonnet';
         'for': '5m',
         labels: labels {
           severity: severity.name,
-        },
+        } + slo.alertLabels,
+        annotations: slo.alertAnnotations,
       }
       for severity in [
         { name: 'warning', percent: slo.warning },

diff --git a/slo-libsonnet/latency-burn.libsonnet b/slo-libsonnet/latency-burn.libsonnet
@@ -10,7 +10,8 @@ local util = import '_util.libsonnet';
       // bucket. As recording rules rely on it.
       latencyTarget: error 'must set latencyTarget latency burn',
       latencyBudget: error 'must set latencyBudget latency burn',
-      labels: [],
+      alertLabels: {},
+      alertAnnotations: {},
       codeSelector: 'code',
       notErrorSelector: '%s!~"5.."' % slo.codeSelector,
     } + param,
@@ -81,10 +82,10 @@ local util = import '_util.libsonnet';
         ],
         labels: util.selectorsToLabels(rulesSelectors) {
           severity: 'critical',
-        },
+        } + slo.alertLabels,
         annotations: {
           message: 'High requests latency budget burn for %s (current value: {{ $value }})' % [std.strReplace(std.join(',', rulesSelectors), '"', '')],
-        },
+        } + slo.alertAnnotations,
       },
       {
         alert: slo.alertName,
@@ -116,10 +117,10 @@ local util = import '_util.libsonnet';
         ],
         labels: util.selectorsToLabels(rulesSelectors) {
           severity: 'warning',
-        },
+        } + slo.alertLabels,
         annotations: {
           message: 'High requests latency budget burn for %s (current value: {{ $value }})' % [std.strReplace(std.join(',', rulesSelectors), '"', '')],
-        },
+        } + slo.alertAnnotations,
       },
     ],
 

diff --git a/slo-libsonnet/latency.libsonnet b/slo-libsonnet/latency.libsonnet
@@ -4,6 +4,8 @@ local util = import '_util.libsonnet';
   latency(param):: {
     local slo = {
       metric: error 'must set metric for latency',
+      alertLabels: {},
+      alertAnnotations: {},
       selectors: error 'must set selectors for latency',
       quantile: error 'must set quantile for latency',
       labels: [],
@@ -37,7 +39,8 @@ local util = import '_util.libsonnet';
       'for': '5m',
       labels: labels {
         severity: 'warning',
-      },
+      } + slo.alertLabels,
+      annotations: slo.alertAnnotations,
     },
 
     alertCritical: {
@@ -47,7 +50,8 @@ local util = import '_util.libsonnet';
       'for': '5m',
       labels: labels {
         severity: 'critical',
-      },
+      } + slo.alertLabels,
+      annotations: slo.alertAnnotations,
     },
 
     grafana: {