fix(chart): default prometheusrules

DoodleScheduling · Sep 22, 2023 · 29becc3 · 29becc3
1 parent e2ed0bc
commit 29becc3
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 8 deletions.
diff --git a/chart/saml-exporter/README.md b/chart/saml-exporter/README.md
@@ -10,13 +10,24 @@ To install the chart with the release name `saml-exporter`:
 helm upgrade saml-exporter --install oci://ghcr.io/doodlescheduling/charts/saml-exporter
 ```
 
-This command deploys the MongoDB Exporter with the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation.
+This command deploys the SAML Exporter with the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation.
 
 ## Using the Chart
 
 The chart comes with a ServiceMonitor (or PodMonitor) for use with the [Prometheus Operator](https://github.com/helm/charts/tree/master/stable/prometheus-operator).
-If you're not using the Prometheus Operator, you can disable the ServiceMonitor by setting `serviceMonitor.enabled` to `false` and instead
-populate the `podAnnotations` as below:
+The chart also bundles sane default PrometheusRules which alert regarding invalid metadata, expiring certificates and unavailable endpoints.
+Prometheus Operator support is disabled by default but may be enabled using:
+
+```
+podMonitor:
+  enabled: true
+
+prometheusRule: 
+  enabled: true
+```
+
+
+If you're not using the Prometheus Operator you may add pod annotations for scraping:
 
 ```yaml
 podAnnotations:

diff --git a/chart/saml-exporter/templates/prometheusrule.yaml b/chart/saml-exporter/templates/prometheusrule.yaml
@@ -2,7 +2,6 @@
 {{- $rulePrefix:= .Values.prometheusRule.rulePrefix }}
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
-metadata:
 metadata:
   name: {{ template "saml-exporter.fullname" . }}
   labels:
@@ -32,7 +31,7 @@ spec:
   - name: saml-exporter.rules
     rules:
     {{- if .Values.prometheusRule.alertOnReadErrors }}
-    - alert: '{{ printf "%s %s" $rulePrefix "X509CertificateReadErrors" | trim }}'
+    - alert: '{{ printf "%s%s" $rulePrefix "X509CertificateReadErrors" | trim }}'
       expr: delta(saml_x509_read_errors[15m]) > 0
       for: 5m
       labels:
@@ -48,7 +47,7 @@ spec:
         {{- end }}
     {{- end }}
     {{- if .Values.prometheusRule.alertOnMetadataErrors }}
-    - alert: '{{ printf "%s %s" $rulePrefix "MetadataError" | trim }}'
+    - alert: '{{ printf "%s%s" $rulePrefix "MetadataError" | trim }}'
       expr: delta(saml_metadata_errors[15m]) > 0
       for: 5m
       labels:
@@ -63,7 +62,7 @@ spec:
         {{- toYaml .Values.prometheusRule.alertExtraAnnotations | nindent 8 }}
         {{- end }}
     {{- end }}
-    - alert: '{{ printf "%s %s" $rulePrefix "X509CertificateRenewal" | trim }}'
+    - alert: '{{ printf "%s%s" $rulePrefix "X509CertificateRenewal" | trim }}'
       expr: count(saml_x509_cert_not_after) by (entityid, use) - count(((saml_x509_cert_not_after - time()) / 86400) < {{ .Values.prometheusRule.warningDaysLeft }}) by (entityid, use) == 0
       for: 15m
       labels:
@@ -77,7 +76,7 @@ spec:
         {{- if .Values.prometheusRule.alertExtraAnnotations }}
         {{- toYaml .Values.prometheusRule.alertExtraAnnotations | nindent 8 }}
         {{- end }}
-    - alert: '{{ printf "%s %s" $rulePrefix "X509CertificateExpiration" | trim }}'
+    - alert: '{{ printf "%s%s" $rulePrefix "X509CertificateExpiration" | trim }}'
       expr: count(saml_x509_cert_not_after) by (entityid, use) - count(((saml_x509_cert_not_after - time()) / 86400) < {{ .Values.prometheusRule.criticalDaysLeft }}) by (entityid, use) == 0
       for: 15m
       labels:
@@ -91,6 +90,20 @@ spec:
         {{- if .Values.prometheusRule.alertExtraAnnotations }}
         {{- toYaml .Values.prometheusRule.alertExtraAnnotations | nindent 8 }}
         {{- end }}
+    - alert: '{{ printf "%s%s" $rulePrefix "UnavailableMetadataEndpoint" | trim }}'
+      expr: sum(rate(http_client_request{job="{{ include "saml-exporter.fullname" . }}", code!~"^2.*"}[5m])) > 0
+      for: 3m
+      labels:
+        severity: {{ .Values.prometheusRule.unavailableMetadataEndpointSeverity }}
+        {{- if .Values.prometheusRule.alertExtraLabels }}
+        {{- toYaml .Values.prometheusRule.alertExtraLabels | nindent 8 }}
+        {{- end }}
+      annotations:
+        summary: A SAML metadata endpoint is unreachable
+        description: A SAML metadata endpoint "{{ "{{" }} $labels.host {{ "}}" }}" is not responding with a 2xx code for over 5m.
+        {{- if .Values.prometheusRule.alertExtraAnnotations }}
+        {{- toYaml .Values.prometheusRule.alertExtraAnnotations | nindent 8 }}
+        {{- end }}
   {{- end }}
 {{- range .Values.prometheusRule.extraAlertGroups }}
   - {{ tpl (toYaml .) $ | indent 4 | trim }}

diff --git a/chart/saml-exporter/values.yaml b/chart/saml-exporter/values.yaml
@@ -153,6 +153,7 @@ prometheusRule:
   metadataErrorsSeverity: warning
   certificateRenewalsSeverity: warning
   certificateExpirationsSeverity: critical
+  unavailableMetadataEndpointSeverity: critical
   warningDaysLeft: 28
   criticalDaysLeft: 14