Adds victoria-metrics declarative alerts

input-output-hk · Jun 21, 2022 · 99b6e38 · 99b6e38
1 parent fc9d4b2
commit 99b6e38
Show file tree

Hide file tree

Showing 15 changed files with 381 additions and 16 deletions.
diff --git a/profiles/monitoring.nix b/profiles/monitoring.nix
@@ -152,21 +152,7 @@ in {
       enable = true;
       externalUrl = "https://monitoring.${domain}/vmalert";
       httpPathPrefix = "/vmalert";
-      rules = [
-        {
-          name = "alerting-pipeline";
-          rules = [{
-            alert = "DeadMansSnitch";
-            expr = "vector(1)";
-            labels = { severity = "critical"; };
-            annotations = {
-              summary = "Alerting DeadMansSnitch.";
-              description =
-                "This is a DeadMansSnitch meant to ensure that the entire Alerting pipeline is functional.";
-            };
-          }];
-        }
-      ];
+      rules = (import ./monitoring/alerts/alerts.nix "https://monitoring.${domain}").groups;
     };
 
     loki = { enable = true; };
@@ -209,7 +195,7 @@ in {
 
         dashboards = [{
           name = "provisioned";
-          options.path = ./monitoring;
+          options.path = ./monitoring/dashboards;
         }];
       };
 

diff --git a/profiles/monitoring/alerts/alerts.nix b/profiles/monitoring/alerts/alerts.nix
@@ -0,0 +1,5 @@
+grafanaUrl: {
+  groups =
+    (import ./deadmans-snitch.nix grafanaUrl).groups ++
+    (import ./victoria-metrics.nix grafanaUrl).groups;
+}
diff --git a/profiles/monitoring/alerts/deadmans-snitch.nix b/profiles/monitoring/alerts/deadmans-snitch.nix
@@ -0,0 +1,17 @@
+grafanaUrl: {
+  groups = [
+    {
+      name = "alerting-pipeline";
+      rules = [{
+        alert = "DeadMansSnitch";
+        expr = "vector(1)";
+        labels = { severity = "critical"; };
+        annotations = {
+          summary = "Alerting DeadMansSnitch.";
+          description =
+            "This is a DeadMansSnitch meant to ensure that the entire Alerting pipeline is functional.";
+        };
+      }];
+    }
+  ];
+}