Merge pull request #37 from livepeer/ya/graf-prom-alerts

[READY] adding alerts channels to the dashboards
livepeer · Oct 26, 2020 · a823ea6 · a823ea6
2 parents 986214d + ff44922
commit a823ea6
Show file tree

Hide file tree

Showing 4 changed files with 596 additions and 7 deletions.
diff --git a/monitoring/Dockerfile b/monitoring/Dockerfile
@@ -94,6 +94,7 @@ COPY ./grafana/livepeer_payments_overview.json $GF_PATHS_DATA/dashboards/livepee
 COPY ./grafana/nvidia-gpu.json $GF_PATHS_DATA/dashboards/nvidia-gpu.json
 COPY ./grafana/kubernetes/lpDebugger.json $GF_PATHS_DATA/dashboards/kubernetes/lpDebugger.json
 COPY ./grafana/kubernetes/livepeer_overview.json $GF_PATHS_DATA/dashboards/kubernetes/k8s_livepeer_overview.json
+COPY ./grafana/kubernetes/node_alerts.json $GF_PATHS_DATA/dashboards/kubernetes/ndoe_alerts.json
 
 
 EXPOSE 3000

diff --git a/monitoring/config-generator/src/generate.js b/monitoring/config-generator/src/generate.js
@@ -92,7 +92,11 @@ function generate() {
         describe: 'the webhook for the Discord notification channel',
         type: 'string',
         default: null
-      }
+      },
+      'grafana-alerts': {
+        describe: 'enables grafana alerts to hook up to the prometheus alertmanager',
+        type: 'boolean'
+      },
     }).argv
 
   if (argv.help || argv.version) {
@@ -107,8 +111,8 @@ function generate() {
   saveYaml('/etc/prometheus', 'alertmanager.yml', getAlertManagerConfig(argv))
   saveYaml('/etc/prometheus', 'rules.yml', getRules(argv.alertGroups))
   saveYaml('/etc/prometheus', 'prometheus.yml', promConfig)
-  if (argv['discord-webhook']) {
-    saveYaml('/etc/grafana/provisioning/notifiers', 'discord.yml', grafanaNotificationChannelsConfig(argv))
+  if (argv['grafana-alerts']) {
+    saveYaml('/etc/grafana/provisioning/notifiers', 'notifiers.yml', grafanaNotificationChannelsConfig(argv))
   }
   fs.writeFileSync(
     path.join('/etc/supervisor.d', 'supervisord.conf'),
@@ -719,8 +723,33 @@ function grafanaNotificationChannelsConfig(params) {
         content: '',
         url: params['discord-webhook']
       }
+    },{
+      name: 'prom-alertmanager',
+      type: 'prometheus-alertmanager',
+      uid: 'prom-alertmanager',
+      org_name: 'Main Org.',
+      is_default: true, 
+      settings: {
+        url: 'http://localhost:9093'
+      }
     }]
   }
 
+  // direct pagerDuty integration
+  // NOTE: these are not activated into the dashboards by default right now. However
+  // the alertmanager is added by default which forwards the alerts to pagerDuty anyway
+  if (params['pagerduty-service-key']) {
+    obj.notifiers.push({
+      name: 'pagerDuty',
+      type: 'pagerduty',
+      uid: 'pagerDuty',
+      org_name: 'Main Org.',
+      is_default: true, 
+      secure_settings: {
+        integrationKey: params['pagerduty-service-key']
+      }
+    })
+  }
+
   return obj
 }
diff --git a/monitoring/grafana/kubernetes/livepeer_overview.json b/monitoring/grafana/kubernetes/livepeer_overview.json
@@ -919,7 +919,10 @@
         "noDataState": "no_data",
         "notifications": [
           {
-            "uid": "cG8gVlDGk"
+            "uid": "discord"
+          },
+          {
+            "uid": "prom-alertmanger"
           }
         ]
       },
@@ -1061,7 +1064,10 @@
         "noDataState": "no_data",
         "notifications": [
           {
-            "uid": "cG8gVlDGk"
+            "uid": "discord"
+          },
+          {
+            "uid": "prom-alertmanger"
           }
         ]
       },
@@ -1210,7 +1216,10 @@
         "noDataState": "no_data",
         "notifications": [
           {
-            "uid": "cG8gVlDGk"
+            "uid": "discord"
+          },
+          {
+            "uid": "prom-alertmanger"
           }
         ]
       },
@@ -1837,7 +1846,10 @@
         "noDataState": "no_data",
         "notifications": [
           {
-            "uid": "cG8gVlDGk"
+            "uid": "discord"
+          },
+          {
+            "uid": "prom-alertmanger"
           }
         ]
       },