imp: add custom metrics grafana-agent scrape

input-output-hk · May 9, 2024 · 436efca · 436efca
1 parent 06239fc
commit 436efca
Show file tree

Hide file tree

Showing 2 changed files with 49 additions and 32 deletions.
diff --git a/flake/nixosModules/profile-cardano-node-custom-metrics.nix b/flake/nixosModules/profile-cardano-node-custom-metrics.nix
@@ -16,8 +16,8 @@
     lib,
     ...
   }: let
-    inherit (lib) mkOption;
-    inherit (lib.types) port str;
+    inherit (lib) mkIf mkOption;
+    inherit (lib.types) bool port str;
     inherit (perNodeCfg.meta) cardanoNodePort hostAddr;
     inherit (perNodeCfg.pkgs) cardano-cli;
 
@@ -31,13 +31,25 @@
         description = "The default netdata statsd listening binding for udp and tcp.";
       };
 
+      enableFilter = mkOption {
+        type = bool;
+        default = true;
+        description = "Whether to filter netdata metrics exported to prometheus.";
+      };
+
       filter = mkOption {
         type = str;
         default = "statsd_cardano*";
         description = "The default netdata prometheus metrics exporter filter.";
       };
 
       port = mkOption {
+        type = port;
+        default = 19999;
+        description = "The default netdata listening port.";
+      };
+
+      statsdPort = mkOption {
         type = port;
         default = 8125;
         description = "The default netdata statsd listening port.";
@@ -47,14 +59,22 @@
     config = {
       services.netdata = {
         enable = true;
-        configText = ''
-          [statsd]
-            bind to = udp:${cfg.address} tcp:${cfg.address}
-            default port = ${toString cfg.port}
 
-          [prometheus:exporter]
-            send charts matching = ${cfg.filter}
-        '';
+        config = {
+          web."default port" = cfg.port;
+
+          statsd = {
+            "bind to" = "udp:${cfg.address} tcp:${cfg.address}";
+            "default port" = cfg.statsdPort;
+          };
+        };
+
+        configDir = mkIf cfg.enableFilter {
+          "exporting.conf" = pkgs.writeText "exporting.conf" ''
+            [prometheus:exporter]
+              send charts matching = ${cfg.filter}
+          '';
+        };
       };
 
       systemd.services.cardano-node-custom-metrics = {
@@ -76,12 +96,12 @@
               UDP=
             fi
 
-            echo "Pushing statsd metrics to port: ${toString cfg.port}; udp=$UDP"
+            echo "Pushing statsd metrics to port: ${toString cfg.statsdPort}; udp=$UDP"
 
             while [ -n "''${1}" ]; do
               printf "%s\n" "''${1}"
               shift
-            done | ncat "''${UDP}" --send-only ${cfg.address} ${toString cfg.port} || return 1
+            done | ncat "''${UDP}" --send-only ${cfg.address} ${toString cfg.statsdPort} || return 1
 
             return 0
           }
@@ -96,8 +116,8 @@
             CARDANO_PING_LATENCY=$(jq '.pongs[-1].sample * 1000' <<< "$CARDANO_PING_OUPUT")
           fi
 
-          echo "cardano_ping_latency_ms:''${CARDANO_PING_LATENCY}|g"
-          statsd "cardano_ping_latency_ms:''${CARDANO_PING_LATENCY}|g"
+          echo "cardano.node_ping_latency_ms:''${CARDANO_PING_LATENCY}|g"
+          statsd "cardano.node_ping_latency_ms:''${CARDANO_PING_LATENCY}|g"
         '';
       };
 

diff --git a/flake/nixosModules/profile-grafana-agent.nix b/flake/nixosModules/profile-grafana-agent.nix
@@ -393,26 +393,23 @@ flake: {
                         ];
                       })
 
-                      # TODO: Uncomment once netdata filter issue is fixed
-                      # Ref: https://github.com/netdata/netdata/issues/17620
-                      #
                       # Metrics exporter: cardano-node-custom-metrics
-                      # (mkIf (cfgSvc ? cardano-node-custom-metrics && cfgSvc.netdata.enable) {
-                      #   job_name = "integrations/cardano-node-custom-metrics";
-                      #   metrics_path = "/api/v1/allmetrics?filter=statsd_cardano*&format=prometheus";
-                      #   params = {
-                      #     format = ["prometheus"];
-                      #     # Filtering here won't work as grafana-agent encodes the pattern match.
-                      #     # Filtering can be configured from the module with the filter option.
-                      #     # filter = ["statsd_cardano*"];
-                      #   };
-                      #   static_configs = [
-                      #     {
-                      #       inherit labels;
-                      #       targets = ["${cfgSvc.cardano-node-custom-metrics.address}:${toString cfgSvc.cardano-node-custom-metrics.port}"];
-                      #     }
-                      #   ];
-                      # })
+                      (mkIf (cfgSvc ? cardano-node-custom-metrics && cfgSvc.netdata.enable) {
+                        job_name = "integrations/cardano-node-custom-metrics";
+                        metrics_path = "/api/v1/allmetrics";
+                        params = {
+                          format = ["prometheus"];
+                          # Filtering here won't work as grafana-agent encodes the pattern match.
+                          # Filtering can be configured from the module with the `enableFilter` and `filter` options.
+                          # filter = ["statsd_cardano*"];
+                        };
+                        static_configs = [
+                          {
+                            inherit labels;
+                            targets = ["${cfgSvc.cardano-node-custom-metrics.address}:${toString cfgSvc.cardano-node-custom-metrics.port}"];
+                          }
+                        ];
+                      })
                     ]
                     # Metrics exporter: cardano-node
                     ++ optionals (cfgSvc ? cardano-node && cfgSvc.cardano-node.enable)