Monitoring and other improvements

flake/nixosModules/module-cardano-node-group.nix

@@ -11,9 +11,8 @@
 #   * This is a cardano-node add-on to the upstream cardano-node nixos service module
 #   * This module assists with group deployments
 #   * The upstream cardano-node nixos service module should still be imported separately
-flake: {
-  flake.nixosModules.module-cardano-node-group = {
-    config,
+{moduleWithSystem, ...}: {
+  flake.nixosModules.module-cardano-node-group = moduleWithSystem ({config, ...}: nixos @ {
     pkgs,
     lib,
     name,
@@ -25,14 +24,14 @@ flake: {
     inherit (types) bool float int;
     inherit (nodeResources) cpuCount memMiB;
 
-    inherit (config.cardano-parts.cluster.group.meta) environmentName;
-    inherit (config.cardano-parts.perNode.lib) cardanoLib;
-    inherit (config.cardano-parts.perNode.meta) cardanoNodePort cardanoNodePrometheusExporterPort hostAddr nodeId;
-    inherit (config.cardano-parts.perNode.pkgs) cardano-node-pkgs;
+    inherit (nixos.config.cardano-parts.cluster.group.meta) environmentName;
+    inherit (nixos.config.cardano-parts.perNode.lib) cardanoLib;
+    inherit (nixos.config.cardano-parts.perNode.meta) cardanoNodePort cardanoNodePrometheusExporterPort hostAddr nodeId;
+    inherit (nixos.config.cardano-parts.perNode.pkgs) cardano-node-pkgs;
     inherit (cardanoLib.environments.${environmentName}.nodeConfig) ByronGenesisFile;
     inherit ((fromJSON (readFile ByronGenesisFile)).protocolConsts) protocolMagic;
 
-    cfg = config.services.cardano-node;
+    cfg = nixos.config.services.cardano-node;
   in {
     # Leave the import of the upstream cardano-node service for
     # cardano-parts consuming repos so that service import can be customized.
@@ -41,7 +40,7 @@ flake: {
     # perNode nixos options as this leads to infinite recursion.
     #
     # imports = [
-    #   config.cardano-parts.perNode.pkgs.cardano-node-service;
+    #   nixos.config.cardano-parts.perNode.pkgs.cardano-node-service;
     # ];
 
     options = {
@@ -69,7 +68,14 @@ flake: {
     };
 
     config = {
-      environment.systemPackages = [cardano-node-pkgs.cardano-cli];
+      environment.systemPackages = [
+        config.cardano-parts.pkgs.bech32
+        cardano-node-pkgs.cardano-cli
+        config.cardano-parts.pkgs.db-analyser
+        config.cardano-parts.pkgs.db-truncater
+        config.cardano-parts.pkgs.db-synthesizer
+      ];
+
       environment.variables = {
         CARDANO_NODE_NETWORK_ID = toString protocolMagic;
         CARDANO_NODE_SOCKET_PATH = cfg.socketPath 0;
@@ -184,5 +190,5 @@ flake: {
         }
       ];
     };
-  };
+  });
 }

flake/nixosModules/module-basic.nix → flake/nixosModules/profile-basic.nix

@@ -1,4 +1,4 @@
-# nixosModule: module-basic
+# nixosModule: profile-basic
 #
 # TODO: Move this to a docs generator
 #
@@ -11,7 +11,7 @@
   moduleWithSystem,
   ...
 }: {
-  flake.nixosModules.module-basic = moduleWithSystem ({system}: {
+  flake.nixosModules.profile-basic = moduleWithSystem ({system}: {
     name,
     pkgs,
     ...
@@ -85,7 +85,11 @@
     };
 
     services = {
-      chrony.enable = true;
+      chrony = {
+        enable = true;
+        extraConfig = "rtcsync";
+      };
+
       cron.enable = true;
       fail2ban.enable = true;
       openssh = {

flake/nixosModules/module-common.nix → flake/nixosModules/profile-common.nix

@@ -1,4 +1,4 @@
-# nixosModule: module-common
+# nixosModule: profile-common
 #
 # TODO: Move this to a docs generator
 #
@@ -17,7 +17,7 @@
   moduleWithSystem,
   ...
 }: {
-  flake.nixosModules.module-common = moduleWithSystem ({system}: {
+  flake.nixosModules.profile-common = moduleWithSystem ({system}: {
     config,
     lib,
     ...

flake/nixosModules/profile-grafana-agent.nix

@@ -0,0 +1,236 @@
+# nixosModule: profile-grafana-agent
+#
+# TODO: Move this to a docs generator
+#
+# Attributes available on nixos module import:
+#
+# Tips:
+#
+{
+  flake.nixosModules.profile-grafana-agent = {
+    config,
+    lib,
+    name,
+    ...
+  }:
+    with builtins;
+    with lib; let
+      inherit (config.cardano-parts.perNode.meta) cardanoNodePrometheusExporterPort hostAddr;
+      inherit (groupCfg) groupName groupFlake;
+      inherit (groupCfg.meta) environmentName;
+
+      groupCfg = config.cardano-parts.cluster.group;
+      groupOutPath = groupFlake.self.outPath;
+
+      pathPrefix = "${groupOutPath}/secrets/monitoring/";
+      trimStorePrefix = path: last (split "/nix/store/[^/]+/" path);
+      verboseTrace = key: traceVerbose ("${name}: using " + (trimStorePrefix key));
+
+      mkSopsSecret = secretsFile: {
+        ${secretsFile} = verboseTrace (pathPrefix + secretsFile + ".enc") {
+          sopsFile = pathPrefix + secretsFile + ".enc";
+        };
+      };
+    in {
+      systemd.services.grafana-agent = {
+        after = ["sops-secrets.service"];
+        wants = ["sops-secrets.service"];
+      };
+
+      sops.secrets =
+        mkSopsSecret "grafana-agent-metrics-url"
+        // mkSopsSecret "grafana-agent-metrics-username"
+        // mkSopsSecret "grafana-agent-metrics-password";
+
+      services.grafana-agent = {
+        enable = true;
+
+        credentials = let
+          sopsPath = name: config.sops.secrets.${name}.path;
+        in {
+          # Loaded as env vars
+          METRICS_REMOTE_WRITE_URL = sopsPath "grafana-agent-metrics-url";
+          METRICS_REMOTE_WRITE_USERNAME = sopsPath "grafana-agent-metrics-username";
+
+          # Loaded as files
+          metrics_remote_write_password = sopsPath "grafana-agent-metrics-password";
+        };
+
+        extraFlags = [
+          "-disable-reporting"
+        ];
+
+        settings = let
+          metrics-client = {
+            basic_auth = {
+              password_file = "\${CREDENTIALS_DIRECTORY}/metrics_remote_write_password";
+              username = "\${METRICS_REMOTE_WRITE_USERNAME}";
+            };
+            url = "\${METRICS_REMOTE_WRITE_URL}";
+          };
+
+          relabelConfig-agent_hostname-instance = [
+            {
+              action = "replace";
+              source_labels = ["agent_hostname"];
+              target_label = "instance";
+            }
+            {
+              action = "labeldrop";
+              regex = "^agent_hostname$";
+            }
+          ];
+        in {
+          integrations = {
+            agent = {
+              enabled = true;
+              metric_relabel_configs = [
+                {
+                  action = "keep";
+                  regex = "^prometheus_target_.*|prometheus_sd_discovered_targets|agent_build.*|agent_wal_samples_appended_total|process_start_time_seconds$";
+                  source_labels = ["__name__"];
+                }
+              ];
+
+              relabel_configs =
+                relabelConfig-agent_hostname-instance
+                ++ [
+                  {
+                    action = "replace";
+                    replacement = "integrations/agent-check";
+                    target_label = "job";
+                  }
+                ];
+            };
+
+            node_exporter = {
+              set_collectors = [
+                "boottime"
+                "conntrack"
+                "cpu"
+                "diskstats"
+                "filefd"
+                "filesystem"
+                "loadavg"
+                "meminfo"
+                "netdev"
+                "netstat"
+                "os"
+                "sockstat"
+                "softnet"
+                "stat"
+                "time"
+                "timex"
+                "uname"
+                "vmstat"
+              ];
+
+              relabel_configs = relabelConfig-agent_hostname-instance;
+
+              metric_relabel_configs = [
+                {
+                  action = "keep";
+                  source_labels = ["__name__"];
+                  regex =
+                    "^"
+                    + concatMapStringsSep "|" (s: "(${s})") [
+                      "node_boot_time_seconds"
+                      "node_context_switches_total"
+                      "node_cpu_seconds_total"
+                      "node_disk_io_time_(seconds|weighted_seconds)_total"
+                      "node_disk_(read|reads|writes|written)_.*"
+                      "node_filefd_.*"
+                      "node_filesystem_.*"
+                      "node_intr_total"
+                      "node_load([[:digit:]]+)"
+                      "node_memory_(Active(|_file|_anon)|Inactive(|_file|_anon))_bytes"
+                      "node_memory_Anon(HugePages|Pages)_bytes"
+                      "node_memory_(Bounce|Committed_AS|CommitLimit|Dirty|Mapped)_bytes"
+                      "node_memory_DirectMap(1G|2M|4k)_bytes"
+                      "node_memory_HugePages_(Free|Rsvd|Surp|Total)"
+                      "node_memory_Hugepagesize_bytes"
+                      "node_memory_(Mem(Available|Free|Total)|Buffers|Cached|SwapTotal)_bytes"
+                      "node_memory_Shmem(|HugPages|PmdMapped)_bytes"
+                      "node_memory_S(Reclaimable|Unreclaim)_bytes"
+                      "node_memory_Vmalloc(Chunk|Total|Used)_bytes"
+                      "node_memory_Writeback(|Tmp)_bytes"
+                      "node_netstat_Icmp6_(InErrors|InMsgs|OutMsgs)"
+                      "node_netstat_Icmp_(InErrors|InMsgs|OutMsgs)"
+                      "node_netstat_IpExt_(InOctets|OutOctets)"
+                      "node_netstat_TcpExt_(ListenDrops|ListenOverflows|TCPSynRetrans)"
+                      "node_netstat_Tcp_(InErrs|InSegs|OutRsts|OutSegs|RetransSegs)"
+                      "node_netstat_Udp6_(InDatagrams|InErrors|NoPorts|OutDatagrams|RcvbufErrors|SndbufErrors)"
+                      "node_netstat_Udp_(InDatagrams|InErrors|NoPorts|OutDatagrams|RcvbufErrors|SndbufErrors)"
+                      "node_netstat_UdpLite_InErrors"
+                      "node_network_.*"
+                      "node_nf_conntrack_entries(|_limit)"
+                      "node_os_info"
+                      "node_sockstat_(FRAG|FRAG6|RAW|RAW6)_inuse"
+                      "node_sockstat_sockets_used"
+                      "node_sockstat_TCP6_inuse"
+                      "node_sockstat_TCP_(alloc|inuse|mem|orphan|tw)"
+                      "node_sockstat_(TCP|UDP)_mem_bytes"
+                      "node_sockstat_UDP_mem"
+                      "node_sockstat_(UDP|UDP6|UDPLITE|UDPLITE6)_inuse"
+                      "node_softnet_(dropped|processed|times_squeezed)_total"
+                      "node_timex_(estimated_error|maxerror|offset)_seconds"
+                      "node_timex_sync_status"
+                      "node_time_zone_offset_seconds"
+                      "node_uname_info"
+                      "node_vmstat_(pgmajfault|pgfault|pgpgin|pgpgout|pswpin|pswpout|oom_kill)"
+                    ]
+                    + "$";
+                }
+                {
+                  action = "drop";
+                  source_labels = ["__name__"];
+                  regex = "^node_filesystem_readonly$";
+                }
+                {
+                  # filesystem collector
+                  action = "keep";
+                  source_labels = ["mountpoint"];
+                  regex = "^|/|/boot|/state|/home|/nix$";
+                }
+                {
+                  # cpu collector
+                  action = "keep";
+                  source_labels = ["mode"];
+                  regex = "^|system|user|iowait|steal|idle$";
+                }
+              ];
+            };
+
+            prometheus_remote_write = [metrics-client];
+          };
+
+          metrics = {
+            configs = [
+              {
+                name = "integrations";
+                remote_write = [metrics-client];
+
+                scrape_configs = [
+                  (mkIf (config.services ? cardano-node && config.services.cardano-node.enable) {
+                    job_name = "integrations/cardano-node";
+                    static_configs = [
+                      {
+                        targets = ["${hostAddr}:${toString cardanoNodePrometheusExporterPort}"];
+                        labels = {
+                          instance = name;
+                          environment = environmentName;
+                          group = groupName;
+                        };
+                      }
+                    ];
+                  })
+                ];
+              }
+            ];
+            global.scrape_interval = "1m";
+            wal_directory = "\${STATE_DIRECTORY}/grafana-agent-wal";
+          };
+        };
+      };
+    };
+}