nixos/prometheus/pushgateway: add module and test

nixos/modules/module-list.nix

@@ -491,6 +491,7 @@
   ./services/monitoring/prometheus/default.nix
   ./services/monitoring/prometheus/alertmanager.nix
   ./services/monitoring/prometheus/exporters.nix
+  ./services/monitoring/prometheus/pushgateway.nix
   ./services/monitoring/riemann.nix
   ./services/monitoring/riemann-dash.nix
   ./services/monitoring/riemann-tools.nix

nixos/modules/services/monitoring/prometheus/pushgateway.nix

@@ -0,0 +1,166 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+
+let
+  cfg = config.services.prometheus.pushgateway;
+
+  cmdlineArgs =
+       opt "web.listen-address" cfg.web.listen-address
+    ++ opt "web.telemetry-path" cfg.web.telemetry-path
+    ++ opt "web.external-url" cfg.web.external-url
+    ++ opt "web.route-prefix" cfg.web.route-prefix
+    ++ optional cfg.persistMetrics ''--persistence.file="/var/lib/${cfg.stateDir}/metrics"''
+    ++ opt "persistence.interval" cfg.persistence.interval
+    ++ opt "log.level" cfg.log.level
+    ++ opt "log.format" cfg.log.format
+    ++ cfg.extraFlags;
+
+  opt = k : v : optional (v != null) ''--${k}="${v}"'';
+
+in {
+  options = {
+    services.prometheus.pushgateway = {
+      enable = mkEnableOption "Prometheus Pushgateway";
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.prometheus-pushgateway;
+        defaultText = "pkgs.prometheus-pushgateway";
+        description = ''
+          Package that should be used for the prometheus pushgateway.
+        '';
+      };
+
+      web.listen-address = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = ''
+          Address to listen on for the web interface, API and telemetry.
+
+          <literal>null</literal> will default to <literal>:9091</literal>.
+        '';
+      };
+
+      web.telemetry-path = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = ''
+          Path under which to expose metrics.
+
+          <literal>null</literal> will default to <literal>/metrics</literal>.
+        '';
+      };
+
+      web.external-url = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = ''
+          The URL under which Pushgateway is externally reachable.
+        '';
+      };
+
+      web.route-prefix = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = ''
+          Prefix for the internal routes of web endpoints.
+
+          Defaults to the path of
+          <option>services.prometheus.pushgateway.web.external-url</option>.
+        '';
+      };
+
+      persistence.interval = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        example = "10m";
+        description = ''
+          The minimum interval at which to write out the persistence file.
+
+          <literal>null</literal> will default to <literal>5m</literal>.
+        '';
+      };
+
+      log.level = mkOption {
+        type = types.nullOr (types.enum ["debug" "info" "warn" "error" "fatal"]);
+        default = null;
+        description = ''
+          Only log messages with the given severity or above.
+
+          <literal>null</literal> will default to <literal>info</literal>.
+        '';
+      };
+
+      log.format = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        example = "logger:syslog?appname=bob&local=7";
+        description = ''
+          Set the log target and format.
+
+          <literal>null</literal> will default to <literal>logger:stderr</literal>.
+        '';
+      };
+
+      extraFlags = mkOption {
+        type = types.listOf types.str;
+        default = [];
+        description = ''
+          Extra commandline options when launching the Pushgateway.
+        '';
+      };
+
+      persistMetrics = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to persist metrics to a file.
+
+          When enabled metrics will be saved to a file called
+          <literal>metrics</literal> in the directory
+          <literal>/var/lib/pushgateway</literal>. The directory below
+          <literal>/var/lib</literal> can be set using
+          <option>services.prometheus.pushgateway.stateDir</option>.
+        '';
+      };
+
+      stateDir = mkOption {
+        type = types.str;
+        default = "pushgateway";
+        description = ''
+          Directory below <literal>/var/lib</literal> to store metrics.
+
+          This directory will be created automatically using systemd's
+          StateDirectory mechanism when
+          <option>services.prometheus.pushgateway.persistMetrics</option>
+          is enabled.
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = !hasPrefix "/" cfg.stateDir;
+        message =
+          "The option services.prometheus.pushgateway.stateDir" +
+          " shouldn't be an absolute directory." +
+          " It should be a directory relative to /var/lib.";
+      }
+    ];
+    systemd.services.pushgateway = {
+      wantedBy = [ "multi-user.target" ];
+      after    = [ "network.target" ];
+      serviceConfig = {
+        Restart  = "always";
+        DynamicUser = true;
+        ExecStart = "${cfg.package}/bin/pushgateway" +
+          optionalString (length cmdlineArgs != 0) (" \\\n  " +
+            concatStringsSep " \\\n  " cmdlineArgs);
+        StateDirectory = if cfg.persistMetrics then cfg.stateDir else null;
+      };
+    };
+  };
+}

nixos/tests/prometheus-2.nix

@@ -3,15 +3,29 @@ import ./make-test.nix {
 
   nodes = {
     one = { pkgs, ... }: {
+      environment.systemPackages = [ pkgs.jq ];
       services.prometheus2 = {
         enable = true;
-        scrapeConfigs = [{
-          job_name = "prometheus";
-          static_configs = [{
-            targets = [ "127.0.0.1:9090" ];
-            labels = { instance = "localhost"; };
-          }];
-        }];
+        scrapeConfigs = [
+          {
+            job_name = "prometheus";
+            static_configs = [
+              {
+                targets = [ "127.0.0.1:9090" ];
+                labels = { instance = "localhost"; };
+              }
+            ];
+          }
+          {
+            job_name = "pushgateway";
+            scrape_interval = "1s";
+            static_configs = [
+              {
+                targets = [ "127.0.0.1:9091" ];
+              }
+            ];
+          }
+        ];
         rules = [
           ''
             groups:
@@ -22,6 +36,12 @@ import ./make-test.nix {
           ''
         ];
       };
+      services.prometheus.pushgateway = {
+        enable = true;
+        persistMetrics = true;
+        persistence.interval = "1s";
+        stateDir = "prometheus-pushgateway";
+      };
     };
   };
 
@@ -30,5 +50,18 @@ import ./make-test.nix {
     $one->waitForUnit("prometheus2.service");
     $one->waitForOpenPort(9090);
     $one->succeed("curl -s http://127.0.0.1:9090/metrics");
+
+    # Let's test if pushing a metric to the pushgateway succeeds
+    # and whether that metric gets ingested by prometheus.
+    $one->waitForUnit("pushgateway.service");
+    $one->succeed(
+      "echo 'some_metric 3.14' | " .
+      "curl --data-binary \@- http://127.0.0.1:9091/metrics/job/some_job");
+    $one->waitUntilSucceeds(
+      "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=some_metric' " .
+      "| jq '.data.result[0].value[1]' | grep '\"3.14\"'");
+
+    # Let's test if the pushgateway persists metrics to the configured location.
+    $one->waitUntilSucceeds("test -e /var/lib/prometheus-pushgateway/metrics");
   '';
 }