beats: Include dashboards, add packetbeat module

nixos/modules/module-list.nix

@@ -392,6 +392,7 @@
   ./services/logging/logcheck.nix
   ./services/logging/logrotate.nix
   ./services/logging/logstash.nix
+  ./services/logging/packetbeat.nix
   ./services/logging/rsyslogd.nix
   ./services/logging/syslog-ng.nix
   ./services/logging/syslogd.nix

nixos/modules/services/logging/journalbeat.nix

@@ -101,6 +101,11 @@ in
             -path.data /var/lib/${cfg.stateDir}/data \
             -path.logs /var/lib/${cfg.stateDir}/logs'';
         Restart = "always";
+        # ElasticSearch takes a while to start, so if the beat runs on the
+        # same machine as ElasticSearch, the beat service becomes fragile and tends to time out.
+        # We cannot define the beat to have a dependency on the elasticsearch.service
+        # because it is not required to be on the same machine.
+        RestartSec = 3;
       };
     };
   };

nixos/modules/services/logging/packetbeat.nix

@@ -0,0 +1,112 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.packetbeat;
+
+  lt6 = builtins.compareVersions cfg.package.version "6" < 0;
+
+  packetbeatYml = pkgs.writeText "packetbeat.yml" ''
+    name: ${cfg.name}
+    tags: ${builtins.toJSON cfg.tags}
+
+    ${optionalString lt6 "packetbeat.cursor_state_file: /var/lib/${cfg.stateDir}/cursor-state"}
+
+    ${cfg.extraConfig}
+  '';
+
+in
+{
+  options = {
+
+    services.packetbeat = {
+
+      enable = mkEnableOption "packetbeat";
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.packetbeat;
+        defaultText = "pkgs.packetbeat";
+        example = literalExample "pkgs.packetbeat7";
+        description = ''
+          The packetbeat package to use
+        '';
+      };
+
+      name = mkOption {
+        type = types.str;
+        default = "packetbeat";
+        description = "Name of the beat";
+      };
+
+      tags = mkOption {
+        type = types.listOf types.str;
+        default = [];
+        description = "Tags to place on the shipped log messages";
+      };
+
+      stateDir = mkOption {
+        type = types.str;
+        default = "packetbeat";
+        description = ''
+          Directory below <literal>/var/lib/</literal> to store packetbeat's
+          own logs and other data. This directory will be created automatically
+          using systemd's StateDirectory mechanism.
+        '';
+      };
+
+      extraConfig = mkOption {
+        type = types.lines;
+        default = optionalString lt6 ''
+          packetbeat:
+            seek_position: cursor
+            cursor_seek_fallback: tail
+            write_cursor_state: true
+            cursor_flush_period: 5s
+            clean_field_names: true
+            convert_to_numbers: false
+            move_metadata_to_field: packet
+            default_type: packet
+        '';
+        description = "Any other configuration options you want to add";
+      };
+
+    };
+  };
+
+  config = mkIf cfg.enable {
+
+    assertions = [
+      {
+        assertion = !hasPrefix "/" cfg.stateDir;
+        message =
+          "The option services.packetbeat.stateDir shouldn't be an absolute directory." +
+          " It should be a directory relative to /var/lib/.";
+      }
+    ];
+
+    systemd.services.packetbeat = {
+      description = "Packetbeat log shipper";
+      wantedBy = [ "multi-user.target" ];
+      preStart = ''
+        mkdir -p ${cfg.stateDir}/data
+        mkdir -p ${cfg.stateDir}/logs
+      '';
+      serviceConfig = {
+        StateDirectory = cfg.stateDir;
+        ExecStart = ''
+          ${cfg.package}/bin/packetbeat run \
+            -c ${packetbeatYml} \
+            -path.data /var/lib/${cfg.stateDir}/data \
+            -path.logs /var/lib/${cfg.stateDir}/logs'';
+        Restart = "always";
+        # ElasticSearch takes a while to start, so if the beat runs on the
+        # same machine as ElasticSearch, the beat service becomes fragile and tends to time out.
+        # We cannot define the beat to have a dependency on the elasticsearch.service
+        # because it is not required to be on the same machine.
+        RestartSec = 3;
+      };
+    };
+  };
+}

nixos/tests/elk.nix

@@ -54,7 +54,22 @@ let
                   - paths: []
                     seek: cursor
                 '');
-              };
+                };
+
+              packetbeat =
+                let lt6 = builtins.compareVersion elk.packetbeat.version "6" < 0;
+                in {
+                  enable = true;
+                  package = elk.packetbeat;
+                  extraConfig = pkgs.lib.mkOptionDefault (''
+                    output.elasticsearch:
+                      hosts: [ "127.0.0.1:9200" ]
+                    packetbeat.flows:
+                      timeout: 30s
+                      period: 10s
+                    setup.dashboards.enabled: true
+                  '');
+                };
 
               logstash = {
                 enable = true;
@@ -173,6 +188,12 @@ let
               total_hits("Supercalifragilisticexpialidocious") + " | grep -v 0"
           )
 
+      with subtest("Packetbeat's sample dashboard is imported"):
+          one.wait_for_unit("packetbeat.service")
+          one.wait_until_succeeds(
+              "curl --silent --show-error http://localhost:5601/api/kibana/dashboards/export?dashboard=MySQL-Errors-ecs"
+          )
+
       with subtest("Elasticsearch-curator works"):
           one.systemctl("stop logstash")
           one.systemctl("start elasticsearch-curator")
@@ -189,12 +210,14 @@ in pkgs.lib.mapAttrs mkElkTest {
       logstash      = pkgs.logstash6;
       kibana        = pkgs.kibana6;
       journalbeat   = pkgs.journalbeat6;
+      packetbeat    = pkgs.packetbeat6;
     }
     else {
       elasticsearch = pkgs.elasticsearch6-oss;
       logstash      = pkgs.logstash6-oss;
       kibana        = pkgs.kibana6-oss;
       journalbeat   = pkgs.journalbeat6;
+      packetbeat    = pkgs.packetbeat6;
     };
   ELK-7 =
     if enableUnfree
@@ -203,11 +226,13 @@ in pkgs.lib.mapAttrs mkElkTest {
       logstash      = pkgs.logstash7;
       kibana        = pkgs.kibana7;
       journalbeat   = pkgs.journalbeat7;
+      packetbeat    = pkgs.packetbeat7;
     }
     else {
       elasticsearch = pkgs.elasticsearch7-oss;
       logstash      = pkgs.logstash7-oss;
       kibana        = pkgs.kibana7-oss;
       journalbeat   = pkgs.journalbeat7;
+      packetbeat    = pkgs.packetbeat7;
     };
 }

pkgs/misc/logging/beats/7.x.nix

@@ -1,4 +1,4 @@
-{ stdenv, lib, fetchFromGitHub, elk7Version, buildGoPackage, libpcap, systemd }:
+{ stdenv, lib, fetchFromGitHub, elk7Version, buildGoPackage, libpcap, systemd, python }:
 
 let beat = package : extraArgs : buildGoPackage (rec {
       name = "${package}-${version}";
@@ -38,6 +38,11 @@ in {
       your application processes, parse on the fly protocols like HTTP, MySQL,
       PostgreSQL, Redis or Thrift and correlate the messages into transactions.
     '';
+    preDistPhases = ["dashboardPhase"];
+    dashboardPhase = ''
+      cp -r ./go/src/github.com/elastic/beats/packetbeat/_meta/* $out/bin
+      ${python}/bin/python ./go/src/github.com/elastic/beats/libbeat/scripts/unpack_dashboards.py --transform encode --glob "$out/bin/kibana/7/dashboard/*.json"
+    '';
   };
   journalbeat7  = beat "journalbeat" {
     meta.description = ''
@@ -48,5 +53,10 @@ in {
     postFixup = let libPath = stdenv.lib.makeLibraryPath [ (lib.getLib systemd) ]; in ''
       patchelf --set-rpath ${libPath} "$out/bin/journalbeat"
     '';
+    preDistPhases = ["dashboardPhase"];
+    dashboardPhase = ''
+      cp -r ./go/src/github.com/elastic/beats/journalbeat/_meta/* $out/bin
+      ${python}/bin/python ./go/src/github.com/elastic/beats/libbeat/scripts/unpack_dashboards.py --transform encode --glob "$out/bin/kibana/7/dashboard/*.json"
+    '';
   };
 }