Add event.dataset to filebeat (#9457)

This allows 6.6 / 6.7 data to be compatible with 7.x.

CHANGELOG.asciidoc

@@ -95,8 +95,8 @@ https://github.com/elastic/beats/compare/v6.5.0...6.x[Check the HEAD diff]
 - Added `detect_null_bytes` selector to detect null bytes from a io.reader. {pull}9210[9210]
 - Added `syslog_host` variable to HAProxy module to allow syslog listener to bind to configured host. {pull}9366[9366]
 - Added support on Traefik for Common Log Format and Combined Log Format mixed which is the default Traefik format {issue}8015[8015] {issue}6111[6111] {pull}8768[8768].
-
 - Allow to force CRI format parsing for better performance {pull}8424[8424]
+- Add event.dataset to module events. {pull}9457[9457]
 
 *Heartbeat*
 

filebeat/_meta/fields.common.yml

@@ -56,6 +56,10 @@
       description: >
         The Filebeat fileset that generated this event.
 
+    - name: event.dataset
+      description: >
+        The Filebeat dataset that generated this event.
+
     - name: syslog.facility
       type: long
       required: false

filebeat/channel/factory.go

@@ -107,6 +107,9 @@ func (f *OutletFactory) Create(p beat.Pipeline, cfg *common.Config, dynFields *c
 	if len(fields) > 0 {
 		fields = common.MapStr{
 			"fileset": fields,
+			"event": common.MapStr{
+				"dataset": config.Module + "." + config.Fileset,
+			},
 		}
 	}
 	if config.Type != "" {

filebeat/docs/fields.asciidoc

@@ -3006,6 +3006,14 @@ The Filebeat module that generated this event.
 The Filebeat fileset that generated this event.
 
 
+--
+
+*`event.dataset`*::
++
+--
+The Filebeat dataset that generated this event.
+
+
 --
 
 *`syslog.facility`*::

filebeat/module/apache2/access/test/test.log-expected.json

@@ -8,6 +8,7 @@
         "apache2.access.response_code": "404", 
         "apache2.access.url": "/favicon.ico", 
         "apache2.access.user_name": "-", 
+        "event.dataset": "apache2.access", 
         "fileset.module": "apache2", 
         "fileset.name": "access", 
         "input.type": "log", 
@@ -33,6 +34,7 @@
         "apache2.access.user_agent.os_minor": "12", 
         "apache2.access.user_agent.os_name": "Mac OS X", 
         "apache2.access.user_name": "-", 
+        "event.dataset": "apache2.access", 
         "fileset.module": "apache2", 
         "fileset.name": "access", 
         "input.type": "log", 
@@ -44,6 +46,7 @@
         "apache2.access.remote_ip": "::1", 
         "apache2.access.response_code": "408", 
         "apache2.access.user_name": "-", 
+        "event.dataset": "apache2.access", 
         "fileset.module": "apache2", 
         "fileset.name": "access", 
         "input.type": "log", 
@@ -68,6 +71,7 @@
         "apache2.access.user_agent.os_name": "Windows 7", 
         "apache2.access.user_agent.patch": "a2", 
         "apache2.access.user_name": "-", 
+        "event.dataset": "apache2.access", 
         "fileset.module": "apache2", 
         "fileset.name": "access", 
         "input.type": "log", 

filebeat/module/apache2/error/test/test.log-expected.json

@@ -4,6 +4,7 @@
         "apache2.error.client": "192.168.33.1", 
         "apache2.error.level": "error", 
         "apache2.error.message": "File does not exist: /var/www/favicon.ico", 
+        "event.dataset": "apache2.error", 
         "fileset.module": "apache2", 
         "fileset.name": "error", 
         "input.type": "log", 
@@ -16,6 +17,7 @@
         "apache2.error.message": "AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", 
         "apache2.error.module": "core", 
         "apache2.error.pid": "11379", 
+        "event.dataset": "apache2.error", 
         "fileset.module": "apache2", 
         "fileset.name": "error", 
         "input.type": "log", 
@@ -30,6 +32,7 @@
         "apache2.error.module": "core", 
         "apache2.error.pid": "35708", 
         "apache2.error.tid": "4328636416", 
+        "event.dataset": "apache2.error", 
         "fileset.module": "apache2", 
         "fileset.name": "error", 
         "input.type": "log", 

filebeat/module/auditd/log/test/test.log-expected.json

@@ -11,6 +11,7 @@
         "auditd.log.ses": "4294967295", 
         "auditd.log.src": "192.168.2.0", 
         "auditd.log.src_prefixlen": "24", 
+        "event.dataset": "auditd.log", 
         "fileset.module": "auditd", 
         "fileset.name": "log", 
         "input.type": "log", 
@@ -45,6 +46,7 @@
         "auditd.log.syscall": "44", 
         "auditd.log.tty": "(none)", 
         "auditd.log.uid": "0", 
+        "event.dataset": "auditd.log", 
         "fileset.module": "auditd", 
         "fileset.name": "log", 
         "input.type": "log", 

filebeat/module/elasticsearch/audit/test/test.log-expected.json

@@ -6,6 +6,7 @@
         "elasticsearch.audit.origin_address": "147.107.128.77", 
         "elasticsearch.audit.principal": "i030648", 
         "elasticsearch.audit.uri": "/_xpack/security/_authenticate", 
+        "event.dataset": "elasticsearch.audit", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "audit", 
         "input.type": "log", 
@@ -22,6 +23,7 @@
         "elasticsearch.audit.principal": "rado", 
         "elasticsearch.audit.uri": "/_xpack/security/_authenticate", 
         "elasticsearch.node.name": "v_VJhjV", 
+        "event.dataset": "elasticsearch.audit", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "audit", 
         "input.type": "log", 
@@ -39,6 +41,7 @@
         "elasticsearch.audit.origin_type": "local_node", 
         "elasticsearch.audit.principal": "_xpack_security", 
         "elasticsearch.audit.request": "ClearScrollRequest", 
+        "event.dataset": "elasticsearch.audit", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "audit", 
         "input.type": "log", 
@@ -54,6 +57,7 @@
         "elasticsearch.audit.origin_address": "172.22.0.3", 
         "elasticsearch.audit.uri": "/_xpack/security/_authenticate", 
         "elasticsearch.node.name": "v_VJhjV", 
+        "event.dataset": "elasticsearch.audit", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "audit", 
         "input.type": "log", 
@@ -69,6 +73,7 @@
         "elasticsearch.audit.origin_address": "147.107.128.77", 
         "elasticsearch.audit.principal": "N078801", 
         "elasticsearch.audit.uri": "/_xpack/security/_authenticate", 
+        "event.dataset": "elasticsearch.audit", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "audit", 
         "input.type": "log", 
@@ -86,6 +91,7 @@
         "elasticsearch.audit.origin_type": "rest", 
         "elasticsearch.audit.principal": "_anonymous", 
         "elasticsearch.audit.request": "MainRequest", 
+        "event.dataset": "elasticsearch.audit", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "audit", 
         "input.type": "log", 
@@ -103,6 +109,7 @@
         "elasticsearch.audit.request_body": "body", 
         "elasticsearch.audit.uri": "/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip", 
         "elasticsearch.node.name": "v_VJhjV", 
+        "event.dataset": "elasticsearch.audit", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "audit", 
         "input.type": "log", 

filebeat/module/elasticsearch/gc/test/test.log-expected.json

@@ -11,6 +11,7 @@
         "elasticsearch.gc.phase.cpu_time.user_sec": "0.01", 
         "elasticsearch.gc.phase.duration_sec": "0.0021716", 
         "elasticsearch.gc.phase.name": "CMS Initial Mark", 
+        "event.dataset": "elasticsearch.gc", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "gc", 
         "input.type": "log", 
@@ -24,6 +25,7 @@
         "elasticsearch.gc.jvm_runtime_sec": "1396138.752", 
         "elasticsearch.gc.stopping_threads_time_sec": "0.0000702", 
         "elasticsearch.gc.threads_total_stop_time_sec": "0.0083760", 
+        "event.dataset": "elasticsearch.gc", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "gc", 
         "input.type": "log", 
@@ -51,6 +53,7 @@
         "elasticsearch.gc.phase.weak_refs_processing_time_sec": "0.0003647", 
         "elasticsearch.gc.young_gen.size_kb": "157248", 
         "elasticsearch.gc.young_gen.used_kb": "113198", 
+        "event.dataset": "elasticsearch.gc", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "gc", 
         "input.type": "log", 

filebeat/module/elasticsearch/server/test/test.log-expected.json

@@ -4,6 +4,7 @@
         "elasticsearch.index.name": "test-filebeat-modules", 
         "elasticsearch.node.name": "vWNJsZ3", 
         "elasticsearch.server.component": "o.e.c.m.MetaDataCreateIndexService", 
+        "event.dataset": "elasticsearch.server", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "server", 
         "input.type": "log", 
@@ -17,6 +18,7 @@
         "@timestamp": "2018-05-17T08:19:35,939", 
         "elasticsearch.node.name": "", 
         "elasticsearch.server.component": "o.e.n.Node", 
+        "event.dataset": "elasticsearch.server", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "server", 
         "input.type": "log", 
@@ -30,6 +32,7 @@
         "@timestamp": "2018-05-17T08:19:36,089", 
         "elasticsearch.node.name": "vWNJsZ3", 
         "elasticsearch.server.component": "o.e.e.NodeEnvironment", 
+        "event.dataset": "elasticsearch.server", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "server", 
         "input.type": "log", 
@@ -43,6 +46,7 @@
         "@timestamp": "2018-05-17T08:19:36,090", 
         "elasticsearch.node.name": "vWNJsZ3", 
         "elasticsearch.server.component": "o.e.e.NodeEnvironment", 
+        "event.dataset": "elasticsearch.server", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "server", 
         "input.type": "log", 
@@ -55,6 +59,7 @@
     {
         "@timestamp": "2018-05-17T08:19:36,116", 
         "elasticsearch.server.component": "o.e.n.Node", 
+        "event.dataset": "elasticsearch.server", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "server", 
         "input.type": "log", 
@@ -68,6 +73,7 @@
         "@timestamp": "2018-05-17T08:23:48,941", 
         "elasticsearch.node.name": "vWNJsZ3", 
         "elasticsearch.server.component": "o.e.c.r.a.DiskThresholdMonitor", 
+        "event.dataset": "elasticsearch.server", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "server", 
         "input.type": "log", 
@@ -82,6 +88,7 @@
         "elasticsearch.index.name": "filebeat-test-input", 
         "elasticsearch.node.name": "vWNJsZ3", 
         "elasticsearch.server.component": "o.e.c.m.MetaDataCreateIndexService", 
+        "event.dataset": "elasticsearch.server", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "server", 
         "input.type": "log", 
@@ -97,6 +104,7 @@
         "elasticsearch.index.name": "filebeat-test-input", 
         "elasticsearch.node.name": "vWNJsZ3", 
         "elasticsearch.server.component": "o.e.c.m.MetaDataMappingService", 
+        "event.dataset": "elasticsearch.server", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "server", 
         "input.type": "log", 
@@ -112,6 +120,7 @@
         "elasticsearch.index.name": ".kibana", 
         "elasticsearch.node.name": "QGY1F5P", 
         "elasticsearch.server.component": "o.e.c.m.MetaDataMappingService", 
+        "event.dataset": "elasticsearch.server", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "server", 
         "input.type": "log", 
@@ -125,6 +134,7 @@
         "@timestamp": "2018-05-17T08:29:25,598", 
         "elasticsearch.node.name": "vWNJsZ3", 
         "elasticsearch.server.component": "o.e.n.Node", 
+        "event.dataset": "elasticsearch.server", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "server", 
         "input.type": "log", 
@@ -138,6 +148,7 @@
         "@timestamp": "2018-05-17T08:29:25,612", 
         "elasticsearch.node.name": "vWNJsZ3", 
         "elasticsearch.server.component": "o.e.n.Node", 
+        "event.dataset": "elasticsearch.server", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "server", 
         "input.type": "log", 
@@ -151,6 +162,7 @@
         "@timestamp": "2018-07-03T11:45:48,548", 
         "elasticsearch.node.name": "srvmulpvlsk252_md", 
         "elasticsearch.server.component": "o.e.d.z.ZenDiscovery", 
+        "event.dataset": "elasticsearch.server", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "server", 
         "input.type": "log", 
@@ -164,6 +176,7 @@
         "@timestamp": "2018-07-03T11:45:48,548", 
         "elasticsearch.node.name": "srvmulpvlsk252_md", 
         "elasticsearch.server.component": "o.e.d.z.ZenDiscovery", 
+        "event.dataset": "elasticsearch.server", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "server", 
         "input.type": "log", 
@@ -179,6 +192,7 @@
     {
         "@timestamp": "2018-07-03T11:45:52,666", 
         "elasticsearch.server.component": "r.suppressed", 
+        "event.dataset": "elasticsearch.server", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "server", 
         "input.type": "log", 
@@ -194,6 +208,7 @@
     {
         "@timestamp": "2018-07-03T11:48:02,552", 
         "elasticsearch.server.component": "r.suppressed", 
+        "event.dataset": "elasticsearch.server", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "server", 
         "input.type": "log", 
@@ -212,6 +227,7 @@
         "elasticsearch.server.component": "o.e.m.j.JvmGcMonitorService", 
         "elasticsearch.server.gc.young.one": "3449979", 
         "elasticsearch.server.gc.young.two": "986594", 
+        "event.dataset": "elasticsearch.server", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "server", 
         "input.type": "log", 
@@ -229,6 +245,7 @@
         "elasticsearch.node.name": "srvmulpvlsk252_md", 
         "elasticsearch.server.component": "o.e.m.j.JvmGcMonitorService", 
         "elasticsearch.server.gc_overhead": "3449992", 
+        "event.dataset": "elasticsearch.server", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "server", 
         "input.type": "log", 
@@ -242,6 +259,7 @@
         "@timestamp": "2018-07-03T11:48:02,541", 
         "elasticsearch.node.name": "srvmulpvlsk252_md", 
         "elasticsearch.server.component": "o.e.a.b.TransportShardBulkAction", 
+        "event.dataset": "elasticsearch.server", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "server", 
         "input.type": "log", 
@@ -255,6 +273,7 @@
         "@timestamp": "2018-07-03T20:10:07,376", 
         "elasticsearch.node.name": "srvmulpvlsk252_md", 
         "elasticsearch.server.component": "o.e.x.m.MonitoringService", 
+        "event.dataset": "elasticsearch.server", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "server", 
         "input.type": "log", 

filebeat/module/elasticsearch/slowlog/test/test.log-expected.json

@@ -13,6 +13,7 @@
         "elasticsearch.slowlog.total_hits": 19435, 
         "elasticsearch.slowlog.total_shards": 1, 
         "elasticsearch.slowlog.types": "", 
+        "event.dataset": "elasticsearch.slowlog", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "slowlog", 
         "input.type": "log", 
@@ -36,6 +37,7 @@
         "elasticsearch.slowlog.total_hits": 19435, 
         "elasticsearch.slowlog.total_shards": 1, 
         "elasticsearch.slowlog.types": "", 
+        "event.dataset": "elasticsearch.slowlog", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "slowlog", 
         "input.type": "log", 
@@ -59,6 +61,7 @@
         "elasticsearch.slowlog.total_hits": 0, 
         "elasticsearch.slowlog.total_shards": 1, 
         "elasticsearch.slowlog.types": "", 
+        "event.dataset": "elasticsearch.slowlog", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "slowlog", 
         "input.type": "log", 
@@ -82,6 +85,7 @@
         "elasticsearch.slowlog.total_hits": 0, 
         "elasticsearch.slowlog.total_shards": 1, 
         "elasticsearch.slowlog.types": "", 
+        "event.dataset": "elasticsearch.slowlog", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "slowlog", 
         "input.type": "log", 
@@ -103,6 +107,7 @@
         "elasticsearch.slowlog.took": "1.4ms", 
         "elasticsearch.slowlog.took_millis": 1, 
         "elasticsearch.slowlog.type": "doc", 
+        "event.dataset": "elasticsearch.slowlog", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "slowlog", 
         "input.type": "log", 
@@ -124,6 +129,7 @@
         "elasticsearch.slowlog.took": "1.7ms", 
         "elasticsearch.slowlog.took_millis": 1, 
         "elasticsearch.slowlog.type": "doc", 
+        "event.dataset": "elasticsearch.slowlog", 
         "fileset.module": "elasticsearch", 
         "fileset.name": "slowlog", 
         "input.type": "log",