Handle ECS-compatible slowlogs emitted by ES 8.0.0+

CHANGELOG.next.asciidoc

@@ -256,6 +256,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Release Google Cloud module as GA. {pull}17511[17511]
 - Improve ECS categorization field mappings for nats module. {issue}16173[16173] {pull}17550[17550]
 - Enhance `elasticsearch/server` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17714[17714]
+- Enhance `elasticsearch/slowlog` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17729[17729]
 
 *Heartbeat*
 

filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml

@@ -12,12 +12,42 @@ processors:
       != 'index_search_slowlog'
 - remove:
     field: elasticsearch.slowlog.type
+- dot_expander:
+    field: service.name
+    path: elasticsearch.slowlog
+- rename:
+    field: elasticsearch.slowlog.service.name
+    target_field: service.name
+    ignore_missing: true
 - rename:
     field: elasticsearch.slowlog.level
     target_field: log.level
+    ignore_missing: true
+- dot_expander:
+    field: log.level
+    path: elasticsearch.slowlog
+- rename:
+    field: elasticsearch.slowlog.log.level
+    target_field: log.level
+    ignore_missing: true
+- dot_expander:
+    field: log.logger
+    path: elasticsearch.slowlog
+- rename:
+    field: elasticsearch.slowlog.log.logger
+    target_field: log.logger
+    ignore_missing: true
+- dot_expander:
+    field: process.thread.name
+    path: elasticsearch.slowlog
+- rename:
+    field: elasticsearch.slowlog.process.thread.name
+    target_field: process.thread.name
+    ignore_missing: true
 - rename:
     field: elasticsearch.slowlog.component
     target_field: elasticsearch.component
+    ignore_missing: true
 - dot_expander:
     field: cluster.name
     path: elasticsearch.slowlog
@@ -68,9 +98,17 @@ processors:
     - \[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\]
 - remove:
     field: elasticsearch.slowlog.message
-- date:
+- rename:
+    field: elasticsearch.slowlog.@timestamp
+    target_field: '@timestamp'
+    ignore_missing: true
+- rename:
     field: elasticsearch.slowlog.timestamp
     target_field: '@timestamp'
+    ignore_missing: true
+- date:
+    field: '@timestamp'
+    target_field: '@timestamp'
     formats:
     - ISO8601
     ignore_failure: true

filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml

@@ -16,7 +16,10 @@ processors:
     if: ctx.first_char == '{'
     name: '{< IngestPipeline "pipeline-json" >}'
 - remove:
-    field: elasticsearch.slowlog.timestamp
+    field:
+    - elasticsearch.slowlog.timestamp
+    - elasticsearch.server.@timestamp
+    ignore_missing: true
 - script:
     lang: painless
     source: ctx.event.duration = Math.round(ctx.elasticsearch.slowlog.duration * params.scale)

filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log

@@ -0,0 +1,2 @@
+{"@timestamp":"2020-04-16T11:20:02.069Z", "log.level":"TRACE",  "id":"5xy3gnEBmUEb0NJ1lijF", "message":"[test_index/M4fNwSWlTfek9m1SNL49Kg]", "source":"{\\\"f", "took":"15.1ms", "took_millis":"15" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][write][T#2]","log.logger":"index.indexing.slowlog.index.M4fNwSWlTfek9m1SNL49Kg","type":"index_indexing_slowlog","cluster.uuid":"HHmOPeWKQlSeaF88DSfFVw","node.id":"wxTr7N_gRWWg3mUdY4spbg","node.name":"integTest-0","cluster.name":"integTest"}
+{"@timestamp":"2020-04-16T11:20:02.777Z", "log.level":"TRACE",  "id":"6By3gnEBmUEb0NJ1mSij", "message":"[test_index/Jsz7IUYMQ9ubo2ahiMgCbQ]", "source":"{\\\"field\\\":123}", "took":"10.4ms", "took_millis":"10" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][write][T#4]","log.logger":"index.indexing.slowlog.index.Jsz7IUYMQ9ubo2ahiMgCbQ","type":"index_indexing_slowlog","cluster.uuid":"HHmOPeWKQlSeaF88DSfFVw","node.id":"wxTr7N_gRWWg3mUdY4spbg","node.name":"integTest-0","cluster.name":"integTest"}

filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log-expected.json

@@ -0,0 +1,58 @@
+[
+    {
+        "@timestamp": "2020-04-16T11:20:02.069Z",
+        "elasticsearch.cluster.name": "integTest",
+        "elasticsearch.cluster.uuid": "HHmOPeWKQlSeaF88DSfFVw",
+        "elasticsearch.index.id": "M4fNwSWlTfek9m1SNL49Kg",
+        "elasticsearch.index.name": "test_index",
+        "elasticsearch.node.id": "wxTr7N_gRWWg3mUdY4spbg",
+        "elasticsearch.node.name": "integTest-0",
+        "elasticsearch.slowlog.id": "5xy3gnEBmUEb0NJ1lijF",
+        "elasticsearch.slowlog.source": "{\\\"f",
+        "elasticsearch.slowlog.took": "15.1ms",
+        "event.category": "database",
+        "event.dataset": "elasticsearch.slowlog",
+        "event.duration": 15000000,
+        "event.kind": "event",
+        "event.module": "elasticsearch",
+        "event.type": "info",
+        "fileset.name": "slowlog",
+        "host.id": "wxTr7N_gRWWg3mUdY4spbg",
+        "input.type": "log",
+        "log.level": "TRACE",
+        "log.logger": "index.indexing.slowlog.index.M4fNwSWlTfek9m1SNL49Kg",
+        "log.offset": 0,
+        "message": "{\"@timestamp\":\"2020-04-16T11:20:02.069Z\", \"log.level\":\"TRACE\",  \"id\":\"5xy3gnEBmUEb0NJ1lijF\", \"message\":\"[test_index/M4fNwSWlTfek9m1SNL49Kg]\", \"source\":\"{\\\\\\\"f\", \"took\":\"15.1ms\", \"took_millis\":\"15\" , \"service.name\":\"ES_ECS\",\"process.thread.name\":\"elasticsearch[integTest-0][write][T#2]\",\"log.logger\":\"index.indexing.slowlog.index.M4fNwSWlTfek9m1SNL49Kg\",\"type\":\"index_indexing_slowlog\",\"cluster.uuid\":\"HHmOPeWKQlSeaF88DSfFVw\",\"node.id\":\"wxTr7N_gRWWg3mUdY4spbg\",\"node.name\":\"integTest-0\",\"cluster.name\":\"integTest\"}",
+        "process.thread.name": "elasticsearch[integTest-0][write][T#2]",
+        "service.name": "ES_ECS",
+        "service.type": "elasticsearch"
+    },
+    {
+        "@timestamp": "2020-04-16T11:20:02.777Z",
+        "elasticsearch.cluster.name": "integTest",
+        "elasticsearch.cluster.uuid": "HHmOPeWKQlSeaF88DSfFVw",
+        "elasticsearch.index.id": "Jsz7IUYMQ9ubo2ahiMgCbQ",
+        "elasticsearch.index.name": "test_index",
+        "elasticsearch.node.id": "wxTr7N_gRWWg3mUdY4spbg",
+        "elasticsearch.node.name": "integTest-0",
+        "elasticsearch.slowlog.id": "6By3gnEBmUEb0NJ1mSij",
+        "elasticsearch.slowlog.source": "{\\\"field\\\":123}",
+        "elasticsearch.slowlog.took": "10.4ms",
+        "event.category": "database",
+        "event.dataset": "elasticsearch.slowlog",
+        "event.duration": 10000000,
+        "event.kind": "event",
+        "event.module": "elasticsearch",
+        "event.type": "info",
+        "fileset.name": "slowlog",
+        "host.id": "wxTr7N_gRWWg3mUdY4spbg",
+        "input.type": "log",
+        "log.level": "TRACE",
+        "log.logger": "index.indexing.slowlog.index.Jsz7IUYMQ9ubo2ahiMgCbQ",
+        "log.offset": 514,
+        "message": "{\"@timestamp\":\"2020-04-16T11:20:02.777Z\", \"log.level\":\"TRACE\",  \"id\":\"6By3gnEBmUEb0NJ1mSij\", \"message\":\"[test_index/Jsz7IUYMQ9ubo2ahiMgCbQ]\", \"source\":\"{\\\\\\\"field\\\\\\\":123}\", \"took\":\"10.4ms\", \"took_millis\":\"10\" , \"service.name\":\"ES_ECS\",\"process.thread.name\":\"elasticsearch[integTest-0][write][T#4]\",\"log.logger\":\"index.indexing.slowlog.index.Jsz7IUYMQ9ubo2ahiMgCbQ\",\"type\":\"index_indexing_slowlog\",\"cluster.uuid\":\"HHmOPeWKQlSeaF88DSfFVw\",\"node.id\":\"wxTr7N_gRWWg3mUdY4spbg\",\"node.name\":\"integTest-0\",\"cluster.name\":\"integTest\"}",
+        "process.thread.name": "elasticsearch[integTest-0][write][T#4]",
+        "service.name": "ES_ECS",
+        "service.type": "elasticsearch"
+    }
+]

filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log

@@ -0,0 +1,2 @@
+{"@timestamp":"2020-04-16T11:20:02.828Z", "log.level":"TRACE",  "id":"null", "message":"[test_index][0]", "search_type":"QUERY_THEN_FETCH", "source":"{\\\"query\\\":{\\\"match_all\\\":{\\\"boost\\\":1.0}}}", "stats":"[]", "took":"10ms", "took_millis":"10", "total_hits":"0 hits", "total_shards":"1" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][search][T#1]","log.logger":"index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ","type":"index_search_slowlog","cluster.uuid":"HHmOPeWKQlSeaF88DSfFVw","node.id":"wxTr7N_gRWWg3mUdY4spbg","node.name":"integTest-0","cluster.name":"integTest"}
+{"@timestamp":"2020-04-16T11:20:02.839Z", "log.level":"TRACE",  "id":"my-identifier", "message":"[test_index][0]", "search_type":"QUERY_THEN_FETCH", "source":"{\\\"query\\\":{\\\"match_all\\\":{\\\"boost\\\":1.0}}}", "stats":"[]", "took":"76.4micros", "took_millis":"0", "total_hits":"0 hits", "total_shards":"1" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][search][T#3]","log.logger":"index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ","type":"index_search_slowlog","cluster.uuid":"HHmOPeWKQlSeaF88DSfFVw","node.id":"wxTr7N_gRWWg3mUdY4spbg","node.name":"integTest-0","cluster.name":"integTest"}

filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log-expected.json

@@ -0,0 +1,66 @@
+[
+    {
+        "@timestamp": "2020-04-16T11:20:02.828Z",
+        "elasticsearch.cluster.name": "integTest",
+        "elasticsearch.cluster.uuid": "HHmOPeWKQlSeaF88DSfFVw",
+        "elasticsearch.index.name": "test_index",
+        "elasticsearch.node.id": "wxTr7N_gRWWg3mUdY4spbg",
+        "elasticsearch.node.name": "integTest-0",
+        "elasticsearch.shard.id": "0",
+        "elasticsearch.slowlog.id": "null",
+        "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH",
+        "elasticsearch.slowlog.source": "{\\\"query\\\":{\\\"match_all\\\":{\\\"boost\\\":1.0}}}",
+        "elasticsearch.slowlog.stats": "[]",
+        "elasticsearch.slowlog.took": "10ms",
+        "elasticsearch.slowlog.total_hits": "0 hits",
+        "elasticsearch.slowlog.total_shards": "1",
+        "event.category": "database",
+        "event.dataset": "elasticsearch.slowlog",
+        "event.duration": 10000000,
+        "event.kind": "event",
+        "event.module": "elasticsearch",
+        "event.type": "info",
+        "fileset.name": "slowlog",
+        "host.id": "wxTr7N_gRWWg3mUdY4spbg",
+        "input.type": "log",
+        "log.level": "TRACE",
+        "log.logger": "index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ",
+        "log.offset": 0,
+        "message": "{\"@timestamp\":\"2020-04-16T11:20:02.828Z\", \"log.level\":\"TRACE\",  \"id\":\"null\", \"message\":\"[test_index][0]\", \"search_type\":\"QUERY_THEN_FETCH\", \"source\":\"{\\\\\\\"query\\\\\\\":{\\\\\\\"match_all\\\\\\\":{\\\\\\\"boost\\\\\\\":1.0}}}\", \"stats\":\"[]\", \"took\":\"10ms\", \"took_millis\":\"10\", \"total_hits\":\"0 hits\", \"total_shards\":\"1\" , \"service.name\":\"ES_ECS\",\"process.thread.name\":\"elasticsearch[integTest-0][search][T#1]\",\"log.logger\":\"index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ\",\"type\":\"index_search_slowlog\",\"cluster.uuid\":\"HHmOPeWKQlSeaF88DSfFVw\",\"node.id\":\"wxTr7N_gRWWg3mUdY4spbg\",\"node.name\":\"integTest-0\",\"cluster.name\":\"integTest\"}",
+        "process.thread.name": "elasticsearch[integTest-0][search][T#1]",
+        "service.name": "ES_ECS",
+        "service.type": "elasticsearch"
+    },
+    {
+        "@timestamp": "2020-04-16T11:20:02.839Z",
+        "elasticsearch.cluster.name": "integTest",
+        "elasticsearch.cluster.uuid": "HHmOPeWKQlSeaF88DSfFVw",
+        "elasticsearch.index.name": "test_index",
+        "elasticsearch.node.id": "wxTr7N_gRWWg3mUdY4spbg",
+        "elasticsearch.node.name": "integTest-0",
+        "elasticsearch.shard.id": "0",
+        "elasticsearch.slowlog.id": "my-identifier",
+        "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH",
+        "elasticsearch.slowlog.source": "{\\\"query\\\":{\\\"match_all\\\":{\\\"boost\\\":1.0}}}",
+        "elasticsearch.slowlog.stats": "[]",
+        "elasticsearch.slowlog.took": "76.4micros",
+        "elasticsearch.slowlog.total_hits": "0 hits",
+        "elasticsearch.slowlog.total_shards": "1",
+        "event.category": "database",
+        "event.dataset": "elasticsearch.slowlog",
+        "event.duration": 0,
+        "event.kind": "event",
+        "event.module": "elasticsearch",
+        "event.type": "info",
+        "fileset.name": "slowlog",
+        "host.id": "wxTr7N_gRWWg3mUdY4spbg",
+        "input.type": "log",
+        "log.level": "TRACE",
+        "log.logger": "index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ",
+        "log.offset": 613,
+        "message": "{\"@timestamp\":\"2020-04-16T11:20:02.839Z\", \"log.level\":\"TRACE\",  \"id\":\"my-identifier\", \"message\":\"[test_index][0]\", \"search_type\":\"QUERY_THEN_FETCH\", \"source\":\"{\\\\\\\"query\\\\\\\":{\\\\\\\"match_all\\\\\\\":{\\\\\\\"boost\\\\\\\":1.0}}}\", \"stats\":\"[]\", \"took\":\"76.4micros\", \"took_millis\":\"0\", \"total_hits\":\"0 hits\", \"total_shards\":\"1\" , \"service.name\":\"ES_ECS\",\"process.thread.name\":\"elasticsearch[integTest-0][search][T#3]\",\"log.logger\":\"index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ\",\"type\":\"index_search_slowlog\",\"cluster.uuid\":\"HHmOPeWKQlSeaF88DSfFVw\",\"node.id\":\"wxTr7N_gRWWg3mUdY4spbg\",\"node.name\":\"integTest-0\",\"cluster.name\":\"integTest\"}",
+        "process.thread.name": "elasticsearch[integTest-0][search][T#3]",
+        "service.name": "ES_ECS",
+        "service.type": "elasticsearch"
+    }
+]