[Filebeat] [MongoDB] Support MongoDB 4.4 json logs (#24774) (#26336)

MongoDB 4.4 uses structured JSON format for logging, add support for this format while keeping support for the old one too. (cherry picked from commit 3752526) Co-authored-by: Tetiana Kravchenko <tanya.kravchenko.v@gmail.com>
elastic · Jun 16, 2021 · 22aec29 · 22aec29
1 parent ed515e1
commit 22aec29
Show file tree

Hide file tree

Showing 12 changed files with 302 additions and 32 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -567,6 +567,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add Content-Type override to aws-s3 input. {issue}25697[25697] {pull}25772[25772]
 - In Cisco Umbrella fileset add users from cisco.umbrella.identities to related.user. {pull}25776[25776]
 - Add fingerprint processor to generate fixed ids for `google_workspace` events. {pull}25841[25841]
+- Update PanOS module to parse HIP Match logs. {issue}24350[24350] {pull}25686[25686]
+- Support MongoDB 4.4 in filebeat's MongoDB module. {issue}20501[20501] {pull}24774[24774]
 - Enhance GCP module to populate orchestrator.* fields for GKE / K8S logs {pull}25368[25368]
 - Move Filebeat azure module to GA. {pull}26114[26114] {pull}26168[26168]
 - Make `filestream` input GA. {pull}26127[26127]

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -95635,6 +95635,30 @@ alias to: message
 
 --
 
+*`mongodb.log.msg`*::
++
+--
+String representing the raw log output message as passed from the server or driver
+
+
+type: text
+
+example: MongoDB starting
+
+--
+
+*`mongodb.log.id`*::
++
+--
+Integer representing the unique identifier of the log statement
+
+
+type: long
+
+example: 4615611
+
+--
+
 [[exported-fields-mssql]]
 == mssql fields
 

diff --git a/filebeat/docs/modules/mongodb.asciidoc b/filebeat/docs/modules/mongodb.asciidoc
@@ -18,7 +18,7 @@ include::../include/gs-link.asciidoc[]
 [float]
 === Compatibility
 
-The +{modulename}+ module was tested with logs from versions v3.2.11 on Debian.
+The +{modulename}+ module was tested with plaintext logs from version v3.2.11 on Debian and json logs from version v4.4.4 on Ubuntu.
 
 include::../include/configuring-intro.asciidoc[]
 

diff --git a/filebeat/module/mongodb/_meta/docs.asciidoc b/filebeat/module/mongodb/_meta/docs.asciidoc
@@ -13,7 +13,7 @@ include::../include/gs-link.asciidoc[]
 [float]
 === Compatibility
 
-The +{modulename}+ module was tested with logs from versions v3.2.11 on Debian.
+The +{modulename}+ module was tested with plaintext logs from version v3.2.11 on Debian and json logs from version v4.4.4 on Ubuntu.
 
 include::../include/configuring-intro.asciidoc[]
 

diff --git a/filebeat/module/mongodb/fields.go b/filebeat/module/mongodb/fields.go
diff --git a/filebeat/module/mongodb/log/_meta/fields.yml b/filebeat/module/mongodb/log/_meta/fields.yml
@@ -22,3 +22,13 @@
     type: alias
     path: message
     migration: true
+  - name: msg
+    description: >
+        String representing the raw log output message as passed from the server or driver
+    example: MongoDB starting
+    type: text
+  - name: id
+    description: >
+        Integer representing the unique identifier of the log statement
+    example: 4615611
+    type: long
diff --git a/filebeat/module/mongodb/log/ingest/pipeline-json.yml b/filebeat/module/mongodb/log/ingest/pipeline-json.yml
@@ -0,0 +1,47 @@
+description: Pipeline for parsing MongoDB logs in JSON format
+processors:
+- json:
+    field: message
+    target_field: mongodb.log
+- date:
+    field: mongodb.log.t.$date
+    target_field: '@timestamp'
+    formats:
+    - yyyy-MM-dd'T'HH:mm:ss.SSSZZZZZ
+- rename:
+    field: mongodb.log.s
+    target_field: log.level
+- rename:
+    field: mongodb.log.c
+    target_field: mongodb.log.component
+- rename:
+    field: mongodb.log.ctx
+    target_field: mongodb.log.context
+- append:
+    field: event.type
+    value: access
+    if: ctx.mongodb.log.component == 'ACCESS'
+- append:
+    field: event.type
+    value: change
+    if: ctx.mongodb.log.component == 'WRITE'
+- append:
+    field: event.type
+    value: info
+    if: ctx.mongodb.log.component != 'WRITE' && ctx.mongodb.log.component != 'ACCESS'
+- append:
+    field: event.type
+    value: error
+    if: ctx.log.level == 'F' || ctx.log.level == 'E'
+- remove:
+    field:
+      - mongodb.log.t
+      - mongodb.log.attr
+      - mongodb.log.tags
+      - mongodb.log.truncated
+      - mongodb.log.size
+    ignore_missing: true
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'
diff --git a/filebeat/module/mongodb/log/ingest/pipeline-plaintext.yml b/filebeat/module/mongodb/log/ingest/pipeline-plaintext.yml
@@ -0,0 +1,34 @@
+description: Pipeline for parsing MongoDB logs in plaintext
+processors:
+- grok:
+    field: message
+    patterns:
+    - '%{TIMESTAMP_ISO8601:mongodb.log.timestamp}%{SPACE}%{MONGO3_SEVERITY:log.level}%{SPACE}%{MONGO3_COMPONENT:mongodb.log.component}%{SPACE}(?:\[%{DATA:mongodb.log.context}\])?%{SPACE}%{GREEDYDATA:message}'
+    ignore_missing: true
+- date:
+    field: mongodb.log.timestamp
+    target_field: '@timestamp'
+    formats:
+    - yyyy-MM-dd'T'HH:mm:ss.SSSZZ
+- remove:
+    field: mongodb.log.timestamp
+- append:
+    field: event.type
+    value: access
+    if: "ctx?.mongodb?.log?.component == 'ACCESS'"
+- append:
+    field: event.type
+    value: change
+    if: "ctx?.mongodb?.log?.component == 'WRITE'"
+- append:
+    field: event.type
+    value: info
+    if: "ctx?.mongodb?.log?.component != 'WRITE' && ctx?.mongodb?.log?.component != 'ACCESS'"
+- append:
+    field: event.type
+    value: error
+    if: "ctx?.log?.level == 'F' || ctx?.log?.level == 'E'"
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'
diff --git a/filebeat/module/mongodb/log/ingest/pipeline.yml b/filebeat/module/mongodb/log/ingest/pipeline.yml
@@ -3,43 +3,30 @@ processors:
 - set:
     field: event.ingested
     value: '{{_ingest.timestamp}}'
-- grok:
-    field: message
-    patterns:
-    - '%{TIMESTAMP_ISO8601:mongodb.log.timestamp}%{SPACE}%{MONGO3_SEVERITY:log.level}%{SPACE}%{MONGO3_COMPONENT:mongodb.log.component}%{SPACE}(?:\[%{DATA:mongodb.log.context}\])?%{SPACE}%{GREEDYDATA:message}'
-    ignore_missing: true
 - rename:
     field: '@timestamp'
     target_field: event.created
-- date:
-    field: mongodb.log.timestamp
-    target_field: '@timestamp'
-    formats:
-    - yyyy-MM-dd'T'HH:mm:ss.SSSZZ
-- remove:
-    field: mongodb.log.timestamp
+- grok:
+    field: message
+    patterns:
+    - ^%{CHAR:first_char}
+    pattern_definitions:
+      CHAR: .
+- pipeline:
+    if: ctx.first_char != '{'
+    name: '{< IngestPipeline "pipeline-plaintext" >}'
+- pipeline:
+    if: ctx.first_char == '{'
+    name: '{< IngestPipeline "pipeline-json" >}'
 - set:
     field: event.kind
     value: event
 - append:
     field: event.category
     value: database
-- append:
-    field: event.type
-    value: access
-    if: "ctx?.mongodb?.log?.component == 'ACCESS'"
-- append:
-    field: event.type
-    value: change
-    if: "ctx?.mongodb?.log?.component == 'WRITE'"
-- append:
-    field: event.type
-    value: info
-    if: "ctx?.mongodb?.log?.component != 'WRITE' && ctx?.mongodb?.log?.component != 'ACCESS'"
-- append:
-    field: event.type
-    value: error
-    if: "ctx?.log?.level == 'F' || ctx?.log?.level == 'E'"
+- remove:
+    field:
+    - first_char
 on_failure:
 - set:
     field: error.message

diff --git a/filebeat/module/mongodb/log/manifest.yml b/filebeat/module/mongodb/log/manifest.yml
@@ -7,5 +7,8 @@ var:
     os.windows:
       - c:\data\log\mongod.log
 
-ingest_pipeline: ingest/pipeline.yml
+ingest_pipeline:
+  - ingest/pipeline.yml
+  - ingest/pipeline-plaintext.yml
+  - ingest/pipeline-json.yml
 input: config/log.yml
diff --git a/filebeat/module/mongodb/log/test/mongodb-ubuntu-4.4.4-json.log b/filebeat/module/mongodb/log/test/mongodb-ubuntu-4.4.4-json.log
@@ -0,0 +1,7 @@
+{"t":{"$date":"2021-03-22T21:21:20.349+00:00"},"s":"I",  "c":"STORAGE",  "id":4615611, "ctx":"initandlisten","msg":"MongoDB starting","attr":{"pid":1,"port":27017,"dbPath":"/data/db","architecture":"64-bit","host":"6150fe65a89c"}}
+{"t":{"$date":"2021-03-22T21:21:20.350+00:00"},"s":"I",  "c":"CONTROL",  "id":23403,   "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"4.4.4","gitVersion":"8db30a63db1a9d84bdcad0c83369623f708e0397","openSSLVersion":"OpenSSL 1.1.1  11 Sep 2018","modules":[],"allocator":"tcmalloc","environment":{"distmod":"ubuntu1804","distarch":"x86_64","target_arch":"x86_64"}}}}
+{"t":{"$date":"2021-03-22T21:21:26.240+00:00"},"s":"I",  "c":"RECOVERY", "id":23987,   "ctx":"initandlisten","msg":"WiredTiger recoveryTimestamp","attr":{"recoveryTimestamp":{"$timestamp":{"t":0,"i":0}}}}
+{"t":{"$date":"2021-03-22T21:21:26.363+00:00"},"s":"I",  "c":"STORAGE",  "id":20320,   "ctx":"initandlisten","msg":"createCollection","attr":{"namespace":"admin.system.version","uuidDisposition":"provided","uuid":{"uuid":{"$uuid":"b383f03c-b97c-4584-87ae-ab1b8ea399c3"}},"options":{"uuid":{"$uuid":"b383f03c-b97c-4584-87ae-ab1b8ea399c3"}}}}
+{"t":{"$date":"2021-03-22T21:21:26.410+00:00"},"s":"I",  "c":"INDEX",    "id":20345,   "ctx":"initandlisten","msg":"Index build: done building","attr":{"buildUUID":null,"namespace":"admin.system.version","index":"_id_","commitTimestamp":{"$timestamp":{"t":0,"i":0}}}}
+{"t":{"$date":"2021-03-22T21:21:26.412+00:00"},"s":"I",  "c":"COMMAND",  "id":20459,   "ctx":"initandlisten","msg":"Setting featureCompatibilityVersion","attr":{"newVersion":"4.4"}}
+{"t":{"$date":"2021-03-22T21:21:26.451+00:00"},"s":"I",  "c":"FTDC",     "id":20625,   "ctx":"initandlisten","msg":"Initializing full-time diagnostic data capture","attr":{"dataDirectory":"/data/db/diagnostic.data"}}
diff --git a/filebeat/module/mongodb/log/test/mongodb-ubuntu-4.4.4-json.log-expected.json b/filebeat/module/mongodb/log/test/mongodb-ubuntu-4.4.4-json.log-expected.json
@@ -0,0 +1,156 @@
+[
+    {
+        "@timestamp": "2021-03-22T21:21:20.349Z",
+        "event.category": [
+            "database"
+        ],
+        "event.dataset": "mongodb.log",
+        "event.kind": "event",
+        "event.module": "mongodb",
+        "event.type": [
+            "info"
+        ],
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.level": "I",
+        "log.offset": 0,
+        "message": "{\"t\":{\"$date\":\"2021-03-22T21:21:20.349+00:00\"},\"s\":\"I\",  \"c\":\"STORAGE\",  \"id\":4615611, \"ctx\":\"initandlisten\",\"msg\":\"MongoDB starting\",\"attr\":{\"pid\":1,\"port\":27017,\"dbPath\":\"/data/db\",\"architecture\":\"64-bit\",\"host\":\"6150fe65a89c\"}}",
+        "mongodb.log.component": "STORAGE",
+        "mongodb.log.context": "initandlisten",
+        "mongodb.log.id": 4615611,
+        "mongodb.log.msg": "MongoDB starting",
+        "service.type": "mongodb"
+    },
+    {
+        "@timestamp": "2021-03-22T21:21:20.350Z",
+        "event.category": [
+            "database"
+        ],
+        "event.dataset": "mongodb.log",
+        "event.kind": "event",
+        "event.module": "mongodb",
+        "event.type": [
+            "info"
+        ],
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.level": "I",
+        "log.offset": 231,
+        "message": "{\"t\":{\"$date\":\"2021-03-22T21:21:20.350+00:00\"},\"s\":\"I\",  \"c\":\"CONTROL\",  \"id\":23403,   \"ctx\":\"initandlisten\",\"msg\":\"Build Info\",\"attr\":{\"buildInfo\":{\"version\":\"4.4.4\",\"gitVersion\":\"8db30a63db1a9d84bdcad0c83369623f708e0397\",\"openSSLVersion\":\"OpenSSL 1.1.1  11 Sep 2018\",\"modules\":[],\"allocator\":\"tcmalloc\",\"environment\":{\"distmod\":\"ubuntu1804\",\"distarch\":\"x86_64\",\"target_arch\":\"x86_64\"}}}}",
+        "mongodb.log.component": "CONTROL",
+        "mongodb.log.context": "initandlisten",
+        "mongodb.log.id": 23403,
+        "mongodb.log.msg": "Build Info",
+        "service.type": "mongodb"
+    },
+    {
+        "@timestamp": "2021-03-22T21:21:26.240Z",
+        "event.category": [
+            "database"
+        ],
+        "event.dataset": "mongodb.log",
+        "event.kind": "event",
+        "event.module": "mongodb",
+        "event.type": [
+            "info"
+        ],
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.level": "I",
+        "log.offset": 621,
+        "message": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.240+00:00\"},\"s\":\"I\",  \"c\":\"RECOVERY\", \"id\":23987,   \"ctx\":\"initandlisten\",\"msg\":\"WiredTiger recoveryTimestamp\",\"attr\":{\"recoveryTimestamp\":{\"$timestamp\":{\"t\":0,\"i\":0}}}}",
+        "mongodb.log.component": "RECOVERY",
+        "mongodb.log.context": "initandlisten",
+        "mongodb.log.id": 23987,
+        "mongodb.log.msg": "WiredTiger recoveryTimestamp",
+        "service.type": "mongodb"
+    },
+    {
+        "@timestamp": "2021-03-22T21:21:26.363Z",
+        "event.category": [
+            "database"
+        ],
+        "event.dataset": "mongodb.log",
+        "event.kind": "event",
+        "event.module": "mongodb",
+        "event.type": [
+            "info"
+        ],
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.level": "I",
+        "log.offset": 826,
+        "message": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.363+00:00\"},\"s\":\"I\",  \"c\":\"STORAGE\",  \"id\":20320,   \"ctx\":\"initandlisten\",\"msg\":\"createCollection\",\"attr\":{\"namespace\":\"admin.system.version\",\"uuidDisposition\":\"provided\",\"uuid\":{\"uuid\":{\"$uuid\":\"b383f03c-b97c-4584-87ae-ab1b8ea399c3\"}},\"options\":{\"uuid\":{\"$uuid\":\"b383f03c-b97c-4584-87ae-ab1b8ea399c3\"}}}}",
+        "mongodb.log.component": "STORAGE",
+        "mongodb.log.context": "initandlisten",
+        "mongodb.log.id": 20320,
+        "mongodb.log.msg": "createCollection",
+        "service.type": "mongodb"
+    },
+    {
+        "@timestamp": "2021-03-22T21:21:26.410Z",
+        "event.category": [
+            "database"
+        ],
+        "event.dataset": "mongodb.log",
+        "event.kind": "event",
+        "event.module": "mongodb",
+        "event.type": [
+            "info"
+        ],
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.level": "I",
+        "log.offset": 1167,
+        "message": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.410+00:00\"},\"s\":\"I\",  \"c\":\"INDEX\",    \"id\":20345,   \"ctx\":\"initandlisten\",\"msg\":\"Index build: done building\",\"attr\":{\"buildUUID\":null,\"namespace\":\"admin.system.version\",\"index\":\"_id_\",\"commitTimestamp\":{\"$timestamp\":{\"t\":0,\"i\":0}}}}",
+        "mongodb.log.component": "INDEX",
+        "mongodb.log.context": "initandlisten",
+        "mongodb.log.id": 20345,
+        "mongodb.log.msg": "Index build: done building",
+        "service.type": "mongodb"
+    },
+    {
+        "@timestamp": "2021-03-22T21:21:26.412Z",
+        "event.category": [
+            "database"
+        ],
+        "event.dataset": "mongodb.log",
+        "event.kind": "event",
+        "event.module": "mongodb",
+        "event.type": [
+            "info"
+        ],
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.level": "I",
+        "log.offset": 1435,
+        "message": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.412+00:00\"},\"s\":\"I\",  \"c\":\"COMMAND\",  \"id\":20459,   \"ctx\":\"initandlisten\",\"msg\":\"Setting featureCompatibilityVersion\",\"attr\":{\"newVersion\":\"4.4\"}}",
+        "mongodb.log.component": "COMMAND",
+        "mongodb.log.context": "initandlisten",
+        "mongodb.log.id": 20459,
+        "mongodb.log.msg": "Setting featureCompatibilityVersion",
+        "service.type": "mongodb"
+    },
+    {
+        "@timestamp": "2021-03-22T21:21:26.451Z",
+        "event.category": [
+            "database"
+        ],
+        "event.dataset": "mongodb.log",
+        "event.kind": "event",
+        "event.module": "mongodb",
+        "event.type": [
+            "info"
+        ],
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.level": "I",
+        "log.offset": 1617,
+        "message": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.451+00:00\"},\"s\":\"I\",  \"c\":\"FTDC\",     \"id\":20625,   \"ctx\":\"initandlisten\",\"msg\":\"Initializing full-time diagnostic data capture\",\"attr\":{\"dataDirectory\":\"/data/db/diagnostic.data\"}}",
+        "mongodb.log.component": "FTDC",
+        "mongodb.log.context": "initandlisten",
+        "mongodb.log.id": 20625,
+        "mongodb.log.msg": "Initializing full-time diagnostic data capture",
+        "service.type": "mongodb"
+    }
+]