diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7e2af891cd2..b694965d3d5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -292,6 +292,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve AWS cloudtrail field mappings {issue}16086[16086] {issue}16110[16110] {pull}17155[17155] - Move azure-eventhub input to GA. {issue}15671[15671] {pull}17313[17313] - Improve ECS categorization field mappings in mongodb module. {issue}16170[16170] {pull}17371[17371] +- Improve ECS categorization field mappings for mssql module. {issue}16171[16171] {pull}17376[17376] *Heartbeat* diff --git a/x-pack/filebeat/module/mssql/log/ingest/pipeline.json b/x-pack/filebeat/module/mssql/log/ingest/pipeline.json deleted file mode 100644 index 05ec4030f4a..00000000000 --- a/x-pack/filebeat/module/mssql/log/ingest/pipeline.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "description": "Pipeline to parse MSSQL logs", - "processors": [ - { - "grok": { - "field": "message", - "patterns": ["%{MSSQL_DATE:date} %{DATA:mssql.log.origin} [ ]*%{GREEDYDATA:msg_temp}"], - "pattern_definitions": { - "MSSQL_DATE":"%{DATA} %{DATA}" - } - } - }, - { - "date": { - "if": "ctx.event.timezone == null", - "field": "date", - "formats": ["yyyy-MM-dd HH:mm:ss.SS"], - "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - } - }, - { - "date": { - "if": "ctx.event.timezone != null", - "field": "date", - "formats": ["yyyy-MM-dd HH:mm:ss.SS"], - "timezone": "{{ event.timezone }}", - "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - } - }, - { - "remove": { - "field":"date", - "ignore_missing": true - } - }, - { - "rename": { - "field": "message", - "target_field": "log.original" - } - }, - { - "rename": { - "field": "msg_temp", - "target_field": "message", - "ignore_missing": true - } - } - ], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml b/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml new file mode 100644 index 00000000000..39a10a9ff99 --- /dev/null +++ b/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml @@ -0,0 +1,50 @@ +description: Pipeline to parse MSSQL logs +processors: +- grok: + field: message + patterns: + - '%{MSSQL_DATE:date} %{DATA:mssql.log.origin} [ ]*%{GREEDYDATA:msg_temp}' + pattern_definitions: + MSSQL_DATE: '%{DATA} %{DATA}' +- date: + if: ctx.event.timezone == null + field: date + formats: + - yyyy-MM-dd HH:mm:ss.SS + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' +- date: + if: ctx.event.timezone != null + field: date + formats: + - yyyy-MM-dd HH:mm:ss.SS + timezone: '{{ event.timezone }}' + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' +- remove: + field: date + ignore_missing: true +- rename: + field: message + target_field: log.original +- rename: + field: msg_temp + target_field: message + ignore_missing: true +- set: + field: event.kind + value: event +- append: + field: event.category + value: database +- append: + field: event.type + value: info +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/mssql/log/manifest.yml b/x-pack/filebeat/module/mssql/log/manifest.yml index fb1b70c9f96..2e90ff36459 100644 --- a/x-pack/filebeat/module/mssql/log/manifest.yml +++ b/x-pack/filebeat/module/mssql/log/manifest.yml @@ -11,5 +11,5 @@ var: os.linux: - /var/opt/mssql/log/error* -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/config.yml diff --git a/x-pack/filebeat/module/mssql/log/test/test.log-expected.json b/x-pack/filebeat/module/mssql/log/test/test.log-expected.json index 4f39989dc86..ed90c872d5a 100644 --- a/x-pack/filebeat/module/mssql/log/test/test.log-expected.json +++ b/x-pack/filebeat/module/mssql/log/test/test.log-expected.json @@ -1,9 +1,16 @@ [ { "@timestamp": "2019-05-03T09:01:09.990-02:00", + "event.category": [ + "database" + ], "event.dataset": "mssql.log", + "event.kind": "event", "event.module": "mssql", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.flags": [ @@ -17,9 +24,16 @@ }, { "@timestamp": "2019-05-03T09:01:09.990-02:00", + "event.category": [ + "database" + ], "event.dataset": "mssql.log", + "event.kind": "event", "event.module": "mssql", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.offset": 226, @@ -30,9 +44,16 @@ }, { "@timestamp": "2019-05-03T09:01:09.990-02:00", + "event.category": [ + "database" + ], "event.dataset": "mssql.log", + "event.kind": "event", "event.module": "mssql", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.offset": 282, @@ -43,9 +64,16 @@ }, { "@timestamp": "2019-05-03T09:01:09.990-02:00", + "event.category": [ + "database" + ], "event.dataset": "mssql.log", + "event.kind": "event", "event.module": "mssql", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.offset": 344, @@ -56,9 +84,16 @@ }, { "@timestamp": "2019-05-03T09:01:10.000-02:00", + "event.category": [ + "database" + ], "event.dataset": "mssql.log", + "event.kind": "event", "event.module": "mssql", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.offset": 400, @@ -69,9 +104,16 @@ }, { "@timestamp": "2019-05-03T09:01:10.000-02:00", + "event.category": [ + "database" + ], "event.dataset": "mssql.log", + "event.kind": "event", "event.module": "mssql", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.offset": 462, @@ -82,9 +124,16 @@ }, { "@timestamp": "2019-05-03T09:01:10.000-02:00", + "event.category": [ + "database" + ], "event.dataset": "mssql.log", + "event.kind": "event", "event.module": "mssql", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.flags": [ @@ -98,9 +147,16 @@ }, { "@timestamp": "2019-05-03T09:01:10.000-02:00", + "event.category": [ + "database" + ], "event.dataset": "mssql.log", + "event.kind": "event", "event.module": "mssql", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.offset": 734, @@ -111,9 +167,16 @@ }, { "@timestamp": "2019-05-03T09:01:10.000-02:00", + "event.category": [ + "database" + ], "event.dataset": "mssql.log", + "event.kind": "event", "event.module": "mssql", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.offset": 1011, @@ -124,9 +187,16 @@ }, { "@timestamp": "2019-05-03T09:01:10.000-02:00", + "event.category": [ + "database" + ], "event.dataset": "mssql.log", + "event.kind": "event", "event.module": "mssql", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.offset": 1166, @@ -137,9 +207,16 @@ }, { "@timestamp": "2019-05-03T09:01:10.000-02:00", + "event.category": [ + "database" + ], "event.dataset": "mssql.log", + "event.kind": "event", "event.module": "mssql", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.offset": 1289, @@ -150,9 +227,16 @@ }, { "@timestamp": "2019-05-03T09:01:10.010-02:00", + "event.category": [ + "database" + ], "event.dataset": "mssql.log", + "event.kind": "event", "event.module": "mssql", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.offset": 1373, @@ -163,9 +247,16 @@ }, { "@timestamp": "2019-05-03T09:01:10.200-02:00", + "event.category": [ + "database" + ], "event.dataset": "mssql.log", + "event.kind": "event", "event.module": "mssql", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.offset": 1435, @@ -176,9 +267,16 @@ }, { "@timestamp": "2019-05-03T09:01:11.930-02:00", + "event.category": [ + "database" + ], "event.dataset": "mssql.log", + "event.kind": "event", "event.module": "mssql", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.offset": 1528, @@ -189,9 +287,16 @@ }, { "@timestamp": "2019-05-03T09:01:12.030-02:00", + "event.category": [ + "database" + ], "event.dataset": "mssql.log", + "event.kind": "event", "event.module": "mssql", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.offset": 1599,