From 31e9a1e2662123c4e8b2632438e599a05c96c7d0 Mon Sep 17 00:00:00 2001 From: Kent Wang Date: Wed, 4 Sep 2019 21:56:09 +0800 Subject: [PATCH] Fix filebeat elasticsearch module ingest timezone (#13367) --- CHANGELOG.next.asciidoc | 3 ++- .../audit/ingest/pipeline-json.json | 10 +++++++++ .../audit/ingest/pipeline-plaintext.json | 22 +++++++++++++++++++ .../elasticsearch/audit/ingest/pipeline.json | 19 ---------------- .../deprecation/ingest/pipeline-json.json | 10 +++++++++ .../ingest/pipeline-plaintext.json | 22 +++++++++++++++++++ .../deprecation/ingest/pipeline.json | 19 ---------------- .../server/ingest/pipeline-json.json | 10 +++++++++ .../server/ingest/pipeline-plaintext.json | 22 +++++++++++++++++++ .../elasticsearch/server/ingest/pipeline.json | 19 ---------------- .../slowlog/ingest/pipeline-json.json | 10 +++++++++ .../slowlog/ingest/pipeline-plaintext.json | 22 +++++++++++++++++++ .../slowlog/ingest/pipeline.json | 19 ---------------- 13 files changed, 130 insertions(+), 77 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index aa49a79e8de..2e2f260c5b2 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -139,8 +139,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix filebeat autodiscover fileset hint for container input. {pull}13296[13296] - Fix incorrect references to index patterns in AWS and CoreDNS dashboards. {pull}13303[13303] - Fix timezone parsing of system module ingest pipelines. {pull}13308[13308] +- Fix timezone parsing of elasticsearch module ingest pipelines. {pull}13367[13367] - Change iis url path grok pattern from URIPATH to NOTSPACE. {issue}12710[12710] {pull}13225[13225] {issue}7951[7951] {pull}13378[13378] -- Add timezone information to apache error fileset. {issue}12772[12772] {pull}13304[13304] - Fix timezone parsing of nginx module ingest pipelines. {pull}13369[13369] - Fix incorrect field references in envoyproxy dashboard {issue}13420[13420] {pull}13421[13421] @@ -297,6 +297,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update PAN-OS fileset to use the ECS NAT fields. {issue}13320[13320] {pull}13330[13330] - Add fields to the Zeek DNS fileset for ECS DNS. {issue}13320[13320] {pull}13324[13324] - Add container image in Kubernetes metadata {pull}13356[13356] {issue}12688[12688] +- Add timezone information to apache error fileset. {issue}12772[12772] {pull}13304[13304] - Add module for ingesting Cisco FTD logs over syslog. {pull}13286[13286] *Heartbeat* diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json index 24cc6d7d0a9..f0cf48a19ff 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json @@ -190,6 +190,16 @@ "target_field": "log.level", "ignore_missing": true } + }, + { + "date": { + "field": "elasticsearch.audit.@timestamp", + "target_field": "@timestamp", + "formats": [ + "ISO8601" + ], + "ignore_failure": true + } } ], "on_failure": [ diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json index 63560c696b8..448028cfa3b 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json @@ -51,6 +51,28 @@ "field": "elasticsearch.audit.sub_action", "ignore_missing": true } + }, + { + "date": { + "field": "elasticsearch.audit.@timestamp", + "target_field": "@timestamp", + "formats": [ + "yyyy-MM-dd'T'HH:mm:ss,SSS" + ], + "ignore_failure": true + } + }, + { + "date": { + "if": "ctx.event.timezone != null", + "field": "elasticsearch.audit.@timestamp", + "target_field": "@timestamp", + "formats": [ + "yyyy-MM-dd'T'HH:mm:ss,SSS" + ], + "timezone": "{{ event.timezone }}", + "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] + } } ], "on_failure": [ diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.json b/filebeat/module/elasticsearch/audit/ingest/pipeline.json index 527545a6bfc..282abdac7be 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline.json +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline.json @@ -30,25 +30,6 @@ "name": "{< IngestPipeline "pipeline-json" >}" } }, - { - "date": { - "field": "elasticsearch.audit.@timestamp", - "target_field": "@timestamp", - "formats": [ - "ISO8601" - ], - "ignore_failure": true - } - }, - { - "date": { - "if": "ctx.event.timezone != null", - "field": "@timestamp", - "formats": ["ISO8601"], - "timezone": "{{ event.timezone }}", - "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - } - }, { "remove": { "field": "elasticsearch.audit.@timestamp" diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.json b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.json index e8a74768c28..00fe1b14f85 100755 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.json +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.json @@ -97,6 +97,16 @@ "field": "elasticsearch.deprecation.message", "target_field": "message" } + }, + { + "date": { + "field": "elasticsearch.deprecation.timestamp", + "target_field": "@timestamp", + "formats": [ + "ISO8601" + ], + "ignore_failure": true + } } ] } diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json index 6fdb52514e8..b3d95fad3c8 100755 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json @@ -19,6 +19,28 @@ "\\[%{TIMESTAMP_ISO8601:elasticsearch.deprecation.timestamp}\\]\\[%{LOGLEVEL:log.level}%{SPACE}*\\]\\[%{DATA:elasticsearch.component}%{SPACE}*\\] %{GREEDYMULTILINE:message}" ] } + }, + { + "date": { + "field": "elasticsearch.deprecation.timestamp", + "target_field": "@timestamp", + "formats": [ + "yyyy-MM-dd'T'HH:mm:ss,SSS" + ], + "ignore_failure": true + } + }, + { + "date": { + "if": "ctx.event.timezone != null", + "field": "elasticsearch.deprecation.timestamp", + "target_field": "@timestamp", + "formats": [ + "yyyy-MM-dd'T'HH:mm:ss,SSS" + ], + "timezone": "{{ event.timezone }}", + "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] + } } ] } diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.json b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.json index 9865e8c144e..59bf8e42868 100644 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.json +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.json @@ -30,25 +30,6 @@ "name": "{< IngestPipeline "pipeline-json" >}" } }, - { - "date": { - "field": "elasticsearch.deprecation.timestamp", - "target_field": "@timestamp", - "formats": [ - "ISO8601" - ], - "ignore_failure": true - } - }, - { - "date": { - "if": "ctx.event.timezone != null", - "field": "@timestamp", - "formats": ["ISO8601"], - "timezone": "{{ event.timezone }}", - "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - } - }, { "remove": { "field": "elasticsearch.deprecation.timestamp" diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-json.json b/filebeat/module/elasticsearch/server/ingest/pipeline-json.json index 25ef397209e..ada7c5063f4 100644 --- a/filebeat/module/elasticsearch/server/ingest/pipeline-json.json +++ b/filebeat/module/elasticsearch/server/ingest/pipeline-json.json @@ -107,6 +107,16 @@ "remove": { "field": "elasticsearch.server.message" } + }, + { + "date": { + "field": "elasticsearch.server.timestamp", + "target_field": "@timestamp", + "formats": [ + "ISO8601" + ], + "ignore_failure": true + } } ] } diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json index 1de5c95345c..0c9779acefb 100755 --- a/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json +++ b/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json @@ -25,6 +25,28 @@ "%{LOG_HEADER}%{SPACE}((\\[%{INDEXNAME:elasticsearch.index.name}\\]|\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\]))?%{SPACE}%{GREEDYMULTILINE:message}" ] } + }, + { + "date": { + "field": "elasticsearch.server.timestamp", + "target_field": "@timestamp", + "formats": [ + "yyyy-MM-dd'T'HH:mm:ss,SSS" + ], + "ignore_failure": true + } + }, + { + "date": { + "if": "ctx.event.timezone != null", + "field": "elasticsearch.server.timestamp", + "target_field": "@timestamp", + "formats": [ + "yyyy-MM-dd'T'HH:mm:ss,SSS" + ], + "timezone": "{{ event.timezone }}", + "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] + } } ] } diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline.json b/filebeat/module/elasticsearch/server/ingest/pipeline.json index 35a17c230dd..de23917aa96 100644 --- a/filebeat/module/elasticsearch/server/ingest/pipeline.json +++ b/filebeat/module/elasticsearch/server/ingest/pipeline.json @@ -54,25 +54,6 @@ "ignore_missing": true } }, - { - "date": { - "field": "elasticsearch.server.timestamp", - "target_field": "@timestamp", - "formats": [ - "ISO8601" - ], - "ignore_failure": true - } - }, - { - "date": { - "if": "ctx.event.timezone != null", - "field": "@timestamp", - "formats": ["ISO8601"], - "timezone": "{{ event.timezone }}", - "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - } - }, { "remove": { "field": "elasticsearch.server.timestamp" diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.json index c51dbadd199..ff0794d0089 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.json +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.json @@ -125,6 +125,16 @@ "remove": { "field": "elasticsearch.slowlog.message" } + }, + { + "date": { + "field": "elasticsearch.slowlog.timestamp", + "target_field": "@timestamp", + "formats": [ + "ISO8601" + ], + "ignore_failure": true + } } ] } diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json index cd3b23f40c9..e58407638e2 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json @@ -20,6 +20,28 @@ "separator": ",", "ignore_missing": true } + }, + { + "date": { + "field": "elasticsearch.slowlog.timestamp", + "target_field": "@timestamp", + "formats": [ + "yyyy-MM-dd'T'HH:mm:ss,SSS" + ], + "ignore_failure": true + } + }, + { + "date": { + "if": "ctx.event.timezone != null", + "field": "elasticsearch.slowlog.timestamp", + "target_field": "@timestamp", + "formats": [ + "yyyy-MM-dd'T'HH:mm:ss,SSS" + ], + "timezone": "{{ event.timezone }}", + "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] + } } ], "on_failure": [ diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json index 940ac7761dd..e36d0fbf1b0 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json @@ -30,25 +30,6 @@ "name": "{< IngestPipeline "pipeline-json" >}" } }, - { - "date": { - "field": "elasticsearch.slowlog.timestamp", - "target_field": "@timestamp", - "formats": [ - "ISO8601" - ], - "ignore_failure": true - } - }, - { - "date": { - "if": "ctx.event.timezone != null", - "field": "@timestamp", - "formats": ["ISO8601"], - "timezone": "{{ event.timezone }}", - "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - } - }, { "remove": { "field": "elasticsearch.slowlog.timestamp"