Fix filebeat elasticsearch module ingest timezone (elastic#13367)

jsoriano · Sep 4, 2019 · 31e9a1e · 31e9a1e
1 parent b1bab96
commit 31e9a1e
Show file tree

Hide file tree

Showing 13 changed files with 130 additions and 77 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -139,8 +139,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix filebeat autodiscover fileset hint for container input. {pull}13296[13296]
 - Fix incorrect references to index patterns in AWS and CoreDNS dashboards. {pull}13303[13303]
 - Fix timezone parsing of system module ingest pipelines. {pull}13308[13308]
+- Fix timezone parsing of elasticsearch module ingest pipelines. {pull}13367[13367]
 - Change iis url path grok pattern from URIPATH to NOTSPACE. {issue}12710[12710] {pull}13225[13225] {issue}7951[7951] {pull}13378[13378]
-- Add timezone information to apache error fileset. {issue}12772[12772] {pull}13304[13304]
 - Fix timezone parsing of nginx module ingest pipelines. {pull}13369[13369]
 - Fix incorrect field references in envoyproxy dashboard {issue}13420[13420] {pull}13421[13421]
 
@@ -297,6 +297,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update PAN-OS fileset to use the ECS NAT fields. {issue}13320[13320] {pull}13330[13330]
 - Add fields to the Zeek DNS fileset for ECS DNS. {issue}13320[13320] {pull}13324[13324]
 - Add container image in Kubernetes metadata {pull}13356[13356] {issue}12688[12688]
+- Add timezone information to apache error fileset. {issue}12772[12772] {pull}13304[13304]
 - Add module for ingesting Cisco FTD logs over syslog. {pull}13286[13286]
 
 *Heartbeat*

diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json
@@ -190,6 +190,16 @@
                 "target_field": "log.level",
                 "ignore_missing": true
             }
+        },
+        {
+            "date": {
+                "field": "elasticsearch.audit.@timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "ISO8601"
+                ],
+                "ignore_failure": true
+            }
         }
     ],
     "on_failure": [

diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json
@@ -51,6 +51,28 @@
                 "field": "elasticsearch.audit.sub_action",
                 "ignore_missing": true
             }
+        },
+        {
+            "date": {
+                "field": "elasticsearch.audit.@timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "yyyy-MM-dd'T'HH:mm:ss,SSS"
+                ],
+                "ignore_failure": true
+            }
+        },
+        {
+            "date": {
+                "if": "ctx.event.timezone != null",
+                "field": "elasticsearch.audit.@timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "yyyy-MM-dd'T'HH:mm:ss,SSS"
+                ],
+                "timezone": "{{ event.timezone }}",
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+            }
         }
     ],
     "on_failure": [

diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.json b/filebeat/module/elasticsearch/audit/ingest/pipeline.json
@@ -30,25 +30,6 @@
                 "name": "{< IngestPipeline "pipeline-json" >}"
             }
         },
-        {
-            "date": {
-                "field": "elasticsearch.audit.@timestamp",
-                "target_field": "@timestamp",
-                "formats": [
-                    "ISO8601"
-                ],
-                "ignore_failure": true
-            }
-        },
-        {
-            "date": {
-                "if": "ctx.event.timezone != null",
-                "field": "@timestamp",
-                "formats": ["ISO8601"],
-                "timezone": "{{ event.timezone }}",
-                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
-            }
-        },
         {
             "remove": {
                 "field": "elasticsearch.audit.@timestamp"

diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.json b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.json
@@ -97,6 +97,16 @@
                 "field": "elasticsearch.deprecation.message",
                 "target_field": "message"
             }
+        },
+        {
+            "date": {
+                "field": "elasticsearch.deprecation.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "ISO8601"
+                ],
+                "ignore_failure": true
+            }
         }
     ]
 }
diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json
@@ -19,6 +19,28 @@
                     "\\[%{TIMESTAMP_ISO8601:elasticsearch.deprecation.timestamp}\\]\\[%{LOGLEVEL:log.level}%{SPACE}*\\]\\[%{DATA:elasticsearch.component}%{SPACE}*\\] %{GREEDYMULTILINE:message}"
                 ]
             }
+        },
+        {
+            "date": {
+                "field": "elasticsearch.deprecation.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "yyyy-MM-dd'T'HH:mm:ss,SSS"
+                ],
+                "ignore_failure": true
+            }
+        },
+        {
+            "date": {
+                "if": "ctx.event.timezone != null",
+                "field": "elasticsearch.deprecation.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "yyyy-MM-dd'T'HH:mm:ss,SSS"
+                ],
+                "timezone": "{{ event.timezone }}",
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+            }
         }
     ]
 }
diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.json b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.json
@@ -30,25 +30,6 @@
                 "name": "{< IngestPipeline "pipeline-json" >}"
             }
         },
-        {
-            "date": {
-                "field": "elasticsearch.deprecation.timestamp",
-                "target_field": "@timestamp",
-                "formats": [
-                    "ISO8601"
-                ],
-                "ignore_failure": true
-            }
-        },
-        {
-            "date": {
-                "if": "ctx.event.timezone != null",
-                "field": "@timestamp",
-                "formats": ["ISO8601"],
-                "timezone": "{{ event.timezone }}",
-                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
-            }
-        },
         {
             "remove": {
                 "field": "elasticsearch.deprecation.timestamp"

diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-json.json b/filebeat/module/elasticsearch/server/ingest/pipeline-json.json
@@ -107,6 +107,16 @@
             "remove": {
                 "field": "elasticsearch.server.message"
             }
+        },
+        {
+            "date": {
+                "field": "elasticsearch.server.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "ISO8601"
+                ],
+                "ignore_failure": true
+            }
         }
     ]
 }
diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json
@@ -25,6 +25,28 @@
                     "%{LOG_HEADER}%{SPACE}((\\[%{INDEXNAME:elasticsearch.index.name}\\]|\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\]))?%{SPACE}%{GREEDYMULTILINE:message}"
                 ]
             }
+        },
+        {
+            "date": {
+                "field": "elasticsearch.server.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "yyyy-MM-dd'T'HH:mm:ss,SSS"
+                ],
+                "ignore_failure": true
+            }
+        },
+        {
+            "date": {
+                "if": "ctx.event.timezone != null",
+                "field": "elasticsearch.server.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "yyyy-MM-dd'T'HH:mm:ss,SSS"
+                ],
+                "timezone": "{{ event.timezone }}",
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+            }
         }
     ]
 }
diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline.json b/filebeat/module/elasticsearch/server/ingest/pipeline.json
@@ -54,25 +54,6 @@
                 "ignore_missing": true
             }
         },
-        {
-            "date": {
-                "field": "elasticsearch.server.timestamp",
-                "target_field": "@timestamp",
-                "formats": [
-                    "ISO8601"
-                ],
-                "ignore_failure": true
-            }
-        },
-        {
-            "date": {
-                "if": "ctx.event.timezone != null",
-                "field": "@timestamp",
-                "formats": ["ISO8601"],
-                "timezone": "{{ event.timezone }}",
-                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
-            }
-        },
         {
             "remove": {
                 "field": "elasticsearch.server.timestamp"

diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.json
@@ -125,6 +125,16 @@
             "remove": {
                 "field": "elasticsearch.slowlog.message"
             }
+        },
+        {
+            "date": {
+                "field": "elasticsearch.slowlog.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "ISO8601"
+                ],
+                "ignore_failure": true
+            }
         }
     ]
 }
diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json
@@ -20,6 +20,28 @@
                 "separator": ",",
                 "ignore_missing": true
             }
+        },
+        {
+            "date": {
+                "field": "elasticsearch.slowlog.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "yyyy-MM-dd'T'HH:mm:ss,SSS"
+                ],
+                "ignore_failure": true
+            }
+        },
+        {
+            "date": {
+                "if": "ctx.event.timezone != null",
+                "field": "elasticsearch.slowlog.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "yyyy-MM-dd'T'HH:mm:ss,SSS"
+                ],
+                "timezone": "{{ event.timezone }}",
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+            }
         }
     ],
     "on_failure": [

diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json
@@ -30,25 +30,6 @@
                 "name": "{< IngestPipeline "pipeline-json" >}"
             }
         },
-        {
-            "date": {
-                "field": "elasticsearch.slowlog.timestamp",
-                "target_field": "@timestamp",
-                "formats": [
-                    "ISO8601"
-                ],
-                "ignore_failure": true
-            }
-        },
-        {
-            "date": {
-                "if": "ctx.event.timezone != null",
-                "field": "@timestamp",
-                "formats": ["ISO8601"],
-                "timezone": "{{ event.timezone }}",
-                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
-            }
-        },
         {
             "remove": {
                 "field": "elasticsearch.slowlog.timestamp"