Make event.original optional for application, security, and system da…

…ta streams
elastic · May 14, 2021 · 885c9f7 · 885c9f7
1 parent e1c09f7
commit 885c9f7
Show file tree

Hide file tree

Showing 14 changed files with 96 additions and 3 deletions.
diff --git a/packages/system/changelog.yml b/packages/system/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.12.5"
+  changes:
+    - description: Make event.original optional for application, security, and system data streams.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/
 - version: "0.12.4"
   changes:
     - description: Fix inconsistent dashboard IDs

diff --git a/packages/system/data_stream/application/agent/stream/httpjson.yml.hbs b/packages/system/data_stream/application/agent/stream/httpjson.yml.hbs
@@ -33,6 +33,9 @@ tags:
 {{#each tags as |tag i|}}
   - {{tag}}
 {{/each}}
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
 {{#contains tags "forwarded"}}
 publisher_pipeline.disable_host: true
 {{/contains}}

diff --git a/packages/system/data_stream/application/agent/stream/winlog.yml.hbs b/packages/system/data_stream/application/agent/stream/winlog.yml.hbs
@@ -1,3 +1,7 @@
 name: Application
 condition: ${host.platform} == 'windows'
-ignore_older: 72h
+ignore_older: 72h
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
diff --git a/packages/system/data_stream/application/elasticsearch/ingest_pipeline/default.yml b/packages/system/data_stream/application/elasticsearch/ingest_pipeline/default.yml
@@ -4,6 +4,11 @@
       - set:
           field: event.ingested
           value: '{{_ingest.timestamp}}'
+      - remove:
+          field: event.original
+          if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+          ignore_failure: true
+          ignore_missing: true
     on_failure:
       - set:
           field: "error.message"

diff --git a/packages/system/data_stream/application/manifest.yml b/packages/system/data_stream/application/manifest.yml
@@ -6,6 +6,15 @@ streams:
     template_path: winlog.yml.hbs
     title: Application
     description: 'Collect Windows application logs'
+    vars:
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false
   - input: httpjson
     title: Windows Application Events via Splunk Enterprise REST API
     description: Collect Application Events via Splunk Enterprise REST API
@@ -19,6 +28,14 @@ streams:
         show_user: true
         required: true
         default: 10s
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false
       - name: search
         type: text
         title: Splunk search string

diff --git a/packages/system/data_stream/security/agent/stream/httpjson.yml.hbs b/packages/system/data_stream/security/agent/stream/httpjson.yml.hbs
@@ -33,6 +33,9 @@ tags:
 {{#each tags as |tag i|}}
   - {{tag}}
 {{/each}}
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
 {{#contains tags "forwarded"}}
 publisher_pipeline.disable_host: true
 {{/contains}}

diff --git a/packages/system/data_stream/security/agent/stream/winlog.yml.hbs b/packages/system/data_stream/security/agent/stream/winlog.yml.hbs
@@ -1,2 +1,6 @@
 name: Security
 condition: ${host.platform} == 'windows'
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
diff --git a/packages/system/data_stream/security/elasticsearch/ingest_pipeline/default.yml b/packages/system/data_stream/security/elasticsearch/ingest_pipeline/default.yml
@@ -3170,6 +3170,12 @@ processors:
       ignore_failure: true
       if: ctx?.winlog?.time_created != null
 
+  - remove:
+      field: event.original
+      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      ignore_failure: true
+      ignore_missing: true
+
 on_failure:
   - set:
       field: error.message

diff --git a/packages/system/data_stream/security/manifest.yml b/packages/system/data_stream/security/manifest.yml
@@ -6,6 +6,15 @@ streams:
     template_path: winlog.yml.hbs
     title: Security
     description: 'Security channel'
+    vars:
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false
   - input: httpjson
     title: Windows Security Events via Splunk Enterprise REST API
     description: Collect Security Events via Splunk Enterprise REST API
@@ -19,6 +28,14 @@ streams:
         show_user: true
         required: true
         default: 10s
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false
       - name: search
         type: text
         title: Splunk search string

diff --git a/packages/system/data_stream/system/agent/stream/httpjson.yml.hbs b/packages/system/data_stream/system/agent/stream/httpjson.yml.hbs
@@ -33,6 +33,9 @@ tags:
 {{#each tags as |tag i|}}
   - {{tag}}
 {{/each}}
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
 {{#contains tags "forwarded"}}
 publisher_pipeline.disable_host: true
 {{/contains}}

diff --git a/packages/system/data_stream/system/agent/stream/winlog.yml.hbs b/packages/system/data_stream/system/agent/stream/winlog.yml.hbs
@@ -1,2 +1,6 @@
 name: System
-condition: ${host.platform} == 'windows'
+condition: ${host.platform} == 'windows'
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
diff --git a/packages/system/data_stream/system/elasticsearch/ingest_pipeline/default.yml b/packages/system/data_stream/system/elasticsearch/ingest_pipeline/default.yml
@@ -4,6 +4,11 @@ processors:
   - set:
       field: event.ingested
       value: '{{_ingest.timestamp}}'
+  - remove:
+      field: event.original
+      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      ignore_failure: true
+      ignore_missing: true
 on_failure:
   - set:
       field: "error.message"

diff --git a/packages/system/data_stream/system/manifest.yml b/packages/system/data_stream/system/manifest.yml
@@ -6,6 +6,15 @@ streams:
     template_path: winlog.yml.hbs
     title: System
     description: 'Collect Windows system logs'
+    vars:
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false
   - input: httpjson
     title: Windows System Events via Splunk Enterprise REST API
     description: Collect System Events via Splunk Enterprise REST API
@@ -19,6 +28,14 @@ streams:
         show_user: true
         required: true
         default: 10s
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false
       - name: search
         type: text
         title: Splunk search string

diff --git a/packages/system/manifest.yml b/packages/system/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: system
 title: System
-version: 0.12.4
+version: 0.12.5
 license: basic
 description: System Integration
 type: integration