diff --git a/packages/system/changelog.yml b/packages/system/changelog.yml index 3074e81a5602..09a40c905197 100644 --- a/packages/system/changelog.yml +++ b/packages/system/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.12.5" + changes: + - description: Make event.original optional for application, security, and system data streams. + type: enhancement + link: https://github.com/elastic/integrations/pull/ - version: "0.12.4" changes: - description: Fix inconsistent dashboard IDs diff --git a/packages/system/data_stream/application/agent/stream/httpjson.yml.hbs b/packages/system/data_stream/application/agent/stream/httpjson.yml.hbs index e5e84c288a63..4699123187c6 100644 --- a/packages/system/data_stream/application/agent/stream/httpjson.yml.hbs +++ b/packages/system/data_stream/application/agent/stream/httpjson.yml.hbs @@ -33,6 +33,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/system/data_stream/application/agent/stream/winlog.yml.hbs b/packages/system/data_stream/application/agent/stream/winlog.yml.hbs index e207b9ffd6eb..05338cafe241 100644 --- a/packages/system/data_stream/application/agent/stream/winlog.yml.hbs +++ b/packages/system/data_stream/application/agent/stream/winlog.yml.hbs @@ -1,3 +1,7 @@ name: Application condition: ${host.platform} == 'windows' -ignore_older: 72h \ No newline at end of file +ignore_older: 72h +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} \ No newline at end of file diff --git a/packages/system/data_stream/application/elasticsearch/ingest_pipeline/default.yml b/packages/system/data_stream/application/elasticsearch/ingest_pipeline/default.yml index d239ad095fff..2554c30c104c 100644 --- a/packages/system/data_stream/application/elasticsearch/ingest_pipeline/default.yml +++ b/packages/system/data_stream/application/elasticsearch/ingest_pipeline/default.yml @@ -4,6 +4,11 @@ - set: field: event.ingested value: '{{_ingest.timestamp}}' + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: "error.message" diff --git a/packages/system/data_stream/application/manifest.yml b/packages/system/data_stream/application/manifest.yml index 3d9d689e7a66..b723baf1316c 100644 --- a/packages/system/data_stream/application/manifest.yml +++ b/packages/system/data_stream/application/manifest.yml @@ -6,6 +6,15 @@ streams: template_path: winlog.yml.hbs title: Application description: 'Collect Windows application logs' + vars: + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false - input: httpjson title: Windows Application Events via Splunk Enterprise REST API description: Collect Application Events via Splunk Enterprise REST API @@ -19,6 +28,14 @@ streams: show_user: true required: true default: 10s + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false - name: search type: text title: Splunk search string diff --git a/packages/system/data_stream/security/agent/stream/httpjson.yml.hbs b/packages/system/data_stream/security/agent/stream/httpjson.yml.hbs index 158e9245d0f2..bc1ae023e7ec 100644 --- a/packages/system/data_stream/security/agent/stream/httpjson.yml.hbs +++ b/packages/system/data_stream/security/agent/stream/httpjson.yml.hbs @@ -33,6 +33,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/system/data_stream/security/agent/stream/winlog.yml.hbs b/packages/system/data_stream/security/agent/stream/winlog.yml.hbs index 9311a7b08874..62e155e83056 100644 --- a/packages/system/data_stream/security/agent/stream/winlog.yml.hbs +++ b/packages/system/data_stream/security/agent/stream/winlog.yml.hbs @@ -1,2 +1,6 @@ name: Security condition: ${host.platform} == 'windows' +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} \ No newline at end of file diff --git a/packages/system/data_stream/security/elasticsearch/ingest_pipeline/default.yml b/packages/system/data_stream/security/elasticsearch/ingest_pipeline/default.yml index 75f9077cfd0c..a6a58a4eb30b 100644 --- a/packages/system/data_stream/security/elasticsearch/ingest_pipeline/default.yml +++ b/packages/system/data_stream/security/elasticsearch/ingest_pipeline/default.yml @@ -3170,6 +3170,12 @@ processors: ignore_failure: true if: ctx?.winlog?.time_created != null + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true + on_failure: - set: field: error.message diff --git a/packages/system/data_stream/security/manifest.yml b/packages/system/data_stream/security/manifest.yml index c2de21a474be..f93ca4f838c8 100644 --- a/packages/system/data_stream/security/manifest.yml +++ b/packages/system/data_stream/security/manifest.yml @@ -6,6 +6,15 @@ streams: template_path: winlog.yml.hbs title: Security description: 'Security channel' + vars: + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false - input: httpjson title: Windows Security Events via Splunk Enterprise REST API description: Collect Security Events via Splunk Enterprise REST API @@ -19,6 +28,14 @@ streams: show_user: true required: true default: 10s + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false - name: search type: text title: Splunk search string diff --git a/packages/system/data_stream/system/agent/stream/httpjson.yml.hbs b/packages/system/data_stream/system/agent/stream/httpjson.yml.hbs index e5e84c288a63..4699123187c6 100644 --- a/packages/system/data_stream/system/agent/stream/httpjson.yml.hbs +++ b/packages/system/data_stream/system/agent/stream/httpjson.yml.hbs @@ -33,6 +33,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/system/data_stream/system/agent/stream/winlog.yml.hbs b/packages/system/data_stream/system/agent/stream/winlog.yml.hbs index 47df93c51d24..74e255530146 100644 --- a/packages/system/data_stream/system/agent/stream/winlog.yml.hbs +++ b/packages/system/data_stream/system/agent/stream/winlog.yml.hbs @@ -1,2 +1,6 @@ name: System -condition: ${host.platform} == 'windows' \ No newline at end of file +condition: ${host.platform} == 'windows' +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} \ No newline at end of file diff --git a/packages/system/data_stream/system/elasticsearch/ingest_pipeline/default.yml b/packages/system/data_stream/system/elasticsearch/ingest_pipeline/default.yml index 9f7e885a2f50..e91dbcf905e3 100644 --- a/packages/system/data_stream/system/elasticsearch/ingest_pipeline/default.yml +++ b/packages/system/data_stream/system/elasticsearch/ingest_pipeline/default.yml @@ -4,6 +4,11 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: "error.message" diff --git a/packages/system/data_stream/system/manifest.yml b/packages/system/data_stream/system/manifest.yml index 6bc5b0c3e285..3a2c3dd761e9 100644 --- a/packages/system/data_stream/system/manifest.yml +++ b/packages/system/data_stream/system/manifest.yml @@ -6,6 +6,15 @@ streams: template_path: winlog.yml.hbs title: System description: 'Collect Windows system logs' + vars: + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false - input: httpjson title: Windows System Events via Splunk Enterprise REST API description: Collect System Events via Splunk Enterprise REST API @@ -19,6 +28,14 @@ streams: show_user: true required: true default: 10s + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false - name: search type: text title: Splunk search string diff --git a/packages/system/manifest.yml b/packages/system/manifest.yml index 89f65eb6f490..b0db50148ab1 100644 --- a/packages/system/manifest.yml +++ b/packages/system/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: system title: System -version: 0.12.4 +version: 0.12.5 license: basic description: System Integration type: integration