c*: ensure event.kind is correctly set for pipeline errors (#6613)

carbon_black_cloud, carbonblack_edr, cef, citrix_waf, cloudflare, crowdstrike and cylance
elastic · Jun 20, 2023 · 69753e1 · 69753e1
1 parent 5ab0222
commit 69753e1
Show file tree

Hide file tree

Showing 38 changed files with 146 additions and 14 deletions.
diff --git a/packages/carbon_black_cloud/changelog.yml b/packages/carbon_black_cloud/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.11.0"
+  changes:
+    - description: Ensure event.kind is correctly set for pipeline errors.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6613
 - version: "1.10.0"
   changes:
     - description: Update package to ECS 8.8.0.

diff --git a/packages/carbon_black_cloud/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/carbon_black_cloud/data_stream/alert/elasticsearch/ingest_pipeline/default.yml
@@ -374,6 +374,9 @@ processors:
         }
         dropEmptyFields(ctx);
 on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
   - append:
       field: error.message
       value: '{{{_ingest.on_failure_message}}}'
diff --git a/...k_cloud/data_stream/asset_vulnerability_summary/elasticsearch/ingest_pipeline/default.yml b/...k_cloud/data_stream/asset_vulnerability_summary/elasticsearch/ingest_pipeline/default.yml
@@ -153,6 +153,9 @@ processors:
     if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
     ignore_missing: true
 on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
   - append:
       field: error.message
       value: '{{{_ingest.on_failure_message}}}'
diff --git a/packages/carbon_black_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/carbon_black_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
@@ -118,6 +118,9 @@ processors:
       ignore_failure: true
       ignore_missing: true
 on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
   - append:
       field: error.message
       value: '{{{_ingest.on_failure_message}}}'
diff --git a/...s/carbon_black_cloud/data_stream/endpoint_event/elasticsearch/ingest_pipeline/default.yml b/...s/carbon_black_cloud/data_stream/endpoint_event/elasticsearch/ingest_pipeline/default.yml
@@ -844,6 +844,9 @@ processors:
           ctx.related.hash = hash;
         }
 on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
   - append:
       field: error.message
       value: '{{{_ingest.on_failure_message}}}'
diff --git a/...es/carbon_black_cloud/data_stream/watchlist_hit/elasticsearch/ingest_pipeline/default.yml b/...es/carbon_black_cloud/data_stream/watchlist_hit/elasticsearch/ingest_pipeline/default.yml
@@ -394,6 +394,9 @@ processors:
           ctx.related.hash = hash;
         }
 on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
   - append:
       field: error.message
       value: '{{{_ingest.on_failure_message}}}'
diff --git a/packages/carbon_black_cloud/manifest.yml b/packages/carbon_black_cloud/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 2.7.0
 name: carbon_black_cloud
 title: VMware Carbon Black Cloud
-version: "1.10.0"
+version: "1.11.0"
 description: Collect logs from VMWare Carbon Black Cloud with Elastic Agent.
 type: integration
 categories:

diff --git a/packages/carbonblack_edr/changelog.yml b/packages/carbonblack_edr/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.11.0"
+  changes:
+    - description: Ensure event.kind is correctly set for pipeline errors.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6613
 - version: "1.10.0"
   changes:
     - description: Update package to ECS 8.8.0.

diff --git a/packages/carbonblack_edr/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/carbonblack_edr/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -867,6 +867,9 @@ processors:
     ignore_missing: true
 
 on_failure:
+- set:
+    field: event.kind
+    value: pipeline_error
 - append:
     field: error.message
     value: '{{ _ingest.on_failure_message }}'
diff --git a/packages/carbonblack_edr/manifest.yml b/packages/carbonblack_edr/manifest.yml
@@ -1,6 +1,6 @@
 name: carbonblack_edr
 title: VMware Carbon Black EDR
-version: "1.10.0"
+version: "1.11.0"
 description: Collect logs from VMware Carbon Black EDR with Elastic Agent.
 type: integration
 format_version: 2.7.0

diff --git a/packages/cef/changelog.yml b/packages/cef/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.11.0"
+  changes:
+    - description: Ensure event.message is correctly set for pipeline errors.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6613
 - version: "2.10.0"
   changes:
     - description: Update package to ECS 8.8.0.

diff --git a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml
@@ -376,7 +376,7 @@ processors:
       tag: convert baseEventCount
       type: long
 on_failure:
-  - set:
+  - append:
       field: error.message
       value: |-
         Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"

diff --git a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -192,7 +192,7 @@ on_failure:
       field:
         - _tmp
       ignore_missing: true
-  - set:
+  - append:
       field: error.message
       value: |-
         Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"

diff --git a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/fp-pipeline.yml b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/fp-pipeline.yml
@@ -22,7 +22,7 @@ processors:
       ignore_empty_value: true
       value: '{{cef.extensions.deviceCustomString4}}'
 on_failure:
-  - set:
+  - append:
       field: error.message
       value: |-
         Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"

diff --git a/packages/cef/manifest.yml b/packages/cef/manifest.yml
@@ -1,6 +1,6 @@
 name: cef
 title: Common Event Format (CEF)
-version: "2.10.0"
+version: "2.11.0"
 description: Collect logs from CEF Logs with Elastic Agent.
 categories:
   - security

diff --git a/packages/citrix_waf/changelog.yml b/packages/citrix_waf/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.7.0"
+  changes:
+    - description: Ensure event.kind is correctly set for pipeline errors.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6613
 - version: "1.6.0"
   changes:
     - description: Update package to ECS 8.8.0.

diff --git a/packages/citrix_waf/data_stream/log/elasticsearch/ingest_pipeline/cef.yml b/packages/citrix_waf/data_stream/log/elasticsearch/ingest_pipeline/cef.yml
@@ -102,3 +102,11 @@ processors:
       field: citrix.extended_kv
       target_field: citrix.extended
       ignore_missing: true
+
+on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
+      field: error.message
+      value: '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/citrix_waf/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/citrix_waf/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -156,6 +156,9 @@ on_failure:
         - _tmp
         - _conf
       ignore_missing: true
+  - set:
+      field: event.kind
+      value: pipeline_error
   - append:
       field: error.message
       value: "{{ _ingest.on_failure_message }}"
diff --git a/packages/citrix_waf/data_stream/log/elasticsearch/ingest_pipeline/native.yml b/packages/citrix_waf/data_stream/log/elasticsearch/ingest_pipeline/native.yml
@@ -22,4 +22,11 @@ processors:
   - set:
       field: citrix.default_class
       value: true
-      if: ctx._tmp?.default == 'default ' # The trailing space is intended.
+      if: ctx._tmp?.default == 'default ' # The trailing space is intended.
+on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
+      field: error.message
+      value: '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/citrix_waf/manifest.yml b/packages/citrix_waf/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 2.7.0
 name: citrix_waf
 title: "Citrix Web App Firewall"
-version: "1.6.0"
+version: "1.7.0"
 description: Ingest events from Citrix Systems Web App Firewall.
 type: integration
 categories:

diff --git a/packages/cloudflare/changelog.yml b/packages/cloudflare/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.9.0"
+  changes:
+    - description: Ensure event.kind is correctly set for pipeline errors.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6613
 - version: "2.8.0"
   changes:
     - description: Adding new Lens dashboards

diff --git a/packages/cloudflare/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
@@ -274,5 +274,8 @@ processors:
       handleMap(ctx);
 on_failure:
 - set:
+    field: event.kind
+    value: pipeline_error
+- append:
     field: error.message
-    value: "{{ _ingest.on_failure_message }}"
+    value: '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/cloudflare/data_stream/logpull/elasticsearch/ingest_pipeline/http.yml b/packages/cloudflare/data_stream/logpull/elasticsearch/ingest_pipeline/http.yml
@@ -502,7 +502,7 @@ processors:
       - json.EdgeServerIP
     ignore_missing: true
 on_failure:
-- set:
+- append:
     field: error.message
     value: |-
       Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"

diff --git a/packages/cloudflare/manifest.yml b/packages/cloudflare/manifest.yml
@@ -1,6 +1,6 @@
 name: cloudflare
 title: Cloudflare
-version: "2.8.0"
+version: "2.9.0"
 description: Collect logs from Cloudflare with Elastic Agent.
 type: integration
 format_version: 2.7.0

diff --git a/packages/crowdstrike/changelog.yml b/packages/crowdstrike/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.14.0"
+  changes:
+    - description: Ensure event.kind is correctly set for pipeline errors.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6613
 - version: "1.13.0"
   changes:
     - description: Update package to ECS 8.8.0 and pkg-spec 2.7.0.

diff --git a/...ages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/auth_activity_audit.yml b/...ages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/auth_activity_audit.yml
@@ -30,3 +30,10 @@ processors:
         def action = ctx?.crowdstrike?.event?.OperationName;
         if (action == null || action == "") return;
         ctx["event.action"] = regex.matcher(action).replaceAll(replacement).toLowerCase();
+on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
+      field: error.message
+      value: '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/default.yml b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/default.yml
@@ -435,5 +435,8 @@ processors:
         handleMap(ctx);
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: '{{ _ingest.on_failure_message }}'
+      value: '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/detection_summary.yml b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/detection_summary.yml
@@ -167,3 +167,10 @@ processors:
       field: threat.tactic.name
       value: "{{{_tmp.threat.tactic.name}}}"
       if: ctx._tmp?.threat?.tactic?.name != null
+on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
+      field: error.message
+      value: '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/firewall_match.yml b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/firewall_match.yml
@@ -133,3 +133,10 @@ processors:
       ignore_missing: true
       ignore_failure: true
       if: ctx?.crowdstrike?.event?.ConnectionDirection != "1"
+on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
+      field: error.message
+      value: '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/incident_summary.yml b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/incident_summary.yml
@@ -25,3 +25,10 @@ processors:
       field: message
       value: "Incident score {{crowdstrike.event.FineScore}}"
       if: ctx?.crowdstrike?.event?.FineScore != null
+on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
+      field: error.message
+      value: '{{{ _ingest.on_failure_message }}}'
diff --git a/...wdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/remote_response_session_end.yml b/...wdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/remote_response_session_end.yml
@@ -21,3 +21,10 @@ processors:
       type: string
       ignore_failure: true
       ignore_missing: true
+on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
+      field: error.message
+      value: '{{{ _ingest.on_failure_message }}}'
diff --git a/...strike/data_stream/falcon/elasticsearch/ingest_pipeline/remote_response_session_start.yml b/...strike/data_stream/falcon/elasticsearch/ingest_pipeline/remote_response_session_start.yml
@@ -21,3 +21,10 @@ processors:
       type: string
       ignore_failure: true
       ignore_missing: true
+on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
+      field: error.message
+      value: '{{{ _ingest.on_failure_message }}}'
diff --git a/...ages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/user_activity_audit.yml b/...ages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/user_activity_audit.yml
@@ -25,3 +25,10 @@ processors:
       ignore_failure: true
       ignore_missing: true
       if: ctx?.crowdstrike?.event?.UserIp != null && ctx?.crowdstrike?.event?.UserIp != ""
+on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
+      field: error.message
+      value: '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/default.yml b/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/default.yml
@@ -2474,5 +2474,8 @@ processors:
         handleMap(ctx);
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
       value: "Processor '{{ _ingest.on_failure_processor_type }}' with tag '{{ _ingest.on_failure_processor_tag }}' failed with message {{ _ingest.on_failure_message }}"
diff --git a/packages/crowdstrike/manifest.yml b/packages/crowdstrike/manifest.yml
@@ -1,6 +1,6 @@
 name: crowdstrike
 title: CrowdStrike
-version: "1.13.0"
+version: "1.14.0"
 description: Collect logs from Crowdstrike with Elastic Agent.
 type: integration
 format_version: 2.7.0

diff --git a/packages/cylance/changelog.yml b/packages/cylance/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.16.0"
+  changes:
+    - description: Ensure event.kind is correctly set for pipeline errors.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6613
 - version: "0.15.0"
   changes:
     - description: Update package to ECS 8.8.0.

diff --git a/packages/cylance/data_stream/protect/elasticsearch/ingest_pipeline/default.yml b/packages/cylance/data_stream/protect/elasticsearch/ingest_pipeline/default.yml
@@ -77,6 +77,9 @@ processors:
       ignore_failure: true
       ignore_missing: true
 on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
   - append:
       field: error.message
       value: "{{ _ingest.on_failure_message }}"
diff --git a/packages/cylance/manifest.yml b/packages/cylance/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 2.7.0
 name: cylance
 title: CylanceProtect Logs
-version: "0.15.0"
+version: "0.16.0"
 description: Collect logs from CylanceProtect devices with Elastic Agent.
 categories: ["security", "edr_xdr"]
 type: integration