From b2c39944045f6a1cf515e5ef3462c2c4c1bccb46 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Tue, 20 Jun 2023 12:02:55 +0930 Subject: [PATCH 1/2] [hi]*: ensure event.kind is correctly set for pipeline errors hashicorp_vault, hid_bravura_monitor, imperva, infoblox_bloxone_ddi, infoblox_nios and iptables --- packages/hashicorp_vault/changelog.yml | 5 +++++ .../audit/elasticsearch/ingest_pipeline/default.yml | 9 ++++++--- .../log/elasticsearch/ingest_pipeline/default.yml | 9 ++++++--- .../log/elasticsearch/ingest_pipeline/json.yml | 9 ++++++--- .../elasticsearch/ingest_pipeline/default.yml | 9 ++++++--- packages/hashicorp_vault/manifest.yml | 2 +- packages/hid_bravura_monitor/changelog.yml | 5 +++++ .../log/elasticsearch/ingest_pipeline/default.yml | 5 ++++- .../winlog/elasticsearch/ingest_pipeline/default.yml | 3 +++ packages/hid_bravura_monitor/manifest.yml | 2 +- packages/imperva/changelog.yml | 5 +++++ .../elasticsearch/ingest_pipeline/default.yml | 7 +++++-- packages/imperva/manifest.yml | 2 +- packages/infoblox_bloxone_ddi/changelog.yml | 5 +++++ .../elasticsearch/ingest_pipeline/default.yml | 5 ++++- .../elasticsearch/ingest_pipeline/default.yml | 5 ++++- .../elasticsearch/ingest_pipeline/default.yml | 5 ++++- packages/infoblox_bloxone_ddi/manifest.yml | 2 +- packages/infoblox_nios/changelog.yml | 5 +++++ .../log/elasticsearch/ingest_pipeline/default.yml | 12 ++++++------ .../elasticsearch/ingest_pipeline/pipeline_audit.yml | 7 +++++++ .../elasticsearch/ingest_pipeline/pipeline_dhcp.yml | 7 +++++++ .../elasticsearch/ingest_pipeline/pipeline_dns.yml | 7 +++++++ packages/infoblox_nios/manifest.yml | 2 +- packages/iptables/changelog.yml | 5 +++++ .../log/elasticsearch/ingest_pipeline/default.yml | 3 +++ packages/iptables/manifest.yml | 2 +- 27 files changed, 114 insertions(+), 30 deletions(-) diff --git a/packages/hashicorp_vault/changelog.yml b/packages/hashicorp_vault/changelog.yml index 5c07d506eb5e..b8ddb909bf99 100644 --- a/packages/hashicorp_vault/changelog.yml +++ b/packages/hashicorp_vault/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.12.0" + changes: + - description: Ensure event.kind is correctly set for pipeline errors. + type: enhancement + link: https://github.com/elastic/integrations/pull/6616 - version: "1.11.0" changes: - description: Update package to ECS 8.8.0. diff --git a/packages/hashicorp_vault/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/hashicorp_vault/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index c80d45f114d8..d83e43347967 100644 --- a/packages/hashicorp_vault/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/hashicorp_vault/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -181,6 +181,9 @@ processors: ignore_failure: true ignore_missing: true on_failure: -- set: - field: error.message - value: '{{ _ingest.on_failure_message }}' + - set: + field: event.kind + value: pipeline_error + - append: + field: error.message + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/hashicorp_vault/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/hashicorp_vault/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 1036e77416cb..a34dc98e4783 100644 --- a/packages/hashicorp_vault/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/hashicorp_vault/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -37,6 +37,9 @@ processors: ignore_failure: true ignore_missing: true on_failure: -- set: - field: error.message - value: '{{ _ingest.on_failure_message }}' + - set: + field: event.kind + value: pipeline_error + - append: + field: error.message + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/hashicorp_vault/data_stream/log/elasticsearch/ingest_pipeline/json.yml b/packages/hashicorp_vault/data_stream/log/elasticsearch/ingest_pipeline/json.yml index e86f09959cda..bd94141f9ec8 100644 --- a/packages/hashicorp_vault/data_stream/log/elasticsearch/ingest_pipeline/json.yml +++ b/packages/hashicorp_vault/data_stream/log/elasticsearch/ingest_pipeline/json.yml @@ -51,6 +51,9 @@ processors: copy_from: hashicorp_vault.log.file_path ignore_failure: true on_failure: -- set: - field: error.message - value: '{{ _ingest.on_failure_message }}' \ No newline at end of file + - set: + field: event.kind + value: pipeline_error + - append: + field: error.message + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/hashicorp_vault/data_stream/metrics/elasticsearch/ingest_pipeline/default.yml b/packages/hashicorp_vault/data_stream/metrics/elasticsearch/ingest_pipeline/default.yml index c8ddbe632ed5..639c9a0ecf34 100644 --- a/packages/hashicorp_vault/data_stream/metrics/elasticsearch/ingest_pipeline/default.yml +++ b/packages/hashicorp_vault/data_stream/metrics/elasticsearch/ingest_pipeline/default.yml @@ -31,6 +31,9 @@ processors: target_field: hashicorp_vault.metrics ignore_missing: true on_failure: -- set: - field: error.message - value: '{{ _ingest.on_failure_message }}' + - set: + field: event.kind + value: pipeline_error + - append: + field: error.message + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/hashicorp_vault/manifest.yml b/packages/hashicorp_vault/manifest.yml index 935b7a869710..d0431a154631 100644 --- a/packages/hashicorp_vault/manifest.yml +++ b/packages/hashicorp_vault/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: hashicorp_vault title: Hashicorp Vault -version: "1.11.0" +version: "1.12.0" license: basic description: Collect logs and metrics from Hashicorp Vault with Elastic Agent. type: integration diff --git a/packages/hid_bravura_monitor/changelog.yml b/packages/hid_bravura_monitor/changelog.yml index e6f26cf27cd9..d2eae61bb521 100644 --- a/packages/hid_bravura_monitor/changelog.yml +++ b/packages/hid_bravura_monitor/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.8.0" + changes: + - description: Ensure event.kind is correctly set for pipeline errors. + type: enhancement + link: https://github.com/elastic/integrations/pull/6616 - version: "1.7.0" changes: - description: Update package to ECS 8.8.0. diff --git a/packages/hid_bravura_monitor/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/hid_bravura_monitor/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 20dd10e6b2bb..667837414da1 100644 --- a/packages/hid_bravura_monitor/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/hid_bravura_monitor/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -192,5 +192,8 @@ processors: ignore_missing: true on_failure: - set: + field: event.kind + value: pipeline_error + - append: field: error.message - value: '{{ _ingest.on_failure_message }}' + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/hid_bravura_monitor/data_stream/winlog/elasticsearch/ingest_pipeline/default.yml b/packages/hid_bravura_monitor/data_stream/winlog/elasticsearch/ingest_pipeline/default.yml index 3baa9243eeb2..9305970d9e92 100644 --- a/packages/hid_bravura_monitor/data_stream/winlog/elasticsearch/ingest_pipeline/default.yml +++ b/packages/hid_bravura_monitor/data_stream/winlog/elasticsearch/ingest_pipeline/default.yml @@ -392,6 +392,9 @@ processors: on_failure: - set: + field: event.kind + value: pipeline_error + - append: field: error.message value: |- Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" diff --git a/packages/hid_bravura_monitor/manifest.yml b/packages/hid_bravura_monitor/manifest.yml index 3a1db8f4bc5b..c3112546d629 100644 --- a/packages/hid_bravura_monitor/manifest.yml +++ b/packages/hid_bravura_monitor/manifest.yml @@ -1,6 +1,6 @@ name: hid_bravura_monitor title: Bravura Monitor -version: "1.7.0" +version: "1.8.0" categories: ["security", "iam"] release: ga description: Collect logs from Bravura Security Fabric with Elastic Agent. diff --git a/packages/imperva/changelog.yml b/packages/imperva/changelog.yml index 59012ec0abd1..316af3b15b60 100644 --- a/packages/imperva/changelog.yml +++ b/packages/imperva/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.16.0" + changes: + - description: Ensure event.kind is correctly set for pipeline errors. + type: enhancement + link: https://github.com/elastic/integrations/pull/6616 - version: "0.15.0" changes: - description: Update package to ECS 8.8.0. diff --git a/packages/imperva/data_stream/securesphere/elasticsearch/ingest_pipeline/default.yml b/packages/imperva/data_stream/securesphere/elasticsearch/ingest_pipeline/default.yml index bd60914c2001..a0f0889abe07 100644 --- a/packages/imperva/data_stream/securesphere/elasticsearch/ingest_pipeline/default.yml +++ b/packages/imperva/data_stream/securesphere/elasticsearch/ingest_pipeline/default.yml @@ -63,6 +63,9 @@ processors: ignore_failure: true ignore_missing: true on_failure: + - set: + field: event.kind + value: pipeline_error - append: - field: error.message - value: "{{ _ingest.on_failure_message }}" + field: error.message + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/imperva/manifest.yml b/packages/imperva/manifest.yml index 9abe7e811cb1..015127beeb7a 100644 --- a/packages/imperva/manifest.yml +++ b/packages/imperva/manifest.yml @@ -1,7 +1,7 @@ format_version: 2.7.0 name: imperva title: Imperva SecureSphere Logs -version: "0.15.0" +version: "0.16.0" description: Collect SecureSphere logs from Imperva devices with Elastic Agent. categories: ["network", "security"] type: integration diff --git a/packages/infoblox_bloxone_ddi/changelog.yml b/packages/infoblox_bloxone_ddi/changelog.yml index 1ac3f963322e..943f30ce1619 100644 --- a/packages/infoblox_bloxone_ddi/changelog.yml +++ b/packages/infoblox_bloxone_ddi/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.5.0" + changes: + - description: Ensure event.kind is correctly set for pipeline errors. + type: enhancement + link: https://github.com/elastic/integrations/pull/6616 - version: "1.4.0" changes: - description: Update package to ECS 8.8.0. diff --git a/packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/elasticsearch/ingest_pipeline/default.yml b/packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/elasticsearch/ingest_pipeline/default.yml index f3b1b6867adb..895a4d62585e 100644 --- a/packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/elasticsearch/ingest_pipeline/default.yml +++ b/packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/elasticsearch/ingest_pipeline/default.yml @@ -239,6 +239,9 @@ processors: } dropEmptyFields(ctx); on_failure: + - set: + field: event.kind + value: pipeline_error - append: field: error.message - value: '{{ _ingest.on_failure_message }}' + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/infoblox_bloxone_ddi/data_stream/dns_config/elasticsearch/ingest_pipeline/default.yml b/packages/infoblox_bloxone_ddi/data_stream/dns_config/elasticsearch/ingest_pipeline/default.yml index 8a21f184c14e..9574bd896ccf 100644 --- a/packages/infoblox_bloxone_ddi/data_stream/dns_config/elasticsearch/ingest_pipeline/default.yml +++ b/packages/infoblox_bloxone_ddi/data_stream/dns_config/elasticsearch/ingest_pipeline/default.yml @@ -1988,6 +1988,9 @@ processors: } dropEmptyFields(ctx); on_failure: + - set: + field: event.kind + value: pipeline_error - append: field: error.message - value: '{{ _ingest.on_failure_message }}' + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/infoblox_bloxone_ddi/data_stream/dns_data/elasticsearch/ingest_pipeline/default.yml b/packages/infoblox_bloxone_ddi/data_stream/dns_data/elasticsearch/ingest_pipeline/default.yml index fb4e2a37696f..8aedc379e928 100644 --- a/packages/infoblox_bloxone_ddi/data_stream/dns_data/elasticsearch/ingest_pipeline/default.yml +++ b/packages/infoblox_bloxone_ddi/data_stream/dns_data/elasticsearch/ingest_pipeline/default.yml @@ -428,6 +428,9 @@ processors: } dropEmptyFields(ctx); on_failure: + - set: + field: event.kind + value: pipeline_error - append: field: error.message - value: '{{ _ingest.on_failure_message }}' + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/infoblox_bloxone_ddi/manifest.yml b/packages/infoblox_bloxone_ddi/manifest.yml index 8b52050f8d94..2b0306294635 100644 --- a/packages/infoblox_bloxone_ddi/manifest.yml +++ b/packages/infoblox_bloxone_ddi/manifest.yml @@ -1,7 +1,7 @@ format_version: 2.7.0 name: infoblox_bloxone_ddi title: Infoblox BloxOne DDI -version: "1.4.0" +version: "1.5.0" description: Collect logs from Infoblox BloxOne DDI with Elastic Agent. type: integration categories: diff --git a/packages/infoblox_nios/changelog.yml b/packages/infoblox_nios/changelog.yml index 9a4b428c0691..d23427d8011d 100644 --- a/packages/infoblox_nios/changelog.yml +++ b/packages/infoblox_nios/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.9.0" + changes: + - description: Ensure event.kind is correctly set for pipeline errors. + type: enhancement + link: https://github.com/elastic/integrations/pull/6616 - version: "1.8.0" changes: - description: Update package to ECS 8.8.0. diff --git a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/default.yml index ff385c7a75c0..537a72a5efe8 100644 --- a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -139,9 +139,9 @@ processors: ignore_failure: true ignore_missing: true on_failure: -- append: - field: error.message - value: '{{{_ingest.on_failure_message}}}' -- set: - field: event.kind - value: pipeline_error \ No newline at end of file + - append: + field: error.message + value: '{{{_ingest.on_failure_message}}}' + - set: + field: event.kind + value: pipeline_error \ No newline at end of file diff --git a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audit.yml b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audit.yml index 19df29db13ba..4a60c6b2875c 100644 --- a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audit.yml +++ b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audit.yml @@ -137,3 +137,10 @@ processors: if: ctx.user?.name != null allow_duplicates: false ignore_failure: true +on_failure: + - set: + field: event.kind + value: pipeline_error + - append: + field: error.message + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dhcp.yml b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dhcp.yml index 62fc980901dc..b20a556d7a72 100644 --- a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dhcp.yml +++ b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dhcp.yml @@ -257,3 +257,10 @@ processors: if: ctx.infoblox_nios?.log?.dhcp?.client_hostname != null allow_duplicates: false ignore_failure: true +on_failure: + - set: + field: event.kind + value: pipeline_error + - append: + field: error.message + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dns.yml b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dns.yml index 45bf2ea0b95e..f81102e50a38 100644 --- a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dns.yml +++ b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dns.yml @@ -245,3 +245,10 @@ processors: - timestamp - repeat_message ignore_missing: true +on_failure: + - set: + field: event.kind + value: pipeline_error + - append: + field: error.message + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/infoblox_nios/manifest.yml b/packages/infoblox_nios/manifest.yml index fd160cd722ad..551d88c24946 100644 --- a/packages/infoblox_nios/manifest.yml +++ b/packages/infoblox_nios/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: infoblox_nios title: Infoblox NIOS -version: "1.8.0" +version: "1.9.0" license: basic description: Collect logs from Infoblox NIOS with Elastic Agent. type: integration diff --git a/packages/iptables/changelog.yml b/packages/iptables/changelog.yml index 380bc654fff8..b8918c5eab5e 100644 --- a/packages/iptables/changelog.yml +++ b/packages/iptables/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.9.0" + changes: + - description: Ensure event.kind is correctly set for pipeline errors. + type: enhancement + link: https://github.com/elastic/integrations/pull/6616 - version: "1.8.0" changes: - description: Update package to ECS 8.8.0. diff --git a/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 2259539b518b..75c17b72ce73 100644 --- a/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -373,6 +373,9 @@ on_failure: field: - _tmp ignore_failure: true + - set: + field: event.kind + value: pipeline_error - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/iptables/manifest.yml b/packages/iptables/manifest.yml index 5c486141a7c1..8a3291d29690 100644 --- a/packages/iptables/manifest.yml +++ b/packages/iptables/manifest.yml @@ -1,6 +1,6 @@ name: iptables title: Iptables -version: "1.8.0" +version: "1.9.0" description: Collect logs from Iptables with Elastic Agent. type: integration icons: From eef9c7089310838924498a16a2467f2103942f90 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Tue, 20 Jun 2023 12:33:39 +0930 Subject: [PATCH 2/2] revert: iptables change reason for revert: the iptables package depends on journald which is failing with ``` { "log.level": "error", "@timestamp": "2023-06-20T02:59:49.371Z", "message": "Input 'journald' failed with: input.go:130: input journald-iptables.log-7d04ae60-0f16-11ee-976c-55635f4b2750 failed (id=journald-iptables.log-7d04ae60-0f16-11ee-976c-55635f4b2750)\n\tinput.go:174: failed to create reader for /run/service_logs/iptables.journal journal (path=/run/service_logs/iptables.journal): reader.go:119: failed to open journal file /run/service_logs/iptables.journal (path=/run/service_logs/iptables.journal): failed to open journals in paths [\"/run/service_logs/iptables.journal\"]: protocol not supported", "component": { "binary": "filebeat", "dataset": "elastic_agent.filebeat", "id": "journald-default", "type": "journald" }, "log": { "source": "journald-default" }, "id": "journald-iptables.log-7d04ae60-0f16-11ee-976c-55635f4b2750", "ecs.version": "1.6.0", "log.logger": "input.journald", "log.origin": { "file.line": 131, "file.name": "compat/compat.go" }, "service.name": "filebeat" } ``` --- packages/iptables/changelog.yml | 5 ----- .../log/elasticsearch/ingest_pipeline/default.yml | 3 --- packages/iptables/manifest.yml | 2 +- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/packages/iptables/changelog.yml b/packages/iptables/changelog.yml index b8918c5eab5e..380bc654fff8 100644 --- a/packages/iptables/changelog.yml +++ b/packages/iptables/changelog.yml @@ -1,9 +1,4 @@ # newer versions go on top -- version: "1.9.0" - changes: - - description: Ensure event.kind is correctly set for pipeline errors. - type: enhancement - link: https://github.com/elastic/integrations/pull/6616 - version: "1.8.0" changes: - description: Update package to ECS 8.8.0. diff --git a/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 75c17b72ce73..2259539b518b 100644 --- a/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -373,9 +373,6 @@ on_failure: field: - _tmp ignore_failure: true - - set: - field: event.kind - value: pipeline_error - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/iptables/manifest.yml b/packages/iptables/manifest.yml index 8a3291d29690..5c486141a7c1 100644 --- a/packages/iptables/manifest.yml +++ b/packages/iptables/manifest.yml @@ -1,6 +1,6 @@ name: iptables title: Iptables -version: "1.9.0" +version: "1.8.0" description: Collect logs from Iptables with Elastic Agent. type: integration icons: