resolve merge conflicts

.buildkite/hooks/pre-command

@@ -12,6 +12,7 @@ export UPLOAD_SAFE_LOGS=${UPLOAD_SAFE_LOGS:-"0"}
 export SERVERLESS=${SERVERLESS:-"false"}
 export STACK_VERSION=${STACK_VERSION:-""}
 export FORCE_CHECK_ALL=${FORCE_CHECK_ALL:-"false"}
+export PUBLISH_COVERAGE_REPORTS=${PUBLISH_COVERAGE_REPORTS:-"false"}
 
 BASE_DIR=$(pwd)
 export BASE_DIR

.buildkite/pipeline.schedule-daily.yml

@@ -35,6 +35,7 @@ steps:
         SKIP_PUBLISHING: "true"
         FORCE_CHECK_ALL: "true"
         STACK_VERSION: 8.12-SNAPSHOT
+        PUBLISH_COVERAGE_REPORTS: "true"
     depends_on:
       - step: "check"
         allow_failure: false

.buildkite/pipeline.yml

@@ -67,6 +67,15 @@ steps:
       cpu: "8"
       memory: "4G"
 
+  - label: ":sonarqube: Continuous Code Inspection"
+    env:
+      VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/integrations/sonar-analyze-token"
+    agents:
+      image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
+    command: ".buildkite/scripts/run_sonar_scanner.sh"
+    artifact_paths:
+      - build/test-coverage/coverage_merged.xml
+
   - label: ":junit: Junit annotate"
     plugins:
       - junit-annotate#v2.4.1:

.buildkite/scripts/common.sh

@@ -15,6 +15,8 @@ export ELASTIC_PACKAGE_BIN=${WORKSPACE}/build/elastic-package
 
 API_BUILDKITE_PIPELINES_URL="https://api.buildkite.com/v2/organizations/elastic/pipelines/"
 
+COVERAGE_FORMAT="generic"
+COVERAGE_OPTIONS="--test-coverage --coverage-format=${COVERAGE_FORMAT}"
 
 running_on_buildkite() {
     if [[ "${BUILDKITE:-"false"}" == "true" ]]; then
@@ -685,11 +687,11 @@ install_package() {
 
 test_package_in_local_stack() {
     local package=$1
-    TEST_OPTIONS="-v --report-format xUnit --report-output file --test-coverage"
+    TEST_OPTIONS="-v --report-format xUnit --report-output file"
 
     echo "Test package: ${package}"
     # Run all test suites
-    ${ELASTIC_PACKAGE_BIN} test ${TEST_OPTIONS}
+    ${ELASTIC_PACKAGE_BIN} test ${TEST_OPTIONS} ${COVERAGE_OPTIONS}
     local ret=$?
     echo ""
     return $ret
@@ -703,10 +705,10 @@ test_package_in_serverless() {
     TEST_OPTIONS="-v --report-format xUnit --report-output file"
 
     echo "Test package: ${package}"
-    if ! ${ELASTIC_PACKAGE_BIN} test asset ${TEST_OPTIONS} --test-coverage ; then
+    if ! ${ELASTIC_PACKAGE_BIN} test asset ${TEST_OPTIONS} ${COVERAGE_OPTIONS}; then
         return 1
     fi
-    if ! ${ELASTIC_PACKAGE_BIN} test static ${TEST_OPTIONS} --test-coverage ; then
+    if ! ${ELASTIC_PACKAGE_BIN} test static ${TEST_OPTIONS} ${COVERAGE_OPTIONS}; then
         return 1
     fi
     # FIXME: adding test-coverage for serverless results in errors like this:

.buildkite/scripts/merge_xml.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Script to merge all the coverage XML files into just one file.
+# It supports XML files using generic test coverage report format:
+# https://docs.sonarsource.com/sonarqube/9.8/analyzing-source-code/test-coverage/generic-test-data/#generic-test-coverage
+
+sourceFolder="build/test-coverage"
+mergedCoverageFileName="coverage_merged.xml"
+
+pushd "${sourceFolder}" > /dev/null
+echo "Generating ${mergedCoverageFileName} into ${sourceFolder}..."
+echo '<?xml version="1.0" encoding="UTF-8"?>' > "${mergedCoverageFileName}"
+echo '<coverage version="1">' >> "${mergedCoverageFileName}"
+
+for file in coverage-*.xml; do
+  if [[ "$file" == "${mergedCoverageFileName}" ]]; then
+      continue
+  fi
+  echo " - Adding ${file}"
+  sed '1d;$d' "$file" | awk '/<file/,/<\/file>/' >> "${mergedCoverageFileName}"
+done
+
+echo '</coverage>' >> "${mergedCoverageFileName}"
+echo 'Done'
+
+popd > /dev/null
+

.buildkite/scripts/run_sonar_scanner.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -euo pipefail
+
+run_sonar_scanner() {
+    echo "--- Download coverage reports and merge them"
+    buildkite-agent artifact download build/test-coverage/coverage-*.xml .
+
+    echo "Merge all coverage reports"
+    .buildkite/scripts/merge_xml.sh
+
+    echo "--- Execute sonar scanner CLI"
+    /scan-source-code.sh
+}
+
+if [[ "${PUBLISH_COVERAGE_REPORTS:-"false"}" == "true" ]]; then
+    echo "Enabled sonnar scanner by PUBLISH_COVERAGE_REPORTS variable (Pipeline ${BUILDKITE_PIPELINE_SLUG})"
+    run_sonar_scanner
+    exit 0
+fi
+
+if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then
+    echo "Run sonar scanner from Pull Request (Pipeline ${BUILDKITE_PIPELINE_SLUG})"
+    run_sonar_scanner
+    exit 0
+fi
+
+echo "Skip coverage report"

.buildkite/scripts/trigger_integrations_in_parallel.sh

@@ -59,6 +59,7 @@ for package in ${PACKAGE_LIST}; do
         UPLOAD_SAFE_LOGS: ${UPLOAD_SAFE_LOGS}
       artifact_paths:
         - build/test-results/*.xml
+        - build/test-coverage/*.xml
         - build/benchmark-results/*.json
         - build/elastic-stack-dump/*/logs/*.log
         - build/elastic-stack-dump/*/logs/fleet-server-internal/**/*

.github/CODEOWNERS

@@ -194,6 +194,7 @@
 /packages/ibmmq @elastic/obs-infraobs-integrations
 /packages/iis @elastic/obs-infraobs-integrations
 /packages/imperva @elastic/sec-deployment-and-devices
+/packages/imperva_cloud_waf @elastic/security-service-integrations
 /packages/influxdb @elastic/obs-infraobs-integrations
 /packages/infoblox @elastic/security-service-integrations
 /packages/infoblox_bloxone_ddi @elastic/security-service-integrations

.gitignore

@@ -6,3 +6,6 @@
 dev/packages
 
 elastic-package
+
+# Folder created by the Sonar Scanner
+.scannerwork/

go.mod

@@ -4,7 +4,7 @@ go 1.21.0
 
 require (
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/elastic/elastic-package v0.96.0
+	github.com/elastic/elastic-package v0.96.1
 	github.com/elastic/go-licenser v0.4.1
 	github.com/elastic/package-registry v1.23.0
 	github.com/magefile/mage v1.15.0

go.sum

@@ -95,8 +95,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/elastic/elastic-integration-corpus-generator-tool v0.9.0 h1:SItTCXxjpeE2kr+3bptl7ZmG3P88pLDCXX23tObteTc=
 github.com/elastic/elastic-integration-corpus-generator-tool v0.9.0/go.mod h1:9Jjc1xsrnMwOA3TfG0arlXRCb0AhShCUU4YtwpBCURI=
-github.com/elastic/elastic-package v0.96.0 h1:RLkNBElCuRZ53/e09HvrZfpwlEYQzazJp77F913mYxM=
-github.com/elastic/elastic-package v0.96.0/go.mod h1:kR2uD/65qcUS8UClGT8QxoJsgdDjzUyI94X/PScF2Jg=
+github.com/elastic/elastic-package v0.96.1 h1:jjV4P+KG2N8yfutN77/VvOwYeU0XmEhExm5uVxtv8m4=
+github.com/elastic/elastic-package v0.96.1/go.mod h1:kR2uD/65qcUS8UClGT8QxoJsgdDjzUyI94X/PScF2Jg=
 github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo=
 github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elastic/go-licenser v0.4.1 h1:1xDURsc8pL5zYT9R29425J3vkHdt4RT5TNEMeRN48x4=

packages/activemq/_dev/build/docs/README.md

@@ -84,7 +84,9 @@ After the integration is successfully configured, clicking on the Assets tab of
 
 ## Troubleshooting
 
-If `host.ip` appears conflicted under the `log-*` or `metrics-*` data view, this issue can be resolved by [reindexing](https://www.elastic.co/guide/en/elasticsearch/reference/current/use-a-data-stream.html#reindex-with-a-data-stream) the indices of the `Audit`, `Log`, `Broker`, `Queue` and `Topic` data streams.
+If `host.ip` is shown conflicted under ``logs-*`` data view, then this issue can be solved by [reindexing](https://www.elastic.co/guide/en/elasticsearch/reference/current/use-a-data-stream.html#reindex-with-a-data-stream) the ``Audit`` and ``Log`` data stream's indices.
+
+If `host.ip` is shown conflicted under ``metrics-*`` data view, then this issue can be solved by [reindexing](https://www.elastic.co/guide/en/elasticsearch/reference/current/tsds-reindex.html) the ``Broker``, ``Queue`` and ``Topic`` data stream's indices.
 
 ## Logs
 

packages/activemq/changelog.yml

@@ -1,9 +1,14 @@
 # newer versions go on top
-- version: "0.16.1"
+- version: "0.16.2"
   changes:
     - description: Inline "by reference" visualizations
       type: enhancement
       link: https://github.com/elastic/integrations/pull/9011
+- version: 0.16.1
+  changes:
+    - description: Update link to the correct reindexing procedure.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/9021
 - version: 0.16.0
   changes:
     - description: Update README to use documentation guidelines.

packages/activemq/docs/README.md

@@ -84,7 +84,9 @@ After the integration is successfully configured, clicking on the Assets tab of
 
 ## Troubleshooting
 
-If `host.ip` appears conflicted under the `log-*` or `metrics-*` data view, this issue can be resolved by [reindexing](https://www.elastic.co/guide/en/elasticsearch/reference/current/use-a-data-stream.html#reindex-with-a-data-stream) the indices of the `Audit`, `Log`, `Broker`, `Queue` and `Topic` data streams.
+If `host.ip` is shown conflicted under ``logs-*`` data view, then this issue can be solved by [reindexing](https://www.elastic.co/guide/en/elasticsearch/reference/current/use-a-data-stream.html#reindex-with-a-data-stream) the ``Audit`` and ``Log`` data stream's indices.
+
+If `host.ip` is shown conflicted under ``metrics-*`` data view, then this issue can be solved by [reindexing](https://www.elastic.co/guide/en/elasticsearch/reference/current/tsds-reindex.html) the ``Broker``, ``Queue`` and ``Topic`` data stream's indices.
 
 ## Logs
 

packages/activemq/manifest.yml

@@ -1,6 +1,6 @@
 name: activemq
 title: ActiveMQ
-version: "0.16.1"
+version: "0.16.2"
 description: Collect logs and metrics from ActiveMQ instances with Elastic Agent.
 type: integration
 icons:

packages/apache_spark/_dev/build/docs/README.md

@@ -65,7 +65,7 @@ Follow the same set of steps for Spark Worker, Driver and Executor.
 
 ### Troubleshooting
 
-If host.ip is shown conflicted under ``metrics-*`` data view, then this issue can be solved by [reindexing](https://www.elastic.co/guide/en/elasticsearch/reference/current/use-a-data-stream.html#reindex-with-a-data-stream) the ``Application``, ``Driver``, ``Executor`` and ``Node`` data stream's indices.
+If host.ip is shown conflicted under ``metrics-*`` data view, then this issue can be solved by [reindexing](https://www.elastic.co/guide/en/elasticsearch/reference/current/tsds-reindex.html) the ``Application``, ``Driver``, ``Executor`` and ``Node`` data stream's indices.
 
 ## Metrics
 

packages/apache_spark/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.0.1"
+  changes:
+    - description: Update the link to the correct reindexing procedure.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/9021
 - version: "1.0.0"
   changes:
     - description: Make Apache Spark GA.

packages/apache_spark/docs/README.md

@@ -65,7 +65,7 @@ Follow the same set of steps for Spark Worker, Driver and Executor.
 
 ### Troubleshooting
 
-If host.ip is shown conflicted under ``metrics-*`` data view, then this issue can be solved by [reindexing](https://www.elastic.co/guide/en/elasticsearch/reference/current/use-a-data-stream.html#reindex-with-a-data-stream) the ``Application``, ``Driver``, ``Executor`` and ``Node`` data stream's indices.
+If host.ip is shown conflicted under ``metrics-*`` data view, then this issue can be solved by [reindexing](https://www.elastic.co/guide/en/elasticsearch/reference/current/tsds-reindex.html) the ``Application``, ``Driver``, ``Executor`` and ``Node`` data stream's indices.
 
 ## Metrics
 

packages/apache_spark/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.0"
 name: apache_spark
 title: Apache Spark
-version: "1.0.0"
+version: "1.0.1"
 description: Collect metrics from Apache Spark with Elastic Agent.
 type: integration
 categories:

packages/cassandra/changelog.yml

@@ -4,6 +4,11 @@
     - description: Inline "by reference" visualizations
       type: enhancement
       link: https://github.com/elastic/integrations/pull/9011
+- version: 1.10.1
+  changes:
+    - description: Update the link to the correct reindexing procedure.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/9021
 - version: 1.10.0
   changes:
     - description: Update the package format_version to 3.0.0.

packages/cassandra/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.0"
 name: cassandra
 title: Cassandra
-version: "1.10.1"
+version: "1.10.2"
 description: This Elastic integration collects logs and metrics from cassandra.
 type: integration
 categories:

packages/cel/changelog.yml

@@ -1,3 +1,8 @@
+- version: "1.8.0"
+  changes:
+    - description: Clarify naming of input.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/9018
 - version: "1.7.1"
   changes:
     - description: Changed owners

packages/cel/docs/README.md

@@ -1,8 +1,8 @@
 # CEL Custom API input integration
 
-The CEL custom API input integration is used to ingest data from custom HTTP and local file-system APIs that do not currently have an existing integration.
+The Common Expression Language (CEL) custom API input integration is used to ingest data from custom HTTP and local file-system APIs that do not currently have an existing integration.
 
-The input itself supports making both HTTP requests and file-system read operations, and keeping running and persistent state on information from the last collected events. The input performs [Common Expression Language](https://opensource.google.com/projects/cel) with a [set of standard extensions](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-cel.html#_cel_extension_libraries) to both obtain input data from the API and then process the data into events that are published to Elasticsearch.
+The input itself supports making both HTTP requests and file-system read operations, and keeping running and persistent state on information from the last collected events. The input performs [Common Expression Language](https://opensource.google.com/projects/cel) expression evaluation with a [set of standard extensions](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-cel.html#_cel_extension_libraries) to both obtain input data from the API and then process the data into events that are published to Elasticsearch.
 
 ## Configuration
 

packages/cel/manifest.yml

@@ -1,9 +1,9 @@
 format_version: "3.0.0"
 name: cel
-title: CEL Custom API
+title: Custom API using Common Expression Language
 description: Collect custom events from an API with Elastic agent
 type: input
-version: "1.7.1"
+version: "1.8.0"
 categories:
   - custom
 conditions:

packages/cisco_ios/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.24.2"
+  changes:
+    - description: Fix time zone configuration not being applied.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/9000
 - version: "1.24.1"
   changes:
     - description: Changed owners

packages/cisco_ios/data_stream/log/_dev/test/pipeline/test-asr920.log-expected.json

@@ -284,7 +284,7 @@
             ]
         },
         {
-            "@timestamp": "2023-08-03T08:11:02.204Z",
+            "@timestamp": "2024-08-03T08:11:02.204Z",
             "cisco": {
                 "ios": {
                     "facility": "LINEPROTO",
@@ -326,7 +326,7 @@
             ]
         },
         {
-            "@timestamp": "2023-08-03T08:08:47.142Z",
+            "@timestamp": "2024-08-03T08:08:47.142Z",
             "cisco": {
                 "ios": {
                     "access_list": "ACL_CE-SECURITY",
@@ -405,7 +405,7 @@
             ]
         },
         {
-            "@timestamp": "2023-08-03T08:04:47.140Z",
+            "@timestamp": "2024-08-03T08:04:47.140Z",
             "cisco": {
                 "ios": {
                     "access_list": "ACL_CE-SECURITY",
@@ -488,7 +488,7 @@
             ]
         },
         {
-            "@timestamp": "2023-08-03T08:09:55.769Z",
+            "@timestamp": "2024-08-03T08:09:55.769Z",
             "cisco": {
                 "ios": {
                     "facility": "SNMP-SW1",

packages/cisco_ios/data_stream/log/_dev/test/pipeline/test-badauth.log-expected.json

@@ -1,7 +1,7 @@
 {
     "expected": [
         {
-            "@timestamp": "2023-12-08T20:07:53.081Z",
+            "@timestamp": "2024-12-08T20:07:53.081Z",
             "cisco": {
                 "ios": {
                     "facility": "TCP",
@@ -62,7 +62,7 @@
             ]
         },
         {
-            "@timestamp": "2023-12-08T20:07:53.081Z",
+            "@timestamp": "2024-12-08T20:07:53.081Z",
             "cisco": {
                 "ios": {
                     "facility": "TCP",
@@ -123,7 +123,7 @@
             ]
         },
         {
-            "@timestamp": "2023-12-08T20:07:53.081Z",
+            "@timestamp": "2024-12-08T20:07:53.081Z",
             "cisco": {
                 "ios": {
                     "facility": "TCP",
@@ -183,7 +183,7 @@
             ]
         },
         {
-            "@timestamp": "2023-12-08T20:07:53.081Z",
+            "@timestamp": "2024-12-08T20:07:53.081Z",
             "cisco": {
                 "ios": {
                     "facility": "TCP",