Merge branch 'elastic:main' into awsfargate_deprecate_memory_field

.buildkite/hooks/pre-command

@@ -12,6 +12,7 @@ export UPLOAD_SAFE_LOGS=${UPLOAD_SAFE_LOGS:-"0"}
 export SERVERLESS=${SERVERLESS:-"false"}
 export STACK_VERSION=${STACK_VERSION:-""}
 export FORCE_CHECK_ALL=${FORCE_CHECK_ALL:-"false"}
+export PUBLISH_COVERAGE_REPORTS=${PUBLISH_COVERAGE_REPORTS:-"false"}
 
 BASE_DIR=$(pwd)
 export BASE_DIR

.buildkite/pipeline.schedule-daily.yml

@@ -35,6 +35,7 @@ steps:
         SKIP_PUBLISHING: "true"
         FORCE_CHECK_ALL: "true"
         STACK_VERSION: 8.12-SNAPSHOT
+        PUBLISH_COVERAGE_REPORTS: "true"
     depends_on:
       - step: "check"
         allow_failure: false

.buildkite/pipeline.serverless.yml

@@ -4,14 +4,13 @@ env:
   SETUP_GVM_VERSION: "v0.5.1"
   LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
   DOCKER_COMPOSE_VERSION: "v2.24.1"
-  DOCKER_VERSION: "24.0.7"
+  DOCKER_VERSION: "false"
   KIND_VERSION: 'v0.20.0'
   K8S_VERSION: 'v1.29.0'
   YQ_VERSION: 'v4.35.2'
   # Elastic package settings
   # Manage docker output/logs
-  ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true"
-  ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true"
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_VERBOSE_OUTPUT: "true"
   # Default license to use by `elastic-package build`
   ELASTIC_PACKAGE_REPOSITORY_LICENSE: "licenses/Elastic-2.0.txt"
   # Link definitions path (full path to be set in the corresponding step)

.buildkite/pipeline.yml

@@ -4,16 +4,15 @@ env:
   SETUP_GVM_VERSION: "v0.5.1"
   LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
   DOCKER_COMPOSE_VERSION: "v2.24.1"
-  DOCKER_VERSION: "24.0.7"
+  DOCKER_VERSION: "false"
   KIND_VERSION: 'v0.20.0'
   K8S_VERSION: 'v1.29.0'
   YQ_VERSION: 'v4.35.2'
   JQ_VERSION: '1.7'
   GH_CLI_VERSION: "2.29.0"
   # Elastic package settings
   # Manage docker output/logs
-  ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true"
-  ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true"
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_VERBOSE_OUTPUT: "true"
   # Default license to use by `elastic-package build`
   ELASTIC_PACKAGE_REPOSITORY_LICENSE: "licenses/Elastic-2.0.txt"
   # Link definitions path (full path to be set in the corresponding step)
@@ -68,6 +67,15 @@ steps:
       cpu: "8"
       memory: "4G"
 
+  - label: ":sonarqube: Continuous Code Inspection"
+    env:
+      VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/integrations/sonar-analyze-token"
+    agents:
+      image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
+    command: ".buildkite/scripts/run_sonar_scanner.sh"
+    artifact_paths:
+      - build/test-coverage/coverage_merged.xml
+
   - label: ":junit: Junit annotate"
     plugins:
       - junit-annotate#v2.4.1:

.buildkite/scripts/common.sh

@@ -15,6 +15,8 @@ export ELASTIC_PACKAGE_BIN=${WORKSPACE}/build/elastic-package
 
 API_BUILDKITE_PIPELINES_URL="https://api.buildkite.com/v2/organizations/elastic/pipelines/"
 
+COVERAGE_FORMAT="generic"
+COVERAGE_OPTIONS="--test-coverage --coverage-format=${COVERAGE_FORMAT}"
 
 running_on_buildkite() {
     if [[ "${BUILDKITE:-"false"}" == "true" ]]; then
@@ -137,6 +139,11 @@ with_mage() {
 
 with_docker() {
     echo "--- Setting up the Docker environment..."
+    echo "- Current docker client version:"
+    docker version -f json  | jq -r '.Client.Version'
+    echo "- Current docker server version:"
+    docker version -f json  | jq -r '.Server.Version'
+
     if [[ "${DOCKER_VERSION:-"false"}" == "false" ]]; then
         echo "Skip docker installation"
         return
@@ -150,7 +157,9 @@ with_docker() {
     local debian_version="5:${DOCKER_VERSION}-1~ubuntu.${ubuntu_version}~${ubuntu_codename}"
 
     sudo sudo mkdir -p /etc/apt/keyrings
-    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+    if [ ! -f /etc/apt/keyrings/docker.gpg ]; then
+        curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+    fi
     echo "deb [arch=${architecture} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${ubuntu_codename} stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
     sudo apt-get update
     sudo DEBIAN_FRONTEND=noninteractive apt-get install --allow-downgrades -y "docker-ce=${debian_version}"
@@ -678,11 +687,11 @@ install_package() {
 
 test_package_in_local_stack() {
     local package=$1
-    TEST_OPTIONS="-v --report-format xUnit --report-output file --test-coverage"
+    TEST_OPTIONS="-v --report-format xUnit --report-output file"
 
     echo "Test package: ${package}"
     # Run all test suites
-    ${ELASTIC_PACKAGE_BIN} test ${TEST_OPTIONS}
+    ${ELASTIC_PACKAGE_BIN} test ${TEST_OPTIONS} ${COVERAGE_OPTIONS}
     local ret=$?
     echo ""
     return $ret
@@ -696,10 +705,10 @@ test_package_in_serverless() {
     TEST_OPTIONS="-v --report-format xUnit --report-output file"
 
     echo "Test package: ${package}"
-    if ! ${ELASTIC_PACKAGE_BIN} test asset ${TEST_OPTIONS} --test-coverage ; then
+    if ! ${ELASTIC_PACKAGE_BIN} test asset ${TEST_OPTIONS} ${COVERAGE_OPTIONS}; then
         return 1
     fi
-    if ! ${ELASTIC_PACKAGE_BIN} test static ${TEST_OPTIONS} --test-coverage ; then
+    if ! ${ELASTIC_PACKAGE_BIN} test static ${TEST_OPTIONS} ${COVERAGE_OPTIONS}; then
         return 1
     fi
     # FIXME: adding test-coverage for serverless results in errors like this:

.buildkite/scripts/merge_xml.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Script to merge all the coverage XML files into just one file.
+# It supports XML files using generic test coverage report format:
+# https://docs.sonarsource.com/sonarqube/9.8/analyzing-source-code/test-coverage/generic-test-data/#generic-test-coverage
+
+sourceFolder="build/test-coverage"
+mergedCoverageFileName="coverage_merged.xml"
+
+pushd "${sourceFolder}" > /dev/null
+echo "Generating ${mergedCoverageFileName} into ${sourceFolder}..."
+echo '<?xml version="1.0" encoding="UTF-8"?>' > "${mergedCoverageFileName}"
+echo '<coverage version="1">' >> "${mergedCoverageFileName}"
+
+for file in coverage-*.xml; do
+  if [[ "$file" == "${mergedCoverageFileName}" ]]; then
+      continue
+  fi
+  echo " - Adding ${file}"
+  sed '1d;$d' "$file" | awk '/<file/,/<\/file>/' >> "${mergedCoverageFileName}"
+done
+
+echo '</coverage>' >> "${mergedCoverageFileName}"
+echo 'Done'
+
+popd > /dev/null
+

.buildkite/scripts/run_sonar_scanner.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -euo pipefail
+
+run_sonar_scanner() {
+    echo "--- Download coverage reports and merge them"
+    buildkite-agent artifact download build/test-coverage/coverage-*.xml .
+
+    echo "Merge all coverage reports"
+    .buildkite/scripts/merge_xml.sh
+
+    echo "--- Execute sonar scanner CLI"
+    /scan-source-code.sh
+}
+
+if [[ "${PUBLISH_COVERAGE_REPORTS:-"false"}" == "true" ]]; then
+    echo "Enabled sonnar scanner by PUBLISH_COVERAGE_REPORTS variable (Pipeline ${BUILDKITE_PIPELINE_SLUG})"
+    run_sonar_scanner
+    exit 0
+fi
+
+if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then
+    echo "Run sonar scanner from Pull Request (Pipeline ${BUILDKITE_PIPELINE_SLUG})"
+    run_sonar_scanner
+    exit 0
+fi
+
+echo "Skip coverage report"

.buildkite/scripts/trigger_integrations_in_parallel.sh

@@ -59,6 +59,7 @@ for package in ${PACKAGE_LIST}; do
         UPLOAD_SAFE_LOGS: ${UPLOAD_SAFE_LOGS}
       artifact_paths:
         - build/test-results/*.xml
+        - build/test-coverage/*.xml
         - build/benchmark-results/*.json
         - build/elastic-stack-dump/*/logs/*.log
         - build/elastic-stack-dump/*/logs/fleet-server-internal/**/*