From 0df4f27d2bedf3c0b8aaca67f71887133b2e0ceb Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 30 Apr 2024 15:44:39 +0200
Subject: [PATCH 01/20] Add pipeline to test with serverless daily

Move workspace and tmp folder vars to pre-command hook

Use path to elastic-package binary in pre-exit hook. PATH variable does
not keep the value from the pipeline step.
---
 .buildkite/hooks/pre-command                  |  30 +++-
 .buildkite/hooks/pre-exit                     |  10 +-
 .buildkite/pipeline.serverless.yml            |  73 ++++++++++
 .buildkite/pipeline.yml                       | 134 ++++++++++--------
 .buildkite/pull-requests.json                 |  16 +++
 .buildkite/scripts/integration_tests.sh       |  60 +++-----
 .buildkite/scripts/release.sh                 |   7 +-
 .buildkite/scripts/test-with-integrations.sh  |   5 -
 .../scripts/test_packages_with_serverless.sh  |  70 +++++++++
 .buildkite/scripts/tooling.sh                 |  91 +++++++++++-
 catalog-info.yaml                             |  51 +++++++
 scripts/test-check-packages.sh                |  35 +++--
 12 files changed, 454 insertions(+), 128 deletions(-)
 create mode 100644 .buildkite/pipeline.serverless.yml
 create mode 100644 .buildkite/scripts/test_packages_with_serverless.sh

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index b051542557..83d3c37c56 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -6,11 +6,18 @@ set -euo pipefail
 GO_VERSION=$(cat .go-version)
 export GO_VERSION
 
+export SERVERLESS=${SERVERLESS:-"false"}
+export WORKSPACE=$(pwd)
+
+
 GCP_SERVICE_ACCOUNT_SECRET_PATH=secret/ci/elastic-elastic-package/gcp-service-account
 AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_account_auth
 GITHUB_TOKEN_VAULT_PATH=kv/ci-shared/platform-ingest/github_token
 PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account
 
+EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa
+EC_DATA_PATH=secret/ci/elastic-elastic-package/ec_data
+
 # variables required for terraform
 export ENVIRONMENT="ci"
 REPO=$(repo_name "${BUILDKITE_REPO}")
@@ -46,13 +53,21 @@ export CREATED_DATE
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 
+export TMP_FOLDER_TEMPLATE_BASE="tmp.${REPO}"
+export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
+
 is_step_required_to_upload_safe_logs() {
-    if [[ "$BUILDKITE_PIPELINE_SLUG" != "elastic-package" ]]; then
+    if [[ "$BUILDKITE_PIPELINE_SLUG" != "elastic-package" && "$BUILDKITE_PIPELINE_SLUG" != "elastic-package-test-serverless" ]]; then
         return 1
     fi
+    # steps from elastic-package
     if [[ "$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives ]]; then
         return 0
     fi
+    # steps from elastic-package-test-serverless
+    if [[ "$BUILDKITE_STEP_KEY" == "test-serverless" ]]; then
+        return 0
+    fi
     return 1
 }
 
@@ -139,3 +154,16 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-cloud-cleanup" && "$BUILDKI
     export ELASTIC_PACKAGE_GCP_EMAIL_SECRET
 fi
 
+
+# TODO: Replace if condition
+# if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package-test-serverless" ]]; then
+if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package" ]]; then
+    if [[ "${BUILDKITE_STEP_KEY}" == "test-serverless" ]]; then
+        EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey "${EC_TOKEN_PATH}")
+        export EC_API_KEY_SECRET
+        EC_HOST_SECRET=$(retry 5 vault kv get -field url "${EC_TOKEN_PATH}")
+        export EC_HOST_SECRET
+        EC_REGION_SECRET=$(retry 5 vault read -field region_qa "${EC_DATA_PATH}")
+        export EC_REGION_SECRET
+    fi
+fi
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index 591b7acfe2..bae53afdee 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -4,6 +4,15 @@ source .buildkite/scripts/tooling.sh
 
 set -euo pipefail
 
+# TODO: change pipeline slug
+if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package" && "${BUILDKITE_STEP_KEY}" == "test-serverless" ]]; then
+    echo "--- Take down the Elastic stack"
+    # BUILDKITE resets PATH contents in pre-exit hook, but elastic-package
+    # is already installed in the test_serverless pipeline step, accessing
+    # directly to the binary
+    EC_API_KEY="${EC_API_KEY_SECRET}" EC_HOST="${EC_HOST_SECRET}" "${HOME}"/go/bin/elastic-package down -v
+fi
+
 unset_secrets
 
 # integrations-parallel-gcp
@@ -15,4 +24,3 @@ unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
 unset ELASTIC_PACKAGE_AWS_SECRET_KEY
 unset AWS_ACCESS_KEY_ID
 unset AWS_SECRET_ACCESS_KEY
-
diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml
new file mode 100644
index 0000000000..56b03ab6e5
--- /dev/null
+++ b/.buildkite/pipeline.serverless.yml
@@ -0,0 +1,73 @@
+env:
+  SETUP_GVM_VERSION: 'v0.5.2' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_VERBOSE_OUTPUT: "true"
+  DOCKER_COMPOSE_VERSION: "v2.24.1"
+  DOCKER_VERSION: "26.1.0"
+  KIND_VERSION: 'v0.20.0'
+  K8S_VERSION: 'v1.29.0'
+  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
+
+steps:
+  - input: "Input values for the variables"
+    key: "input-variables"
+    fields:
+    - select: "SERVERLESS_PROJECT"
+      key: "SERVERLESS_PROJECT"
+      options:
+        - label: "observability"
+          value: "observability"
+        - label: "security"
+          value: "security"
+      default: "observability"
+    if: "build.source == 'ui'"
+
+  - wait: ~
+    if: "build.source == 'ui'"
+    allow_dependency_failure: false
+
+  - label: ":go: Run check-static"
+    key: check-static
+    command: "make check-static"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+
+  - label: ":go: :linux: Run unit tests"
+    key: unit-tests-linux
+    command: "make test-go-ci"
+    artifact_paths:
+      - "build/test-results/*.xml"
+      - "build/test-coverage/*.xml"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+
+  - wait: ~
+    continue_on_failure: true
+
+  - label: ":pipeline: Test Serverless Integration tests"
+    key: test-serverless
+    command: ".buildkite/scripts/test_packages_with_serverless.sh"
+    env:
+      SERVERLESS_PROJECT: "${SERVERLESS_PROJECT:-observability}"
+      UPLOAD_SAFE_LOGS: 1
+    artifact_paths:
+      - build/test-results/*.xml
+      - build/test-coverage/coverage-*.xml
+    depends_on:
+      - step: check-static
+        allow_failure: false
+      - step: unit-tests-linux
+        allow_failure: false
+
+  - wait: ~
+    continue_on_failure: true
+
+  - label: ":junit: Junit annotate"
+    plugins:
+      - junit-annotate#v2.4.1:
+          artifacts: "build/test-results/*.xml"
+    agents:
+      provider: "gcp"  # junit plugin requires docker
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 42e868699a..96f88c556c 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -9,76 +9,88 @@ env:
   WINDOWS_AGENT_IMAGE: "family/ci-windows-2022"
 
 steps:
-  - label: ":go: Run check-static"
-    key: check-static
-    command: "make check-static"
-    agents:
-      image: "${LINUX_AGENT_IMAGE}"
-      cpu: "8"
-      memory: "4G"
+  # - label: ":go: Run check-static"
+  #   key: check-static
+  #   command: "make check-static"
+  #   agents:
+  #     image: "${LINUX_AGENT_IMAGE}"
+  #     cpu: "8"
+  #     memory: "4G"
 
-  - label: ":go: :linux: Run unit tests"
-    key: unit-tests-linux
-    command: "make test-go-ci"
-    artifact_paths:
-      - "build/test-results/*.xml"
-      - "build/test-coverage/*.xml"
-    agents:
-      image: "${LINUX_AGENT_IMAGE}"
-      cpu: "8"
-      memory: "4G"
+  # - label: ":go: :linux: Run unit tests"
+  #   key: unit-tests-linux
+  #   command: "make test-go-ci"
+  #   artifact_paths:
+  #     - "build/test-results/*.xml"
+  #     - "build/test-coverage/*.xml"
+  #   agents:
+  #     image: "${LINUX_AGENT_IMAGE}"
+  #     cpu: "8"
+  #     memory: "4G"
 
-  - label: ":go: :windows: Run unit tests"
-    key: unit-tests-windows
-    command: ".buildkite/scripts/unit_tests_windows.ps1"
+  # - label: ":go: :windows: Run unit tests"
+  #   key: unit-tests-windows
+  #   command: ".buildkite/scripts/unit_tests_windows.ps1"
+  #   agents:
+  #     provider: "gcp"
+  #     image: "${WINDOWS_AGENT_IMAGE}"
+  #   artifact_paths:
+  #     - "TEST-unit-windows.xml"
+
+  - label: ":package: test serverless"
+    key: test-serverless
+    command: ".buildkite/scripts/test_packages_with_serverless.sh"
     agents:
       provider: "gcp"
-      image: "${WINDOWS_AGENT_IMAGE}"
+    env:
+      SERVERLESS_PROJECT: observability
+      UPLOAD_SAFE_LOGS: 1
     artifact_paths:
-      - "TEST-unit-windows.xml"
+      - build/test-results/*.xml
+      - build/test-coverage/coverage-*.xml
 
-  - wait: ~
-    continue_on_failure: true
+  # - wait: ~
+  #   continue_on_failure: true
 
-  - label: ":pipeline: Trigger Integration tests"
-    command: ".buildkite/pipeline.trigger.integration.tests.sh | buildkite-agent pipeline upload"
-    depends_on:
-      - step: check-static
-        allow_failure: false
-      - step: unit-tests-linux
-        allow_failure: false
-      - step: unit-tests-windows
-        allow_failure: false
+  # - label: ":pipeline: Trigger Integration tests"
+  #   command: ".buildkite/pipeline.trigger.integration.tests.sh | buildkite-agent pipeline upload"
+  #   depends_on:
+  #     - step: check-static
+  #       allow_failure: false
+  #     - step: unit-tests-linux
+  #       allow_failure: false
+  #     - step: unit-tests-windows
+  #       allow_failure: false
 
-  - wait: ~
-    continue_on_failure: true
+  # - wait: ~
+  #   continue_on_failure: true
 
-  - label: ":junit: Transform windows paths to linux for Junit plugin"
-    commands:
-      - buildkite-agent artifact download "*-windows.xml" . --step unit-tests-windows
-      - mkdir -p build/test-results
-      - for file in $(ls *-windows.xml); do mv $$file build/test-results/; done
-    agents:
-      image: "${LINUX_AGENT_IMAGE}"
-      cpu: "8"
-      memory: "4G"
-    artifact_paths:
-      - "build/test-results/*.xml"
+  # - label: ":junit: Transform windows paths to linux for Junit plugin"
+  #   commands:
+  #     - buildkite-agent artifact download "*-windows.xml" . --step unit-tests-windows
+  #     - mkdir -p build/test-results
+  #     - for file in $(ls *-windows.xml); do mv $$file build/test-results/; done
+  #   agents:
+  #     image: "${LINUX_AGENT_IMAGE}"
+  #     cpu: "8"
+  #     memory: "4G"
+  #   artifact_paths:
+  #     - "build/test-results/*.xml"
 
-  - wait: ~
-    continue_on_failure: true
+  # - wait: ~
+  #   continue_on_failure: true
 
-  - label: ":junit: Junit annotate"
-    plugins:
-      - junit-annotate#v2.4.1:
-          artifacts: "build/test-results/*.xml"
-    agents:
-      provider: "gcp"  # junit plugin requires docker
+  # - label: ":junit: Junit annotate"
+  #   plugins:
+  #     - junit-annotate#v2.4.1:
+  #         artifacts: "build/test-results/*.xml"
+  #   agents:
+  #     provider: "gcp"  # junit plugin requires docker
 
-  - label: ":github: Release"
-    key: "release"
-    if: |
-      build.tag =~ /^v[0-9]+[.][0-9]+[.][0-9]+$$/
-    command: ".buildkite/scripts/release.sh"
-    agents:
-      provider: "gcp"
+  # - label: ":github: Release"
+  #   key: "release"
+  #   if: |
+  #     build.tag =~ /^v[0-9]+[.][0-9]+[.][0-9]+$$/
+  #   command: ".buildkite/scripts/release.sh"
+  #   agents:
+  #     provider: "gcp"
diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json
index 1960b6a69d..51bbc9c9b7 100644
--- a/.buildkite/pull-requests.json
+++ b/.buildkite/pull-requests.json
@@ -31,6 +31,22 @@
         "skip_target_branches": [ ],
         "skip_ci_on_only_changed": [ ],
         "always_require_ci_on_changed": [ ]
+     },
+     {
+        "enabled": true,
+        "pipelineSlug": "elastic-package-test-serverless",
+        "allow_org_users": true,
+        "allowed_repo_permissions": ["admin", "write"],
+        "allowed_list": [ ],
+        "set_commit_status": false,
+        "build_on_commit": false,
+        "build_on_comment": true,
+        "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:test)\\W+(?:serverless))$",
+        "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:test)\\W+(?:serverless))$",
+        "skip_ci_labels": [ ],
+        "skip_target_branches": [ ],
+        "skip_ci_on_only_changed": [ ],
+        "always_require_ci_on_changed": [ ]
      }
    ]
 }
diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh
index a8466db85a..6b70ffdded 100755
--- a/.buildkite/scripts/integration_tests.sh
+++ b/.buildkite/scripts/integration_tests.sh
@@ -5,9 +5,6 @@ source .buildkite/scripts/tooling.sh
 
 set -euo pipefail
 
-WORKSPACE="$(pwd)"
-TMP_FOLDER_TEMPLATE_BASE="tmp.elastic-package"
-
 cleanup() {
     local error_code=$?
 
@@ -39,15 +36,15 @@ PARALLEL_TARGET="test-check-packages-parallel"
 FALSE_POSITIVES_TARGET="test-check-packages-false-positives"
 KIND_TARGET="test-check-packages-with-kind"
 SYSTEM_TEST_FLAGS_TARGET="test-system-test-flags"
-TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
 GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
 ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT=${ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT:-"false"}
 
 REPO_NAME=$(repo_name "${BUILDKITE_REPO}")
-REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
+export REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
 TARGET=""
 PACKAGE=""
-while getopts ":t:p:h" o; do
+SERVERLESS="false"
+while getopts ":t:p:sh" o; do
     case "${o}" in
         t)
             TARGET=${OPTARG}
@@ -55,6 +52,9 @@ while getopts ":t:p:h" o; do
         p)
             PACKAGE=${OPTARG}
             ;;
+        s)
+            SERVERLESS="true"
+            ;;
         h)
             usage
             exit 0
@@ -78,44 +78,20 @@ if [[ "${TARGET}" == "" ]]; then
     exit 1
 fi
 
-google_cloud_auth_safe_logs() {
-    local gsUtilLocation=""
-    gsUtilLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE}")
-
-    local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
-
-    echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > "${secretFileLocation}"
-
-    google_cloud_auth "${secretFileLocation}"
-}
-
-upload_safe_logs() {
-    local bucket="$1"
-    local source="$2"
-    local target="$3"
-
-    if ! ls ${source} 2>&1 > /dev/null ; then
-        echo "upload_safe_logs: artifacts files not found, nothing will be archived"
-        return
-    fi
-
-    google_cloud_auth_safe_logs
-
-    gsutil cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}"
-
-    google_cloud_logout_active_account
-}
-
 add_bin_path
 
-echo "--- install go"
-with_go
+if [[ "$SERVERLESS" == "false" ]]; then
+    # If packages are tested with Serverless, these action are already performed
+    # here: .buildkite/scripts/test_packages_with_serverless.sh
+    echo "--- install go"
+    with_go
 
-echo "--- install docker"
-with_docker
+    echo "--- install docker"
+    with_docker
 
-echo "--- install docker-compose plugin"
-with_docker_compose_plugin
+    echo "--- install docker-compose plugin"
+    with_docker_compose_plugin
+fi
 
 if [[ "${TARGET}" == "${KIND_TARGET}" || "${TARGET}" == "${SYSTEM_TEST_FLAGS_TARGET}" ]]; then
     echo "--- install kubectl & kind"
@@ -128,7 +104,7 @@ if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSIT
 
     # allow to fail this command, to be able to upload safe logs
     set +e
-    make PACKAGE_UNDER_TEST="${PACKAGE}" "${TARGET}"
+    make SERVERLESS="${SERVERLESS}" PACKAGE_UNDER_TEST="${PACKAGE}" "${TARGET}"
     testReturnCode=$?
     set -e
 
@@ -162,7 +138,7 @@ if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSIT
     fi
 
     if [ $testReturnCode != 0 ]; then
-        echo "make PACKAGE_UDER_TEST=${PACKAGE} ${TARGET} failed with ${testReturnCode}"
+        echo "make SERVERLESS=${SERVERLESS} PACKAGE_UDER_TEST=${PACKAGE} ${TARGET} failed with ${testReturnCode}"
         exit ${testReturnCode}
     fi
 
diff --git a/.buildkite/scripts/release.sh b/.buildkite/scripts/release.sh
index af033dec13..a015b52f59 100755
--- a/.buildkite/scripts/release.sh
+++ b/.buildkite/scripts/release.sh
@@ -1,5 +1,8 @@
 #!/bin/bash
 
+source .buildkite/scripts/install_deps.sh
+source .buildkite/scripts/tooling.sh
+
 set -euo pipefail
 
 cleanup() {
@@ -7,11 +10,9 @@ cleanup() {
 }
 trap cleanup exit
 
-WORKSPACE="/tmp/bin-buildkite/"
+export WORKSPACE="/tmp/bin-buildkite/"
 
 VERSION=""
-source .buildkite/scripts/install_deps.sh
-source .buildkite/scripts/tooling.sh
 
 add_bin_path
 with_go
diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index e1eadb53b7..cd4ce6ddc7 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -3,11 +3,6 @@ source .buildkite/scripts/install_deps.sh
 
 set -euo pipefail
 
-WORKSPACE="$(pwd)"
-
-TMP_FOLDER_TEMPLATE_BASE="tmp.${GITHUB_PR_BASE_REPO}"
-TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
-
 cleanup() {
     echo "Deleting temporal files..."
     cd "${WORKSPACE}"
diff --git a/.buildkite/scripts/test_packages_with_serverless.sh b/.buildkite/scripts/test_packages_with_serverless.sh
new file mode 100644
index 0000000000..71ce1311b1
--- /dev/null
+++ b/.buildkite/scripts/test_packages_with_serverless.sh
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+source .buildkite/scripts/tooling.sh
+source .buildkite/scripts/install_deps.sh
+
+set -euo pipefail
+
+UPLOAD_SAFE_LOGS=${UPLOAD_SAFE_LOGS:-"0"}
+
+SKIPPED_PACKAGES_FILE_PATH="${WORKSPACE}/skipped_packages.txt"
+FAILED_PACKAGES_FILE_PATH="${WORKSPACE}/failed_packages.txt"
+
+export SERVERLESS="true"
+SERVERLESS_PROJECT=${SERVERLESS_PROJECT:-"observability"}
+
+echo "Running packages on Serverles project type: ${SERVERLESS_PROJECT}"
+if running_on_buildkite; then
+    SERVERLESS_PROJECT="$(buildkite-agent meta-data get SERVERLESS_PROJECT --default ${SERVERLESS_PROJECT:-"observability"})"
+    buildkite-agent annotate "Serverless Project: ${SERVERLESS_PROJECT}" --context "ctx-info-${SERVERLESS_PROJECT}" --style "info"
+fi
+
+add_bin_path
+
+echo "--- install go"
+with_go
+
+echo "--- Install docker"
+with_docker
+echo "--- Install docker-compose"
+with_docker_compose_plugin
+
+echo "--- Install elastic-package"
+# Required to start the Serverless Elastic stack
+make install
+
+prepare_serverless_stack
+
+echo "Waiting time to avoid getaddrinfo ENOTFOUND errors..."
+sleep 120
+echo "Done."
+
+list_packages() {
+    find test/packages/parallel -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort
+}
+
+any_package_failing=0
+
+for package in $(list_packages); do
+    echo "--- Test package: ${package}"
+    if ! .buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p "${package}" -s ; then
+        echo "- ${package}" >> "${FAILED_PACKAGES_FILE_PATH}"
+        any_package_failing=1
+    fi
+done
+
+if running_on_buildkite ; then
+    if [ -f "${SKIPPED_PACKAGES_FILE_PATH}" ]; then
+        create_collapsed_annotation "Skipped packages in ${SERVERLESS_PROJECT}" "${SKIPPED_PACKAGES_FILE_PATH}" "info" "ctx-skipped-packages-${SERVERLESS_PROJECT}"
+    fi
+
+    if [ -f "${FAILED_PACKAGES_FILE_PATH}" ]; then
+        create_collapsed_annotation "Failed packages in ${SERVERLESS_PROJECT}" "${FAILED_PACKAGES_FILE_PATH}" "error" "ctx-failed-packages-${SERVERLESS_PROJECT}"
+    fi
+fi
+
+if [ $any_package_failing -eq 1 ] ; then
+    echo "These packages have failed:"
+    cat "${FAILED_PACKAGES_FILE_PATH}"
+    exit 1
+fi
diff --git a/.buildkite/scripts/tooling.sh b/.buildkite/scripts/tooling.sh
index 0c18d9502f..32e0e9f54d 100755
--- a/.buildkite/scripts/tooling.sh
+++ b/.buildkite/scripts/tooling.sh
@@ -18,11 +18,17 @@ repo_name() {
 }
 
 buildkite_pr_branch_build_id() {
-    if [ "${BUILDKITE_PULL_REQUEST}" == "false" ]; then
+    if [ "${BUILDKITE_PULL_REQUEST}" != "false" ]; then
+        echo "PR-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_BUILD_NUMBER}"
+        return
+    fi
+
+    if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package" ]]; then
         echo "${BUILDKITE_BRANCH}-${BUILDKITE_BUILD_NUMBER}"
         return
     fi
-    echo "PR-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_BUILD_NUMBER}"
+    # Other pipelines
+    echo "${BUILDKITE_BRANCH}-${BUILDKITE_PIPELINE_SLUG}-${BUILDKITE_BUILD_NUMBER}"
 }
 
 google_cloud_auth() {
@@ -74,3 +80,84 @@ running_on_buildkite() {
     fi
     return 1
 }
+
+create_elastic_package_profile() {
+    local name="$1"
+    elastic-package profiles create "${name}"
+    elastic-package profiles use "${name}"
+}
+
+prepare_serverless_stack() {
+    echo "--- Prepare serverless stack"
+    local stack_version=${STACK_VERSION:-""}
+
+    local args="-v"
+    if [ -n "${stack_version}" ]; then
+        args="${args} --version ${stack_version}"
+    fi
+
+    # Currently, if STACK_VERSION is not defined, for serverless it will be
+    # used as Elastic stack version (for agents) the default version in elastic-package
+
+    # Creating a new profile allow to set a specific name for the serverless project
+    local profile_name="elastic-package-${BUILDKITE_PIPELINE_SLUG}-${BUILDKITE_BUILD_NUMBER}-${SERVERLESS_PROJECT}"
+    if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then
+        profile_name="elastic-package-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_BUILD_NUMBER}-${SERVERLESS_PROJECT}"
+    fi
+    create_elastic_package_profile "${profile_name}"
+
+    export EC_API_KEY=${EC_API_KEY_SECRET}
+    export EC_HOST=${EC_HOST_SECRET}
+
+    echo "Boot up the Elastic stack"
+    # grep command required to remove password from the output
+    if ! elastic-package stack up \
+        -d \
+        ${args} \
+        --provider serverless \
+        -U "stack.serverless.region=${EC_REGION_SECRET},stack.serverless.type=${SERVERLESS_PROJECT}" 2>&1 | grep -E -v "^Password: " ; then
+        return 1
+    fi
+    echo ""
+    elastic-package stack status
+    echo ""
+}
+
+google_cloud_auth_safe_logs() {
+    local gsUtilLocation=""
+    gsUtilLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE}")
+
+    local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
+
+    echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > "${secretFileLocation}"
+
+    google_cloud_auth "${secretFileLocation}"
+}
+
+upload_safe_logs() {
+    local bucket="$1"
+    local source="$2"
+    local target="$3"
+
+    if ! ls ${source} 2>&1 > /dev/null ; then
+        echo "upload_safe_logs: artifacts files not found, nothing will be archived"
+        return
+    fi
+
+    google_cloud_auth_safe_logs
+
+    gsutil cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}"
+
+    google_cloud_logout_active_account
+}
+
+clean_safe_logs() {
+    rm -rf "${WORKSPACE}/build/elastic-stack-dump"
+    rm -rf "${WORKSPACE}/build/container-logs"
+}
+
+cleanup() {
+  echo "Deleting temporary files..."
+  rm -rf ${WORKSPACE}/${TMP_FOLDER_TEMPLATE_BASE}.*
+  echo "Done."
+}
diff --git a/catalog-info.yaml b/catalog-info.yaml
index 4af584497c..eca70afe5b 100644
--- a/catalog-info.yaml
+++ b/catalog-info.yaml
@@ -155,3 +155,54 @@ spec:
         everyone:
           access_level: BUILD_AND_READ
 
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-elastic-package-test-serverless
+  description: Pipeline to test elastic-package with Serverless projects
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/elastic-package-test-serverless
+
+spec:
+  type: buildkite-pipeline
+  owner: group:ingest-fp
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: elastic-package-test-serverless
+      description: Pipeline to test elastic-package with Serverless projects
+    spec:
+      pipeline_file: ".buildkite/pipeline.serverless.yml"
+      provider_settings:
+        build_pull_request_forks: false
+        # Managed by buildkite-pr-bot
+        build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot
+        publish_commit_status: false # do not update status of commits for this pipeline
+        build_tags: false
+        build_branches: false
+        filter_enabled: true
+        filter_condition: >-
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null && build.source == 'api')
+      cancel_intermediate_builds: true
+      cancel_intermediate_builds_branch_filter: '!main'
+      skip_intermediate_builds: true
+      skip_intermediate_builds_branch_filter: '!main'
+      repository: elastic/elastic-package
+      schedules:
+        Test Serverless Daily:
+          branch: main
+          cronline: "00 5 * * *"
+          message: Test Serverless Daily
+          env:
+            SERVERLESS_PROJECT: observability
+      teams:
+        ingest-fp:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ
+
diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh
index 5628c7a428..386d12466f 100755
--- a/scripts/test-check-packages.sh
+++ b/scripts/test-check-packages.sh
@@ -20,8 +20,10 @@ cleanup() {
     kind delete cluster || true
   fi
 
-  # Take down the stack
-  elastic-package stack down -v
+  if [[ "$SERVERLESS" != "true" ]]; then
+      # Take down the stack
+      elastic-package stack down -v
+  fi
 
   if [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ]; then
     # Delete the logstash profile
@@ -45,6 +47,7 @@ ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT=${ELASTIC_PACKAGE_TEST_ENABLE_INDE
 export ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT
 ELASTIC_PACKAGE_LINKS_FILE_PATH="$(pwd)/scripts/links_table.yml"
 export ELASTIC_PACKAGE_LINKS_FILE_PATH
+export SERVERLESS=${SERVERLESS:-"false"}
 
 OLDPWD=$PWD
 # Build/check packages
@@ -68,13 +71,15 @@ if [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ]; then
   echo "stack.logstash_enabled: true" >> ~/.elastic-package/profiles/logstash/config.yml
 fi
 
-# Update the stack
-elastic-package stack update -v
+if [[ "${SERVERLESS}" != "true" ]]; then
+  # Update the stack
+  elastic-package stack update -v
 
-# Boot up the stack
-elastic-package stack up -d -v
+  # Boot up the stack
+  elastic-package stack up -d -v
 
-elastic-package stack status
+  elastic-package stack status
+fi
 
 if [ "${PACKAGE_TEST_TYPE:-other}" == "with-kind" ]; then
   # Boot up the kind cluster
@@ -113,13 +118,17 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do
           echo "Package \"${package_to_test}\" skipped: not supported with Elastic Agent running in the stack (missing capabilities)."
           exit # as it is run in a subshell, it cannot be used "continue"
       fi
+
+      if [[ "${SERVERLESS}" == "true" ]]; then
+          # skip system tests
+          elastic-package test asset -v --report-format xUnit --report-output file --defer-cleanup 1s  --test-coverage --coverage-format=generic
+          elastic-package test static -v --report-format xUnit --report-output file --defer-cleanup 1s  --test-coverage --coverage-format=generic
+          elastic-package test pipeline -v --report-format xUnit --report-output file --defer-cleanup 1s  --test-coverage --coverage-format=generic
+          exit # as it is run in a subshell, it cannot be used "continue"
+      fi
+      # Run all tests
       # defer-cleanup is set to a short period to verify that the option is available
-      elastic-package test -v \
-          --report-format xUnit \
-          --report-output file \
-          --defer-cleanup 1s \
-          --test-coverage \
-          --coverage-format=generic
+      elastic-package test -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage --coverage-format=generic
     fi
   )
 cd -

From 96cbd809d2e00e706b122f8ae707e3af58c07a2d Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 30 Apr 2024 18:40:12 +0200
Subject: [PATCH 02/20] Fix errors

---
 .buildkite/hooks/pre-exit          |  2 +-
 .buildkite/pipeline.serverless.yml | 10 +++++++++-
 .buildkite/scripts/tooling.sh      | 17 +++++++++++++++++
 scripts/test-check-packages.sh     |  4 +++-
 4 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index bae53afdee..60b1aff14c 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -10,7 +10,7 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package" && "${BUILDKITE_STEP_KEY
     # BUILDKITE resets PATH contents in pre-exit hook, but elastic-package
     # is already installed in the test_serverless pipeline step, accessing
     # directly to the binary
-    EC_API_KEY="${EC_API_KEY_SECRET}" EC_HOST="${EC_HOST_SECRET}" "${HOME}"/go/bin/elastic-package down -v
+    EC_API_KEY="${EC_API_KEY_SECRET}" EC_HOST="${EC_HOST_SECRET}" "${HOME}"/go/bin/elastic-package stack down -v
 fi
 
 unset_secrets
diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml
index 56b03ab6e5..7a57631a5a 100644
--- a/.buildkite/pipeline.serverless.yml
+++ b/.buildkite/pipeline.serverless.yml
@@ -1,11 +1,19 @@
 env:
   SETUP_GVM_VERSION: 'v0.5.2' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
-  ELASTIC_PACKAGE_COMPOSE_DISABLE_VERBOSE_OUTPUT: "true"
   DOCKER_COMPOSE_VERSION: "v2.24.1"
   DOCKER_VERSION: "26.1.0"
   KIND_VERSION: 'v0.20.0'
   K8S_VERSION: 'v1.29.0'
   LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
+  # Elastic package settings
+  # Manage docker output/logs
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_VERBOSE_OUTPUT: "true"
+  # Default license to use by `elastic-package build`
+  ELASTIC_PACKAGE_REPOSITORY_LICENSE: "licenses/Elastic-2.0.txt"
+  # Link definitions path (full path to be set in the corresponding step)
+  ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
+  # Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields
+  ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true"
 
 steps:
   - input: "Input values for the variables"
diff --git a/.buildkite/scripts/tooling.sh b/.buildkite/scripts/tooling.sh
index 32e0e9f54d..ea7d8cc25b 100755
--- a/.buildkite/scripts/tooling.sh
+++ b/.buildkite/scripts/tooling.sh
@@ -161,3 +161,20 @@ cleanup() {
   rm -rf ${WORKSPACE}/${TMP_FOLDER_TEMPLATE_BASE}.*
   echo "Done."
 }
+
+create_collapsed_annotation() {
+    local title="$1"
+    local file="$2"
+    local style="$3"
+    local context="$4"
+
+    local annotation_file="tmp.annotation.md"
+    echo "<details><summary>${title}</summary>" >> ${annotation_file}
+    echo -e "\n\n" >> ${annotation_file}
+    cat "${file}" >> ${annotation_file}
+    echo "</details>" >> ${annotation_file}
+
+    cat ${annotation_file} | buildkite-agent annotate --style "${style}" --context "${context}"
+
+    rm -f ${annotation_file}
+}
diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh
index 386d12466f..a291c4e180 100755
--- a/scripts/test-check-packages.sh
+++ b/scripts/test-check-packages.sh
@@ -123,7 +123,9 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do
           # skip system tests
           elastic-package test asset -v --report-format xUnit --report-output file --defer-cleanup 1s  --test-coverage --coverage-format=generic
           elastic-package test static -v --report-format xUnit --report-output file --defer-cleanup 1s  --test-coverage --coverage-format=generic
-          elastic-package test pipeline -v --report-format xUnit --report-output file --defer-cleanup 1s  --test-coverage --coverage-format=generic
+          # FIXME: adding test-coverage for serverless results in errors like this:
+          # Error: error running package pipeline tests: could not complete test run: error calculating pipeline coverage: error fetching pipeline stats for code coverage calculations: need exactly one ES node in stats response (got 4)
+          elastic-package test pipeline -v --report-format xUnit --report-output file --defer-cleanup 1s
           exit # as it is run in a subshell, it cannot be used "continue"
       fi
       # Run all tests

From 54a8f12112d5ed83ccf92f2f9a87c799fcdbf6b8 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 30 Apr 2024 19:07:22 +0200
Subject: [PATCH 03/20] Update labels

---
 .buildkite/scripts/integration_tests.sh             | 6 +++++-
 .buildkite/scripts/test_packages_with_serverless.sh | 1 -
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh
index 6b70ffdded..76d30c5b9f 100755
--- a/.buildkite/scripts/integration_tests.sh
+++ b/.buildkite/scripts/integration_tests.sh
@@ -98,6 +98,10 @@ if [[ "${TARGET}" == "${KIND_TARGET}" || "${TARGET}" == "${SYSTEM_TEST_FLAGS_TAR
     with_kubernetes
 fi
 
+label="${TARGET}"
+if [ -n "${PACKAGE_UNDER_TEST}" ]; then
+    label="${LABEL} - ${PACKAGE_UNDER_TEST}"
+fi
 echo "--- Run integration test ${TARGET}"
 if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSITIVES_TARGET}" ]]; then
     make install
@@ -138,7 +142,7 @@ if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSIT
     fi
 
     if [ $testReturnCode != 0 ]; then
-        echo "make SERVERLESS=${SERVERLESS} PACKAGE_UDER_TEST=${PACKAGE} ${TARGET} failed with ${testReturnCode}"
+        echo "make SERVERLESS=${SERVERLESS} PACKAGE_UNDER_TEST=${PACKAGE} ${TARGET} failed with ${testReturnCode}"
         exit ${testReturnCode}
     fi
 
diff --git a/.buildkite/scripts/test_packages_with_serverless.sh b/.buildkite/scripts/test_packages_with_serverless.sh
index 71ce1311b1..f754f34dd1 100644
--- a/.buildkite/scripts/test_packages_with_serverless.sh
+++ b/.buildkite/scripts/test_packages_with_serverless.sh
@@ -46,7 +46,6 @@ list_packages() {
 any_package_failing=0
 
 for package in $(list_packages); do
-    echo "--- Test package: ${package}"
     if ! .buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p "${package}" -s ; then
         echo "- ${package}" >> "${FAILED_PACKAGES_FILE_PATH}"
         any_package_failing=1

From e2b9e9aa4b4cc6496c5de789608fac3aa28449c5 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 30 Apr 2024 19:16:41 +0200
Subject: [PATCH 04/20] Move traps to pre-exit hook

---
 .buildkite/hooks/pre-exit                    |  2 ++
 .buildkite/scripts/integration_tests.sh      | 19 -------------------
 .buildkite/scripts/test-with-integrations.sh |  9 ---------
 3 files changed, 2 insertions(+), 28 deletions(-)

diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index 60b1aff14c..676336193e 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -13,6 +13,8 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package" && "${BUILDKITE_STEP_KEY
     EC_API_KEY="${EC_API_KEY_SECRET}" EC_HOST="${EC_HOST_SECRET}" "${HOME}"/go/bin/elastic-package stack down -v
 fi
 
+cleanup
+google_cloud_logout_active_account
 unset_secrets
 
 # integrations-parallel-gcp
diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh
index 76d30c5b9f..5b1b27f49d 100755
--- a/.buildkite/scripts/integration_tests.sh
+++ b/.buildkite/scripts/integration_tests.sh
@@ -5,25 +5,6 @@ source .buildkite/scripts/tooling.sh
 
 set -euo pipefail
 
-cleanup() {
-    local error_code=$?
-
-    if [ $error_code != 0 ] ; then
-        # if variable is defined run the logout
-        if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then
-             google_cloud_logout_active_account
-        fi
-    fi
-
-    echo "Deleting temporal files..."
-    cd "${WORKSPACE}"
-    rm -rf "${TMP_FOLDER_TEMPLATE_BASE}.*"
-    echo "Done."
-
-    exit $error_code
-}
-trap cleanup EXIT
-
 usage() {
     echo "$0 [-t <target>] [-h]"
     echo "Trigger integration tests related to a target in Makefile"
diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index cd4ce6ddc7..8a7018153a 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -3,15 +3,6 @@ source .buildkite/scripts/install_deps.sh
 
 set -euo pipefail
 
-cleanup() {
-    echo "Deleting temporal files..."
-    cd "${WORKSPACE}"
-    rm -rf "${TMP_FOLDER_TEMPLATE_BASE}.*"
-    echo "Done."
-}
-
-trap cleanup EXIT
-
 add_bin_path
 
 echo "--- install gh cli"

From 1fa152e6aa2386e3ca8a588f2dc32cd467a6378a Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 30 Apr 2024 19:29:20 +0200
Subject: [PATCH 05/20] Fix unbound variable

---
 .buildkite/hooks/pre-exit               | 1 +
 .buildkite/pipeline.serverless.yml      | 2 +-
 .buildkite/scripts/integration_tests.sh | 4 ++--
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index 676336193e..ec5b0c041a 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -13,6 +13,7 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package" && "${BUILDKITE_STEP_KEY
     EC_API_KEY="${EC_API_KEY_SECRET}" EC_HOST="${EC_HOST_SECRET}" "${HOME}"/go/bin/elastic-package stack down -v
 fi
 
+echo "--- Cleanup"
 cleanup
 google_cloud_logout_active_account
 unset_secrets
diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml
index 7a57631a5a..a1ccae690a 100644
--- a/.buildkite/pipeline.serverless.yml
+++ b/.buildkite/pipeline.serverless.yml
@@ -55,7 +55,7 @@ steps:
   - wait: ~
     continue_on_failure: true
 
-  - label: ":pipeline: Test Serverless Integration tests"
+  - label: ":elastic: Serverless Integration tests"
     key: test-serverless
     command: ".buildkite/scripts/test_packages_with_serverless.sh"
     env:
diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh
index 5b1b27f49d..4799944f1c 100755
--- a/.buildkite/scripts/integration_tests.sh
+++ b/.buildkite/scripts/integration_tests.sh
@@ -80,8 +80,8 @@ if [[ "${TARGET}" == "${KIND_TARGET}" || "${TARGET}" == "${SYSTEM_TEST_FLAGS_TAR
 fi
 
 label="${TARGET}"
-if [ -n "${PACKAGE_UNDER_TEST}" ]; then
-    label="${LABEL} - ${PACKAGE_UNDER_TEST}"
+if [ -n "${PACKAGE}" ]; then
+    label="${LABEL} - ${PACKAGE}"
 fi
 echo "--- Run integration test ${TARGET}"
 if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSITIVES_TARGET}" ]]; then

From 5586dadab075442d86f058dac430d6355c529fbf Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 30 Apr 2024 19:30:26 +0200
Subject: [PATCH 06/20] Restore main pipeline

---
 .buildkite/pipeline.yml | 130 ++++++++++++++++++++--------------------
 1 file changed, 65 insertions(+), 65 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 96f88c556c..63a0c6beff 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -9,33 +9,36 @@ env:
   WINDOWS_AGENT_IMAGE: "family/ci-windows-2022"
 
 steps:
-  # - label: ":go: Run check-static"
-  #   key: check-static
-  #   command: "make check-static"
-  #   agents:
-  #     image: "${LINUX_AGENT_IMAGE}"
-  #     cpu: "8"
-  #     memory: "4G"
+  - label: ":go: Run check-static"
+    key: check-static
+    command: "make check-static"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+
+  - label: ":go: :linux: Run unit tests"
+    key: unit-tests-linux
+    command: "make test-go-ci"
+    artifact_paths:
+      - "build/test-results/*.xml"
+      - "build/test-coverage/*.xml"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
 
-  # - label: ":go: :linux: Run unit tests"
-  #   key: unit-tests-linux
-  #   command: "make test-go-ci"
-  #   artifact_paths:
-  #     - "build/test-results/*.xml"
-  #     - "build/test-coverage/*.xml"
-  #   agents:
-  #     image: "${LINUX_AGENT_IMAGE}"
-  #     cpu: "8"
-  #     memory: "4G"
+  - label: ":go: :windows: Run unit tests"
+    key: unit-tests-windows
+    command: ".buildkite/scripts/unit_tests_windows.ps1"
+    agents:
+      provider: "gcp"
+      image: "${WINDOWS_AGENT_IMAGE}"
+    artifact_paths:
+      - "TEST-unit-windows.xml"
 
-  # - label: ":go: :windows: Run unit tests"
-  #   key: unit-tests-windows
-  #   command: ".buildkite/scripts/unit_tests_windows.ps1"
-  #   agents:
-  #     provider: "gcp"
-  #     image: "${WINDOWS_AGENT_IMAGE}"
-  #   artifact_paths:
-  #     - "TEST-unit-windows.xml"
+  - wait: ~
+    continue_on_failure: true
 
   - label: ":package: test serverless"
     key: test-serverless
@@ -49,48 +52,45 @@ steps:
       - build/test-results/*.xml
       - build/test-coverage/coverage-*.xml
 
-  # - wait: ~
-  #   continue_on_failure: true
+  - label: ":pipeline: Trigger Integration tests"
+    command: ".buildkite/pipeline.trigger.integration.tests.sh | buildkite-agent pipeline upload"
+    depends_on:
+      - step: check-static
+        allow_failure: false
+      - step: unit-tests-linux
+        allow_failure: false
+      - step: unit-tests-windows
+        allow_failure: false
 
-  # - label: ":pipeline: Trigger Integration tests"
-  #   command: ".buildkite/pipeline.trigger.integration.tests.sh | buildkite-agent pipeline upload"
-  #   depends_on:
-  #     - step: check-static
-  #       allow_failure: false
-  #     - step: unit-tests-linux
-  #       allow_failure: false
-  #     - step: unit-tests-windows
-  #       allow_failure: false
+  - wait: ~
+    continue_on_failure: true
 
-  # - wait: ~
-  #   continue_on_failure: true
-
-  # - label: ":junit: Transform windows paths to linux for Junit plugin"
-  #   commands:
-  #     - buildkite-agent artifact download "*-windows.xml" . --step unit-tests-windows
-  #     - mkdir -p build/test-results
-  #     - for file in $(ls *-windows.xml); do mv $$file build/test-results/; done
-  #   agents:
-  #     image: "${LINUX_AGENT_IMAGE}"
-  #     cpu: "8"
-  #     memory: "4G"
-  #   artifact_paths:
-  #     - "build/test-results/*.xml"
+  - label: ":junit: Transform windows paths to linux for Junit plugin"
+    commands:
+      - buildkite-agent artifact download "*-windows.xml" . --step unit-tests-windows
+      - mkdir -p build/test-results
+      - for file in $(ls *-windows.xml); do mv $$file build/test-results/; done
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+    artifact_paths:
+      - "build/test-results/*.xml"
 
-  # - wait: ~
-  #   continue_on_failure: true
+  - wait: ~
+    continue_on_failure: true
 
-  # - label: ":junit: Junit annotate"
-  #   plugins:
-  #     - junit-annotate#v2.4.1:
-  #         artifacts: "build/test-results/*.xml"
-  #   agents:
-  #     provider: "gcp"  # junit plugin requires docker
+  - label: ":junit: Junit annotate"
+    plugins:
+      - junit-annotate#v2.4.1:
+          artifacts: "build/test-results/*.xml"
+    agents:
+      provider: "gcp"  # junit plugin requires docker
 
-  # - label: ":github: Release"
-  #   key: "release"
-  #   if: |
-  #     build.tag =~ /^v[0-9]+[.][0-9]+[.][0-9]+$$/
-  #   command: ".buildkite/scripts/release.sh"
-  #   agents:
-  #     provider: "gcp"
+  - label: ":github: Release"
+    key: "release"
+    if: |
+      build.tag =~ /^v[0-9]+[.][0-9]+[.][0-9]+$$/
+    command: ".buildkite/scripts/release.sh"
+    agents:
+      provider: "gcp"

From 1c4eaa93f9275343cbb985cf50455b9a05ff57a0 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 30 Apr 2024 19:42:35 +0200
Subject: [PATCH 07/20] Update label for each package

---
 .buildkite/scripts/integration_tests.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh
index 4799944f1c..aeb62093b3 100755
--- a/.buildkite/scripts/integration_tests.sh
+++ b/.buildkite/scripts/integration_tests.sh
@@ -81,9 +81,9 @@ fi
 
 label="${TARGET}"
 if [ -n "${PACKAGE}" ]; then
-    label="${LABEL} - ${PACKAGE}"
+    label="${label} - ${PACKAGE}"
 fi
-echo "--- Run integration test ${TARGET}"
+echo "--- Run integration test ${label}"
 if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSITIVES_TARGET}" ]]; then
     make install
 

From e2db8383c4e1626b307d91df713e60f5679d1a99 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 30 Apr 2024 20:29:12 +0200
Subject: [PATCH 08/20] Move just clean up files

---
 .buildkite/hooks/pre-exit               |  1 -
 .buildkite/scripts/integration_tests.sh | 14 ++++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index ec5b0c041a..76863a1ac0 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -15,7 +15,6 @@ fi
 
 echo "--- Cleanup"
 cleanup
-google_cloud_logout_active_account
 unset_secrets
 
 # integrations-parallel-gcp
diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh
index aeb62093b3..b8085cecf9 100755
--- a/.buildkite/scripts/integration_tests.sh
+++ b/.buildkite/scripts/integration_tests.sh
@@ -5,6 +5,20 @@ source .buildkite/scripts/tooling.sh
 
 set -euo pipefail
 
+ensure_logout() {
+    local error_code=$?
+
+    if [ $error_code != 0 ] ; then
+        # if variable is defined run the logout
+        if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then
+             google_cloud_logout_active_account
+        fi
+    fi
+
+    exit $error_code
+}
+trap ensure_logout EXIT
+
 usage() {
     echo "$0 [-t <target>] [-h]"
     echo "Trigger integration tests related to a target in Makefile"

From e61efc9b46de20a89646fda250a91598727a4c62 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 2 May 2024 10:27:27 +0200
Subject: [PATCH 09/20] Add comment in pull request

---
 .buildkite/scripts/test-with-integrations.sh   |  8 ++++----
 .../scripts/test_packages_with_serverless.sh   | 18 ++++++++++++++++++
 .buildkite/scripts/tooling.sh                  | 11 +++++++++++
 3 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh
index 8a7018153a..7da2e08566 100755
--- a/.buildkite/scripts/test-with-integrations.sh
+++ b/.buildkite/scripts/test-with-integrations.sh
@@ -165,10 +165,10 @@ add_pr_comment() {
     local source_pr_number="$1"
     local integrations_pr_link="$2"
 
-    retry 3 \
-        gh pr comment "${source_pr_number}" \
-        --body "Created or updated PR in integrations repository to test this version. Check ${integrations_pr_link}" \
-        --repo "${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}"
+    add_github_comment \
+        "${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}" \
+        "${source_pr_number}" \
+        "Created or updated PR in integrations repository to test this version. Check ${integrations_pr_link}"
 }
 
 
diff --git a/.buildkite/scripts/test_packages_with_serverless.sh b/.buildkite/scripts/test_packages_with_serverless.sh
index f754f34dd1..7a348892ef 100644
--- a/.buildkite/scripts/test_packages_with_serverless.sh
+++ b/.buildkite/scripts/test_packages_with_serverless.sh
@@ -13,12 +13,23 @@ FAILED_PACKAGES_FILE_PATH="${WORKSPACE}/failed_packages.txt"
 export SERVERLESS="true"
 SERVERLESS_PROJECT=${SERVERLESS_PROJECT:-"observability"}
 
+add_pr_comment() {
+    local source_pr_number="$1"
+    local buildkite_build="$2"
+
+    add_github_comment \
+        "${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}" \
+        "${source_pr_number}" \
+        "Triggered serverless pipeline: ${buildkite_build}"
+}
+
 echo "Running packages on Serverles project type: ${SERVERLESS_PROJECT}"
 if running_on_buildkite; then
     SERVERLESS_PROJECT="$(buildkite-agent meta-data get SERVERLESS_PROJECT --default ${SERVERLESS_PROJECT:-"observability"})"
     buildkite-agent annotate "Serverless Project: ${SERVERLESS_PROJECT}" --context "ctx-info-${SERVERLESS_PROJECT}" --style "info"
 fi
 
+
 add_bin_path
 
 echo "--- install go"
@@ -29,6 +40,13 @@ with_docker
 echo "--- Install docker-compose"
 with_docker_compose_plugin
 
+if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then
+    echo "--- install gh cli"
+    with_github_cli
+
+    add_pr_comment "${BUILDKITE_PULL_REQUEST}" "${BUILDKITE_BUILD_URL}"
+fi
+
 echo "--- Install elastic-package"
 # Required to start the Serverless Elastic stack
 make install
diff --git a/.buildkite/scripts/tooling.sh b/.buildkite/scripts/tooling.sh
index ea7d8cc25b..a8842091de 100755
--- a/.buildkite/scripts/tooling.sh
+++ b/.buildkite/scripts/tooling.sh
@@ -178,3 +178,14 @@ create_collapsed_annotation() {
 
     rm -f ${annotation_file}
 }
+
+add_github_comment() {
+    local repository="$1"
+    local pr_id="$2"
+    local message="$3"
+
+    retry 3 \
+        gh pr comment "${pr_id}" \
+        --body "${message}" \
+        --repo "${repository}"
+}

From 799fa1872accdb369522a6903ae796413fd951a5 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 2 May 2024 10:54:18 +0200
Subject: [PATCH 10/20] Add gh cli version

---
 .buildkite/pipeline.serverless.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml
index a1ccae690a..f6e1004220 100644
--- a/.buildkite/pipeline.serverless.yml
+++ b/.buildkite/pipeline.serverless.yml
@@ -5,6 +5,7 @@ env:
   KIND_VERSION: 'v0.20.0'
   K8S_VERSION: 'v1.29.0'
   LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
+  GH_CLI_VERSION: "2.29.0"
   # Elastic package settings
   # Manage docker output/logs
   ELASTIC_PACKAGE_COMPOSE_DISABLE_VERBOSE_OUTPUT: "true"

From fe01122ed695e8362ef398e8d0ac3d9c774acf12 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 2 May 2024 10:55:29 +0200
Subject: [PATCH 11/20] Add gh cli version as env

---
 .buildkite/pipeline.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 63a0c6beff..b7e1c9c064 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -47,6 +47,7 @@ steps:
       provider: "gcp"
     env:
       SERVERLESS_PROJECT: observability
+      GH_CLI_VERSION: "2.29.0"
       UPLOAD_SAFE_LOGS: 1
     artifact_paths:
       - build/test-results/*.xml

From 1e7458efbcb810e452cb8300f6e14f0fb2ead40e Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 2 May 2024 11:13:43 +0200
Subject: [PATCH 12/20] Add missing env. var

---
 .buildkite/hooks/pre-command | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 83d3c37c56..deff2a8d28 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -165,5 +165,8 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package" ]]; then
         export EC_HOST_SECRET
         EC_REGION_SECRET=$(retry 5 vault read -field region_qa "${EC_DATA_PATH}")
         export EC_REGION_SECRET
+
+        GITHUB_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
+        export GITHUB_TOKEN
     fi
 fi

From 6469336e97b92798deff6d15dcb2ffefb05e138a Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 2 May 2024 12:04:19 +0200
Subject: [PATCH 13/20] Remove debug changes

---
 .buildkite/hooks/pre-command | 20 +++++++++++---------
 .buildkite/hooks/pre-exit    |  3 +--
 .buildkite/pipeline.yml      | 13 -------------
 3 files changed, 12 insertions(+), 24 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index deff2a8d28..7255f62d9d 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -60,13 +60,17 @@ is_step_required_to_upload_safe_logs() {
     if [[ "$BUILDKITE_PIPELINE_SLUG" != "elastic-package" && "$BUILDKITE_PIPELINE_SLUG" != "elastic-package-test-serverless" ]]; then
         return 1
     fi
-    # steps from elastic-package
-    if [[ "$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives ]]; then
-        return 0
+
+    if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" ]]; then
+        if [[ "$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives ]]; then
+            return 0
+        fi
     fi
-    # steps from elastic-package-test-serverless
-    if [[ "$BUILDKITE_STEP_KEY" == "test-serverless" ]]; then
-        return 0
+
+    if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-test-serverless" ]]; then
+        if [[ "$BUILDKITE_STEP_KEY" == "test-serverless" ]]; then
+            return 0
+        fi
     fi
     return 1
 }
@@ -155,9 +159,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-cloud-cleanup" && "$BUILDKI
 fi
 
 
-# TODO: Replace if condition
-# if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package-test-serverless" ]]; then
-if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package" ]]; then
+if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package-test-serverless" ]]; then
     if [[ "${BUILDKITE_STEP_KEY}" == "test-serverless" ]]; then
         EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey "${EC_TOKEN_PATH}")
         export EC_API_KEY_SECRET
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index 76863a1ac0..92f0f4d7d9 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -4,8 +4,7 @@ source .buildkite/scripts/tooling.sh
 
 set -euo pipefail
 
-# TODO: change pipeline slug
-if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package" && "${BUILDKITE_STEP_KEY}" == "test-serverless" ]]; then
+if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package-test-serverless" && "${BUILDKITE_STEP_KEY}" == "test-serverless" ]]; then
     echo "--- Take down the Elastic stack"
     # BUILDKITE resets PATH contents in pre-exit hook, but elastic-package
     # is already installed in the test_serverless pipeline step, accessing
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index b7e1c9c064..42e868699a 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -40,19 +40,6 @@ steps:
   - wait: ~
     continue_on_failure: true
 
-  - label: ":package: test serverless"
-    key: test-serverless
-    command: ".buildkite/scripts/test_packages_with_serverless.sh"
-    agents:
-      provider: "gcp"
-    env:
-      SERVERLESS_PROJECT: observability
-      GH_CLI_VERSION: "2.29.0"
-      UPLOAD_SAFE_LOGS: 1
-    artifact_paths:
-      - build/test-results/*.xml
-      - build/test-coverage/coverage-*.xml
-
   - label: ":pipeline: Trigger Integration tests"
     command: ".buildkite/pipeline.trigger.integration.tests.sh | buildkite-agent pipeline upload"
     depends_on:

From 0abd723171ea335f45669dd1ffd0e6fab2d53b43 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 2 May 2024 13:00:09 +0200
Subject: [PATCH 14/20] Add email failure notification just for non pull
 requests builds

---
 .buildkite/pipeline.serverless.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml
index f6e1004220..c3d4043c92 100644
--- a/.buildkite/pipeline.serverless.yml
+++ b/.buildkite/pipeline.serverless.yml
@@ -1,4 +1,5 @@
 env:
+  NOTIFY_TO: "ecosystem-team@elastic.co"
   SETUP_GVM_VERSION: 'v0.5.2' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
   DOCKER_COMPOSE_VERSION: "v2.24.1"
   DOCKER_VERSION: "26.1.0"
@@ -80,3 +81,7 @@ steps:
           artifacts: "build/test-results/*.xml"
     agents:
       provider: "gcp"  # junit plugin requires docker
+
+notify:
+  - email: "$NOTIFY_TO"
+    if: "build.state == 'failed' && build.env('BUILDKITE_PULL_REQUEST') == 'false'"

From 28462ecdc7a99e4c0cec8163a6d11a1a1ead15f5 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 2 May 2024 15:43:31 +0200
Subject: [PATCH 15/20] Add documentation about this new pipeline

---
 README.md                   | 20 ++++++++++++++++++++
 tools/readme/readme.md.tmpl | 20 ++++++++++++++++++++
 2 files changed, 40 insertions(+)

diff --git a/README.md b/README.md
index dca14997ec..34b0944b2a 100644
--- a/README.md
+++ b/README.md
@@ -114,6 +114,26 @@ Usually, this process would require the following manual steps:
     - Creating this PR would automatically trigger a new Jenkins pipeline.
 
 
+### Testing with Elastic serverless
+
+While working on a branch, it might be interesting to test your changes using
+a project created in [Elastic serverless](https://docs.elastic.co/serverless), instead of spinning up a local
+Elastic stack. To do so, you can add a new comment while developing in your Pull request
+a comment like `test serverless`.
+
+Adding that comment in your Pull Request will create a new build of this
+[Buildkite pipeline](https://buildkite.com/elastic/elastic-package-test-serverless).
+This pipeline creates a new Serverless project and run some tests with the packages defined
+in the `test/packages/parallel` folder. Currently, there are some differences with respect to testing
+with a local Elastic stack:
+- System tests are not executed.
+- Disabled comparison of results in pipeline tests to avoid errors related to GeoIP fields
+- Pipeline tests cannot be executed with coverage flags.
+
+At the same time, this pipeline is going to be triggered daily to test the latest contents
+of the main branch with an Elastic serverless project.
+
+
 ## Commands
 
 `elastic-package` currently offers the commands listed below.
diff --git a/tools/readme/readme.md.tmpl b/tools/readme/readme.md.tmpl
index 3f5b817a77..bb3b805587 100644
--- a/tools/readme/readme.md.tmpl
+++ b/tools/readme/readme.md.tmpl
@@ -114,6 +114,26 @@ Usually, this process would require the following manual steps:
     - Creating this PR would automatically trigger a new Jenkins pipeline.
 
 
+### Testing with Elastic serverless
+
+While working on a branch, it might be interesting to test your changes using
+a project created in [Elastic serverless](https://docs.elastic.co/serverless), instead of spinning up a local
+Elastic stack. To do so, you can add a new comment while developing in your Pull request
+a comment like `test serverless`.
+
+Adding that comment in your Pull Request will create a new build of this
+[Buildkite pipeline](https://buildkite.com/elastic/elastic-package-test-serverless).
+This pipeline creates a new Serverless project and run some tests with the packages defined
+in the `test/packages/parallel` folder. Currently, there are some differences with respect to testing
+with a local Elastic stack:
+- System tests are not executed.
+- Disabled comparison of results in pipeline tests to avoid errors related to GeoIP fields
+- Pipeline tests cannot be executed with coverage flags.
+
+At the same time, this pipeline is going to be triggered daily to test the latest contents
+of the main branch with an Elastic serverless project.
+
+
 ## Commands
 
 `elastic-package` currently offers the commands listed below.

From 4a0f705667403676c08d532956b2f0d88219aca7 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 2 May 2024 17:10:07 +0200
Subject: [PATCH 16/20] Move code

---
 .buildkite/scripts/tooling.sh | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/.buildkite/scripts/tooling.sh b/.buildkite/scripts/tooling.sh
index a8842091de..3151774dfc 100755
--- a/.buildkite/scripts/tooling.sh
+++ b/.buildkite/scripts/tooling.sh
@@ -89,15 +89,6 @@ create_elastic_package_profile() {
 
 prepare_serverless_stack() {
     echo "--- Prepare serverless stack"
-    local stack_version=${STACK_VERSION:-""}
-
-    local args="-v"
-    if [ -n "${stack_version}" ]; then
-        args="${args} --version ${stack_version}"
-    fi
-
-    # Currently, if STACK_VERSION is not defined, for serverless it will be
-    # used as Elastic stack version (for agents) the default version in elastic-package
 
     # Creating a new profile allow to set a specific name for the serverless project
     local profile_name="elastic-package-${BUILDKITE_PIPELINE_SLUG}-${BUILDKITE_BUILD_NUMBER}-${SERVERLESS_PROJECT}"
@@ -110,6 +101,14 @@ prepare_serverless_stack() {
     export EC_HOST=${EC_HOST_SECRET}
 
     echo "Boot up the Elastic stack"
+    # Currently, if STACK_VERSION is not defined, for serverless it will be
+    # used as Elastic stack version (for agents) the default version in elastic-package
+    local stack_version=${STACK_VERSION:-""}
+    local args="-v"
+    if [ -n "${stack_version}" ]; then
+        args="${args} --version ${stack_version}"
+    fi
+
     # grep command required to remove password from the output
     if ! elastic-package stack up \
         -d \

From 4dd82be77e8f5ec166112b6c45aebeeca789299a Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 2 May 2024 17:14:20 +0200
Subject: [PATCH 17/20] Add comment about skipping some elastic-package
 commands

---
 scripts/test-check-packages.sh | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh
index a291c4e180..40422e7ba3 100755
--- a/scripts/test-check-packages.sh
+++ b/scripts/test-check-packages.sh
@@ -20,6 +20,10 @@ cleanup() {
     kind delete cluster || true
   fi
 
+  # In case it is tested with Elatic serverless, there should be just one Elastic stack
+  # started to test all packages. In our CI, this Elastic serverless stack is started 
+  # at the beginning of the pipeline and must be running for all packages without stopping it between
+  # packages.
   if [[ "$SERVERLESS" != "true" ]]; then
       # Take down the stack
       elastic-package stack down -v
@@ -71,6 +75,9 @@ if [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ]; then
   echo "stack.logstash_enabled: true" >> ~/.elastic-package/profiles/logstash/config.yml
 fi
 
+# In case it is tested with Elatic serverless, there should be just one Elastic stack
+# started to test all packages. In our CI, this Elastic serverless stack is started 
+# at the beginning of the pipeline and must be running for all packages.
 if [[ "${SERVERLESS}" != "true" ]]; then
   # Update the stack
   elastic-package stack update -v

From 1d2bde4060dfb8487e3254a45d871497b05a3561 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 2 May 2024 17:15:46 +0200
Subject: [PATCH 18/20] Remove unit tests step

---
 .buildkite/pipeline.serverless.yml | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml
index c3d4043c92..82e1fe58f2 100644
--- a/.buildkite/pipeline.serverless.yml
+++ b/.buildkite/pipeline.serverless.yml
@@ -43,17 +43,6 @@ steps:
       cpu: "8"
       memory: "4G"
 
-  - label: ":go: :linux: Run unit tests"
-    key: unit-tests-linux
-    command: "make test-go-ci"
-    artifact_paths:
-      - "build/test-results/*.xml"
-      - "build/test-coverage/*.xml"
-    agents:
-      image: "${LINUX_AGENT_IMAGE}"
-      cpu: "8"
-      memory: "4G"
-
   - wait: ~
     continue_on_failure: true
 
@@ -69,8 +58,6 @@ steps:
     depends_on:
       - step: check-static
         allow_failure: false
-      - step: unit-tests-linux
-        allow_failure: false
 
   - wait: ~
     continue_on_failure: true

From 311164f2c47a660eb8c7c38be6031a0878ced7b4 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 2 May 2024 17:31:54 +0200
Subject: [PATCH 19/20] Remvoe check-static step

---
 .buildkite/pipeline.serverless.yml | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml
index 82e1fe58f2..83c537e35a 100644
--- a/.buildkite/pipeline.serverless.yml
+++ b/.buildkite/pipeline.serverless.yml
@@ -35,17 +35,6 @@ steps:
     if: "build.source == 'ui'"
     allow_dependency_failure: false
 
-  - label: ":go: Run check-static"
-    key: check-static
-    command: "make check-static"
-    agents:
-      image: "${LINUX_AGENT_IMAGE}"
-      cpu: "8"
-      memory: "4G"
-
-  - wait: ~
-    continue_on_failure: true
-
   - label: ":elastic: Serverless Integration tests"
     key: test-serverless
     command: ".buildkite/scripts/test_packages_with_serverless.sh"
@@ -55,9 +44,6 @@ steps:
     artifact_paths:
       - build/test-results/*.xml
       - build/test-coverage/coverage-*.xml
-    depends_on:
-      - step: check-static
-        allow_failure: false
 
   - wait: ~
     continue_on_failure: true

From a05e4a921a4cac20245097cf13dc337e7197bc63 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 2 May 2024 17:34:34 +0200
Subject: [PATCH 20/20] Rephrased comment

---
 .buildkite/scripts/test_packages_with_serverless.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/test_packages_with_serverless.sh b/.buildkite/scripts/test_packages_with_serverless.sh
index 7a348892ef..6dde90745a 100644
--- a/.buildkite/scripts/test_packages_with_serverless.sh
+++ b/.buildkite/scripts/test_packages_with_serverless.sh
@@ -53,7 +53,7 @@ make install
 
 prepare_serverless_stack
 
-echo "Waiting time to avoid getaddrinfo ENOTFOUND errors..."
+echo "Waiting time to avoid getaddrinfo ENOTFOUND errors if any..."
 sleep 120
 echo "Done."