From ef445158d0dafd36e48761e6cd52ab5ec7ef4e37 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 3 Oct 2023 16:27:44 +0200
Subject: [PATCH 001/139] Add an empty schedule-daily job

---
 .buildkite/pipeline.schedule-daily.yml | 10 +++++
 .buildkite/pull-requests.json          | 52 +++++++++++++++++---------
 catalog-info.yaml                      | 48 ++++++++++++++++++++++++
 3 files changed, 92 insertions(+), 18 deletions(-)
 create mode 100644 .buildkite/pipeline.schedule-daily.yml

diff --git a/.buildkite/pipeline.schedule-daily.yml b/.buildkite/pipeline.schedule-daily.yml
new file mode 100644
index 00000000000..9f300156c10
--- /dev/null
+++ b/.buildkite/pipeline.schedule-daily.yml
@@ -0,0 +1,10 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+name: integrations-schedule-daily
+
+env:
+  SETUP_GVM_VERSION: "v0.5.1"
+
+# The pipeline is triggered by the scheduler every day
+steps:
+  - label: "Placeholder"
+    command: echo "Hello!"
diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json
index f18e9c0a66a..7921e85205c 100644
--- a/.buildkite/pull-requests.json
+++ b/.buildkite/pull-requests.json
@@ -1,20 +1,36 @@
 {
-    "jobs": [
-        {
-            "enabled": true,
-            "pipelineSlug": "integrations",
-            "allow_org_users": true,
-            "allowed_repo_permissions": ["admin", "write"],
-            "allowed_list": [ ],
-            "set_commit_status": true,
-            "build_on_commit": true,
-            "build_on_comment": true,
-            "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test$",
-            "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test$",
-            "skip_ci_labels": [  ],
-            "skip_target_branches": [ ],
-            "skip_ci_on_only_changed": [ ],
-            "always_require_ci_on_changed": [ ]
-        }
-    ]
+  "jobs": [
+    {
+      "enabled": true,
+      "pipelineSlug": "integrations",
+      "allow_org_users": true,
+      "allowed_repo_permissions": ["admin", "write"],
+      "allowed_list": ["dependabot[bot]", "mergify[bot]"],
+      "set_commit_status": true,
+      "build_on_commit": true,
+      "build_on_comment": true,
+      "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test$",
+      "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test$",
+      "skip_ci_labels": [],
+      "skip_target_branches": [],
+      "skip_ci_on_only_changed": [],
+      "always_require_ci_on_changed": []
+    },
+    {
+      "enabled": false,
+      "pipelineSlug": "integrations-schedule-daily",
+      "allow_org_users": true,
+      "allowed_repo_permissions": ["admin", "write"],
+      "allowed_list": [],
+      "set_commit_status": false,
+      "build_on_commit": false,
+      "build_on_comment": false,
+      "trigger_comment_regex": "",
+      "always_trigger_comment_regex": "",
+      "skip_ci_labels": [],
+      "skip_target_branches": [],
+      "skip_ci_on_only_changed": [],
+      "always_require_ci_on_changed": []
+    }
+  ]
 }
diff --git a/catalog-info.yaml b/catalog-info.yaml
index b0259d4829f..b62b5643173 100644
--- a/catalog-info.yaml
+++ b/catalog-info.yaml
@@ -63,3 +63,51 @@ spec:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
           access_level: READ_ONLY
+
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: pipeline-integrations-schedule-daily
+  description: 'Daily pipeline for the Integrations project'
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/integrations-schedule-daily
+
+spec:
+  type: buildkite-pipeline
+  owner: group:ingest-fp
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: integrations-schedule-daily
+      description: 'Daily pipeline for the Integrations project'
+    spec:
+      pipeline_file: ".buildkite/pipeline.schedule-daily.yml"
+      schedules:
+        main_daily:
+          branch: "main"
+          cronline: "@daily"
+          message: "Run the daily jobs"
+      provider_settings:
+        trigger_mode: none # don't trigger jobs from github activity
+        build_pull_request_forks: false
+        build_pull_requests: true
+        build_tags: false
+        filter_enabled: true
+        filter_condition: >-
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
+      repository: elastic/integrations
+      cancel_intermediate_builds: true
+      cancel_intermediate_builds_branch_filter: '!main'
+      skip_intermediate_builds: true
+      skip_intermediate_builds_branch_filter: '!main'
+      teams:
+        ingest-fp:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY
+

From 554d5a0b074d3b2c6d6ad7e684f7879c9437cff1 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 3 Oct 2023 18:11:06 +0200
Subject: [PATCH 002/139] Add first steps

---
 .buildkite/hooks/post-checkout              |  54 +++++++
 .buildkite/hooks/pre-command                |   9 ++
 .buildkite/hooks/pre-exit                   |   7 +
 .buildkite/pipeline.yml                     |  33 ++++-
 .buildkite/scripts/check_sources.sh         |   9 ++
 .buildkite/scripts/common.sh                | 154 ++++++++++++++++++++
 .buildkite/scripts/run_integrations_sync.sh |  48 ++++++
 7 files changed, 312 insertions(+), 2 deletions(-)
 create mode 100755 .buildkite/hooks/post-checkout
 create mode 100755 .buildkite/hooks/pre-command
 create mode 100755 .buildkite/hooks/pre-exit
 create mode 100755 .buildkite/scripts/check_sources.sh
 create mode 100755 .buildkite/scripts/common.sh
 create mode 100755 .buildkite/scripts/run_integrations_sync.sh

diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout
new file mode 100755
index 00000000000..b1572902664
--- /dev/null
+++ b/.buildkite/hooks/post-checkout
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+set -euo pipefail
+
+checkout_merge() {
+    local target_branch=$1
+    local pr_commit=$2
+    local merge_branch=$3
+
+    if [[ -z "${target_branch}" ]]; then
+        echo "No pull request target branch"
+        exit 1
+    fi
+
+    git fetch -v origin "${target_branch}"
+    git checkout FETCH_HEAD
+    echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+
+    # create temporal branch to merge the PR with the target branch
+    git checkout -b ${merge_branch}
+    echo "New branch created: $(git rev-parse --abbrev-ref HEAD)"
+
+    # set author identity so it can be run git merge
+    git config user.name "github-merged-pr-post-checkout"
+    git config user.email "auto-merge@buildkite"
+
+    git merge --no-edit "${BUILDKITE_COMMIT}" || {
+        local merge_result=$?
+        echo "Merge failed: ${merge_result}"
+        git merge --abort
+        exit ${merge_result}
+    }
+}
+
+pull_request="${BUILDKITE_PULL_REQUEST:-false}"
+
+if [[ "${pull_request}" == "false" ]]; then
+    echo "Not a pull request, skipping"
+    exit 0
+fi
+
+TARGET_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-master}"
+PR_COMMIT="${BUILDKITE_COMMIT}"
+PR_ID=${BUILDKITE_PULL_REQUEST}
+MERGE_BRANCH="pr_merge_${PR_ID}"
+
+checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}"
+
+echo "Commit information"
+
+git --no-pager log --format=%B -n 1
+
+# Ensure buildkite groups are rendered
+echo ""
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
new file mode 100755
index 00000000000..15d96a64d13
--- /dev/null
+++ b/.buildkite/hooks/pre-command
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+export BASE_DIR=$(pwd)
+export GO_VERSION=$(cat .go-version)
+export startTimeInMillis="$(date +%s%3N)"
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
new file mode 100755
index 00000000000..2bb85494013
--- /dev/null
+++ b/.buildkite/hooks/pre-exit
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 34321b61161..b90a40e3fdc 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,5 +1,34 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
+env:
+  SETUP_GVM_VERSION: "v0.5.1"
+  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
+  DOCKER_COMPOSE_VERSION: "v2.17.2"
+  KIND_VERSION: 'v0.20.0'
+  K8S_VERSION: 'v1.27.3'
+  # Elastic package settings
+  # Manage docker output/logs
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true"
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true"
+  # Default license to use by `elastic-package build`
+  ELASTIC_PACKAGE_REPOSITORY_LICENSE: "licenses/Elastic-2.0.txt"
+  # Link definitions path
+  ELASTIC_PACKAGE_LINKS_FILE_PATH: "${BASE_DIR}/links_table.yml"
+
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+  - label: ":white_check_mark: Check go sources"
+    key: "check"
+    command: ".buildkite/scripts/check_sources.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+
+  - label: "Check"
+    key: "test-serverless"
+    command: ".buildkite/scripts/run_integrations_sync.sh"
+    agents:
+      provider: "gcp"
+    depends_on:
+      - step: "check"
+        allow_failure: true
diff --git a/.buildkite/scripts/check_sources.sh b/.buildkite/scripts/check_sources.sh
new file mode 100755
index 00000000000..44363eda529
--- /dev/null
+++ b/.buildkite/scripts/check_sources.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+with_mage
+
+mage -debug check
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
new file mode 100755
index 00000000000..ecbf9fb996e
--- /dev/null
+++ b/.buildkite/scripts/common.sh
@@ -0,0 +1,154 @@
+#!/bin/bash
+
+set -euo pipefail
+
+WORKSPACE="$(pwd)"
+REPO="integrations"
+TMP_FOLDER_TEMPLATE_BASE="tmp.${REPO}"
+platform_type="$(uname)"
+hw_type="$(uname -m)"
+
+retry() {
+  local retries=$1
+  shift
+  local count=0
+  until "$@"; do
+    exit=$?
+    wait=$((2 ** count))
+    count=$((count + 1))
+    if [ $count -lt "$retries" ]; then
+      >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+      sleep $wait
+    else
+      >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+      return $exit
+    fi
+  done
+  return 0
+}
+
+cleanup() {
+  echo "Deleting temporary files..."
+  rm -rf ${WORKSPACE}/${TMP_FOLDER_TEMPLATE_BASE}.*
+  echo "Done."
+}
+
+unset_secrets () {
+  for var in $(printenv | sed 's;=.*;;' | sort); do
+    if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then
+        unset "$var"
+    fi
+  done
+}
+
+check_platform_architeture() {
+  case "${hw_type}" in
+    "x86_64")
+      arch_type="amd64"
+      ;;
+    "aarch64")
+      arch_type="arm64"
+      ;;
+    "arm64")
+      arch_type="arm64"
+      ;;
+    *)
+    echo "The current platform/OS type is unsupported yet"
+    ;;
+  esac
+}
+
+create_workspace() {
+  if [[ ! -d "${WORKSPACE}" ]]; then
+    mkdir -p ${WORKSPACE}
+  fi
+}
+
+add_bin_path() {
+  echo "Adding PATH to the environment variables..."
+  create_workspace
+  export PATH="${PATH}:${WORKSPACE}"
+}
+
+with_go() {
+  echo "Setting up the Go environment..."
+  create_workspace
+  check_platform_architeture
+  local platform_type_lowercase="${platform_type,,}"
+  retry 5 curl -sL -o ${WORKSPACE}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${platform_type_lowercase}-${arch_type}"
+  chmod +x ${WORKSPACE}/gvm
+  eval "$(gvm $(cat .go-version))"
+  go version
+  which go
+  export PATH="${PATH}:$(go env GOPATH):$(go env GOPATH)/bin"
+}
+
+with_mage() {
+    with_go
+
+    local install_packages=(
+            "github.com/magefile/mage"
+            "github.com/elastic/go-licenser"
+            "golang.org/x/tools/cmd/goimports"
+            "github.com/jstemmer/go-junit-report"
+            "gotest.tools/gotestsum"
+    )
+    for pkg in "${install_packages[@]}"; do
+        go install "${pkg}@latest"
+    done
+    mage --version
+}
+
+
+with_go_junit_report() {
+    go install github.com/jstemmer/go-junit-report/v2@latest
+}
+
+with_docker_compose() {
+    mkdir -p ${WORKSPACE}/bin
+    retry 5 curl -SL -o ${WORKSPACE}/bin/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64"
+    chmod +x ${WORKSPACE}/bin/docker-compose
+    docker-compose version
+}
+
+with_kubernetes() {
+    mkdir -p ${WORKSPACE}/bin
+    retry 5 curl -sSLo ${WORKSPACE}/bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
+    chmod +x ${WORKSPACE}/bin/kind
+    kind version
+    which kind
+
+    mkdir -p ${WORKSPACE}/bin
+    retry 5 curl -sSLo ${WORKSPACE}/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
+    chmod +x ${WORKSPACE}/bin/kubectl
+    kubectl version --client
+    which kubectl
+}
+
+google_cloud_upload_auth() {
+  local secretFileLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX")/google-cloud-credentials.json
+  echo "${PRIVATE_INFRA_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
+  gcloud auth activate-service-account --key-file ${secretFileLocation} 2> /dev/null
+  export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
+}
+
+google_cloud_signing_auth() {
+  local secretFileLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX")/google-cloud-credentials.json
+  echo "${SIGNING_PACKAGES_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
+  gcloud auth activate-service-account --key-file ${secretFileLocation} 2> /dev/null
+  export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
+}
+
+google_cloud_logout_active_account() {
+  local active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null)
+  if [ -n "$active_account" ]; then
+    echo "Logging out from GCP for active account"
+    gcloud auth revoke $active_account > /dev/null 2>&1
+    if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then
+      unset GOOGLE_APPLICATION_CREDENTIALS
+    fi
+    cleanup
+  else
+    echo "No active GCP accounts found."
+  fi
+}
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
new file mode 100755
index 00000000000..3dbb91399b2
--- /dev/null
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+source .buildkite/scripts/common.sh
+
+if [ ! -d packages ]; then
+    buildkite-agent annotate "Missing packages folder" --style "error"
+    exit 1
+fi
+
+use_elastic_package() {
+    echo "Installing elastic-package"
+    mkdir -p build
+    go build -o build/elastic-package github.com/elastic/elastic-package
+}
+
+check_git_diff() {
+    cd ${WORKSPACE}
+    echo "git update-index"
+    git update-index --refresh
+    echo "git diff-index"
+    git diff-index --exit-code HEAD --
+}
+
+is_pr_affected() {
+    echo "1"
+}
+
+is_pr() {
+    if [ "${BUILDKITE_PULL_REQUEST}" == "false" ]; then
+        return 0
+    fi
+    return 1
+}
+
+
+add_bin_path
+
+with_mage
+with_docker_compose
+
+
+cd packages
+for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
+    integration=$(basename ${it})
+    echo "Package ${integration}"
+
+done
+

From dcabe5be5baf0dd977e2581a4bf34b6a527e763d Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 3 Oct 2023 19:04:05 +0200
Subject: [PATCH 003/139] Avoid running all packages in Jenkins

---
 .ci/Jenkinsfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index fa13f663739..47c371c8a41 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -278,6 +278,8 @@ def checkGitDiff() {
 }
 
 def isPrAffected(integrationName) {
+  return false // TODO: remove after tests
+
   def manifest = readYaml(file: "${integrationName}/manifest.yml")
 
   // Packages supported in Kibana >= 8.0.0, shouldn't be included in daily tests of the stack 7.x

From ce9e0f19f943b802fab2ea67d7034ed4b83c28cc Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 3 Oct 2023 19:11:17 +0200
Subject: [PATCH 004/139] Fix mage and gvm installation

---
 .buildkite/scripts/common.sh | 20 ++++----------------
 1 file changed, 4 insertions(+), 16 deletions(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index ecbf9fb996e..b70fa01baf1 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -58,25 +58,18 @@ check_platform_architeture() {
   esac
 }
 
-create_workspace() {
-  if [[ ! -d "${WORKSPACE}" ]]; then
-    mkdir -p ${WORKSPACE}
-  fi
-}
-
 add_bin_path() {
+  mkdir -p ${WORKSPACE}/bin
   echo "Adding PATH to the environment variables..."
-  create_workspace
-  export PATH="${PATH}:${WORKSPACE}"
+  export PATH="${PATH}:${WORKSPACE}/bin"
 }
 
 with_go() {
   echo "Setting up the Go environment..."
-  create_workspace
   check_platform_architeture
   local platform_type_lowercase="${platform_type,,}"
-  retry 5 curl -sL -o ${WORKSPACE}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${platform_type_lowercase}-${arch_type}"
-  chmod +x ${WORKSPACE}/gvm
+  retry 5 curl -sL -o ${WORKSPACE}/bin/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${platform_type_lowercase}-${arch_type}"
+  chmod +x ${WORKSPACE}/bin/gvm
   eval "$(gvm $(cat .go-version))"
   go version
   which go
@@ -99,11 +92,6 @@ with_mage() {
     mage --version
 }
 
-
-with_go_junit_report() {
-    go install github.com/jstemmer/go-junit-report/v2@latest
-}
-
 with_docker_compose() {
     mkdir -p ${WORKSPACE}/bin
     retry 5 curl -SL -o ${WORKSPACE}/bin/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64"

From e9af47e18c7ac761c9abbb01c0287789b79966b8 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 3 Oct 2023 19:21:14 +0200
Subject: [PATCH 005/139] test

---
 .buildkite/scripts/check_sources.sh |  1 +
 .buildkite/scripts/common.sh        | 33 +++++++++++++++++------------
 2 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/.buildkite/scripts/check_sources.sh b/.buildkite/scripts/check_sources.sh
index 44363eda529..9bc87edc42a 100755
--- a/.buildkite/scripts/check_sources.sh
+++ b/.buildkite/scripts/check_sources.sh
@@ -4,6 +4,7 @@ source .buildkite/scripts/common.sh
 
 set -euo pipefail
 
+add_bin_path
 with_mage
 
 mage -debug check
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index b70fa01baf1..40be3983854 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -3,6 +3,7 @@
 set -euo pipefail
 
 WORKSPACE="$(pwd)"
+BIN_FOLDER="${WORKSPACE}/bin"
 REPO="integrations"
 TMP_FOLDER_TEMPLATE_BASE="tmp.${REPO}"
 platform_type="$(uname)"
@@ -58,18 +59,24 @@ check_platform_architeture() {
   esac
 }
 
+create_bin_folder() {
+  mkdir -p ${BIN_FOLDER}
+}
+
 add_bin_path() {
-  mkdir -p ${WORKSPACE}/bin
+  create_bin_folder
   echo "Adding PATH to the environment variables..."
-  export PATH="${PATH}:${WORKSPACE}/bin"
+  export PATH="${PATH}:${BIN_FOLDER}"
 }
 
 with_go() {
+  create_bin_folder
   echo "Setting up the Go environment..."
   check_platform_architeture
   local platform_type_lowercase="${platform_type,,}"
-  retry 5 curl -sL -o ${WORKSPACE}/bin/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${platform_type_lowercase}-${arch_type}"
-  chmod +x ${WORKSPACE}/bin/gvm
+  echo " GVM ${SETUP_GVM_VERSION} (platform ${platform_type_lowercase} arch ${arch_type}"
+  retry 5 curl -sL -o ${BIN_FOLDER}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${platform_type_lowercase}-${arch_type}"
+  chmod +x ${BIN_FOLDER}/gvm
   eval "$(gvm $(cat .go-version))"
   go version
   which go
@@ -77,6 +84,7 @@ with_go() {
 }
 
 with_mage() {
+    create_bin_folder
     with_go
 
     local install_packages=(
@@ -93,22 +101,21 @@ with_mage() {
 }
 
 with_docker_compose() {
-    mkdir -p ${WORKSPACE}/bin
-    retry 5 curl -SL -o ${WORKSPACE}/bin/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64"
-    chmod +x ${WORKSPACE}/bin/docker-compose
+    create_bin_folder
+    retry 5 curl -SL -o ${BIN_FOLDER}/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64"
+    chmod +x ${BIN_FOLDER}/docker-compose
     docker-compose version
 }
 
 with_kubernetes() {
-    mkdir -p ${WORKSPACE}/bin
-    retry 5 curl -sSLo ${WORKSPACE}/bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
-    chmod +x ${WORKSPACE}/bin/kind
+    create_bin_folder
+    retry 5 curl -sSLo ${BIN_FOLDER}/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
+    chmod +x ${BIN_FOLDER}/kind
     kind version
     which kind
 
-    mkdir -p ${WORKSPACE}/bin
-    retry 5 curl -sSLo ${WORKSPACE}/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
-    chmod +x ${WORKSPACE}/bin/kubectl
+    retry 5 curl -sSLo ${BIN_FOLDER}/kubectl "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
+    chmod +x ${BIN_FOLDER}/kubectl
     kubectl version --client
     which kubectl
 }

From 884bed7ae9029d95bb57b9e466263c3fc561da04 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 4 Oct 2023 16:28:48 +0200
Subject: [PATCH 006/139] Add check-git-diff to check sources

---
 .buildkite/hooks/pre-exit                   | 3 ++-
 .buildkite/scripts/check_sources.sh         | 2 ++
 .buildkite/scripts/common.sh                | 9 +++++++++
 .buildkite/scripts/run_integrations_sync.sh | 9 +--------
 4 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index 2bb85494013..baacc7e5c38 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -4,4 +4,5 @@ source .buildkite/scripts/common.sh
 
 set -euo pipefail
 
-
+unset_secrets
+cleanup
diff --git a/.buildkite/scripts/check_sources.sh b/.buildkite/scripts/check_sources.sh
index 9bc87edc42a..cb1261a1616 100755
--- a/.buildkite/scripts/check_sources.sh
+++ b/.buildkite/scripts/check_sources.sh
@@ -8,3 +8,5 @@ add_bin_path
 with_mage
 
 mage -debug check
+
+check_git_diff
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 40be3983854..3d23783f54c 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -147,3 +147,12 @@ google_cloud_logout_active_account() {
     echo "No active GCP accounts found."
   fi
 }
+
+check_git_diff() {
+    cd ${WORKSPACE}
+    echo "git update-index"
+    git update-index --refresh
+    echo "git diff-index"
+    git diff-index --exit-code HEAD --
+}
+
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 3dbb91399b2..118bd00b5d3 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -13,14 +13,6 @@ use_elastic_package() {
     go build -o build/elastic-package github.com/elastic/elastic-package
 }
 
-check_git_diff() {
-    cd ${WORKSPACE}
-    echo "git update-index"
-    git update-index --refresh
-    echo "git diff-index"
-    git diff-index --exit-code HEAD --
-}
-
 is_pr_affected() {
     echo "1"
 }
@@ -37,6 +29,7 @@ add_bin_path
 
 with_mage
 with_docker_compose
+with_kubernetes
 
 
 cd packages

From a1882c15768cc5e5a6567a17bd48bb26f17da222 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 4 Oct 2023 16:58:39 +0200
Subject: [PATCH 007/139] Add check command

---
 .buildkite/pipeline.yml                     |  5 +++--
 .buildkite/scripts/common.sh                |  2 +-
 .buildkite/scripts/run_integrations_sync.sh | 13 ++++++++++++-
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index b90a40e3fdc..78abe9e51e6 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -24,8 +24,9 @@ steps:
       cpu: "8"
       memory: "4G"
 
-  - label: "Check"
-    key: "test-serverless"
+  # to be moved to schedule-daily job
+  - label: "Check integrations"
+    key: "test-integrations-sync"
     command: ".buildkite/scripts/run_integrations_sync.sh"
     agents:
       provider: "gcp"
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 3d23783f54c..b2d37d4763d 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -102,7 +102,7 @@ with_mage() {
 
 with_docker_compose() {
     create_bin_folder
-    retry 5 curl -SL -o ${BIN_FOLDER}/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64"
+    retry 5 curl -sSL -o ${BIN_FOLDER}/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64"
     chmod +x ${BIN_FOLDER}/docker-compose
     docker-compose version
 }
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 118bd00b5d3..aa20767f1c5 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -13,6 +13,10 @@ use_elastic_package() {
     go build -o build/elastic-package github.com/elastic/elastic-package
 }
 
+prepare_stack() {
+    echo "Prepare stack"
+}
+
 is_pr_affected() {
     echo "1"
 }
@@ -31,11 +35,18 @@ with_mage
 with_docker_compose
 with_kubernetes
 
+use_elastic_package
 
 cd packages
 for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
     integration=$(basename ${it})
-    echo "Package ${integration}"
+    echo "Package ${integration}: check"
+
+    pushd ${integration} 2> /dev/null
+
+    echo "Check integration: ${integration}"
+    ../../build/elastic-package check -v
 
+    popd 2> /dev/null
 done
 

From 6dd38bd17ffc8997aa60cb812085f44fabefb708 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 4 Oct 2023 17:11:40 +0200
Subject: [PATCH 008/139] Show links table info

---
 .buildkite/scripts/run_integrations_sync.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index aa20767f1c5..6a7f30ccb14 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -44,6 +44,11 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
 
     pushd ${integration} 2> /dev/null
 
+    echo "Links file path: ${ELASTIC_PACKAGE_LINKS_FILE_PATH}"
+    directory=$(dirname ${ELASTIC_PACKAGE_LINKS_FILE_PATH})
+    echo "Folder links file path: $directory"
+    ls -l  $directory
+
     echo "Check integration: ${integration}"
     ../../build/elastic-package check -v
 

From 81cd4f4fe957b9c09aaef281cf46ab5b5095ebc5 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 4 Oct 2023 17:26:37 +0200
Subject: [PATCH 009/139] test

---
 .buildkite/scripts/run_integrations_sync.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 6a7f30ccb14..315b81a2cd9 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -45,6 +45,7 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
     pushd ${integration} 2> /dev/null
 
     echo "Links file path: ${ELASTIC_PACKAGE_LINKS_FILE_PATH}"
+    echo "Base dir: ${BASE_DIR}"
     directory=$(dirname ${ELASTIC_PACKAGE_LINKS_FILE_PATH})
     echo "Folder links file path: $directory"
     ls -l  $directory

From 24f2596c78dcb8e5f9615d6f9ed0742d71b653cb Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 4 Oct 2023 17:37:43 +0200
Subject: [PATCH 010/139] Test with relative path from root repository
 directory

---
 .buildkite/pipeline.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 78abe9e51e6..a74c97eaf34 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -12,8 +12,8 @@ env:
   ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true"
   # Default license to use by `elastic-package build`
   ELASTIC_PACKAGE_REPOSITORY_LICENSE: "licenses/Elastic-2.0.txt"
-  # Link definitions path
-  ELASTIC_PACKAGE_LINKS_FILE_PATH: "${BASE_DIR}/links_table.yml"
+  # Link definitions path (use relative path from working directory)
+  ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
 
 steps:
   - label: ":white_check_mark: Check go sources"

From 5234eaa72c200632af0ad444a0f4890c910aa39a Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 4 Oct 2023 17:53:35 +0200
Subject: [PATCH 011/139] Set links file path in pre-command hook

---
 .buildkite/hooks/pre-command | 1 +
 .buildkite/pipeline.yml      | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 15d96a64d13..9f8a9cb7a0a 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -7,3 +7,4 @@ set -euo pipefail
 export BASE_DIR=$(pwd)
 export GO_VERSION=$(cat .go-version)
 export startTimeInMillis="$(date +%s%3N)"
+export ELASTIC_PACKAGE_LINKS_FILE_PATH=${BASE_DIR}/${ELASTIC_PACKAGE_LINKS_FILE_PATH}
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index a74c97eaf34..826874e0681 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -12,7 +12,7 @@ env:
   ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true"
   # Default license to use by `elastic-package build`
   ELASTIC_PACKAGE_REPOSITORY_LICENSE: "licenses/Elastic-2.0.txt"
-  # Link definitions path (use relative path from working directory)
+  # Link definitions path (full path to be set in the corresponding step)
   ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
 
 steps:

From 7b36121e120a3222f01a92d66a02e7df0b2aa14f Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 4 Oct 2023 18:14:05 +0200
Subject: [PATCH 012/139] Set links file full path just if exists var

---
 .buildkite/hooks/pre-command | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 9f8a9cb7a0a..bf71684a0de 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -7,4 +7,8 @@ set -euo pipefail
 export BASE_DIR=$(pwd)
 export GO_VERSION=$(cat .go-version)
 export startTimeInMillis="$(date +%s%3N)"
-export ELASTIC_PACKAGE_LINKS_FILE_PATH=${BASE_DIR}/${ELASTIC_PACKAGE_LINKS_FILE_PATH}
+
+if [ -n "${ELASTIC_PACKAGE_LINKS_FILE_PATH+x}" ]; then
+    # first upload pipeline does not have the environment variables defined in the YAML
+    export ELASTIC_PACKAGE_LINKS_FILE_PATH=${BASE_DIR}/${ELASTIC_PACKAGE_LINKS_FILE_PATH}
+fi

From 17ff5d122ddb34b6b7ed468186c5b26ff2b59da4 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 4 Oct 2023 19:26:06 +0200
Subject: [PATCH 013/139] Install yq and test spec version

---
 .buildkite/pipeline.yml                     |  1 +
 .buildkite/scripts/common.sh                | 15 ++++++++
 .buildkite/scripts/run_integrations_sync.sh | 38 ++++++++++++++++++---
 3 files changed, 49 insertions(+), 5 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 826874e0681..26d3dbe34b6 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -6,6 +6,7 @@ env:
   DOCKER_COMPOSE_VERSION: "v2.17.2"
   KIND_VERSION: 'v0.20.0'
   K8S_VERSION: 'v1.27.3'
+  YQ_VERSION: 'v4.35.2'
   # Elastic package settings
   # Manage docker output/logs
   ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true"
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index b2d37d4763d..fd3d69c0249 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -156,3 +156,18 @@ check_git_diff() {
     git diff-index --exit-code HEAD --
 }
 
+with_yq() {
+    check_platform_architeture
+    local platform_type_lowercase="${platform_type,,}"
+    local binary="yq_${platform_type_lowercase}_${arch_type}
+
+    retry 5 curl -sSL -o ${BIN_FOLDER}/yq.tar.gz "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${binary}.tar.gz"
+
+    tar -C ${BIN_FOLDER} -xpf ${BIN_FOLDER}/yq.tar.gz ./yq_linux_amd64
+
+    mv ${BIN_FOLDER}/yq_linux_amd64 ${BIN_FOLDER}/yq
+    chmod +x ${BIN_FOLDER}/yq
+    yq --version
+
+    rm -rf ${BIN_FOLDER}/yq.tar.gz
+}
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 315b81a2cd9..0a33e2249f2 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -15,6 +15,27 @@ use_elastic_package() {
 
 prepare_stack() {
     echo "Prepare stack"
+
+    local args=""
+    if [ -n "${STACK_VERSION+x}" ]; then
+        args="${args} --version ${STACK_VERSION}"
+    fi
+
+    echo "Update the Elastic stack"
+    ../../build/elastic-package stack update ${args}
+
+    echo "Boot up the Elastic stack"
+    ../../build/elastic-package stack up -d ${args}
+}
+
+is_spec_3_0_0() {
+    local pkg_spec=$(cat manifest.yml | yq '.format_version')
+    local major_version=$(echo $pkg_spec | cut -d '.' -f 1)
+
+    if [ $major_version -ge 3 ]; then
+        return 0
+    fi
+    return 1
 }
 
 is_pr_affected() {
@@ -31,12 +52,15 @@ is_pr() {
 
 add_bin_path
 
+with_yq
 with_mage
 with_docker_compose
 with_kubernetes
 
 use_elastic_package
 
+prepare_stack
+
 cd packages
 for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
     integration=$(basename ${it})
@@ -44,15 +68,19 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
 
     pushd ${integration} 2> /dev/null
 
-    echo "Links file path: ${ELASTIC_PACKAGE_LINKS_FILE_PATH}"
-    echo "Base dir: ${BASE_DIR}"
-    directory=$(dirname ${ELASTIC_PACKAGE_LINKS_FILE_PATH})
-    echo "Folder links file path: $directory"
-    ls -l  $directory
+    if [ ! is_spec_3_0_0 ]; then
+        echo "Not v3 spec version. Skipped"
+        continue
+    fi
 
     echo "Check integration: ${integration}"
     ../../build/elastic-package check -v
 
+    # echo "Test integration: ${integration}"
+    #  # eval "$(../../build/elastic-package stack shellinit)"
+    # ../../build/elastic-package test -v --report-format xUnit --report-output file --test-coverage
+
     popd 2> /dev/null
+    exit 0
 done
 

From 22e414c2b474a1483c0f893fcd3427f6175736e2 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 4 Oct 2023 22:20:07 +0200
Subject: [PATCH 014/139] add missing double quotes

---
 .buildkite/scripts/common.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index fd3d69c0249..7c1ca154bcf 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -159,7 +159,7 @@ check_git_diff() {
 with_yq() {
     check_platform_architeture
     local platform_type_lowercase="${platform_type,,}"
-    local binary="yq_${platform_type_lowercase}_${arch_type}
+    local binary="yq_${platform_type_lowercase}_${arch_type}"
 
     retry 5 curl -sSL -o ${BIN_FOLDER}/yq.tar.gz "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${binary}.tar.gz"
 

From dea56620cb5fe80cae955b185afdde645e33a03d Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 4 Oct 2023 22:40:07 +0200
Subject: [PATCH 015/139] Use variable for elastic-package binary path

---
 .buildkite/scripts/run_integrations_sync.sh | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 0a33e2249f2..e64a91c7c4d 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -10,7 +10,7 @@ fi
 use_elastic_package() {
     echo "Installing elastic-package"
     mkdir -p build
-    go build -o build/elastic-package github.com/elastic/elastic-package
+    go build -o ${ELASTIC_PACKAGE_BIN} github.com/elastic/elastic-package
 }
 
 prepare_stack() {
@@ -22,10 +22,10 @@ prepare_stack() {
     fi
 
     echo "Update the Elastic stack"
-    ../../build/elastic-package stack update ${args}
+    ${ELASTIC_PACKAGE_BIN} stack update ${args}
 
     echo "Boot up the Elastic stack"
-    ../../build/elastic-package stack up -d ${args}
+    ${ELASTIC_PACKAGE_BIN} stack up -d ${args}
 }
 
 is_spec_3_0_0() {
@@ -57,6 +57,7 @@ with_mage
 with_docker_compose
 with_kubernetes
 
+ELASTIC_PACKAGE_BIN=${WORKSPACE}/build/elastic-package
 use_elastic_package
 
 prepare_stack
@@ -74,11 +75,11 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
     fi
 
     echo "Check integration: ${integration}"
-    ../../build/elastic-package check -v
+    ${ELASTIC_PACKAGE_BIN} check -v
 
     # echo "Test integration: ${integration}"
     #  # eval "$(../../build/elastic-package stack shellinit)"
-    # ../../build/elastic-package test -v --report-format xUnit --report-output file --test-coverage
+    # ${ELASTIC_PACKAGE_BIN} test -v --report-format xUnit --report-output file --test-coverage
 
     popd 2> /dev/null
     exit 0

From 87f110e5b175b1fe96aa03797036734237d1f19c Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 4 Oct 2023 23:01:28 +0200
Subject: [PATCH 016/139] Add verbose to stack commands

---
 .buildkite/scripts/run_integrations_sync.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index e64a91c7c4d..921337f1d41 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -16,16 +16,18 @@ use_elastic_package() {
 prepare_stack() {
     echo "Prepare stack"
 
-    local args=""
+    local args="-v"
     if [ -n "${STACK_VERSION+x}" ]; then
         args="${args} --version ${STACK_VERSION}"
     fi
 
     echo "Update the Elastic stack"
     ${ELASTIC_PACKAGE_BIN} stack update ${args}
+    echo ""
 
     echo "Boot up the Elastic stack"
     ${ELASTIC_PACKAGE_BIN} stack up -d ${args}
+    echo ""
 }
 
 is_spec_3_0_0() {

From 14d5ed8ab2bc7934dc153adf1223a25d1483fcb0 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 5 Oct 2023 17:28:33 +0200
Subject: [PATCH 017/139] Added a draft version of publish package storage step

---
 .buildkite/pipeline.yml                     |  8 ++++
 .buildkite/scripts/common.sh                | 18 +++++++
 .buildkite/scripts/publish_packages.sh      | 53 +++++++++++++++++++++
 .buildkite/scripts/run_integrations_sync.sh |  9 +---
 4 files changed, 81 insertions(+), 7 deletions(-)
 create mode 100644 .buildkite/scripts/publish_packages.sh

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 26d3dbe34b6..bc244a9aaa0 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -25,6 +25,14 @@ steps:
       cpu: "8"
       memory: "4G"
 
+  - label: "Publish packages"
+    key: "check"
+    command: ".buildkite/scripts/publish_packages.sh" # TODO: missing signature and publishing loops
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+
   # to be moved to schedule-daily job
   - label: "Check integrations"
     key: "test-integrations-sync"
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 7c1ca154bcf..0e6e6b3a95d 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -8,6 +8,7 @@ REPO="integrations"
 TMP_FOLDER_TEMPLATE_BASE="tmp.${REPO}"
 platform_type="$(uname)"
 hw_type="$(uname -m)"
+export ELASTIC_PACKAGE_BIN=${WORKSPACE}/build/elastic-package
 
 retry() {
   local retries=$1
@@ -171,3 +172,20 @@ with_yq() {
 
     rm -rf ${BIN_FOLDER}/yq.tar.gz
 }
+
+use_elastic_package() {
+    echo "Installing elastic-package"
+    mkdir -p build
+    go build -o ${ELASTIC_PACKAGE_BIN} github.com/elastic/elastic-package
+}
+
+is_already_published() {
+    local packageZip=$1
+
+    if curl -s --head https://package-storage.elastic.co/artifacts/packages/${packageZip} | grep -q "HTTP/2 200" ; then
+        echo "- Already published ${packageZip}"
+        return 0
+    fi
+    echo "- Not published ${packageZip}"
+    return 1
+}
diff --git a/.buildkite/scripts/publish_packages.sh b/.buildkite/scripts/publish_packages.sh
new file mode 100644
index 00000000000..79718c9c90f
--- /dev/null
+++ b/.buildkite/scripts/publish_packages.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+source .buildkite/scripts/common.sh
+set -euo pipefail
+
+if [ ${SKIP_PUBLISHING:-"false"} == "true" ] ; then
+    echo "packageStoragePublish: skipping because skip_publishing param is ${SKIP_PUBLISHING}"
+    exit 0
+fi
+
+if [ "${BUILDKITE_PULL_REQUEST}" == "false" ]; then
+    if [[ "${BUILDKITE_BRANCH}" == "main" || "${BUILDKITE_BRANCH}" =~ ^backport- ]]; then
+        echo "packageStoragePublish: not the main branch or a backport branch, nothing will be published"
+        exit 0
+    fi
+fi
+
+use_elastic_package
+
+unpublished="false"
+
+cd packages
+for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
+    integration=$(basename ${it})
+    echo "Package ${integration}: check"
+
+    version=$(cat manifest.yml | yq .version)
+    name=$(cat manifest.yml | yq .name)
+
+    package_zip="${name}-${version}.zip"
+
+    if is_already_published ${packageZip} ; then
+        echo "Skipping. ${packageZip} already published"
+        continue
+    fi
+
+    pushd ${integration} > /dev/null
+    echo "Build integration as zip: ${integration}"
+    ${ELASTIC_PACKAGE_BIN} check
+    ${ELASTIC_PACKAGE_BIN} build --zip
+    popd > /dev/null
+
+    unpublished="true"
+done
+
+if [ "${unpublished}" == "false" ]; then
+    echo "All packages are in sync"
+    exit 0
+fi
+
+
+# TODO require signing: to be based on elastic-package
+# TODO require publishing: to be based on elastic-package
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 921337f1d41..3a4d41ecfc1 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -2,17 +2,13 @@
 
 source .buildkite/scripts/common.sh
 
+set -euo pipefail
+
 if [ ! -d packages ]; then
     buildkite-agent annotate "Missing packages folder" --style "error"
     exit 1
 fi
 
-use_elastic_package() {
-    echo "Installing elastic-package"
-    mkdir -p build
-    go build -o ${ELASTIC_PACKAGE_BIN} github.com/elastic/elastic-package
-}
-
 prepare_stack() {
     echo "Prepare stack"
 
@@ -59,7 +55,6 @@ with_mage
 with_docker_compose
 with_kubernetes
 
-ELASTIC_PACKAGE_BIN=${WORKSPACE}/build/elastic-package
 use_elastic_package
 
 prepare_stack

From 130e672984bd93a379d14edae9f6f7cb94ea598c Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 5 Oct 2023 17:39:29 +0200
Subject: [PATCH 018/139] Change step key

---
 .buildkite/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index bc244a9aaa0..cd119633942 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -26,7 +26,7 @@ steps:
       memory: "4G"
 
   - label: "Publish packages"
-    key: "check"
+    key: "publish-packages"
     command: ".buildkite/scripts/publish_packages.sh" # TODO: missing signature and publishing loops
     agents:
       image: "${LINUX_AGENT_IMAGE}"

From 93a410490ed2b29a81bdd8277d6ed1a7291ac3e1 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 5 Oct 2023 17:44:15 +0200
Subject: [PATCH 019/139] Add dependencies

---
 .buildkite/pipeline.yml                | 5 ++++-
 .buildkite/scripts/publish_packages.sh | 1 +
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index cd119633942..512a39d510d 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -32,6 +32,9 @@ steps:
       image: "${LINUX_AGENT_IMAGE}"
       cpu: "8"
       memory: "4G"
+    depends_on:
+      - step: "check"
+        allow_failure: false
 
   # to be moved to schedule-daily job
   - label: "Check integrations"
@@ -41,4 +44,4 @@ steps:
       provider: "gcp"
     depends_on:
       - step: "check"
-        allow_failure: true
+        allow_failure: false
diff --git a/.buildkite/scripts/publish_packages.sh b/.buildkite/scripts/publish_packages.sh
index 79718c9c90f..9287b5d71a9 100644
--- a/.buildkite/scripts/publish_packages.sh
+++ b/.buildkite/scripts/publish_packages.sh
@@ -15,6 +15,7 @@ if [ "${BUILDKITE_PULL_REQUEST}" == "false" ]; then
     fi
 fi
 
+with_yq
 use_elastic_package
 
 unpublished="false"

From a3a07abc832c07a9169fd1343149c372cc563657 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 5 Oct 2023 18:07:34 +0200
Subject: [PATCH 020/139] Change agent for publish packages

---
 .buildkite/pipeline.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 512a39d510d..13ab2cc6ee4 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -29,9 +29,7 @@ steps:
     key: "publish-packages"
     command: ".buildkite/scripts/publish_packages.sh" # TODO: missing signature and publishing loops
     agents:
-      image: "${LINUX_AGENT_IMAGE}"
-      cpu: "8"
-      memory: "4G"
+      provider: "gcp"
     depends_on:
       - step: "check"
         allow_failure: false

From 5b54f1c07dc495ed05bf2dd7429dc2f1a1dfe68a Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 5 Oct 2023 20:23:58 +0200
Subject: [PATCH 021/139] Add debug

---
 .buildkite/scripts/common.sh           | 1 +
 .buildkite/scripts/publish_packages.sh | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 0e6e6b3a95d..6f164b0558a 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -162,6 +162,7 @@ with_yq() {
     local platform_type_lowercase="${platform_type,,}"
     local binary="yq_${platform_type_lowercase}_${arch_type}"
 
+    echo curl -sSL -o ${BIN_FOLDER}/yq.tar.gz "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${binary}.tar.gz"
     retry 5 curl -sSL -o ${BIN_FOLDER}/yq.tar.gz "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${binary}.tar.gz"
 
     tar -C ${BIN_FOLDER} -xpf ${BIN_FOLDER}/yq.tar.gz ./yq_linux_amd64
diff --git a/.buildkite/scripts/publish_packages.sh b/.buildkite/scripts/publish_packages.sh
index 9287b5d71a9..839e8a5d60e 100644
--- a/.buildkite/scripts/publish_packages.sh
+++ b/.buildkite/scripts/publish_packages.sh
@@ -15,6 +15,7 @@ if [ "${BUILDKITE_PULL_REQUEST}" == "false" ]; then
     fi
 fi
 
+add_bin_path
 with_yq
 use_elastic_package
 

From 4dee0a91f9789570cb7c9d122db99c0d38836121 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 5 Oct 2023 20:29:50 +0200
Subject: [PATCH 022/139] Add go in publish_packages

---
 .buildkite/scripts/common.sh           | 1 +
 .buildkite/scripts/publish_packages.sh | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 6f164b0558a..cb84408ed12 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -162,6 +162,7 @@ with_yq() {
     local platform_type_lowercase="${platform_type,,}"
     local binary="yq_${platform_type_lowercase}_${arch_type}"
 
+    # TODO: remove debug
     echo curl -sSL -o ${BIN_FOLDER}/yq.tar.gz "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${binary}.tar.gz"
     retry 5 curl -sSL -o ${BIN_FOLDER}/yq.tar.gz "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${binary}.tar.gz"
 
diff --git a/.buildkite/scripts/publish_packages.sh b/.buildkite/scripts/publish_packages.sh
index 839e8a5d60e..95f1e7516b6 100644
--- a/.buildkite/scripts/publish_packages.sh
+++ b/.buildkite/scripts/publish_packages.sh
@@ -16,6 +16,7 @@ if [ "${BUILDKITE_PULL_REQUEST}" == "false" ]; then
 fi
 
 add_bin_path
+with_go
 with_yq
 use_elastic_package
 

From d11957bff02feb62c5322607024b243bfed32153 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 5 Oct 2023 20:32:23 +0200
Subject: [PATCH 023/139] Check python

---
 .buildkite/scripts/run_integrations_sync.sh | 29 +++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 3a4d41ecfc1..9508e725022 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -9,12 +9,33 @@ if [ ! -d packages ]; then
     exit 1
 fi
 
+kibana_version() {
+    kibana_version=$(cat manifest.yml | yq ".conditions.kibana.version")
+    if [ $kibana_version != "null" ]; then
+        echo $kibana_version
+        return
+    fi
+
+    kibana_version=$(cat manifest.yml | yq ".conditions.\"kibana.version\"")
+    if [ $kibana_version != "null" ]; then
+        echo $kibana_version
+        return
+    fi
+
+    echo "null"
+}
+
 prepare_stack() {
     echo "Prepare stack"
 
     local args="-v"
     if [ -n "${STACK_VERSION+x}" ]; then
         args="${args} --version ${STACK_VERSION}"
+    else
+        kibana_constraint=$(kibana_version)
+        if [ "$condition" != "null" ]; then
+            # FIXME
+        fi
     fi
 
     echo "Update the Elastic stack"
@@ -59,6 +80,14 @@ use_elastic_package
 
 prepare_stack
 
+echo "Checking python command..."
+if ! command -v python &> /dev/null ; then
+    echo "⚠️  python is not installed"
+fi
+if ! command -v python3 &> /dev/null ; then
+    echo "⚠️  python3 is not installed"
+fi
+
 cd packages
 for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
     integration=$(basename ${it})

From e9f864655a269dc28dda304cd7923dfb8651caaa Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 5 Oct 2023 20:38:13 +0200
Subject: [PATCH 024/139] Add a noop command

---
 .buildkite/scripts/run_integrations_sync.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 9508e725022..065ce8ba67c 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -35,6 +35,7 @@ prepare_stack() {
         kibana_constraint=$(kibana_version)
         if [ "$condition" != "null" ]; then
             # FIXME
+            true
         fi
     fi
 

From d39157649058fc0256b07cfda4b8489b66587552 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 5 Oct 2023 20:39:28 +0200
Subject: [PATCH 025/139] Change directory before reading manifest

---
 .buildkite/scripts/publish_packages.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.buildkite/scripts/publish_packages.sh b/.buildkite/scripts/publish_packages.sh
index 95f1e7516b6..97c2e7a1d6a 100644
--- a/.buildkite/scripts/publish_packages.sh
+++ b/.buildkite/scripts/publish_packages.sh
@@ -27,6 +27,8 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
     integration=$(basename ${it})
     echo "Package ${integration}: check"
 
+    pushd ${integration} > /dev/null
+
     version=$(cat manifest.yml | yq .version)
     name=$(cat manifest.yml | yq .name)
 
@@ -37,7 +39,6 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
         continue
     fi
 
-    pushd ${integration} > /dev/null
     echo "Build integration as zip: ${integration}"
     ${ELASTIC_PACKAGE_BIN} check
     ${ELASTIC_PACKAGE_BIN} build --zip

From 76cf40a8769f9fa3d0b254bd729361cd0643905d Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 5 Oct 2023 21:31:23 +0200
Subject: [PATCH 026/139] Fix variable name

---
 .buildkite/scripts/publish_packages.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/publish_packages.sh b/.buildkite/scripts/publish_packages.sh
index 97c2e7a1d6a..6f27f513102 100644
--- a/.buildkite/scripts/publish_packages.sh
+++ b/.buildkite/scripts/publish_packages.sh
@@ -34,8 +34,8 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
 
     package_zip="${name}-${version}.zip"
 
-    if is_already_published ${packageZip} ; then
-        echo "Skipping. ${packageZip} already published"
+    if is_already_published ${package_zip} ; then
+        echo "Skipping. ${package_zip} already published"
         continue
     fi
 

From 7914b73c8a843685c07c19889afd6526986d6736 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 5 Oct 2023 21:33:47 +0200
Subject: [PATCH 027/139] Remove kibana.version check from prepare stack

---
 .buildkite/scripts/run_integrations_sync.sh | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 065ce8ba67c..8bb5716661e 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -31,12 +31,13 @@ prepare_stack() {
     local args="-v"
     if [ -n "${STACK_VERSION+x}" ]; then
         args="${args} --version ${STACK_VERSION}"
-    else
-        kibana_constraint=$(kibana_version)
-        if [ "$condition" != "null" ]; then
-            # FIXME
-            true
-        fi
+    # TODO What stack version to use (for agents)?
+    # else
+    #     kibana_constraint=$(kibana_version)
+    #     if [ "$condition" != "null" ]; then
+    #         # FIXME
+    #         true
+    #     fi
     fi
 
     echo "Update the Elastic stack"
@@ -79,8 +80,6 @@ with_kubernetes
 
 use_elastic_package
 
-prepare_stack
-
 echo "Checking python command..."
 if ! command -v python &> /dev/null ; then
     echo "⚠️  python is not installed"
@@ -89,6 +88,8 @@ if ! command -v python3 &> /dev/null ; then
     echo "⚠️  python3 is not installed"
 fi
 
+prepare_stack
+
 cd packages
 for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
     integration=$(basename ${it})

From 239a162786162096547d4ec50fc230e58fb80014 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 5 Oct 2023 21:47:53 +0200
Subject: [PATCH 028/139] Add missing popd

---
 .buildkite/scripts/publish_packages.sh      | 1 +
 .buildkite/scripts/run_integrations_sync.sh | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/.buildkite/scripts/publish_packages.sh b/.buildkite/scripts/publish_packages.sh
index 6f27f513102..6c97250662e 100644
--- a/.buildkite/scripts/publish_packages.sh
+++ b/.buildkite/scripts/publish_packages.sh
@@ -36,6 +36,7 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
 
     if is_already_published ${package_zip} ; then
         echo "Skipping. ${package_zip} already published"
+        popd > /dev/null
         continue
     fi
 
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 8bb5716661e..4414d5f41e1 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -83,9 +83,13 @@ use_elastic_package
 echo "Checking python command..."
 if ! command -v python &> /dev/null ; then
     echo "⚠️  python is not installed"
+else
+    echo "🐍 python is installed"
 fi
 if ! command -v python3 &> /dev/null ; then
     echo "⚠️  python3 is not installed"
+else
+    echo "🐍 python3 is installed"
 fi
 
 prepare_stack

From 85e394e8d878f290b8a8acd41c4dfe7c962ca97c Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 10 Oct 2023 17:16:39 +0200
Subject: [PATCH 029/139] Fix condition about publishing step

---
 .buildkite/scripts/publish_packages.sh | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/publish_packages.sh b/.buildkite/scripts/publish_packages.sh
index 6c97250662e..36b6fe5c1c9 100644
--- a/.buildkite/scripts/publish_packages.sh
+++ b/.buildkite/scripts/publish_packages.sh
@@ -8,11 +8,24 @@ if [ ${SKIP_PUBLISHING:-"false"} == "true" ] ; then
     exit 0
 fi
 
-if [ "${BUILDKITE_PULL_REQUEST}" == "false" ]; then
-    if [[ "${BUILDKITE_BRANCH}" == "main" || "${BUILDKITE_BRANCH}" =~ ^backport- ]]; then
-        echo "packageStoragePublish: not the main branch or a backport branch, nothing will be published"
-        exit 0
+skipPublishing() {
+    if [[ "${BUILDKITE_PULL_REQUEST}" == "true" ]]; then
+        return 0
     fi
+
+    if [[ "${BUILDKITE_BRANCH}" == "main" ]]; then
+        return 1
+    fi
+    if [[ "${BUILDKITE_BRANCH}" =~ ^backport- ]]; then
+        return 1
+    fi
+
+    return 0
+}
+
+if skipPublishing ; the
+    echo "packageStoragePublish: not the main branch or a backport branch, nothing will be published"
+    exit 0
 fi
 
 add_bin_path

From e4b343b1a930ab3e0814adb6379812708e5dd6a1 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 10 Oct 2023 17:38:36 +0200
Subject: [PATCH 030/139] Fix if syntax

---
 .buildkite/scripts/publish_packages.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/publish_packages.sh b/.buildkite/scripts/publish_packages.sh
index 36b6fe5c1c9..cb56b6f91b9 100644
--- a/.buildkite/scripts/publish_packages.sh
+++ b/.buildkite/scripts/publish_packages.sh
@@ -23,7 +23,7 @@ skipPublishing() {
     return 0
 }
 
-if skipPublishing ; the
+if skipPublishing ; then
     echo "packageStoragePublish: not the main branch or a backport branch, nothing will be published"
     exit 0
 fi

From 14e90f5054d0d87583e6cc1be42d6298f12972ce Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 11 Oct 2023 13:30:44 +0200
Subject: [PATCH 031/139] Added isPrAffected method

---
 .buildkite/scripts/run_integrations_sync.sh | 114 +++++++++++++++++---
 1 file changed, 102 insertions(+), 12 deletions(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 4414d5f41e1..c8dbc440104 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -4,21 +4,61 @@ source .buildkite/scripts/common.sh
 
 set -euo pipefail
 
+# default values
+STACK_VERSION=${STACK_VERSION:-""}
+FORCE_CHECK_ALL=${FORCE_CHECK_ALL:-"false"}
+SKIP_PUBLISHING=${SKIP_PUBLISHING:-"false"}
+
+
 if [ ! -d packages ]; then
     buildkite-agent annotate "Missing packages folder" --style "error"
     exit 1
 fi
 
-kibana_version() {
-    kibana_version=$(cat manifest.yml | yq ".conditions.kibana.version")
+kibana_version_manifest() {
+    local kibana_version=$(cat manifest.yml | yq ".conditions.kibana.version")
     if [ $kibana_version != "null" ]; then
-        echo $kibana_version
+        echo "${kibana_version}"
         return
     fi
 
     kibana_version=$(cat manifest.yml | yq ".conditions.\"kibana.version\"")
     if [ $kibana_version != "null" ]; then
-        echo $kibana_version
+        echo "${kibana_version}"
+        return
+    fi
+
+    echo "null"
+}
+
+is_unsupported_stack() {
+    if [ "${STACK_VERSION}" == "" ]; then
+        return 1
+    fi
+
+    local kibana_version=$(kibana_version_manifest)
+    if [ "${kibana_version}" == "null" ]; then
+        return 1
+    fi
+    if [[ ! ${kibana_version} =~ \^7\. && ${STACK_VERSION} =~ ^7\. ]]; then
+        return 0
+    fi
+    if [[ ! ${kibana_version} =~ \^8\. && ${STACK_VERSION} =~ ^8\. ]]; then
+        return 0
+    fi
+    return 1
+}
+
+oldest_supported_version() {
+    local kibana_version=$(cat manifest.yml | yq ".conditions.kibana.version")
+    if [ $kibana_version != "null" ]; then
+        python3 .buildkite/scripts/find_oldest_supported_version --manifest manifest.yml
+        return
+    fi
+
+    kibana_version=$(cat manifest.yml | yq ".conditions.\"kibana.version\"")
+    if [ $kibana_version != "null" ]; then
+        python3 .buildkite/scripts/find_oldest_supported_version --manifest manifest.yml
         return
     fi
 
@@ -31,13 +71,8 @@ prepare_stack() {
     local args="-v"
     if [ -n "${STACK_VERSION+x}" ]; then
         args="${args} --version ${STACK_VERSION}"
-    # TODO What stack version to use (for agents)?
+    # TODO What stack version to use (for agents) in serverless?
     # else
-    #     kibana_constraint=$(kibana_version)
-    #     if [ "$condition" != "null" ]; then
-    #         # FIXME
-    #         true
-    #     fi
     fi
 
     echo "Update the Elastic stack"
@@ -53,14 +88,69 @@ is_spec_3_0_0() {
     local pkg_spec=$(cat manifest.yml | yq '.format_version')
     local major_version=$(echo $pkg_spec | cut -d '.' -f 1)
 
-    if [ $major_version -ge 3 ]; then
+    if [ ${major_version} -ge 3 ]; then
         return 0
     fi
     return 1
 }
 
+get_from_changeset() {
+    if [ "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" != "false" ]; then
+        # pull request
+        echo "origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
+        return
+    fi
+    # main or backport branches
+    previous_commit=$(git rev-parse --verify FETCH_HEAD~1)
+    echo "${previous_commit}"
+}
+
+get_to_changeset() {
+    echo "${BUILDKITE_COMMIT}"
+}
+
 is_pr_affected() {
-    echo "1"
+    local integration="${1}"
+
+    if is_unsupported_stack ; then
+        echo "[${integrationName}] PR is not affected: unsupported stack (${STACK_VERSION})"
+        return 1
+    fi
+
+    if [[ ${FORCE_CHECK_ALL} == "true" ]];then
+        echo "[${integration}] PR is affected: \"force_check_all\" parameter enabled"
+        return 0
+    fi
+
+    # setting default values for a PR
+    # TODO: get previous built commit as in Jenkins (groovy)
+    # def from = env.CHANGE_TARGET?.trim() ? "origin/${env.CHANGE_TARGET}" : "${env.GIT_PREVIOUS_COMMIT?.trim() ? env.GIT_PREVIOUS_COMMIT : env.GIT_BASE_COMMIT}"
+    local from="$(get_from_changeset)"
+    local to="$(get_to_changeset)"
+
+    # TODO: If running for an integration branch (main, backport-*) check with
+    # GIT_PREVIOUS_SUCCESSFUL_COMMIT to check if the branch is still healthy.
+    # If this value is not available, check with last commit.
+    if [[ ${BUILDKITE_BRANCH} == "main" || ${BUILDKITE_BRANCH} =~ ^backport- ]]; then
+        echo "[${integration}] PR is affected: running on ${BUILDKITE_BRANCH} branch"
+        # TODO: get previous successful commit as in Jenkins (groovy)
+        # from = env.GIT_PREVIOUS_SUCCESSFUL_COMMIT?.trim() ? env.GIT_PREVIOUS_SUCCESSFUL_COMMIT : "origin/${env.BRANCH_NAME}^"
+        from="origin/${BUILDKITE_BRANCH}^"
+        to="origin/${BUILDKITE_BRANCH}"
+    fi
+
+    echo "[${integration}] git-diff: check non-package files"
+    if git diff --name-only $(git merge-base ${from} ${to}) ${to} | egrep '^(packages/|.github/CODEOWNERS)' ; then
+        echo "[${integration}] PR is affected: found non-package files"
+        return 0
+    fi
+    echo "[${integration}] git-diff: check package files"
+    if git diff --name-only $(git merge-base ${from} ${to}) ${to} | egrep '^packages/${integration}/' ; then
+        echo "[${integration}] PR is affected: found package files"
+        return 0
+    fi
+    echo "[${integration}] PR is not affected"
+    return 1
 }
 
 is_pr() {

From 7171b79645345b1a631e35e46a29ba42edc195ba Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 11 Oct 2023 13:36:50 +0200
Subject: [PATCH 032/139] Add check is_pr_affected

---
 .buildkite/scripts/run_integrations_sync.sh | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index c8dbc440104..ae6efe78556 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -8,6 +8,7 @@ set -euo pipefail
 STACK_VERSION=${STACK_VERSION:-""}
 FORCE_CHECK_ALL=${FORCE_CHECK_ALL:-"false"}
 SKIP_PUBLISHING=${SKIP_PUBLISHING:-"false"}
+SERVERLESS=${SERVERLESS:-"false"}
 
 
 if [ ! -d packages ]; then
@@ -113,7 +114,7 @@ is_pr_affected() {
     local integration="${1}"
 
     if is_unsupported_stack ; then
-        echo "[${integrationName}] PR is not affected: unsupported stack (${STACK_VERSION})"
+        echo "[${integration}] PR is not affected: unsupported stack (${STACK_VERSION})"
         return 1
     fi
 
@@ -191,9 +192,15 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
 
     pushd ${integration} 2> /dev/null
 
-    if [ ! is_spec_3_0_0 ]; then
-        echo "Not v3 spec version. Skipped"
-        continue
+    if ! is_pr_affected ${integration} ; then
+        echo "[${integration}] Skipped"
+    fi
+
+    if [[ ${SERVERLESS} == "true" ]] ; then
+        if ! is_spec_3_0_0 ]]; then
+            echo "Not v3 spec version. Skipped"
+            continue
+        fi
     fi
 
     echo "Check integration: ${integration}"

From 3910340d1eeed270033085d7ba15ac172db15d46 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 11 Oct 2023 13:44:44 +0200
Subject: [PATCH 033/139] Create kind cluster if needed

---
 .buildkite/scripts/common.sh                |  2 ++
 .buildkite/scripts/run_integrations_sync.sh | 19 +++++++++++++++++++
 2 files changed, 21 insertions(+)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index cb84408ed12..0618a74cdee 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -110,11 +110,13 @@ with_docker_compose() {
 
 with_kubernetes() {
     create_bin_folder
+    echo "Install kind"
     retry 5 curl -sSLo ${BIN_FOLDER}/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
     chmod +x ${BIN_FOLDER}/kind
     kind version
     which kind
 
+    echo "Install kubectl"
     retry 5 curl -sSLo ${BIN_FOLDER}/kubectl "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
     chmod +x ${BIN_FOLDER}/kubectl
     kubectl version --client
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index ae6efe78556..07abf4c76e0 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -161,6 +161,21 @@ is_pr() {
     return 1
 }
 
+kubernetes_service_deployer_used() {
+    echo "Check if Kubernetes service deployer is used"
+    find . -type d | egrep '_dev/deploy/k8s$'
+}
+
+create_kind_cluster() {
+    echo "Create kind cluster"
+    kind create cluster --config ${WORKSPACE}/kind-config.yaml --image kindest/node:${K8S_VERSION}
+}
+
+
+delete_kind_cluster() {
+    echo "Delete kind cluster"
+    kind delete cluster || true
+}
 
 add_bin_path
 
@@ -203,6 +218,10 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
         fi
     fi
 
+    if kubernetes_service_deployer_used ; then
+        create_kind_cluster
+    fi
+
     echo "Check integration: ${integration}"
     ${ELASTIC_PACKAGE_BIN} check -v
 

From 0773224e067c53465c0a95be47f2cba3132cf550 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 11 Oct 2023 13:48:22 +0200
Subject: [PATCH 034/139] Change STACK_VERSION condition

---
 .buildkite/scripts/run_integrations_sync.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 07abf4c76e0..4f5c6e3909e 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -70,7 +70,7 @@ prepare_stack() {
     echo "Prepare stack"
 
     local args="-v"
-    if [ -n "${STACK_VERSION+x}" ]; then
+    if [ -n "${STACK_VERSION}" ]; then
         args="${args} --version ${STACK_VERSION}"
     # TODO What stack version to use (for agents) in serverless?
     # else

From faa583a0c99a50b06f404cb889622cd7e7ceebc7 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 11 Oct 2023 14:23:48 +0200
Subject: [PATCH 035/139] Report failure without exiting

---
 .buildkite/scripts/publish_packages.sh | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/publish_packages.sh b/.buildkite/scripts/publish_packages.sh
index cb56b6f91b9..aac8b43293d 100644
--- a/.buildkite/scripts/publish_packages.sh
+++ b/.buildkite/scripts/publish_packages.sh
@@ -23,6 +23,21 @@ skipPublishing() {
     return 0
 }
 
+check_and_build_package() {
+    ${ELASTIC_PACKAGE_BIN} check
+    ${ELASTIC_PACKAGE_BIN} build --zip
+}
+
+report_build_failure() {
+    local integration="${1}"
+    echo "Build package ${integration}failed"
+
+    # if running in Buildkite , add an annotation
+    if [ -n "$BUILDKITE_BRANCH" ]; then
+        buildkite-agent annotate "Build package ${integration} failed" --style "warning"
+    fi
+}
+
 if skipPublishing ; then
     echo "packageStoragePublish: not the main branch or a backport branch, nothing will be published"
     exit 0
@@ -54,8 +69,7 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
     fi
 
     echo "Build integration as zip: ${integration}"
-    ${ELASTIC_PACKAGE_BIN} check
-    ${ELASTIC_PACKAGE_BIN} build --zip
+    check_and_build_package || report_build_failure ${integration}
     popd > /dev/null
 
     unpublished="true"

From bccd11e1d519115f3a5bbbaac582f1e8b212e9a1 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 11 Oct 2023 14:57:54 +0200
Subject: [PATCH 036/139] Scaffolding for signing and publishing

---
 .buildkite/hooks/pre-command                  |  35 +++++
 .buildkite/hooks/pre-exit                     |  10 ++
 .buildkite/scripts/common.sh                  |   8 +
 .buildkite/scripts/publish_packages.sh        |  87 ++++++++---
 .../scripts/triggerJenkinsJob/check_job.sh    |  16 ++
 .buildkite/scripts/triggerJenkinsJob/go.mod   |   7 +
 .buildkite/scripts/triggerJenkinsJob/go.sum   |  18 +++
 .../triggerJenkinsJob/jenkins/jenkins.go      | 138 +++++++++++++++++
 .../triggerJenkinsJob/jenkins/retry.go        |  48 ++++++
 .buildkite/scripts/triggerJenkinsJob/main.go  | 119 ++++++++++++++
 .../scripts/triggerJenkinsJob/main.go.backup  | 146 ++++++++++++++++++
 .../scripts/triggerJenkinsJob/run_jenkins.sh  |  18 +++
 .../scripts/triggerJenkinsJob/trigger_job.sh  |  20 +++
 13 files changed, 645 insertions(+), 25 deletions(-)
 create mode 100755 .buildkite/scripts/triggerJenkinsJob/check_job.sh
 create mode 100644 .buildkite/scripts/triggerJenkinsJob/go.mod
 create mode 100644 .buildkite/scripts/triggerJenkinsJob/go.sum
 create mode 100644 .buildkite/scripts/triggerJenkinsJob/jenkins/jenkins.go
 create mode 100644 .buildkite/scripts/triggerJenkinsJob/jenkins/retry.go
 create mode 100644 .buildkite/scripts/triggerJenkinsJob/main.go
 create mode 100644 .buildkite/scripts/triggerJenkinsJob/main.go.backup
 create mode 100755 .buildkite/scripts/triggerJenkinsJob/run_jenkins.sh
 create mode 100755 .buildkite/scripts/triggerJenkinsJob/trigger_job.sh

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index bf71684a0de..65c54c2c64c 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -8,7 +8,42 @@ export BASE_DIR=$(pwd)
 export GO_VERSION=$(cat .go-version)
 export startTimeInMillis="$(date +%s%3N)"
 
+export REPO_NAME=$(repo_name "${BUILDKITE_REPO}")
+export TMP_FOLDER_TEMPLATE_BASE="tmp.${REPO_NAME}"
+export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
+
+JENKINS_API_TOKEN_PATH=kv/ci-shared/platform-ingest/jenkins_api_tokens
+SIGNING_PACKAGES_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/signing_packages_gcs_artifacts_credentials
+PACKAGE_UPLOADER_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/package_storage_uploader
+
 if [ -n "${ELASTIC_PACKAGE_LINKS_FILE_PATH+x}" ]; then
     # first upload pipeline does not have the environment variables defined in the YAML
     export ELASTIC_PACKAGE_LINKS_FILE_PATH=${BASE_DIR}/${ELASTIC_PACKAGE_LINKS_FILE_PATH}
 fi
+
+# TODO: add schedule-daily too ?
+if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
+    if [ ${BUILDKITE_PIPELINE_STEP} == "publish-packages" ]; then
+        export JENKINS_USERNAME_SECRET=$(retry 5 vault kv get -field username ${JENKINS_API_TOKEN_PATH})
+        export JENKINS_HOST_SECRET=$(retry 5 vault kv get -field internal_ci_host ${JENKINS_API_TOKEN_PATH})
+        export JENKINS_TOKEN=$(retry 5 vault kv get -field internal_ci ${JENKINS_API_TOKEN_PATH})
+
+        # signing job
+        export SIGNING_PACKAGES_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field value ${SIGNING_PACKAGES_GCS_CREDENTIALS_PATH})
+
+        # publishing job
+        export PACKAGE_UPLOADER_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field value ${PACKAGE_UPLOADER_GCS_CREDENTIALS_PATH})
+    fi
+fi
+
+# TODO: add schedule-daily too
+if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
+    if [ ${BUILDKITE_PIPELINE_STEP} == "test-integrations-sync" ]; then
+        export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+        export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+
+        # Environment variables required by the service deployer
+        export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
+        export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
+    fi
+fi
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index baacc7e5c38..aae5c01412f 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -6,3 +6,13 @@ set -euo pipefail
 
 unset_secrets
 cleanup
+
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
+    if [[ "$BUILDKITE_STEP_KEY" == "test-integrations-sync" ]]; then
+       unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
+       unset ELASTIC_PACKAGE_AWS_SECRET_KEY
+       unset AWS_ACCESS_KEY_ID
+       unset AWS_SECRET_ACCESS_KEY
+    fi
+fi
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 0618a74cdee..89e22d8590c 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -193,3 +193,11 @@ is_already_published() {
     echo "- Not published ${packageZip}"
     return 1
 }
+
+repo_name() {
+    # Example of URL: git@github.com:acme-inc/my-project.git
+    local repoUrl=$1
+
+    orgAndRepo=$(echo $repoUrl | cut -d':' -f 2)
+    echo "$(basename ${orgAndRepo} .git)"
+}
diff --git a/.buildkite/scripts/publish_packages.sh b/.buildkite/scripts/publish_packages.sh
index aac8b43293d..0cc4ee58826 100644
--- a/.buildkite/scripts/publish_packages.sh
+++ b/.buildkite/scripts/publish_packages.sh
@@ -8,6 +8,19 @@ if [ ${SKIP_PUBLISHING:-"false"} == "true" ] ; then
     exit 0
 fi
 
+export BUILD_TAG="buildkite-${BUILDKITE_PIPELINE_SLUG}-${BUILDKITE_BUILD_NUMBER}"
+export REPO_BUILD_TAG="${REPO_NAME}/${BUILD_TAG}"
+
+JENKINS_TRIGGER_PATH=".buildkite/scripts/triggerJenkinsJob"
+GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
+
+# signing
+INFRA_SIGNING_BUCKET_NAME='internal-ci-artifacts'
+INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER="${REPO_BUILD_TAG}/signed-artifacts"
+INFRA_SIGNING_BUCKET_ARTIFACTS_PATH="gs://${INFRA_SIGNING_BUCKET_NAME}/${REPO_BUILD_TAG}"
+INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH="gs://${INFRA_SIGNING_BUCKET_NAME}/${INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER}"
+
+
 skipPublishing() {
     if [[ "${BUILDKITE_PULL_REQUEST}" == "true" ]]; then
         return 0
@@ -38,11 +51,56 @@ report_build_failure() {
     fi
 }
 
+build_packages() {
+    pushd packages > /dev/null
+
+    for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
+        integration=$(basename ${it})
+        echo "Package ${integration}: check"
+
+        pushd ${integration} > /dev/null
+
+        version=$(cat manifest.yml | yq .version)
+        name=$(cat manifest.yml | yq .name)
+
+        package_zip="${name}-${version}.zip"
+
+        if is_already_published ${package_zip} ; then
+            echo "Skipping. ${package_zip} already published"
+            popd > /dev/null
+            continue
+        fi
+
+        echo "Build integration as zip: ${integration}"
+        check_and_build_package || report_build_failure ${integration}
+        popd > /dev/null
+
+        unpublished="true"
+    done
+    popd > /dev/null
+}
+
+sign_packages() {
+    echo "Signing packages"
+    # TODO require signing: to be based on elastic-package
+}
+
+publish_packages() {
+    echo "Publishing packages"
+    # TODO require publishing: to be based on elastic-package
+}
+
 if skipPublishing ; then
     echo "packageStoragePublish: not the main branch or a backport branch, nothing will be published"
     exit 0
 fi
 
+echo "Checking gsutil command..."
+if ! command -v gsutil &> /dev/null ; then
+    echo "⚠️  gsutil is not installed"
+    exit 1
+fi
+
 add_bin_path
 with_go
 with_yq
@@ -50,36 +108,15 @@ use_elastic_package
 
 unpublished="false"
 
-cd packages
-for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
-    integration=$(basename ${it})
-    echo "Package ${integration}: check"
-
-    pushd ${integration} > /dev/null
+# build packages
 
-    version=$(cat manifest.yml | yq .version)
-    name=$(cat manifest.yml | yq .name)
+build_packages
 
-    package_zip="${name}-${version}.zip"
-
-    if is_already_published ${package_zip} ; then
-        echo "Skipping. ${package_zip} already published"
-        popd > /dev/null
-        continue
-    fi
-
-    echo "Build integration as zip: ${integration}"
-    check_and_build_package || report_build_failure ${integration}
-    popd > /dev/null
-
-    unpublished="true"
-done
 
 if [ "${unpublished}" == "false" ]; then
     echo "All packages are in sync"
     exit 0
 fi
 
-
-# TODO require signing: to be based on elastic-package
-# TODO require publishing: to be based on elastic-package
+sign_packages
+publish_packages
diff --git a/.buildkite/scripts/triggerJenkinsJob/check_job.sh b/.buildkite/scripts/triggerJenkinsJob/check_job.sh
new file mode 100755
index 00000000000..99fcf1ab1b6
--- /dev/null
+++ b/.buildkite/scripts/triggerJenkinsJob/check_job.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+export JENKINS_TOKEN=11c7b833b6ad3ed0328b80a0317774760f
+export JENKINS_HOST_SECRET="http://localhost:8081"
+export JENKINS_USERNAME_SECRET="admin"
+
+CRUMB=$(curl -u "admin:admin" \
+    -s 'http://localhost:8081/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
+
+curl -s \
+    -H "$CRUMB" \
+    -XPOST \
+    -u "${JENKINS_USERNAME_SECRET}:${JENKINS_TOKEN}" \
+    "${JENKINS_HOST_SECRET}/job/test-job/api/json" | jq -r '.inQueue'
+
+exit 0
diff --git a/.buildkite/scripts/triggerJenkinsJob/go.mod b/.buildkite/scripts/triggerJenkinsJob/go.mod
new file mode 100644
index 00000000000..4690be7da13
--- /dev/null
+++ b/.buildkite/scripts/triggerJenkinsJob/go.mod
@@ -0,0 +1,7 @@
+module github.com/elastic/trigger-jenkins-buildkite-plugin
+
+go 1.21.0
+
+require github.com/bndr/gojenkins v1.1.0
+
+require golang.org/x/net v0.7.0 // indirect
diff --git a/.buildkite/scripts/triggerJenkinsJob/go.sum b/.buildkite/scripts/triggerJenkinsJob/go.sum
new file mode 100644
index 00000000000..43a3692ff00
--- /dev/null
+++ b/.buildkite/scripts/triggerJenkinsJob/go.sum
@@ -0,0 +1,18 @@
+github.com/bndr/gojenkins v1.1.0 h1:TWyJI6ST1qDAfH33DQb3G4mD8KkrBfyfSUoZBHQAvPI=
+github.com/bndr/gojenkins v1.1.0/go.mod h1:QeskxN9F/Csz0XV/01IC8y37CapKKWvOHa0UHLLX1fM=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/.buildkite/scripts/triggerJenkinsJob/jenkins/jenkins.go b/.buildkite/scripts/triggerJenkinsJob/jenkins/jenkins.go
new file mode 100644
index 00000000000..7a8f32cdd82
--- /dev/null
+++ b/.buildkite/scripts/triggerJenkinsJob/jenkins/jenkins.go
@@ -0,0 +1,138 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package jenkins
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/bndr/gojenkins"
+)
+
+type JenkinsClient struct {
+	client *gojenkins.Jenkins
+}
+
+type Options struct {
+	WaitingTime    time.Duration
+	MaxWaitingTime time.Duration
+	GrowthFactor   float64
+	Retries        int
+}
+
+func NewJenkinsClient(ctx context.Context, host, user, token string) (*JenkinsClient, error) {
+	jenkins, err := gojenkins.CreateJenkins(nil, host, user, token).Init(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("client coult not be created: %w", err)
+	}
+
+	return &JenkinsClient{
+		client: jenkins,
+	}, nil
+}
+
+func (j *JenkinsClient) RunJob(ctx context.Context, jobName string, async bool, params map[string]string, opts Options) error {
+	log.Printf("Building job %s", jobName)
+	var queueId int64
+
+	r := retry(func(ctx context.Context) error {
+		var err error
+		queueId, err = j.client.BuildJob(ctx, jobName, params)
+		if err != nil {
+			return fmt.Errorf("error running job %s: %w", jobName, err)
+		}
+
+		if queueId != 0 {
+			return nil
+		}
+		return fmt.Errorf("already running %s?", jobName)
+
+	}, opts.Retries, opts.GrowthFactor, opts.WaitingTime, opts.MaxWaitingTime)
+
+	if err := r(ctx); err != nil {
+		return err
+	}
+
+	build, err := j.getBuildFromJobAndQueueID(ctx, jobName, queueId)
+	if err != nil {
+		return err
+	}
+	log.Printf("Job triggered %s/%d\n", jobName, build.GetBuildNumber())
+
+	if async {
+		return nil
+	}
+
+	log.Printf("Waiting to be finished %s\n", build.GetUrl())
+	err = j.waitForBuildFinished(ctx, build)
+	if err != nil {
+		return fmt.Errorf("not finished job %s/%d: %w", jobName, build.GetBuildNumber(), err)
+	}
+
+	log.Printf("Build %s finished with result: %s\n", build.GetUrl(), build.GetResult())
+
+	if build.GetResult() != gojenkins.STATUS_SUCCESS {
+		return fmt.Errorf("build %s finished with result %s", build.GetUrl(), build.GetResult())
+	}
+	return nil
+}
+
+func (j *JenkinsClient) getBuildFromJobAndQueueID(ctx context.Context, jobName string, queueId int64) (*gojenkins.Build, error) {
+	job, err := j.client.GetJob(ctx, jobName)
+	if err != nil {
+		return nil, fmt.Errorf("not able to get job %s: %w", jobName, err)
+	}
+
+	build, err := j.getBuildFromQueueID(ctx, job, queueId)
+	if err != nil {
+		return nil, fmt.Errorf("not able to get build from %s: %w", jobName, err)
+	}
+	return build, nil
+}
+
+// based on https://github.com/bndr/gojenkins/blob/master/jenkins.go#L282
+func (j *JenkinsClient) getBuildFromQueueID(ctx context.Context, job *gojenkins.Job, queueid int64) (*gojenkins.Build, error) {
+	task, err := j.client.GetQueueItem(ctx, queueid)
+	if err != nil {
+		return nil, err
+	}
+	// Jenkins queue API has about 4.7second quiet period
+	for task.Raw.Executable.Number == 0 {
+		select {
+		case <-time.After(1000 * time.Millisecond):
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+		_, err = task.Poll(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	build, err := job.GetBuild(ctx, task.Raw.Executable.Number)
+	if err != nil {
+		return nil, fmt.Errorf("not able to retrieve build %s", task.Raw.Executable.Number, err)
+	}
+	return build, nil
+}
+
+func (j *JenkinsClient) waitForBuildFinished(ctx context.Context, build *gojenkins.Build) error {
+	const waitingPeriod = 10000 * time.Millisecond
+	for build.IsRunning(ctx) {
+		log.Printf("Build still running, waiting for %s...", waitingPeriod)
+		select {
+		case <-time.After(waitingPeriod):
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+		_, err := build.Poll(ctx)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
diff --git a/.buildkite/scripts/triggerJenkinsJob/jenkins/retry.go b/.buildkite/scripts/triggerJenkinsJob/jenkins/retry.go
new file mode 100644
index 00000000000..7be270c93db
--- /dev/null
+++ b/.buildkite/scripts/triggerJenkinsJob/jenkins/retry.go
@@ -0,0 +1,48 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package jenkins
+
+import (
+	"context"
+	"log"
+	"math"
+	"time"
+)
+
+type retryableFunction func(context.Context) error
+
+func minDuration(a, b time.Duration) time.Duration {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+func retry(f retryableFunction, retries int, growthFactor float64, delay, maxDelay time.Duration) retryableFunction {
+	return func(ctx context.Context) error {
+		delaySeconds := delay.Seconds()
+		for r := 0; ; r++ {
+			err := f(ctx)
+			if err == nil || r >= retries {
+				// Return when there is no error or the maximum amount
+				// of retries is reached.
+				return err
+			}
+
+			waitingTimeSeconds := math.Pow(growthFactor, float64(r)) * delaySeconds
+			waitingTime := time.Duration(waitingTimeSeconds) * time.Second
+			waitingTime = minDuration(waitingTime, maxDelay)
+
+			log.Printf("Function failed, retrying in %v -> %.2f", waitingTime, waitingTimeSeconds)
+
+			select {
+			case <-time.After(waitingTime):
+			case <-ctx.Done():
+				return ctx.Err()
+			}
+		}
+		return nil
+	}
+}
diff --git a/.buildkite/scripts/triggerJenkinsJob/main.go b/.buildkite/scripts/triggerJenkinsJob/main.go
new file mode 100644
index 00000000000..f85866deccd
--- /dev/null
+++ b/.buildkite/scripts/triggerJenkinsJob/main.go
@@ -0,0 +1,119 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/elastic/trigger-jenkins-buildkite-plugin/jenkins"
+)
+
+const (
+	publishingRemoteJob = "package_storage/job/publishing-job-remote"
+	signingJob          = "elastic+unified-release+master+sign-artifacts-with-gpg"
+
+	publishJobKey = "publish"
+	signJobKey    = "sign"
+)
+
+var allowedJenkinsJobs = map[string]string{
+	publishJobKey: publishingRemoteJob,
+	signJobKey:    signingJob,
+}
+
+var (
+	jenkinsHost  = os.Getenv("JENKINS_HOST_SECRET")
+	jenkinsUser  = os.Getenv("JENKINS_USERNAME_SECRET")
+	jenkinsToken = os.Getenv("JENKINS_TOKEN")
+)
+
+func jenkinsJobOptions() []string {
+	keys := make([]string, 0, len(allowedJenkinsJobs))
+	for k := range allowedJenkinsJobs {
+		keys = append(keys, k)
+	}
+	return keys
+}
+
+func main() {
+	jenkinsJob := flag.String("jenkins-job", "", fmt.Sprintf("Jenkins job to trigger. Allowed values: %s", strings.Join(jenkinsJobOptions(), " ,")))
+	waitingTime := flag.Duration("waiting-time", 5*time.Second, fmt.Sprintf("Waiting period between each retry"))
+	growthFactor := flag.Float64("growth-factor", 1.25, fmt.Sprintf("Growth-Factor used for exponential backoff delays"))
+	retries := flag.Int("retries", 20, fmt.Sprintf("Number of retries to trigger the job"))
+	maxWaitingTime := flag.Duration("max-waiting-time", 60*time.Minute, fmt.Sprintf("Maximum waiting time per each retry"))
+
+	folderPath := flag.String("folder", "", "Path to artifacts folder")
+	zipPackagePath := flag.String("package", "", "Path to zip package file (*.zip)")
+	sigPackagePath := flag.String("signature", "", "Path to the signature file of the package file (*.zip.sig)")
+	async := flag.Bool("async", false, "Run async the Jenkins job")
+	flag.Parse()
+
+	if _, ok := allowedJenkinsJobs[*jenkinsJob]; !ok {
+		log.Fatal("Invalid jenkins job")
+	}
+
+	log.Printf("Triggering job: %s", allowedJenkinsJobs[*jenkinsJob])
+
+	ctx := context.Background()
+	client, err := jenkins.NewJenkinsClient(ctx, jenkinsHost, jenkinsUser, jenkinsToken)
+	if err != nil {
+		log.Fatalf("error creating jenkins client")
+	}
+
+	opts := jenkins.Options{
+		WaitingTime:    *waitingTime,
+		Retries:        *retries,
+		GrowthFactor:   *growthFactor,
+		MaxWaitingTime: *maxWaitingTime,
+	}
+
+	switch *jenkinsJob {
+	case publishJobKey:
+		err = runPublishingRemoteJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], *zipPackagePath, *sigPackagePath, opts)
+	case signJobKey:
+		err = runSignPackageJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], *folderPath, opts)
+	default:
+		log.Fatal("unsupported jenkins job")
+	}
+
+	if err != nil {
+		log.Fatalf("Error: %s", err)
+	}
+}
+
+func runSignPackageJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, folderPath string, opts jenkins.Options) error {
+	if folderPath == "" {
+		return fmt.Errorf("missing parameter --gcs_input_path for")
+	}
+	params := map[string]string{
+		"gcs_input_path": folderPath,
+	}
+
+	return client.RunJob(ctx, jobName, async, params, opts)
+}
+
+func runPublishingRemoteJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, packagePath, signaturePath string, opts jenkins.Options) error {
+	if packagePath == "" {
+		return fmt.Errorf("missing parameter --gs_package_build_zip_path")
+	}
+	if signaturePath == "" {
+		return fmt.Errorf("missing parameter --gs_package_signature_path")
+	}
+
+	// Run the job with some parameters
+	params := map[string]string{
+		"dry_run":                   "true",
+		"gs_package_build_zip_path": packagePath,
+		"gs_package_signature_path": signaturePath,
+	}
+
+	return client.RunJob(ctx, jobName, async, params, opts)
+}
diff --git a/.buildkite/scripts/triggerJenkinsJob/main.go.backup b/.buildkite/scripts/triggerJenkinsJob/main.go.backup
new file mode 100644
index 00000000000..dc21d099eb6
--- /dev/null
+++ b/.buildkite/scripts/triggerJenkinsJob/main.go.backup
@@ -0,0 +1,146 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"math/rand"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/elastic/trigger-jenkins-buildkite-plugin/jenkins"
+)
+
+const (
+	publishingRemoteJob = "package_storage/job/publishing-job-remote"
+	signingJob          = "elastic+unified-release+master+sign-artifacts-with-gpg"
+	testJob             = "test-job"
+
+	publishJobKey = "publish"
+	signJobKey    = "sign"
+	testJobKey    = "test"
+)
+
+var allowedJenkinsJobs = map[string]string{
+	publishJobKey: publishingRemoteJob,
+	signJobKey:    signingJob,
+	testJobKey:    testJob,
+}
+
+var (
+	jenkinsHost  = os.Getenv("JENKINS_HOST_SECRET")
+	jenkinsUser  = os.Getenv("JENKINS_USERNAME_SECRET")
+	jenkinsToken = os.Getenv("JENKINS_TOKEN")
+)
+
+func jenkinsJobOptions() []string {
+	keys := make([]string, 0, len(allowedJenkinsJobs))
+	for k := range allowedJenkinsJobs {
+		keys = append(keys, k)
+	}
+	return keys
+}
+
+func main() {
+	jenkinsJob := flag.String("jenkins-job", "", fmt.Sprintf("Jenkins job to trigger. Allowed values: %s", strings.Join(jenkinsJobOptions(), " ,")))
+	waitingTime := flag.Duration("waiting-time", 30*time.Second, fmt.Sprintf("Waiting period between each retry"))
+	retries := flag.Int("retries", 10, fmt.Sprintf("Number of retries to trigger the job"))
+	folderPath := flag.String("folder", "", "Path to artifacts folder")
+	zipPackagePath := flag.String("package", "", "Path to zip package file (*.zip)")
+	sigPackagePath := flag.String("signature", "", "Path to the signature file of the package file (*.zip.sig)")
+	async := flag.Bool("async", false, "Run async the Jenkins job")
+	flag.Parse()
+
+	if _, ok := allowedJenkinsJobs[*jenkinsJob]; !ok {
+		log.Fatal("Invalid jenkins job")
+	}
+
+	log.Printf("Triggering job: %s", allowedJenkinsJobs[*jenkinsJob])
+
+	ctx := context.Background()
+	client, err := jenkins.NewJenkinsClient(ctx, jenkinsHost, jenkinsUser, jenkinsToken)
+	if err != nil {
+		log.Fatalf("error creating jenkins client")
+	}
+
+	opts := jenkins.Options{
+		WaitingTime: *waitingTime,
+		Retries:     *retries,
+	}
+
+	switch *jenkinsJob {
+	case publishJobKey:
+		err = runPublishingRemoteJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], *zipPackagePath, *sigPackagePath, opts)
+	case signJobKey:
+		err = runSignPackageJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], *folderPath, opts)
+	case testJobKey:
+		err = runTestJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], "aabbccddeeff", opts)
+		// err = runTestJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], StringWithCharset(20, charset), opts)
+	default:
+		log.Fatal("unsupported jenkins job")
+	}
+
+	if err != nil {
+		log.Fatalf("Error: %s", err)
+	}
+}
+
+const charset = "abcdefghijklmnopqrstuvwxyz" +
+	"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+
+var seededRand *rand.Rand = rand.New(
+	rand.NewSource(time.Now().UnixNano()))
+
+func StringWithCharset(length int, charset string) string {
+	b := make([]byte, length)
+	for i := range b {
+		b[i] = charset[seededRand.Intn(len(charset))]
+	}
+	return string(b)
+}
+
+func runSignPackageJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, folderPath string, opts jenkins.Options) error {
+	if folderPath == "" {
+		return fmt.Errorf("missing parameter --gcs_input_path for")
+	}
+	params := map[string]string{
+		"gcs_input_path": folderPath,
+	}
+
+	return client.RunJob(ctx, jobName, async, params, opts)
+}
+
+func runPublishingRemoteJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, packagePath, signaturePath string, opts jenkins.Options) error {
+	if packagePath == "" {
+		return fmt.Errorf("missing parameter --gs_package_build_zip_path")
+	}
+	if signaturePath == "" {
+		return fmt.Errorf("missing parameter --gs_package_signature_path")
+	}
+
+	// Run the job with some parameters
+	params := map[string]string{
+		"dry_run":                   "true",
+		"gs_package_build_zip_path": packagePath,
+		"gs_package_signature_path": signaturePath,
+	}
+
+	return client.RunJob(ctx, jobName, async, params, opts)
+}
+
+func runTestJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, test string, opts jenkins.Options) error {
+
+	// Run the job with some parameters
+	params := map[string]string{
+		"dry_run": "true",
+		"test":    test,
+	}
+
+	return client.RunJob(ctx, jobName, async, params, opts)
+}
diff --git a/.buildkite/scripts/triggerJenkinsJob/run_jenkins.sh b/.buildkite/scripts/triggerJenkinsJob/run_jenkins.sh
new file mode 100755
index 00000000000..3cc7d085f43
--- /dev/null
+++ b/.buildkite/scripts/triggerJenkinsJob/run_jenkins.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+    # -it \
+    # --entrypoint /bin/bash \
+
+if ! docker volume ls | grep -q  jenkins-home ; then
+    echo "Create volume"
+    docker volume create jenkins-home
+fi
+
+docker run \
+    --name jenkins \
+    --rm  \
+    --user 1000:1000 \
+    -p 8081:8080 \
+    -p 50000:50000 \
+    -v jenkins-home:/var/jenkins_home \
+    jenkins/jenkins:2.346.3
diff --git a/.buildkite/scripts/triggerJenkinsJob/trigger_job.sh b/.buildkite/scripts/triggerJenkinsJob/trigger_job.sh
new file mode 100755
index 00000000000..40e1cd5696c
--- /dev/null
+++ b/.buildkite/scripts/triggerJenkinsJob/trigger_job.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+export JENKINS_TOKEN=11c7b833b6ad3ed0328b80a0317774760f
+export JENKINS_HOST_SECRET="http://localhost:8081"
+export JENKINS_USERNAME_SECRET="admin"
+
+go run main.go -jenkins-job test -retries 20 -waiting-time 5s -growth-factor 1.25 -max-waiting-time 60m
+
+exit 0
+
+CRUMB=$(curl -u "admin:admin" \
+    -s 'http://localhost:8081/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
+
+curl -v \
+    -H "$CRUMB" \
+    -XPOST \
+    -u "${JENKINS_USERNAME_SECRET}:${JENKINS_TOKEN}" \
+    "${JENKINS_HOST_SECRET}/job/test-job/buildWithParameters?dry_run=true&test=aaab"
+
+exit 0

From 2ef49f2b3ffe2d6f8896af96f53b4246a4a7091a Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 11 Oct 2023 17:33:40 +0200
Subject: [PATCH 037/139] Collapse some outputs

---
 .buildkite/hooks/pre-command                |  7 +++++
 .buildkite/hooks/pre-exit                   | 27 +++++++++++------
 .buildkite/scripts/common.sh                | 10 +++----
 .buildkite/scripts/run_integrations_sync.sh | 32 +++++++--------------
 4 files changed, 41 insertions(+), 35 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 65c54c2c64c..daad09db884 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -16,6 +16,9 @@ JENKINS_API_TOKEN_PATH=kv/ci-shared/platform-ingest/jenkins_api_tokens
 SIGNING_PACKAGES_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/signing_packages_gcs_artifacts_credentials
 PACKAGE_UPLOADER_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/package_storage_uploader
 
+EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa
+EC_DATA_PATH=secret/ci/elastic-integrations/ec_data
+
 if [ -n "${ELASTIC_PACKAGE_LINKS_FILE_PATH+x}" ]; then
     # first upload pipeline does not have the environment variables defined in the YAML
     export ELASTIC_PACKAGE_LINKS_FILE_PATH=${BASE_DIR}/${ELASTIC_PACKAGE_LINKS_FILE_PATH}
@@ -45,5 +48,9 @@ if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
         # Environment variables required by the service deployer
         export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
         export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
+
+        export EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey ${EC_TOKEN_PATH})
+        export EC_HOST_SECRET=$(retry 5 vault kv get -field url ${EC_TOKEN_PATH})
+        export EC_REGION_SECRET=$(retry 5 vault read -field region_qa) ${EC_DATA_PATH}
     fi
 fi
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index aae5c01412f..a6686ccfb95 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -4,15 +4,26 @@ source .buildkite/scripts/common.sh
 
 set -euo pipefail
 
-unset_secrets
-cleanup
-
-
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
     if [[ "$BUILDKITE_STEP_KEY" == "test-integrations-sync" ]]; then
-       unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
-       unset ELASTIC_PACKAGE_AWS_SECRET_KEY
-       unset AWS_ACCESS_KEY_ID
-       unset AWS_SECRET_ACCESS_KEY
+        unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
+        unset ELASTIC_PACKAGE_AWS_SECRET_KEY
+        unset AWS_ACCESS_KEY_ID
+        unset AWS_SECRET_ACCESS_KEY
+
+        if [ -f ${ELASTIC_PACKAGE_BIN} ]; then
+            export EC_API_KEY=${EC_API_KEY_SECRET}
+            export EC_HOST=${EC_HOST_SECRET}
+
+            ${ELASTIC_PACKAGE_BIN} stack down -v
+
+            unset EC_API_KEY
+            unset EC_HOST
+        fi
     fi
 fi
+
+unset_secrets
+cleanup
+
+
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 89e22d8590c..ce1ec95b2bb 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -4,8 +4,6 @@ set -euo pipefail
 
 WORKSPACE="$(pwd)"
 BIN_FOLDER="${WORKSPACE}/bin"
-REPO="integrations"
-TMP_FOLDER_TEMPLATE_BASE="tmp.${REPO}"
 platform_type="$(uname)"
 hw_type="$(uname -m)"
 export ELASTIC_PACKAGE_BIN=${WORKSPACE}/build/elastic-package
@@ -72,7 +70,7 @@ add_bin_path() {
 
 with_go() {
   create_bin_folder
-  echo "Setting up the Go environment..."
+  echo "--- Setting up the Go environment..."
   check_platform_architeture
   local platform_type_lowercase="${platform_type,,}"
   echo " GVM ${SETUP_GVM_VERSION} (platform ${platform_type_lowercase} arch ${arch_type}"
@@ -110,13 +108,13 @@ with_docker_compose() {
 
 with_kubernetes() {
     create_bin_folder
-    echo "Install kind"
+    echo "--- Install kind"
     retry 5 curl -sSLo ${BIN_FOLDER}/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
     chmod +x ${BIN_FOLDER}/kind
     kind version
     which kind
 
-    echo "Install kubectl"
+    echo "--- Install kubectl"
     retry 5 curl -sSLo ${BIN_FOLDER}/kubectl "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
     chmod +x ${BIN_FOLDER}/kubectl
     kubectl version --client
@@ -178,7 +176,7 @@ with_yq() {
 }
 
 use_elastic_package() {
-    echo "Installing elastic-package"
+    echo "--- Installing elastic-package"
     mkdir -p build
     go build -o ${ELASTIC_PACKAGE_BIN} github.com/elastic/elastic-package
 }
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 4f5c6e3909e..8f5db7612f1 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -9,6 +9,7 @@ STACK_VERSION=${STACK_VERSION:-""}
 FORCE_CHECK_ALL=${FORCE_CHECK_ALL:-"false"}
 SKIP_PUBLISHING=${SKIP_PUBLISHING:-"false"}
 SERVERLESS=${SERVERLESS:-"false"}
+SERVERLESS_PROJECT=${SERVERLESS_PROJECT:-"observability"}
 
 
 if [ ! -d packages ]; then
@@ -66,8 +67,8 @@ oldest_supported_version() {
     echo "null"
 }
 
-prepare_stack() {
-    echo "Prepare stack"
+prepare_serverless_stack() {
+    echo "--- Prepare serverless stack"
 
     local args="-v"
     if [ -n "${STACK_VERSION}" ]; then
@@ -76,11 +77,12 @@ prepare_stack() {
     # else
     fi
 
-    echo "Update the Elastic stack"
-    ${ELASTIC_PACKAGE_BIN} stack update ${args}
-    echo ""
+    export EC_API_KEY=${EC_API_KEY_SECRET}
+    export EC_HOST=${EC_HOST_SECRET}
+
 
     echo "Boot up the Elastic stack"
+    # ${ELASTIC_PACKAGE_BIN} stack up -d ${args} --provider serverless -U stack.serverless.region=${EC_REGION_SECRET} -U stack.serverless.type=${SERVERLESS_PROJECT}
     ${ELASTIC_PACKAGE_BIN} stack up -d ${args}
     echo ""
 }
@@ -167,13 +169,13 @@ kubernetes_service_deployer_used() {
 }
 
 create_kind_cluster() {
-    echo "Create kind cluster"
+    echo "--- Create kind cluster"
     kind create cluster --config ${WORKSPACE}/kind-config.yaml --image kindest/node:${K8S_VERSION}
 }
 
 
 delete_kind_cluster() {
-    echo "Delete kind cluster"
+    echo "--- Delete kind cluster"
     kind delete cluster || true
 }
 
@@ -186,24 +188,12 @@ with_kubernetes
 
 use_elastic_package
 
-echo "Checking python command..."
-if ! command -v python &> /dev/null ; then
-    echo "⚠️  python is not installed"
-else
-    echo "🐍 python is installed"
-fi
-if ! command -v python3 &> /dev/null ; then
-    echo "⚠️  python3 is not installed"
-else
-    echo "🐍 python3 is installed"
-fi
-
-prepare_stack
+prepare_serverless_stack
 
 cd packages
 for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
     integration=$(basename ${it})
-    echo "Package ${integration}: check"
+    echo "--- Package ${integration}: check"
 
     pushd ${integration} 2> /dev/null
 

From a6ae16a04849ebb4a17120acc90df080f572c521 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 11 Oct 2023 17:35:40 +0200
Subject: [PATCH 038/139] Fix vars

---
 .buildkite/hooks/pre-command | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index daad09db884..93ac84d257b 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -26,7 +26,7 @@ fi
 
 # TODO: add schedule-daily too ?
 if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
-    if [ ${BUILDKITE_PIPELINE_STEP} == "publish-packages" ]; then
+    if [ ${BUILDKITE_STEP_KEY} == "publish-packages" ]; then
         export JENKINS_USERNAME_SECRET=$(retry 5 vault kv get -field username ${JENKINS_API_TOKEN_PATH})
         export JENKINS_HOST_SECRET=$(retry 5 vault kv get -field internal_ci_host ${JENKINS_API_TOKEN_PATH})
         export JENKINS_TOKEN=$(retry 5 vault kv get -field internal_ci ${JENKINS_API_TOKEN_PATH})
@@ -41,7 +41,7 @@ fi
 
 # TODO: add schedule-daily too
 if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
-    if [ ${BUILDKITE_PIPELINE_STEP} == "test-integrations-sync" ]; then
+    if [ ${BUILDKITE_STEP_KEY} == "test-integrations-sync" ]; then
         export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
         export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
 

From 47c3a56d0ba1aaab6ce71fd45e8904963d25774e Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 11 Oct 2023 17:47:52 +0200
Subject: [PATCH 039/139] Fix vault command

---
 .buildkite/hooks/pre-command | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 93ac84d257b..29e16f50447 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -51,6 +51,6 @@ if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
 
         export EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey ${EC_TOKEN_PATH})
         export EC_HOST_SECRET=$(retry 5 vault kv get -field url ${EC_TOKEN_PATH})
-        export EC_REGION_SECRET=$(retry 5 vault read -field region_qa) ${EC_DATA_PATH}
+        export EC_REGION_SECRET=$(retry 5 vault read -field region_qa ${EC_DATA_PATH})
     fi
 fi

From 33b1f259d2e0400b9225d77ee265d238f56d927f Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 11 Oct 2023 18:09:08 +0200
Subject: [PATCH 040/139] Removed for now trigger jenkins scripts

---
 .buildkite/scripts/publish_packages.sh        |   2 -
 .../scripts/triggerJenkinsJob/check_job.sh    |  16 --
 .buildkite/scripts/triggerJenkinsJob/go.mod   |   7 -
 .buildkite/scripts/triggerJenkinsJob/go.sum   |  18 ---
 .../triggerJenkinsJob/jenkins/jenkins.go      | 138 -----------------
 .../triggerJenkinsJob/jenkins/retry.go        |  48 ------
 .buildkite/scripts/triggerJenkinsJob/main.go  | 119 --------------
 .../scripts/triggerJenkinsJob/main.go.backup  | 146 ------------------
 .../scripts/triggerJenkinsJob/run_jenkins.sh  |  18 ---
 .../scripts/triggerJenkinsJob/trigger_job.sh  |  20 ---
 10 files changed, 532 deletions(-)
 delete mode 100755 .buildkite/scripts/triggerJenkinsJob/check_job.sh
 delete mode 100644 .buildkite/scripts/triggerJenkinsJob/go.mod
 delete mode 100644 .buildkite/scripts/triggerJenkinsJob/go.sum
 delete mode 100644 .buildkite/scripts/triggerJenkinsJob/jenkins/jenkins.go
 delete mode 100644 .buildkite/scripts/triggerJenkinsJob/jenkins/retry.go
 delete mode 100644 .buildkite/scripts/triggerJenkinsJob/main.go
 delete mode 100644 .buildkite/scripts/triggerJenkinsJob/main.go.backup
 delete mode 100755 .buildkite/scripts/triggerJenkinsJob/run_jenkins.sh
 delete mode 100755 .buildkite/scripts/triggerJenkinsJob/trigger_job.sh

diff --git a/.buildkite/scripts/publish_packages.sh b/.buildkite/scripts/publish_packages.sh
index 0cc4ee58826..d469fc31f63 100644
--- a/.buildkite/scripts/publish_packages.sh
+++ b/.buildkite/scripts/publish_packages.sh
@@ -108,8 +108,6 @@ use_elastic_package
 
 unpublished="false"
 
-# build packages
-
 build_packages
 
 
diff --git a/.buildkite/scripts/triggerJenkinsJob/check_job.sh b/.buildkite/scripts/triggerJenkinsJob/check_job.sh
deleted file mode 100755
index 99fcf1ab1b6..00000000000
--- a/.buildkite/scripts/triggerJenkinsJob/check_job.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash
-
-export JENKINS_TOKEN=11c7b833b6ad3ed0328b80a0317774760f
-export JENKINS_HOST_SECRET="http://localhost:8081"
-export JENKINS_USERNAME_SECRET="admin"
-
-CRUMB=$(curl -u "admin:admin" \
-    -s 'http://localhost:8081/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
-
-curl -s \
-    -H "$CRUMB" \
-    -XPOST \
-    -u "${JENKINS_USERNAME_SECRET}:${JENKINS_TOKEN}" \
-    "${JENKINS_HOST_SECRET}/job/test-job/api/json" | jq -r '.inQueue'
-
-exit 0
diff --git a/.buildkite/scripts/triggerJenkinsJob/go.mod b/.buildkite/scripts/triggerJenkinsJob/go.mod
deleted file mode 100644
index 4690be7da13..00000000000
--- a/.buildkite/scripts/triggerJenkinsJob/go.mod
+++ /dev/null
@@ -1,7 +0,0 @@
-module github.com/elastic/trigger-jenkins-buildkite-plugin
-
-go 1.21.0
-
-require github.com/bndr/gojenkins v1.1.0
-
-require golang.org/x/net v0.7.0 // indirect
diff --git a/.buildkite/scripts/triggerJenkinsJob/go.sum b/.buildkite/scripts/triggerJenkinsJob/go.sum
deleted file mode 100644
index 43a3692ff00..00000000000
--- a/.buildkite/scripts/triggerJenkinsJob/go.sum
+++ /dev/null
@@ -1,18 +0,0 @@
-github.com/bndr/gojenkins v1.1.0 h1:TWyJI6ST1qDAfH33DQb3G4mD8KkrBfyfSUoZBHQAvPI=
-github.com/bndr/gojenkins v1.1.0/go.mod h1:QeskxN9F/Csz0XV/01IC8y37CapKKWvOHa0UHLLX1fM=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/.buildkite/scripts/triggerJenkinsJob/jenkins/jenkins.go b/.buildkite/scripts/triggerJenkinsJob/jenkins/jenkins.go
deleted file mode 100644
index 7a8f32cdd82..00000000000
--- a/.buildkite/scripts/triggerJenkinsJob/jenkins/jenkins.go
+++ /dev/null
@@ -1,138 +0,0 @@
-// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-// or more contributor license agreements. Licensed under the Elastic License;
-// you may not use this file except in compliance with the Elastic License.
-
-package jenkins
-
-import (
-	"context"
-	"fmt"
-	"log"
-	"time"
-
-	"github.com/bndr/gojenkins"
-)
-
-type JenkinsClient struct {
-	client *gojenkins.Jenkins
-}
-
-type Options struct {
-	WaitingTime    time.Duration
-	MaxWaitingTime time.Duration
-	GrowthFactor   float64
-	Retries        int
-}
-
-func NewJenkinsClient(ctx context.Context, host, user, token string) (*JenkinsClient, error) {
-	jenkins, err := gojenkins.CreateJenkins(nil, host, user, token).Init(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("client coult not be created: %w", err)
-	}
-
-	return &JenkinsClient{
-		client: jenkins,
-	}, nil
-}
-
-func (j *JenkinsClient) RunJob(ctx context.Context, jobName string, async bool, params map[string]string, opts Options) error {
-	log.Printf("Building job %s", jobName)
-	var queueId int64
-
-	r := retry(func(ctx context.Context) error {
-		var err error
-		queueId, err = j.client.BuildJob(ctx, jobName, params)
-		if err != nil {
-			return fmt.Errorf("error running job %s: %w", jobName, err)
-		}
-
-		if queueId != 0 {
-			return nil
-		}
-		return fmt.Errorf("already running %s?", jobName)
-
-	}, opts.Retries, opts.GrowthFactor, opts.WaitingTime, opts.MaxWaitingTime)
-
-	if err := r(ctx); err != nil {
-		return err
-	}
-
-	build, err := j.getBuildFromJobAndQueueID(ctx, jobName, queueId)
-	if err != nil {
-		return err
-	}
-	log.Printf("Job triggered %s/%d\n", jobName, build.GetBuildNumber())
-
-	if async {
-		return nil
-	}
-
-	log.Printf("Waiting to be finished %s\n", build.GetUrl())
-	err = j.waitForBuildFinished(ctx, build)
-	if err != nil {
-		return fmt.Errorf("not finished job %s/%d: %w", jobName, build.GetBuildNumber(), err)
-	}
-
-	log.Printf("Build %s finished with result: %s\n", build.GetUrl(), build.GetResult())
-
-	if build.GetResult() != gojenkins.STATUS_SUCCESS {
-		return fmt.Errorf("build %s finished with result %s", build.GetUrl(), build.GetResult())
-	}
-	return nil
-}
-
-func (j *JenkinsClient) getBuildFromJobAndQueueID(ctx context.Context, jobName string, queueId int64) (*gojenkins.Build, error) {
-	job, err := j.client.GetJob(ctx, jobName)
-	if err != nil {
-		return nil, fmt.Errorf("not able to get job %s: %w", jobName, err)
-	}
-
-	build, err := j.getBuildFromQueueID(ctx, job, queueId)
-	if err != nil {
-		return nil, fmt.Errorf("not able to get build from %s: %w", jobName, err)
-	}
-	return build, nil
-}
-
-// based on https://github.com/bndr/gojenkins/blob/master/jenkins.go#L282
-func (j *JenkinsClient) getBuildFromQueueID(ctx context.Context, job *gojenkins.Job, queueid int64) (*gojenkins.Build, error) {
-	task, err := j.client.GetQueueItem(ctx, queueid)
-	if err != nil {
-		return nil, err
-	}
-	// Jenkins queue API has about 4.7second quiet period
-	for task.Raw.Executable.Number == 0 {
-		select {
-		case <-time.After(1000 * time.Millisecond):
-		case <-ctx.Done():
-			return nil, ctx.Err()
-		}
-		_, err = task.Poll(ctx)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	build, err := job.GetBuild(ctx, task.Raw.Executable.Number)
-	if err != nil {
-		return nil, fmt.Errorf("not able to retrieve build %s", task.Raw.Executable.Number, err)
-	}
-	return build, nil
-}
-
-func (j *JenkinsClient) waitForBuildFinished(ctx context.Context, build *gojenkins.Build) error {
-	const waitingPeriod = 10000 * time.Millisecond
-	for build.IsRunning(ctx) {
-		log.Printf("Build still running, waiting for %s...", waitingPeriod)
-		select {
-		case <-time.After(waitingPeriod):
-		case <-ctx.Done():
-			return ctx.Err()
-		}
-		_, err := build.Poll(ctx)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
diff --git a/.buildkite/scripts/triggerJenkinsJob/jenkins/retry.go b/.buildkite/scripts/triggerJenkinsJob/jenkins/retry.go
deleted file mode 100644
index 7be270c93db..00000000000
--- a/.buildkite/scripts/triggerJenkinsJob/jenkins/retry.go
+++ /dev/null
@@ -1,48 +0,0 @@
-// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-// or more contributor license agreements. Licensed under the Elastic License;
-// you may not use this file except in compliance with the Elastic License.
-
-package jenkins
-
-import (
-	"context"
-	"log"
-	"math"
-	"time"
-)
-
-type retryableFunction func(context.Context) error
-
-func minDuration(a, b time.Duration) time.Duration {
-	if a < b {
-		return a
-	}
-	return b
-}
-
-func retry(f retryableFunction, retries int, growthFactor float64, delay, maxDelay time.Duration) retryableFunction {
-	return func(ctx context.Context) error {
-		delaySeconds := delay.Seconds()
-		for r := 0; ; r++ {
-			err := f(ctx)
-			if err == nil || r >= retries {
-				// Return when there is no error or the maximum amount
-				// of retries is reached.
-				return err
-			}
-
-			waitingTimeSeconds := math.Pow(growthFactor, float64(r)) * delaySeconds
-			waitingTime := time.Duration(waitingTimeSeconds) * time.Second
-			waitingTime = minDuration(waitingTime, maxDelay)
-
-			log.Printf("Function failed, retrying in %v -> %.2f", waitingTime, waitingTimeSeconds)
-
-			select {
-			case <-time.After(waitingTime):
-			case <-ctx.Done():
-				return ctx.Err()
-			}
-		}
-		return nil
-	}
-}
diff --git a/.buildkite/scripts/triggerJenkinsJob/main.go b/.buildkite/scripts/triggerJenkinsJob/main.go
deleted file mode 100644
index f85866deccd..00000000000
--- a/.buildkite/scripts/triggerJenkinsJob/main.go
+++ /dev/null
@@ -1,119 +0,0 @@
-// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-// or more contributor license agreements. Licensed under the Elastic License;
-// you may not use this file except in compliance with the Elastic License.
-
-package main
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"log"
-	"os"
-	"strings"
-	"time"
-
-	"github.com/elastic/trigger-jenkins-buildkite-plugin/jenkins"
-)
-
-const (
-	publishingRemoteJob = "package_storage/job/publishing-job-remote"
-	signingJob          = "elastic+unified-release+master+sign-artifacts-with-gpg"
-
-	publishJobKey = "publish"
-	signJobKey    = "sign"
-)
-
-var allowedJenkinsJobs = map[string]string{
-	publishJobKey: publishingRemoteJob,
-	signJobKey:    signingJob,
-}
-
-var (
-	jenkinsHost  = os.Getenv("JENKINS_HOST_SECRET")
-	jenkinsUser  = os.Getenv("JENKINS_USERNAME_SECRET")
-	jenkinsToken = os.Getenv("JENKINS_TOKEN")
-)
-
-func jenkinsJobOptions() []string {
-	keys := make([]string, 0, len(allowedJenkinsJobs))
-	for k := range allowedJenkinsJobs {
-		keys = append(keys, k)
-	}
-	return keys
-}
-
-func main() {
-	jenkinsJob := flag.String("jenkins-job", "", fmt.Sprintf("Jenkins job to trigger. Allowed values: %s", strings.Join(jenkinsJobOptions(), " ,")))
-	waitingTime := flag.Duration("waiting-time", 5*time.Second, fmt.Sprintf("Waiting period between each retry"))
-	growthFactor := flag.Float64("growth-factor", 1.25, fmt.Sprintf("Growth-Factor used for exponential backoff delays"))
-	retries := flag.Int("retries", 20, fmt.Sprintf("Number of retries to trigger the job"))
-	maxWaitingTime := flag.Duration("max-waiting-time", 60*time.Minute, fmt.Sprintf("Maximum waiting time per each retry"))
-
-	folderPath := flag.String("folder", "", "Path to artifacts folder")
-	zipPackagePath := flag.String("package", "", "Path to zip package file (*.zip)")
-	sigPackagePath := flag.String("signature", "", "Path to the signature file of the package file (*.zip.sig)")
-	async := flag.Bool("async", false, "Run async the Jenkins job")
-	flag.Parse()
-
-	if _, ok := allowedJenkinsJobs[*jenkinsJob]; !ok {
-		log.Fatal("Invalid jenkins job")
-	}
-
-	log.Printf("Triggering job: %s", allowedJenkinsJobs[*jenkinsJob])
-
-	ctx := context.Background()
-	client, err := jenkins.NewJenkinsClient(ctx, jenkinsHost, jenkinsUser, jenkinsToken)
-	if err != nil {
-		log.Fatalf("error creating jenkins client")
-	}
-
-	opts := jenkins.Options{
-		WaitingTime:    *waitingTime,
-		Retries:        *retries,
-		GrowthFactor:   *growthFactor,
-		MaxWaitingTime: *maxWaitingTime,
-	}
-
-	switch *jenkinsJob {
-	case publishJobKey:
-		err = runPublishingRemoteJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], *zipPackagePath, *sigPackagePath, opts)
-	case signJobKey:
-		err = runSignPackageJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], *folderPath, opts)
-	default:
-		log.Fatal("unsupported jenkins job")
-	}
-
-	if err != nil {
-		log.Fatalf("Error: %s", err)
-	}
-}
-
-func runSignPackageJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, folderPath string, opts jenkins.Options) error {
-	if folderPath == "" {
-		return fmt.Errorf("missing parameter --gcs_input_path for")
-	}
-	params := map[string]string{
-		"gcs_input_path": folderPath,
-	}
-
-	return client.RunJob(ctx, jobName, async, params, opts)
-}
-
-func runPublishingRemoteJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, packagePath, signaturePath string, opts jenkins.Options) error {
-	if packagePath == "" {
-		return fmt.Errorf("missing parameter --gs_package_build_zip_path")
-	}
-	if signaturePath == "" {
-		return fmt.Errorf("missing parameter --gs_package_signature_path")
-	}
-
-	// Run the job with some parameters
-	params := map[string]string{
-		"dry_run":                   "true",
-		"gs_package_build_zip_path": packagePath,
-		"gs_package_signature_path": signaturePath,
-	}
-
-	return client.RunJob(ctx, jobName, async, params, opts)
-}
diff --git a/.buildkite/scripts/triggerJenkinsJob/main.go.backup b/.buildkite/scripts/triggerJenkinsJob/main.go.backup
deleted file mode 100644
index dc21d099eb6..00000000000
--- a/.buildkite/scripts/triggerJenkinsJob/main.go.backup
+++ /dev/null
@@ -1,146 +0,0 @@
-// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-// or more contributor license agreements. Licensed under the Elastic License;
-// you may not use this file except in compliance with the Elastic License.
-
-package main
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"log"
-	"math/rand"
-	"os"
-	"strings"
-	"time"
-
-	"github.com/elastic/trigger-jenkins-buildkite-plugin/jenkins"
-)
-
-const (
-	publishingRemoteJob = "package_storage/job/publishing-job-remote"
-	signingJob          = "elastic+unified-release+master+sign-artifacts-with-gpg"
-	testJob             = "test-job"
-
-	publishJobKey = "publish"
-	signJobKey    = "sign"
-	testJobKey    = "test"
-)
-
-var allowedJenkinsJobs = map[string]string{
-	publishJobKey: publishingRemoteJob,
-	signJobKey:    signingJob,
-	testJobKey:    testJob,
-}
-
-var (
-	jenkinsHost  = os.Getenv("JENKINS_HOST_SECRET")
-	jenkinsUser  = os.Getenv("JENKINS_USERNAME_SECRET")
-	jenkinsToken = os.Getenv("JENKINS_TOKEN")
-)
-
-func jenkinsJobOptions() []string {
-	keys := make([]string, 0, len(allowedJenkinsJobs))
-	for k := range allowedJenkinsJobs {
-		keys = append(keys, k)
-	}
-	return keys
-}
-
-func main() {
-	jenkinsJob := flag.String("jenkins-job", "", fmt.Sprintf("Jenkins job to trigger. Allowed values: %s", strings.Join(jenkinsJobOptions(), " ,")))
-	waitingTime := flag.Duration("waiting-time", 30*time.Second, fmt.Sprintf("Waiting period between each retry"))
-	retries := flag.Int("retries", 10, fmt.Sprintf("Number of retries to trigger the job"))
-	folderPath := flag.String("folder", "", "Path to artifacts folder")
-	zipPackagePath := flag.String("package", "", "Path to zip package file (*.zip)")
-	sigPackagePath := flag.String("signature", "", "Path to the signature file of the package file (*.zip.sig)")
-	async := flag.Bool("async", false, "Run async the Jenkins job")
-	flag.Parse()
-
-	if _, ok := allowedJenkinsJobs[*jenkinsJob]; !ok {
-		log.Fatal("Invalid jenkins job")
-	}
-
-	log.Printf("Triggering job: %s", allowedJenkinsJobs[*jenkinsJob])
-
-	ctx := context.Background()
-	client, err := jenkins.NewJenkinsClient(ctx, jenkinsHost, jenkinsUser, jenkinsToken)
-	if err != nil {
-		log.Fatalf("error creating jenkins client")
-	}
-
-	opts := jenkins.Options{
-		WaitingTime: *waitingTime,
-		Retries:     *retries,
-	}
-
-	switch *jenkinsJob {
-	case publishJobKey:
-		err = runPublishingRemoteJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], *zipPackagePath, *sigPackagePath, opts)
-	case signJobKey:
-		err = runSignPackageJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], *folderPath, opts)
-	case testJobKey:
-		err = runTestJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], "aabbccddeeff", opts)
-		// err = runTestJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], StringWithCharset(20, charset), opts)
-	default:
-		log.Fatal("unsupported jenkins job")
-	}
-
-	if err != nil {
-		log.Fatalf("Error: %s", err)
-	}
-}
-
-const charset = "abcdefghijklmnopqrstuvwxyz" +
-	"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
-
-var seededRand *rand.Rand = rand.New(
-	rand.NewSource(time.Now().UnixNano()))
-
-func StringWithCharset(length int, charset string) string {
-	b := make([]byte, length)
-	for i := range b {
-		b[i] = charset[seededRand.Intn(len(charset))]
-	}
-	return string(b)
-}
-
-func runSignPackageJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, folderPath string, opts jenkins.Options) error {
-	if folderPath == "" {
-		return fmt.Errorf("missing parameter --gcs_input_path for")
-	}
-	params := map[string]string{
-		"gcs_input_path": folderPath,
-	}
-
-	return client.RunJob(ctx, jobName, async, params, opts)
-}
-
-func runPublishingRemoteJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, packagePath, signaturePath string, opts jenkins.Options) error {
-	if packagePath == "" {
-		return fmt.Errorf("missing parameter --gs_package_build_zip_path")
-	}
-	if signaturePath == "" {
-		return fmt.Errorf("missing parameter --gs_package_signature_path")
-	}
-
-	// Run the job with some parameters
-	params := map[string]string{
-		"dry_run":                   "true",
-		"gs_package_build_zip_path": packagePath,
-		"gs_package_signature_path": signaturePath,
-	}
-
-	return client.RunJob(ctx, jobName, async, params, opts)
-}
-
-func runTestJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, test string, opts jenkins.Options) error {
-
-	// Run the job with some parameters
-	params := map[string]string{
-		"dry_run": "true",
-		"test":    test,
-	}
-
-	return client.RunJob(ctx, jobName, async, params, opts)
-}
diff --git a/.buildkite/scripts/triggerJenkinsJob/run_jenkins.sh b/.buildkite/scripts/triggerJenkinsJob/run_jenkins.sh
deleted file mode 100755
index 3cc7d085f43..00000000000
--- a/.buildkite/scripts/triggerJenkinsJob/run_jenkins.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-
-    # -it \
-    # --entrypoint /bin/bash \
-
-if ! docker volume ls | grep -q  jenkins-home ; then
-    echo "Create volume"
-    docker volume create jenkins-home
-fi
-
-docker run \
-    --name jenkins \
-    --rm  \
-    --user 1000:1000 \
-    -p 8081:8080 \
-    -p 50000:50000 \
-    -v jenkins-home:/var/jenkins_home \
-    jenkins/jenkins:2.346.3
diff --git a/.buildkite/scripts/triggerJenkinsJob/trigger_job.sh b/.buildkite/scripts/triggerJenkinsJob/trigger_job.sh
deleted file mode 100755
index 40e1cd5696c..00000000000
--- a/.buildkite/scripts/triggerJenkinsJob/trigger_job.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-
-export JENKINS_TOKEN=11c7b833b6ad3ed0328b80a0317774760f
-export JENKINS_HOST_SECRET="http://localhost:8081"
-export JENKINS_USERNAME_SECRET="admin"
-
-go run main.go -jenkins-job test -retries 20 -waiting-time 5s -growth-factor 1.25 -max-waiting-time 60m
-
-exit 0
-
-CRUMB=$(curl -u "admin:admin" \
-    -s 'http://localhost:8081/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
-
-curl -v \
-    -H "$CRUMB" \
-    -XPOST \
-    -u "${JENKINS_USERNAME_SECRET}:${JENKINS_TOKEN}" \
-    "${JENKINS_HOST_SECRET}/job/test-job/buildWithParameters?dry_run=true&test=aaab"
-
-exit 0

From f6a69a2c37011a1efc09ef0d92cb43e681d63dfc Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 11 Oct 2023 18:09:28 +0200
Subject: [PATCH 041/139] Run tests for some packages

---
 .buildkite/scripts/run_integrations_sync.sh | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 8f5db7612f1..7c2c3b22119 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -190,6 +190,9 @@ use_elastic_package
 
 prepare_serverless_stack
 
+num_packages=0  # TODO: to be removed
+maximum_packages=5
+
 cd packages
 for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
     integration=$(basename ${it})
@@ -215,11 +218,15 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
     echo "Check integration: ${integration}"
     ${ELASTIC_PACKAGE_BIN} check -v
 
-    # echo "Test integration: ${integration}"
+    echo "Test integration: ${integration}"
     #  # eval "$(../../build/elastic-package stack shellinit)"
-    # ${ELASTIC_PACKAGE_BIN} test -v --report-format xUnit --report-output file --test-coverage
+    ${ELASTIC_PACKAGE_BIN} test -v --report-format xUnit --report-output file --test-coverage
 
+    # TODO: debug to be removed
+    num_packages=$((num_packages+1))
     popd 2> /dev/null
-    exit 0
+    if [ $num_packages -eq ${maximum_packages} ]; then
+        exit 0
+    fi
 done
 

From 3ea83bc3f8da68ebb6a40185b37aed78a650baef Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 11 Oct 2023 18:37:09 +0200
Subject: [PATCH 042/139] Debug messages

---
 .buildkite/scripts/run_integrations_sync.sh | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 7c2c3b22119..0fa087fe607 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -91,6 +91,9 @@ is_spec_3_0_0() {
     local pkg_spec=$(cat manifest.yml | yq '.format_version')
     local major_version=$(echo $pkg_spec | cut -d '.' -f 1)
 
+    echo "pkg_spec ${pkg_spec}"
+    echo "major_version ${major_version}"
+
     if [ ${major_version} -ge 3 ]; then
         return 0
     fi
@@ -200,10 +203,6 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
 
     pushd ${integration} 2> /dev/null
 
-    if ! is_pr_affected ${integration} ; then
-        echo "[${integration}] Skipped"
-    fi
-
     if [[ ${SERVERLESS} == "true" ]] ; then
         if ! is_spec_3_0_0 ]]; then
             echo "Not v3 spec version. Skipped"
@@ -211,6 +210,11 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
         fi
     fi
 
+    if ! is_pr_affected ${integration} ; then
+        echo "[${integration}] Skipped"
+        # continue # TODO enable this skip after testing
+    fi
+
     if kubernetes_service_deployer_used ; then
         create_kind_cluster
     fi

From 4b91ed54b88b25d77ac4a9207d1459fc9159d93b Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 11 Oct 2023 18:55:14 +0200
Subject: [PATCH 043/139] Add environment vartiable

---
 .buildkite/pipeline.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 13ab2cc6ee4..c02ddc3a607 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -38,6 +38,8 @@ steps:
   - label: "Check integrations"
     key: "test-integrations-sync"
     command: ".buildkite/scripts/run_integrations_sync.sh"
+    env:
+      SERVERLESS: true
     agents:
       provider: "gcp"
     depends_on:

From 77e4a338d6227e02408010d3bda37a25a41fda47 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 11 Oct 2023 19:05:41 +0200
Subject: [PATCH 044/139] Add missing popd command

---
 .buildkite/scripts/run_integrations_sync.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 0fa087fe607..5b21c5c3f67 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -196,7 +196,7 @@ prepare_serverless_stack
 num_packages=0  # TODO: to be removed
 maximum_packages=5
 
-cd packages
+pushd packages > /dev/null
 for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
     integration=$(basename ${it})
     echo "--- Package ${integration}: check"
@@ -206,12 +206,14 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
     if [[ ${SERVERLESS} == "true" ]] ; then
         if ! is_spec_3_0_0 ]]; then
             echo "Not v3 spec version. Skipped"
+            popd 2> /dev/null
             continue
         fi
     fi
 
     if ! is_pr_affected ${integration} ; then
         echo "[${integration}] Skipped"
+        # popd 2> /dev/null
         # continue # TODO enable this skip after testing
     fi
 
@@ -230,7 +232,8 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
     num_packages=$((num_packages+1))
     popd 2> /dev/null
     if [ $num_packages -eq ${maximum_packages} ]; then
-        exit 0
+        break
     fi
 done
+popd > /dev/null
 

From 64969a1b1bea72e97a9d8813dbd9d13e7ded24d3 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 11 Oct 2023 19:36:36 +0200
Subject: [PATCH 045/139] Ensure kind cluster is deleted

Change PATH order to be able to use the versions set in the given
environment variables.
Increase number of test packages up to 15
---
 .buildkite/hooks/pre-exit                   |  4 ++++
 .buildkite/scripts/common.sh                | 14 +++++++++++++-
 .buildkite/scripts/run_integrations_sync.sh | 16 +---------------
 3 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index a6686ccfb95..d2504b7469e 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -11,6 +11,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
         unset AWS_ACCESS_KEY_ID
         unset AWS_SECRET_ACCESS_KEY
 
+        # Ensure that kind cluster is deleted
+        delete_kind_cluster
+
+        # Ensure elastic stack is stopped
         if [ -f ${ELASTIC_PACKAGE_BIN} ]; then
             export EC_API_KEY=${EC_API_KEY_SECRET}
             export EC_HOST=${EC_HOST_SECRET}
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index ce1ec95b2bb..39e712aab83 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -65,7 +65,7 @@ create_bin_folder() {
 add_bin_path() {
   create_bin_folder
   echo "Adding PATH to the environment variables..."
-  export PATH="${PATH}:${BIN_FOLDER}"
+  export PATH="${BIN_FOLDER}:${PATH}"  # TODO: set bin folder after PATH
 }
 
 with_go() {
@@ -199,3 +199,15 @@ repo_name() {
     orgAndRepo=$(echo $repoUrl | cut -d':' -f 2)
     echo "$(basename ${orgAndRepo} .git)"
 }
+
+
+create_kind_cluster() {
+    echo "--- Create kind cluster"
+    kind create cluster --config ${WORKSPACE}/kind-config.yaml --image kindest/node:${K8S_VERSION}
+}
+
+
+delete_kind_cluster() {
+    echo "--- Delete kind cluster"
+    kind delete cluster || true
+}
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 5b21c5c3f67..cadf2461324 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -80,7 +80,6 @@ prepare_serverless_stack() {
     export EC_API_KEY=${EC_API_KEY_SECRET}
     export EC_HOST=${EC_HOST_SECRET}
 
-
     echo "Boot up the Elastic stack"
     # ${ELASTIC_PACKAGE_BIN} stack up -d ${args} --provider serverless -U stack.serverless.region=${EC_REGION_SECRET} -U stack.serverless.type=${SERVERLESS_PROJECT}
     ${ELASTIC_PACKAGE_BIN} stack up -d ${args}
@@ -91,9 +90,6 @@ is_spec_3_0_0() {
     local pkg_spec=$(cat manifest.yml | yq '.format_version')
     local major_version=$(echo $pkg_spec | cut -d '.' -f 1)
 
-    echo "pkg_spec ${pkg_spec}"
-    echo "major_version ${major_version}"
-
     if [ ${major_version} -ge 3 ]; then
         return 0
     fi
@@ -171,16 +167,6 @@ kubernetes_service_deployer_used() {
     find . -type d | egrep '_dev/deploy/k8s$'
 }
 
-create_kind_cluster() {
-    echo "--- Create kind cluster"
-    kind create cluster --config ${WORKSPACE}/kind-config.yaml --image kindest/node:${K8S_VERSION}
-}
-
-
-delete_kind_cluster() {
-    echo "--- Delete kind cluster"
-    kind delete cluster || true
-}
 
 add_bin_path
 
@@ -194,7 +180,7 @@ use_elastic_package
 prepare_serverless_stack
 
 num_packages=0  # TODO: to be removed
-maximum_packages=5
+maximum_packages=15
 
 pushd packages > /dev/null
 for it in $(find . -maxdepth 1 -mindepth 1 -type d); do

From 1be824d850a14727b504804bfb3f12c13e45ed3b Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 11 Oct 2023 20:19:07 +0200
Subject: [PATCH 046/139] Add delete kind cluster step also after testing
 package

---
 .buildkite/scripts/run_integrations_sync.sh | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index cadf2461324..3f0164f248e 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -203,17 +203,24 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
         # continue # TODO enable this skip after testing
     fi
 
+    echo "Check integration: ${integration}"
+    ${ELASTIC_PACKAGE_BIN} check -v
+
+    use_kind=0
     if kubernetes_service_deployer_used ; then
+        use_kind=1
         create_kind_cluster
     fi
 
-    echo "Check integration: ${integration}"
-    ${ELASTIC_PACKAGE_BIN} check -v
 
     echo "Test integration: ${integration}"
     #  # eval "$(../../build/elastic-package stack shellinit)"
     ${ELASTIC_PACKAGE_BIN} test -v --report-format xUnit --report-output file --test-coverage
 
+    if [ ${use_kind} -eq 1 ]; then
+        delete_kind_cluster
+    fi
+
     # TODO: debug to be removed
     num_packages=$((num_packages+1))
     popd 2> /dev/null

From 98fa4c0d3d6959e77f09bd2cb9530bcf1b6ae6dc Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 16 Oct 2023 16:34:01 +0200
Subject: [PATCH 047/139] Test with more packages

---
 .buildkite/scripts/run_integrations_sync.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 3f0164f248e..dcc7cb663b1 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -180,7 +180,7 @@ use_elastic_package
 prepare_serverless_stack
 
 num_packages=0  # TODO: to be removed
-maximum_packages=15
+maximum_packages=25
 
 pushd packages > /dev/null
 for it in $(find . -maxdepth 1 -mindepth 1 -type d); do

From c636d6cbd25bc7b30ccff6429aa54065da49ccf7 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 16 Oct 2023 16:44:12 +0200
Subject: [PATCH 048/139] Move helper functions to common library

Add FORCE_CHECK_CALL environment variable in pipeline step
---
 .buildkite/pipeline.yml                     |   1 +
 .buildkite/scripts/common.sh                | 150 +++++++++++++++++++
 .buildkite/scripts/run_integrations_sync.sh | 151 +-------------------
 3 files changed, 152 insertions(+), 150 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index c02ddc3a607..419d1ae42d2 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -40,6 +40,7 @@ steps:
     command: ".buildkite/scripts/run_integrations_sync.sh"
     env:
       SERVERLESS: true
+      FORCE_CHECK_ALL: true
     agents:
       provider: "gcp"
     depends_on:
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 39e712aab83..22927d2119a 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -211,3 +211,153 @@ delete_kind_cluster() {
     echo "--- Delete kind cluster"
     kind delete cluster || true
 }
+
+kibana_version_manifest() {
+    local kibana_version=$(cat manifest.yml | yq ".conditions.kibana.version")
+    if [ $kibana_version != "null" ]; then
+        echo "${kibana_version}"
+        return
+    fi
+
+    kibana_version=$(cat manifest.yml | yq ".conditions.\"kibana.version\"")
+    if [ $kibana_version != "null" ]; then
+        echo "${kibana_version}"
+        return
+    fi
+
+    echo "null"
+}
+
+is_unsupported_stack() {
+    if [ "${STACK_VERSION}" == "" ]; then
+        return 1
+    fi
+
+    local kibana_version=$(kibana_version_manifest)
+    if [ "${kibana_version}" == "null" ]; then
+        return 1
+    fi
+    if [[ ! ${kibana_version} =~ \^7\. && ${STACK_VERSION} =~ ^7\. ]]; then
+        return 0
+    fi
+    if [[ ! ${kibana_version} =~ \^8\. && ${STACK_VERSION} =~ ^8\. ]]; then
+        return 0
+    fi
+    return 1
+}
+
+oldest_supported_version() {
+    local kibana_version=$(cat manifest.yml | yq ".conditions.kibana.version")
+    if [ $kibana_version != "null" ]; then
+        python3 .buildkite/scripts/find_oldest_supported_version --manifest manifest.yml
+        return
+    fi
+
+    kibana_version=$(cat manifest.yml | yq ".conditions.\"kibana.version\"")
+    if [ $kibana_version != "null" ]; then
+        python3 .buildkite/scripts/find_oldest_supported_version --manifest manifest.yml
+        return
+    fi
+
+    echo "null"
+}
+
+prepare_serverless_stack() {
+    echo "--- Prepare serverless stack"
+
+    local args="-v"
+    if [ -n "${STACK_VERSION}" ]; then
+        args="${args} --version ${STACK_VERSION}"
+    # TODO What stack version to use (for agents) in serverless?
+    # else
+    fi
+
+    export EC_API_KEY=${EC_API_KEY_SECRET}
+    export EC_HOST=${EC_HOST_SECRET}
+
+    echo "Boot up the Elastic stack"
+    # ${ELASTIC_PACKAGE_BIN} stack up -d ${args} --provider serverless -U stack.serverless.region=${EC_REGION_SECRET} -U stack.serverless.type=${SERVERLESS_PROJECT}
+    ${ELASTIC_PACKAGE_BIN} stack up -d ${args}
+    echo ""
+}
+
+is_spec_3_0_0() {
+    local pkg_spec=$(cat manifest.yml | yq '.format_version')
+    local major_version=$(echo $pkg_spec | cut -d '.' -f 1)
+
+    if [ ${major_version} -ge 3 ]; then
+        return 0
+    fi
+    return 1
+}
+
+get_from_changeset() {
+    if [ "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" != "false" ]; then
+        # pull request
+        echo "origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
+        return
+    fi
+    # main or backport branches
+    previous_commit=$(git rev-parse --verify FETCH_HEAD~1)
+    echo "${previous_commit}"
+}
+
+get_to_changeset() {
+    echo "${BUILDKITE_COMMIT}"
+}
+
+is_pr_affected() {
+    local integration="${1}"
+
+    if is_unsupported_stack ; then
+        echo "[${integration}] PR is not affected: unsupported stack (${STACK_VERSION})"
+        return 1
+    fi
+
+    if [[ ${FORCE_CHECK_ALL} == "true" ]];then
+        echo "[${integration}] PR is affected: \"force_check_all\" parameter enabled"
+        return 0
+    fi
+
+    # setting default values for a PR
+    # TODO: get previous built commit as in Jenkins (groovy)
+    # def from = env.CHANGE_TARGET?.trim() ? "origin/${env.CHANGE_TARGET}" : "${env.GIT_PREVIOUS_COMMIT?.trim() ? env.GIT_PREVIOUS_COMMIT : env.GIT_BASE_COMMIT}"
+    local from="$(get_from_changeset)"
+    local to="$(get_to_changeset)"
+
+    # TODO: If running for an integration branch (main, backport-*) check with
+    # GIT_PREVIOUS_SUCCESSFUL_COMMIT to check if the branch is still healthy.
+    # If this value is not available, check with last commit.
+    if [[ ${BUILDKITE_BRANCH} == "main" || ${BUILDKITE_BRANCH} =~ ^backport- ]]; then
+        echo "[${integration}] PR is affected: running on ${BUILDKITE_BRANCH} branch"
+        # TODO: get previous successful commit as in Jenkins (groovy)
+        # from = env.GIT_PREVIOUS_SUCCESSFUL_COMMIT?.trim() ? env.GIT_PREVIOUS_SUCCESSFUL_COMMIT : "origin/${env.BRANCH_NAME}^"
+        from="origin/${BUILDKITE_BRANCH}^"
+        to="origin/${BUILDKITE_BRANCH}"
+    fi
+
+    echo "[${integration}] git-diff: check non-package files"
+    if git diff --name-only $(git merge-base ${from} ${to}) ${to} | egrep '^(packages/|.github/CODEOWNERS)' ; then
+        echo "[${integration}] PR is affected: found non-package files"
+        return 0
+    fi
+    echo "[${integration}] git-diff: check package files"
+    if git diff --name-only $(git merge-base ${from} ${to}) ${to} | egrep '^packages/${integration}/' ; then
+        echo "[${integration}] PR is affected: found package files"
+        return 0
+    fi
+    echo "[${integration}] PR is not affected"
+    return 1
+}
+
+is_pr() {
+    if [ "${BUILDKITE_PULL_REQUEST}" == "false" ]; then
+        return 0
+    fi
+    return 1
+}
+
+kubernetes_service_deployer_used() {
+    echo "Check if Kubernetes service deployer is used"
+    find . -type d | egrep '_dev/deploy/k8s$'
+}
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index dcc7cb663b1..19fa8c2104e 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -13,160 +13,11 @@ SERVERLESS_PROJECT=${SERVERLESS_PROJECT:-"observability"}
 
 
 if [ ! -d packages ]; then
+    echo "Missing packages folder"
     buildkite-agent annotate "Missing packages folder" --style "error"
     exit 1
 fi
 
-kibana_version_manifest() {
-    local kibana_version=$(cat manifest.yml | yq ".conditions.kibana.version")
-    if [ $kibana_version != "null" ]; then
-        echo "${kibana_version}"
-        return
-    fi
-
-    kibana_version=$(cat manifest.yml | yq ".conditions.\"kibana.version\"")
-    if [ $kibana_version != "null" ]; then
-        echo "${kibana_version}"
-        return
-    fi
-
-    echo "null"
-}
-
-is_unsupported_stack() {
-    if [ "${STACK_VERSION}" == "" ]; then
-        return 1
-    fi
-
-    local kibana_version=$(kibana_version_manifest)
-    if [ "${kibana_version}" == "null" ]; then
-        return 1
-    fi
-    if [[ ! ${kibana_version} =~ \^7\. && ${STACK_VERSION} =~ ^7\. ]]; then
-        return 0
-    fi
-    if [[ ! ${kibana_version} =~ \^8\. && ${STACK_VERSION} =~ ^8\. ]]; then
-        return 0
-    fi
-    return 1
-}
-
-oldest_supported_version() {
-    local kibana_version=$(cat manifest.yml | yq ".conditions.kibana.version")
-    if [ $kibana_version != "null" ]; then
-        python3 .buildkite/scripts/find_oldest_supported_version --manifest manifest.yml
-        return
-    fi
-
-    kibana_version=$(cat manifest.yml | yq ".conditions.\"kibana.version\"")
-    if [ $kibana_version != "null" ]; then
-        python3 .buildkite/scripts/find_oldest_supported_version --manifest manifest.yml
-        return
-    fi
-
-    echo "null"
-}
-
-prepare_serverless_stack() {
-    echo "--- Prepare serverless stack"
-
-    local args="-v"
-    if [ -n "${STACK_VERSION}" ]; then
-        args="${args} --version ${STACK_VERSION}"
-    # TODO What stack version to use (for agents) in serverless?
-    # else
-    fi
-
-    export EC_API_KEY=${EC_API_KEY_SECRET}
-    export EC_HOST=${EC_HOST_SECRET}
-
-    echo "Boot up the Elastic stack"
-    # ${ELASTIC_PACKAGE_BIN} stack up -d ${args} --provider serverless -U stack.serverless.region=${EC_REGION_SECRET} -U stack.serverless.type=${SERVERLESS_PROJECT}
-    ${ELASTIC_PACKAGE_BIN} stack up -d ${args}
-    echo ""
-}
-
-is_spec_3_0_0() {
-    local pkg_spec=$(cat manifest.yml | yq '.format_version')
-    local major_version=$(echo $pkg_spec | cut -d '.' -f 1)
-
-    if [ ${major_version} -ge 3 ]; then
-        return 0
-    fi
-    return 1
-}
-
-get_from_changeset() {
-    if [ "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" != "false" ]; then
-        # pull request
-        echo "origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
-        return
-    fi
-    # main or backport branches
-    previous_commit=$(git rev-parse --verify FETCH_HEAD~1)
-    echo "${previous_commit}"
-}
-
-get_to_changeset() {
-    echo "${BUILDKITE_COMMIT}"
-}
-
-is_pr_affected() {
-    local integration="${1}"
-
-    if is_unsupported_stack ; then
-        echo "[${integration}] PR is not affected: unsupported stack (${STACK_VERSION})"
-        return 1
-    fi
-
-    if [[ ${FORCE_CHECK_ALL} == "true" ]];then
-        echo "[${integration}] PR is affected: \"force_check_all\" parameter enabled"
-        return 0
-    fi
-
-    # setting default values for a PR
-    # TODO: get previous built commit as in Jenkins (groovy)
-    # def from = env.CHANGE_TARGET?.trim() ? "origin/${env.CHANGE_TARGET}" : "${env.GIT_PREVIOUS_COMMIT?.trim() ? env.GIT_PREVIOUS_COMMIT : env.GIT_BASE_COMMIT}"
-    local from="$(get_from_changeset)"
-    local to="$(get_to_changeset)"
-
-    # TODO: If running for an integration branch (main, backport-*) check with
-    # GIT_PREVIOUS_SUCCESSFUL_COMMIT to check if the branch is still healthy.
-    # If this value is not available, check with last commit.
-    if [[ ${BUILDKITE_BRANCH} == "main" || ${BUILDKITE_BRANCH} =~ ^backport- ]]; then
-        echo "[${integration}] PR is affected: running on ${BUILDKITE_BRANCH} branch"
-        # TODO: get previous successful commit as in Jenkins (groovy)
-        # from = env.GIT_PREVIOUS_SUCCESSFUL_COMMIT?.trim() ? env.GIT_PREVIOUS_SUCCESSFUL_COMMIT : "origin/${env.BRANCH_NAME}^"
-        from="origin/${BUILDKITE_BRANCH}^"
-        to="origin/${BUILDKITE_BRANCH}"
-    fi
-
-    echo "[${integration}] git-diff: check non-package files"
-    if git diff --name-only $(git merge-base ${from} ${to}) ${to} | egrep '^(packages/|.github/CODEOWNERS)' ; then
-        echo "[${integration}] PR is affected: found non-package files"
-        return 0
-    fi
-    echo "[${integration}] git-diff: check package files"
-    if git diff --name-only $(git merge-base ${from} ${to}) ${to} | egrep '^packages/${integration}/' ; then
-        echo "[${integration}] PR is affected: found package files"
-        return 0
-    fi
-    echo "[${integration}] PR is not affected"
-    return 1
-}
-
-is_pr() {
-    if [ "${BUILDKITE_PULL_REQUEST}" == "false" ]; then
-        return 0
-    fi
-    return 1
-}
-
-kubernetes_service_deployer_used() {
-    echo "Check if Kubernetes service deployer is used"
-    find . -type d | egrep '_dev/deploy/k8s$'
-}
-
 
 add_bin_path
 

From 38cae12ce60b540941aa70f72e40daa9793eea6e Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 16 Oct 2023 17:41:43 +0200
Subject: [PATCH 049/139] Add check about capabilities

---
 .buildkite/pipeline.yml                     | 17 ++++++-
 .buildkite/scripts/common.sh                | 55 ++++++++++++++++++---
 .buildkite/scripts/run_integrations_sync.sh | 11 +++--
 3 files changed, 70 insertions(+), 13 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 419d1ae42d2..d5141d66e18 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -40,9 +40,24 @@ steps:
     command: ".buildkite/scripts/run_integrations_sync.sh"
     env:
       SERVERLESS: true
+      SERVERLESS_PROJECT: observability
       FORCE_CHECK_ALL: true
     agents:
       provider: "gcp"
     depends_on:
-      - step: "check"
+      - step: "publish-packages"
         allow_failure: false
+          #
+  # to be moved to schedule-daily job
+  # - label: "Check integrations"
+  #   key: "test-integrations-sync"
+  #   command: ".buildkite/scripts/run_integrations_sync.sh"
+  #   env:
+  #     SERVERLESS: true
+  #     SERVERLESS_PROJECT: security
+  #     FORCE_CHECK_ALL: true
+  #   agents:
+  #     provider: "gcp"
+  #   depends_on:
+  #     - step: "publish-packages"
+  #       allow_failure: false
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 22927d2119a..527d39d0335 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -228,22 +228,58 @@ kibana_version_manifest() {
     echo "null"
 }
 
-is_unsupported_stack() {
+capabilities_manifest() {
+    cat manifest.yml | yq ".conditions.elastic.capabilitites"
+}
+
+is_supported_capability() {
+    if [ "${SERVERLESS_PROJECT}" == "" ]; then
+        return 0
+    fi
+
+    local capabilities=$(capabilities_manifest)
+
+    # if no capabilities defined, it is available iavailable all projects
+    if [[  "${capabilities}" == "null" ]]; then
+        return 0
+    fi
+
+    if [[ ${SERVERLESS_PROJECT} == "observability" ]]; then
+        if echo ${capabilities} |egrep 'apm|observability|uptime' ; then
+            return 0
+        else
+            return 1
+        fi
+    fi
+
+    if [[ ${SERVERLESS_PROJECT} == "security" ]]; then
+        if echo ${capabilities} |egrep 'security' ; then
+            return 0
+        else
+            return 1
+        fi
+    fi
+
+    echo "Unsupported serverless project"
+    return 1
+}
+
+is_supported_stack() {
     if [ "${STACK_VERSION}" == "" ]; then
-        return 1
+        return 0
     fi
 
     local kibana_version=$(kibana_version_manifest)
     if [ "${kibana_version}" == "null" ]; then
-        return 1
+        return 0
     fi
     if [[ ! ${kibana_version} =~ \^7\. && ${STACK_VERSION} =~ ^7\. ]]; then
-        return 0
+        return 1
     fi
     if [[ ! ${kibana_version} =~ \^8\. && ${STACK_VERSION} =~ ^8\. ]]; then
-        return 0
+        return 1
     fi
-    return 1
+    return 0
 }
 
 oldest_supported_version() {
@@ -309,11 +345,16 @@ get_to_changeset() {
 is_pr_affected() {
     local integration="${1}"
 
-    if is_unsupported_stack ; then
+    if ! is_supported_stack ; then
         echo "[${integration}] PR is not affected: unsupported stack (${STACK_VERSION})"
         return 1
     fi
 
+    if ! is_supported_capability ; then
+        echo "[${integration}] PR is not affected: capabilities not mached with the project (${SERVERLESS_PROJECT})"
+        return 1
+    fi
+
     if [[ ${FORCE_CHECK_ALL} == "true" ]];then
         echo "[${integration}] PR is affected: \"force_check_all\" parameter enabled"
         return 0
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 19fa8c2104e..26f33248f69 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -50,8 +50,8 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
 
     if ! is_pr_affected ${integration} ; then
         echo "[${integration}] Skipped"
-        # popd 2> /dev/null
-        # continue # TODO enable this skip after testing
+        popd 2> /dev/null
+        continue
     fi
 
     echo "Check integration: ${integration}"
@@ -63,18 +63,19 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
         create_kind_cluster
     fi
 
-
     echo "Test integration: ${integration}"
-    #  # eval "$(../../build/elastic-package stack shellinit)"
     ${ELASTIC_PACKAGE_BIN} test -v --report-format xUnit --report-output file --test-coverage
 
+    # TODO: add benchmarks support
+
     if [ ${use_kind} -eq 1 ]; then
         delete_kind_cluster
     fi
 
+    popd 2> /dev/null
+
     # TODO: debug to be removed
     num_packages=$((num_packages+1))
-    popd 2> /dev/null
     if [ $num_packages -eq ${maximum_packages} ]; then
         break
     fi

From 354d0010b2122b8b1182d0e22027bd1a947d9a3b Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 16 Oct 2023 18:00:26 +0200
Subject: [PATCH 050/139] Add junit as artifacts and step

---
 .buildkite/pipeline.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index d5141d66e18..f0ff47f2f5d 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -44,6 +44,8 @@ steps:
       FORCE_CHECK_ALL: true
     agents:
       provider: "gcp"
+    artifact_paths:
+      - build/results/*.xml
     depends_on:
       - step: "publish-packages"
         allow_failure: false
@@ -61,3 +63,10 @@ steps:
   #   depends_on:
   #     - step: "publish-packages"
   #       allow_failure: false
+
+  - label: ":junit: Junit annotate"
+    plugins:
+      - junit-annotate#v2.4.1:
+          artifacts: "build/results/*.xml"
+    agents:
+      provider: "gcp"  # junit plugin requires docker

From 068447a3277869b06ec736c7a921a31f59e34b3c Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 16 Oct 2023 18:01:46 +0200
Subject: [PATCH 051/139] Add teardown per package

---
 .buildkite/scripts/common.sh                | 23 +++++++++++++++++++++
 .buildkite/scripts/run_integrations_sync.sh |  5 ++++-
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 527d39d0335..40549b15db5 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -402,3 +402,26 @@ kubernetes_service_deployer_used() {
     echo "Check if Kubernetes service deployer is used"
     find . -type d | egrep '_dev/deploy/k8s$'
 }
+
+teardown_serverless_test_package() {
+    local integration=$1
+    local dump_directory="${WORKSPACE}/build/elastic-stack-dump/${integration}"
+
+    echo "Collect Elastic stack logs"
+    ${ELASTIC_PACKAGE_BIN} stack dump -v --output ${dump_directory}
+
+    # TODO: upload insecure logs
+}
+
+teardown_test_package() {
+    local integration=$1
+    local dump_directory="${WORKSPACE}/build/elastic-stack-dump/${integration}"
+
+    echo "Collect Elastic stack logs"
+    ${ELASTIC_PACKAGE_BIN} stack dump -v --output ${dump_directory}
+
+    # TODO: upload insecure logs
+
+    echo "Take down the Elastic stack"
+    ${ELASTIC_PACKAGE_BIN} stack down -v
+}
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 26f33248f69..efde8983784 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -66,12 +66,15 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
     echo "Test integration: ${integration}"
     ${ELASTIC_PACKAGE_BIN} test -v --report-format xUnit --report-output file --test-coverage
 
-    # TODO: add benchmarks support
+    # TODO: add benchmarks support (https://github.com/elastic/integrations/blob/befdc5cb752a08aaf5f79b0d9bdb68588ade9f27/.ci/Jenkinsfile#L180)
+    # ${ELASTIC_PACKAGE_BIN} benchmark pipeline -v --report-format json --report-output file
 
     if [ ${use_kind} -eq 1 ]; then
         delete_kind_cluster
     fi
 
+    teardown_serverless_test_package
+
     popd 2> /dev/null
 
     # TODO: debug to be removed

From fb7e8d8731d42f7f8700758ce4f315addc6d38b9 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 16 Oct 2023 18:47:23 +0200
Subject: [PATCH 052/139] Add wait step to execute junit as last step

---
 .buildkite/pipeline.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index f0ff47f2f5d..8d47af4ee64 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -64,6 +64,9 @@ steps:
   #     - step: "publish-packages"
   #       allow_failure: false
 
+  - wait: ~
+    continue_on_failure: true
+
   - label: ":junit: Junit annotate"
     plugins:
       - junit-annotate#v2.4.1:

From df41a6025552ba3584f7f93dbbb47c18f777b840 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 16 Oct 2023 19:05:32 +0200
Subject: [PATCH 053/139] Add integration name for teardown

---
 .buildkite/scripts/run_integrations_sync.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index efde8983784..5143efc2ce4 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -73,7 +73,7 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
         delete_kind_cluster
     fi
 
-    teardown_serverless_test_package
+    teardown_serverless_test_package ${integration}
 
     popd 2> /dev/null
 

From 67e076d503094407cb601a36f77835d1af8cb523 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 16 Oct 2023 19:18:41 +0200
Subject: [PATCH 054/139] Remove output from pushd and popd

---
 .buildkite/scripts/run_integrations_sync.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 5143efc2ce4..6aa6d148437 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -38,19 +38,19 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
     integration=$(basename ${it})
     echo "--- Package ${integration}: check"
 
-    pushd ${integration} 2> /dev/null
+    pushd ${integration} > /dev/null
 
     if [[ ${SERVERLESS} == "true" ]] ; then
         if ! is_spec_3_0_0 ]]; then
             echo "Not v3 spec version. Skipped"
-            popd 2> /dev/null
+            popd > /dev/null
             continue
         fi
     fi
 
     if ! is_pr_affected ${integration} ; then
         echo "[${integration}] Skipped"
-        popd 2> /dev/null
+        popd > /dev/null
         continue
     fi
 
@@ -75,7 +75,7 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
 
     teardown_serverless_test_package ${integration}
 
-    popd 2> /dev/null
+    popd > /dev/null
 
     # TODO: debug to be removed
     num_packages=$((num_packages+1))

From 2ed855d2ef181d3433ad1e24beb42e826c85a4ce Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 16 Oct 2023 19:20:55 +0200
Subject: [PATCH 055/139] Check total packages examined

---
 .buildkite/scripts/run_integrations_sync.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 6aa6d148437..4456990c114 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -30,6 +30,7 @@ use_elastic_package
 
 prepare_serverless_stack
 
+packages_visited=0
 num_packages=0  # TODO: to be removed
 maximum_packages=25
 
@@ -39,6 +40,7 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
     echo "--- Package ${integration}: check"
 
     pushd ${integration} > /dev/null
+    packages_visited=$((packages_visited+1))
 
     if [[ ${SERVERLESS} == "true" ]] ; then
         if ! is_spec_3_0_0 ]]; then
@@ -85,3 +87,4 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
 done
 popd > /dev/null
 
+echo "Total packages examined: ${packages_visited}"

From dc04c42c7d8210f9df838b65fe125d32106b5e84 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 17 Oct 2023 00:13:17 +0200
Subject: [PATCH 056/139] Allow to fail a package to continue with the
 following packages

---
 .buildkite/scripts/run_integrations_sync.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 4456990c114..5a0e688a389 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -66,7 +66,11 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
     fi
 
     echo "Test integration: ${integration}"
+    set +e
     ${ELASTIC_PACKAGE_BIN} test -v --report-format xUnit --report-output file --test-coverage
+    exit_code=$?
+    echo "[$integration] failed testing (exit code $exit_code)"
+    set -è
 
     # TODO: add benchmarks support (https://github.com/elastic/integrations/blob/befdc5cb752a08aaf5f79b0d9bdb68588ade9f27/.ci/Jenkinsfile#L180)
     # ${ELASTIC_PACKAGE_BIN} benchmark pipeline -v --report-format json --report-output file

From 3eadd981606dc8040a2f5975464009391911d8ac Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 17 Oct 2023 11:48:05 +0200
Subject: [PATCH 057/139] Check errors

---
 .buildkite/scripts/run_integrations_sync.sh | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 5a0e688a389..da11919657f 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -56,21 +56,27 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
         continue
     fi
 
-    echo "Check integration: ${integration}"
-    ${ELASTIC_PACKAGE_BIN} check -v
-
     use_kind=0
     if kubernetes_service_deployer_used ; then
         use_kind=1
         create_kind_cluster
     fi
 
-    echo "Test integration: ${integration}"
     set +e
-    ${ELASTIC_PACKAGE_BIN} test -v --report-format xUnit --report-output file --test-coverage
+    echo "Check integration: ${integration}"
+    ${ELASTIC_PACKAGE_BIN} check -v
     exit_code=$?
-    echo "[$integration] failed testing (exit code $exit_code)"
-    set -è
+
+    if [ ${exit_code} -eq 0 ]; then
+        echo "Test integration: ${integration}"
+        ${ELASTIC_PACKAGE_BIN} test -v --report-format xUnit --report-output file --test-coverage
+        exit_code=$?
+    fi
+
+    if [ ${exit_code} -ne 0 ]; then
+        echo "[$integration] failed (exit code $exit_code)"
+    fi
+     set -e
 
     # TODO: add benchmarks support (https://github.com/elastic/integrations/blob/befdc5cb752a08aaf5f79b0d9bdb68588ade9f27/.ci/Jenkinsfile#L180)
     # ${ELASTIC_PACKAGE_BIN} benchmark pipeline -v --report-format json --report-output file

From 07a9aa1659a3df4c0227090dbf0d014f18852420 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 17 Oct 2023 16:43:16 +0200
Subject: [PATCH 058/139] Test without system tests

---
 .buildkite/scripts/common.sh                | 19 +++++++++++++++++++
 .buildkite/scripts/run_integrations_sync.sh | 21 +++------------------
 2 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 40549b15db5..38fe1d7c70d 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -425,3 +425,22 @@ teardown_test_package() {
     echo "Take down the Elastic stack"
     ${ELASTIC_PACKAGE_BIN} stack down -v
 }
+
+list_all_directories() {
+    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {}
+}
+
+check_and_test_packages() {
+    local integration=$1
+    echo "Check integration: ${integration}"
+    ${ELASTIC_PACKAGE_BIN} check -v
+
+    echo "Install integration: ${integration}"
+    ${ELASTIC_PACKAGE_BIN} install -v
+
+    echo "Test integration: ${integration}"
+    TEST_OPTIONS="-v --report-format xUnit --report-output file --test-coverage"
+    ${ELASTIC_PACKAGE_BIN} test asset ${TEST_OPTIONS}
+    ${ELASTIC_PACKAGE_BIN} test static ${TEST_OPTIONS}
+    ${ELASTIC_PACKAGE_BIN} test pipeline ${TEST_OPTIONS}
+}
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index da11919657f..17ea4e8f4fd 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -18,7 +18,6 @@ if [ ! -d packages ]; then
     exit 1
 fi
 
-
 add_bin_path
 
 with_yq
@@ -35,8 +34,8 @@ num_packages=0  # TODO: to be removed
 maximum_packages=25
 
 pushd packages > /dev/null
-for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
-    integration=$(basename ${it})
+
+for integration in $(list_all_directories); do
     echo "--- Package ${integration}: check"
 
     pushd ${integration} > /dev/null
@@ -62,21 +61,7 @@ for it in $(find . -maxdepth 1 -mindepth 1 -type d); do
         create_kind_cluster
     fi
 
-    set +e
-    echo "Check integration: ${integration}"
-    ${ELASTIC_PACKAGE_BIN} check -v
-    exit_code=$?
-
-    if [ ${exit_code} -eq 0 ]; then
-        echo "Test integration: ${integration}"
-        ${ELASTIC_PACKAGE_BIN} test -v --report-format xUnit --report-output file --test-coverage
-        exit_code=$?
-    fi
-
-    if [ ${exit_code} -ne 0 ]; then
-        echo "[$integration] failed (exit code $exit_code)"
-    fi
-     set -e
+    check_install_and_test_package ${integration} || buildkite-agent annotate "Package ${integration} failed" --style "error"
 
     # TODO: add benchmarks support (https://github.com/elastic/integrations/blob/befdc5cb752a08aaf5f79b0d9bdb68588ade9f27/.ci/Jenkinsfile#L180)
     # ${ELASTIC_PACKAGE_BIN} benchmark pipeline -v --report-format json --report-output file

From a8f4b2bb2fb521d8757827469b6f4c7db1ba53fe Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 17 Oct 2023 16:59:50 +0200
Subject: [PATCH 059/139] Add test-results xml as artifact paths

---
 .buildkite/pipeline.yml      | 12 ++++++------
 .buildkite/scripts/common.sh |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 8d47af4ee64..7e7f1b3c3be 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -35,8 +35,8 @@ steps:
         allow_failure: false
 
   # to be moved to schedule-daily job
-  - label: "Check integrations"
-    key: "test-integrations-sync"
+  - label: "Check integrations in serverless"
+    key: "trigger-integrations-serverless"
     command: ".buildkite/scripts/run_integrations_sync.sh"
     env:
       SERVERLESS: true
@@ -44,12 +44,12 @@ steps:
       FORCE_CHECK_ALL: true
     agents:
       provider: "gcp"
-    artifact_paths:
-      - build/results/*.xml
     depends_on:
       - step: "publish-packages"
         allow_failure: false
-          #
+    artifact_paths:
+      - build/test-results/*.xml
+
   # to be moved to schedule-daily job
   # - label: "Check integrations"
   #   key: "test-integrations-sync"
@@ -70,6 +70,6 @@ steps:
   - label: ":junit: Junit annotate"
     plugins:
       - junit-annotate#v2.4.1:
-          artifacts: "build/results/*.xml"
+          artifacts: "build/test-results/*.xml"
     agents:
       provider: "gcp"  # junit plugin requires docker
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 38fe1d7c70d..7ee48c42ce9 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -430,7 +430,7 @@ list_all_directories() {
     find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {}
 }
 
-check_and_test_packages() {
+check_install_and_test_packages() {
     local integration=$1
     echo "Check integration: ${integration}"
     ${ELASTIC_PACKAGE_BIN} check -v

From ed99dc4a4158b3f0a5dba0e23567bdd996902e9c Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 17 Oct 2023 17:08:41 +0200
Subject: [PATCH 060/139] Fix hooks

---
 .buildkite/hooks/pre-command |  2 +-
 .buildkite/hooks/pre-exit    |  4 +---
 .buildkite/pipeline.yml      | 17 +++++++++--------
 3 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 29e16f50447..eb3c6cff9f0 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -41,7 +41,7 @@ fi
 
 # TODO: add schedule-daily too
 if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
-    if [ ${BUILDKITE_STEP_KEY} == "test-integrations-sync" ]; then
+    if [ ${BUILDKITE_STEP_KEY} == "test-integrations-serverless" ]; then
         export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
         export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
 
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index d2504b7469e..76dea960a9a 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -5,7 +5,7 @@ source .buildkite/scripts/common.sh
 set -euo pipefail
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
-    if [[ "$BUILDKITE_STEP_KEY" == "test-integrations-sync" ]]; then
+    if [[ "$BUILDKITE_STEP_KEY" == "test-integrations-serverless" ]]; then
         unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
         unset ELASTIC_PACKAGE_AWS_SECRET_KEY
         unset AWS_ACCESS_KEY_ID
@@ -29,5 +29,3 @@ fi
 
 unset_secrets
 cleanup
-
-
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 7e7f1b3c3be..0ced61f7625 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -25,14 +25,15 @@ steps:
       cpu: "8"
       memory: "4G"
 
-  - label: "Publish packages"
-    key: "publish-packages"
-    command: ".buildkite/scripts/publish_packages.sh" # TODO: missing signature and publishing loops
-    agents:
-      provider: "gcp"
-    depends_on:
-      - step: "check"
-        allow_failure: false
+  # TODO: remove comments
+  # - label: "Publish packages"
+  #   key: "publish-packages"
+  #   command: ".buildkite/scripts/publish_packages.sh" # TODO: missing signature and publishing loops
+  #   agents:
+  #     provider: "gcp"
+  #   depends_on:
+  #     - step: "check"
+  #       allow_failure: false
 
   # to be moved to schedule-daily job
   - label: "Check integrations in serverless"

From cf400727b07b0f64acd406898e611aa575e9c418 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 17 Oct 2023 17:27:14 +0200
Subject: [PATCH 061/139] Change depends_on temporarily

---
 .buildkite/pipeline.yml | 18 ++----------------
 1 file changed, 2 insertions(+), 16 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 0ced61f7625..99adf3091b1 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -46,24 +46,10 @@ steps:
     agents:
       provider: "gcp"
     depends_on:
-      - step: "publish-packages"
+      - step: "check"   # TODO: change to publish-packages if re-added
         allow_failure: false
     artifact_paths:
-      - build/test-results/*.xml
-
-  # to be moved to schedule-daily job
-  # - label: "Check integrations"
-  #   key: "test-integrations-sync"
-  #   command: ".buildkite/scripts/run_integrations_sync.sh"
-  #   env:
-  #     SERVERLESS: true
-  #     SERVERLESS_PROJECT: security
-  #     FORCE_CHECK_ALL: true
-  #   agents:
-  #     provider: "gcp"
-  #   depends_on:
-  #     - step: "publish-packages"
-  #       allow_failure: false
+      - "build/test-results/*.xml"
 
   - wait: ~
     continue_on_failure: true

From d2efeb3ff7981d21d9fcd4bdf37da4def4bd6382 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 17 Oct 2023 17:34:56 +0200
Subject: [PATCH 062/139] Update key

---
 .buildkite/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 99adf3091b1..295075b119c 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -37,7 +37,7 @@ steps:
 
   # to be moved to schedule-daily job
   - label: "Check integrations in serverless"
-    key: "trigger-integrations-serverless"
+    key: "test-integrations-serverless"
     command: ".buildkite/scripts/run_integrations_sync.sh"
     env:
       SERVERLESS: true

From 570b14d0627ff6b69632dc48abda03ad26aefc29 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 17 Oct 2023 18:17:20 +0200
Subject: [PATCH 063/139] Fix typo in function name

---
 .buildkite/scripts/common.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 7ee48c42ce9..a4730b7ecd2 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -430,7 +430,7 @@ list_all_directories() {
     find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {}
 }
 
-check_install_and_test_packages() {
+check_install_and_test_package() {
     local integration=$1
     echo "Check integration: ${integration}"
     ${ELASTIC_PACKAGE_BIN} check -v

From 0125ff954657b1c4a3ba172f043a2b2f5671c44e Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 17 Oct 2023 18:33:10 +0200
Subject: [PATCH 064/139] Test with all packages

---
 .buildkite/scripts/run_integrations_sync.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 17ea4e8f4fd..0c3c7458e73 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -30,8 +30,8 @@ use_elastic_package
 prepare_serverless_stack
 
 packages_visited=0
-num_packages=0  # TODO: to be removed
-maximum_packages=25
+# num_packages=0  # TODO: to be removed
+# maximum_packages=25
 
 pushd packages > /dev/null
 
@@ -75,10 +75,10 @@ for integration in $(list_all_directories); do
     popd > /dev/null
 
     # TODO: debug to be removed
-    num_packages=$((num_packages+1))
-    if [ $num_packages -eq ${maximum_packages} ]; then
-        break
-    fi
+    # num_packages=$((num_packages+1))
+    # if [ $num_packages -eq ${maximum_packages} ]; then
+    #     break
+    # fi
 done
 popd > /dev/null
 

From 1234f6c7b0ce6cde37096ae8f782d10c99bc9c6a Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 17 Oct 2023 19:16:51 +0200
Subject: [PATCH 065/139] Add more info in annotations

---
 .buildkite/scripts/run_integrations_sync.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 0c3c7458e73..d33d1d9a486 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -51,6 +51,7 @@ for integration in $(list_all_directories); do
 
     if ! is_pr_affected ${integration} ; then
         echo "[${integration}] Skipped"
+        buildkite-agent annotate "Package ${integration} skipped" --context "ctx-skipped-packages" --style "info" --append
         popd > /dev/null
         continue
     fi
@@ -61,7 +62,7 @@ for integration in $(list_all_directories); do
         create_kind_cluster
     fi
 
-    check_install_and_test_package ${integration} || buildkite-agent annotate "Package ${integration} failed" --style "error"
+    check_install_and_test_package ${integration} || buildkite-agent annotate "Package ${integration} failed" --context "ctx-failed-packages" --style "error" --append
 
     # TODO: add benchmarks support (https://github.com/elastic/integrations/blob/befdc5cb752a08aaf5f79b0d9bdb68588ade9f27/.ci/Jenkinsfile#L180)
     # ${ELASTIC_PACKAGE_BIN} benchmark pipeline -v --report-format json --report-output file

From e862f84d9a714446f7a35adbb136cb1b1be4dfd1 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 17 Oct 2023 19:26:47 +0200
Subject: [PATCH 066/139] Add stack status

---
 .buildkite/scripts/common.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index a4730b7ecd2..c4abf386f74 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -315,6 +315,8 @@ prepare_serverless_stack() {
     # ${ELASTIC_PACKAGE_BIN} stack up -d ${args} --provider serverless -U stack.serverless.region=${EC_REGION_SECRET} -U stack.serverless.type=${SERVERLESS_PROJECT}
     ${ELASTIC_PACKAGE_BIN} stack up -d ${args}
     echo ""
+    ${ELASTIC_PACKAGE_BIN} stack status
+    echo ""
 }
 
 is_spec_3_0_0() {

From 8cc8602caff160b01a639c212fa10e55370603ff Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 17 Oct 2023 19:30:51 +0200
Subject: [PATCH 067/139] Sort folders

---
 .buildkite/scripts/common.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index c4abf386f74..178c6439314 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -429,7 +429,7 @@ teardown_test_package() {
 }
 
 list_all_directories() {
-    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {}
+    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort
 }
 
 check_install_and_test_package() {

From 31fca7f45e5a76984d1a5ec830f14f3586d93c23 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 17 Oct 2023 20:06:04 +0200
Subject: [PATCH 068/139] Test annotations

---
 .buildkite/scripts/common.sh                |  2 +-
 .buildkite/scripts/run_integrations_sync.sh | 13 +++++++++++--
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 178c6439314..9f8920037eb 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -429,7 +429,7 @@ teardown_test_package() {
 }
 
 list_all_directories() {
-    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort
+    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |egrep "logstash|azure|osquery_manager|nginx"
 }
 
 check_install_and_test_package() {
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index d33d1d9a486..4d94fdec330 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -51,7 +51,7 @@ for integration in $(list_all_directories); do
 
     if ! is_pr_affected ${integration} ; then
         echo "[${integration}] Skipped"
-        buildkite-agent annotate "Package ${integration} skipped" --context "ctx-skipped-packages" --style "info" --append
+        echo "- ${integration}" >> skipped_packages.txt
         popd > /dev/null
         continue
     fi
@@ -62,7 +62,7 @@ for integration in $(list_all_directories); do
         create_kind_cluster
     fi
 
-    check_install_and_test_package ${integration} || buildkite-agent annotate "Package ${integration} failed" --context "ctx-failed-packages" --style "error" --append
+    check_install_and_test_package ${integration} || echo "- ${integration}" >> failed_packages.txt
 
     # TODO: add benchmarks support (https://github.com/elastic/integrations/blob/befdc5cb752a08aaf5f79b0d9bdb68588ade9f27/.ci/Jenkinsfile#L180)
     # ${ELASTIC_PACKAGE_BIN} benchmark pipeline -v --report-format json --report-output file
@@ -83,4 +83,13 @@ for integration in $(list_all_directories); do
 done
 popd > /dev/null
 
+if [ -f skipped_packages.txt ]; then
+    sed -i '1s/^/Skipped packages:\n/' skipped_packages.txt
+    cat skipped_packages.txt | buildkite-agetn annotate --style info --append --ontext "ctx-skipped-packages"
+fi
+
+if [ -f failed_packages.txt ]; then
+    sed -i '1s/^/Failed packages:\n/' failed_packages.txt
+    cat failed_packages.txt | buildkite-agetn annotate --style error --append --context "ctx-failed-packages"
+fi
 echo "Total packages examined: ${packages_visited}"

From fb789c34a92af04e4f985efd3c6e672546a4dab1 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 17 Oct 2023 20:11:17 +0200
Subject: [PATCH 069/139] Add more debug messages

---
 .buildkite/scripts/common.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 9f8920037eb..a046150b5f5 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -241,21 +241,27 @@ is_supported_capability() {
 
     # if no capabilities defined, it is available iavailable all projects
     if [[  "${capabilities}" == "null" ]]; then
+        echo "No capabilities defined"
         return 0
     fi
 
+    echo "Project: ${SERVERLESS_PROJECT} capabilities: ${capabilities}"
     if [[ ${SERVERLESS_PROJECT} == "observability" ]]; then
         if echo ${capabilities} |egrep 'apm|observability|uptime' ; then
+            echo "Supported"
             return 0
         else
+            echo "Not Supported"
             return 1
         fi
     fi
 
     if [[ ${SERVERLESS_PROJECT} == "security" ]]; then
         if echo ${capabilities} |egrep 'security' ; then
+            echo "Supported"
             return 0
         else
+            echo "Not Supported"
             return 1
         fi
     fi

From 4868d6a01ee5c1817f803c642c2d813c8ba936a1 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 10:39:49 +0200
Subject: [PATCH 070/139] Fix typo

---
 .buildkite/scripts/common.sh                | 2 +-
 .buildkite/scripts/run_integrations_sync.sh | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index a046150b5f5..fe26ed62212 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -229,7 +229,7 @@ kibana_version_manifest() {
 }
 
 capabilities_manifest() {
-    cat manifest.yml | yq ".conditions.elastic.capabilitites"
+    cat manifest.yml | yq ".conditions.elastic.capabilities"
 }
 
 is_supported_capability() {
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 4d94fdec330..5f35d805b60 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -62,7 +62,9 @@ for integration in $(list_all_directories); do
         create_kind_cluster
     fi
 
-    check_install_and_test_package ${integration} || echo "- ${integration}" >> failed_packages.txt
+    if ! check_install_and_test_package ${integration} ; then
+        echo "- ${integration}" >> failed_packages.txt
+    fi
 
     # TODO: add benchmarks support (https://github.com/elastic/integrations/blob/befdc5cb752a08aaf5f79b0d9bdb68588ade9f27/.ci/Jenkinsfile#L180)
     # ${ELASTIC_PACKAGE_BIN} benchmark pipeline -v --report-format json --report-output file
@@ -84,11 +86,13 @@ done
 popd > /dev/null
 
 if [ -f skipped_packages.txt ]; then
+    echo "Found skipped_packages.txt"  # TODO: remove
     sed -i '1s/^/Skipped packages:\n/' skipped_packages.txt
     cat skipped_packages.txt | buildkite-agetn annotate --style info --append --ontext "ctx-skipped-packages"
 fi
 
 if [ -f failed_packages.txt ]; then
+    echo "Found failed_packages.txt"  # TODO: remove
     sed -i '1s/^/Failed packages:\n/' failed_packages.txt
     cat failed_packages.txt | buildkite-agetn annotate --style error --append --context "ctx-failed-packages"
 fi

From 96480bc07ff1c5811c04b3ff88e7a44eeaa603f9 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 11:02:30 +0200
Subject: [PATCH 071/139] Add variables for skipped and failed paths

---
 .buildkite/scripts/run_integrations_sync.sh | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 5f35d805b60..5331de3bc8b 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -12,6 +12,9 @@ SERVERLESS=${SERVERLESS:-"false"}
 SERVERLESS_PROJECT=${SERVERLESS_PROJECT:-"observability"}
 
 
+SKIPPED_PACKAGES_FILE_PATH="${WORKSPACE}/skipped_packages.txt"
+FAILED_PACKAGES_FILE_PATH="${WORKSPACE}/failed_packages.txt"
+
 if [ ! -d packages ]; then
     echo "Missing packages folder"
     buildkite-agent annotate "Missing packages folder" --style "error"
@@ -44,6 +47,7 @@ for integration in $(list_all_directories); do
     if [[ ${SERVERLESS} == "true" ]] ; then
         if ! is_spec_3_0_0 ]]; then
             echo "Not v3 spec version. Skipped"
+            echo "- ${integration} (spec <3.0.0)" >> ${SKIPPED_PACKAGES_FILE_PATH}
             popd > /dev/null
             continue
         fi
@@ -51,7 +55,7 @@ for integration in $(list_all_directories); do
 
     if ! is_pr_affected ${integration} ; then
         echo "[${integration}] Skipped"
-        echo "- ${integration}" >> skipped_packages.txt
+        echo "- ${integration}" >> ${SKIPPED_PACKAGES_FILE_PATH}
         popd > /dev/null
         continue
     fi
@@ -63,7 +67,7 @@ for integration in $(list_all_directories); do
     fi
 
     if ! check_install_and_test_package ${integration} ; then
-        echo "- ${integration}" >> failed_packages.txt
+        echo "- ${integration}" >> ${FAILED_PACKAGES_FILE_PATH}
     fi
 
     # TODO: add benchmarks support (https://github.com/elastic/integrations/blob/befdc5cb752a08aaf5f79b0d9bdb68588ade9f27/.ci/Jenkinsfile#L180)
@@ -85,15 +89,15 @@ for integration in $(list_all_directories); do
 done
 popd > /dev/null
 
-if [ -f skipped_packages.txt ]; then
+if [ -f ${SKIPPED_PACKAGES_FILE_PATH} ]; then
     echo "Found skipped_packages.txt"  # TODO: remove
-    sed -i '1s/^/Skipped packages:\n/' skipped_packages.txt
-    cat skipped_packages.txt | buildkite-agetn annotate --style info --append --ontext "ctx-skipped-packages"
+    sed -i '1s/^/Skipped packages:\n/' ${SKIPPED_PACKAGES_FILE_PATH}
+    cat ${SKIPPED_PACKAGES_FILE_PATH} | buildkite-agent annotate --style info --append --ontext "ctx-skipped-packages"
 fi
 
-if [ -f failed_packages.txt ]; then
+if [ -f ${FAILED_PACKAGES_FILE_PATH} ]; then
     echo "Found failed_packages.txt"  # TODO: remove
-    sed -i '1s/^/Failed packages:\n/' failed_packages.txt
-    cat failed_packages.txt | buildkite-agetn annotate --style error --append --context "ctx-failed-packages"
+    sed -i '1s/^/Failed packages:\n/' ${FAILED_PACKAGES_FILE_PATH}
+    cat ${FAILED_PACKAGES_FILE_PATH} | buildkite-agent annotate --style error --append --context "ctx-failed-packages"
 fi
 echo "Total packages examined: ${packages_visited}"

From 2c95411361667dc620cdf90f081143baf88ababf Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 11:13:40 +0200
Subject: [PATCH 072/139] Fix parameter

---
 .buildkite/scripts/run_integrations_sync.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 5331de3bc8b..621aef56d9e 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -92,7 +92,7 @@ popd > /dev/null
 if [ -f ${SKIPPED_PACKAGES_FILE_PATH} ]; then
     echo "Found skipped_packages.txt"  # TODO: remove
     sed -i '1s/^/Skipped packages:\n/' ${SKIPPED_PACKAGES_FILE_PATH}
-    cat ${SKIPPED_PACKAGES_FILE_PATH} | buildkite-agent annotate --style info --append --ontext "ctx-skipped-packages"
+    cat ${SKIPPED_PACKAGES_FILE_PATH} | buildkite-agent annotate --style info --append --context "ctx-skipped-packages"
 fi
 
 if [ -f ${FAILED_PACKAGES_FILE_PATH} ]; then

From 53bb5162c55a6015ce2c9a1d4039919a6816a02d Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 11:57:01 +0200
Subject: [PATCH 073/139] Update reason skipped

---
 .buildkite/scripts/common.sh                | 23 +++++++++++++++++++--
 .buildkite/scripts/run_integrations_sync.sh | 20 ++++--------------
 2 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index fe26ed62212..99f2a07e811 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -350,6 +350,8 @@ get_to_changeset() {
     echo "${BUILDKITE_COMMIT}"
 }
 
+# TODO: it is required to have GIT_PREVIOUS_COMMIT and GIT_PREVIOUS_SUCCESSFUL_COMMIT
+# as in Jenkins to set the right from (changesets)
 is_pr_affected() {
     local integration="${1}"
 
@@ -438,17 +440,34 @@ list_all_directories() {
     find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |egrep "logstash|azure|osquery_manager|nginx"
 }
 
-check_install_and_test_package() {
+check_package() {
     local integration=$1
     echo "Check integration: ${integration}"
     ${ELASTIC_PACKAGE_BIN} check -v
+    echo ""
+}
 
+install_package() {
+    local integration=$1
     echo "Install integration: ${integration}"
     ${ELASTIC_PACKAGE_BIN} install -v
+    echo ""
+}
 
-    echo "Test integration: ${integration}"
+test_package_no_system() {
+    local integration=$1
     TEST_OPTIONS="-v --report-format xUnit --report-output file --test-coverage"
+
+    echo "Test integration: ${integration}"
     ${ELASTIC_PACKAGE_BIN} test asset ${TEST_OPTIONS}
     ${ELASTIC_PACKAGE_BIN} test static ${TEST_OPTIONS}
     ${ELASTIC_PACKAGE_BIN} test pipeline ${TEST_OPTIONS}
+    echo ""
+}
+
+run_tests_package() {
+    local integration=$1
+    check_package ${integration}
+    install_package ${integration}
+    test_package_no_system ${integration}
 }
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 621aef56d9e..9141ce40843 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -32,17 +32,12 @@ use_elastic_package
 
 prepare_serverless_stack
 
-packages_visited=0
-# num_packages=0  # TODO: to be removed
-# maximum_packages=25
-
 pushd packages > /dev/null
 
 for integration in $(list_all_directories); do
     echo "--- Package ${integration}: check"
 
     pushd ${integration} > /dev/null
-    packages_visited=$((packages_visited+1))
 
     if [[ ${SERVERLESS} == "true" ]] ; then
         if ! is_spec_3_0_0 ]]; then
@@ -53,9 +48,9 @@ for integration in $(list_all_directories); do
         fi
     fi
 
-    if ! is_pr_affected ${integration} ; then
-        echo "[${integration}] Skipped"
-        echo "- ${integration}" >> ${SKIPPED_PACKAGES_FILE_PATH}
+    if ! reason=$(is_pr_affected ${integration}) ; then
+        echo "${reason}"
+        echo "- ${reason}" >> ${SKIPPED_PACKAGES_FILE_PATH}
         popd > /dev/null
         continue
     fi
@@ -66,7 +61,7 @@ for integration in $(list_all_directories); do
         create_kind_cluster
     fi
 
-    if ! check_install_and_test_package ${integration} ; then
+    if ! run_tests_package ${integration} ; then
         echo "- ${integration}" >> ${FAILED_PACKAGES_FILE_PATH}
     fi
 
@@ -80,12 +75,6 @@ for integration in $(list_all_directories); do
     teardown_serverless_test_package ${integration}
 
     popd > /dev/null
-
-    # TODO: debug to be removed
-    # num_packages=$((num_packages+1))
-    # if [ $num_packages -eq ${maximum_packages} ]; then
-    #     break
-    # fi
 done
 popd > /dev/null
 
@@ -100,4 +89,3 @@ if [ -f ${FAILED_PACKAGES_FILE_PATH} ]; then
     sed -i '1s/^/Failed packages:\n/' ${FAILED_PACKAGES_FILE_PATH}
     cat ${FAILED_PACKAGES_FILE_PATH} | buildkite-agent annotate --style error --append --context "ctx-failed-packages"
 fi
-echo "Total packages examined: ${packages_visited}"

From fdc9a8f206f351d95e687880d4e1c3ea378f7164 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 12:18:35 +0200
Subject: [PATCH 074/139] Add step for security

---
 .buildkite/hooks/pre-command           | 15 +++++++++++++
 .buildkite/hooks/pre-exit              | 25 ++++++++++++++++++++-
 .buildkite/pipeline.schedule-daily.yml | 31 ++++++++++++++++++++++++--
 .buildkite/pipeline.yml                | 19 ++++++++++++++--
 4 files changed, 85 insertions(+), 5 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index eb3c6cff9f0..94e999d65c9 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -41,6 +41,21 @@ fi
 
 # TODO: add schedule-daily too
 if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
+    if [[ ${BUILDKITE_STEP_KEY} =~ test-integrations-serverless-(obs|security) ]]; then
+        export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+        export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+
+        # Environment variables required by the service deployer
+        export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
+        export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
+
+        export EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey ${EC_TOKEN_PATH})
+        export EC_HOST_SECRET=$(retry 5 vault kv get -field url ${EC_TOKEN_PATH})
+        export EC_REGION_SECRET=$(retry 5 vault read -field region_qa ${EC_DATA_PATH})
+    fi
+fi
+
+if [ ${BUILDKITE_PIPELINE_SLUG} == "schedule-daily" ]; then
     if [ ${BUILDKITE_STEP_KEY} == "test-integrations-serverless" ]; then
         export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
         export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index 76dea960a9a..1fed2e0f1ff 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -5,7 +5,30 @@ source .buildkite/scripts/common.sh
 set -euo pipefail
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
-    if [[ "$BUILDKITE_STEP_KEY" == "test-integrations-serverless" ]]; then
+    if [[ "$BUILDKITE_STEP_KEY" =~ test-integrations-serverless-(obs|security) ]]; then
+        unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
+        unset ELASTIC_PACKAGE_AWS_SECRET_KEY
+        unset AWS_ACCESS_KEY_ID
+        unset AWS_SECRET_ACCESS_KEY
+
+        # Ensure that kind cluster is deleted
+        delete_kind_cluster
+
+        # Ensure elastic stack is stopped
+        if [ -f ${ELASTIC_PACKAGE_BIN} ]; then
+            export EC_API_KEY=${EC_API_KEY_SECRET}
+            export EC_HOST=${EC_HOST_SECRET}
+
+            ${ELASTIC_PACKAGE_BIN} stack down -v
+
+            unset EC_API_KEY
+            unset EC_HOST
+        fi
+    fi
+fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-schedule-daily" ]]; then
+    if [[ "$BUILDKITE_STEP_KEY" =~ test-integrations-serverless-(obs|security) ]]; then
         unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
         unset ELASTIC_PACKAGE_AWS_SECRET_KEY
         unset AWS_ACCESS_KEY_ID
diff --git a/.buildkite/pipeline.schedule-daily.yml b/.buildkite/pipeline.schedule-daily.yml
index 9f300156c10..49be1498f4a 100644
--- a/.buildkite/pipeline.schedule-daily.yml
+++ b/.buildkite/pipeline.schedule-daily.yml
@@ -6,5 +6,32 @@ env:
 
 # The pipeline is triggered by the scheduler every day
 steps:
-  - label: "Placeholder"
-    command: echo "Hello!"
+  - label: ":white_check_mark: Check go sources"
+    key: "check"
+    command: ".buildkite/scripts/check_sources.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+
+  - label: "Check integrations in serverless - project: Observability"
+    key: "trigger-integrations-serverless-obs"
+    command: ".buildkite/scripts/run_integrations_sync.sh"
+    env:
+      SERVERLESS: true
+      SERVERLESS_PROJECT: observability
+      FORCE_CHECK_ALL: true
+    depends_on:
+      - step: "check-sources"
+        allow_failure: false
+
+  - label: "Check integrations in serverless - project: security"
+    key: "trigger-integrations-serverless-security"
+    command: ".buildkite/scripts/run_integrations_sync.sh"
+    env:
+      SERVERLESS: true
+      SERVERLESS_PROJECT: security
+      FORCE_CHECK_ALL: true
+    depends_on:
+      - step: "check-sources"
+        allow_failure: false
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 295075b119c..7a4e3ae6bba 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -36,8 +36,8 @@ steps:
   #       allow_failure: false
 
   # to be moved to schedule-daily job
-  - label: "Check integrations in serverless"
-    key: "test-integrations-serverless"
+  - label: "Check integrations in serverless - project observability"
+    key: "test-integrations-serverless-obs"
     command: ".buildkite/scripts/run_integrations_sync.sh"
     env:
       SERVERLESS: true
@@ -51,6 +51,21 @@ steps:
     artifact_paths:
       - "build/test-results/*.xml"
 
+  - label: "Check integrations in serverless - project security"
+    key: "test-integrations-serverless-security"
+    command: ".buildkite/scripts/run_integrations_sync.sh"
+    env:
+      SERVERLESS: true
+      SERVERLESS_PROJECT: security
+      FORCE_CHECK_ALL: true
+    agents:
+      provider: "gcp"
+    depends_on:
+      - step: "check"   # TODO: change to publish-packages if re-added
+        allow_failure: false
+    artifact_paths:
+      - "build/test-results/*.xml"
+
   - wait: ~
     continue_on_failure: true
 

From 5aff1b5005f563cb358f2d8f6cba95a88eefa8cc Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 12:24:42 +0200
Subject: [PATCH 075/139] Remove debug messages about capabilities

---
 .buildkite/scripts/common.sh | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 99f2a07e811..c8332c3d2b9 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -241,32 +241,25 @@ is_supported_capability() {
 
     # if no capabilities defined, it is available iavailable all projects
     if [[  "${capabilities}" == "null" ]]; then
-        echo "No capabilities defined"
         return 0
     fi
 
-    echo "Project: ${SERVERLESS_PROJECT} capabilities: ${capabilities}"
     if [[ ${SERVERLESS_PROJECT} == "observability" ]]; then
         if echo ${capabilities} |egrep 'apm|observability|uptime' ; then
-            echo "Supported"
             return 0
         else
-            echo "Not Supported"
             return 1
         fi
     fi
 
     if [[ ${SERVERLESS_PROJECT} == "security" ]]; then
         if echo ${capabilities} |egrep 'security' ; then
-            echo "Supported"
             return 0
         else
-            echo "Not Supported"
             return 1
         fi
     fi
 
-    echo "Unsupported serverless project"
     return 1
 }
 

From 48d4933d21ce9f8d2b67c56f669ce0d332d94eb3 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 12:27:42 +0200
Subject: [PATCH 076/139] Adapt annotations

---
 .buildkite/scripts/run_integrations_sync.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 9141ce40843..21f83814bb6 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -80,12 +80,12 @@ popd > /dev/null
 
 if [ -f ${SKIPPED_PACKAGES_FILE_PATH} ]; then
     echo "Found skipped_packages.txt"  # TODO: remove
-    sed -i '1s/^/Skipped packages:\n/' ${SKIPPED_PACKAGES_FILE_PATH}
-    cat ${SKIPPED_PACKAGES_FILE_PATH} | buildkite-agent annotate --style info --append --context "ctx-skipped-packages"
+    sed -i "1s/^/Skipped packages in ${SERVERLESS_PROJECT}:\n/" ${SKIPPED_PACKAGES_FILE_PATH}
+    cat ${SKIPPED_PACKAGES_FILE_PATH} | buildkite-agent annotate --style info --append --context "ctx-skipped-packages-${SERVERLESS_PROJECT}"
 fi
 
 if [ -f ${FAILED_PACKAGES_FILE_PATH} ]; then
     echo "Found failed_packages.txt"  # TODO: remove
-    sed -i '1s/^/Failed packages:\n/' ${FAILED_PACKAGES_FILE_PATH}
-    cat ${FAILED_PACKAGES_FILE_PATH} | buildkite-agent annotate --style error --append --context "ctx-failed-packages"
+    sed -i "1s/^/Failed packages in ${SERVERLESS_PROJECT}:\n/" ${FAILED_PACKAGES_FILE_PATH}
+    cat ${FAILED_PACKAGES_FILE_PATH} | buildkite-agent annotate --style error --append --context "ctx-failed-packages-${SERVERLESS_PROJECT}"
 fi

From 541078f4606fb94665076df0d7775491b187e0cc Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 13:11:04 +0200
Subject: [PATCH 077/139] Keep same message for all skipped packages

---
 .buildkite/scripts/run_integrations_sync.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 21f83814bb6..97fe39433c4 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -42,7 +42,7 @@ for integration in $(list_all_directories); do
     if [[ ${SERVERLESS} == "true" ]] ; then
         if ! is_spec_3_0_0 ]]; then
             echo "Not v3 spec version. Skipped"
-            echo "- ${integration} (spec <3.0.0)" >> ${SKIPPED_PACKAGES_FILE_PATH}
+            echo "- [${integration}] spec <3.0.0" >> ${SKIPPED_PACKAGES_FILE_PATH}
             popd > /dev/null
             continue
         fi

From 1146f0033cd1b32fdfb6fddc74609b6ec5742a42 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 13:11:35 +0200
Subject: [PATCH 078/139] Test with all packages

---
 .buildkite/scripts/common.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index c8332c3d2b9..7fa9008de20 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -430,7 +430,7 @@ teardown_test_package() {
 }
 
 list_all_directories() {
-    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |egrep "logstash|azure|osquery_manager|nginx"
+    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort #|egrep "logstash|azure|osquery_manager|nginx"
 }
 
 check_package() {

From 51817e60b09901ff82542e66bba826b417b417e0 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 14:45:19 +0200
Subject: [PATCH 079/139] Move message

---
 .buildkite/scripts/common.sh                | 1 -
 .buildkite/scripts/run_integrations_sync.sh | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 7fa9008de20..14c32cedd88 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -402,7 +402,6 @@ is_pr() {
 }
 
 kubernetes_service_deployer_used() {
-    echo "Check if Kubernetes service deployer is used"
     find . -type d | egrep '_dev/deploy/k8s$'
 }
 
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 97fe39433c4..5ede16f6b3b 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -57,6 +57,7 @@ for integration in $(list_all_directories); do
 
     use_kind=0
     if kubernetes_service_deployer_used ; then
+        echo "Kubernetes service deployer is used. Creating Kind cluster"
         use_kind=1
         create_kind_cluster
     fi

From f1f0e7fc2e8c1ab825548567ec00cc5e08e1ff43 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 14:49:24 +0200
Subject: [PATCH 080/139] Test just with observability

---
 .buildkite/pipeline.yml | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 7a4e3ae6bba..ab0e62f9dc2 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -51,20 +51,20 @@ steps:
     artifact_paths:
       - "build/test-results/*.xml"
 
-  - label: "Check integrations in serverless - project security"
-    key: "test-integrations-serverless-security"
-    command: ".buildkite/scripts/run_integrations_sync.sh"
-    env:
-      SERVERLESS: true
-      SERVERLESS_PROJECT: security
-      FORCE_CHECK_ALL: true
-    agents:
-      provider: "gcp"
-    depends_on:
-      - step: "check"   # TODO: change to publish-packages if re-added
-        allow_failure: false
-    artifact_paths:
-      - "build/test-results/*.xml"
+  # - label: "Check integrations in serverless - project security"
+  #   key: "test-integrations-serverless-security"
+  #   command: ".buildkite/scripts/run_integrations_sync.sh"
+  #   env:
+  #     SERVERLESS: true
+  #     SERVERLESS_PROJECT: security
+  #     FORCE_CHECK_ALL: true
+  #   agents:
+  #     provider: "gcp"
+  #   depends_on:
+  #     - step: "check"   # TODO: change to publish-packages if re-added
+  #       allow_failure: false
+  #   artifact_paths:
+  #     - "build/test-results/*.xml"
 
   - wait: ~
     continue_on_failure: true

From b0dac2a7f6ef62afedb9839f330701fca1ff910a Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 15:10:14 +0200
Subject: [PATCH 081/139] Create elastic-package profile

Creating a new profile makes that the serverless project created in the
cloud would also change the name.
---
 .buildkite/hooks/pre-command |  2 +-
 .buildkite/scripts/common.sh | 14 +++++++++++---
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 94e999d65c9..f2f25b1e929 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -56,7 +56,7 @@ if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
 fi
 
 if [ ${BUILDKITE_PIPELINE_SLUG} == "schedule-daily" ]; then
-    if [ ${BUILDKITE_STEP_KEY} == "test-integrations-serverless" ]; then
+    if [[ ${BUILDKITE_STEP_KEY} =~ test-integrations-serverless-(obs|security) ]]; then
         export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
         export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
 
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 14c32cedd88..e6150d2ebd5 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -297,6 +297,12 @@ oldest_supported_version() {
     echo "null"
 }
 
+create_elastic_package_profile() {
+    local name="$1"
+    ${ELASTIC_PACKAGE_BIN} profiles create "${name}"
+    ${ELASTIC_PACKAGE_BIN} profiles use "${name}"
+}
+
 prepare_serverless_stack() {
     echo "--- Prepare serverless stack"
 
@@ -307,12 +313,14 @@ prepare_serverless_stack() {
     # else
     fi
 
+    create_elastic_package_profile "integrations-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_BUILD_NUMBER}"
+
     export EC_API_KEY=${EC_API_KEY_SECRET}
     export EC_HOST=${EC_HOST_SECRET}
 
     echo "Boot up the Elastic stack"
-    # ${ELASTIC_PACKAGE_BIN} stack up -d ${args} --provider serverless -U stack.serverless.region=${EC_REGION_SECRET} -U stack.serverless.type=${SERVERLESS_PROJECT}
-    ${ELASTIC_PACKAGE_BIN} stack up -d ${args}
+    ${ELASTIC_PACKAGE_BIN} stack up -d ${args} --provider serverless -U stack.serverless.region=${EC_REGION_SECRET},stack.serverless.type=${SERVERLESS_PROJECT}
+    # ${ELASTIC_PACKAGE_BIN} stack up -d ${args}
     echo ""
     ${ELASTIC_PACKAGE_BIN} stack status
     echo ""
@@ -429,7 +437,7 @@ teardown_test_package() {
 }
 
 list_all_directories() {
-    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort #|egrep "logstash|azure|osquery_manager|nginx"
+    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |egrep "logstash|azure|osquery_manager|nginx"
 }
 
 check_package() {

From 0dcdb3052088b1ab698f2ee3c91514c22c4806fc Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 15:27:09 +0200
Subject: [PATCH 082/139] Add collapsed annotations

---
 .buildkite/scripts/common.sh                | 16 ++++++++++++++++
 .buildkite/scripts/run_integrations_sync.sh |  6 ++----
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index e6150d2ebd5..d56f098f0de 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -471,3 +471,19 @@ run_tests_package() {
     install_package ${integration}
     test_package_no_system ${integration}
 }
+
+create_collapsed_annotation() {
+    local title="$1"
+    local file="$2"
+    local style="$3"
+    local context="$4"
+
+    local annotation_file="tmp.annotation.md"
+    echo "<details><summary>${title}</summary>" >> ${annotation_file}
+    cat ${file} >> ${annotation_file}
+    echo "</details>" >> ${annotation_file}
+
+    cat ${annotation_file} | buildkite-agent annotate --style "${style}" --context "${context}"
+
+    rm -f ${annotation_file}
+}
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 5ede16f6b3b..66e871b8ac5 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -81,12 +81,10 @@ popd > /dev/null
 
 if [ -f ${SKIPPED_PACKAGES_FILE_PATH} ]; then
     echo "Found skipped_packages.txt"  # TODO: remove
-    sed -i "1s/^/Skipped packages in ${SERVERLESS_PROJECT}:\n/" ${SKIPPED_PACKAGES_FILE_PATH}
-    cat ${SKIPPED_PACKAGES_FILE_PATH} | buildkite-agent annotate --style info --append --context "ctx-skipped-packages-${SERVERLESS_PROJECT}"
+    create_collapsed_annotation "Skipped packages in ${SERVERLESS_PROJECT}" ${SKIPPED_PACKAGES_FILE_PATH} "info" "ctx-skipped-packages-${SERVERLESS_PROJECT}"
 fi
 
 if [ -f ${FAILED_PACKAGES_FILE_PATH} ]; then
     echo "Found failed_packages.txt"  # TODO: remove
-    sed -i "1s/^/Failed packages in ${SERVERLESS_PROJECT}:\n/" ${FAILED_PACKAGES_FILE_PATH}
-    cat ${FAILED_PACKAGES_FILE_PATH} | buildkite-agent annotate --style error --append --context "ctx-failed-packages-${SERVERLESS_PROJECT}"
+    create_collapsed_annotation "Failed packages in ${SERVERLESS_PROJECT}" ${FAILED_PACKAGES_FILE_PATH} "error" "ctx-failed-packages-${SERVERLESS_PROJECT}"
 fi

From 1565dfd44807bbfff5a30be49b6607e8e3d9a0b4 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 15:27:58 +0200
Subject: [PATCH 083/139] Revert test serverless

---
 .buildkite/scripts/common.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index d56f098f0de..226af5672ea 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -319,8 +319,8 @@ prepare_serverless_stack() {
     export EC_HOST=${EC_HOST_SECRET}
 
     echo "Boot up the Elastic stack"
-    ${ELASTIC_PACKAGE_BIN} stack up -d ${args} --provider serverless -U stack.serverless.region=${EC_REGION_SECRET},stack.serverless.type=${SERVERLESS_PROJECT}
-    # ${ELASTIC_PACKAGE_BIN} stack up -d ${args}
+    # ${ELASTIC_PACKAGE_BIN} stack up -d ${args} --provider serverless -U stack.serverless.region=${EC_REGION_SECRET},stack.serverless.type=${SERVERLESS_PROJECT}
+    ${ELASTIC_PACKAGE_BIN} stack up -d ${args}
     echo ""
     ${ELASTIC_PACKAGE_BIN} stack status
     echo ""

From d6b5acf15fa71ddcbc71f827257015e686fae419 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 15:50:14 +0200
Subject: [PATCH 084/139] Add empty lines to test markdown in collapsed
 annotations

---
 .buildkite/scripts/common.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 226af5672ea..b8337e2fb99 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -480,6 +480,8 @@ create_collapsed_annotation() {
 
     local annotation_file="tmp.annotation.md"
     echo "<details><summary>${title}</summary>" >> ${annotation_file}
+    echo "" >> ${annotation_file}
+    echo "" >> ${annotation_file}
     cat ${file} >> ${annotation_file}
     echo "</details>" >> ${annotation_file}
 

From e6eabb6e9759dc2a348cbaf741ed4bb019076dec Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 16:03:42 +0200
Subject: [PATCH 085/139] First test with serverless project

---
 .buildkite/scripts/common.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index b8337e2fb99..51828fe0eea 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -319,8 +319,8 @@ prepare_serverless_stack() {
     export EC_HOST=${EC_HOST_SECRET}
 
     echo "Boot up the Elastic stack"
-    # ${ELASTIC_PACKAGE_BIN} stack up -d ${args} --provider serverless -U stack.serverless.region=${EC_REGION_SECRET},stack.serverless.type=${SERVERLESS_PROJECT}
-    ${ELASTIC_PACKAGE_BIN} stack up -d ${args}
+    ${ELASTIC_PACKAGE_BIN} stack up -d ${args} --provider serverless -U stack.serverless.region=${EC_REGION_SECRET},stack.serverless.type=${SERVERLESS_PROJECT}
+    # ${ELASTIC_PACKAGE_BIN} stack up -d ${args}
     echo ""
     ${ELASTIC_PACKAGE_BIN} stack status
     echo ""

From 827f75c99bb7278356489ba40d88b2c105af65a5 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 16:28:57 +0200
Subject: [PATCH 086/139] Remove password from output

---
 .buildkite/scripts/common.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 51828fe0eea..7d1deb3b80c 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -319,7 +319,11 @@ prepare_serverless_stack() {
     export EC_HOST=${EC_HOST_SECRET}
 
     echo "Boot up the Elastic stack"
-    ${ELASTIC_PACKAGE_BIN} stack up -d ${args} --provider serverless -U stack.serverless.region=${EC_REGION_SECRET},stack.serverless.type=${SERVERLESS_PROJECT}
+    ${ELASTIC_PACKAGE_BIN} stack up \
+        -d \
+        ${args} \
+        --provider serverless \
+        -U stack.serverless.region=${EC_REGION_SECRET},stack.serverless.type=${SERVERLESS_PROJECT} 2>&1 | egrep -v "^Password: " # To remove password from the output
     # ${ELASTIC_PACKAGE_BIN} stack up -d ${args}
     echo ""
     ${ELASTIC_PACKAGE_BIN} stack status
@@ -437,7 +441,7 @@ teardown_test_package() {
 }
 
 list_all_directories() {
-    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |egrep "logstash|azure|osquery_manager|nginx"
+    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |egrep "logstash|azure|osquery_manager|nginx|1password"
 }
 
 check_package() {

From 39ecda2d63d9ec648566038f8f68b3d4dadfcabc Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 17:20:49 +0200
Subject: [PATCH 087/139] Test all packages in observability project

---
 .buildkite/scripts/common.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 7d1deb3b80c..808041dbd8d 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -441,7 +441,7 @@ teardown_test_package() {
 }
 
 list_all_directories() {
-    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |egrep "logstash|azure|osquery_manager|nginx|1password"
+    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort # |egrep "logstash|azure|osquery_manager|nginx|1password"
 }
 
 check_package() {

From b3e1e6e70230b9468e0089cb1428064c393975cc Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 18:41:59 +0200
Subject: [PATCH 088/139] Add exit code if any package failed

---
 .buildkite/scripts/common.sh                |  2 +-
 .buildkite/scripts/run_integrations_sync.sh | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 808041dbd8d..79ab5c70fbb 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -441,7 +441,7 @@ teardown_test_package() {
 }
 
 list_all_directories() {
-    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort # |egrep "logstash|azure|osquery_manager|nginx|1password"
+    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort  |egrep "netskope|logstash"
 }
 
 check_package() {
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 66e871b8ac5..2f21a66aeb4 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -34,6 +34,8 @@ prepare_serverless_stack
 
 pushd packages > /dev/null
 
+any_package_failing=0
+
 for integration in $(list_all_directories); do
     echo "--- Package ${integration}: check"
 
@@ -63,7 +65,9 @@ for integration in $(list_all_directories); do
     fi
 
     if ! run_tests_package ${integration} ; then
+        echo "[${integration}] run_tests_package failed"
         echo "- ${integration}" >> ${FAILED_PACKAGES_FILE_PATH}
+        any_package_failing=1
     fi
 
     # TODO: add benchmarks support (https://github.com/elastic/integrations/blob/befdc5cb752a08aaf5f79b0d9bdb68588ade9f27/.ci/Jenkinsfile#L180)
@@ -88,3 +92,9 @@ if [ -f ${FAILED_PACKAGES_FILE_PATH} ]; then
     echo "Found failed_packages.txt"  # TODO: remove
     create_collapsed_annotation "Failed packages in ${SERVERLESS_PROJECT}" ${FAILED_PACKAGES_FILE_PATH} "error" "ctx-failed-packages-${SERVERLESS_PROJECT}"
 fi
+
+if [ $any_package_failing -eq 1 ] ; then
+    echo "These packages have failed:"
+    cat ${FAILED_PACKAGES_FILE_PATH}
+    exit 1
+fi

From 87c00cfffc84c8e6c854621b567961eb3e0fba39 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 18:52:00 +0200
Subject: [PATCH 089/139] Add returns to run_tests_package

---
 .buildkite/scripts/common.sh | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 79ab5c70fbb..40c75707e50 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -471,9 +471,16 @@ test_package_no_system() {
 
 run_tests_package() {
     local integration=$1
-    check_package ${integration}
-    install_package ${integration}
-    test_package_no_system ${integration}
+    if ! check_package ${integration} ; then
+        return 1
+    fi
+    if ! install_package ${integration} ; then
+        return 1
+    fi
+    if ! test_package_no_system ${integration} ; then
+        return 1
+    fi
+    return 0
 }
 
 create_collapsed_annotation() {

From d541590b3fa7057f951094e1236463385deea6ea Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 19:12:52 +0200
Subject: [PATCH 090/139] Add more returns

---
 .buildkite/scripts/common.sh | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 40c75707e50..65c8ffbe649 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -463,10 +463,17 @@ test_package_no_system() {
     TEST_OPTIONS="-v --report-format xUnit --report-output file --test-coverage"
 
     echo "Test integration: ${integration}"
-    ${ELASTIC_PACKAGE_BIN} test asset ${TEST_OPTIONS}
-    ${ELASTIC_PACKAGE_BIN} test static ${TEST_OPTIONS}
-    ${ELASTIC_PACKAGE_BIN} test pipeline ${TEST_OPTIONS}
+    if ! ${ELASTIC_PACKAGE_BIN} test asset ${TEST_OPTIONS} ; then
+        return 1
+    fi
+    if ! ${ELASTIC_PACKAGE_BIN} test static ${TEST_OPTIONS} ; then
+        return 1
+    fi
+    if ! ${ELASTIC_PACKAGE_BIN} test pipeline ${TEST_OPTIONS} ; then
+        return 1
+    fi
     echo ""
+    return 0
 }
 
 run_tests_package() {

From cff69048e99bd974e66f1c2eca5e63d535aba004 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 18 Oct 2023 19:29:52 +0200
Subject: [PATCH 091/139] Test all packages observability

---
 .buildkite/scripts/common.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 65c8ffbe649..ed2b388b7c5 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -441,7 +441,7 @@ teardown_test_package() {
 }
 
 list_all_directories() {
-    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort  |egrep "netskope|logstash"
+    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort  #|egrep "netskope|logstash"
 }
 
 check_package() {

From c5a14dc8998c0c1f647fae647b44a1b8cc633c3b Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 19 Oct 2023 10:14:12 +0200
Subject: [PATCH 092/139] Replace elastic-package

---
 go.mod |  47 +++----
 go.sum | 398 ++++++++++++++++++++++++++++++++++++++++++++++++++-------
 2 files changed, 377 insertions(+), 68 deletions(-)

diff --git a/go.mod b/go.mod
index f0f9f7572db..69f6d733841 100644
--- a/go.mod
+++ b/go.mod
@@ -10,7 +10,7 @@ require (
 	github.com/magefile/mage v1.15.0
 	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.8.4
-	golang.org/x/tools v0.13.0
+	golang.org/x/tools v0.14.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
@@ -43,10 +43,10 @@ require (
 	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/creasty/defaults v1.7.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/elastic/elastic-integration-corpus-generator-tool v0.5.0 // indirect
+	github.com/elastic/elastic-integration-corpus-generator-tool v0.7.0 // indirect
 	github.com/elastic/go-elasticsearch/v7 v7.17.10 // indirect
 	github.com/elastic/go-resource v0.1.1 // indirect
 	github.com/elastic/go-sysinfo v1.9.0 // indirect
@@ -54,7 +54,7 @@ require (
 	github.com/elastic/go-windows v1.0.1 // indirect
 	github.com/elastic/gojsonschema v1.2.1 // indirect
 	github.com/elastic/kbncontent v0.1.1 // indirect
-	github.com/elastic/package-spec/v2 v2.13.0 // indirect
+	github.com/elastic/package-spec/v3 v3.0.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.10.1 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.7.0 // indirect
@@ -76,7 +76,7 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.1.2 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-github/v32 v32.1.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
@@ -94,14 +94,14 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.4 // indirect
 	github.com/huandu/xstrings v1.4.0 // indirect
-	github.com/imdario/mergo v0.3.15 // indirect
+	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jedib0t/go-pretty v4.3.0+incompatible // indirect
 	github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/klauspost/pgzip v1.2.5 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
@@ -129,7 +129,7 @@ require (
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pierrec/lz4/v4 v4.1.17 // indirect
 	github.com/pkg/xattr v0.4.9 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/prometheus/client_golang v1.16.0 // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect
@@ -137,11 +137,12 @@ require (
 	github.com/prometheus/procfs v0.10.1 // indirect
 	github.com/rivo/uniseg v0.4.3 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/shirou/gopsutil/v3 v3.23.8 // indirect
+	github.com/shirou/gopsutil/v3 v3.23.9 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
-	github.com/spf13/cast v1.5.0 // indirect
+	github.com/spf13/afero v1.10.0 // indirect
+	github.com/spf13/cast v1.5.1 // indirect
 	github.com/spf13/cobra v1.7.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/objx v0.5.1 // indirect
@@ -164,13 +165,13 @@ require (
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.25.0 // indirect
-	golang.org/x/crypto v0.13.0 // indirect
-	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/net v0.15.0 // indirect
-	golang.org/x/oauth2 v0.10.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/sys v0.12.0 // indirect
-	golang.org/x/term v0.12.0 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/mod v0.13.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/oauth2 v0.12.0 // indirect
+	golang.org/x/sync v0.4.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/term v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
@@ -183,13 +184,13 @@ require (
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	helm.sh/helm/v3 v3.13.0 // indirect
+	helm.sh/helm/v3 v3.13.1 // indirect
 	howett.net/plist v1.0.0 // indirect
-	k8s.io/api v0.28.2 // indirect
+	k8s.io/api v0.28.3 // indirect
 	k8s.io/apiextensions-apiserver v0.28.2 // indirect
-	k8s.io/apimachinery v0.28.2 // indirect
-	k8s.io/cli-runtime v0.28.2 // indirect
-	k8s.io/client-go v0.28.2 // indirect
+	k8s.io/apimachinery v0.28.3 // indirect
+	k8s.io/cli-runtime v0.28.3 // indirect
+	k8s.io/client-go v0.28.3 // indirect
 	k8s.io/component-base v0.28.2 // indirect
 	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
@@ -201,3 +202,5 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
+
+replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20231019081141-bddd6a1963e7
diff --git a/go.sum b/go.sum
index bf42a827e6a..363ea3b5a5c 100644
--- a/go.sum
+++ b/go.sum
@@ -1,24 +1,61 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
+cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
+cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY=
 cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk=
 cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
 cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg=
 cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
 cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
 cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y=
 cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU=
 cloud.google.com/go/kms v1.12.1 h1:xZmZuwy2cwzsocmKDOPu4BL7umg8QXagQx6fKVmf45U=
 cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
 cloud.google.com/go/pubsub v1.33.0 h1:6SPCPvWav64tj0sVX/+npCBKhUi/UjJehy9op/V3p2g=
 cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 cloud.google.com/go/storage v1.32.0 h1:5w6DxEGOnktmJHarxAOUywxVW9lbNWIzlzzUltG/3+o=
 cloud.google.com/go/storage v1.32.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/OxflYkiD8=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ=
 github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
 github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
@@ -81,6 +118,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=
 github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
 github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
@@ -94,17 +132,16 @@ github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr
 github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdBA=
 github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY=
 github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s=
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/elastic/elastic-integration-corpus-generator-tool v0.5.0 h1:Me2T3/O4nASmdjmfaKYaiJaGq8zVhasjfZi3il5p/gs=
-github.com/elastic/elastic-integration-corpus-generator-tool v0.5.0/go.mod h1:uf9N86y+UACGybdEhZLpwZ93XHWVhsYZAA4c2T2v6YM=
-github.com/elastic/elastic-package v0.88.0 h1:mZVrf8Rzjslj05EAWk+k9TgmTuZCM0tBLIjsqRi8xVk=
-github.com/elastic/elastic-package v0.88.0/go.mod h1:h/mpD65Vt8suyoVAHhXvTqrsbo9J24q2StMXh7QtULc=
+github.com/elastic/elastic-integration-corpus-generator-tool v0.7.0 h1:yYvNv5XxBgoYINZr/jKJl98pX7SNKMPYeZ8bWOeM/Vw=
+github.com/elastic/elastic-integration-corpus-generator-tool v0.7.0/go.mod h1:WF7gU0qRVj1i//XtQD5i2ZGcGiuiVO6DJNDCRh/o2xk=
 github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo=
 github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elastic/go-licenser v0.4.1 h1:1xDURsc8pL5zYT9R29425J3vkHdt4RT5TNEMeRN48x4=
@@ -123,13 +160,14 @@ github.com/elastic/kbncontent v0.1.1 h1:X6ZXKPTW3MwFZPLMZcPAbpUIsKQhFImhBGKOd83+
 github.com/elastic/kbncontent v0.1.1/go.mod h1:kOPREITK9gSJsiw/WKe7QWSO+PRiZMyEFQCw+CMLAHI=
 github.com/elastic/package-registry v1.21.0 h1:yl2LCXVvRrUI5sOuBHKyZr0N1+8hwLCWEynmj91yQJw=
 github.com/elastic/package-registry v1.21.0/go.mod h1:KMaIWgFlxMKRd1I8W8NMCLrQLKOvYwDxp1F9KioneU4=
-github.com/elastic/package-spec/v2 v2.13.0 h1:loZ0xZOzsobV4LIQZphavRY/CPMKJK463psSUGjZBxg=
-github.com/elastic/package-spec/v2 v2.13.0/go.mod h1:OcEnGWmL05XtMAb8qJweiVaePwMb2GQhFFpiYFTgzcc=
+github.com/elastic/package-spec/v3 v3.0.0 h1:t4domWWtALwms0pnWKHLceNY5LZe2EKPgRzdfBPLcas=
+github.com/elastic/package-spec/v3 v3.0.0/go.mod h1:proxSLyZsrgNOnWMqg01ROXcnU8b8DRa0OiY4T/dN90=
 github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ=
 github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
@@ -144,13 +182,16 @@ github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBD
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
-github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
+github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
+github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsouza/fake-gcs-server v1.47.4 h1:gfBhBxEra20/Om02cvcyL8EnekV8KDb01Yffjat6AKQ=
 github.com/fsouza/fake-gcs-server v1.47.4/go.mod h1:vqUZbI12uy9IkRQ54Q4p5AniQsSiUq8alO9Nv2egMmA=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
 github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -172,14 +213,24 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -196,6 +247,8 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
 github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
 github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
@@ -204,14 +257,17 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
 github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@@ -220,10 +276,25 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
 github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg=
 github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4=
 github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc=
@@ -237,8 +308,11 @@ github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
 github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM=
 github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
 github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
+github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
 github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4=
 github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
@@ -258,14 +332,18 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
 github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog=
 github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
 github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU=
 github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
-github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
-github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
+github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
+github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PHJzaeDodcfvRAbIo=
@@ -277,6 +355,8 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -284,14 +364,15 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
-github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
+github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
 github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
 github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -357,6 +438,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/mrodm/elastic-package v0.53.1-0.20231019081141-bddd6a1963e7 h1:n0U731qeFJoCo80liZf8fZ2hRa1/U0nmL7LQfWS2DYs=
+github.com/mrodm/elastic-package v0.53.1-0.20231019081141-bddd6a1963e7/go.mod h1:MkUrD3iIh8/YTNv/z5+EJvgWknnI1qAlYZF+7vAy3JQ=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
@@ -371,8 +454,8 @@ github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE=
 github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM=
 github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
 github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
-github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY=
-github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww=
+github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
+github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
@@ -382,10 +465,12 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
 github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
 github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
 github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
@@ -401,6 +486,7 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ
 github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw=
 github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
@@ -409,8 +495,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
 github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
-github.com/shirou/gopsutil/v3 v3.23.8 h1:xnATPiybo6GgdRoC4YoGnxXZFRc3dqQTGi73oLvvBrE=
-github.com/shirou/gopsutil/v3 v3.23.8/go.mod h1:7hmCaBn+2ZwaZOr6jmPBZDfawwMGuo1id3C6aM8EDqQ=
+github.com/shirou/gopsutil/v3 v3.23.9 h1:ZI5bWVeu2ep4/DIxB4U9okeYJ7zp/QLTO4auRb/ty/E=
+github.com/shirou/gopsutil/v3 v3.23.9/go.mod h1:x/NWSb71eMcjFIO0vhyGW5nZ7oSIgVjrCnADckb85GA=
 github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
 github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
 github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
@@ -420,9 +506,11 @@ github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5g
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY=
+github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
-github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
+github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA=
+github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48=
 github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
 github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -467,7 +555,9 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMx
 github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=
 github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
@@ -489,6 +579,12 @@ go.elastic.co/fastjson v1.1.0/go.mod h1:boNGISWMjQsUPy/t6yqt2/1Wx4YNPSe+mZjlyw9v
 go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8=
 go.mongodb.org/mongo-driver v1.11.1 h1:QP0znIRTuL0jf1oBQoAoM0C6ZJfBK4kx0Uumtv1A7w8=
 go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
@@ -505,43 +601,91 @@ go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
 go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c=
 go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
+golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@@ -550,35 +694,73 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
-golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
-golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
+golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4=
+golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
+golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -595,19 +777,23 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
-golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
@@ -616,40 +802,139 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
-golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
+golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
+google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
+google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
 google.golang.org/api v0.134.0 h1:ktL4Goua+UBgoP1eL1/60LwZJqa1sIzkLmvoR3hR6Gw=
 google.golang.org/api v0.134.0/go.mod h1:sjRL3UnjTx5UqNQS9EWr9N8p7xbHpy1k0XGRLCf3Spk=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130 h1:Au6te5hbKUV8pIYWHqOUZ1pva5qK/rwbIhoXEUB9Lu8=
 google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:O9kGHb51iE/nOGvQaDUuadVYqovW56s5emA88lQnj6Y=
 google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130 h1:XVeBY8d/FaK4848myy41HBqnDwvxeV3zMZhwN1TvAMU=
@@ -657,11 +942,22 @@ google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130/go.
 google.golang.org/genproto/googleapis/rpc v0.0.0-20230720185612-659f7aaaa771 h1:Z8qdAF9GFsmcUuWQ5KVYIpP3PCKydn/YKORnghIalu4=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20230720185612-659f7aaaa771/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
+google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
 google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw=
@@ -674,6 +970,7 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
 google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
@@ -684,6 +981,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/hjson/hjson-go.v3 v3.0.1/go.mod h1:X6zrTSVeImfwfZLfgQdInl9mWjqPqgH90jom9nym/lw=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
@@ -701,22 +999,27 @@ gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-helm.sh/helm/v3 v3.13.0 h1:XPJKIU30K4JTQ6VX/6e0hFAmEIonYa8E7wx5aqv4xOc=
-helm.sh/helm/v3 v3.13.0/go.mod h1:2PBEKsMWKLVZTojUOqMS3Eadv5mP43FBWrRgLNkNm9Y=
+helm.sh/helm/v3 v3.13.1 h1:DG+XLGzBJeZvMLlMbm6bPDLV1dGaVW9eZsDoUd1/LM0=
+helm.sh/helm/v3 v3.13.1/go.mod h1:TdQRMiq46CSWcc68Hb0uVhvAWusaN90YwAV54cz6JzU=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM=
 howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
-k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw=
-k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg=
+k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM=
+k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc=
 k8s.io/apiextensions-apiserver v0.28.2 h1:J6/QRWIKV2/HwBhHRVITMLYoypCoPY1ftigDM0Kn+QU=
 k8s.io/apiextensions-apiserver v0.28.2/go.mod h1:5tnkxLGa9nefefYzWuAlWZ7RZYuN/765Au8cWLA6SRg=
-k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ=
-k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU=
-k8s.io/cli-runtime v0.28.2 h1:64meB2fDj10/ThIMEJLO29a1oujSm0GQmKzh1RtA/uk=
-k8s.io/cli-runtime v0.28.2/go.mod h1:bTpGOvpdsPtDKoyfG4EG041WIyFZLV9qq4rPlkyYfDA=
-k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY=
-k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY=
+k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A=
+k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8=
+k8s.io/cli-runtime v0.28.3 h1:lvuJYVkwCqHEvpS6KuTZsUVwPePFjBfSGvuaLl2SxzA=
+k8s.io/cli-runtime v0.28.3/go.mod h1:jeX37ZPjIcENVuXDDTskG3+FnVuZms5D9omDXS/2Jjc=
+k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4=
+k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo=
 k8s.io/component-base v0.28.2 h1:Yc1yU+6AQSlpJZyvehm/NkJBII72rzlEsd6MkBQ+G0E=
 k8s.io/component-base v0.28.2/go.mod h1:4IuQPQviQCg3du4si8GpMrhAIegxpsgPngPRR/zWpzc=
 k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
@@ -727,6 +1030,9 @@ k8s.io/kubectl v0.28.2 h1:fOWOtU6S0smdNjG1PB9WFbqEIMlkzU5ahyHkc7ESHgM=
 k8s.io/kubectl v0.28.2/go.mod h1:6EQWTPySF1fn7yKoQZHYf9TPwIl2AygHEcJoxFekr64=
 k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk=
 k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0=

From a2ee7eb98042a70afe13b51042208669cafbe18b Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 19 Oct 2023 12:27:13 +0200
Subject: [PATCH 093/139] Add new pipeline for serverless

---
 .buildkite/hooks/pre-command           |  4 +--
 .buildkite/hooks/pre-exit              |  4 +--
 .buildkite/pipeline.schedule-daily.yml | 20 ++++++------
 .buildkite/pull-requests.json          | 16 ++++++++++
 catalog-info.yaml                      | 42 ++++++++++++++++++++++++++
 5 files changed, 71 insertions(+), 15 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index f2f25b1e929..4da68fddf8a 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -55,8 +55,8 @@ if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
     fi
 fi
 
-if [ ${BUILDKITE_PIPELINE_SLUG} == "schedule-daily" ]; then
-    if [[ ${BUILDKITE_STEP_KEY} =~ test-integrations-serverless-(obs|security) ]]; then
+if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations-serverless" ]; then
+    if [[ ${BUILDKITE_STEP_KEY} =~ run-integrations-serverless-(obs|security) ]]; then
         export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
         export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
 
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index 1fed2e0f1ff..86df5b7b9b4 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -27,8 +27,8 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
     fi
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-schedule-daily" ]]; then
-    if [[ "$BUILDKITE_STEP_KEY" =~ test-integrations-serverless-(obs|security) ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-serverless" ]]; then
+    if [[ "$BUILDKITE_STEP_KEY" =~ run-integrations-serverless-(obs|security) ]]; then
         unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
         unset ELASTIC_PACKAGE_AWS_SECRET_KEY
         unset AWS_ACCESS_KEY_ID
diff --git a/.buildkite/pipeline.schedule-daily.yml b/.buildkite/pipeline.schedule-daily.yml
index 49be1498f4a..fbfa4087fba 100644
--- a/.buildkite/pipeline.schedule-daily.yml
+++ b/.buildkite/pipeline.schedule-daily.yml
@@ -16,22 +16,20 @@ steps:
 
   - label: "Check integrations in serverless - project: Observability"
     key: "trigger-integrations-serverless-obs"
-    command: ".buildkite/scripts/run_integrations_sync.sh"
-    env:
-      SERVERLESS: true
-      SERVERLESS_PROJECT: observability
-      FORCE_CHECK_ALL: true
+    trigger: "integrations-serverless"
+    build:
+      env:
+        SERVERLESS_PROJECT: observability
     depends_on:
       - step: "check-sources"
         allow_failure: false
 
-  - label: "Check integrations in serverless - project: security"
+  - label: "Check integrations in serverless - project: Security"
     key: "trigger-integrations-serverless-security"
-    command: ".buildkite/scripts/run_integrations_sync.sh"
-    env:
-      SERVERLESS: true
-      SERVERLESS_PROJECT: security
-      FORCE_CHECK_ALL: true
+    trigger: "integrations-serverless"
+    build:
+      env:
+        SERVERLESS_PROJECT: security
     depends_on:
       - step: "check-sources"
         allow_failure: false
diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json
index 7921e85205c..404452152a0 100644
--- a/.buildkite/pull-requests.json
+++ b/.buildkite/pull-requests.json
@@ -31,6 +31,22 @@
       "skip_target_branches": [],
       "skip_ci_on_only_changed": [],
       "always_require_ci_on_changed": []
+    },
+    {
+      "enabled": false,
+      "pipelineSlug": "integrations-serverless",
+      "allow_org_users": true,
+      "allowed_repo_permissions": ["admin", "write"],
+      "allowed_list": [],
+      "set_commit_status": false,
+      "build_on_commit": false,
+      "build_on_comment": false,
+      "trigger_comment_regex": "",
+      "always_trigger_comment_regex": "",
+      "skip_ci_labels": [],
+      "skip_target_branches": [],
+      "skip_ci_on_only_changed": [],
+      "always_require_ci_on_changed": []
     }
   ]
 }
diff --git a/catalog-info.yaml b/catalog-info.yaml
index b62b5643173..ce660c5f754 100644
--- a/catalog-info.yaml
+++ b/catalog-info.yaml
@@ -111,3 +111,45 @@ spec:
         everyone:
           access_level: READ_ONLY
 
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: pipeline-integrations-serverless
+  description: 'Pipeline for the Integrations project in serverless'
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/integrations-serverless
+
+spec:
+  type: buildkite-pipeline
+  owner: group:ingest-fp
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: integrations-serverless
+      description: 'Pipeline for the Integrations project in serverless'
+    spec:
+      pipeline_file: ".buildkite/pipeline.serverless.yml"
+      provider_settings:
+        trigger_mode: none # don't trigger jobs from github activity
+        build_pull_request_forks: false
+        build_pull_requests: true
+        build_tags: false
+        filter_enabled: true
+        filter_condition: >-
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
+      repository: elastic/integrations
+      cancel_intermediate_builds: true
+      cancel_intermediate_builds_branch_filter: '!main'
+      skip_intermediate_builds: true
+      skip_intermediate_builds_branch_filter: '!main'
+      teams:
+        ingest-fp:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY
+

From cd70b8777a6640cedb6c53fbea81fe507a5ebbb7 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 19 Oct 2023 12:45:52 +0200
Subject: [PATCH 094/139] Add security

---
 .buildkite/pipeline.yml | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index ab0e62f9dc2..4e60011fca4 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -26,7 +26,7 @@ steps:
       memory: "4G"
 
   # TODO: remove comments
-  # - label: "Publish packages"
+  # - label: "Publish packages - INCOMPLETE"
   #   key: "publish-packages"
   #   command: ".buildkite/scripts/publish_packages.sh" # TODO: missing signature and publishing loops
   #   agents:
@@ -51,20 +51,20 @@ steps:
     artifact_paths:
       - "build/test-results/*.xml"
 
-  # - label: "Check integrations in serverless - project security"
-  #   key: "test-integrations-serverless-security"
-  #   command: ".buildkite/scripts/run_integrations_sync.sh"
-  #   env:
-  #     SERVERLESS: true
-  #     SERVERLESS_PROJECT: security
-  #     FORCE_CHECK_ALL: true
-  #   agents:
-  #     provider: "gcp"
-  #   depends_on:
-  #     - step: "check"   # TODO: change to publish-packages if re-added
-  #       allow_failure: false
-  #   artifact_paths:
-  #     - "build/test-results/*.xml"
+  - label: "Check integrations in serverless - project security"
+    key: "test-integrations-serverless-security"
+    command: ".buildkite/scripts/run_integrations_sync.sh"
+    env:
+      SERVERLESS: true
+      SERVERLESS_PROJECT: security
+      FORCE_CHECK_ALL: true
+    agents:
+      provider: "gcp"
+    depends_on:
+      - step: "check"   # TODO: change to publish-packages if re-added
+        allow_failure: false
+    artifact_paths:
+      - "build/test-results/*.xml"
 
   - wait: ~
     continue_on_failure: true

From 333ad653a2e0873a6b059586067600ea07b006c8 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 19 Oct 2023 15:55:54 +0200
Subject: [PATCH 095/139] Remove test coverage for pipeline tests

---
 .buildkite/scripts/common.sh | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index ed2b388b7c5..032f1e968f9 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -458,17 +458,19 @@ install_package() {
     echo ""
 }
 
-test_package_no_system() {
+test_package_in_serverless() {
     local integration=$1
-    TEST_OPTIONS="-v --report-format xUnit --report-output file --test-coverage"
+    TEST_OPTIONS="-v --report-format xUnit --report-output file"
 
     echo "Test integration: ${integration}"
-    if ! ${ELASTIC_PACKAGE_BIN} test asset ${TEST_OPTIONS} ; then
+    if ! ${ELASTIC_PACKAGE_BIN} test asset ${TEST_OPTIONS} --test-coverage ; then
         return 1
     fi
-    if ! ${ELASTIC_PACKAGE_BIN} test static ${TEST_OPTIONS} ; then
+    if ! ${ELASTIC_PACKAGE_BIN} test static ${TEST_OPTIONS} --test-coverage ; then
         return 1
     fi
+    # FIXME: adding test-coverage for serverless results in errors like this:
+    # Error: error running package pipeline tests: could not complete test run: error calculating pipeline coverage: error fetching pipeline stats for code coverage calculations: need exactly one ES node in stats response (got 4)
     if ! ${ELASTIC_PACKAGE_BIN} test pipeline ${TEST_OPTIONS} ; then
         return 1
     fi
@@ -484,8 +486,10 @@ run_tests_package() {
     if ! install_package ${integration} ; then
         return 1
     fi
-    if ! test_package_no_system ${integration} ; then
-        return 1
+    if [[ $SERVERLESS == "true" ]]; then
+        if ! test_package_in_serverless ${integration} ; then
+            return 1
+        fi
     fi
     return 0
 }

From a4123ca66355064cb5cc415bed4ebba684311131 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 19 Oct 2023 16:11:37 +0200
Subject: [PATCH 096/139] Add project to profile

---
 .buildkite/scripts/common.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 032f1e968f9..11c54f8c886 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -313,7 +313,7 @@ prepare_serverless_stack() {
     # else
     fi
 
-    create_elastic_package_profile "integrations-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_BUILD_NUMBER}"
+    create_elastic_package_profile "integrations-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_BUILD_NUMBER}-${SERVERLESS_PROJECT}"
 
     export EC_API_KEY=${EC_API_KEY_SECRET}
     export EC_HOST=${EC_HOST_SECRET}

From 425b9800504833ef67e57846edc463e733a21c16 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 19 Oct 2023 16:14:20 +0200
Subject: [PATCH 097/139] Add a new section header

---
 .buildkite/hooks/pre-exit | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index 86df5b7b9b4..b4af958ee0b 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -16,6 +16,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
 
         # Ensure elastic stack is stopped
         if [ -f ${ELASTIC_PACKAGE_BIN} ]; then
+            echo "--- Take down the Elastic stack"
             export EC_API_KEY=${EC_API_KEY_SECRET}
             export EC_HOST=${EC_HOST_SECRET}
 
@@ -39,6 +40,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-serverless" ]]; then
 
         # Ensure elastic stack is stopped
         if [ -f ${ELASTIC_PACKAGE_BIN} ]; then
+            echo "--- Take down the Elastic stack"
             export EC_API_KEY=${EC_API_KEY_SECRET}
             export EC_HOST=${EC_HOST_SECRET}
 

From f8d3ba997d448ea2c9bc7cb1386d94e25fdef0b2 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 19 Oct 2023 16:18:58 +0200
Subject: [PATCH 098/139] Add timeout for steps

---
 .buildkite/pipeline.serverless.yml | 39 ++++++++++++++++++++++++++++++
 .buildkite/pipeline.yml            |  2 ++
 2 files changed, 41 insertions(+)
 create mode 100644 .buildkite/pipeline.serverless.yml

diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml
new file mode 100644
index 00000000000..40dedbee642
--- /dev/null
+++ b/.buildkite/pipeline.serverless.yml
@@ -0,0 +1,39 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  SETUP_GVM_VERSION: "v0.5.1"
+  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
+  DOCKER_COMPOSE_VERSION: "v2.17.2"
+  KIND_VERSION: 'v0.20.0'
+  K8S_VERSION: 'v1.27.3'
+  YQ_VERSION: 'v4.35.2'
+  # Elastic package settings
+  # Manage docker output/logs
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true"
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true"
+  # Default license to use by `elastic-package build`
+  ELASTIC_PACKAGE_REPOSITORY_LICENSE: "licenses/Elastic-2.0.txt"
+  # Link definitions path (full path to be set in the corresponding step)
+  ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
+
+steps:
+  - label: ":white_check_mark: Check go sources"
+    key: "check"
+    command: ".buildkite/scripts/check_sources.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+
+  - label: "Check integrations in serverless - project ${SERVERLESS_PROJECT}"
+    key: "run-integrations-serverless-project"
+    command: ".buildkite/scripts/run_integrations_sync.sh"
+    timeout_in_minutes: 120
+    env:
+      SERVERLESS: true
+      FORCE_CHECK_ALL: true
+    agents:
+      provider: "gcp"
+    depends_on:
+      - step: "check"
+        allow_failure: false
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 4e60011fca4..8c16d12df7b 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -39,6 +39,7 @@ steps:
   - label: "Check integrations in serverless - project observability"
     key: "test-integrations-serverless-obs"
     command: ".buildkite/scripts/run_integrations_sync.sh"
+    timeout_in_minutes: 120
     env:
       SERVERLESS: true
       SERVERLESS_PROJECT: observability
@@ -54,6 +55,7 @@ steps:
   - label: "Check integrations in serverless - project security"
     key: "test-integrations-serverless-security"
     command: ".buildkite/scripts/run_integrations_sync.sh"
+    timeout_in_minutes: 120
     env:
       SERVERLESS: true
       SERVERLESS_PROJECT: security

From dd326d7db0576d88767a28d1317368e854130338 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 19 Oct 2023 16:40:43 +0200
Subject: [PATCH 099/139] Add paths as artifacts

---
 .buildkite/pipeline.serverless.yml | 4 ++++
 .buildkite/pipeline.yml            | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml
index 40dedbee642..bf15b1e04e3 100644
--- a/.buildkite/pipeline.serverless.yml
+++ b/.buildkite/pipeline.serverless.yml
@@ -34,6 +34,10 @@ steps:
       FORCE_CHECK_ALL: true
     agents:
       provider: "gcp"
+    artifact_paths:
+      - "build/test-results/*.xml"
+      - "build/elastic-stack-dump/*/logs/*.log"
+      - "build/elastic-stack-dump/*/logs/fleet-server-internal/**/*"
     depends_on:
       - step: "check"
         allow_failure: false
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 8c16d12df7b..98710764583 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -51,6 +51,8 @@ steps:
         allow_failure: false
     artifact_paths:
       - "build/test-results/*.xml"
+      - "build/elastic-stack-dump/*/logs/*.log"
+      - "build/elastic-stack-dump/*/logs/fleet-server-internal/**/*"
 
   - label: "Check integrations in serverless - project security"
     key: "test-integrations-serverless-security"

From 221783224b5da48440d9062de16a958037a51290 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 19 Oct 2023 17:57:34 +0200
Subject: [PATCH 100/139] Upload safe logs

---
 .buildkite/hooks/pre-command           |  9 +++
 .buildkite/pipeline.serverless.yml     | 14 ++++-
 .buildkite/pipeline.yml                |  6 +-
 .buildkite/scripts/common.sh           | 85 +++++++++++++++++++++++---
 .buildkite/scripts/publish_packages.sh |  1 -
 .ci/Jenkinsfile                        |  2 +-
 6 files changed, 104 insertions(+), 13 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 4da68fddf8a..a5beb1309c3 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -12,9 +12,12 @@ export REPO_NAME=$(repo_name "${BUILDKITE_REPO}")
 export TMP_FOLDER_TEMPLATE_BASE="tmp.${REPO_NAME}"
 export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
 
+REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
+
 JENKINS_API_TOKEN_PATH=kv/ci-shared/platform-ingest/jenkins_api_tokens
 SIGNING_PACKAGES_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/signing_packages_gcs_artifacts_credentials
 PACKAGE_UPLOADER_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/package_storage_uploader
+PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials
 
 EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa
 EC_DATA_PATH=secret/ci/elastic-integrations/ec_data
@@ -45,6 +48,9 @@ if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
         export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
         export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
 
+        export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
+        export JOB_GCS_BUCKET_INTERNAL="fleet-ci-temp-internal"
+
         # Environment variables required by the service deployer
         export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
         export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
@@ -60,6 +66,9 @@ if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations-serverless" ]; then
         export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
         export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
 
+        export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
+        export JOB_GCS_BUCKET_INTERNAL="fleet-ci-temp-internal"
+
         # Environment variables required by the service deployer
         export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
         export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml
index bf15b1e04e3..c67f60202a7 100644
--- a/.buildkite/pipeline.serverless.yml
+++ b/.buildkite/pipeline.serverless.yml
@@ -36,8 +36,18 @@ steps:
       provider: "gcp"
     artifact_paths:
       - "build/test-results/*.xml"
-      - "build/elastic-stack-dump/*/logs/*.log"
-      - "build/elastic-stack-dump/*/logs/fleet-server-internal/**/*"
+      # - "build/elastic-stack-dump/*/logs/*.log"
+      # - "build/elastic-stack-dump/*/logs/fleet-server-internal/**/*"
     depends_on:
       - step: "check"
         allow_failure: false
+
+  - wait: ~
+    continue_on_failure: true
+
+  - label: ":junit: Junit annotate"
+    plugins:
+      - junit-annotate#v2.4.1:
+          artifacts: "build/test-results/*.xml"
+    agents:
+      provider: "gcp"  # junit plugin requires docker
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 98710764583..74791e4eedd 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -51,8 +51,8 @@ steps:
         allow_failure: false
     artifact_paths:
       - "build/test-results/*.xml"
-      - "build/elastic-stack-dump/*/logs/*.log"
-      - "build/elastic-stack-dump/*/logs/fleet-server-internal/**/*"
+      # - "build/elastic-stack-dump/*/logs/*.log"
+      # - "build/elastic-stack-dump/*/logs/fleet-server-internal/**/*"
 
   - label: "Check integrations in serverless - project security"
     key: "test-integrations-serverless-security"
@@ -69,6 +69,8 @@ steps:
         allow_failure: false
     artifact_paths:
       - "build/test-results/*.xml"
+      # - "build/elastic-stack-dump/*/logs/*.log"
+      # - "build/elastic-stack-dump/*/logs/fleet-server-internal/**/*"
 
   - wait: ~
     continue_on_failure: true
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 11c54f8c886..d77de033702 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -6,6 +6,7 @@ WORKSPACE="$(pwd)"
 BIN_FOLDER="${WORKSPACE}/bin"
 platform_type="$(uname)"
 hw_type="$(uname -m)"
+GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
 export ELASTIC_PACKAGE_BIN=${WORKSPACE}/build/elastic-package
 
 retry() {
@@ -122,14 +123,14 @@ with_kubernetes() {
 }
 
 google_cloud_upload_auth() {
-  local secretFileLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX")/google-cloud-credentials.json
+  local secretFileLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX")/${GOOGLE_CREDENTIALS_FILENAME}
   echo "${PRIVATE_INFRA_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
   gcloud auth activate-service-account --key-file ${secretFileLocation} 2> /dev/null
   export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
 }
 
 google_cloud_signing_auth() {
-  local secretFileLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX")/google-cloud-credentials.json
+  local secretFileLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX")/${GOOGLE_CREDENTIALS_FILENAME}
   echo "${SIGNING_PACKAGES_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
   gcloud auth activate-service-account --key-file ${secretFileLocation} 2> /dev/null
   export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
@@ -424,7 +425,7 @@ teardown_serverless_test_package() {
     echo "Collect Elastic stack logs"
     ${ELASTIC_PACKAGE_BIN} stack dump -v --output ${dump_directory}
 
-    # TODO: upload insecure logs
+    upload_safe_logs_from_package ${integration}
 }
 
 teardown_test_package() {
@@ -434,28 +435,34 @@ teardown_test_package() {
     echo "Collect Elastic stack logs"
     ${ELASTIC_PACKAGE_BIN} stack dump -v --output ${dump_directory}
 
-    # TODO: upload insecure logs
+    upload_safe_logs_from_package ${integration}
 
     echo "Take down the Elastic stack"
     ${ELASTIC_PACKAGE_BIN} stack down -v
 }
 
 list_all_directories() {
-    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort  #|egrep "netskope|logstash"
+    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort  |egrep "netskope|logstash|ti_rapid7"
 }
 
 check_package() {
     local integration=$1
     echo "Check integration: ${integration}"
-    ${ELASTIC_PACKAGE_BIN} check -v
+    if ! ${ELASTIC_PACKAGE_BIN} check -v ; then
+        return 1
+    fi
     echo ""
+    return 0
 }
 
 install_package() {
     local integration=$1
     echo "Install integration: ${integration}"
-    ${ELASTIC_PACKAGE_BIN} install -v
+    if ! ${ELASTIC_PACKAGE_BIN} install -v ; then
+        return 1
+    fi
     echo ""
+    return 0
 }
 
 test_package_in_serverless() {
@@ -511,3 +518,67 @@ create_collapsed_annotation() {
 
     rm -f ${annotation_file}
 }
+
+google_cloud_auth_safe_logs() {
+    local gsUtilLocation=$(mktemp -d -p ${WORKSPACE} -t ${TMP_FOLDER_TEMPLATE})
+    local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
+
+    echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
+
+    gcloud auth activate-service-account --key-file ${secretFileLocation} 2> /dev/null
+    export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
+}
+
+upload_safe_logs() {
+    local bucket="$1"
+    local source="$2"
+    local target="$3"
+
+    if ! ls ${source} 2>&1 > /dev/null ; then
+        echo "upload_safe_logs: artifacts files not found, nothing will be archived"
+        return
+    fi
+
+    google_cloud_auth_safe_logs
+
+    gsutil cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}"
+
+    google_cloud_logout_active_account
+}
+
+buildkite_pr_branch_build_id() {
+    if [ "${BUILDKITE_PULL_REQUEST}" == "false" ]; then
+        echo "${BUILDKITE_BRANCH}-${BUILDKITE_BUILD_NUMBER}"
+        return
+    fi
+    echo "PR-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_BUILD_NUMBER}"
+}
+
+upload_safe_logs_from_package() {
+    if [[ "${UPLOAD_SAFE_LOGS}" -eq 1 ]] ; then
+        return
+    fi
+
+    local integration=$1
+    local parent_folder="insecure-logs"
+    if [[ "${SERVERLESS_PROJECT}" != "" ]]; then
+        #TODO remove this if when just triggered from pipeline-serverless
+        parent_folder="insecure-serverless-${SERVERLESS_PROJECT}-logs"
+    fi
+
+    upload_safe_logs \
+        "${JOB_GCS_BUCKET_INTERNAL}" \
+        "build/elastic-stack-dump/${integration}/logs/elastic-agent-internal/*.*" \
+        "${parent_folder}/${integration}/elastic-agent-logs/"
+
+    # required for <8.6.0
+    upload_safe_logs \
+        "${JOB_GCS_BUCKET_INTERNAL}" \
+        "build/elastic-stack-dump/${integration}/logs/elastic-agent-internal/default/*" \
+        "${parent_folder}/${integration}/elastic-agent-logs/default/"
+
+    upload_safe_logs \
+        "${JOB_GCS_BUCKET_INTERNAL}" \
+        "build/container-logs/*.log" \
+        "${parent_folder}/${integration}/container-logs/"
+}
diff --git a/.buildkite/scripts/publish_packages.sh b/.buildkite/scripts/publish_packages.sh
index d469fc31f63..ed20c3f392b 100644
--- a/.buildkite/scripts/publish_packages.sh
+++ b/.buildkite/scripts/publish_packages.sh
@@ -12,7 +12,6 @@ export BUILD_TAG="buildkite-${BUILDKITE_PIPELINE_SLUG}-${BUILDKITE_BUILD_NUMBER}
 export REPO_BUILD_TAG="${REPO_NAME}/${BUILD_TAG}"
 
 JENKINS_TRIGGER_PATH=".buildkite/scripts/triggerJenkinsJob"
-GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
 
 # signing
 INFRA_SIGNING_BUCKET_NAME='internal-ci-artifacts'
diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 47c371c8a41..d93bc7de4b6 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -143,7 +143,7 @@ pipeline {
                             archiveArtifacts(allowEmptyArchive: true, artifacts: "build/elastic-stack-dump/${it}/logs/*.log, build/elastic-stack-dump/${it}/logs/fleet-server-internal/**/*")
                             archiveArtifactsSafe("insecure-logs/${it}/elastic-agent-logs/", "build/elastic-stack-dump/${it}/logs/elastic-agent-internal/*.*")
                             archiveArtifactsSafe("insecure-logs/${it}/elastic-agent-logs/default/", "build/elastic-stack-dump/${it}/logs/elastic-agent-internal/default/*")
-                            archiveArtifactsSafe("insecure-logs/${it}/container-logs", "build/container-logs/*.log")
+                            archiveArtifactsSafe("insecure-logs/${it}/container-logs/", "build/container-logs/*.log")
                             sh(label: "Take down the Elastic stack", script: 'build/elastic-package stack down -v')
                             stashCoverageReport()
                           }

From e2981b09212242623637256124bb27f9800e52a9 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 19 Oct 2023 18:07:21 +0200
Subject: [PATCH 101/139] Update elastic-package

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 3f80f2daf28..bf08dc48e86 100644
--- a/go.mod
+++ b/go.mod
@@ -203,4 +203,4 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
-replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20231019081141-bddd6a1963e7
+replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20231019160543-ca99433cb9f9
diff --git a/go.sum b/go.sum
index 363ea3b5a5c..240486dd1bf 100644
--- a/go.sum
+++ b/go.sum
@@ -438,8 +438,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
-github.com/mrodm/elastic-package v0.53.1-0.20231019081141-bddd6a1963e7 h1:n0U731qeFJoCo80liZf8fZ2hRa1/U0nmL7LQfWS2DYs=
-github.com/mrodm/elastic-package v0.53.1-0.20231019081141-bddd6a1963e7/go.mod h1:MkUrD3iIh8/YTNv/z5+EJvgWknnI1qAlYZF+7vAy3JQ=
+github.com/mrodm/elastic-package v0.53.1-0.20231019160543-ca99433cb9f9 h1:e8xNNcQ/qAwR+kQz/YOg/k01LvdSWEvwewwyvUWus1Q=
+github.com/mrodm/elastic-package v0.53.1-0.20231019160543-ca99433cb9f9/go.mod h1:MkUrD3iIh8/YTNv/z5+EJvgWknnI1qAlYZF+7vAy3JQ=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=

From b7fd3cd386a334b98ac5956abb8d964fcd2ec223 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 19 Oct 2023 18:12:50 +0200
Subject: [PATCH 102/139] Add variable to upload safe logs

---
 .buildkite/hooks/pre-command       | 1 +
 .buildkite/pipeline.serverless.yml | 1 +
 .buildkite/pipeline.yml            | 2 ++
 3 files changed, 4 insertions(+)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index a5beb1309c3..23d28ab2893 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -4,6 +4,7 @@ source .buildkite/scripts/common.sh
 
 set -euo pipefail
 
+export UPLOAD_SAFE_LOGS=${UPLOAD_SAFE_LOGS:-"0"}
 export BASE_DIR=$(pwd)
 export GO_VERSION=$(cat .go-version)
 export startTimeInMillis="$(date +%s%3N)"
diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml
index c67f60202a7..3192295960e 100644
--- a/.buildkite/pipeline.serverless.yml
+++ b/.buildkite/pipeline.serverless.yml
@@ -32,6 +32,7 @@ steps:
     env:
       SERVERLESS: true
       FORCE_CHECK_ALL: true
+      UPLOAD_SAFE_LOGS: 1
     agents:
       provider: "gcp"
     artifact_paths:
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 74791e4eedd..be0753f887d 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -44,6 +44,7 @@ steps:
       SERVERLESS: true
       SERVERLESS_PROJECT: observability
       FORCE_CHECK_ALL: true
+      UPLOAD_SAFE_LOGS: 1
     agents:
       provider: "gcp"
     depends_on:
@@ -62,6 +63,7 @@ steps:
       SERVERLESS: true
       SERVERLESS_PROJECT: security
       FORCE_CHECK_ALL: true
+      UPLOAD_SAFE_LOGS: 1
     agents:
       provider: "gcp"
     depends_on:

From 88eda5a258a3de1dd9306db32d29ef107bccacbe Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 19 Oct 2023 18:42:53 +0200
Subject: [PATCH 103/139] Change condition

---
 .buildkite/scripts/common.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index d77de033702..42745afe825 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -555,7 +555,7 @@ buildkite_pr_branch_build_id() {
 }
 
 upload_safe_logs_from_package() {
-    if [[ "${UPLOAD_SAFE_LOGS}" -eq 1 ]] ; then
+    if [[ "${UPLOAD_SAFE_LOGS}" -eq 0 ]] ; then
         return
     fi
 

From 57aebff1d4a1d1ea8c2633f248e45d3f80e9609c Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 19 Oct 2023 18:44:30 +0200
Subject: [PATCH 104/139] Test function safe logs

---
 .buildkite/scripts/common.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 42745afe825..3c0f3170c72 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -541,7 +541,9 @@ upload_safe_logs() {
 
     google_cloud_auth_safe_logs
 
-    gsutil cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}"
+    echo "Repo build tag: $REPO_BUILD_TAG"
+    echo "Target path: $target"
+    # gsutil cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}"
 
     google_cloud_logout_active_account
 }

From 5d519913d5ca7a9705bc7e35acbbdd62c238a48b Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 19 Oct 2023 19:04:00 +0200
Subject: [PATCH 105/139] Fix path for logs

---
 .buildkite/scripts/common.sh                | 23 ++++++++++++++-------
 .buildkite/scripts/run_integrations_sync.sh |  3 ++-
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 3c0f3170c72..44c802e8982 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -420,22 +420,24 @@ kubernetes_service_deployer_used() {
 
 teardown_serverless_test_package() {
     local integration=$1
-    local dump_directory="${WORKSPACE}/build/elastic-stack-dump/${integration}"
+    local build_directory="${WORKSPACE}/build"
+    local dump_directory="${build_directory}/elastic-stack-dump/${integration}"
 
     echo "Collect Elastic stack logs"
     ${ELASTIC_PACKAGE_BIN} stack dump -v --output ${dump_directory}
 
-    upload_safe_logs_from_package ${integration}
+    upload_safe_logs_from_package ${integration} ${build_directory}
 }
 
 teardown_test_package() {
     local integration=$1
-    local dump_directory="${WORKSPACE}/build/elastic-stack-dump/${integration}"
+    local build_directory="${WORKSPACE}/build"
+    local dump_directory="${build_directory}/elastic-stack-dump/${integration}"
 
     echo "Collect Elastic stack logs"
     ${ELASTIC_PACKAGE_BIN} stack dump -v --output ${dump_directory}
 
-    upload_safe_logs_from_package ${integration}
+    upload_safe_logs_from_package ${integration} ${build_directory}
 
     echo "Take down the Elastic stack"
     ${ELASTIC_PACKAGE_BIN} stack down -v
@@ -556,12 +558,19 @@ buildkite_pr_branch_build_id() {
     echo "PR-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_BUILD_NUMBER}"
 }
 
+clean_safe_logs() {
+    rm -rf ${WORKSPACE}/build/elastic-stack-dump
+    rm -rf ${WORKSPACE}/build/container-logs
+}
+
 upload_safe_logs_from_package() {
     if [[ "${UPLOAD_SAFE_LOGS}" -eq 0 ]] ; then
         return
     fi
 
     local integration=$1
+    local build_directory=$2
+
     local parent_folder="insecure-logs"
     if [[ "${SERVERLESS_PROJECT}" != "" ]]; then
         #TODO remove this if when just triggered from pipeline-serverless
@@ -570,17 +579,17 @@ upload_safe_logs_from_package() {
 
     upload_safe_logs \
         "${JOB_GCS_BUCKET_INTERNAL}" \
-        "build/elastic-stack-dump/${integration}/logs/elastic-agent-internal/*.*" \
+        "${build_directory}/elastic-stack-dump/${integration}/logs/elastic-agent-internal/*.*" \
         "${parent_folder}/${integration}/elastic-agent-logs/"
 
     # required for <8.6.0
     upload_safe_logs \
         "${JOB_GCS_BUCKET_INTERNAL}" \
-        "build/elastic-stack-dump/${integration}/logs/elastic-agent-internal/default/*" \
+        "${build_directory}/elastic-stack-dump/${integration}/logs/elastic-agent-internal/default/*" \
         "${parent_folder}/${integration}/elastic-agent-logs/default/"
 
     upload_safe_logs \
         "${JOB_GCS_BUCKET_INTERNAL}" \
-        "build/container-logs/*.log" \
+        "${build_directory}/container-logs/*.log" \
         "${parent_folder}/${integration}/container-logs/"
 }
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 2f21a66aeb4..3f3e28ef349 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -38,9 +38,10 @@ any_package_failing=0
 
 for integration in $(list_all_directories); do
     echo "--- Package ${integration}: check"
-
     pushd ${integration} > /dev/null
 
+    clean_safe_logs
+
     if [[ ${SERVERLESS} == "true" ]] ; then
         if ! is_spec_3_0_0 ]]; then
             echo "Not v3 spec version. Skipped"

From 439f9f03b002f5a79fa75c03eb9c8874116ffa62 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 19 Oct 2023 19:19:21 +0200
Subject: [PATCH 106/139] Missing to export a variable

---
 .buildkite/hooks/pre-command | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 23d28ab2893..1304a14057c 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -13,7 +13,7 @@ export REPO_NAME=$(repo_name "${BUILDKITE_REPO}")
 export TMP_FOLDER_TEMPLATE_BASE="tmp.${REPO_NAME}"
 export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
 
-REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
+export REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
 
 JENKINS_API_TOKEN_PATH=kv/ci-shared/platform-ingest/jenkins_api_tokens
 SIGNING_PACKAGES_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/signing_packages_gcs_artifacts_credentials

From 262cbe2947d697a058fda41669049b1a29ad0472 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 19 Oct 2023 19:34:39 +0200
Subject: [PATCH 107/139] Remove test for upload safe logs

---
 .buildkite/scripts/common.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 44c802e8982..d65082e9284 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -543,9 +543,7 @@ upload_safe_logs() {
 
     google_cloud_auth_safe_logs
 
-    echo "Repo build tag: $REPO_BUILD_TAG"
-    echo "Target path: $target"
-    # gsutil cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}"
+    gsutil cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}"
 
     google_cloud_logout_active_account
 }

From 0f7d52d9e80e3a8edf439863eb820ae62e0c38d4 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 19 Oct 2023 19:45:24 +0200
Subject: [PATCH 108/139] Update elastic-package

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index bf08dc48e86..de6ba1cd139 100644
--- a/go.mod
+++ b/go.mod
@@ -203,4 +203,4 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
-replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20231019160543-ca99433cb9f9
+replace github.com/elastic/elastic-package => github.com/elastic/elastic-package v0.90.1-0.20231019174243-af194a4df954
diff --git a/go.sum b/go.sum
index 240486dd1bf..81ccde772ae 100644
--- a/go.sum
+++ b/go.sum
@@ -142,6 +142,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/elastic/elastic-integration-corpus-generator-tool v0.7.0 h1:yYvNv5XxBgoYINZr/jKJl98pX7SNKMPYeZ8bWOeM/Vw=
 github.com/elastic/elastic-integration-corpus-generator-tool v0.7.0/go.mod h1:WF7gU0qRVj1i//XtQD5i2ZGcGiuiVO6DJNDCRh/o2xk=
+github.com/elastic/elastic-package v0.90.1-0.20231019174243-af194a4df954 h1:jQd3+Y2KP/DSioLkYfqLCMGS/hELHFHTrDDPT1BwReI=
+github.com/elastic/elastic-package v0.90.1-0.20231019174243-af194a4df954/go.mod h1:MkUrD3iIh8/YTNv/z5+EJvgWknnI1qAlYZF+7vAy3JQ=
 github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo=
 github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elastic/go-licenser v0.4.1 h1:1xDURsc8pL5zYT9R29425J3vkHdt4RT5TNEMeRN48x4=
@@ -438,8 +440,6 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
-github.com/mrodm/elastic-package v0.53.1-0.20231019160543-ca99433cb9f9 h1:e8xNNcQ/qAwR+kQz/YOg/k01LvdSWEvwewwyvUWus1Q=
-github.com/mrodm/elastic-package v0.53.1-0.20231019160543-ca99433cb9f9/go.mod h1:MkUrD3iIh8/YTNv/z5+EJvgWknnI1qAlYZF+7vAy3JQ=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=

From 3f7eebabf70c5339b7a9fc681c845d262f7fecf3 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 19 Oct 2023 19:45:36 +0200
Subject: [PATCH 109/139] Test with all packages

---
 .buildkite/scripts/common.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index d65082e9284..8a8ec04ef7b 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -444,7 +444,7 @@ teardown_test_package() {
 }
 
 list_all_directories() {
-    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort  |egrep "netskope|logstash|ti_rapid7"
+    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort # |egrep "netskope|logstash|ti_rapid7"
 }
 
 check_package() {

From e3716fd86f391cd217feaa9aef845ab14ed3bd11 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 23 Oct 2023 16:48:21 +0200
Subject: [PATCH 110/139] Update elastic-package

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index de6ba1cd139..5196632b289 100644
--- a/go.mod
+++ b/go.mod
@@ -203,4 +203,4 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
-replace github.com/elastic/elastic-package => github.com/elastic/elastic-package v0.90.1-0.20231019174243-af194a4df954
+replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20231023144223-2e1f0a1077ec
diff --git a/go.sum b/go.sum
index 81ccde772ae..fc3018691b1 100644
--- a/go.sum
+++ b/go.sum
@@ -142,8 +142,6 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/elastic/elastic-integration-corpus-generator-tool v0.7.0 h1:yYvNv5XxBgoYINZr/jKJl98pX7SNKMPYeZ8bWOeM/Vw=
 github.com/elastic/elastic-integration-corpus-generator-tool v0.7.0/go.mod h1:WF7gU0qRVj1i//XtQD5i2ZGcGiuiVO6DJNDCRh/o2xk=
-github.com/elastic/elastic-package v0.90.1-0.20231019174243-af194a4df954 h1:jQd3+Y2KP/DSioLkYfqLCMGS/hELHFHTrDDPT1BwReI=
-github.com/elastic/elastic-package v0.90.1-0.20231019174243-af194a4df954/go.mod h1:MkUrD3iIh8/YTNv/z5+EJvgWknnI1qAlYZF+7vAy3JQ=
 github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo=
 github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elastic/go-licenser v0.4.1 h1:1xDURsc8pL5zYT9R29425J3vkHdt4RT5TNEMeRN48x4=
@@ -440,6 +438,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/mrodm/elastic-package v0.53.1-0.20231023144223-2e1f0a1077ec h1:TVtK7tCD7QHmvGQFdtMfRZZy4a5pHYdVyI6kqXbQxKY=
+github.com/mrodm/elastic-package v0.53.1-0.20231023144223-2e1f0a1077ec/go.mod h1:MkUrD3iIh8/YTNv/z5+EJvgWknnI1qAlYZF+7vAy3JQ=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=

From 293b70a2fa5e9ae65c597f11ab08548c483c3c06 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 23 Oct 2023 16:49:40 +0200
Subject: [PATCH 111/139] Add exception for fleet_server

---
 .buildkite/scripts/run_integrations_sync.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index 3f3e28ef349..a13732fc487 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -43,6 +43,11 @@ for integration in $(list_all_directories); do
     clean_safe_logs
 
     if [[ ${SERVERLESS} == "true" ]] ; then
+        if [[ "${integration}" == "fleet_server" ]]; then
+            echo "- [${integration}] not supported" >> ${SKIPPED_PACKAGES_FILE_PATH}
+            popd > /dev/null
+            continue
+        fi
         if ! is_spec_3_0_0 ]]; then
             echo "Not v3 spec version. Skipped"
             echo "- [${integration}] spec <3.0.0" >> ${SKIPPED_PACKAGES_FILE_PATH}

From f9b74bc5e0d7e61fe66a83a2d0a0f23630f2778e Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 23 Oct 2023 16:58:52 +0200
Subject: [PATCH 112/139] Add variables in case it is needed to send messages
 to slack

---
 catalog-info.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/catalog-info.yaml b/catalog-info.yaml
index ce660c5f754..de0357b50dd 100644
--- a/catalog-info.yaml
+++ b/catalog-info.yaml
@@ -105,6 +105,12 @@ spec:
       cancel_intermediate_builds_branch_filter: '!main'
       skip_intermediate_builds: true
       skip_intermediate_builds_branch_filter: '!main'
+      # env:
+      #   ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+#     #     SLACK_NOTIFICATIONS_CHANNEL: '#beats-build' as the current jenkins schedule-daily?
+      #   SLACK_NOTIFICATIONS_CHANNEL: '#ingest-test-notifications'   #TODO: will be changed before GO-PROD
+      #   SLACK_NOTIFICATIONS_ALL_BRANCHES: 'true'
+      #   SLACK_NOTIFICATIONS_ON_SUCCESS: 'true'
       teams:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ

From 3e298e17f3e37d0420bcfa975ad77c42da4be030 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 23 Oct 2023 18:33:01 +0200
Subject: [PATCH 113/139] Add input steps

---
 .buildkite/pipeline.serverless.yml          | 19 ++++++++++-
 .buildkite/scripts/common.sh                | 18 +++++++---
 .buildkite/scripts/run_integrations_sync.sh | 37 +++++++++++++++------
 3 files changed, 57 insertions(+), 17 deletions(-)

diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml
index 3192295960e..5618905c049 100644
--- a/.buildkite/pipeline.serverless.yml
+++ b/.buildkite/pipeline.serverless.yml
@@ -17,6 +17,23 @@ env:
   ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
 
 steps:
+  - input: "Input values for the variables"
+    key: "input-variables"
+    fields:
+    - select: "SERVERLESS_PROJECT"
+      key: "SERVERLESS_PROJECT"
+      options:
+        - label: "observability"
+          value: "observability"
+        - label: "security"
+          value: "security"
+      default: "observability"
+    if: "build.source == 'ui'"
+
+  - wait: ~
+    if: "build.source == 'ui'"
+    allow_dependency_failure: false
+
   - label: ":white_check_mark: Check go sources"
     key: "check"
     command: ".buildkite/scripts/check_sources.sh"
@@ -25,7 +42,7 @@ steps:
       cpu: "8"
       memory: "4G"
 
-  - label: "Check integrations in serverless - project ${SERVERLESS_PROJECT}"
+  - label: "Check integrations in serverless"
     key: "run-integrations-serverless-project"
     command: ".buildkite/scripts/run_integrations_sync.sh"
     timeout_in_minutes: 120
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 8a8ec04ef7b..b1e98f0d048 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -9,6 +9,13 @@ hw_type="$(uname -m)"
 GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
 export ELASTIC_PACKAGE_BIN=${WORKSPACE}/build/elastic-package
 
+running_on_buildkite() {
+    if [[ "${BUILDKITE:-"false"}" == "true" ]]; then
+        return 0
+    fi
+    return 1
+}
+
 retry() {
   local retries=$1
   shift
@@ -138,16 +145,17 @@ google_cloud_signing_auth() {
 
 google_cloud_logout_active_account() {
   local active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null)
-  if [ -n "$active_account" ]; then
+    if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then
     echo "Logging out from GCP for active account"
     gcloud auth revoke $active_account > /dev/null 2>&1
-    if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then
-      unset GOOGLE_APPLICATION_CREDENTIALS
-    fi
-    cleanup
   else
     echo "No active GCP accounts found."
   fi
+
+  if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then
+    rm -rf ${GOOGLE_APPLICATION_CREDENTIALS}
+    unset GOOGLE_APPLICATION_CREDENTIALS
+  fi
 }
 
 check_git_diff() {
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_sync.sh
index a13732fc487..f3bdacf44c1 100755
--- a/.buildkite/scripts/run_integrations_sync.sh
+++ b/.buildkite/scripts/run_integrations_sync.sh
@@ -8,16 +8,29 @@ set -euo pipefail
 STACK_VERSION=${STACK_VERSION:-""}
 FORCE_CHECK_ALL=${FORCE_CHECK_ALL:-"false"}
 SKIP_PUBLISHING=${SKIP_PUBLISHING:-"false"}
+SKIPPED_PACKAGES_FILE_PATH="${WORKSPACE}/skipped_packages.txt"
+FAILED_PACKAGES_FILE_PATH="${WORKSPACE}/failed_packages.txt"
 SERVERLESS=${SERVERLESS:-"false"}
-SERVERLESS_PROJECT=${SERVERLESS_PROJECT:-"observability"}
 
+if running_on_buildkite; then
+    # just get the value from meta-data if it is running on Buildkite
+    if buildkite-agent meta-data exists SERVERLESS_PROJECT; then
+        SERVERLESS_PROJECT="$(buildkite-agent meta-data get SERVERLESS_PROJECT)"
+    fi
+fi
+
+SERVERLESS_PROJECT=${SERVERLESS_PROJECT:-"observability"}
+echo "Running packages on Serverles project type: ${SERVERLESS_PROJECT}"
+if running_on_buildkite; then
+    buildkite-agent annotate "Serverless Project: ${SERVERLESS_PROJECT}" --style "info"
+fi
 
-SKIPPED_PACKAGES_FILE_PATH="${WORKSPACE}/skipped_packages.txt"
-FAILED_PACKAGES_FILE_PATH="${WORKSPACE}/failed_packages.txt"
 
 if [ ! -d packages ]; then
     echo "Missing packages folder"
-    buildkite-agent annotate "Missing packages folder" --style "error"
+    if running_on_buildkite ; then
+        buildkite-agent annotate "Missing packages folder" --style "error"
+    fi
     exit 1
 fi
 
@@ -28,6 +41,7 @@ with_mage
 with_docker_compose
 with_kubernetes
 
+
 use_elastic_package
 
 prepare_serverless_stack
@@ -44,6 +58,7 @@ for integration in $(list_all_directories); do
 
     if [[ ${SERVERLESS} == "true" ]] ; then
         if [[ "${integration}" == "fleet_server" ]]; then
+            echo "fleet_server not supported. Skipped"
             echo "- [${integration}] not supported" >> ${SKIPPED_PACKAGES_FILE_PATH}
             popd > /dev/null
             continue
@@ -89,14 +104,14 @@ for integration in $(list_all_directories); do
 done
 popd > /dev/null
 
-if [ -f ${SKIPPED_PACKAGES_FILE_PATH} ]; then
-    echo "Found skipped_packages.txt"  # TODO: remove
-    create_collapsed_annotation "Skipped packages in ${SERVERLESS_PROJECT}" ${SKIPPED_PACKAGES_FILE_PATH} "info" "ctx-skipped-packages-${SERVERLESS_PROJECT}"
-fi
+if running_on_buildkite ; then
+    if [ -f ${SKIPPED_PACKAGES_FILE_PATH} ]; then
+        create_collapsed_annotation "Skipped packages in ${SERVERLESS_PROJECT}" ${SKIPPED_PACKAGES_FILE_PATH} "info" "ctx-skipped-packages-${SERVERLESS_PROJECT}"
+    fi
 
-if [ -f ${FAILED_PACKAGES_FILE_PATH} ]; then
-    echo "Found failed_packages.txt"  # TODO: remove
-    create_collapsed_annotation "Failed packages in ${SERVERLESS_PROJECT}" ${FAILED_PACKAGES_FILE_PATH} "error" "ctx-failed-packages-${SERVERLESS_PROJECT}"
+    if [ -f ${FAILED_PACKAGES_FILE_PATH} ]; then
+        create_collapsed_annotation "Failed packages in ${SERVERLESS_PROJECT}" ${FAILED_PACKAGES_FILE_PATH} "error" "ctx-failed-packages-${SERVERLESS_PROJECT}"
+    fi
 fi
 
 if [ $any_package_failing -eq 1 ] ; then

From 3a04e0536443688dc3b9fef949e77152ffffe702 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 23 Oct 2023 18:34:00 +0200
Subject: [PATCH 114/139] Rename script

---
 .buildkite/pipeline.serverless.yml                            | 2 +-
 .buildkite/pipeline.yml                                       | 4 ++--
 ...tegrations_sync.sh => run_integrations_with_serverless.sh} | 0
 3 files changed, 3 insertions(+), 3 deletions(-)
 rename .buildkite/scripts/{run_integrations_sync.sh => run_integrations_with_serverless.sh} (100%)

diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml
index 5618905c049..c1cb670317b 100644
--- a/.buildkite/pipeline.serverless.yml
+++ b/.buildkite/pipeline.serverless.yml
@@ -44,7 +44,7 @@ steps:
 
   - label: "Check integrations in serverless"
     key: "run-integrations-serverless-project"
-    command: ".buildkite/scripts/run_integrations_sync.sh"
+    command: ".buildkite/scripts/run_integrations_with_serverless.sh"
     timeout_in_minutes: 120
     env:
       SERVERLESS: true
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index be0753f887d..24b15c6ef15 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -38,7 +38,7 @@ steps:
   # to be moved to schedule-daily job
   - label: "Check integrations in serverless - project observability"
     key: "test-integrations-serverless-obs"
-    command: ".buildkite/scripts/run_integrations_sync.sh"
+    command: ".buildkite/scripts/run_integrations_with_serverless.sh"
     timeout_in_minutes: 120
     env:
       SERVERLESS: true
@@ -57,7 +57,7 @@ steps:
 
   - label: "Check integrations in serverless - project security"
     key: "test-integrations-serverless-security"
-    command: ".buildkite/scripts/run_integrations_sync.sh"
+    command: ".buildkite/scripts/run_integrations_with_serverless.sh"
     timeout_in_minutes: 120
     env:
       SERVERLESS: true
diff --git a/.buildkite/scripts/run_integrations_sync.sh b/.buildkite/scripts/run_integrations_with_serverless.sh
similarity index 100%
rename from .buildkite/scripts/run_integrations_sync.sh
rename to .buildkite/scripts/run_integrations_with_serverless.sh

From e54eae677432f7812715eb1e4436a9b774542d41 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 23 Oct 2023 18:34:51 +0200
Subject: [PATCH 115/139] Remove debug statement from Jenkins

---
 .ci/Jenkinsfile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index d93bc7de4b6..f8996fbfbb6 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -278,8 +278,6 @@ def checkGitDiff() {
 }
 
 def isPrAffected(integrationName) {
-  return false // TODO: remove after tests
-
   def manifest = readYaml(file: "${integrationName}/manifest.yml")
 
   // Packages supported in Kibana >= 8.0.0, shouldn't be included in daily tests of the stack 7.x

From d6d41d50f966c83014db89b919bc50d3b6335e71 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 23 Oct 2023 18:51:57 +0200
Subject: [PATCH 116/139] Add script to find oldest supported version

---
 .../scripts/find_oldest_supported_version.py  | 272 ++++++++++++++++++
 1 file changed, 272 insertions(+)
 create mode 100644 .buildkite/scripts/find_oldest_supported_version.py

diff --git a/.buildkite/scripts/find_oldest_supported_version.py b/.buildkite/scripts/find_oldest_supported_version.py
new file mode 100644
index 00000000000..3a7be4d04b4
--- /dev/null
+++ b/.buildkite/scripts/find_oldest_supported_version.py
@@ -0,0 +1,272 @@
+#!/bin/env python3
+import argparse
+import requests
+import sys
+import yaml
+import unittest
+
+VERSION_URL = "https://artifacts-api.elastic.co/v1/versions?x-elastic-no-kpi=true"
+
+
+def fetch_version():
+    return requests.get(VERSION_URL).json()
+
+
+def find_oldest_supported_version(kibana_version_condition: str) -> str:
+    # The logic of this function is copied from https://github.com/elastic/apm-pipeline-library/blob/main/vars/findOldestSupportedVersion.groovy
+
+    if "||" in kibana_version_condition and kibana_version_condition.index("||") >= 0:
+        return handle_or(kibana_version_condition)
+
+    available_versions = fetch_version()
+    version = remove_operator(kibana_version_condition)
+    parts = version.split(".")
+
+    # If this is specifying a major or a minor only, check with the zero version.
+    while len(parts) < 3:
+        version += ".0"
+        parts.append("0")
+
+    major, minor, patch = parts[0], parts[1], parts[2]
+
+    # Use the snapshot if this is the last patch version.
+    next_patch = ".".join((major, minor, str(int(patch)+1)))
+    next_patch_exists = (
+        next_patch in available_versions.get("versions", []) or
+        f"{next_patch}-SNAPSHOT" in available_versions.get("versions", [])
+    )
+
+    snapshot_version = f"{version}-SNAPSHOT"
+    if not next_patch_exists and (snapshot_version in available_versions.get("versions", [])):
+        return snapshot_version
+
+    # Use the version as is if it exists.
+    if version in available_versions.get("version", []):
+        return version
+
+    # Old minors may not be available in artifacts-api, if it is older
+    # than the others in the same major, return the version as is.
+    older = True
+    for available_version in available_versions.get("versions", []):
+        available_parts = available_version.split(".")
+        if len(available_parts) < 2:
+            continue
+
+        available_major = available_parts[0]
+        available_minor = available_parts[1]
+        if major == available_major and minor > available_minor:
+            older = False
+            break
+    if older:
+        return version
+
+    # If no version has been found so far, try with the snapshot of the next version
+    # in the current major.
+    major_snapshot = f"{major}.x-SNAPSHOT"
+    if major_snapshot in available_versions.get("aliases", []):
+        return major_snapshot
+
+    # Otherwise, return it, whatever this is.
+    return version
+
+
+def remove_operator(kibana_version_condition: str) -> str:
+    if kibana_version_condition[0].isdigit():
+        return kibana_version_condition
+    elif kibana_version_condition.startswith("^") or kibana_version_condition.startswith("~"):
+        return kibana_version_condition[1:]
+    elif kibana_version_condition.startswith(">="):
+        return kibana_version_condition[2:]
+    raise Exception("kibana version condition supports only ^, ~ and >= operators")
+
+
+def handle_or(kibana_version_condition: str):
+    if "||" not in kibana_version_condition:
+        raise Exception(f"no conditions found in '{kibana_version_condition}'")
+
+    conditions = kibana_version_condition.split("||")
+    result = ""
+    for cond in conditions:
+        candidate = find_oldest_supported_version(cond)
+        if result == "" or candidate < result:
+            result = candidate
+
+    return result
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Prepare Elastic stack")
+    parser.add_argument("--manifest-path",
+                        required=False,
+                        default="manifest.yml",
+                        help="path of manifest file")
+    parser.add_argument("--test",
+                        required=False,
+                        action="store_true",
+                        default=False,
+                        help="trigger test")
+
+    args, unknown = parser.parse_known_args()
+    # Set this for unittest
+    sys.argv[1:] = unknown
+    return args
+
+
+def run(cfg: argparse.Namespace):
+    with open(cfg.manifest_path, "r") as src:
+        manifest_doc = yaml.safe_load(src)
+
+    kibana_version_condition = ""
+    if "kibana.version" in manifest_doc["conditions"]:
+        kibana_version_condition = manifest_doc["conditions"]["kibana.version"]
+    elif "kibana" in manifest_doc["conditions"]:
+        kibana_version_condition = manifest_doc["conditions"]["kibana"]["version"]
+
+    if kibana_version_condition:
+        print(find_oldest_supported_version(kibana_version_condition), end="")
+    else:
+        print("null")
+
+
+def main():
+    cfg = parse_args()
+
+    if cfg.test:
+        unittest.main()
+    else:
+        run(cfg)
+
+
+# Test: meant to run locally with --test
+
+class TestFindOldestSupportVersion(unittest.TestCase):
+    """Testcase for find_oldest_supported_version."""
+
+    mock_data = {
+        "versions": [
+            "7.17.10",
+            "7.17.11",
+            "7.17.12",
+            "7.17.13-SNAPSHOT",
+            "7.17.13",
+            "7.17.14-SNAPSHOT",
+            "8.7.0",
+            "8.7.1",
+            "8.8.0",
+            "8.8.1",
+            "8.8.2",
+            "8.9.0",
+            "8.9.1-SNAPSHOT",
+            "8.9.1",
+            "8.9.2-SNAPSHOT",
+            "8.9.2",
+            "8.10.0-SNAPSHOT",
+            "8.10.0",
+            "8.10.1-SNAPSHOT",
+            "8.11.0-SNAPSHOT"
+        ],
+        "aliases": [
+            "7.17-SNAPSHOT",
+            "7.17",
+            "8.7",
+            "8.8",
+            "8.9-SNAPSHOT",
+            "8.9",
+            "8.10-SNAPSHOT",
+            "8.10",
+            "8.11-SNAPSHOT"
+        ],
+        "manifests": {
+            "last-update-time": "Thu, 14 Sep 2023 16:03:46 UTC",
+            "seconds-since-last-update": 107
+        }
+    }
+
+    def setUp(self):
+        super().setUp()
+        global fetch_version
+        self._fetch_version = fetch_version
+        def fetch_version(): return self.mock_data
+
+    def tearDown(self):
+        global fetch_version
+        fetch_version = self._fetch_version
+        super().tearDown()
+
+    def test_next_patch_does_not_exits_and_available_version_contains_snapshot(self):
+        self.assertEqual(find_oldest_supported_version("7.17.14"), "7.17.14-SNAPSHOT")
+        self.assertEqual(find_oldest_supported_version("8.9.2"), "8.9.2-SNAPSHOT")
+        self.assertEqual(find_oldest_supported_version("8.10.1"), "8.10.1-SNAPSHOT")
+        self.assertEqual(find_oldest_supported_version("8.11.0"), "8.11.0-SNAPSHOT")
+
+    def test_available_version_contains_kibana_version(self):
+        self.assertEqual(find_oldest_supported_version("7.17.10"), "7.17.10")
+        self.assertEqual(find_oldest_supported_version("7.17.11"), "7.17.11")
+        self.assertEqual(find_oldest_supported_version("7.17.12"), "7.17.12")
+        self.assertEqual(find_oldest_supported_version("7.17.13"), "7.17.13")
+        self.assertEqual(find_oldest_supported_version("8.7.1"), "8.7.1")
+        self.assertEqual(find_oldest_supported_version("8.10.0"), "8.10.0")
+
+    def test_too_old_to_be_in_api(self):
+        self.assertEqual(find_oldest_supported_version("7.16.0"), "7.16.0")
+        self.assertEqual(find_oldest_supported_version("8.6.0"), "8.6.0")
+
+    def test_or(self):
+        self.assertEqual(find_oldest_supported_version("8.6.0||8.7.0"), "8.6.0")
+        self.assertEqual(find_oldest_supported_version("8.9.2||8.9.1||7.17.14"), "7.17.14-SNAPSHOT")
+
+    def test_mix(self):
+        self.assertEqual(find_oldest_supported_version("^8.6.0||~8.7.0"), "8.6.0")
+        self.assertEqual(find_oldest_supported_version("8.9.2||8.9.1||7.17.14"), "7.17.14-SNAPSHOT")
+        self.assertEqual(find_oldest_supported_version(
+            "~8.9.2||>=8.11.0||7.17.14"), "7.17.14-SNAPSHOT")
+
+
+class TestRemoveOperator(unittest.TestCase):
+    """Testcase for remove_operator."""
+
+    def test_no_operator(self):
+        self.assertEqual(remove_operator("1.0.0"), "1.0.0")
+
+    def test_circumflex(self):
+        self.assertEqual(remove_operator("^1.0.0"), "1.0.0")
+
+    def test_tilda(self):
+        self.assertEqual(remove_operator("~1.0.0"), "1.0.0")
+
+    def test_greater_or_equal(self):
+        self.assertEqual(remove_operator(">=1.0.0"), "1.0.0")
+
+    def test_unknown(self):
+        with self.assertRaises(Exception):
+            remove_operator("<=1.0.0")
+        with self.assertRaises(Exception):
+            remove_operator("==1.0.0")
+
+
+class TestHandleOr(unittest.TestCase):
+    """Testcase for handle_or."""
+
+    def test_single_condition(self):
+        with self.assertRaises(Exception):
+            handle_or("0.0.1")
+
+    def test_happy_path(self):
+        # Mock temporarly with the identiy function.
+        global find_oldest_supported_version
+        old_func = find_oldest_supported_version
+        def find_oldest_supported_version(x): return x
+
+        self.assertEqual(handle_or("0.1||0.2"), "0.1")
+        self.assertEqual(handle_or("0.2||0.2.1||0.2.3-alpha"), "0.2")
+        self.assertEqual(handle_or("1.2||1.2.0||3.2.3-alpha"), "1.2")
+
+        # restore mock
+        find_oldest_supported_version = old_func
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        pass

From e3897241f947137e3c313ca15f718d7f928ec482 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 23 Oct 2023 19:03:09 +0200
Subject: [PATCH 117/139] Set git pager as cat

---
 .buildkite/hooks/pre-command | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 1304a14057c..bf16a3a1a3b 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -4,6 +4,9 @@ source .buildkite/scripts/common.sh
 
 set -euo pipefail
 
+# Avoid any pager when running git commands
+git config --global core.pager 'cat'
+
 export UPLOAD_SAFE_LOGS=${UPLOAD_SAFE_LOGS:-"0"}
 export BASE_DIR=$(pwd)
 export GO_VERSION=$(cat .go-version)

From df6fb5e88d70b61c74aee56862271544f77355df Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 23 Oct 2023 19:04:21 +0200
Subject: [PATCH 118/139] Remove unused variable

---
 .buildkite/hooks/pre-command | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index bf16a3a1a3b..897a67a2941 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -10,7 +10,6 @@ git config --global core.pager 'cat'
 export UPLOAD_SAFE_LOGS=${UPLOAD_SAFE_LOGS:-"0"}
 export BASE_DIR=$(pwd)
 export GO_VERSION=$(cat .go-version)
-export startTimeInMillis="$(date +%s%3N)"
 
 export REPO_NAME=$(repo_name "${BUILDKITE_REPO}")
 export TMP_FOLDER_TEMPLATE_BASE="tmp.${REPO_NAME}"

From fd667b1f01e697575bd8845900a4b0b180af1aca Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 23 Oct 2023 19:12:26 +0200
Subject: [PATCH 119/139] Re-order and add comments into common.sh

---
 .buildkite/scripts/common.sh | 81 +++++++++++++++++++-----------------
 1 file changed, 42 insertions(+), 39 deletions(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index b1e98f0d048..54bc43b93fc 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -49,6 +49,14 @@ unset_secrets () {
   done
 }
 
+repo_name() {
+    # Example of URL: git@github.com:acme-inc/my-project.git
+    local repoUrl=$1
+
+    orgAndRepo=$(echo $repoUrl | cut -d':' -f 2)
+    echo "$(basename ${orgAndRepo} .git)"
+}
+
 check_platform_architeture() {
   case "${hw_type}" in
     "x86_64")
@@ -66,6 +74,7 @@ check_platform_architeture() {
   esac
 }
 
+# Helpers to install required tools
 create_bin_folder() {
   mkdir -p ${BIN_FOLDER}
 }
@@ -129,6 +138,24 @@ with_kubernetes() {
     which kubectl
 }
 
+with_yq() {
+    check_platform_architeture
+    local platform_type_lowercase="${platform_type,,}"
+    local binary="yq_${platform_type_lowercase}_${arch_type}"
+
+    retry 5 curl -sSL -o ${BIN_FOLDER}/yq.tar.gz "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${binary}.tar.gz"
+
+    tar -C ${BIN_FOLDER} -xpf ${BIN_FOLDER}/yq.tar.gz ./yq_linux_amd64
+
+    mv ${BIN_FOLDER}/yq_linux_amd64 ${BIN_FOLDER}/yq
+    chmod +x ${BIN_FOLDER}/yq
+    yq --version
+
+    rm -rf ${BIN_FOLDER}/yq.tar.gz
+}
+
+
+## Logging and logout from Google Cloud
 google_cloud_upload_auth() {
   local secretFileLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX")/${GOOGLE_CREDENTIALS_FILENAME}
   echo "${PRIVATE_INFRA_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
@@ -143,6 +170,16 @@ google_cloud_signing_auth() {
   export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
 }
 
+google_cloud_auth_safe_logs() {
+    local gsUtilLocation=$(mktemp -d -p ${WORKSPACE} -t ${TMP_FOLDER_TEMPLATE})
+    local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
+
+    echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
+
+    gcloud auth activate-service-account --key-file ${secretFileLocation} 2> /dev/null
+    export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
+}
+
 google_cloud_logout_active_account() {
   local active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null)
     if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then
@@ -158,6 +195,7 @@ google_cloud_logout_active_account() {
   fi
 }
 
+## Helpers for integrations pipelines
 check_git_diff() {
     cd ${WORKSPACE}
     echo "git update-index"
@@ -166,24 +204,6 @@ check_git_diff() {
     git diff-index --exit-code HEAD --
 }
 
-with_yq() {
-    check_platform_architeture
-    local platform_type_lowercase="${platform_type,,}"
-    local binary="yq_${platform_type_lowercase}_${arch_type}"
-
-    # TODO: remove debug
-    echo curl -sSL -o ${BIN_FOLDER}/yq.tar.gz "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${binary}.tar.gz"
-    retry 5 curl -sSL -o ${BIN_FOLDER}/yq.tar.gz "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${binary}.tar.gz"
-
-    tar -C ${BIN_FOLDER} -xpf ${BIN_FOLDER}/yq.tar.gz ./yq_linux_amd64
-
-    mv ${BIN_FOLDER}/yq_linux_amd64 ${BIN_FOLDER}/yq
-    chmod +x ${BIN_FOLDER}/yq
-    yq --version
-
-    rm -rf ${BIN_FOLDER}/yq.tar.gz
-}
-
 use_elastic_package() {
     echo "--- Installing elastic-package"
     mkdir -p build
@@ -201,15 +221,6 @@ is_already_published() {
     return 1
 }
 
-repo_name() {
-    # Example of URL: git@github.com:acme-inc/my-project.git
-    local repoUrl=$1
-
-    orgAndRepo=$(echo $repoUrl | cut -d':' -f 2)
-    echo "$(basename ${orgAndRepo} .git)"
-}
-
-
 create_kind_cluster() {
     echo "--- Create kind cluster"
     kind create cluster --config ${WORKSPACE}/kind-config.yaml --image kindest/node:${K8S_VERSION}
@@ -318,10 +329,12 @@ prepare_serverless_stack() {
     local args="-v"
     if [ -n "${STACK_VERSION}" ]; then
         args="${args} --version ${STACK_VERSION}"
-    # TODO What stack version to use (for agents) in serverless?
-    # else
     fi
 
+    # Currently, if STACK_VERSION is not defined, for serverless it will be
+    # used as Elastic stack version (for agents) the default version in elastic-package
+
+    # Creating a new profile allow to set a specific name for the serverless project
     create_elastic_package_profile "integrations-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_BUILD_NUMBER}-${SERVERLESS_PROJECT}"
 
     export EC_API_KEY=${EC_API_KEY_SECRET}
@@ -529,16 +542,6 @@ create_collapsed_annotation() {
     rm -f ${annotation_file}
 }
 
-google_cloud_auth_safe_logs() {
-    local gsUtilLocation=$(mktemp -d -p ${WORKSPACE} -t ${TMP_FOLDER_TEMPLATE})
-    local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
-
-    echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
-
-    gcloud auth activate-service-account --key-file ${secretFileLocation} 2> /dev/null
-    export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
-}
-
 upload_safe_logs() {
     local bucket="$1"
     local source="$2"

From 7dffa6e10ce459da4c770b13374aff93d1d2a3da Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 24 Oct 2023 10:32:50 +0200
Subject: [PATCH 120/139] Remove leftover

---
 .buildkite/scripts/common.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 54bc43b93fc..7cb61485f3e 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -346,7 +346,6 @@ prepare_serverless_stack() {
         ${args} \
         --provider serverless \
         -U stack.serverless.region=${EC_REGION_SECRET},stack.serverless.type=${SERVERLESS_PROJECT} 2>&1 | egrep -v "^Password: " # To remove password from the output
-    # ${ELASTIC_PACKAGE_BIN} stack up -d ${args}
     echo ""
     ${ELASTIC_PACKAGE_BIN} stack status
     echo ""

From afaebcbc1e96fec73a554056f408a04915be38cb Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 24 Oct 2023 10:34:50 +0200
Subject: [PATCH 121/139] Add note about not using system tests

---
 .buildkite/scripts/common.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 7cb61485f3e..389d1ea47cd 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -487,6 +487,9 @@ install_package() {
     return 0
 }
 
+# Currently, system tests are not run in serverless to avoid lasting the build
+# too much time, since all packages are run in the same step one by one.
+# Packages are tested one by one to avoid creating more than 100 projects for one build.
 test_package_in_serverless() {
     local integration=$1
     TEST_OPTIONS="-v --report-format xUnit --report-output file"

From 73febabea9dd236a364130434e9c04434123bbf6 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 24 Oct 2023 10:36:38 +0200
Subject: [PATCH 122/139] Remove extra characters

---
 .buildkite/scripts/run_integrations_with_serverless.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/run_integrations_with_serverless.sh b/.buildkite/scripts/run_integrations_with_serverless.sh
index f3bdacf44c1..23f3bfe184f 100755
--- a/.buildkite/scripts/run_integrations_with_serverless.sh
+++ b/.buildkite/scripts/run_integrations_with_serverless.sh
@@ -63,7 +63,7 @@ for integration in $(list_all_directories); do
             popd > /dev/null
             continue
         fi
-        if ! is_spec_3_0_0 ]]; then
+        if ! is_spec_3_0_0 ; then
             echo "Not v3 spec version. Skipped"
             echo "- [${integration}] spec <3.0.0" >> ${SKIPPED_PACKAGES_FILE_PATH}
             popd > /dev/null

From c0fbc5738d95333ee98a36b644fbcd091962823d Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 24 Oct 2023 10:43:44 +0200
Subject: [PATCH 123/139] Add context for info annotation

---
 .buildkite/scripts/run_integrations_with_serverless.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/run_integrations_with_serverless.sh b/.buildkite/scripts/run_integrations_with_serverless.sh
index 23f3bfe184f..1e8e15b82d4 100755
--- a/.buildkite/scripts/run_integrations_with_serverless.sh
+++ b/.buildkite/scripts/run_integrations_with_serverless.sh
@@ -5,12 +5,13 @@ source .buildkite/scripts/common.sh
 set -euo pipefail
 
 # default values
+SERVERLESS=${SERVERLESS:-"false"}
 STACK_VERSION=${STACK_VERSION:-""}
 FORCE_CHECK_ALL=${FORCE_CHECK_ALL:-"false"}
 SKIP_PUBLISHING=${SKIP_PUBLISHING:-"false"}
+
 SKIPPED_PACKAGES_FILE_PATH="${WORKSPACE}/skipped_packages.txt"
 FAILED_PACKAGES_FILE_PATH="${WORKSPACE}/failed_packages.txt"
-SERVERLESS=${SERVERLESS:-"false"}
 
 if running_on_buildkite; then
     # just get the value from meta-data if it is running on Buildkite
@@ -22,7 +23,7 @@ fi
 SERVERLESS_PROJECT=${SERVERLESS_PROJECT:-"observability"}
 echo "Running packages on Serverles project type: ${SERVERLESS_PROJECT}"
 if running_on_buildkite; then
-    buildkite-agent annotate "Serverless Project: ${SERVERLESS_PROJECT}" --style "info"
+    buildkite-agent annotate "Serverless Project: ${SERVERLESS_PROJECT}" --context "ctx-info-${SERVERLESS_PROJECT}" --style "info"
 fi
 
 

From aae1db9e868c20b48ef377ae37b8b808a3be6efd Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 24 Oct 2023 10:44:20 +0200
Subject: [PATCH 124/139] Move to a function code run in each iteration

---
 .../run_integrations_with_serverless.sh       | 42 ++++++++++---------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/.buildkite/scripts/run_integrations_with_serverless.sh b/.buildkite/scripts/run_integrations_with_serverless.sh
index 1e8e15b82d4..6c55c955ce9 100755
--- a/.buildkite/scripts/run_integrations_with_serverless.sh
+++ b/.buildkite/scripts/run_integrations_with_serverless.sh
@@ -42,37 +42,30 @@ with_mage
 with_docker_compose
 with_kubernetes
 
+check_package_in_serverless() {
+    local package="$1"
 
-use_elastic_package
-
-prepare_serverless_stack
-
-pushd packages > /dev/null
-
-any_package_failing=0
-
-for integration in $(list_all_directories); do
-    echo "--- Package ${integration}: check"
-    pushd ${integration} > /dev/null
+    echo "--- Package ${package}: check"
+    pushd ${package} > /dev/null
 
     clean_safe_logs
 
     if [[ ${SERVERLESS} == "true" ]] ; then
-        if [[ "${integration}" == "fleet_server" ]]; then
+        if [[ "${package}" == "fleet_server" ]]; then
             echo "fleet_server not supported. Skipped"
-            echo "- [${integration}] not supported" >> ${SKIPPED_PACKAGES_FILE_PATH}
+            echo "- [${package}] not supported" >> ${SKIPPED_PACKAGES_FILE_PATH}
             popd > /dev/null
             continue
         fi
         if ! is_spec_3_0_0 ; then
             echo "Not v3 spec version. Skipped"
-            echo "- [${integration}] spec <3.0.0" >> ${SKIPPED_PACKAGES_FILE_PATH}
+            echo "- [${package}] spec <3.0.0" >> ${SKIPPED_PACKAGES_FILE_PATH}
             popd > /dev/null
             continue
         fi
     fi
 
-    if ! reason=$(is_pr_affected ${integration}) ; then
+    if ! reason=$(is_pr_affected ${package}) ; then
         echo "${reason}"
         echo "- ${reason}" >> ${SKIPPED_PACKAGES_FILE_PATH}
         popd > /dev/null
@@ -86,9 +79,9 @@ for integration in $(list_all_directories); do
         create_kind_cluster
     fi
 
-    if ! run_tests_package ${integration} ; then
-        echo "[${integration}] run_tests_package failed"
-        echo "- ${integration}" >> ${FAILED_PACKAGES_FILE_PATH}
+    if ! run_tests_package ${package} ; then
+        echo "[${package}] run_tests_package failed"
+        echo "- ${package}" >> ${FAILED_PACKAGES_FILE_PATH}
         any_package_failing=1
     fi
 
@@ -99,9 +92,20 @@ for integration in $(list_all_directories); do
         delete_kind_cluster
     fi
 
-    teardown_serverless_test_package ${integration}
+    teardown_serverless_test_package ${package}
 
     popd > /dev/null
+}
+
+use_elastic_package
+
+prepare_serverless_stack
+
+any_package_failing=0
+
+pushd packages > /dev/null
+for package in $(list_all_directories); do
+    check_package_in_serverless ${package}
 done
 popd > /dev/null
 

From 1737413ed4f255545295ef47fa4ea7bed065b0af Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 24 Oct 2023 11:52:33 +0200
Subject: [PATCH 125/139] Rename

---
 .buildkite/scripts/common.sh | 66 ++++++++++++++++++------------------
 1 file changed, 33 insertions(+), 33 deletions(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 389d1ea47cd..308a1f0387b 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -379,20 +379,20 @@ get_to_changeset() {
 # TODO: it is required to have GIT_PREVIOUS_COMMIT and GIT_PREVIOUS_SUCCESSFUL_COMMIT
 # as in Jenkins to set the right from (changesets)
 is_pr_affected() {
-    local integration="${1}"
+    local package="${1}"
 
     if ! is_supported_stack ; then
-        echo "[${integration}] PR is not affected: unsupported stack (${STACK_VERSION})"
+        echo "[${package}] PR is not affected: unsupported stack (${STACK_VERSION})"
         return 1
     fi
 
     if ! is_supported_capability ; then
-        echo "[${integration}] PR is not affected: capabilities not mached with the project (${SERVERLESS_PROJECT})"
+        echo "[${package}] PR is not affected: capabilities not mached with the project (${SERVERLESS_PROJECT})"
         return 1
     fi
 
     if [[ ${FORCE_CHECK_ALL} == "true" ]];then
-        echo "[${integration}] PR is affected: \"force_check_all\" parameter enabled"
+        echo "[${package}] PR is affected: \"force_check_all\" parameter enabled"
         return 0
     fi
 
@@ -406,24 +406,24 @@ is_pr_affected() {
     # GIT_PREVIOUS_SUCCESSFUL_COMMIT to check if the branch is still healthy.
     # If this value is not available, check with last commit.
     if [[ ${BUILDKITE_BRANCH} == "main" || ${BUILDKITE_BRANCH} =~ ^backport- ]]; then
-        echo "[${integration}] PR is affected: running on ${BUILDKITE_BRANCH} branch"
+        echo "[${package}] PR is affected: running on ${BUILDKITE_BRANCH} branch"
         # TODO: get previous successful commit as in Jenkins (groovy)
         # from = env.GIT_PREVIOUS_SUCCESSFUL_COMMIT?.trim() ? env.GIT_PREVIOUS_SUCCESSFUL_COMMIT : "origin/${env.BRANCH_NAME}^"
         from="origin/${BUILDKITE_BRANCH}^"
         to="origin/${BUILDKITE_BRANCH}"
     fi
 
-    echo "[${integration}] git-diff: check non-package files"
+    echo "[${package}] git-diff: check non-package files"
     if git diff --name-only $(git merge-base ${from} ${to}) ${to} | egrep '^(packages/|.github/CODEOWNERS)' ; then
-        echo "[${integration}] PR is affected: found non-package files"
+        echo "[${package}] PR is affected: found non-package files"
         return 0
     fi
-    echo "[${integration}] git-diff: check package files"
-    if git diff --name-only $(git merge-base ${from} ${to}) ${to} | egrep '^packages/${integration}/' ; then
-        echo "[${integration}] PR is affected: found package files"
+    echo "[${package}] git-diff: check package files"
+    if git diff --name-only $(git merge-base ${from} ${to}) ${to} | egrep '^packages/${package}/' ; then
+        echo "[${package}] PR is affected: found package files"
         return 0
     fi
-    echo "[${integration}] PR is not affected"
+    echo "[${package}] PR is not affected"
     return 1
 }
 
@@ -439,25 +439,25 @@ kubernetes_service_deployer_used() {
 }
 
 teardown_serverless_test_package() {
-    local integration=$1
+    local package=$1
     local build_directory="${WORKSPACE}/build"
-    local dump_directory="${build_directory}/elastic-stack-dump/${integration}"
+    local dump_directory="${build_directory}/elastic-stack-dump/${package}"
 
     echo "Collect Elastic stack logs"
     ${ELASTIC_PACKAGE_BIN} stack dump -v --output ${dump_directory}
 
-    upload_safe_logs_from_package ${integration} ${build_directory}
+    upload_safe_logs_from_package ${package} ${build_directory}
 }
 
 teardown_test_package() {
-    local integration=$1
+    local package=$1
     local build_directory="${WORKSPACE}/build"
-    local dump_directory="${build_directory}/elastic-stack-dump/${integration}"
+    local dump_directory="${build_directory}/elastic-stack-dump/${package}"
 
     echo "Collect Elastic stack logs"
     ${ELASTIC_PACKAGE_BIN} stack dump -v --output ${dump_directory}
 
-    upload_safe_logs_from_package ${integration} ${build_directory}
+    upload_safe_logs_from_package ${package} ${build_directory}
 
     echo "Take down the Elastic stack"
     ${ELASTIC_PACKAGE_BIN} stack down -v
@@ -468,8 +468,8 @@ list_all_directories() {
 }
 
 check_package() {
-    local integration=$1
-    echo "Check integration: ${integration}"
+    local package=$1
+    echo "Check package: ${package}"
     if ! ${ELASTIC_PACKAGE_BIN} check -v ; then
         return 1
     fi
@@ -478,8 +478,8 @@ check_package() {
 }
 
 install_package() {
-    local integration=$1
-    echo "Install integration: ${integration}"
+    local package=$1
+    echo "Install package: ${package}"
     if ! ${ELASTIC_PACKAGE_BIN} install -v ; then
         return 1
     fi
@@ -491,10 +491,10 @@ install_package() {
 # too much time, since all packages are run in the same step one by one.
 # Packages are tested one by one to avoid creating more than 100 projects for one build.
 test_package_in_serverless() {
-    local integration=$1
+    local package=$1
     TEST_OPTIONS="-v --report-format xUnit --report-output file"
 
-    echo "Test integration: ${integration}"
+    echo "Test package: ${package}"
     if ! ${ELASTIC_PACKAGE_BIN} test asset ${TEST_OPTIONS} --test-coverage ; then
         return 1
     fi
@@ -511,15 +511,15 @@ test_package_in_serverless() {
 }
 
 run_tests_package() {
-    local integration=$1
-    if ! check_package ${integration} ; then
+    local package=$1
+    if ! check_package ${package} ; then
         return 1
     fi
-    if ! install_package ${integration} ; then
+    if ! install_package ${package} ; then
         return 1
     fi
     if [[ $SERVERLESS == "true" ]]; then
-        if ! test_package_in_serverless ${integration} ; then
+        if ! test_package_in_serverless ${package} ; then
             return 1
         fi
     fi
@@ -579,7 +579,7 @@ upload_safe_logs_from_package() {
         return
     fi
 
-    local integration=$1
+    local package=$1
     local build_directory=$2
 
     local parent_folder="insecure-logs"
@@ -590,17 +590,17 @@ upload_safe_logs_from_package() {
 
     upload_safe_logs \
         "${JOB_GCS_BUCKET_INTERNAL}" \
-        "${build_directory}/elastic-stack-dump/${integration}/logs/elastic-agent-internal/*.*" \
-        "${parent_folder}/${integration}/elastic-agent-logs/"
+        "${build_directory}/elastic-stack-dump/${package}/logs/elastic-agent-internal/*.*" \
+        "${parent_folder}/${package}/elastic-agent-logs/"
 
     # required for <8.6.0
     upload_safe_logs \
         "${JOB_GCS_BUCKET_INTERNAL}" \
-        "${build_directory}/elastic-stack-dump/${integration}/logs/elastic-agent-internal/default/*" \
-        "${parent_folder}/${integration}/elastic-agent-logs/default/"
+        "${build_directory}/elastic-stack-dump/${package}/logs/elastic-agent-internal/default/*" \
+        "${parent_folder}/${package}/elastic-agent-logs/default/"
 
     upload_safe_logs \
         "${JOB_GCS_BUCKET_INTERNAL}" \
         "${build_directory}/container-logs/*.log" \
-        "${parent_folder}/${integration}/container-logs/"
+        "${parent_folder}/${package}/container-logs/"
 }

From 41af02ce854d89ab2c5deba66f70549caddcd206 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 24 Oct 2023 11:53:48 +0200
Subject: [PATCH 126/139] Add debug

---
 .buildkite/scripts/run_integrations_with_serverless.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/scripts/run_integrations_with_serverless.sh b/.buildkite/scripts/run_integrations_with_serverless.sh
index 6c55c955ce9..d42daf171bf 100755
--- a/.buildkite/scripts/run_integrations_with_serverless.sh
+++ b/.buildkite/scripts/run_integrations_with_serverless.sh
@@ -44,6 +44,7 @@ with_kubernetes
 
 check_package_in_serverless() {
     local package="$1"
+    echo "Current directory: $(pwd)"
 
     echo "--- Package ${package}: check"
     pushd ${package} > /dev/null

From 8b3ddf427e47163b121b5bb325edd2cb1be0ba2a Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 24 Oct 2023 12:08:36 +0200
Subject: [PATCH 127/139] Change continue for return - not in a loop anymore

---
 .buildkite/scripts/run_integrations_with_serverless.sh | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/run_integrations_with_serverless.sh b/.buildkite/scripts/run_integrations_with_serverless.sh
index d42daf171bf..aa700659411 100755
--- a/.buildkite/scripts/run_integrations_with_serverless.sh
+++ b/.buildkite/scripts/run_integrations_with_serverless.sh
@@ -44,7 +44,6 @@ with_kubernetes
 
 check_package_in_serverless() {
     local package="$1"
-    echo "Current directory: $(pwd)"
 
     echo "--- Package ${package}: check"
     pushd ${package} > /dev/null
@@ -56,13 +55,13 @@ check_package_in_serverless() {
             echo "fleet_server not supported. Skipped"
             echo "- [${package}] not supported" >> ${SKIPPED_PACKAGES_FILE_PATH}
             popd > /dev/null
-            continue
+            return
         fi
         if ! is_spec_3_0_0 ; then
             echo "Not v3 spec version. Skipped"
             echo "- [${package}] spec <3.0.0" >> ${SKIPPED_PACKAGES_FILE_PATH}
             popd > /dev/null
-            continue
+            return
         fi
     fi
 
@@ -70,7 +69,7 @@ check_package_in_serverless() {
         echo "${reason}"
         echo "- ${reason}" >> ${SKIPPED_PACKAGES_FILE_PATH}
         popd > /dev/null
-        continue
+        return
     fi
 
     use_kind=0

From e06e6dcea4901174e42db1071976fcf42be5cae7 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 24 Oct 2023 12:30:27 +0200
Subject: [PATCH 128/139] Update elastic-package

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index b983591cd39..3a9262c4293 100644
--- a/go.mod
+++ b/go.mod
@@ -203,4 +203,4 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
-replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20231023144223-2e1f0a1077ec
+replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20231024082918-7ca8edbc560e
diff --git a/go.sum b/go.sum
index fc3018691b1..6f5d91ef290 100644
--- a/go.sum
+++ b/go.sum
@@ -438,8 +438,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
-github.com/mrodm/elastic-package v0.53.1-0.20231023144223-2e1f0a1077ec h1:TVtK7tCD7QHmvGQFdtMfRZZy4a5pHYdVyI6kqXbQxKY=
-github.com/mrodm/elastic-package v0.53.1-0.20231023144223-2e1f0a1077ec/go.mod h1:MkUrD3iIh8/YTNv/z5+EJvgWknnI1qAlYZF+7vAy3JQ=
+github.com/mrodm/elastic-package v0.53.1-0.20231024082918-7ca8edbc560e h1:DAaAjdVAZedHuclu8HOkQ2ggLled5KIgBcqh8Bfetog=
+github.com/mrodm/elastic-package v0.53.1-0.20231024082918-7ca8edbc560e/go.mod h1:MkUrD3iIh8/YTNv/z5+EJvgWknnI1qAlYZF+7vAy3JQ=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=

From 3b751a851664db787a649cb46bbde0765336545c Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 24 Oct 2023 12:32:42 +0200
Subject: [PATCH 129/139] Add environment variable to avoid comapre results in
 pipeline tests

---
 .buildkite/pipeline.serverless.yml | 2 ++
 .buildkite/pipeline.yml            | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml
index c1cb670317b..66d7a72b12e 100644
--- a/.buildkite/pipeline.serverless.yml
+++ b/.buildkite/pipeline.serverless.yml
@@ -15,6 +15,8 @@ env:
   ELASTIC_PACKAGE_REPOSITORY_LICENSE: "licenses/Elastic-2.0.txt"
   # Link definitions path (full path to be set in the corresponding step)
   ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
+  # Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields
+  ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true"
 
 steps:
   - input: "Input values for the variables"
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 24b15c6ef15..c658fd4a771 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -15,6 +15,8 @@ env:
   ELASTIC_PACKAGE_REPOSITORY_LICENSE: "licenses/Elastic-2.0.txt"
   # Link definitions path (full path to be set in the corresponding step)
   ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
+  # Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields
+  ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true"
 
 steps:
   - label: ":white_check_mark: Check go sources"

From 74bd61c0c0f3ca68fe17f22de95a1668ac3406c6 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 24 Oct 2023 17:57:32 +0200
Subject: [PATCH 130/139] Remove elastic-package replacement dep

---
 go.mod | 2 --
 go.sum | 4 ++--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/go.mod b/go.mod
index 9805e66d284..42eed06b68b 100644
--- a/go.mod
+++ b/go.mod
@@ -202,5 +202,3 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
-
-replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20231024082918-7ca8edbc560e
diff --git a/go.sum b/go.sum
index 6f5d91ef290..d763872eb2d 100644
--- a/go.sum
+++ b/go.sum
@@ -142,6 +142,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/elastic/elastic-integration-corpus-generator-tool v0.7.0 h1:yYvNv5XxBgoYINZr/jKJl98pX7SNKMPYeZ8bWOeM/Vw=
 github.com/elastic/elastic-integration-corpus-generator-tool v0.7.0/go.mod h1:WF7gU0qRVj1i//XtQD5i2ZGcGiuiVO6DJNDCRh/o2xk=
+github.com/elastic/elastic-package v0.91.1 h1:YKRFBUd3cQUH9zsTLLiKABU7+XRJ5XwmWaSC4N8OXnQ=
+github.com/elastic/elastic-package v0.91.1/go.mod h1:MkUrD3iIh8/YTNv/z5+EJvgWknnI1qAlYZF+7vAy3JQ=
 github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo=
 github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elastic/go-licenser v0.4.1 h1:1xDURsc8pL5zYT9R29425J3vkHdt4RT5TNEMeRN48x4=
@@ -438,8 +440,6 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
-github.com/mrodm/elastic-package v0.53.1-0.20231024082918-7ca8edbc560e h1:DAaAjdVAZedHuclu8HOkQ2ggLled5KIgBcqh8Bfetog=
-github.com/mrodm/elastic-package v0.53.1-0.20231024082918-7ca8edbc560e/go.mod h1:MkUrD3iIh8/YTNv/z5+EJvgWknnI1qAlYZF+7vAy3JQ=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=

From 8162b7dff4857d86fcab97fe06f8b52fd0a29024 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 24 Oct 2023 19:14:39 +0200
Subject: [PATCH 131/139] Add TODOs and revisted scripts and hooks for new
 serverless pipeline

---
 .buildkite/hooks/pre-command                        |  8 ++++----
 .buildkite/hooks/pre-exit                           |  5 +++--
 .buildkite/pipeline.serverless.yml                  |  4 ++--
 .buildkite/pipeline.yml                             | 13 ++++++++++---
 .buildkite/scripts/common.sh                        |  9 +++++++--
 ...less.sh => test_integrations_with_serverless.sh} |  0
 6 files changed, 26 insertions(+), 13 deletions(-)
 rename .buildkite/scripts/{run_integrations_with_serverless.sh => test_integrations_with_serverless.sh} (100%)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 897a67a2941..d6441492dc2 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -30,7 +30,6 @@ if [ -n "${ELASTIC_PACKAGE_LINKS_FILE_PATH+x}" ]; then
     export ELASTIC_PACKAGE_LINKS_FILE_PATH=${BASE_DIR}/${ELASTIC_PACKAGE_LINKS_FILE_PATH}
 fi
 
-# TODO: add schedule-daily too ?
 if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
     if [ ${BUILDKITE_STEP_KEY} == "publish-packages" ]; then
         export JENKINS_USERNAME_SECRET=$(retry 5 vault kv get -field username ${JENKINS_API_TOKEN_PATH})
@@ -45,9 +44,9 @@ if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
     fi
 fi
 
-# TODO: add schedule-daily too
 if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
-    if [[ ${BUILDKITE_STEP_KEY} =~ test-integrations-serverless-(obs|security) ]]; then
+    # FIXME: update condition depending on the pipeline steps triggered
+    if [[ ${BUILDKITE_STEP_KEY} == "test-integrations" || "${BUILDKITE_STEP_KEY}" =~ test-integrations-serverless-(obs|security) ]]; then
         export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
         export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
 
@@ -58,6 +57,7 @@ if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
         export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
         export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
 
+        # TODO: To be removed
         export EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey ${EC_TOKEN_PATH})
         export EC_HOST_SECRET=$(retry 5 vault kv get -field url ${EC_TOKEN_PATH})
         export EC_REGION_SECRET=$(retry 5 vault read -field region_qa ${EC_DATA_PATH})
@@ -65,7 +65,7 @@ if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
 fi
 
 if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations-serverless" ]; then
-    if [[ ${BUILDKITE_STEP_KEY} =~ run-integrations-serverless-(obs|security) ]]; then
+    if [[ ${BUILDKITE_STEP_KEY} == "test-integrations-serverless-project" ]]; then
         export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
         export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
 
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index b4af958ee0b..2ff79eb3fee 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -5,7 +5,8 @@ source .buildkite/scripts/common.sh
 set -euo pipefail
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
-    if [[ "$BUILDKITE_STEP_KEY" =~ test-integrations-serverless-(obs|security) ]]; then
+    # FIXME: update condition depending on the pipeline steps triggered
+    if [[ "$BUILDKITE_STEP_KEY" == "test-integrations" || "${BUILDKITE_STEP_KEY}" =~ test-integrations-serverless-(obs|security) ]]; then
         unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
         unset ELASTIC_PACKAGE_AWS_SECRET_KEY
         unset AWS_ACCESS_KEY_ID
@@ -29,7 +30,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-serverless" ]]; then
-    if [[ "$BUILDKITE_STEP_KEY" =~ run-integrations-serverless-(obs|security) ]]; then
+    if [[ "$BUILDKITE_STEP_KEY" == "test-integrations-serverless-project" ]]; then
         unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
         unset ELASTIC_PACKAGE_AWS_SECRET_KEY
         unset AWS_ACCESS_KEY_ID
diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml
index 66d7a72b12e..57b678a234f 100644
--- a/.buildkite/pipeline.serverless.yml
+++ b/.buildkite/pipeline.serverless.yml
@@ -45,8 +45,8 @@ steps:
       memory: "4G"
 
   - label: "Check integrations in serverless"
-    key: "run-integrations-serverless-project"
-    command: ".buildkite/scripts/run_integrations_with_serverless.sh"
+    key: "test-integrations-serverless-project"
+    command: ".buildkite/scripts/test_integrations_with_serverless.sh"
     timeout_in_minutes: 120
     env:
       SERVERLESS: true
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index c658fd4a771..552cfdee308 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -27,7 +27,7 @@ steps:
       cpu: "8"
       memory: "4G"
 
-  # TODO: remove comments
+  # TODO: Pending to migrate stages from https://github.com/elastic/integrations/blob/993537b80456edf2035f2a4826031841116c2019/.ci/Jenkinsfile
   # - label: "Publish packages - INCOMPLETE"
   #   key: "publish-packages"
   #   command: ".buildkite/scripts/publish_packages.sh" # TODO: missing signature and publishing loops
@@ -37,10 +37,17 @@ steps:
   #     - step: "check"
   #       allow_failure: false
 
+  # - label: "Trigger integrations"
+  #   key: "trigger-integrations"
+  #   command: ".buildkite/scripts/trigger_integrations_in_parallel.sh" # TODO: missing script
+  #   depends_on:
+  #     - step: "publish-packages"
+  #       allow_failure: false
+
   # to be moved to schedule-daily job
   - label: "Check integrations in serverless - project observability"
     key: "test-integrations-serverless-obs"
-    command: ".buildkite/scripts/run_integrations_with_serverless.sh"
+    command: ".buildkite/scripts/test_integrations_with_serverless.sh"
     timeout_in_minutes: 120
     env:
       SERVERLESS: true
@@ -59,7 +66,7 @@ steps:
 
   - label: "Check integrations in serverless - project security"
     key: "test-integrations-serverless-security"
-    command: ".buildkite/scripts/run_integrations_with_serverless.sh"
+    command: ".buildkite/scripts/test_integrations_with_serverless.sh"
     timeout_in_minutes: 120
     env:
       SERVERLESS: true
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 308a1f0387b..2b0e4f4ab57 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -523,6 +523,8 @@ run_tests_package() {
             return 1
         fi
     fi
+
+    # TODO add if block to test packages locally as Jenkins performs
     return 0
 }
 
@@ -563,7 +565,9 @@ upload_safe_logs() {
 
 buildkite_pr_branch_build_id() {
     if [ "${BUILDKITE_PULL_REQUEST}" == "false" ]; then
-        echo "${BUILDKITE_BRANCH}-${BUILDKITE_BUILD_NUMBER}"
+        # add pipeline slug ad build_id to distinguish between integration and integrations-serverless builds
+        # when both are executed using main branch
+        echo "${BUILDKITE_BRANCH}-${BUILDKITE_PIPELINE_SLUG}-${BUILDKITE_BUILD_NUMBER}"
         return
     fi
     echo "PR-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_BUILD_NUMBER}"
@@ -583,8 +587,9 @@ upload_safe_logs_from_package() {
     local build_directory=$2
 
     local parent_folder="insecure-logs"
+
+    #TODO remove this if when just triggered from integrations-serverless pipeline
     if [[ "${SERVERLESS_PROJECT}" != "" ]]; then
-        #TODO remove this if when just triggered from pipeline-serverless
         parent_folder="insecure-serverless-${SERVERLESS_PROJECT}-logs"
     fi
 
diff --git a/.buildkite/scripts/run_integrations_with_serverless.sh b/.buildkite/scripts/test_integrations_with_serverless.sh
similarity index 100%
rename from .buildkite/scripts/run_integrations_with_serverless.sh
rename to .buildkite/scripts/test_integrations_with_serverless.sh

From 3030df257c6551c42c93f42b4ed5ee391bbd340a Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 24 Oct 2023 19:58:59 +0200
Subject: [PATCH 132/139] Remove serverless steps from main pipeline (Pull
 Requests)

---
 .buildkite/hooks/pre-command |  7 +----
 .buildkite/hooks/pre-exit    |  8 +-----
 .buildkite/pipeline.yml      | 55 ++++++------------------------------
 .buildkite/scripts/common.sh |  5 ----
 4 files changed, 10 insertions(+), 65 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index d6441492dc2..3acf1c5cd90 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -46,7 +46,7 @@ fi
 
 if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
     # FIXME: update condition depending on the pipeline steps triggered
-    if [[ ${BUILDKITE_STEP_KEY} == "test-integrations" || "${BUILDKITE_STEP_KEY}" =~ test-integrations-serverless-(obs|security) ]]; then
+    if [[ ${BUILDKITE_STEP_KEY} == "test-integrations" ]]; then
         export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
         export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
 
@@ -56,11 +56,6 @@ if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
         # Environment variables required by the service deployer
         export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
         export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
-
-        # TODO: To be removed
-        export EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey ${EC_TOKEN_PATH})
-        export EC_HOST_SECRET=$(retry 5 vault kv get -field url ${EC_TOKEN_PATH})
-        export EC_REGION_SECRET=$(retry 5 vault read -field region_qa ${EC_DATA_PATH})
     fi
 fi
 
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index 2ff79eb3fee..7952763d649 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -6,7 +6,7 @@ set -euo pipefail
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
     # FIXME: update condition depending on the pipeline steps triggered
-    if [[ "$BUILDKITE_STEP_KEY" == "test-integrations" || "${BUILDKITE_STEP_KEY}" =~ test-integrations-serverless-(obs|security) ]]; then
+    if [[ "$BUILDKITE_STEP_KEY" == "test-integrations" ]]; then
         unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
         unset ELASTIC_PACKAGE_AWS_SECRET_KEY
         unset AWS_ACCESS_KEY_ID
@@ -18,13 +18,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
         # Ensure elastic stack is stopped
         if [ -f ${ELASTIC_PACKAGE_BIN} ]; then
             echo "--- Take down the Elastic stack"
-            export EC_API_KEY=${EC_API_KEY_SECRET}
-            export EC_HOST=${EC_HOST_SECRET}
-
             ${ELASTIC_PACKAGE_BIN} stack down -v
-
-            unset EC_API_KEY
-            unset EC_HOST
         fi
     fi
 fi
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 552cfdee308..a9d78b8e6cb 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -44,51 +44,12 @@ steps:
   #     - step: "publish-packages"
   #       allow_failure: false
 
-  # to be moved to schedule-daily job
-  - label: "Check integrations in serverless - project observability"
-    key: "test-integrations-serverless-obs"
-    command: ".buildkite/scripts/test_integrations_with_serverless.sh"
-    timeout_in_minutes: 120
-    env:
-      SERVERLESS: true
-      SERVERLESS_PROJECT: observability
-      FORCE_CHECK_ALL: true
-      UPLOAD_SAFE_LOGS: 1
-    agents:
-      provider: "gcp"
-    depends_on:
-      - step: "check"   # TODO: change to publish-packages if re-added
-        allow_failure: false
-    artifact_paths:
-      - "build/test-results/*.xml"
-      # - "build/elastic-stack-dump/*/logs/*.log"
-      # - "build/elastic-stack-dump/*/logs/fleet-server-internal/**/*"
-
-  - label: "Check integrations in serverless - project security"
-    key: "test-integrations-serverless-security"
-    command: ".buildkite/scripts/test_integrations_with_serverless.sh"
-    timeout_in_minutes: 120
-    env:
-      SERVERLESS: true
-      SERVERLESS_PROJECT: security
-      FORCE_CHECK_ALL: true
-      UPLOAD_SAFE_LOGS: 1
-    agents:
-      provider: "gcp"
-    depends_on:
-      - step: "check"   # TODO: change to publish-packages if re-added
-        allow_failure: false
-    artifact_paths:
-      - "build/test-results/*.xml"
-      # - "build/elastic-stack-dump/*/logs/*.log"
-      # - "build/elastic-stack-dump/*/logs/fleet-server-internal/**/*"
+  # - wait: ~
+  #   continue_on_failure: true
 
-  - wait: ~
-    continue_on_failure: true
-
-  - label: ":junit: Junit annotate"
-    plugins:
-      - junit-annotate#v2.4.1:
-          artifacts: "build/test-results/*.xml"
-    agents:
-      provider: "gcp"  # junit plugin requires docker
+  # - label: ":junit: Junit annotate"
+  #   plugins:
+  #     - junit-annotate#v2.4.1:
+  #         artifacts: "build/test-results/*.xml"
+  #   agents:
+  #     provider: "gcp"  # junit plugin requires docker
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 2b0e4f4ab57..631d9e9fb89 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -588,11 +588,6 @@ upload_safe_logs_from_package() {
 
     local parent_folder="insecure-logs"
 
-    #TODO remove this if when just triggered from integrations-serverless pipeline
-    if [[ "${SERVERLESS_PROJECT}" != "" ]]; then
-        parent_folder="insecure-serverless-${SERVERLESS_PROJECT}-logs"
-    fi
-
     upload_safe_logs \
         "${JOB_GCS_BUCKET_INTERNAL}" \
         "${build_directory}/elastic-stack-dump/${package}/logs/elastic-agent-internal/*.*" \

From bf445341c124e4c6182dce56478b377e49657019 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 25 Oct 2023 16:05:15 +0200
Subject: [PATCH 133/139] Changes from code review

---
 .buildkite/pipeline.schedule-daily.yml |  5 +++--
 .buildkite/scripts/common.sh           | 24 ++++++++++++++----------
 .buildkite/scripts/publish_packages.sh |  2 +-
 3 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/.buildkite/pipeline.schedule-daily.yml b/.buildkite/pipeline.schedule-daily.yml
index fbfa4087fba..f64001d85a0 100644
--- a/.buildkite/pipeline.schedule-daily.yml
+++ b/.buildkite/pipeline.schedule-daily.yml
@@ -3,6 +3,7 @@ name: integrations-schedule-daily
 
 env:
   SETUP_GVM_VERSION: "v0.5.1"
+  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
 
 # The pipeline is triggered by the scheduler every day
 steps:
@@ -21,7 +22,7 @@ steps:
       env:
         SERVERLESS_PROJECT: observability
     depends_on:
-      - step: "check-sources"
+      - step: "check"
         allow_failure: false
 
   - label: "Check integrations in serverless - project: Security"
@@ -31,5 +32,5 @@ steps:
       env:
         SERVERLESS_PROJECT: security
     depends_on:
-      - step: "check-sources"
+      - step: "check"
         allow_failure: false
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 631d9e9fb89..5d571cbaa66 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -6,6 +6,8 @@ WORKSPACE="$(pwd)"
 BIN_FOLDER="${WORKSPACE}/bin"
 platform_type="$(uname)"
 hw_type="$(uname -m)"
+platform_type_lowercase="${platform_type,,}"
+
 GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
 export ELASTIC_PACKAGE_BIN=${WORKSPACE}/build/elastic-package
 
@@ -57,7 +59,7 @@ repo_name() {
     echo "$(basename ${orgAndRepo} .git)"
 }
 
-check_platform_architeture() {
+check_platform_architecture() {
   case "${hw_type}" in
     "x86_64")
       arch_type="amd64"
@@ -88,8 +90,7 @@ add_bin_path() {
 with_go() {
   create_bin_folder
   echo "--- Setting up the Go environment..."
-  check_platform_architeture
-  local platform_type_lowercase="${platform_type,,}"
+  check_platform_architecture
   echo " GVM ${SETUP_GVM_VERSION} (platform ${platform_type_lowercase} arch ${arch_type}"
   retry 5 curl -sL -o ${BIN_FOLDER}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${platform_type_lowercase}-${arch_type}"
   chmod +x ${BIN_FOLDER}/gvm
@@ -118,29 +119,33 @@ with_mage() {
 
 with_docker_compose() {
     create_bin_folder
-    retry 5 curl -sSL -o ${BIN_FOLDER}/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64"
+    check_platform_architecture
+
+    echo "--- Setting up the Docker-compose environment..."
+    retry 5 curl -sSL -o ${WORKSPACE}/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-${platform_type_lowercase}-${hw_type}"
     chmod +x ${BIN_FOLDER}/docker-compose
     docker-compose version
 }
 
 with_kubernetes() {
     create_bin_folder
+    check_platform_architecture
+
     echo "--- Install kind"
-    retry 5 curl -sSLo ${BIN_FOLDER}/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
+    retry 5 curl -sSLo ${BIN_FOLDER}/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${platform_type_lowercase}-${arch_type}"
     chmod +x ${BIN_FOLDER}/kind
     kind version
     which kind
 
     echo "--- Install kubectl"
-    retry 5 curl -sSLo ${BIN_FOLDER}/kubectl "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
+    retry 5 curl -sSLo ${BIN_FOLDER}/kubectl "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/${platform_type_lowercase}/${arch_type}/kubectl"
     chmod +x ${BIN_FOLDER}/kubectl
     kubectl version --client
     which kubectl
 }
 
 with_yq() {
-    check_platform_architeture
-    local platform_type_lowercase="${platform_type,,}"
+    check_platform_architecture
     local binary="yq_${platform_type_lowercase}_${arch_type}"
 
     retry 5 curl -sSL -o ${BIN_FOLDER}/yq.tar.gz "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${binary}.tar.gz"
@@ -154,7 +159,6 @@ with_yq() {
     rm -rf ${BIN_FOLDER}/yq.tar.gz
 }
 
-
 ## Logging and logout from Google Cloud
 google_cloud_upload_auth() {
   local secretFileLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX")/${GOOGLE_CREDENTIALS_FILENAME}
@@ -428,7 +432,7 @@ is_pr_affected() {
 }
 
 is_pr() {
-    if [ "${BUILDKITE_PULL_REQUEST}" == "false" ]; then
+    if [[ "${BUILDKITE_PULL_REQUEST}" == "false" || "${BUILDKITE_TAG}" == "" ]]; then
         return 0
     fi
     return 1
diff --git a/.buildkite/scripts/publish_packages.sh b/.buildkite/scripts/publish_packages.sh
index ed20c3f392b..241781796a3 100644
--- a/.buildkite/scripts/publish_packages.sh
+++ b/.buildkite/scripts/publish_packages.sh
@@ -21,7 +21,7 @@ INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH="gs://${INFRA_SIGNING_BUCKET_NAME}/${
 
 
 skipPublishing() {
-    if [[ "${BUILDKITE_PULL_REQUEST}" == "true" ]]; then
+    if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then
         return 0
     fi
 

From 94fa9718fc5640992754bd2f5517352053c6dedc Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 25 Oct 2023 16:11:43 +0200
Subject: [PATCH 134/139] Revert "Remove serverless steps from main pipeline
 (Pull Requests)"

This reverts commit 3030df257c6551c42c93f42b4ed5ee391bbd340a.
---
 .buildkite/hooks/pre-command |  7 ++++-
 .buildkite/hooks/pre-exit    |  8 +++++-
 .buildkite/pipeline.yml      | 55 ++++++++++++++++++++++++++++++------
 .buildkite/scripts/common.sh |  5 ++++
 4 files changed, 65 insertions(+), 10 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 3acf1c5cd90..d6441492dc2 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -46,7 +46,7 @@ fi
 
 if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
     # FIXME: update condition depending on the pipeline steps triggered
-    if [[ ${BUILDKITE_STEP_KEY} == "test-integrations" ]]; then
+    if [[ ${BUILDKITE_STEP_KEY} == "test-integrations" || "${BUILDKITE_STEP_KEY}" =~ test-integrations-serverless-(obs|security) ]]; then
         export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
         export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
 
@@ -56,6 +56,11 @@ if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
         # Environment variables required by the service deployer
         export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
         export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
+
+        # TODO: To be removed
+        export EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey ${EC_TOKEN_PATH})
+        export EC_HOST_SECRET=$(retry 5 vault kv get -field url ${EC_TOKEN_PATH})
+        export EC_REGION_SECRET=$(retry 5 vault read -field region_qa ${EC_DATA_PATH})
     fi
 fi
 
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index 7952763d649..2ff79eb3fee 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -6,7 +6,7 @@ set -euo pipefail
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
     # FIXME: update condition depending on the pipeline steps triggered
-    if [[ "$BUILDKITE_STEP_KEY" == "test-integrations" ]]; then
+    if [[ "$BUILDKITE_STEP_KEY" == "test-integrations" || "${BUILDKITE_STEP_KEY}" =~ test-integrations-serverless-(obs|security) ]]; then
         unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
         unset ELASTIC_PACKAGE_AWS_SECRET_KEY
         unset AWS_ACCESS_KEY_ID
@@ -18,7 +18,13 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
         # Ensure elastic stack is stopped
         if [ -f ${ELASTIC_PACKAGE_BIN} ]; then
             echo "--- Take down the Elastic stack"
+            export EC_API_KEY=${EC_API_KEY_SECRET}
+            export EC_HOST=${EC_HOST_SECRET}
+
             ${ELASTIC_PACKAGE_BIN} stack down -v
+
+            unset EC_API_KEY
+            unset EC_HOST
         fi
     fi
 fi
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index a9d78b8e6cb..552cfdee308 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -44,12 +44,51 @@ steps:
   #     - step: "publish-packages"
   #       allow_failure: false
 
-  # - wait: ~
-  #   continue_on_failure: true
+  # to be moved to schedule-daily job
+  - label: "Check integrations in serverless - project observability"
+    key: "test-integrations-serverless-obs"
+    command: ".buildkite/scripts/test_integrations_with_serverless.sh"
+    timeout_in_minutes: 120
+    env:
+      SERVERLESS: true
+      SERVERLESS_PROJECT: observability
+      FORCE_CHECK_ALL: true
+      UPLOAD_SAFE_LOGS: 1
+    agents:
+      provider: "gcp"
+    depends_on:
+      - step: "check"   # TODO: change to publish-packages if re-added
+        allow_failure: false
+    artifact_paths:
+      - "build/test-results/*.xml"
+      # - "build/elastic-stack-dump/*/logs/*.log"
+      # - "build/elastic-stack-dump/*/logs/fleet-server-internal/**/*"
 
-  # - label: ":junit: Junit annotate"
-  #   plugins:
-  #     - junit-annotate#v2.4.1:
-  #         artifacts: "build/test-results/*.xml"
-  #   agents:
-  #     provider: "gcp"  # junit plugin requires docker
+  - label: "Check integrations in serverless - project security"
+    key: "test-integrations-serverless-security"
+    command: ".buildkite/scripts/test_integrations_with_serverless.sh"
+    timeout_in_minutes: 120
+    env:
+      SERVERLESS: true
+      SERVERLESS_PROJECT: security
+      FORCE_CHECK_ALL: true
+      UPLOAD_SAFE_LOGS: 1
+    agents:
+      provider: "gcp"
+    depends_on:
+      - step: "check"   # TODO: change to publish-packages if re-added
+        allow_failure: false
+    artifact_paths:
+      - "build/test-results/*.xml"
+      # - "build/elastic-stack-dump/*/logs/*.log"
+      # - "build/elastic-stack-dump/*/logs/fleet-server-internal/**/*"
+
+  - wait: ~
+    continue_on_failure: true
+
+  - label: ":junit: Junit annotate"
+    plugins:
+      - junit-annotate#v2.4.1:
+          artifacts: "build/test-results/*.xml"
+    agents:
+      provider: "gcp"  # junit plugin requires docker
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 5d571cbaa66..1c909f12ff3 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -592,6 +592,11 @@ upload_safe_logs_from_package() {
 
     local parent_folder="insecure-logs"
 
+    #TODO remove this if when just triggered from integrations-serverless pipeline
+    if [[ "${SERVERLESS_PROJECT}" != "" ]]; then
+        parent_folder="insecure-serverless-${SERVERLESS_PROJECT}-logs"
+    fi
+
     upload_safe_logs \
         "${JOB_GCS_BUCKET_INTERNAL}" \
         "${build_directory}/elastic-stack-dump/${package}/logs/elastic-agent-internal/*.*" \

From 22fc1ad7ee9e64c5df5868476324a1609bffe73a Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 25 Oct 2023 16:22:40 +0200
Subject: [PATCH 135/139] Fix target docker-compose path

---
 .buildkite/scripts/common.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 1c909f12ff3..9df435020a4 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -122,7 +122,7 @@ with_docker_compose() {
     check_platform_architecture
 
     echo "--- Setting up the Docker-compose environment..."
-    retry 5 curl -sSL -o ${WORKSPACE}/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-${platform_type_lowercase}-${hw_type}"
+    retry 5 curl -sSL -o ${BIN_FOLDER}/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-${platform_type_lowercase}-${hw_type}"
     chmod +x ${BIN_FOLDER}/docker-compose
     docker-compose version
 }

From 5f9559988928bdb2e048095e884c046e0efc6741 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <marrodmo@gmail.com>
Date: Thu, 26 Oct 2023 18:11:03 +0200
Subject: [PATCH 136/139] Apply suggestions from code review

Co-authored-by: sharbuz <87968844+sharbuz@users.noreply.github.com>
---
 .buildkite/scripts/common.sh | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 9df435020a4..4ae8d5ebe18 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -150,9 +150,9 @@ with_yq() {
 
     retry 5 curl -sSL -o ${BIN_FOLDER}/yq.tar.gz "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${binary}.tar.gz"
 
-    tar -C ${BIN_FOLDER} -xpf ${BIN_FOLDER}/yq.tar.gz ./yq_linux_amd64
+    tar -C ${BIN_FOLDER} -xpf ${BIN_FOLDER}/yq.tar.gz ./${binary}
 
-    mv ${BIN_FOLDER}/yq_linux_amd64 ${BIN_FOLDER}/yq
+    mv ${BIN_FOLDER}/${binary} ${BIN_FOLDER}/yq
     chmod +x ${BIN_FOLDER}/yq
     yq --version
 
@@ -540,8 +540,7 @@ create_collapsed_annotation() {
 
     local annotation_file="tmp.annotation.md"
     echo "<details><summary>${title}</summary>" >> ${annotation_file}
-    echo "" >> ${annotation_file}
-    echo "" >> ${annotation_file}
+    echo -e "\n\n" >> ${annotation_file}
     cat ${file} >> ${annotation_file}
     echo "</details>" >> ${annotation_file}
 

From 7e0172a2c35a9ca934991e9a73536e3ecdca4369 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 26 Oct 2023 18:18:28 +0200
Subject: [PATCH 137/139] Enable slack notifications to
 ingest-test-notifications

---
 catalog-info.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/catalog-info.yaml b/catalog-info.yaml
index de0357b50dd..55118198fdc 100644
--- a/catalog-info.yaml
+++ b/catalog-info.yaml
@@ -105,12 +105,12 @@ spec:
       cancel_intermediate_builds_branch_filter: '!main'
       skip_intermediate_builds: true
       skip_intermediate_builds_branch_filter: '!main'
-      # env:
-      #   ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
-#     #     SLACK_NOTIFICATIONS_CHANNEL: '#beats-build' as the current jenkins schedule-daily?
-      #   SLACK_NOTIFICATIONS_CHANNEL: '#ingest-test-notifications'   #TODO: will be changed before GO-PROD
-      #   SLACK_NOTIFICATIONS_ALL_BRANCHES: 'true'
-      #   SLACK_NOTIFICATIONS_ON_SUCCESS: 'true'
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+        # SLACK_NOTIFICATIONS_CHANNEL: '#beats-build' as the current jenkins schedule-daily?
+        SLACK_NOTIFICATIONS_CHANNEL: '#ingest-test-notifications'   #TODO: will be changed before GO-PROD
+        SLACK_NOTIFICATIONS_ALL_BRANCHES: 'true'
+        SLACK_NOTIFICATIONS_ON_SUCCESS: 'true'
       teams:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ

From f7b74ae2e04238b9fd7471d84f9dd9584f2fc6f7 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 26 Oct 2023 18:20:22 +0200
Subject: [PATCH 138/139] Revert "Revert "Remove serverless steps from main
 pipeline (Pull Requests)""

This reverts commit 94fa9718fc5640992754bd2f5517352053c6dedc.
---
 .buildkite/hooks/pre-command |  7 +----
 .buildkite/hooks/pre-exit    |  8 +-----
 .buildkite/pipeline.yml      | 55 ++++++------------------------------
 .buildkite/scripts/common.sh |  5 ----
 4 files changed, 10 insertions(+), 65 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index d6441492dc2..3acf1c5cd90 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -46,7 +46,7 @@ fi
 
 if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
     # FIXME: update condition depending on the pipeline steps triggered
-    if [[ ${BUILDKITE_STEP_KEY} == "test-integrations" || "${BUILDKITE_STEP_KEY}" =~ test-integrations-serverless-(obs|security) ]]; then
+    if [[ ${BUILDKITE_STEP_KEY} == "test-integrations" ]]; then
         export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
         export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
 
@@ -56,11 +56,6 @@ if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
         # Environment variables required by the service deployer
         export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
         export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
-
-        # TODO: To be removed
-        export EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey ${EC_TOKEN_PATH})
-        export EC_HOST_SECRET=$(retry 5 vault kv get -field url ${EC_TOKEN_PATH})
-        export EC_REGION_SECRET=$(retry 5 vault read -field region_qa ${EC_DATA_PATH})
     fi
 fi
 
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index 2ff79eb3fee..7952763d649 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -6,7 +6,7 @@ set -euo pipefail
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
     # FIXME: update condition depending on the pipeline steps triggered
-    if [[ "$BUILDKITE_STEP_KEY" == "test-integrations" || "${BUILDKITE_STEP_KEY}" =~ test-integrations-serverless-(obs|security) ]]; then
+    if [[ "$BUILDKITE_STEP_KEY" == "test-integrations" ]]; then
         unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
         unset ELASTIC_PACKAGE_AWS_SECRET_KEY
         unset AWS_ACCESS_KEY_ID
@@ -18,13 +18,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
         # Ensure elastic stack is stopped
         if [ -f ${ELASTIC_PACKAGE_BIN} ]; then
             echo "--- Take down the Elastic stack"
-            export EC_API_KEY=${EC_API_KEY_SECRET}
-            export EC_HOST=${EC_HOST_SECRET}
-
             ${ELASTIC_PACKAGE_BIN} stack down -v
-
-            unset EC_API_KEY
-            unset EC_HOST
         fi
     fi
 fi
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 552cfdee308..a9d78b8e6cb 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -44,51 +44,12 @@ steps:
   #     - step: "publish-packages"
   #       allow_failure: false
 
-  # to be moved to schedule-daily job
-  - label: "Check integrations in serverless - project observability"
-    key: "test-integrations-serverless-obs"
-    command: ".buildkite/scripts/test_integrations_with_serverless.sh"
-    timeout_in_minutes: 120
-    env:
-      SERVERLESS: true
-      SERVERLESS_PROJECT: observability
-      FORCE_CHECK_ALL: true
-      UPLOAD_SAFE_LOGS: 1
-    agents:
-      provider: "gcp"
-    depends_on:
-      - step: "check"   # TODO: change to publish-packages if re-added
-        allow_failure: false
-    artifact_paths:
-      - "build/test-results/*.xml"
-      # - "build/elastic-stack-dump/*/logs/*.log"
-      # - "build/elastic-stack-dump/*/logs/fleet-server-internal/**/*"
-
-  - label: "Check integrations in serverless - project security"
-    key: "test-integrations-serverless-security"
-    command: ".buildkite/scripts/test_integrations_with_serverless.sh"
-    timeout_in_minutes: 120
-    env:
-      SERVERLESS: true
-      SERVERLESS_PROJECT: security
-      FORCE_CHECK_ALL: true
-      UPLOAD_SAFE_LOGS: 1
-    agents:
-      provider: "gcp"
-    depends_on:
-      - step: "check"   # TODO: change to publish-packages if re-added
-        allow_failure: false
-    artifact_paths:
-      - "build/test-results/*.xml"
-      # - "build/elastic-stack-dump/*/logs/*.log"
-      # - "build/elastic-stack-dump/*/logs/fleet-server-internal/**/*"
+  # - wait: ~
+  #   continue_on_failure: true
 
-  - wait: ~
-    continue_on_failure: true
-
-  - label: ":junit: Junit annotate"
-    plugins:
-      - junit-annotate#v2.4.1:
-          artifacts: "build/test-results/*.xml"
-    agents:
-      provider: "gcp"  # junit plugin requires docker
+  # - label: ":junit: Junit annotate"
+  #   plugins:
+  #     - junit-annotate#v2.4.1:
+  #         artifacts: "build/test-results/*.xml"
+  #   agents:
+  #     provider: "gcp"  # junit plugin requires docker
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index 4ae8d5ebe18..30f72681c5e 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -591,11 +591,6 @@ upload_safe_logs_from_package() {
 
     local parent_folder="insecure-logs"
 
-    #TODO remove this if when just triggered from integrations-serverless pipeline
-    if [[ "${SERVERLESS_PROJECT}" != "" ]]; then
-        parent_folder="insecure-serverless-${SERVERLESS_PROJECT}-logs"
-    fi
-
     upload_safe_logs \
         "${JOB_GCS_BUCKET_INTERNAL}" \
         "${build_directory}/elastic-stack-dump/${package}/logs/elastic-agent-internal/*.*" \

From a9c3fa0563fe938fe47230f70d07c60493eb5871 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 26 Oct 2023 19:21:26 +0200
Subject: [PATCH 139/139] Change permissions of publish script

---
 .buildkite/scripts/publish_packages.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 .buildkite/scripts/publish_packages.sh

diff --git a/.buildkite/scripts/publish_packages.sh b/.buildkite/scripts/publish_packages.sh
old mode 100644
new mode 100755