From 37921bc414183ee9db285e5466319ef26c21bc61 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 1 Feb 2023 12:39:36 +0100
Subject: [PATCH 01/39] Add release step

In jenkins it is needed, but in Buildkite as the repository is
not shallow, the --unshallow parameter should be removed
---
 .buildkite/pipeline.yml       | 12 +++++++++++-
 .buildkite/scripts/release.sh | 15 +++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)
 create mode 100644 .buildkite/scripts/release.sh

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index cfdf57f398..7fe7bd297d 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -19,10 +19,20 @@ steps:
   - wait: ~
     continue_on_failure: true
 
-  - label: "Junit annotate"
+  - label: ":junit: Junit annotate"
     plugins:
       - junit-annotate#v2.4.1:
           artifacts: "build/test-results/*.xml"
     agents:
       provider: "gcp"
 
+  - label: ":github: Release"
+    # TODO enable this conditional
+    # if: |
+    #   build.tag =~ /^v[0-9]+[.][0-9]+[.][0-9]+$$/ &&  build.branch == "main"
+    # TODO GITHUB_TOKEN environment var
+    command: ".buildkite/scripts/release.sh"
+    agents:
+      image: "golang:1.19.5"
+      cpu: "8"
+      memory: "4G"
diff --git a/.buildkite/scripts/release.sh b/.buildkite/scripts/release.sh
new file mode 100644
index 0000000000..2c5bedd273
--- /dev/null
+++ b/.buildkite/scripts/release.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -euo pipefail
+
+echo "--- fetching tags"
+# Ensure that tags are present so goreleaser can build the changelog from the last release.
+git rev-parse --is-shallow-repository
+git fetch origin --tags
+
+echo "--- running goreleaser"
+# Run latest version of goreleaser
+# TODO Remove check command once this stage is removed from Jenkinsfile
+# curl -sL https://git.io/goreleaser | bash
+curl -sL https://git.io/goreleaser | bash -s -- check
+

From 97b04045df1e04fae4fc67b841ba6d7d473fb502 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 1 Feb 2023 13:32:09 +0100
Subject: [PATCH 02/39] Add integration test stage with just one make command

This requires to install in the VM created in GCP gvm, golang,
docker-compose , kind and kubectl binaries.
Golang installation requires to add some environment variables to the
PATH as GOPATH.
---
 .buildkite/pipeline.yml                 | 12 ++++++
 .buildkite/scripts/integration_tests.sh | 52 +++++++++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 .buildkite/scripts/integration_tests.sh

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 7fe7bd297d..9a4df51a91 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,3 +1,9 @@
+env:
+  SETUP_GVM_VERSION: 'v0.5.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
+  DOCKER_COMPOSE_VERSION: "v2.15.1"
+  KIND_VERSION: 'v0.17.0'
+  K8S_VERSION: 'v1.26.0'
+
 steps:
   - label: ":go: Run check-static"
     command: "make check-static"
@@ -26,6 +32,12 @@ steps:
     agents:
       provider: "gcp"
 
+  - label: ":go: Integration tests"
+    # command: ".buildkite/pipeline.trigger.integration.tests.sh | buildkite-agent pipeline upload"
+    command: ".buildkite/scripts/integration_tests.sh"
+    agents:
+      provider: "gcp"
+
   - label: ":github: Release"
     # TODO enable this conditional
     # if: |
diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh
new file mode 100644
index 0000000000..50eb7ca343
--- /dev/null
+++ b/.buildkite/scripts/integration_tests.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+set -euo pipefail
+
+with_kubernetes() {
+    # FIXME add retry logic
+    mkdir -p ${WORKSPACE}/bin
+    curl -sSLo ${WORKSPACE}/bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
+    chmod +x ${WORKSPACE}/bin/kind
+    kind version
+    which kind
+
+    mkdir -p ${WORKSPACE}/bin
+    curl -sSLo ${WORKSPACE}/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
+    chmod +x ${WORKSPACE}/bin/kubectl
+    kubectl version --client
+    which kubectl
+}
+
+with_go() {
+    # FIXME add retry logic
+    mkdir -p ${WORKSPACE}/bin
+    curl -sL -o ${WORKSPACE}/bin/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-linux-amd64"
+    chmod +x ${WORKSPACE}/bin/gvm
+    eval "$(gvm $(cat .go-version))"
+    go version
+    which go
+}
+
+with_docker_compose() {
+    # FIXME add retry logic
+    mkdir -p ${WORKSPACE}/bin
+    curl -SL -o ${WORKSPACE}/bin/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64"
+    chmod +x ${WORKSPACE}/bin/docker-compose
+    docker-compose version
+}
+
+
+echo "Current path: $(pwd)"
+WORKSPACE="$(pwd)"
+export PATH="${WORKSPACE}/bin:${PATH}"
+echo "Path: $PATH"
+
+echo "--- install go"
+with_go
+export PATH="$(go env GOPATH)/bin:${PATH}"
+
+echo "--- install docker-compose"
+with_docker_compose
+
+echo "--- Run integration test stack-command-default"
+make install test-stack-command-default check-git-clean

From f6c85447a484619175e8a6c987dd9bd8f92b070f Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 1 Feb 2023 19:12:27 +0100
Subject: [PATCH 03/39] Update permissions scripts

---
 .buildkite/scripts/integration_tests.sh | 0
 .buildkite/scripts/release.sh           | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 .buildkite/scripts/integration_tests.sh
 mode change 100644 => 100755 .buildkite/scripts/release.sh

diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh
old mode 100644
new mode 100755
diff --git a/.buildkite/scripts/release.sh b/.buildkite/scripts/release.sh
old mode 100644
new mode 100755

From 998b1cfd8814950abf6d6c52f4c03d7aee2b6788 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 2 Feb 2023 10:48:17 +0100
Subject: [PATCH 04/39] Upload dynamically integration tests

---
 .../pipeline.trigger.integration.tests.sh     | 24 ++++++++++++
 .buildkite/pipeline.yml                       | 13 ++++---
 .buildkite/scripts/integration_tests.sh       | 39 ++++++++++++++++++-
 3 files changed, 68 insertions(+), 8 deletions(-)
 create mode 100755 .buildkite/pipeline.trigger.integration.tests.sh

diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
new file mode 100755
index 0000000000..c7102e5bbd
--- /dev/null
+++ b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# exit immediately on failure, or if an undefined variable is used
+set -eu
+
+# begin the pipeline.yml file
+echo "steps:"
+
+# Integration tests related stack command
+STACK_COMMAND_TESTS = (
+  test-stack-command-default
+  test-stack-command-oldest
+)
+
+for test in ${STACK_COMMAND_TESTS[@]}; do
+    echo "  - label: :go: Running integration test: ${test}"
+    echo "    command: ./buildkite/scripts/integration_tests.sh -t ${test}"
+    echo "    agents:"
+    echo "      provider: \"gcp\""
+    echo "    artifact_paths:"
+    echo "      - build/elastic-stack-dump/stack/*/logs/*.log"
+    echo "      - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*.log"
+    echo "      - build/elastic-stack-status/*/*"
+done
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 9a4df51a91..1711c289cb 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -25,6 +25,13 @@ steps:
   - wait: ~
     continue_on_failure: true
 
+  - label: ":pipeline: Trigger Integration tests"
+    command: ".buildkite/pipeline.trigger.integration.tests.sh | buildkite-agent pipeline upload"
+    # command: ".buildkite/scripts/integration_tests.sh"
+
+  - wait: ~
+    continue_on_failure: true
+
   - label: ":junit: Junit annotate"
     plugins:
       - junit-annotate#v2.4.1:
@@ -32,12 +39,6 @@ steps:
     agents:
       provider: "gcp"
 
-  - label: ":go: Integration tests"
-    # command: ".buildkite/pipeline.trigger.integration.tests.sh | buildkite-agent pipeline upload"
-    command: ".buildkite/scripts/integration_tests.sh"
-    agents:
-      provider: "gcp"
-
   - label: ":github: Release"
     # TODO enable this conditional
     # if: |
diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh
index 50eb7ca343..4dfc1d0706 100755
--- a/.buildkite/scripts/integration_tests.sh
+++ b/.buildkite/scripts/integration_tests.sh
@@ -2,6 +2,13 @@
 
 set -euo pipefail
 
+usage() {
+    echo "$0 [-t <target>] [-h]"
+    echo "Trigger integration tests related to a target in Makefile"
+    echo -e "\t-t <target>: Makefile target. Mandatory"
+    echo -e "\t-h: Show this message"
+}
+
 with_kubernetes() {
     # FIXME add retry logic
     mkdir -p ${WORKSPACE}/bin
@@ -35,6 +42,34 @@ with_docker_compose() {
     docker-compose version
 }
 
+TARGET=""
+while getopts ":t:h" o; do
+    case "${o}" in
+        t)
+            TARGET=${OPTARG}
+            ;;
+        h)
+            usage
+            exit 0
+            ;;
+        \?)
+            echo "Invalid option ${OPTARG}"
+            usage
+            exit 1
+            ;;
+        :)
+            echo "Missing argument for -${OPTARG}"
+            usage
+            exit 1
+            ;;
+    esac
+done
+
+if [[ "${TARGET}" == "" ]]; then
+    echo "Missing target"
+    usage
+    exit 1
+fi
 
 echo "Current path: $(pwd)"
 WORKSPACE="$(pwd)"
@@ -48,5 +83,5 @@ export PATH="$(go env GOPATH)/bin:${PATH}"
 echo "--- install docker-compose"
 with_docker_compose
 
-echo "--- Run integration test stack-command-default"
-make install test-stack-command-default check-git-clean
+echo "--- Run integration test ${TARGET}"
+make install ${TARGET} check-git-clean

From 663bab8205cade441d7eb9392e936113df04bd68 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 2 Feb 2023 11:06:11 +0100
Subject: [PATCH 05/39] Fix trigger pipeline script

---
 .../pipeline.trigger.integration.tests.sh     | 21 +++++++++++--------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
index c7102e5bbd..b39cc11fb5 100755
--- a/.buildkite/pipeline.trigger.integration.tests.sh
+++ b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -5,20 +5,23 @@ set -eu
 
 # begin the pipeline.yml file
 echo "steps:"
+echo "  - group: \":terminal: Integration test suite\""
+echo "    key: \"integration-tests\""
+echo "    steps:"
 
 # Integration tests related stack command
-STACK_COMMAND_TESTS = (
+STACK_COMMAND_TESTS=(
   test-stack-command-default
   test-stack-command-oldest
 )
 
 for test in ${STACK_COMMAND_TESTS[@]}; do
-    echo "  - label: :go: Running integration test: ${test}"
-    echo "    command: ./buildkite/scripts/integration_tests.sh -t ${test}"
-    echo "    agents:"
-    echo "      provider: \"gcp\""
-    echo "    artifact_paths:"
-    echo "      - build/elastic-stack-dump/stack/*/logs/*.log"
-    echo "      - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*.log"
-    echo "      - build/elastic-stack-status/*/*"
+    echo "      - label: \":go: Running integration test: ${test}\""
+    echo "        command: ./.buildkite/scripts/integration_tests.sh -t ${test}"
+    echo "        agents:"
+    echo "          provider: \"gcp\""
+    echo "        artifact_paths:"
+    echo "          - build/elastic-stack-dump/stack/*/logs/*.log"
+    echo "          - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*.log"
+    echo "          - build/elastic-stack-status/*/*"
 done

From 6f2a1881b47fb34cf854653d7374171dc6077b93 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 2 Feb 2023 11:36:39 +0100
Subject: [PATCH 06/39] Add more stages into integration tests

---
 .../pipeline.trigger.integration.tests.sh     | 40 ++++++++++++++++++-
 1 file changed, 38 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
index b39cc11fb5..72a774dbb2 100755
--- a/.buildkite/pipeline.trigger.integration.tests.sh
+++ b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -11,8 +11,11 @@ echo "    steps:"
 
 # Integration tests related stack command
 STACK_COMMAND_TESTS=(
-  test-stack-command-default
-  test-stack-command-oldest
+    test-stack-command-default
+    test-stack-command-oldest
+    test-stack-command-7x
+    test-stack-command-86
+    test-stack-command-8x
 )
 
 for test in ${STACK_COMMAND_TESTS[@]}; do
@@ -25,3 +28,36 @@ for test in ${STACK_COMMAND_TESTS[@]}; do
     echo "          - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*.log"
     echo "          - build/elastic-stack-status/*/*"
 done
+
+CHECK_PACKAGES_TESTS=(
+    test-check-packages-other
+    test-check-packages-with-kind
+    test-check-packages-with-custom-agent
+    test-check-packages-benchmarks
+)
+for test in ${CHECK_PACKAGES_TESTS[@]}; do
+    echo "      - label: \":go: Running integration test: ${test}\""
+    echo "        command: ./.buildkite/scripts/integration_tests.sh -t ${test}"
+    echo "        agents:"
+    echo "          provider: \"gcp\""
+    echo "        artifact_paths:"
+    echo "          - build/test-results/*.xml"
+    echo "          - build/elastic-stack-dump/stack/*/logs/*.log"
+    echo "          - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*.log"
+    if [[ $test =~ with-kind ]]; then
+        echo "          - build/kubectl-dump.txt"
+    fi
+done
+
+echo "      - label: \":go: Running integration test: test-build-zip\""
+echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-build-zip"
+echo "        agents:"
+echo "          provider: \"gcp\""
+echo "        artifact_paths:"
+echo "          - build/elastic-stack-dump/stack/*/logs/*.log"
+echo "          - build/packages/*.sig"
+
+echo "      - label: \":go: Running integration test: test-profiles-command\""
+echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-profiles-command"
+echo "        agents:"
+echo "          provider: \"gcp\""

From fc6229de883425f3bcdcc810fa66e0c623cdbb5d Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 2 Feb 2023 12:12:23 +0100
Subject: [PATCH 07/39] Add check parallel targets

---
 .../pipeline.trigger.integration.tests.sh     | 29 +++++++++++++++++--
 .buildkite/scripts/integration_tests.sh       | 21 +++++++++++++-
 2 files changed, 46 insertions(+), 4 deletions(-)

diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
index 72a774dbb2..0a0f8447e6 100755
--- a/.buildkite/pipeline.trigger.integration.tests.sh
+++ b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -42,13 +42,36 @@ for test in ${CHECK_PACKAGES_TESTS[@]}; do
     echo "          provider: \"gcp\""
     echo "        artifact_paths:"
     echo "          - build/test-results/*.xml"
-    echo "          - build/elastic-stack-dump/stack/*/logs/*.log"
-    echo "          - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*.log"
-    if [[ $test =~ with-kind ]]; then
+    echo "          - build/elastic-stack-dump/stack/check-*/logs/*.log"
+    echo "          - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/*.log"
+    if [[ $test =~ with-kind$ ]]; then
         echo "          - build/kubectl-dump.txt"
     fi
 done
 
+pushd test/packages/parallel > /dev/null
+for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
+    package_name=$(basename ${package})
+    if [[ "${package_name}" =~ ^(aws|gcp)$ ]]; then
+        # TODO: missing secrets
+        continue
+    fi
+    echo "      - label: \":go: Running integration test: ${package_name}\""
+    echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}"
+    echo "        agents:"
+    echo "          provider: \"gcp\""
+    echo "        artifact_paths:"
+    echo "          - build/test-results/*.xml"
+    echo "          - build/elastic-stack-dump/stack/check-*/logs/*.log"
+    echo "          - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/*.log"
+    echo "          - insecure-logs/${package_name}"
+    echo "          - build/elastic-stack-dump/stack/check-${package_name}/logs/elastic-agent-internal/*"
+    echo "          - insecure-logs/${package_name}/container-logs"
+    echo "          - build/container-logs/*.log"
+done
+
+popd > /dev/null
+
 echo "      - label: \":go: Running integration test: test-build-zip\""
 echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-build-zip"
 echo "        agents:"
diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh
index 4dfc1d0706..896927a07e 100755
--- a/.buildkite/scripts/integration_tests.sh
+++ b/.buildkite/scripts/integration_tests.sh
@@ -2,10 +2,14 @@
 
 set -euo pipefail
 
+PARALLEL_TARGET="test-check-packages-parallel"
+KIND_TARGET="test-check-packages-with-kind"
+
 usage() {
     echo "$0 [-t <target>] [-h]"
     echo "Trigger integration tests related to a target in Makefile"
     echo -e "\t-t <target>: Makefile target. Mandatory"
+    echo -e "\t-p <package>: Package (required for test-check-packages-parallel target)."
     echo -e "\t-h: Show this message"
 }
 
@@ -43,11 +47,15 @@ with_docker_compose() {
 }
 
 TARGET=""
-while getopts ":t:h" o; do
+PACKAGE=""
+while getopts ":t:p:h" o; do
     case "${o}" in
         t)
             TARGET=${OPTARG}
             ;;
+        p)
+            PACKAGE=${OPTARG}
+            ;;
         h)
             usage
             exit 0
@@ -83,5 +91,16 @@ export PATH="$(go env GOPATH)/bin:${PATH}"
 echo "--- install docker-compose"
 with_docker_compose
 
+if [[ "${TARGET}" == "${KIND_TARGET}" ]]; then
+    echo "--- install kubectl & kind"
+    with_kubernetes
+fi
+
 echo "--- Run integration test ${TARGET}"
+if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then
+    make install
+    make PACKAGE_UNDER_TEST=${PACKAGE} ${TARGET}
+    exit 0
+fi
+
 make install ${TARGET} check-git-clean

From 03dcd6bbd21e1369a2f4c5f695b5dd0796d8e039 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 6 Feb 2023 17:42:33 +0100
Subject: [PATCH 08/39] Remove commented command

---
 .buildkite/pipeline.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 1711c289cb..1ff011a2cb 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -27,7 +27,6 @@ steps:
 
   - label: ":pipeline: Trigger Integration tests"
     command: ".buildkite/pipeline.trigger.integration.tests.sh | buildkite-agent pipeline upload"
-    # command: ".buildkite/scripts/integration_tests.sh"
 
   - wait: ~
     continue_on_failure: true

From d2f3cab80f594820ffa4cfbf7111180a2e38c71b Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 7 Feb 2023 11:11:18 +0100
Subject: [PATCH 09/39] Add comment in junit plugin

---
 .buildkite/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 1ff011a2cb..681d81a3d5 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -36,7 +36,7 @@ steps:
       - junit-annotate#v2.4.1:
           artifacts: "build/test-results/*.xml"
     agents:
-      provider: "gcp"
+      provider: "gcp"  # junit plugin requires docker
 
   - label: ":github: Release"
     # TODO enable this conditional

From 98c37adf8907300efc387d29cd274dea20f5eac7 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 7 Feb 2023 11:26:45 +0100
Subject: [PATCH 10/39] Test run unit tests windows

Add chocolatey profile

Create target for windows in Makefile
---
 .buildkite/pipeline.yml                   |  9 ++++++++-
 .buildkite/scripts/unit_tests_windows.ps1 | 20 ++++++++++++++++++++
 Makefile                                  | 11 +++++++++++
 3 files changed, 39 insertions(+), 1 deletion(-)
 create mode 100644 .buildkite/scripts/unit_tests_windows.ps1

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 681d81a3d5..97542fef6f 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -12,7 +12,7 @@ steps:
       cpu: "8"
       memory: "4G"
 
-  - label: ":go: Run unit tests"
+  - label: ":go: :linux: Run unit tests"
     command: "make test-go-ci"
     artifact_paths:
       - "build/test-results/*.xml"
@@ -22,6 +22,13 @@ steps:
       cpu: "8"
       memory: "4G"
 
+  - label: ":go: :windows: Run unit tests"
+    key: unit-tests-windows
+    command: ".buildkite/scripts/unit_tests_windows.ps1"
+    agents:
+      provider: "gcp"
+      image: "family/ci-windows-2022"
+
   - wait: ~
     continue_on_failure: true
 
diff --git a/.buildkite/scripts/unit_tests_windows.ps1 b/.buildkite/scripts/unit_tests_windows.ps1
new file mode 100644
index 0000000000..f60c84a676
--- /dev/null
+++ b/.buildkite/scripts/unit_tests_windows.ps1
@@ -0,0 +1,20 @@
+
+
+echo "--- Installing make"
+choco install -y make
+
+echo "--- Installing golang"
+choco install -y golang --version 1.19.5
+
+echo "--- Updating session environment"
+# refreshenv requires to have chocolatey profile installed
+$env:ChocolateyInstall = Convert-Path "$((Get-Command choco).Path)\..\.."
+Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
+
+refreshenv
+
+echo "--- Running unit tests"
+go version
+make test-go-ci-win
+
+exit 1
diff --git a/Makefile b/Makefile
index 60f1559f52..292848d688 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,7 @@
 CODE_COVERAGE_REPORT_FOLDER = $(PWD)/build/test-coverage
+WIN_CODE_COVERAGE_REPORT_FOLDER = $(PWD)\build\test-coverage
 CODE_COVERAGE_REPORT_NAME_UNIT = $(CODE_COVERAGE_REPORT_FOLDER)/coverage-unit-report
+WIN_CODE_COVERAGE_REPORT_NAME_UNIT = $(CODE_COVERAGE_REPORT_FOLDER)\coverage-unit-report
 VERSION_IMPORT_PATH = github.com/elastic/elastic-package/internal/version
 VERSION_COMMIT_HASH = `git describe --always --long --dirty`
 VERSION_BUILD_TIME = `date +%s`
@@ -38,6 +40,9 @@ update: update-readme
 $(CODE_COVERAGE_REPORT_FOLDER):
 	mkdir -p $@
 
+$(WIN_CODE_COVERAGE_REPORT_FOLDER):
+	mkdir -p $@
+
 test-go: $(CODE_COVERAGE_REPORT_FOLDER)
 	# -count=1 is included to invalidate the test cache. This way, if you run "make test-go" multiple times
 	# you will get fresh test results each time. For instance, changing the source of mocked packages
@@ -50,6 +55,12 @@ test-go-ci: $(CODE_COVERAGE_REPORT_FOLDER)
 	go run gotest.tools/gotestsum --junitfile "$(PWD)/build/test-results/TEST-unit.xml" -- -count=1 -coverprofile=$(CODE_COVERAGE_REPORT_NAME_UNIT).out ./...
 	go run github.com/boumenot/gocover-cobertura < $(CODE_COVERAGE_REPORT_NAME_UNIT).out > $(CODE_COVERAGE_REPORT_NAME_UNIT).xml
 
+test-go-ci-win: $(WIN_CODE_COVERAGE_REPORT_FOLDER)
+	mkdir -p $(PWD)\build\test-results
+	mkdir -p $(PWD)\build\test-coverage
+	go run gotest.tools/gotestsum --junitfile "$(PWD)\build\test-results\TEST-unit.xml" -- -count=1 -coverprofile=$(WIN_CODE_COVERAGE_REPORT_NAME_UNIT).out ./...
+	go run github.com/boumenot/gocover-cobertura < $(WIN_CODE_COVERAGE_REPORT_NAME_UNIT).out > $(WIN_CODE_COVERAGE_REPORT_NAME_UNIT).xml
+
 test-stack-command-default:
 	./scripts/test-stack-command.sh
 

From f6637a5afaa00ef910ae159a938d3e4d50ab11cc Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 7 Feb 2023 11:33:18 +0100
Subject: [PATCH 11/39] Do not run integration tests if there are failures

---
 .buildkite/pipeline.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 97542fef6f..0b58117834 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -6,6 +6,7 @@ env:
 
 steps:
   - label: ":go: Run check-static"
+    key: check-static
     command: "make check-static"
     agents:
       image: "golang:1.19.5"
@@ -13,6 +14,7 @@ steps:
       memory: "4G"
 
   - label: ":go: :linux: Run unit tests"
+    key: unit-tests-linux
     command: "make test-go-ci"
     artifact_paths:
       - "build/test-results/*.xml"
@@ -34,6 +36,13 @@ steps:
 
   - label: ":pipeline: Trigger Integration tests"
     command: ".buildkite/pipeline.trigger.integration.tests.sh | buildkite-agent pipeline upload"
+    depends_on:
+      - step: check-static
+        allow_failure: false
+      - step: unit-tests-linux
+        allow_failure: false
+      - step: unit-tests-windows
+        allow_failure: false
 
   - wait: ~
     continue_on_failure: true

From 44ca5f48c80e857ee84b3e4d1a6cac2f4675746f Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 7 Feb 2023 18:21:26 +0100
Subject: [PATCH 12/39] Disable windows step

---
 .buildkite/pipeline.yml                   | 12 ++++++------
 .buildkite/scripts/unit_tests_windows.ps1 | 20 --------------------
 2 files changed, 6 insertions(+), 26 deletions(-)
 delete mode 100644 .buildkite/scripts/unit_tests_windows.ps1

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 0b58117834..61777d428b 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -24,12 +24,12 @@ steps:
       cpu: "8"
       memory: "4G"
 
-  - label: ":go: :windows: Run unit tests"
-    key: unit-tests-windows
-    command: ".buildkite/scripts/unit_tests_windows.ps1"
-    agents:
-      provider: "gcp"
-      image: "family/ci-windows-2022"
+  # - label: ":go: :windows: Run unit tests"
+  #   key: unit-tests-windows
+  #   command: ".buildkite/scripts/unit_tests_windows.ps1"
+  #   agents:
+  #     provider: "gcp"
+  #     image: "family/ci-windows-2022"
 
   - wait: ~
     continue_on_failure: true
diff --git a/.buildkite/scripts/unit_tests_windows.ps1 b/.buildkite/scripts/unit_tests_windows.ps1
deleted file mode 100644
index f60c84a676..0000000000
--- a/.buildkite/scripts/unit_tests_windows.ps1
+++ /dev/null
@@ -1,20 +0,0 @@
-
-
-echo "--- Installing make"
-choco install -y make
-
-echo "--- Installing golang"
-choco install -y golang --version 1.19.5
-
-echo "--- Updating session environment"
-# refreshenv requires to have chocolatey profile installed
-$env:ChocolateyInstall = Convert-Path "$((Get-Command choco).Path)\..\.."
-Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
-
-refreshenv
-
-echo "--- Running unit tests"
-go version
-make test-go-ci-win
-
-exit 1

From 5454e7c9a8954d247ab759875be14db44591e7ed Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 7 Feb 2023 19:23:17 +0100
Subject: [PATCH 13/39] Add gcp integration test

---
 .buildkite/hooks/pre-command                  | 31 +++++++++++++++++++
 .../pipeline.trigger.integration.tests.sh     |  3 +-
 .buildkite/scripts/integration_tests.sh       |  4 +++
 3 files changed, 37 insertions(+), 1 deletion(-)
 create mode 100644 .buildkite/hooks/pre-command

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
new file mode 100644
index 0000000000..8c1ff27385
--- /dev/null
+++ b/.buildkite/hooks/pre-command
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -euo pipefail
+
+function retry {
+  local retries=$1
+  shift
+
+  local count=0
+  until "$@"; do
+    exit=$?
+    wait=$((2 ** count))
+    count=$((count + 1))
+    if [ $count -lt "$retries" ]; then
+      >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+      sleep $wait
+    else
+      >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+      return $exit
+    fi
+  done
+  return 0
+}
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp" ]]; then
+  export ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field project "secret/ci/elastic-elastic-package/gcp-service-account-temp")
+  export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field google_credentials "secret/ci/elastic-elastic-package/gcp-service-account-temp")
+  export TEST_SECRET="test"
+  export GOOGLE_CREDENTIALS=${TEST_SECRET}
+  export GCP_PROJECT_ID=${TEST_SECRET}
+fi
diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
index 0a0f8447e6..f6e3fdf4ac 100755
--- a/.buildkite/pipeline.trigger.integration.tests.sh
+++ b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -52,11 +52,12 @@ done
 pushd test/packages/parallel > /dev/null
 for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     package_name=$(basename ${package})
-    if [[ "${package_name}" =~ ^(aws|gcp)$ ]]; then
+    if [[ "${package_name}" =~ ^aws$ ]]; then
         # TODO: missing secrets
         continue
     fi
     echo "      - label: \":go: Running integration test: ${package_name}\""
+    echo "        key: \"integration-parallel-${package_name}\""
     echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}"
     echo "        agents:"
     echo "          provider: \"gcp\""
diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh
index 896927a07e..fcb1a0ef9f 100755
--- a/.buildkite/scripts/integration_tests.sh
+++ b/.buildkite/scripts/integration_tests.sh
@@ -98,6 +98,10 @@ fi
 
 echo "--- Run integration test ${TARGET}"
 if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then
+    if [[ "${PACKAGE}" == "gcp" ]]; then
+      echo "Project test: $GCP_PROJECT_ID"
+      echo "Google credentials test: $GOOGLE_CREDENTIALS"
+    fi
     make install
     make PACKAGE_UNDER_TEST=${PACKAGE} ${TARGET}
     exit 0

From 93d85c3bee63b90662f64f4fbfc06ac08f58bf87 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 7 Feb 2023 19:41:45 +0100
Subject: [PATCH 14/39] Remove windows dep

---
 .buildkite/hooks/pre-command | 2 +-
 .buildkite/pipeline.yml      | 2 --
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 8c1ff27385..e56b5f15de 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -25,7 +25,7 @@ function retry {
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp" ]]; then
   export ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field project "secret/ci/elastic-elastic-package/gcp-service-account-temp")
   export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field google_credentials "secret/ci/elastic-elastic-package/gcp-service-account-temp")
-  export TEST_SECRET="test"
+  export TEST_SECRET="testtest"
   export GOOGLE_CREDENTIALS=${TEST_SECRET}
   export GCP_PROJECT_ID=${TEST_SECRET}
 fi
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 61777d428b..84b9e485b1 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -41,8 +41,6 @@ steps:
         allow_failure: false
       - step: unit-tests-linux
         allow_failure: false
-      - step: unit-tests-windows
-        allow_failure: false
 
   - wait: ~
     continue_on_failure: true

From 20cc88ca94241bbbf12089dfc8b0f370a2e84407 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 7 Feb 2023 19:51:09 +0100
Subject: [PATCH 15/39] Set env. vars

---
 .buildkite/hooks/pre-command            | 7 ++++---
 .buildkite/scripts/integration_tests.sh | 4 ----
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index e56b5f15de..f99f96291a 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -25,7 +25,8 @@ function retry {
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp" ]]; then
   export ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field project "secret/ci/elastic-elastic-package/gcp-service-account-temp")
   export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field google_credentials "secret/ci/elastic-elastic-package/gcp-service-account-temp")
-  export TEST_SECRET="testtest"
-  export GOOGLE_CREDENTIALS=${TEST_SECRET}
-  export GCP_PROJECT_ID=${TEST_SECRET}
+
+  # Environment variables required by the service deployer
+  export GOOGLE_CREDENTIALS=${ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET}
+  export GCP_PROJECT_ID=${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}
 fi
diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh
index fcb1a0ef9f..896927a07e 100755
--- a/.buildkite/scripts/integration_tests.sh
+++ b/.buildkite/scripts/integration_tests.sh
@@ -98,10 +98,6 @@ fi
 
 echo "--- Run integration test ${TARGET}"
 if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then
-    if [[ "${PACKAGE}" == "gcp" ]]; then
-      echo "Project test: $GCP_PROJECT_ID"
-      echo "Google credentials test: $GOOGLE_CREDENTIALS"
-    fi
     make install
     make PACKAGE_UNDER_TEST=${PACKAGE} ${TARGET}
     exit 0

From 693fc9782e464f07b06b2f2175054725d8d47400 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 7 Feb 2023 20:11:51 +0100
Subject: [PATCH 16/39] Remove changes in Makefile

---
 Makefile | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/Makefile b/Makefile
index 292848d688..60f1559f52 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,5 @@
 CODE_COVERAGE_REPORT_FOLDER = $(PWD)/build/test-coverage
-WIN_CODE_COVERAGE_REPORT_FOLDER = $(PWD)\build\test-coverage
 CODE_COVERAGE_REPORT_NAME_UNIT = $(CODE_COVERAGE_REPORT_FOLDER)/coverage-unit-report
-WIN_CODE_COVERAGE_REPORT_NAME_UNIT = $(CODE_COVERAGE_REPORT_FOLDER)\coverage-unit-report
 VERSION_IMPORT_PATH = github.com/elastic/elastic-package/internal/version
 VERSION_COMMIT_HASH = `git describe --always --long --dirty`
 VERSION_BUILD_TIME = `date +%s`
@@ -40,9 +38,6 @@ update: update-readme
 $(CODE_COVERAGE_REPORT_FOLDER):
 	mkdir -p $@
 
-$(WIN_CODE_COVERAGE_REPORT_FOLDER):
-	mkdir -p $@
-
 test-go: $(CODE_COVERAGE_REPORT_FOLDER)
 	# -count=1 is included to invalidate the test cache. This way, if you run "make test-go" multiple times
 	# you will get fresh test results each time. For instance, changing the source of mocked packages
@@ -55,12 +50,6 @@ test-go-ci: $(CODE_COVERAGE_REPORT_FOLDER)
 	go run gotest.tools/gotestsum --junitfile "$(PWD)/build/test-results/TEST-unit.xml" -- -count=1 -coverprofile=$(CODE_COVERAGE_REPORT_NAME_UNIT).out ./...
 	go run github.com/boumenot/gocover-cobertura < $(CODE_COVERAGE_REPORT_NAME_UNIT).out > $(CODE_COVERAGE_REPORT_NAME_UNIT).xml
 
-test-go-ci-win: $(WIN_CODE_COVERAGE_REPORT_FOLDER)
-	mkdir -p $(PWD)\build\test-results
-	mkdir -p $(PWD)\build\test-coverage
-	go run gotest.tools/gotestsum --junitfile "$(PWD)\build\test-results\TEST-unit.xml" -- -count=1 -coverprofile=$(WIN_CODE_COVERAGE_REPORT_NAME_UNIT).out ./...
-	go run github.com/boumenot/gocover-cobertura < $(WIN_CODE_COVERAGE_REPORT_NAME_UNIT).out > $(WIN_CODE_COVERAGE_REPORT_NAME_UNIT).xml
-
 test-stack-command-default:
 	./scripts/test-stack-command.sh
 

From 6d2be6f9550efc0b974174e35594ef298db171c0 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 7 Feb 2023 20:21:53 +0100
Subject: [PATCH 17/39] Set same docker-compose version as in Jenkins

---
 .buildkite/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 84b9e485b1..4c24cc2f7e 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,6 +1,6 @@
 env:
   SETUP_GVM_VERSION: 'v0.5.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
-  DOCKER_COMPOSE_VERSION: "v2.15.1"
+  DOCKER_COMPOSE_VERSION: "1.25.5" # "v2.15.1"
   KIND_VERSION: 'v0.17.0'
   K8S_VERSION: 'v1.26.0'
 

From 7fd2313823fc89319d7a88ddbf6b71bf8182cedd Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 8 Feb 2023 12:14:28 +0100
Subject: [PATCH 18/39] Use temporal credentials for GCP

---
 .buildkite/hooks/pre-command | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index f99f96291a..5118dc0bce 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -22,9 +22,11 @@ function retry {
   return 0
 }
 
+GCP_SERVICE_ACCOUNT_SECRET_PATH=secret/ci/elastic-elastic-package/gcp-service-account
+
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp" ]]; then
-  export ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field project "secret/ci/elastic-elastic-package/gcp-service-account-temp")
-  export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field google_credentials "secret/ci/elastic-elastic-package/gcp-service-account-temp")
+  export ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectId ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
+  export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
 
   # Environment variables required by the service deployer
   export GOOGLE_CREDENTIALS=${ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET}

From 4ff623386ffcc24f42e1d8b0d876065c11478ea3 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 8 Feb 2023 18:34:24 +0100
Subject: [PATCH 19/39] Add secrets for aws

---
 .buildkite/hooks/pre-command                  | 57 ++++++++++++-------
 .../pipeline.trigger.integration.tests.sh     |  4 --
 .buildkite/scripts/integration_tests.sh       |  4 ++
 3 files changed, 39 insertions(+), 26 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 5118dc0bce..1131a03294 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -3,32 +3,45 @@
 set -euo pipefail
 
 function retry {
-  local retries=$1
-  shift
-
-  local count=0
-  until "$@"; do
-    exit=$?
-    wait=$((2 ** count))
-    count=$((count + 1))
-    if [ $count -lt "$retries" ]; then
-      >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
-      sleep $wait
-    else
-      >&2 echo "Retry $count/$retries exited $exit, no more retries left."
-      return $exit
-    fi
-  done
-  return 0
+    local retries=$1
+    shift
+
+    local count=0
+    until "$@"; do
+        exit=$?
+        wait=$((2 ** count))
+        count=$((count + 1))
+        if [ $count -lt "$retries" ]; then
+            >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+            sleep $wait
+        else
+            >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+            return $exit
+        fi
+    done
+    return 0
 }
 
 GCP_SERVICE_ACCOUNT_SECRET_PATH=secret/ci/elastic-elastic-package/gcp-service-account
+AWS_SERVICE_ACCOUNT_SECRET_PATH=secret/ci/elastic-elastic-package/elastic-temp-aws-account-auth
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp" ]]; then
-  export ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectId ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
-  export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectId ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
+
+    # Environment variables required by the service deployer
+    export GOOGLE_CREDENTIALS=${ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET}
+    export GCP_PROJECT_ID=${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}
+fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "integration-parallel-aws" ]]; then
+    export ELASTIC_PACKAGE_AWS_SECRET_KEY_SECRET=$(retry 5 vault read -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_AWS_ACCESS_KEY_SECRET=$(retry 5 vault read -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+
+    export TEST_ACCESS_KEY="foobartest"
+    export TEST_SECRET_KEY="foobarother"
 
-  # Environment variables required by the service deployer
-  export GOOGLE_CREDENTIALS=${ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET}
-  export GCP_PROJECT_ID=${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}
+    # Environment variables required by the service deployer
+    export AWS_SECRET_ACCESS_KEY=${TEST_SECRET_KEY}
+    export AWS_ACCESS_KEY_ID=${TEST_ACCESS_KEY}
 fi
diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
index f6e3fdf4ac..d886f7b202 100755
--- a/.buildkite/pipeline.trigger.integration.tests.sh
+++ b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -52,10 +52,6 @@ done
 pushd test/packages/parallel > /dev/null
 for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     package_name=$(basename ${package})
-    if [[ "${package_name}" =~ ^aws$ ]]; then
-        # TODO: missing secrets
-        continue
-    fi
     echo "      - label: \":go: Running integration test: ${package_name}\""
     echo "        key: \"integration-parallel-${package_name}\""
     echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}"
diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh
index 896927a07e..b0f0cd25e9 100755
--- a/.buildkite/scripts/integration_tests.sh
+++ b/.buildkite/scripts/integration_tests.sh
@@ -98,6 +98,10 @@ fi
 
 echo "--- Run integration test ${TARGET}"
 if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then
+    if [[ "${PACKAGE}" == "aws" ]]; then
+        echo "Test dummy key: ${AWS_SECRET_ACCESS_KEY} - ${AWS_ACCESS_KEY_ID}"
+        exit 0
+    fi
     make install
     make PACKAGE_UNDER_TEST=${PACKAGE} ${TARGET}
     exit 0

From 3d30f3314dcf75a9dc836b02753cad3694ab848c Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 9 Feb 2023 09:53:23 +0100
Subject: [PATCH 20/39] Remove dummy secrets

---
 .buildkite/hooks/pre-command            | 14 +++++++-------
 .buildkite/scripts/integration_tests.sh |  4 ----
 2 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 1131a03294..795f80c049 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -25,6 +25,9 @@ function retry {
 GCP_SERVICE_ACCOUNT_SECRET_PATH=secret/ci/elastic-elastic-package/gcp-service-account
 AWS_SERVICE_ACCOUNT_SECRET_PATH=secret/ci/elastic-elastic-package/elastic-temp-aws-account-auth
 
+# Secrets must be redacted
+# https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
+
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp" ]]; then
     export ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectId ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
     export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
@@ -35,13 +38,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" =
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "integration-parallel-aws" ]]; then
-    export ELASTIC_PACKAGE_AWS_SECRET_KEY_SECRET=$(retry 5 vault read -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
-    export ELASTIC_PACKAGE_AWS_ACCESS_KEY_SECRET=$(retry 5 vault read -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
-
-    export TEST_ACCESS_KEY="foobartest"
-    export TEST_SECRET_KEY="foobarother"
+    export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault read -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault read -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
 
     # Environment variables required by the service deployer
-    export AWS_SECRET_ACCESS_KEY=${TEST_SECRET_KEY}
-    export AWS_ACCESS_KEY_ID=${TEST_ACCESS_KEY}
+    export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
+    export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
 fi
diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh
index b0f0cd25e9..896927a07e 100755
--- a/.buildkite/scripts/integration_tests.sh
+++ b/.buildkite/scripts/integration_tests.sh
@@ -98,10 +98,6 @@ fi
 
 echo "--- Run integration test ${TARGET}"
 if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then
-    if [[ "${PACKAGE}" == "aws" ]]; then
-        echo "Test dummy key: ${AWS_SECRET_ACCESS_KEY} - ${AWS_ACCESS_KEY_ID}"
-        exit 0
-    fi
     make install
     make PACKAGE_UNDER_TEST=${PACKAGE} ${TARGET}
     exit 0

From d8c528e50a3567e3a655f73b4217c3dfbffdf87a Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 9 Feb 2023 10:35:04 +0100
Subject: [PATCH 21/39] Test junit failures from multiple steps

---
 .../pipeline.trigger.integration.tests.sh     | 38 +++++++++----------
 .../data_stream/test/sample_event.json        |  3 +-
 .../other/long_integers/docs/README.md        |  3 +-
 .../data_stream/memory/sample_event.json      |  3 +-
 .../with-custom-agent/oracle/docs/README.md   |  3 +-
 .../data_stream/pod/sample_event.json         |  3 +-
 6 files changed, 29 insertions(+), 24 deletions(-)

diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
index d886f7b202..89116ab313 100755
--- a/.buildkite/pipeline.trigger.integration.tests.sh
+++ b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -49,25 +49,25 @@ for test in ${CHECK_PACKAGES_TESTS[@]}; do
     fi
 done
 
-pushd test/packages/parallel > /dev/null
-for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
-    package_name=$(basename ${package})
-    echo "      - label: \":go: Running integration test: ${package_name}\""
-    echo "        key: \"integration-parallel-${package_name}\""
-    echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}"
-    echo "        agents:"
-    echo "          provider: \"gcp\""
-    echo "        artifact_paths:"
-    echo "          - build/test-results/*.xml"
-    echo "          - build/elastic-stack-dump/stack/check-*/logs/*.log"
-    echo "          - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/*.log"
-    echo "          - insecure-logs/${package_name}"
-    echo "          - build/elastic-stack-dump/stack/check-${package_name}/logs/elastic-agent-internal/*"
-    echo "          - insecure-logs/${package_name}/container-logs"
-    echo "          - build/container-logs/*.log"
-done
-
-popd > /dev/null
+# pushd test/packages/parallel > /dev/null
+# for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
+#     package_name=$(basename ${package})
+#     echo "      - label: \":go: Running integration test: ${package_name}\""
+#     echo "        key: \"integration-parallel-${package_name}\""
+#     echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}"
+#     echo "        agents:"
+#     echo "          provider: \"gcp\""
+#     echo "        artifact_paths:"
+#     echo "          - build/test-results/*.xml"
+#     echo "          - build/elastic-stack-dump/stack/check-*/logs/*.log"
+#     echo "          - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/*.log"
+#     echo "          - insecure-logs/${package_name}"
+#     echo "          - build/elastic-stack-dump/stack/check-${package_name}/logs/elastic-agent-internal/*"
+#     echo "          - insecure-logs/${package_name}/container-logs"
+#     echo "          - build/container-logs/*.log"
+# done
+#
+# popd > /dev/null
 
 echo "      - label: \":go: Running integration test: test-build-zip\""
 echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-build-zip"
diff --git a/test/packages/other/long_integers/data_stream/test/sample_event.json b/test/packages/other/long_integers/data_stream/test/sample_event.json
index a87b8ec861..d2b6cfc234 100644
--- a/test/packages/other/long_integers/data_stream/test/sample_event.json
+++ b/test/packages/other/long_integers/data_stream/test/sample_event.json
@@ -7,5 +7,6 @@
         9007199254740993,
         18014398509481985,
         9223372036854773807
-    ]
+    ],
+    "foo": "bar"
 }
\ No newline at end of file
diff --git a/test/packages/other/long_integers/docs/README.md b/test/packages/other/long_integers/docs/README.md
index abf9e0a265..274a32d845 100644
--- a/test/packages/other/long_integers/docs/README.md
+++ b/test/packages/other/long_integers/docs/README.md
@@ -12,7 +12,8 @@ An example event for `test` looks as following:
         9007199254740993,
         18014398509481985,
         9223372036854773807
-    ]
+    ],
+    "foo": "bar"
 }
 ```
 
diff --git a/test/packages/with-custom-agent/oracle/data_stream/memory/sample_event.json b/test/packages/with-custom-agent/oracle/data_stream/memory/sample_event.json
index 48dd5186f6..b6b9ecb552 100644
--- a/test/packages/with-custom-agent/oracle/data_stream/memory/sample_event.json
+++ b/test/packages/with-custom-agent/oracle/data_stream/memory/sample_event.json
@@ -97,5 +97,6 @@
     "service": {
         "address": "elastic-package-service_oracle_1:1521",
         "type": "sql"
-    }
+    },
+    "foo": "bar"
 }
\ No newline at end of file
diff --git a/test/packages/with-custom-agent/oracle/docs/README.md b/test/packages/with-custom-agent/oracle/docs/README.md
index 711be0d30c..17e0a1b243 100644
--- a/test/packages/with-custom-agent/oracle/docs/README.md
+++ b/test/packages/with-custom-agent/oracle/docs/README.md
@@ -191,7 +191,8 @@ An example event for `memory` looks as following:
     "service": {
         "address": "elastic-package-service_oracle_1:1521",
         "type": "sql"
-    }
+    },
+    "foo": "bar"
 }
 ```
 
diff --git a/test/packages/with-kind/kubernetes/data_stream/pod/sample_event.json b/test/packages/with-kind/kubernetes/data_stream/pod/sample_event.json
index 62e44029da..281028c341 100644
--- a/test/packages/with-kind/kubernetes/data_stream/pod/sample_event.json
+++ b/test/packages/with-kind/kubernetes/data_stream/pod/sample_event.json
@@ -121,5 +121,6 @@
         "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf",
         "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a",
         "name": "minikube"
-    }
+    },
+    "foo": "bar"
 }
\ No newline at end of file

From 2b8a93cc122c85a5c400edde347472b0b399b51d Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 9 Feb 2023 12:16:58 +0100
Subject: [PATCH 22/39] Revert "Test junit failures from multiple steps"

This reverts commit d8c528e50a3567e3a655f73b4217c3dfbffdf87a.
---
 .../pipeline.trigger.integration.tests.sh     | 38 +++++++++----------
 .../data_stream/test/sample_event.json        |  3 +-
 .../other/long_integers/docs/README.md        |  3 +-
 .../data_stream/memory/sample_event.json      |  3 +-
 .../with-custom-agent/oracle/docs/README.md   |  3 +-
 .../data_stream/pod/sample_event.json         |  3 +-
 6 files changed, 24 insertions(+), 29 deletions(-)

diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
index 89116ab313..d886f7b202 100755
--- a/.buildkite/pipeline.trigger.integration.tests.sh
+++ b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -49,25 +49,25 @@ for test in ${CHECK_PACKAGES_TESTS[@]}; do
     fi
 done
 
-# pushd test/packages/parallel > /dev/null
-# for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
-#     package_name=$(basename ${package})
-#     echo "      - label: \":go: Running integration test: ${package_name}\""
-#     echo "        key: \"integration-parallel-${package_name}\""
-#     echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}"
-#     echo "        agents:"
-#     echo "          provider: \"gcp\""
-#     echo "        artifact_paths:"
-#     echo "          - build/test-results/*.xml"
-#     echo "          - build/elastic-stack-dump/stack/check-*/logs/*.log"
-#     echo "          - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/*.log"
-#     echo "          - insecure-logs/${package_name}"
-#     echo "          - build/elastic-stack-dump/stack/check-${package_name}/logs/elastic-agent-internal/*"
-#     echo "          - insecure-logs/${package_name}/container-logs"
-#     echo "          - build/container-logs/*.log"
-# done
-#
-# popd > /dev/null
+pushd test/packages/parallel > /dev/null
+for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
+    package_name=$(basename ${package})
+    echo "      - label: \":go: Running integration test: ${package_name}\""
+    echo "        key: \"integration-parallel-${package_name}\""
+    echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}"
+    echo "        agents:"
+    echo "          provider: \"gcp\""
+    echo "        artifact_paths:"
+    echo "          - build/test-results/*.xml"
+    echo "          - build/elastic-stack-dump/stack/check-*/logs/*.log"
+    echo "          - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/*.log"
+    echo "          - insecure-logs/${package_name}"
+    echo "          - build/elastic-stack-dump/stack/check-${package_name}/logs/elastic-agent-internal/*"
+    echo "          - insecure-logs/${package_name}/container-logs"
+    echo "          - build/container-logs/*.log"
+done
+
+popd > /dev/null
 
 echo "      - label: \":go: Running integration test: test-build-zip\""
 echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-build-zip"
diff --git a/test/packages/other/long_integers/data_stream/test/sample_event.json b/test/packages/other/long_integers/data_stream/test/sample_event.json
index d2b6cfc234..a87b8ec861 100644
--- a/test/packages/other/long_integers/data_stream/test/sample_event.json
+++ b/test/packages/other/long_integers/data_stream/test/sample_event.json
@@ -7,6 +7,5 @@
         9007199254740993,
         18014398509481985,
         9223372036854773807
-    ],
-    "foo": "bar"
+    ]
 }
\ No newline at end of file
diff --git a/test/packages/other/long_integers/docs/README.md b/test/packages/other/long_integers/docs/README.md
index 274a32d845..abf9e0a265 100644
--- a/test/packages/other/long_integers/docs/README.md
+++ b/test/packages/other/long_integers/docs/README.md
@@ -12,8 +12,7 @@ An example event for `test` looks as following:
         9007199254740993,
         18014398509481985,
         9223372036854773807
-    ],
-    "foo": "bar"
+    ]
 }
 ```
 
diff --git a/test/packages/with-custom-agent/oracle/data_stream/memory/sample_event.json b/test/packages/with-custom-agent/oracle/data_stream/memory/sample_event.json
index b6b9ecb552..48dd5186f6 100644
--- a/test/packages/with-custom-agent/oracle/data_stream/memory/sample_event.json
+++ b/test/packages/with-custom-agent/oracle/data_stream/memory/sample_event.json
@@ -97,6 +97,5 @@
     "service": {
         "address": "elastic-package-service_oracle_1:1521",
         "type": "sql"
-    },
-    "foo": "bar"
+    }
 }
\ No newline at end of file
diff --git a/test/packages/with-custom-agent/oracle/docs/README.md b/test/packages/with-custom-agent/oracle/docs/README.md
index 17e0a1b243..711be0d30c 100644
--- a/test/packages/with-custom-agent/oracle/docs/README.md
+++ b/test/packages/with-custom-agent/oracle/docs/README.md
@@ -191,8 +191,7 @@ An example event for `memory` looks as following:
     "service": {
         "address": "elastic-package-service_oracle_1:1521",
         "type": "sql"
-    },
-    "foo": "bar"
+    }
 }
 ```
 
diff --git a/test/packages/with-kind/kubernetes/data_stream/pod/sample_event.json b/test/packages/with-kind/kubernetes/data_stream/pod/sample_event.json
index 281028c341..62e44029da 100644
--- a/test/packages/with-kind/kubernetes/data_stream/pod/sample_event.json
+++ b/test/packages/with-kind/kubernetes/data_stream/pod/sample_event.json
@@ -121,6 +121,5 @@
         "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf",
         "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a",
         "name": "minikube"
-    },
-    "foo": "bar"
+    }
 }
\ No newline at end of file

From 6a89acec2c6dae51f6247571e4e5bc8ff363619d Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 9 Feb 2023 16:24:01 +0100
Subject: [PATCH 23/39] Test release step conditional

---
 .buildkite/pipeline.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 4c24cc2f7e..a7cae3e6fb 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -53,9 +53,10 @@ steps:
       provider: "gcp"  # junit plugin requires docker
 
   - label: ":github: Release"
-    # TODO enable this conditional
-    # if: |
-    #   build.tag =~ /^v[0-9]+[.][0-9]+[.][0-9]+$$/ &&  build.branch == "main"
+    key: "release"
+    # build.tag =~ /^v[0-9]+\.[0-9]+\.[0-9]+$$/ && build.branch == "main"
+    if: |
+      build.tag != null && build.branch == "main"
     # TODO GITHUB_TOKEN environment var
     command: ".buildkite/scripts/release.sh"
     agents:

From 4c33798e7091c7f8c45f60efddc78fd576096b7e Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 9 Feb 2023 18:06:01 +0100
Subject: [PATCH 24/39] Test release step

---
 .buildkite/hooks/pre-command | 5 +++++
 .buildkite/pipeline.yml      | 5 ++---
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 795f80c049..77c8787e61 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -24,6 +24,7 @@ function retry {
 
 GCP_SERVICE_ACCOUNT_SECRET_PATH=secret/ci/elastic-elastic-package/gcp-service-account
 AWS_SERVICE_ACCOUNT_SECRET_PATH=secret/ci/elastic-elastic-package/elastic-temp-aws-account-auth
+GITHUB_TOKEN_VAULT_PATH=kv/ci-shared/platform-ingest/github_token
 
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
@@ -45,3 +46,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" =
     export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
     export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
 fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "release" ]]; then
+    export GITHUB_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
+fi
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index a7cae3e6fb..621a1ac2ee 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -55,9 +55,8 @@ steps:
   - label: ":github: Release"
     key: "release"
     # build.tag =~ /^v[0-9]+\.[0-9]+\.[0-9]+$$/ && build.branch == "main"
-    if: |
-      build.tag != null && build.branch == "main"
-    # TODO GITHUB_TOKEN environment var
+    # if: |
+    #   build.tag != null && build.branch == "main"
     command: ".buildkite/scripts/release.sh"
     agents:
       image: "golang:1.19.5"

From 0895e2f8cc673ff33c6055829654ea82743e5fc3 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 9 Feb 2023 19:07:59 +0100
Subject: [PATCH 25/39] Re-add if condition in release step

---
 .buildkite/pipeline.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 621a1ac2ee..efb760f3f3 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -55,8 +55,8 @@ steps:
   - label: ":github: Release"
     key: "release"
     # build.tag =~ /^v[0-9]+\.[0-9]+\.[0-9]+$$/ && build.branch == "main"
-    # if: |
-    #   build.tag != null && build.branch == "main"
+    if: |
+      build.tag != null && build.branch == "main"
     command: ".buildkite/scripts/release.sh"
     agents:
       image: "golang:1.19.5"

From 644d36b756444e4d458609b009fb53c52a4d8479 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Fri, 10 Feb 2023 11:10:13 +0100
Subject: [PATCH 26/39] Remove paths from artifacts

---
 .buildkite/pipeline.trigger.integration.tests.sh | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
index d886f7b202..04a7e94496 100755
--- a/.buildkite/pipeline.trigger.integration.tests.sh
+++ b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -61,10 +61,6 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     echo "          - build/test-results/*.xml"
     echo "          - build/elastic-stack-dump/stack/check-*/logs/*.log"
     echo "          - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/*.log"
-    echo "          - insecure-logs/${package_name}"
-    echo "          - build/elastic-stack-dump/stack/check-${package_name}/logs/elastic-agent-internal/*"
-    echo "          - insecure-logs/${package_name}/container-logs"
-    echo "          - build/container-logs/*.log"
 done
 
 popd > /dev/null

From 30fb51b51b7b0982e253b09b86c3c33a37847bf8 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Fri, 10 Feb 2023 13:32:35 +0100
Subject: [PATCH 27/39] Remove windows leftovers

---
 .buildkite/pipeline.yml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index efb760f3f3..e68663af4b 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -24,13 +24,6 @@ steps:
       cpu: "8"
       memory: "4G"
 
-  # - label: ":go: :windows: Run unit tests"
-  #   key: unit-tests-windows
-  #   command: ".buildkite/scripts/unit_tests_windows.ps1"
-  #   agents:
-  #     provider: "gcp"
-  #     image: "family/ci-windows-2022"
-
   - wait: ~
     continue_on_failure: true
 

From aa08b370b6e378d818d14b2747fbe916a65e9ad8 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Fri, 10 Feb 2023 14:12:48 +0100
Subject: [PATCH 28/39] Updated build.tag regex

---
 .buildkite/pipeline.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index e68663af4b..d3bced3418 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -47,9 +47,9 @@ steps:
 
   - label: ":github: Release"
     key: "release"
-    # build.tag =~ /^v[0-9]+\.[0-9]+\.[0-9]+$$/ && build.branch == "main"
+    # build.tag != null && build.branch == "main"
     if: |
-      build.tag != null && build.branch == "main"
+      build.tag =~ /^v[0-9]+\.[0-9]+\.[0-9]+$$/ && build.branch == "main"
     command: ".buildkite/scripts/release.sh"
     agents:
       image: "golang:1.19.5"

From 0e3c48a0020fd82ece428214d125b4969027fd76 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Fri, 10 Feb 2023 14:16:45 +0100
Subject: [PATCH 29/39] Update regex

---
 .buildkite/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index d3bced3418..53c66fc35d 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -49,7 +49,7 @@ steps:
     key: "release"
     # build.tag != null && build.branch == "main"
     if: |
-      build.tag =~ /^v[0-9]+\.[0-9]+\.[0-9]+$$/ && build.branch == "main"
+      build.tag =~ /^v[0-9]+[.][0-9]+[.][0-9]+$$/ && build.branch == "main"
     command: ".buildkite/scripts/release.sh"
     agents:
       image: "golang:1.19.5"

From 00089c2804ad68d9c2de1ed72a1283bde5e8b8bf Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Mon, 13 Feb 2023 15:31:29 +0100
Subject: [PATCH 30/39] Remove release step from Jenkins

---
 .buildkite/scripts/release.sh      |  4 +---
 .ci/Jenkinsfile                    | 17 -----------------
 .ci/package-storage-publish.groovy |  1 -
 3 files changed, 1 insertion(+), 21 deletions(-)

diff --git a/.buildkite/scripts/release.sh b/.buildkite/scripts/release.sh
index 2c5bedd273..afb3d18f48 100755
--- a/.buildkite/scripts/release.sh
+++ b/.buildkite/scripts/release.sh
@@ -9,7 +9,5 @@ git fetch origin --tags
 
 echo "--- running goreleaser"
 # Run latest version of goreleaser
-# TODO Remove check command once this stage is removed from Jenkinsfile
-# curl -sL https://git.io/goreleaser | bash
-curl -sL https://git.io/goreleaser | bash -s -- check
+curl -sL https://git.io/goreleaser | bash
 
diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index de7de19137..ea14834c2d 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -11,7 +11,6 @@ pipeline {
     REPO = "elastic-package"
     BASE_DIR="src/github.com/elastic/elastic-package"
     JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba"
-    GITHUB_TOKEN_CREDENTIALS = "2a9602aa-ab9f-4e52-baf3-b71ca88469c7"
     PIPELINE_LOG_LEVEL='INFO'
     AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth'
     HOME = "${env.WORKSPACE}"
@@ -117,22 +116,6 @@ pipeline {
             }
           }
         }
-        stage('Release') {
-          when {
-            tag pattern: '(v)?\\d+\\.\\d+\\.\\d+', comparator: 'REGEXP'
-          }
-          steps {
-            dir("${BASE_DIR}"){
-              withMageEnv(){
-                withCredentials([string(credentialsId: "${GITHUB_TOKEN_CREDENTIALS}", variable: 'GITHUB_TOKEN')]) {
-                  // Ensure that tags are present so goreleaser can build the changelog from the last release.
-                  gitCmd(cmd: 'fetch', args: '--unshallow --tags')
-                  sh 'curl -sL https://git.io/goreleaser | bash'
-                }
-              }
-            }
-          }
-        }
       }
     }
   }
diff --git a/.ci/package-storage-publish.groovy b/.ci/package-storage-publish.groovy
index 9dc4d16015..8c6066d6d4 100644
--- a/.ci/package-storage-publish.groovy
+++ b/.ci/package-storage-publish.groovy
@@ -10,7 +10,6 @@ pipeline {
 
     BASE_DIR="src/github.com/elastic/elastic-package"
     JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba"
-    GITHUB_TOKEN_CREDENTIALS = "2a9602aa-ab9f-4e52-baf3-b71ca88469c7"
     PIPELINE_LOG_LEVEL='INFO'
 
     // Signing

From 0feb72a1e964b2583ff9426a6fe5760556a97355 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 14 Feb 2023 13:11:56 +0100
Subject: [PATCH 31/39] Add post-checkout hook

---
 .buildkite/hooks/post-checkout | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 .buildkite/hooks/post-checkout

diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout
new file mode 100644
index 0000000000..b0e9bebcac
--- /dev/null
+++ b/.buildkite/hooks/post-checkout
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -euo pipefail
+
+merge() {
+    local pr_id=$1
+    local branch=$2
+
+    echo "Checking out reference refs/pull/${pr_id}/merge into ${branch}"
+    git fetch origin +refs/pull/${pr_id}/merge:${branch}
+    git checkout ${branch}
+}
+
+pull_request="${BUILDKITE_PULL_REQUEST:-false}"
+
+if [[ "${pull_request}" == "false" ]]; then
+    echo "Not a pull request, skipping"
+    exit 0
+fi
+
+# use merge changeset
+PR_ID=${BUILDKITE_PULL_REQUEST}
+MERGE_BRANCH=${pr_merge_${PR_ID}}
+
+checkout_merge ${PR_ID} ${MERGE_BRANCH}
+
+echo "--- Commit information"
+git log --format=%B -n 1

From daa4de855db0a358dbeec89af8d2200eaf369e5f Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 14 Feb 2023 13:13:13 +0100
Subject: [PATCH 32/39] fix branch name var

---
 .buildkite/hooks/post-checkout | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout
index b0e9bebcac..54dcab808c 100644
--- a/.buildkite/hooks/post-checkout
+++ b/.buildkite/hooks/post-checkout
@@ -20,7 +20,7 @@ fi
 
 # use merge changeset
 PR_ID=${BUILDKITE_PULL_REQUEST}
-MERGE_BRANCH=${pr_merge_${PR_ID}}
+MERGE_BRANCH="pr_merge_${PR_ID}"
 
 checkout_merge ${PR_ID} ${MERGE_BRANCH}
 

From 8387042ce45b543d734d4c75c7b3266fc5254020 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 14 Feb 2023 13:14:14 +0100
Subject: [PATCH 33/39] update function name

---
 .buildkite/hooks/post-checkout | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout
index 54dcab808c..a8e0b7412b 100644
--- a/.buildkite/hooks/post-checkout
+++ b/.buildkite/hooks/post-checkout
@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-merge() {
+checkout_merge() {
     local pr_id=$1
     local branch=$2
 

From 9159ddc4aae42222a06efbec0c1b0b730df86ea5 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 14 Feb 2023 13:23:44 +0100
Subject: [PATCH 34/39] Update echo message

---
 .buildkite/hooks/post-checkout | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout
index a8e0b7412b..799cbc6c34 100644
--- a/.buildkite/hooks/post-checkout
+++ b/.buildkite/hooks/post-checkout
@@ -24,5 +24,5 @@ MERGE_BRANCH="pr_merge_${PR_ID}"
 
 checkout_merge ${PR_ID} ${MERGE_BRANCH}
 
-echo "--- Commit information"
+echo "Commit information"
 git log --format=%B -n 1

From 93efa175a7d17ac31d57cdb0d1d18f2a80bdff6c Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 14 Feb 2023 16:50:38 +0100
Subject: [PATCH 35/39] Update post-checkout hook

Instead of using pull/id/merge , a merge commit is created manually.
Github reference is not updated in case there is a conflict and it could
cause that Buildkite runs the pipeline in an old changeset.
---
 .buildkite/hooks/post-checkout | 38 +++++++++++++++++++++++++++-------
 1 file changed, 30 insertions(+), 8 deletions(-)

diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout
index 799cbc6c34..85bc44f1fc 100644
--- a/.buildkite/hooks/post-checkout
+++ b/.buildkite/hooks/post-checkout
@@ -3,12 +3,33 @@
 set -euo pipefail
 
 checkout_merge() {
-    local pr_id=$1
-    local branch=$2
-
-    echo "Checking out reference refs/pull/${pr_id}/merge into ${branch}"
-    git fetch origin +refs/pull/${pr_id}/merge:${branch}
-    git checkout ${branch}
+    local target_branch=$1
+    local pr_commit=$2
+    local merge_branch=$3
+
+    if [[ -z "${target_branch}" ]]; then
+        echo "No pull request target branch"
+        exit 1
+    fi
+
+    git fetch -v origin "${target_branch}"
+    git checkout FETCH_HEAD
+    echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+
+    # create temporal branch to merge the PR with the target branch
+    git checkout -b ${merge_branch}
+    echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+
+    # env vars to ensure merge is non-interactive
+    GIT_AUTHOR_NAME="github-merged-pr-post-checkout" \
+    GIT_COMMITTER_EMAIL="auto-merge@buildkite" \
+    GIT_COMMITTER_NAME="github-merged-pr-post-checkout" \
+    git merge --no-edit "${BUILDKITE_COMMIT}" || {
+        local merge_result=$?
+        echo "Merge failed: ${merge_result}"
+        git merge --abort
+        exit ${merge_result}
+    }
 }
 
 pull_request="${BUILDKITE_PULL_REQUEST:-false}"
@@ -18,11 +39,12 @@ if [[ "${pull_request}" == "false" ]]; then
     exit 0
 fi
 
-# use merge changeset
+TARGET_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-master}"
+PR_COMMIT="${BUILDKITE_COMMIT}"
 PR_ID=${BUILDKITE_PULL_REQUEST}
 MERGE_BRANCH="pr_merge_${PR_ID}"
 
-checkout_merge ${PR_ID} ${MERGE_BRANCH}
+checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}"
 
 echo "Commit information"
 git log --format=%B -n 1

From 670cd89e8cf8c95e6f83b541dc822d44309947f0 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 14 Feb 2023 17:02:11 +0100
Subject: [PATCH 36/39] Change message

---
 .buildkite/hooks/post-checkout | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout
index 85bc44f1fc..d901ff2a07 100644
--- a/.buildkite/hooks/post-checkout
+++ b/.buildkite/hooks/post-checkout
@@ -18,7 +18,7 @@ checkout_merge() {
 
     # create temporal branch to merge the PR with the target branch
     git checkout -b ${merge_branch}
-    echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+    echo "New branch created: $(git rev-parse --abbrev-ref HEAD)"
 
     # env vars to ensure merge is non-interactive
     GIT_AUTHOR_NAME="github-merged-pr-post-checkout" \

From c153d441e1b629b622801cc89d5f04fd2b176d45 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 14 Feb 2023 17:08:01 +0100
Subject: [PATCH 37/39] set author name/email

---
 .buildkite/hooks/post-checkout | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout
index d901ff2a07..08916f0446 100644
--- a/.buildkite/hooks/post-checkout
+++ b/.buildkite/hooks/post-checkout
@@ -20,10 +20,10 @@ checkout_merge() {
     git checkout -b ${merge_branch}
     echo "New branch created: $(git rev-parse --abbrev-ref HEAD)"
 
-    # env vars to ensure merge is non-interactive
-    GIT_AUTHOR_NAME="github-merged-pr-post-checkout" \
-    GIT_COMMITTER_EMAIL="auto-merge@buildkite" \
-    GIT_COMMITTER_NAME="github-merged-pr-post-checkout" \
+    # set author identity so it can be run git merge
+    git config user.name "github-merged-pr-post-checkout"
+    git config user.email "auto-merge@buildkite"
+
     git merge --no-edit "${BUILDKITE_COMMIT}" || {
         local merge_result=$?
         echo "Merge failed: ${merge_result}"

From 1385a94043a42ad48f6008eae34d4a1b01757b73 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 15 Feb 2023 11:22:55 +0100
Subject: [PATCH 38/39] Update path aws account

---
 .buildkite/hooks/pre-command | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 77c8787e61..ec76774033 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -23,7 +23,7 @@ function retry {
 }
 
 GCP_SERVICE_ACCOUNT_SECRET_PATH=secret/ci/elastic-elastic-package/gcp-service-account
-AWS_SERVICE_ACCOUNT_SECRET_PATH=secret/ci/elastic-elastic-package/elastic-temp-aws-account-auth
+AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_account_auth
 GITHUB_TOKEN_VAULT_PATH=kv/ci-shared/platform-ingest/github_token
 
 # Secrets must be redacted
@@ -39,8 +39,8 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" =
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "integration-parallel-aws" ]]; then
-    export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault read -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
-    export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault read -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
 
     # Environment variables required by the service deployer
     export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}

From 3740ebe60d9fa9ab216408fa68d91bb6a3f74986 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 15 Feb 2023 18:29:04 +0100
Subject: [PATCH 39/39] Test to ensure pre-command group is rendered

---
 .buildkite/hooks/post-checkout | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout
index 08916f0446..67ae21491b 100644
--- a/.buildkite/hooks/post-checkout
+++ b/.buildkite/hooks/post-checkout
@@ -48,3 +48,6 @@ checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}"
 
 echo "Commit information"
 git log --format=%B -n 1
+
+# Ensure buildkite groups are rendered
+echo ""