From 85e69c207f47f0dc968cc88a91ff8bf7b8f15403 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 21 Feb 2024 19:26:36 +0100
Subject: [PATCH 01/33] Add pipeline to cleanup cloud resources

---
 .buildkite/configs/cleanup.aws.yml            | 54 +++++++++++++++++++
 .buildkite/configs/cleanup.gcp.yml            | 37 +++++++++++++
 .buildkite/hooks/pre-command                  | 16 ++++++
 ...pipeline.elastic-package-cloud-cleanup.yml |  9 ++++
 .buildkite/scripts/cloud-cleanup.sh           | 25 +++++++++
 catalog-info.yaml                             | 45 ++++++++++++++++
 6 files changed, 186 insertions(+)
 create mode 100644 .buildkite/configs/cleanup.aws.yml
 create mode 100644 .buildkite/configs/cleanup.gcp.yml
 create mode 100644 .buildkite/pipeline.elastic-package-cloud-cleanup.yml
 create mode 100644 .buildkite/scripts/cloud-cleanup.sh

diff --git a/.buildkite/configs/cleanup.aws.yml b/.buildkite/configs/cleanup.aws.yml
new file mode 100644
index 0000000000..5838fdb771
--- /dev/null
+++ b/.buildkite/configs/cleanup.aws.yml
@@ -0,0 +1,54 @@
+---
+version: "1.0"
+
+accounts:
+  - name: "${ACCOUNT_PROJECT}"
+    driver: "aws"
+    options:
+      key: '${ACCOUNT_KEY}'
+      secret: '${ACCOUNT_SECRET}'
+
+scanners:
+  - account_name: "${ACCOUNT_PROJECT}"
+    resources:
+      - type: 'node'
+        regions:
+          - us-east-1
+        filters:
+          # - type: "<"
+          #   pointer: "/created_at"
+          #   param: "${CREATION_DATE}"
+          #   converters:
+          #     param: "date"
+          #     value: "date"
+          - type: "regex"
+            pointer: "/extra/tags/repo"
+            param: "^(elastic-package|integrations)"
+          - type: "="
+            pointer: "/extra/tags/environment"
+            param: "ci"
+          - type: "regex"
+            pointer: "/name"
+            param: "^elastic-package-(.*)"
+          - type: "!="
+            pointer: "/state"
+            param: "terminated"
+      - type: 'object_storage_bucket'
+        regions:
+          - us-east-1
+        filters:
+          # - type: "<"
+          #   pointer: "/created_at"
+          #   param: "${CREATION_DATE}"
+          #   converters:
+          #     param: "date"
+          #     value: "date"
+          - type: "regex"
+            pointer: "/extra/tags/repo"
+            param: "^(elastic-package|integrations)"
+          - type: "="
+            pointer: "/extra/tags/environment"
+            param: "ci"
+          - type: "regex"
+            pointer: "/name"
+            param: "^elastic-package-(.*)"
diff --git a/.buildkite/configs/cleanup.gcp.yml b/.buildkite/configs/cleanup.gcp.yml
new file mode 100644
index 0000000000..d08cb7b630
--- /dev/null
+++ b/.buildkite/configs/cleanup.gcp.yml
@@ -0,0 +1,37 @@
+---
+version: "1.0"
+
+accounts:
+  - name: "${ACCOUNT_PROJECT}"
+    driver: "gce"
+    options:
+      key: "${ACCOUNT_KEY}"
+      secret: "${ACCOUNT_SECRET}"
+      project: "${ACCOUNT_PROJECT}"
+
+scanners:
+  - account_name: "${ACCOUNT_PROJECT}"
+    resources:
+      - type: "node"
+        regions:
+          - "us-east1"
+        filters:
+          - type: "<"
+            pointer: "/extra/creationTimestamp"
+            param: "${CREATION_DATE}"
+            converters:
+              param: "date"
+              value: "date"
+          - type: "="
+            pointer: "/extra/labels/repo"
+            param: "elastic-package"
+          - type: "regex"
+            pointer: "/name"
+            param: "^elastic-package-(.*)"
+          - type: "!="
+            pointer: "/state"
+            param: "unknown"
+          - type: "!="
+            pointer: "/state"
+            param: "terminated"
+
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index c4dc9ed9ed..ebfddeb0ce 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -87,3 +87,19 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-test-with-integrations" &&
     GITHUB_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
     export GITHUB_TOKEN
 fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-cloud-cleanup" ]]; then
+    ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_AWS_SECRET_KEY
+    ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_AWS_ACCESS_KEY
+    ELASTIC_PACKAGE_AWS_USER_SECRET=$(retry 5 vault kv get -field user ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_AWS_USER_SECRET
+
+    ELASTIC_PACKAGE_GCP_KEY_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH} | jq -r '.private_key')
+    export ELASTIC_PACKAGE_GCP_KEY_SECRET
+    ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH} | jq -r '.project_id')
+    export ELASTIC_PACKAGE_GCP_PROJECT_SECRET
+    ELASTIC_PACKAGE_GCP_EMAIL_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH} | jq -r '.client_email')
+    export ELASTIC_PACKAGE_GCP_EMAIL_SECRET
+fi
diff --git a/.buildkite/pipeline.elastic-package-cloud-cleanup.yml b/.buildkite/pipeline.elastic-package-cloud-cleanup.yml
new file mode 100644
index 0000000000..7702925e97
--- /dev/null
+++ b/.buildkite/pipeline.elastic-package-cloud-cleanup.yml
@@ -0,0 +1,9 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+# Removes stale Cloud resources (AWS and GCP) having matching labels, name prefixes and older than 24 hours
+steps:
+  - label: "Cloud Cleanup"
+    key: "cloud-cleanup"
+    command: ".buildkite/scripts/cloud-cleanup.sh"
+    agents:
+      provider: "gcp"
diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
new file mode 100644
index 0000000000..af9220cc4a
--- /dev/null
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+export CREATION_DATE=$(date -Is -d "24 hours ago")
+
+echo "--- Cleaning up AWS resources..."
+docker run -v $(pwd)/.buildkite/misc/gce-cleanup.yml:/etc/cloud-reaper/config.yml \
+  -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_AWS_SECRET_KEY}" \
+  -e ACCOUNT_KEY="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}" \
+  -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_AWS_USER_SECRET}"
+  -e CREATION_DATE=${CREATION_DATE} \
+  ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
+    --config /etc/cloud-reaper/config.yml \
+    plan
+
+echo "--- Cleaning up GCP resources..."
+docker run -v $(pwd)/.buildkite/misc/gce-cleanup.yml:/etc/cloud-reaper/config.yml \
+  -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_GCP_KEY_SECRET}" \
+  -e ACCOUNT_KEY="${ELASTIC_PACKAGE_GCP_EMAIL_SECRET}" \
+  -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}"
+  -e CREATION_DATE=${CREATION_DATE} \
+  ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
+    --config /etc/cloud-reaper/config.yml \
+    plan
diff --git a/catalog-info.yaml b/catalog-info.yaml
index 664f8a7fa7..274829680c 100644
--- a/catalog-info.yaml
+++ b/catalog-info.yaml
@@ -108,3 +108,48 @@ spec:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
           access_level: READ_ONLY
+
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-elastic-package-cloud-cleanup
+  description: Clean up stale cloud resources
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/elastic-package-cloud-cleanup
+
+spec:
+  type: buildkite-pipeline
+  owner: group:ingest-fp
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: elastic-package-cloud-cleanup
+      description: Buildkite pipeline for cleaning stale resource in cloud providers
+    spec:
+      pipeline_file: ".buildkite/pipeline.elastic-package-cloud-cleanup.yml"
+      provider_settings:
+        build_pull_request_forks: false
+        build_pull_requests: false # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot
+        publish_commit_status: false # do not update status of commits for this pipeline
+        build_tags: false
+        build_branches: false
+        filter_enabled: true
+        filter_condition: >-
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
+      repository: elastic/elastic-package
+      schedules:
+        Daily main:
+          branch: main
+          cronline: "@daily"
+          message: Daily Cloud cleanup
+      teams:
+        ingest-fp:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ
+

From 7a21ecab6bde23bab5e1ee5d672d6b57d1f209af Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 21 Feb 2024 19:31:27 +0100
Subject: [PATCH 02/33] Test in main pipeline

---
 .buildkite/hooks/pre-command                  | 28 +++++-
 ...pipeline.elastic-package-cloud-cleanup.yml |  2 +
 .buildkite/pipeline.yml                       | 97 ++++++++++---------
 .buildkite/scripts/cloud-cleanup.sh           |  3 +-
 catalog-info.yaml                             |  6 +-
 5 files changed, 84 insertions(+), 52 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index ebfddeb0ce..eb6d2279c1 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -96,10 +96,32 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-cloud-cleanup" ]]; then
     ELASTIC_PACKAGE_AWS_USER_SECRET=$(retry 5 vault kv get -field user ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
     export ELASTIC_PACKAGE_AWS_USER_SECRET
 
-    ELASTIC_PACKAGE_GCP_KEY_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH} | jq -r '.private_key')
+    ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET
+    ELASTIC_PACKAGE_GCP_KEY_SECRET=$(echo "${ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET}" | jq -r '.private_key')
+    export ELASTIC_PACKAGE_GCP_KEY_SECRET
+    ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectId ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_GCP_PROJECT_SECRET
+    ELASTIC_PACKAGE_GCP_EMAIL_SECRET=$(retry 5 vault read -field username ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_GCP_EMAIL_SECRET
+fi
+
+## TODO: Remove before merging
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "cloud-cleanup" ]]; then
+    ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_AWS_SECRET_KEY
+    ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_AWS_ACCESS_KEY
+    ELASTIC_PACKAGE_AWS_USER_SECRET=$(retry 5 vault kv get -field user ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_AWS_USER_SECRET
+
+    ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET
+    ELASTIC_PACKAGE_GCP_KEY_SECRET=$(echo "${ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET}" | jq -r '.private_key')
     export ELASTIC_PACKAGE_GCP_KEY_SECRET
-    ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH} | jq -r '.project_id')
+    ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectID ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
     export ELASTIC_PACKAGE_GCP_PROJECT_SECRET
-    ELASTIC_PACKAGE_GCP_EMAIL_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH} | jq -r '.client_email')
+    ELASTIC_PACKAGE_GCP_EMAIL_SECRET=$(retry 5 vault read -field username ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
     export ELASTIC_PACKAGE_GCP_EMAIL_SECRET
+
 fi
diff --git a/.buildkite/pipeline.elastic-package-cloud-cleanup.yml b/.buildkite/pipeline.elastic-package-cloud-cleanup.yml
index 7702925e97..77daceabdc 100644
--- a/.buildkite/pipeline.elastic-package-cloud-cleanup.yml
+++ b/.buildkite/pipeline.elastic-package-cloud-cleanup.yml
@@ -1,6 +1,8 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 # Removes stale Cloud resources (AWS and GCP) having matching labels, name prefixes and older than 24 hours
+name: elastic-package-cloud-cleanup
+
 steps:
   - label: "Cloud Cleanup"
     key: "cloud-cleanup"
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 42e868699a..5db2d97196 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -28,57 +28,62 @@ steps:
       cpu: "8"
       memory: "4G"
 
-  - label: ":go: :windows: Run unit tests"
-    key: unit-tests-windows
-    command: ".buildkite/scripts/unit_tests_windows.ps1"
+  # - label: ":go: :windows: Run unit tests"
+  #   key: unit-tests-windows
+  #   command: ".buildkite/scripts/unit_tests_windows.ps1"
+  #   agents:
+  #     provider: "gcp"
+  #     image: "${WINDOWS_AGENT_IMAGE}"
+  #   artifact_paths:
+  #     - "TEST-unit-windows.xml"
+
+  - label: "Cloud Cleanup"
+    key: "cloud-cleanup"
+    command: ".buildkite/scripts/cloud-cleanup.sh"
     agents:
       provider: "gcp"
-      image: "${WINDOWS_AGENT_IMAGE}"
-    artifact_paths:
-      - "TEST-unit-windows.xml"
-
-  - wait: ~
-    continue_on_failure: true
+  # - wait: ~
+  #   continue_on_failure: true
 
-  - label: ":pipeline: Trigger Integration tests"
-    command: ".buildkite/pipeline.trigger.integration.tests.sh | buildkite-agent pipeline upload"
-    depends_on:
-      - step: check-static
-        allow_failure: false
-      - step: unit-tests-linux
-        allow_failure: false
-      - step: unit-tests-windows
-        allow_failure: false
+  # - label: ":pipeline: Trigger Integration tests"
+  #   command: ".buildkite/pipeline.trigger.integration.tests.sh | buildkite-agent pipeline upload"
+  #   depends_on:
+  #     - step: check-static
+  #       allow_failure: false
+  #     - step: unit-tests-linux
+  #       allow_failure: false
+  #     - step: unit-tests-windows
+  #       allow_failure: false
 
-  - wait: ~
-    continue_on_failure: true
+  # - wait: ~
+  #   continue_on_failure: true
 
-  - label: ":junit: Transform windows paths to linux for Junit plugin"
-    commands:
-      - buildkite-agent artifact download "*-windows.xml" . --step unit-tests-windows
-      - mkdir -p build/test-results
-      - for file in $(ls *-windows.xml); do mv $$file build/test-results/; done
-    agents:
-      image: "${LINUX_AGENT_IMAGE}"
-      cpu: "8"
-      memory: "4G"
-    artifact_paths:
-      - "build/test-results/*.xml"
+  # - label: ":junit: Transform windows paths to linux for Junit plugin"
+  #   commands:
+  #     - buildkite-agent artifact download "*-windows.xml" . --step unit-tests-windows
+  #     - mkdir -p build/test-results
+  #     - for file in $(ls *-windows.xml); do mv $$file build/test-results/; done
+  #   agents:
+  #     image: "${LINUX_AGENT_IMAGE}"
+  #     cpu: "8"
+  #     memory: "4G"
+  #   artifact_paths:
+  #     - "build/test-results/*.xml"
 
-  - wait: ~
-    continue_on_failure: true
+  # - wait: ~
+  #   continue_on_failure: true
 
-  - label: ":junit: Junit annotate"
-    plugins:
-      - junit-annotate#v2.4.1:
-          artifacts: "build/test-results/*.xml"
-    agents:
-      provider: "gcp"  # junit plugin requires docker
+  # - label: ":junit: Junit annotate"
+  #   plugins:
+  #     - junit-annotate#v2.4.1:
+  #         artifacts: "build/test-results/*.xml"
+  #   agents:
+  #     provider: "gcp"  # junit plugin requires docker
 
-  - label: ":github: Release"
-    key: "release"
-    if: |
-      build.tag =~ /^v[0-9]+[.][0-9]+[.][0-9]+$$/
-    command: ".buildkite/scripts/release.sh"
-    agents:
-      provider: "gcp"
+  # - label: ":github: Release"
+  #   key: "release"
+  #   if: |
+  #     build.tag =~ /^v[0-9]+[.][0-9]+[.][0-9]+$$/
+  #   command: ".buildkite/scripts/release.sh"
+  #   agents:
+  #     provider: "gcp"
diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index af9220cc4a..bfd7bbacfd 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -2,7 +2,8 @@
 
 set -euo pipefail
 
-export CREATION_DATE=$(date -Is -d "24 hours ago")
+# TODO: change to 24 hours ago
+export CREATION_DATE=$(date -Is -d "1 minute ago")
 
 echo "--- Cleaning up AWS resources..."
 docker run -v $(pwd)/.buildkite/misc/gce-cleanup.yml:/etc/cloud-reaper/config.yml \
diff --git a/catalog-info.yaml b/catalog-info.yaml
index 274829680c..f3b6a235a1 100644
--- a/catalog-info.yaml
+++ b/catalog-info.yaml
@@ -114,7 +114,7 @@ spec:
 apiVersion: backstage.io/v1alpha1
 kind: Resource
 metadata:
-  name: buildkite-elastic-package-cloud-cleanup
+  name: buildkite-pipeline-elastic-package-cloud-cleanup
   description: Clean up stale cloud resources
   links:
     - title: Pipeline
@@ -141,11 +141,13 @@ spec:
         filter_enabled: true
         filter_condition: >-
           build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
+      cancel_intermediate_builds: false  # do not cancel any build to avoid inconsistent states
+      skip_intermediate_builds: true     # just need to run the latest commit
       repository: elastic/elastic-package
       schedules:
         Daily main:
           branch: main
-          cronline: "@daily"
+          cronline: "00 1 * * *"
           message: Daily Cloud cleanup
       teams:
         ingest-fp:

From 95b0cb27d1b284da1d8b925b946b8be98f39bf7b Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 22 Feb 2024 11:54:56 +0100
Subject: [PATCH 03/33] Update secrets in pre-hook

---
 .buildkite/hooks/pre-command | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index eb6d2279c1..35dcd6a8ea 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -47,7 +47,7 @@ export CREATED_DATE
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && ("$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives) ]]; then
-    PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
+    PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH} | jq -c)
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET
     export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
 fi
@@ -55,7 +55,7 @@ fi
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp" ]]; then
     ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectId ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
     export ELASTIC_PACKAGE_GCP_PROJECT_SECRET
-    ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
+    ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH} | jq -c)
     export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET
 
     # Environment variables required by the service deployer
@@ -96,9 +96,9 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-cloud-cleanup" ]]; then
     ELASTIC_PACKAGE_AWS_USER_SECRET=$(retry 5 vault kv get -field user ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
     export ELASTIC_PACKAGE_AWS_USER_SECRET
 
-    ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
+    ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH} | jq -c)
     export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET
-    ELASTIC_PACKAGE_GCP_KEY_SECRET=$(echo "${ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET}" | jq -r '.private_key')
+    ELASTIC_PACKAGE_GCP_KEY_SECRET=$(echo "${ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET}" | jq -r '.private_key' | tr -d '\n')
     export ELASTIC_PACKAGE_GCP_KEY_SECRET
     ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectId ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
     export ELASTIC_PACKAGE_GCP_PROJECT_SECRET
@@ -115,9 +115,9 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" =
     ELASTIC_PACKAGE_AWS_USER_SECRET=$(retry 5 vault kv get -field user ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
     export ELASTIC_PACKAGE_AWS_USER_SECRET
 
-    ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
+    ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH} | jq -c)
     export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET
-    ELASTIC_PACKAGE_GCP_KEY_SECRET=$(echo "${ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET}" | jq -r '.private_key')
+    ELASTIC_PACKAGE_GCP_KEY_SECRET=$(echo "${ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET}" | jq -r '.private_key' | tr -d '\n')
     export ELASTIC_PACKAGE_GCP_KEY_SECRET
     ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectID ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
     export ELASTIC_PACKAGE_GCP_PROJECT_SECRET

From bf30001a49b1ea7e88670aec1a82986631925f0a Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 22 Feb 2024 12:11:25 +0100
Subject: [PATCH 04/33] Enable filter by date

---
 .buildkite/configs/cleanup.aws.yml | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/.buildkite/configs/cleanup.aws.yml b/.buildkite/configs/cleanup.aws.yml
index 5838fdb771..745030aff7 100644
--- a/.buildkite/configs/cleanup.aws.yml
+++ b/.buildkite/configs/cleanup.aws.yml
@@ -15,12 +15,12 @@ scanners:
         regions:
           - us-east-1
         filters:
-          # - type: "<"
-          #   pointer: "/created_at"
-          #   param: "${CREATION_DATE}"
-          #   converters:
-          #     param: "date"
-          #     value: "date"
+          - type: "<"
+            pointer: "/created_at"
+            param: "${CREATION_DATE}"
+            converters:
+              param: "date"
+              value: "date"
           - type: "regex"
             pointer: "/extra/tags/repo"
             param: "^(elastic-package|integrations)"
@@ -37,12 +37,12 @@ scanners:
         regions:
           - us-east-1
         filters:
-          # - type: "<"
-          #   pointer: "/created_at"
-          #   param: "${CREATION_DATE}"
-          #   converters:
-          #     param: "date"
-          #     value: "date"
+          - type: "<"
+            pointer: "/created_at"
+            param: "${CREATION_DATE}"
+            converters:
+              param: "date"
+              value: "date"
           - type: "regex"
             pointer: "/extra/tags/repo"
             param: "^(elastic-package|integrations)"

From 8beb94298eb8ad0c9459f17a5e1ee790da137b93 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 22 Feb 2024 12:54:19 +0100
Subject: [PATCH 05/33] Fix field name

---
 .buildkite/hooks/pre-command | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 35dcd6a8ea..76dc12b68e 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -119,7 +119,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" =
     export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET
     ELASTIC_PACKAGE_GCP_KEY_SECRET=$(echo "${ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET}" | jq -r '.private_key' | tr -d '\n')
     export ELASTIC_PACKAGE_GCP_KEY_SECRET
-    ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectID ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
+    ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectId ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
     export ELASTIC_PACKAGE_GCP_PROJECT_SECRET
     ELASTIC_PACKAGE_GCP_EMAIL_SECRET=$(retry 5 vault read -field username ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
     export ELASTIC_PACKAGE_GCP_EMAIL_SECRET

From b4d1b9cd73ab7058b43a2ff760cf41a52d20108c Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 22 Feb 2024 12:58:43 +0100
Subject: [PATCH 06/33] Add missing backslash

---
 .buildkite/scripts/cloud-cleanup.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index bfd7bbacfd..ca83b0bee3 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -9,7 +9,7 @@ echo "--- Cleaning up AWS resources..."
 docker run -v $(pwd)/.buildkite/misc/gce-cleanup.yml:/etc/cloud-reaper/config.yml \
   -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_AWS_SECRET_KEY}" \
   -e ACCOUNT_KEY="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}" \
-  -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_AWS_USER_SECRET}"
+  -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_AWS_USER_SECRET}" \
   -e CREATION_DATE=${CREATION_DATE} \
   ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
     --config /etc/cloud-reaper/config.yml \
@@ -19,7 +19,7 @@ echo "--- Cleaning up GCP resources..."
 docker run -v $(pwd)/.buildkite/misc/gce-cleanup.yml:/etc/cloud-reaper/config.yml \
   -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_GCP_KEY_SECRET}" \
   -e ACCOUNT_KEY="${ELASTIC_PACKAGE_GCP_EMAIL_SECRET}" \
-  -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}"
+  -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}" \
   -e CREATION_DATE=${CREATION_DATE} \
   ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
     --config /etc/cloud-reaper/config.yml \

From b5bcb16b6ac171e9c7f0650e65e8a32c16b49fff Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 22 Feb 2024 13:01:46 +0100
Subject: [PATCH 07/33] Add environment variable for docker registry

---
 .buildkite/pipeline.elastic-package-cloud-cleanup.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.buildkite/pipeline.elastic-package-cloud-cleanup.yml b/.buildkite/pipeline.elastic-package-cloud-cleanup.yml
index 77daceabdc..f82442703a 100644
--- a/.buildkite/pipeline.elastic-package-cloud-cleanup.yml
+++ b/.buildkite/pipeline.elastic-package-cloud-cleanup.yml
@@ -3,6 +3,9 @@
 # Removes stale Cloud resources (AWS and GCP) having matching labels, name prefixes and older than 24 hours
 name: elastic-package-cloud-cleanup
 
+env:
+  DOCKER_REGISTRY: docker.elastic.co
+
 steps:
   - label: "Cloud Cleanup"
     key: "cloud-cleanup"

From c96510b033610657d7ef5eb6d0ab683b928731dc Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 22 Feb 2024 13:04:43 +0100
Subject: [PATCH 08/33] Add docker registry var for test step

---
 .buildkite/pipeline.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 5db2d97196..0db926e983 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -40,6 +40,8 @@ steps:
   - label: "Cloud Cleanup"
     key: "cloud-cleanup"
     command: ".buildkite/scripts/cloud-cleanup.sh"
+    env:
+      DOCKER_REGISTRY: docker.elastic.co
     agents:
       provider: "gcp"
   # - wait: ~

From 2f50da87648d90d2d012e4d06dccf26135ca5570 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 22 Feb 2024 13:08:55 +0100
Subject: [PATCH 09/33] Set the correct paths for configs

---
 .buildkite/scripts/cloud-cleanup.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index ca83b0bee3..4baf2bc97f 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -6,7 +6,7 @@ set -euo pipefail
 export CREATION_DATE=$(date -Is -d "1 minute ago")
 
 echo "--- Cleaning up AWS resources..."
-docker run -v $(pwd)/.buildkite/misc/gce-cleanup.yml:/etc/cloud-reaper/config.yml \
+docker run -v $(pwd)/.buildkite/configs/cleanup.aws.yml:/etc/cloud-reaper/config.yml \
   -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_AWS_SECRET_KEY}" \
   -e ACCOUNT_KEY="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}" \
   -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_AWS_USER_SECRET}" \
@@ -16,7 +16,7 @@ docker run -v $(pwd)/.buildkite/misc/gce-cleanup.yml:/etc/cloud-reaper/config.ym
     plan
 
 echo "--- Cleaning up GCP resources..."
-docker run -v $(pwd)/.buildkite/misc/gce-cleanup.yml:/etc/cloud-reaper/config.yml \
+docker run -v $(pwd)/.buildkite/configs/cleanup.gcp.yml:/etc/cloud-reaper/config.yml \
   -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_GCP_KEY_SECRET}" \
   -e ACCOUNT_KEY="${ELASTIC_PACKAGE_GCP_EMAIL_SECRET}" \
   -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}" \

From edebdea4c9686913f011f7428d44c68b831c68e9 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 22 Feb 2024 13:22:05 +0100
Subject: [PATCH 10/33] Skip filter by date in aws nodes

---
 .buildkite/configs/cleanup.aws.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.buildkite/configs/cleanup.aws.yml b/.buildkite/configs/cleanup.aws.yml
index 745030aff7..7a7c5ec2ef 100644
--- a/.buildkite/configs/cleanup.aws.yml
+++ b/.buildkite/configs/cleanup.aws.yml
@@ -15,12 +15,12 @@ scanners:
         regions:
           - us-east-1
         filters:
-          - type: "<"
-            pointer: "/created_at"
-            param: "${CREATION_DATE}"
-            converters:
-              param: "date"
-              value: "date"
+          # - type: "<"
+          #   pointer: "/created_at"
+          #   param: "${CREATION_DATE}"
+          #   converters:
+          #     param: "date"
+          #     value: "date"
           - type: "regex"
             pointer: "/extra/tags/repo"
             param: "^(elastic-package|integrations)"

From 68f9591bc02d162c8f3c207766ced00655a2b6e1 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <marrodmo@gmail.com>
Date: Thu, 22 Feb 2024 15:20:44 +0100
Subject: [PATCH 11/33] Remove "value" from converters section

Co-authored-by: Adrien Mannocci <adrien.mannocci@gmail.com>
---
 .buildkite/configs/cleanup.aws.yml | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/.buildkite/configs/cleanup.aws.yml b/.buildkite/configs/cleanup.aws.yml
index 7a7c5ec2ef..71680e64b0 100644
--- a/.buildkite/configs/cleanup.aws.yml
+++ b/.buildkite/configs/cleanup.aws.yml
@@ -15,12 +15,11 @@ scanners:
         regions:
           - us-east-1
         filters:
-          # - type: "<"
-          #   pointer: "/created_at"
-          #   param: "${CREATION_DATE}"
-          #   converters:
-          #     param: "date"
-          #     value: "date"
+          - type: "<"
+            pointer: "/created_at"
+            param: "${CREATION_DATE}"
+            converters:
+              param: "date"
           - type: "regex"
             pointer: "/extra/tags/repo"
             param: "^(elastic-package|integrations)"

From 215fea6f062ee768fbe2144fadc7120128aceb13 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 22 Feb 2024 15:59:06 +0100
Subject: [PATCH 12/33] Add SQS queues

---
 .buildkite/configs/cleanup.aws.yml  | 19 +++++++++++++++++++
 .buildkite/scripts/cloud-cleanup.sh | 19 ++++++++++---------
 2 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/.buildkite/configs/cleanup.aws.yml b/.buildkite/configs/cleanup.aws.yml
index 71680e64b0..08d668f37f 100644
--- a/.buildkite/configs/cleanup.aws.yml
+++ b/.buildkite/configs/cleanup.aws.yml
@@ -51,3 +51,22 @@ scanners:
           - type: "regex"
             pointer: "/name"
             param: "^elastic-package-(.*)"
+      - type: 'queue'
+        regions:
+          - us-east-1
+        filters:
+          - type: "<"
+            pointer: "/extra/tags/created_at"
+            param: "${CREATION_DATE}"
+            converters:
+              param: "date"
+              value: "date_epoch_ms"
+          - type: "regex"
+            pointer: "/extra/tags/repo"
+            param: "^(elastic-package|integrations)"
+          - type: "="
+            pointer: "/extra/tags/environment"
+            param: "ci"
+          - type: "regex"
+            pointer: "/id"
+            param: "^https://(.*)/elastic-package-(.*)"
diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index 4baf2bc97f..49ec17823f 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -5,6 +5,16 @@ set -euo pipefail
 # TODO: change to 24 hours ago
 export CREATION_DATE=$(date -Is -d "1 minute ago")
 
+echo "--- Cleaning up GCP resources..."
+docker run -v $(pwd)/.buildkite/configs/cleanup.gcp.yml:/etc/cloud-reaper/config.yml \
+  -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_GCP_KEY_SECRET}" \
+  -e ACCOUNT_KEY="${ELASTIC_PACKAGE_GCP_EMAIL_SECRET}" \
+  -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}" \
+  -e CREATION_DATE=${CREATION_DATE} \
+  ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
+    --config /etc/cloud-reaper/config.yml \
+    plan
+
 echo "--- Cleaning up AWS resources..."
 docker run -v $(pwd)/.buildkite/configs/cleanup.aws.yml:/etc/cloud-reaper/config.yml \
   -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_AWS_SECRET_KEY}" \
@@ -15,12 +25,3 @@ docker run -v $(pwd)/.buildkite/configs/cleanup.aws.yml:/etc/cloud-reaper/config
     --config /etc/cloud-reaper/config.yml \
     plan
 
-echo "--- Cleaning up GCP resources..."
-docker run -v $(pwd)/.buildkite/configs/cleanup.gcp.yml:/etc/cloud-reaper/config.yml \
-  -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_GCP_KEY_SECRET}" \
-  -e ACCOUNT_KEY="${ELASTIC_PACKAGE_GCP_EMAIL_SECRET}" \
-  -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}" \
-  -e CREATION_DATE=${CREATION_DATE} \
-  ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
-    --config /etc/cloud-reaper/config.yml \
-    plan

From 5322187fc18765d96289886b975270c9fb7a75a1 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 22 Feb 2024 16:05:59 +0100
Subject: [PATCH 13/33] Add state filter for nodes

---
 .buildkite/configs/cleanup.aws.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.buildkite/configs/cleanup.aws.yml b/.buildkite/configs/cleanup.aws.yml
index 08d668f37f..911d6bb620 100644
--- a/.buildkite/configs/cleanup.aws.yml
+++ b/.buildkite/configs/cleanup.aws.yml
@@ -29,6 +29,9 @@ scanners:
           - type: "regex"
             pointer: "/name"
             param: "^elastic-package-(.*)"
+          - type: "!="
+            pointer: "/state"
+            param: "unknown"
           - type: "!="
             pointer: "/state"
             param: "terminated"

From 45107fdd47c6327f282a3cb5151f5d4aed618e4a Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 22 Feb 2024 16:07:07 +0100
Subject: [PATCH 14/33] Add environment filter for nodes gcp

---
 .buildkite/configs/cleanup.gcp.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.buildkite/configs/cleanup.gcp.yml b/.buildkite/configs/cleanup.gcp.yml
index d08cb7b630..1dd01f24be 100644
--- a/.buildkite/configs/cleanup.gcp.yml
+++ b/.buildkite/configs/cleanup.gcp.yml
@@ -25,6 +25,9 @@ scanners:
           - type: "="
             pointer: "/extra/labels/repo"
             param: "elastic-package"
+          - type: "="
+            pointer: "/extra/labels/environment"
+            param: "ci"
           - type: "regex"
             pointer: "/name"
             param: "^elastic-package-(.*)"

From d7d47d4c25a8763fcdf993bc8fddbcfc8e10ad3f Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 22 Feb 2024 16:11:11 +0100
Subject: [PATCH 15/33] Rename pipeline file

---
 ...tic-package-cloud-cleanup.yml => pipeline.cloud-cleanup.yml} | 0
 catalog-info.yaml                                               | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename .buildkite/{pipeline.elastic-package-cloud-cleanup.yml => pipeline.cloud-cleanup.yml} (100%)

diff --git a/.buildkite/pipeline.elastic-package-cloud-cleanup.yml b/.buildkite/pipeline.cloud-cleanup.yml
similarity index 100%
rename from .buildkite/pipeline.elastic-package-cloud-cleanup.yml
rename to .buildkite/pipeline.cloud-cleanup.yml
diff --git a/catalog-info.yaml b/catalog-info.yaml
index f3b6a235a1..201c992691 100644
--- a/catalog-info.yaml
+++ b/catalog-info.yaml
@@ -131,7 +131,7 @@ spec:
       name: elastic-package-cloud-cleanup
       description: Buildkite pipeline for cleaning stale resource in cloud providers
     spec:
-      pipeline_file: ".buildkite/pipeline.elastic-package-cloud-cleanup.yml"
+      pipeline_file: ".buildkite/pipeline.cloud-cleanup.yml"
       provider_settings:
         build_pull_request_forks: false
         build_pull_requests: false # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot

From cc966475e222e0b3c864ecea06775d95c219343f Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 22 Feb 2024 16:13:23 +0100
Subject: [PATCH 16/33] Add email notificaiton - currently disabled

---
 .buildkite/pipeline.cloud-cleanup.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.buildkite/pipeline.cloud-cleanup.yml b/.buildkite/pipeline.cloud-cleanup.yml
index f82442703a..c852986b2b 100644
--- a/.buildkite/pipeline.cloud-cleanup.yml
+++ b/.buildkite/pipeline.cloud-cleanup.yml
@@ -5,6 +5,7 @@ name: elastic-package-cloud-cleanup
 
 env:
   DOCKER_REGISTRY: docker.elastic.co
+  NOTIFY_TO: "ecosystem-team@elastic.co"
 
 steps:
   - label: "Cloud Cleanup"
@@ -12,3 +13,8 @@ steps:
     command: ".buildkite/scripts/cloud-cleanup.sh"
     agents:
       provider: "gcp"
+
+# TODO: enable before merging
+# notify:
+#   - email: "$NOTIFY_TO"
+#     if: "build.state == 'failed' && build.env('BUILDKITE_PULL_REQUEST') == 'false'"

From d548a7b67103fb691d76ce82320bd66af168a038 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 22 Feb 2024 17:11:30 +0100
Subject: [PATCH 17/33] Install awscli

---
 .buildkite/hooks/pre-exit           |  6 ++++++
 .buildkite/scripts/cloud-cleanup.sh | 15 +++++++++++++++
 2 files changed, 21 insertions(+)

diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index 02346d1f0d..2196186008 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -15,3 +15,9 @@ unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
 unset ELASTIC_PACKAGE_AWS_SECRET_KEY
 unset AWS_ACCESS_KEY_ID
 unset AWS_SECRET_ACCESS_KEY
+
+# cloud-cleanup
+unset ELASTIC_PACKAGE_AWS_USER_SECRET
+unset ELASTIC_PACKAGE_GCP_KEY_SECRET
+unset ELASTIC_PACKAGE_GCP_EMAIL_SECRET
+unset ELASTIC_PACKAGE_GCP_PROJECT_SECRET
diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index 49ec17823f..c1b859d91d 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -25,3 +25,18 @@ docker run -v $(pwd)/.buildkite/configs/cleanup.aws.yml:/etc/cloud-reaper/config
     --config /etc/cloud-reaper/config.yml \
     plan
 
+echo "--- Cleaning up other AWS resources"
+echo "--- Installing awscli"
+if ! which aws; then
+  curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+  unzip awscliv2.zip
+  sudo ./aws/install
+  rm -rf awscliv2.zip aws
+  aws --version
+fi
+
+export AWS_ACCESS_KEY_ID="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}"
+export AWS_SECRET_ACCESS_KEY="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}"
+export AWS_DEFAULT_REGION=us-east-1
+
+echo "--- Cleaning up Redshift resources"

From 890a985e5f9c6c8fd2d02ae488b5181625b8046b Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 22 Feb 2024 17:46:47 +0100
Subject: [PATCH 18/33] Set quiet for unzip

---
 .buildkite/scripts/cloud-cleanup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index c1b859d91d..02bf31f797 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -29,7 +29,7 @@ echo "--- Cleaning up other AWS resources"
 echo "--- Installing awscli"
 if ! which aws; then
   curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-  unzip awscliv2.zip
+  unzip -q awscliv2.zip
   sudo ./aws/install
   rm -rf awscliv2.zip aws
   aws --version

From 214a2c719c1d619f00f8d5048e9229dd80ee3948 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 22 Feb 2024 18:02:17 +0100
Subject: [PATCH 19/33] Add validation commands

---
 .buildkite/scripts/cloud-cleanup.sh | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index 02bf31f797..1485d399ff 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -6,6 +6,17 @@ set -euo pipefail
 export CREATION_DATE=$(date -Is -d "1 minute ago")
 
 echo "--- Cleaning up GCP resources..."
+echo "Validating configuration"
+docker run -v $(pwd)/.buildkite/configs/cleanup.gcp.yml:/etc/cloud-reaper/config.yml \
+  -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_GCP_KEY_SECRET}" \
+  -e ACCOUNT_KEY="${ELASTIC_PACKAGE_GCP_EMAIL_SECRET}" \
+  -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}" \
+  -e CREATION_DATE=${CREATION_DATE} \
+  ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
+    --config /etc/cloud-reaper/config.yml \
+    validate
+
+echo "Scanning resources"
 docker run -v $(pwd)/.buildkite/configs/cleanup.gcp.yml:/etc/cloud-reaper/config.yml \
   -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_GCP_KEY_SECRET}" \
   -e ACCOUNT_KEY="${ELASTIC_PACKAGE_GCP_EMAIL_SECRET}" \
@@ -16,6 +27,17 @@ docker run -v $(pwd)/.buildkite/configs/cleanup.gcp.yml:/etc/cloud-reaper/config
     plan
 
 echo "--- Cleaning up AWS resources..."
+echo "Validating configuration"
+docker run -v $(pwd)/.buildkite/configs/cleanup.aws.yml:/etc/cloud-reaper/config.yml \
+  -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_AWS_SECRET_KEY}" \
+  -e ACCOUNT_KEY="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}" \
+  -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_AWS_USER_SECRET}" \
+  -e CREATION_DATE=${CREATION_DATE} \
+  ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
+    --config /etc/cloud-reaper/config.yml \
+    validate
+
+echo "Scanning resources"
 docker run -v $(pwd)/.buildkite/configs/cleanup.aws.yml:/etc/cloud-reaper/config.yml \
   -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_AWS_SECRET_KEY}" \
   -e ACCOUNT_KEY="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}" \
@@ -28,7 +50,7 @@ docker run -v $(pwd)/.buildkite/configs/cleanup.aws.yml:/etc/cloud-reaper/config
 echo "--- Cleaning up other AWS resources"
 echo "--- Installing awscli"
 if ! which aws; then
-  curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+  curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
   unzip -q awscliv2.zip
   sudo ./aws/install
   rm -rf awscliv2.zip aws

From 30e551f3707f3ffb6ca7c675790ef480d603c601 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Fri, 23 Feb 2024 11:00:52 +0100
Subject: [PATCH 20/33] Show errors

---
 .buildkite/scripts/cloud-cleanup.sh | 49 +++++++++++++++++++++++------
 .buildkite/scripts/install_deps.sh  | 14 +++++++++
 .buildkite/scripts/tooling.sh       |  7 +++++
 3 files changed, 61 insertions(+), 9 deletions(-)

diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index 1485d399ff..083cce927d 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -1,10 +1,28 @@
 #!/usr/bin/env bash
 
+source .buildkite/scripts/install_deps.sh
+source .buildkite/scripts/tooling.sh
+
 set -euo pipefail
 
+AWS_RESOURCES_FILE="aws.resources.txt"
+GCP_RESOURCES_FILE="gcp.resources.txt"
+
 # TODO: change to 24 hours ago
 export CREATION_DATE=$(date -Is -d "1 minute ago")
 
+resource_to_delete=0
+
+any_resources_to_delete() {
+    local file=$1
+    local number=0
+    number=$(tail -n 4 "${file}" | wc -l)
+    if [ "${number}" -eq 0 ]; then
+        return 1
+    fi
+    return 0
+}
+
 echo "--- Cleaning up GCP resources..."
 echo "Validating configuration"
 docker run -v $(pwd)/.buildkite/configs/cleanup.gcp.yml:/etc/cloud-reaper/config.yml \
@@ -24,7 +42,11 @@ docker run -v $(pwd)/.buildkite/configs/cleanup.gcp.yml:/etc/cloud-reaper/config
   -e CREATION_DATE=${CREATION_DATE} \
   ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
     --config /etc/cloud-reaper/config.yml \
-    plan
+    plan | tee "${GCP_RESOURCES_FILE}"
+
+if any_resources_to_delete "${GCP_RESOURCES_FILE}"; then
+    resources_to_delete=1
+fi
 
 echo "--- Cleaning up AWS resources..."
 echo "Validating configuration"
@@ -45,17 +67,26 @@ docker run -v $(pwd)/.buildkite/configs/cleanup.aws.yml:/etc/cloud-reaper/config
   -e CREATION_DATE=${CREATION_DATE} \
   ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
     --config /etc/cloud-reaper/config.yml \
-    plan
+    plan | tee "${AWS_RESOURCES_FILE}"
+
+if ! any_resources_to_delete "${AWS_RESOURCES_FILE}" ; then
+    resource_to_delete=1
+fi
+
+if [ "${resource_to_delete}" -eq 1 ]; then
+    message="There are resources to be deleted"
+    if running_on_buildkite ; then
+         buildkite-agent annotate \
+             "There are resources to be deleted" \
+             --style "error"
+    fi
+    echo "There are resources to be deleted"
+    exit 1
+fi
 
 echo "--- Cleaning up other AWS resources"
 echo "--- Installing awscli"
-if ! which aws; then
-  curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-  unzip -q awscliv2.zip
-  sudo ./aws/install
-  rm -rf awscliv2.zip aws
-  aws --version
-fi
+with_aws_cli
 
 export AWS_ACCESS_KEY_ID="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}"
 export AWS_SECRET_ACCESS_KEY="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}"
diff --git a/.buildkite/scripts/install_deps.sh b/.buildkite/scripts/install_deps.sh
index e68c86a10c..52bf90e861 100755
--- a/.buildkite/scripts/install_deps.sh
+++ b/.buildkite/scripts/install_deps.sh
@@ -147,3 +147,17 @@ with_jq() {
     chmod +x "${WORKSPACE}/bin/jq"
     jq --version
 }
+
+with_aws_cli() {
+    check_platform_architecture
+
+    if ! which aws; then
+        curl -s "https://awscli.amazonaws.com/awscli-exe-${platform_type_lowercase}-${arch_type}.zip" -o "awscliv2.zip"
+        unzip -q awscliv2.zip
+        sudo ./aws/install
+        rm -rf awscliv2.zip aws
+        aws --version
+        return
+    fi
+    echo "\"aws\" already installed"
+}
diff --git a/.buildkite/scripts/tooling.sh b/.buildkite/scripts/tooling.sh
index 1eb38337cd..0c18d9502f 100755
--- a/.buildkite/scripts/tooling.sh
+++ b/.buildkite/scripts/tooling.sh
@@ -67,3 +67,10 @@ google_cloud_logout_active_account() {
     unset GOOGLE_APPLICATION_CREDENTIALS
   fi
 }
+
+running_on_buildkite() {
+    if [[ "${BUILDKITE:-"false"}" == "true" ]]; then
+        return 0
+    fi
+    return 1
+}

From 9632fb22f33752bcbe7e8f52c3b81f6ffb4b1975 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Fri, 23 Feb 2024 11:08:40 +0100
Subject: [PATCH 21/33] Fix url to download aws binary

---
 .buildkite/scripts/install_deps.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/install_deps.sh b/.buildkite/scripts/install_deps.sh
index 52bf90e861..c4ed488735 100755
--- a/.buildkite/scripts/install_deps.sh
+++ b/.buildkite/scripts/install_deps.sh
@@ -152,7 +152,7 @@ with_aws_cli() {
     check_platform_architecture
 
     if ! which aws; then
-        curl -s "https://awscli.amazonaws.com/awscli-exe-${platform_type_lowercase}-${arch_type}.zip" -o "awscliv2.zip"
+        curl -s "https://awscli.amazonaws.com/awscli-exe-${platform_type_lowercase}-${hw_type}.zip" -o "awscliv2.zip"
         unzip -q awscliv2.zip
         sudo ./aws/install
         rm -rf awscliv2.zip aws

From 4b92dc022b827a9c7aaad48e13d6fe29b2014f6b Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Fri, 23 Feb 2024 12:37:05 +0100
Subject: [PATCH 22/33] Set variables for period

---
 .buildkite/scripts/cloud-cleanup.sh | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index 083cce927d..c97cc99810 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -1,15 +1,14 @@
 #!/usr/bin/env bash
 
 source .buildkite/scripts/install_deps.sh
-source .buildkite/scripts/tooling.sh
 
 set -euo pipefail
 
 AWS_RESOURCES_FILE="aws.resources.txt"
 GCP_RESOURCES_FILE="gcp.resources.txt"
 
-# TODO: change to 24 hours ago
-export CREATION_DATE=$(date -Is -d "1 minute ago")
+RESOURCE_RETENTION_PERIOD="${RESOURCE_RETENTION_PERIOD:-"24 hours"}"
+export CREATION_DATE=$(date -Is -d "${RESOURCE_RETENTION_PERIOD} ago")
 
 resource_to_delete=0
 
@@ -84,6 +83,7 @@ if [ "${resource_to_delete}" -eq 1 ]; then
     exit 1
 fi
 
+# TODO: List and delete the required resources using aws cli
 echo "--- Cleaning up other AWS resources"
 echo "--- Installing awscli"
 with_aws_cli
@@ -92,4 +92,7 @@ export AWS_ACCESS_KEY_ID="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}"
 export AWS_SECRET_ACCESS_KEY="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}"
 export AWS_DEFAULT_REGION=us-east-1
 
-echo "--- Cleaning up Redshift resources"
+echo "+++ Cleaning up Redshift resources"
+echo "+++ Cleaning up IAM roles"
+echo "+++ Cleaning up IAM policies"
+echo "+++ Cleaning up Schedulers"

From 92879a33ba1b4e71345570ee2097b30e1bd5e698 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Fri, 23 Feb 2024 12:43:01 +0100
Subject: [PATCH 23/33] Refactor functions and rename variable for date

---
 .buildkite/hooks/pre-exit             |  5 --
 .buildkite/pipeline.cloud-cleanup.yml |  3 +
 .buildkite/scripts/cloud-cleanup.sh   | 88 +++++++++++++++------------
 3 files changed, 51 insertions(+), 45 deletions(-)

diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index 2196186008..591b7acfe2 100755
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -16,8 +16,3 @@ unset ELASTIC_PACKAGE_AWS_SECRET_KEY
 unset AWS_ACCESS_KEY_ID
 unset AWS_SECRET_ACCESS_KEY
 
-# cloud-cleanup
-unset ELASTIC_PACKAGE_AWS_USER_SECRET
-unset ELASTIC_PACKAGE_GCP_KEY_SECRET
-unset ELASTIC_PACKAGE_GCP_EMAIL_SECRET
-unset ELASTIC_PACKAGE_GCP_PROJECT_SECRET
diff --git a/.buildkite/pipeline.cloud-cleanup.yml b/.buildkite/pipeline.cloud-cleanup.yml
index c852986b2b..1999917e4e 100644
--- a/.buildkite/pipeline.cloud-cleanup.yml
+++ b/.buildkite/pipeline.cloud-cleanup.yml
@@ -11,6 +11,9 @@ steps:
   - label: "Cloud Cleanup"
     key: "cloud-cleanup"
     command: ".buildkite/scripts/cloud-cleanup.sh"
+    env:
+      # TODO: change to "24 hours"
+      RESOURCE_RETENTION_PERIOD: "1 minute"
     agents:
       provider: "gcp"
 
diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index c97cc99810..eccbf0c6a8 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -8,7 +8,7 @@ AWS_RESOURCES_FILE="aws.resources.txt"
 GCP_RESOURCES_FILE="gcp.resources.txt"
 
 RESOURCE_RETENTION_PERIOD="${RESOURCE_RETENTION_PERIOD:-"24 hours"}"
-export CREATION_DATE=$(date -Is -d "${RESOURCE_RETENTION_PERIOD} ago")
+export DELETE_RESOURCES_BEFORE_DATE=$(date -Is -d "${RESOURCE_RETENTION_PERIOD} ago")
 
 resource_to_delete=0
 
@@ -22,51 +22,59 @@ any_resources_to_delete() {
     return 0
 }
 
+cloud_reaper_aws() {
+    echo "Validating configuration"
+    docker run -v $(pwd)/.buildkite/configs/cleanup.aws.yml:/etc/cloud-reaper/config.yml \
+      -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_AWS_SECRET_KEY}" \
+      -e ACCOUNT_KEY="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}" \
+      -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_AWS_USER_SECRET}" \
+      -e CREATION_DATE="${DELETE_RESOURCES_BEFORE_DATE}" \
+      ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
+        --config /etc/cloud-reaper/config.yml \
+        validate
+
+    echo "Scanning resources"
+    docker run -v $(pwd)/.buildkite/configs/cleanup.aws.yml:/etc/cloud-reaper/config.yml \
+      -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_AWS_SECRET_KEY}" \
+      -e ACCOUNT_KEY="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}" \
+      -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_AWS_USER_SECRET}" \
+      -e CREATION_DATE="${DELETE_RESOURCES_BEFORE_DATE}" \
+      ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
+        --config /etc/cloud-reaper/config.yml \
+        plan | tee "${AWS_RESOURCES_FILE}"
+}
+
+cloud_reaper_gcp() {
+    echo "Validating configuration"
+    docker run -v $(pwd)/.buildkite/configs/cleanup.gcp.yml:/etc/cloud-reaper/config.yml \
+      -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_GCP_KEY_SECRET}" \
+      -e ACCOUNT_KEY="${ELASTIC_PACKAGE_GCP_EMAIL_SECRET}" \
+      -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}" \
+      -e CREATION_DATE="${DELETE_RESOURCES_BEFORE_DATE}" \
+      ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
+        --config /etc/cloud-reaper/config.yml \
+        validate
+
+    echo "Scanning resources"
+    docker run -v $(pwd)/.buildkite/configs/cleanup.gcp.yml:/etc/cloud-reaper/config.yml \
+      -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_GCP_KEY_SECRET}" \
+      -e ACCOUNT_KEY="${ELASTIC_PACKAGE_GCP_EMAIL_SECRET}" \
+      -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}" \
+      -e CREATION_DATE="${DELETE_RESOURCES_BEFORE_DATE}" \
+      ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
+        --config /etc/cloud-reaper/config.yml \
+        plan | tee "${GCP_RESOURCES_FILE}"
+}
+
 echo "--- Cleaning up GCP resources..."
-echo "Validating configuration"
-docker run -v $(pwd)/.buildkite/configs/cleanup.gcp.yml:/etc/cloud-reaper/config.yml \
-  -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_GCP_KEY_SECRET}" \
-  -e ACCOUNT_KEY="${ELASTIC_PACKAGE_GCP_EMAIL_SECRET}" \
-  -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}" \
-  -e CREATION_DATE=${CREATION_DATE} \
-  ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
-    --config /etc/cloud-reaper/config.yml \
-    validate
-
-echo "Scanning resources"
-docker run -v $(pwd)/.buildkite/configs/cleanup.gcp.yml:/etc/cloud-reaper/config.yml \
-  -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_GCP_KEY_SECRET}" \
-  -e ACCOUNT_KEY="${ELASTIC_PACKAGE_GCP_EMAIL_SECRET}" \
-  -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}" \
-  -e CREATION_DATE=${CREATION_DATE} \
-  ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
-    --config /etc/cloud-reaper/config.yml \
-    plan | tee "${GCP_RESOURCES_FILE}"
+cloud_reaper_gcp
 
 if any_resources_to_delete "${GCP_RESOURCES_FILE}"; then
     resources_to_delete=1
 fi
 
 echo "--- Cleaning up AWS resources..."
-echo "Validating configuration"
-docker run -v $(pwd)/.buildkite/configs/cleanup.aws.yml:/etc/cloud-reaper/config.yml \
-  -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_AWS_SECRET_KEY}" \
-  -e ACCOUNT_KEY="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}" \
-  -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_AWS_USER_SECRET}" \
-  -e CREATION_DATE=${CREATION_DATE} \
-  ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
-    --config /etc/cloud-reaper/config.yml \
-    validate
-
-echo "Scanning resources"
-docker run -v $(pwd)/.buildkite/configs/cleanup.aws.yml:/etc/cloud-reaper/config.yml \
-  -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_AWS_SECRET_KEY}" \
-  -e ACCOUNT_KEY="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}" \
-  -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_AWS_USER_SECRET}" \
-  -e CREATION_DATE=${CREATION_DATE} \
-  ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
-    --config /etc/cloud-reaper/config.yml \
-    plan | tee "${AWS_RESOURCES_FILE}"
+cloud_reaper_aws
 
 if ! any_resources_to_delete "${AWS_RESOURCES_FILE}" ; then
     resource_to_delete=1
@@ -92,7 +100,7 @@ export AWS_ACCESS_KEY_ID="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}"
 export AWS_SECRET_ACCESS_KEY="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}"
 export AWS_DEFAULT_REGION=us-east-1
 
-echo "+++ Cleaning up Redshift resources"
+echo "+++ Cleaning up Redshift clusters"
 echo "+++ Cleaning up IAM roles"
 echo "+++ Cleaning up IAM policies"
 echo "+++ Cleaning up Schedulers"

From d06503c63ad92ed7db9a4ecb9b4fbce52761c435 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Fri, 23 Feb 2024 14:16:02 +0100
Subject: [PATCH 24/33] Update collapsed section pending

---
 .buildkite/scripts/cloud-cleanup.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index eccbf0c6a8..312406f729 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -100,7 +100,7 @@ export AWS_ACCESS_KEY_ID="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}"
 export AWS_SECRET_ACCESS_KEY="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}"
 export AWS_DEFAULT_REGION=us-east-1
 
-echo "+++ Cleaning up Redshift clusters"
-echo "+++ Cleaning up IAM roles"
-echo "+++ Cleaning up IAM policies"
-echo "+++ Cleaning up Schedulers"
+echo "--- TODO: Cleaning up Redshift clusters"
+echo "--- TODO: Cleaning up IAM roles"
+echo "--- TODO: Cleaning up IAM policies"
+echo "--- TODO: Cleaning up Schedulers"

From cbba05d7cc13c0c64c5ed9fdbd6119c99c9852f3 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Fri, 23 Feb 2024 14:17:29 +0100
Subject: [PATCH 25/33] Update retention period values

---
 .buildkite/pipeline.cloud-cleanup.yml | 3 +--
 .buildkite/pipeline.yml               | 1 +
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipeline.cloud-cleanup.yml b/.buildkite/pipeline.cloud-cleanup.yml
index 1999917e4e..9a8458305c 100644
--- a/.buildkite/pipeline.cloud-cleanup.yml
+++ b/.buildkite/pipeline.cloud-cleanup.yml
@@ -12,8 +12,7 @@ steps:
     key: "cloud-cleanup"
     command: ".buildkite/scripts/cloud-cleanup.sh"
     env:
-      # TODO: change to "24 hours"
-      RESOURCE_RETENTION_PERIOD: "1 minute"
+      RESOURCE_RETENTION_PERIOD: "24 hours"
     agents:
       provider: "gcp"
 
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 0db926e983..7ee897a400 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -42,6 +42,7 @@ steps:
     command: ".buildkite/scripts/cloud-cleanup.sh"
     env:
       DOCKER_REGISTRY: docker.elastic.co
+      RESOURCE_RETENTION_PERIOD: "1 minute"
     agents:
       provider: "gcp"
   # - wait: ~

From 13ad1c51ec9ecaecaebf6d118d20019728d1fc8a Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Fri, 23 Feb 2024 15:36:08 +0100
Subject: [PATCH 26/33] Add dry-run option

---
 .buildkite/pipeline.cloud-cleanup.yml |  8 ++--
 .buildkite/pipeline.yml               |  2 +
 .buildkite/scripts/cloud-cleanup.sh   | 56 +++++++++++++++++----------
 3 files changed, 41 insertions(+), 25 deletions(-)

diff --git a/.buildkite/pipeline.cloud-cleanup.yml b/.buildkite/pipeline.cloud-cleanup.yml
index 9a8458305c..cb36298db1 100644
--- a/.buildkite/pipeline.cloud-cleanup.yml
+++ b/.buildkite/pipeline.cloud-cleanup.yml
@@ -13,10 +13,10 @@ steps:
     command: ".buildkite/scripts/cloud-cleanup.sh"
     env:
       RESOURCE_RETENTION_PERIOD: "24 hours"
+      DRY_RUN: "true"
     agents:
       provider: "gcp"
 
-# TODO: enable before merging
-# notify:
-#   - email: "$NOTIFY_TO"
-#     if: "build.state == 'failed' && build.env('BUILDKITE_PULL_REQUEST') == 'false'"
+notify:
+  - email: "$NOTIFY_TO"
+    if: "build.state == 'failed' && build.env('BUILDKITE_PULL_REQUEST') == 'false'"
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 7ee897a400..7afc35d3e8 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -43,8 +43,10 @@ steps:
     env:
       DOCKER_REGISTRY: docker.elastic.co
       RESOURCE_RETENTION_PERIOD: "1 minute"
+      DRY_RUN: "true"
     agents:
       provider: "gcp"
+
   # - wait: ~
   #   continue_on_failure: true
 
diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index 312406f729..b133ef2caa 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -10,8 +10,17 @@ GCP_RESOURCES_FILE="gcp.resources.txt"
 RESOURCE_RETENTION_PERIOD="${RESOURCE_RETENTION_PERIOD:-"24 hours"}"
 export DELETE_RESOURCES_BEFORE_DATE=$(date -Is -d "${RESOURCE_RETENTION_PERIOD} ago")
 
+CLOUD_REAPER_IMAGE="${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper"
+
 resource_to_delete=0
 
+COMMAND="validate"
+if [[ "${DRY_RUN}" != "true" ]]; then
+    COMMAND="plan" # TODO: to be changed to "destroy --confirm"
+else
+    COMMAND="plan"
+fi
+
 any_resources_to_delete() {
     local file=$1
     local number=0
@@ -24,56 +33,60 @@ any_resources_to_delete() {
 
 cloud_reaper_aws() {
     echo "Validating configuration"
-    docker run -v $(pwd)/.buildkite/configs/cleanup.aws.yml:/etc/cloud-reaper/config.yml \
+    docker run --rm -v $(pwd)/.buildkite/configs/cleanup.aws.yml:/etc/cloud-reaper/config.yml \
       -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_AWS_SECRET_KEY}" \
       -e ACCOUNT_KEY="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}" \
       -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_AWS_USER_SECRET}" \
       -e CREATION_DATE="${DELETE_RESOURCES_BEFORE_DATE}" \
-      ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
-        --config /etc/cloud-reaper/config.yml \
-        validate
+      "${CLOUD_REAPER_IMAGE}" \
+        cloud-reaper \
+          --config /etc/cloud-reaper/config.yml \
+          validate
 
     echo "Scanning resources"
-    docker run -v $(pwd)/.buildkite/configs/cleanup.aws.yml:/etc/cloud-reaper/config.yml \
+    docker run --rm -v $(pwd)/.buildkite/configs/cleanup.aws.yml:/etc/cloud-reaper/config.yml \
       -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_AWS_SECRET_KEY}" \
       -e ACCOUNT_KEY="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}" \
       -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_AWS_USER_SECRET}" \
       -e CREATION_DATE="${DELETE_RESOURCES_BEFORE_DATE}" \
-      ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
-        --config /etc/cloud-reaper/config.yml \
-        plan | tee "${AWS_RESOURCES_FILE}"
+      "${CLOUD_REAPER_IMAGE}" \
+        cloud-reaper \
+          --config /etc/cloud-reaper/config.yml \
+          ${COMMAND} | tee "${AWS_RESOURCES_FILE}"
 }
 
 cloud_reaper_gcp() {
     echo "Validating configuration"
-    docker run -v $(pwd)/.buildkite/configs/cleanup.gcp.yml:/etc/cloud-reaper/config.yml \
+    docker run --rm -v $(pwd)/.buildkite/configs/cleanup.gcp.yml:/etc/cloud-reaper/config.yml \
       -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_GCP_KEY_SECRET}" \
       -e ACCOUNT_KEY="${ELASTIC_PACKAGE_GCP_EMAIL_SECRET}" \
       -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}" \
       -e CREATION_DATE="${DELETE_RESOURCES_BEFORE_DATE}" \
-      ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
-        --config /etc/cloud-reaper/config.yml \
-        validate
+      "${CLOUD_REAPER_IMAGE}" \
+        cloud-reaper \
+          --config /etc/cloud-reaper/config.yml \
+          validate
 
     echo "Scanning resources"
-    docker run -v $(pwd)/.buildkite/configs/cleanup.gcp.yml:/etc/cloud-reaper/config.yml \
+    docker run --rm -v $(pwd)/.buildkite/configs/cleanup.gcp.yml:/etc/cloud-reaper/config.yml \
       -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_GCP_KEY_SECRET}" \
       -e ACCOUNT_KEY="${ELASTIC_PACKAGE_GCP_EMAIL_SECRET}" \
       -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}" \
       -e CREATION_DATE="${DELETE_RESOURCES_BEFORE_DATE}" \
-      ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper \
-        --config /etc/cloud-reaper/config.yml \
-        plan | tee "${GCP_RESOURCES_FILE}"
+      "${CLOUD_REAPER_IMAGE}" \
+        cloud-reaper \
+          --config /etc/cloud-reaper/config.yml \
+          ${COMMAND} | tee "${GCP_RESOURCES_FILE}"
 }
 
-echo "--- Cleaning up GCP resources..."
+echo "--- Cleaning up GCP resources older than ${DELETE_RESOURCES_BEFORE_DATE}..."
 cloud_reaper_gcp
 
 if any_resources_to_delete "${GCP_RESOURCES_FILE}"; then
     resources_to_delete=1
 fi
 
-echo "--- Cleaning up AWS resources..."
+echo "--- Cleaning up AWS resources older than ${DELETE_RESOURCES_BEFORE_DATE}..."
 cloud_reaper_aws
 
 if ! any_resources_to_delete "${AWS_RESOURCES_FILE}" ; then
@@ -82,17 +95,18 @@ fi
 
 if [ "${resource_to_delete}" -eq 1 ]; then
     message="There are resources to be deleted"
+    echo "${message}"
     if running_on_buildkite ; then
          buildkite-agent annotate \
-             "There are resources to be deleted" \
+             "${message}" \
+             --context "ctx-cloud-reaper-error" \
              --style "error"
     fi
-    echo "There are resources to be deleted"
     exit 1
 fi
 
 # TODO: List and delete the required resources using aws cli
-echo "--- Cleaning up other AWS resources"
+echo "--- Cleaning up other AWS resources older than ${DELETE_RESOURCES_BEFORE_DATE}"
 echo "--- Installing awscli"
 with_aws_cli
 

From 3fd3eb702ef360f2ac5cf03a8c5c21a657cd350f Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Fri, 23 Feb 2024 16:38:36 +0100
Subject: [PATCH 27/33] Fix docker image name

---
 .buildkite/scripts/cloud-cleanup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index b133ef2caa..a5d5b6db1e 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -10,7 +10,7 @@ GCP_RESOURCES_FILE="gcp.resources.txt"
 RESOURCE_RETENTION_PERIOD="${RESOURCE_RETENTION_PERIOD:-"24 hours"}"
 export DELETE_RESOURCES_BEFORE_DATE=$(date -Is -d "${RESOURCE_RETENTION_PERIOD} ago")
 
-CLOUD_REAPER_IMAGE="${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper"
+CLOUD_REAPER_IMAGE="${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0"
 
 resource_to_delete=0
 

From 68c6ff1f04ebf4b3cacc99566289037264bd3dbd Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Fri, 23 Feb 2024 17:00:29 +0100
Subject: [PATCH 28/33] Restore all steps in main pipeline

Pending to remove cloud cleanup step.
---
 .buildkite/pipeline.yml | 95 +++++++++++++++++++++--------------------
 1 file changed, 48 insertions(+), 47 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 7afc35d3e8..270e133122 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -28,15 +28,16 @@ steps:
       cpu: "8"
       memory: "4G"
 
-  # - label: ":go: :windows: Run unit tests"
-  #   key: unit-tests-windows
-  #   command: ".buildkite/scripts/unit_tests_windows.ps1"
-  #   agents:
-  #     provider: "gcp"
-  #     image: "${WINDOWS_AGENT_IMAGE}"
-  #   artifact_paths:
-  #     - "TEST-unit-windows.xml"
+  - label: ":go: :windows: Run unit tests"
+    key: unit-tests-windows
+    command: ".buildkite/scripts/unit_tests_windows.ps1"
+    agents:
+      provider: "gcp"
+      image: "${WINDOWS_AGENT_IMAGE}"
+    artifact_paths:
+      - "TEST-unit-windows.xml"
 
+  # TODO: remove step
   - label: "Cloud Cleanup"
     key: "cloud-cleanup"
     command: ".buildkite/scripts/cloud-cleanup.sh"
@@ -47,48 +48,48 @@ steps:
     agents:
       provider: "gcp"
 
-  # - wait: ~
-  #   continue_on_failure: true
+  - wait: ~
+    continue_on_failure: true
 
-  # - label: ":pipeline: Trigger Integration tests"
-  #   command: ".buildkite/pipeline.trigger.integration.tests.sh | buildkite-agent pipeline upload"
-  #   depends_on:
-  #     - step: check-static
-  #       allow_failure: false
-  #     - step: unit-tests-linux
-  #       allow_failure: false
-  #     - step: unit-tests-windows
-  #       allow_failure: false
+  - label: ":pipeline: Trigger Integration tests"
+    command: ".buildkite/pipeline.trigger.integration.tests.sh | buildkite-agent pipeline upload"
+    depends_on:
+      - step: check-static
+        allow_failure: false
+      - step: unit-tests-linux
+        allow_failure: false
+      - step: unit-tests-windows
+        allow_failure: false
 
-  # - wait: ~
-  #   continue_on_failure: true
+  - wait: ~
+    continue_on_failure: true
 
-  # - label: ":junit: Transform windows paths to linux for Junit plugin"
-  #   commands:
-  #     - buildkite-agent artifact download "*-windows.xml" . --step unit-tests-windows
-  #     - mkdir -p build/test-results
-  #     - for file in $(ls *-windows.xml); do mv $$file build/test-results/; done
-  #   agents:
-  #     image: "${LINUX_AGENT_IMAGE}"
-  #     cpu: "8"
-  #     memory: "4G"
-  #   artifact_paths:
-  #     - "build/test-results/*.xml"
+  - label: ":junit: Transform windows paths to linux for Junit plugin"
+    commands:
+      - buildkite-agent artifact download "*-windows.xml" . --step unit-tests-windows
+      - mkdir -p build/test-results
+      - for file in $(ls *-windows.xml); do mv $$file build/test-results/; done
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+    artifact_paths:
+      - "build/test-results/*.xml"
 
-  # - wait: ~
-  #   continue_on_failure: true
+  - wait: ~
+    continue_on_failure: true
 
-  # - label: ":junit: Junit annotate"
-  #   plugins:
-  #     - junit-annotate#v2.4.1:
-  #         artifacts: "build/test-results/*.xml"
-  #   agents:
-  #     provider: "gcp"  # junit plugin requires docker
+  - label: ":junit: Junit annotate"
+    plugins:
+      - junit-annotate#v2.4.1:
+          artifacts: "build/test-results/*.xml"
+    agents:
+      provider: "gcp"  # junit plugin requires docker
 
-  # - label: ":github: Release"
-  #   key: "release"
-  #   if: |
-  #     build.tag =~ /^v[0-9]+[.][0-9]+[.][0-9]+$$/
-  #   command: ".buildkite/scripts/release.sh"
-  #   agents:
-  #     provider: "gcp"
+  - label: ":github: Release"
+    key: "release"
+    if: |
+      build.tag =~ /^v[0-9]+[.][0-9]+[.][0-9]+$$/
+    command: ".buildkite/scripts/release.sh"
+    agents:
+      provider: "gcp"

From b1026b04f230d7b6e5ac068d06490224906b3094 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 5 Mar 2024 10:21:40 +0100
Subject: [PATCH 29/33] Remove debug leftovers

---
 .buildkite/hooks/pre-command | 21 +--------------------
 .buildkite/pipeline.yml      | 11 -----------
 2 files changed, 1 insertion(+), 31 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 76dc12b68e..e6128c1c75 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -88,7 +88,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-test-with-integrations" &&
     export GITHUB_TOKEN
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-cloud-cleanup" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-cloud-cleanup" && "$BUILDKITE_STEP_KEY" == "cloud-cleanup" ]]; then
     ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
     export ELASTIC_PACKAGE_AWS_SECRET_KEY
     ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
@@ -106,22 +106,3 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-cloud-cleanup" ]]; then
     export ELASTIC_PACKAGE_GCP_EMAIL_SECRET
 fi
 
-## TODO: Remove before merging
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "cloud-cleanup" ]]; then
-    ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
-    export ELASTIC_PACKAGE_AWS_SECRET_KEY
-    ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
-    export ELASTIC_PACKAGE_AWS_ACCESS_KEY
-    ELASTIC_PACKAGE_AWS_USER_SECRET=$(retry 5 vault kv get -field user ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
-    export ELASTIC_PACKAGE_AWS_USER_SECRET
-
-    ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH} | jq -c)
-    export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET
-    ELASTIC_PACKAGE_GCP_KEY_SECRET=$(echo "${ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET}" | jq -r '.private_key' | tr -d '\n')
-    export ELASTIC_PACKAGE_GCP_KEY_SECRET
-    ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectId ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
-    export ELASTIC_PACKAGE_GCP_PROJECT_SECRET
-    ELASTIC_PACKAGE_GCP_EMAIL_SECRET=$(retry 5 vault read -field username ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
-    export ELASTIC_PACKAGE_GCP_EMAIL_SECRET
-
-fi
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 270e133122..42e868699a 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -37,17 +37,6 @@ steps:
     artifact_paths:
       - "TEST-unit-windows.xml"
 
-  # TODO: remove step
-  - label: "Cloud Cleanup"
-    key: "cloud-cleanup"
-    command: ".buildkite/scripts/cloud-cleanup.sh"
-    env:
-      DOCKER_REGISTRY: docker.elastic.co
-      RESOURCE_RETENTION_PERIOD: "1 minute"
-      DRY_RUN: "true"
-    agents:
-      provider: "gcp"
-
   - wait: ~
     continue_on_failure: true
 

From bbe1265d0157dd1d2047a9af0ff5f86f25620347 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 5 Mar 2024 15:50:35 +0100
Subject: [PATCH 30/33] Keep same name for resources to delete var

---
 .buildkite/scripts/cloud-cleanup.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index a5d5b6db1e..1d2c351b68 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -12,7 +12,7 @@ export DELETE_RESOURCES_BEFORE_DATE=$(date -Is -d "${RESOURCE_RETENTION_PERIOD}
 
 CLOUD_REAPER_IMAGE="${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0"
 
-resource_to_delete=0
+resources_to_delete=0
 
 COMMAND="validate"
 if [[ "${DRY_RUN}" != "true" ]]; then
@@ -90,10 +90,10 @@ echo "--- Cleaning up AWS resources older than ${DELETE_RESOURCES_BEFORE_DATE}..
 cloud_reaper_aws
 
 if ! any_resources_to_delete "${AWS_RESOURCES_FILE}" ; then
-    resource_to_delete=1
+    resources_to_delete=1
 fi
 
-if [ "${resource_to_delete}" -eq 1 ]; then
+if [ "${resources_to_delete}" -eq 1 ]; then
     message="There are resources to be deleted"
     echo "${message}"
     if running_on_buildkite ; then

From 688936161976614d5009a60c4c244c477306fa54 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 5 Mar 2024 16:26:31 +0100
Subject: [PATCH 31/33] Fix condition to check stale resources in AWS

---
 .buildkite/scripts/cloud-cleanup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index 1d2c351b68..02fd88742f 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -89,7 +89,7 @@ fi
 echo "--- Cleaning up AWS resources older than ${DELETE_RESOURCES_BEFORE_DATE}..."
 cloud_reaper_aws
 
-if ! any_resources_to_delete "${AWS_RESOURCES_FILE}" ; then
+if any_resources_to_delete "${AWS_RESOURCES_FILE}" ; then
     resources_to_delete=1
 fi
 

From cc0ffaaee93eecf7981ec4e9be360258b98f4b17 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 5 Mar 2024 16:32:49 +0100
Subject: [PATCH 32/33] Add echo messages

---
 .buildkite/scripts/cloud-cleanup.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index 02fd88742f..e2b2206683 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -83,6 +83,7 @@ echo "--- Cleaning up GCP resources older than ${DELETE_RESOURCES_BEFORE_DATE}..
 cloud_reaper_gcp
 
 if any_resources_to_delete "${GCP_RESOURCES_FILE}"; then
+    echo "Pending GCP resources"
     resources_to_delete=1
 fi
 
@@ -90,6 +91,7 @@ echo "--- Cleaning up AWS resources older than ${DELETE_RESOURCES_BEFORE_DATE}..
 cloud_reaper_aws
 
 if any_resources_to_delete "${AWS_RESOURCES_FILE}" ; then
+    echo "Pending AWS resources"
     resources_to_delete=1
 fi
 

From 5138a38cba31322d1d92ebb424bed8d84d09cf3d Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 5 Mar 2024 16:39:07 +0100
Subject: [PATCH 33/33] Just take into account lines starting with line 4

---
 .buildkite/scripts/cloud-cleanup.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index e2b2206683..ab73877323 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -24,7 +24,11 @@ fi
 any_resources_to_delete() {
     local file=$1
     local number=0
-    number=$(tail -n 4 "${file}" | wc -l)
+    # First three lines are like:
+    # ⇒ Loading configuration...
+    # ✓ Succeeded to load configuration
+    # Scanning resources... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% 0:00:00
+    number=$(tail -n +4 "${file}" | wc -l)
     if [ "${number}" -eq 0 ]; then
         return 1
     fi