From 2d3194c1e3d60dde6facb8c42e7dfee62e73c3c8 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 24 Apr 2024 16:37:34 +0300
Subject: [PATCH] added retries

---
 .buildkite/auditbeat/auditbeat-pipeline.yml   |  4 ++
 .buildkite/heartbeat/heartbeat-pipeline.yml   | 48 +++++++++++++++++
 .buildkite/libbeat/pipeline.libbeat.yml       | 24 +++++++++
 .buildkite/metricbeat/pipeline.yml            | 44 ++++++++++++++++
 .buildkite/packetbeat/pipeline.packetbeat.yml | 40 ++++++++++++++
 .buildkite/winlogbeat/pipeline.winlogbeat.yml | 24 +++++++++
 .../x-pack/pipeline.xpack.auditbeat.yml       | 40 ++++++++++++++
 .../x-pack/pipeline.xpack.dockerlogbeat.yml   |  8 +++
 .buildkite/x-pack/pipeline.xpack.filebeat.yml | 44 ++++++++++++++++
 .../x-pack/pipeline.xpack.heartbeat.yml       | 38 +++++++++++++-
 .buildkite/x-pack/pipeline.xpack.libbeat.yml  | 36 +++++++++++++
 .../x-pack/pipeline.xpack.metricbeat.yml      | 40 ++++++++++++++
 .../x-pack/pipeline.xpack.osquerybeat.yml     | 36 +++++++++++++
 .../x-pack/pipeline.xpack.packetbeat.yml      | 52 +++++++++++++++++++
 .../x-pack/pipeline.xpack.winlogbeat.yml      | 24 +++++++++
 15 files changed, 501 insertions(+), 1 deletion(-)

diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
index 2c5cab90e8af..ef33977c7eff 100644
--- a/.buildkite/auditbeat/auditbeat-pipeline.yml
+++ b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -113,6 +113,10 @@ steps:
           make -C auditbeat crosscompile
         env:
           GOX_FLAGS: "-arch amd64"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
index 8091b2eead17..e3bf05615a49 100644
--- a/.buildkite/heartbeat/heartbeat-pipeline.yml
+++ b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -30,6 +30,10 @@ steps:
         command: |
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -45,6 +49,10 @@ steps:
         command: |
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9}"
@@ -61,6 +69,10 @@ steps:
         command: |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -78,6 +90,10 @@ steps:
         command: |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -94,6 +110,10 @@ steps:
         command:  |
           cd heartbeat
           mage goIntegTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -109,6 +129,10 @@ steps:
         command: |
           cd heartbeat
           mage pythonIntegTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -130,6 +154,10 @@ steps:
         command: |
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
@@ -151,6 +179,10 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -168,6 +200,10 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
@@ -188,6 +224,10 @@ steps:
         command:  |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -205,6 +245,10 @@ steps:
         command: |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -222,6 +266,10 @@ steps:
         command: |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml
index 040ad9b1d669..a15bdb7286c3 100644
--- a/.buildkite/libbeat/pipeline.libbeat.yml
+++ b/.buildkite/libbeat/pipeline.libbeat.yml
@@ -21,6 +21,10 @@ steps:
           set -euo pipefail
           cd libbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -38,6 +42,10 @@ steps:
           set -euo pipefail
           cd libbeat
           mage goIntegTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -55,6 +63,10 @@ steps:
           set -euo pipefail
           cd libbeat
           mage pythonIntegTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -72,6 +84,10 @@ steps:
           set -euo pipefail
           cd libbeat
           make crosscompile
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -89,6 +105,10 @@ steps:
           set -euo pipefail
           cd libbeat
           make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' GOTEST_OUTPUT_OPTIONS=' | go-junit-report > libbeat-stress-test.xml' stress-tests
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -108,6 +128,10 @@ steps:
           set -euo pipefail
           cd libbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml
index 1fb6bfcc2370..945405cd6b31 100644
--- a/.buildkite/metricbeat/pipeline.yml
+++ b/.buildkite/metricbeat/pipeline.yml
@@ -32,6 +32,10 @@ steps:
       - label: ":linux: Ubuntu Unit Tests"
         key: "mandatory-linux-unit-test"
         command: "cd metricbeat && mage build unitTest"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -62,6 +66,10 @@ steps:
           echo "~~~ Running tests"
           export KUBECONFIG="$$PWD/kubecfg"
           cd metricbeat && mage goIntegTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -92,6 +100,10 @@ steps:
           echo "~~~ Running tests"
           export KUBECONFIG="$$PWD/kubecfg"
           cd metricbeat && mage pythonIntegTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -106,6 +118,10 @@ steps:
       - label: ":negative_squared_cross_mark: Cross compile"
         key: "mandatory-cross-compile"
         command: "make -C metricbeat crosscompile"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -122,6 +138,10 @@ steps:
           Set-Location -Path metricbeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -140,6 +160,10 @@ steps:
           Set-Location -Path metricbeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -162,6 +186,10 @@ steps:
           Set-Location -Path metricbeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -180,6 +208,10 @@ steps:
           Set-Location -Path metricbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -198,6 +230,10 @@ steps:
           Set-Location -Path metricbeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -221,6 +257,10 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd metricbeat && mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -238,6 +278,10 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd metricbeat && mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml
index c0f5c1e1a735..cdd28991af96 100644
--- a/.buildkite/packetbeat/pipeline.packetbeat.yml
+++ b/.buildkite/packetbeat/pipeline.packetbeat.yml
@@ -28,6 +28,10 @@ steps:
         command: |
           cd packetbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -43,6 +47,10 @@ steps:
         command: |
           cd packetbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9_X86_64}"
@@ -58,6 +66,10 @@ steps:
         command: |
           Set-Location -Path packetbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -75,6 +87,10 @@ steps:
         command: |
           Set-Location -Path packetbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -96,6 +112,10 @@ steps:
         command: |
           Set-Location -Path packetbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -114,6 +134,10 @@ steps:
           Set-Location -Path packetbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -132,6 +156,10 @@ steps:
           Set-Location -Path packetbeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -156,6 +184,10 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd packetbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -174,6 +206,10 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd packetbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
@@ -188,6 +224,10 @@ steps:
         key: "linux-arm64-unit-tests-extended"
         command: "cd packetbeat && mage build unitTest"
         if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml
index c71858b45b0a..2b59299724fb 100644
--- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml
+++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml
@@ -24,6 +24,10 @@ steps:
       - label: ":ubuntu: Winlogbeat Crossccompile"
         key: "mandatory-cross-compile"
         command: "make -C winlogbeat crosscompile"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -40,6 +44,10 @@ steps:
           Set-Location -Path winlogbeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -58,6 +66,10 @@ steps:
           Set-Location -Path winlogbeat
           mage build unitTest
         key: "mandatory-win-2019-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -76,6 +88,10 @@ steps:
           Set-Location -Path winlogbeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -99,6 +115,10 @@ steps:
           Set-Location -Path winlogbeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -117,6 +137,10 @@ steps:
           Set-Location -Path winlogbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml
index 36fcb9bebd99..bfce65f75797 100644
--- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml
@@ -36,6 +36,10 @@ steps:
           echo "~~~ Will run tests with env var MODULE=$$MODULE"
           cd x-pack/auditbeat
           mage update build test
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -52,6 +56,10 @@ steps:
         command: |
           cd x-pack/auditbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9_X86_64}"
@@ -68,6 +76,10 @@ steps:
           Set-Location -Path x-pack/auditbeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -86,6 +98,10 @@ steps:
           Set-Location -Path x-pack/auditbeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -108,6 +124,10 @@ steps:
           Set-Location -Path x-pack/auditbeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -126,6 +146,10 @@ steps:
           Set-Location -Path x-pack/auditbeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -144,6 +168,10 @@ steps:
           Set-Location -Path x-pack/auditbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -167,6 +195,10 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/auditbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -183,6 +215,10 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/auditbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
@@ -201,6 +237,10 @@ steps:
         command: |
           cd x-pack/auditbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml
index 05aee81e4d80..525b13281f1b 100644
--- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml
@@ -28,6 +28,10 @@ steps:
       - label: ":ubuntu: Xpack/Dockerlogbeat Ubuntu Unit Tests"
         key: "mandatory-linux-unit-test"
         command: "cd x-pack/dockerlogbeat && mage build unitTest"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -44,6 +48,10 @@ steps:
         command: "cd x-pack/dockerlogbeat && mage goIntegTest"
         env:
           MODULE: $MODULE
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
index 795302bc2d99..39c4c8607717 100644
--- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
@@ -30,6 +30,10 @@ steps:
         command: |
           cd x-pack/filebeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -50,6 +54,10 @@ steps:
           defineModuleFromTheChangeSet x-pack/filebeat
           echo "~~~ Will run tests with env var MODULE=$$MODULE"
           cd x-pack/filebeat && mage goIntegTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -70,6 +78,10 @@ steps:
           defineModuleFromTheChangeSet x-pack/filebeat
           echo "~~~ Running tests with env var MODULE=$$MODULE"
           cd x-pack/filebeat && mage pythonIntegTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -86,6 +98,10 @@ steps:
           Set-Location -Path x-pack/filebeat
           mage build unitTest
         key: "x-pack-filebeat-mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -104,6 +120,10 @@ steps:
           Set-Location -Path x-pack/filebeat
           mage build unitTest
         key: "x-pack-filebeat-mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -122,6 +142,10 @@ steps:
         command: |
           cd x-pack/filebeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
@@ -142,6 +166,10 @@ steps:
           Set-Location -Path x-pack/filebeat
           mage build unitTest
         key: "x-pack-filebeat-extended-win-2019-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -160,6 +188,10 @@ steps:
           Set-Location -Path x-pack/filebeat
           mage build unitTest
         key: "x-pack-filebeat-extended-win-10-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -178,6 +210,10 @@ steps:
           Set-Location -Path x-pack/filebeat
           mage build unitTest
         key: "x-pack-filebeat-extended-win-11-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -200,6 +236,10 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/filebeat && mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -217,6 +257,10 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/filebeat && mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml
index 107dfa65f1b2..0065399460b2 100644
--- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml
@@ -39,6 +39,10 @@ steps:
           echo "~~~ Running tests"
           cd x-pack/heartbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -59,6 +63,10 @@ steps:
           echo "~~~ Running tests"
           cd x-pack/heartbeat
           mage goIntegTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -76,6 +84,10 @@ steps:
         command: |
           Set-Location -Path x-pack/heartbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -95,6 +107,10 @@ steps:
         command: |
           Set-Location -Path x-pack/heartbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -118,6 +134,10 @@ steps:
           Set-Location -Path x-pack/heartbeat
           mage build test
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -136,6 +156,10 @@ steps:
           Set-Location -Path x-pack/heartbeat
           mage build test
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -153,6 +177,10 @@ steps:
         command: |
           Set-Location -Path x-pack/heartbeat
           mage build test
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         key: "extended-win-2019-unit-tests"
         agents:
           provider: "gcp"
@@ -166,7 +194,7 @@ steps:
         notify:
           - github_commit_status:
               context: "x-pack/heartbeat: Windows 2019 Unit Tests"
-  
+
   - group: "x-pack/heartbeat MacOS Extended Tests"
     key: "x-pack-heartbeat-extended-tests-macos"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
@@ -179,6 +207,10 @@ steps:
           installNodeJsDependencies
           cd x-pack/heartbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -197,6 +229,10 @@ steps:
           installNodeJsDependencies
           cd x-pack/heartbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml
index 14316a3ecd70..615c27d8fded 100644
--- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml
@@ -26,6 +26,10 @@ steps:
         command: |
           cd x-pack/libbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -42,6 +46,10 @@ steps:
         command: |
           cd x-pack/libbeat
           mage goIntegTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -58,6 +66,10 @@ steps:
         command: |
           cd x-pack/libbeat
           mage pythonIntegTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -74,6 +86,10 @@ steps:
           Set-Location -Path x-pack/libbeat
           mage -w reader\etw build goUnitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -92,6 +108,10 @@ steps:
           Set-Location -Path x-pack/libbeat
           mage -w reader\etw build goUnitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -114,6 +134,10 @@ steps:
           Set-Location -Path x-pack/libbeat
           mage -w reader\etw build goUnitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -132,6 +156,10 @@ steps:
           Set-Location -Path x-pack/libbeat
           mage -w reader\etw build goUnitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -150,6 +178,10 @@ steps:
           Set-Location -Path x-pack/libbeat
           mage -w reader\etw build goUnitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -172,6 +204,10 @@ steps:
         command: |
           cd x-pack/libbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
index 317b9069c556..73974178d1e7 100644
--- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
@@ -30,6 +30,10 @@ steps:
         command: |
           cd x-pack/metricbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -50,6 +54,10 @@ steps:
           defineModuleFromTheChangeSet x-pack/metricbeat
           echo "~~~ Will run tests with env var MODULE=$$MODULE"
           cd x-pack/metricbeat && mage goIntegTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -70,6 +78,10 @@ steps:
           defineModuleFromTheChangeSet x-pack/metricbeat
           echo "~~~ Running tests with env var MODULE=$$MODULE"
           cd x-pack/metricbeat && mage pythonIntegTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -86,6 +98,10 @@ steps:
           Set-Location -Path x-pack/metricbeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -104,6 +120,10 @@ steps:
           Set-Location -Path x-pack/metricbeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -126,6 +146,10 @@ steps:
           Set-Location -Path x-pack/metricbeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -144,6 +168,10 @@ steps:
           Set-Location -Path x-pack/metricbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -162,6 +190,10 @@ steps:
           Set-Location -Path x-pack/metricbeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -185,6 +217,10 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/metricbeat && mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -202,6 +238,10 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/metricbeat && mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
diff --git a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml
index 8c9137cb423e..a564951b3017 100644
--- a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml
@@ -30,6 +30,10 @@ steps:
         command: |
           cd x-pack/osquerybeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -46,6 +50,10 @@ steps:
         command: |
           cd x-pack/osquerybeat
           mage goIntegTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -62,6 +70,10 @@ steps:
           Set-Location -Path x-pack/osquerybeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -80,6 +92,10 @@ steps:
           Set-Location -Path x-pack/osquerybeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -102,6 +118,10 @@ steps:
           Set-Location -Path x-pack/osquerybeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -120,6 +140,10 @@ steps:
           Set-Location -Path x-pack/osquerybeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -138,6 +162,10 @@ steps:
           Set-Location -Path x-pack/osquerybeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -160,6 +188,10 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/osquerybeat && mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -175,6 +207,10 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/osquerybeat && mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml
index 77fdf2af8483..15254ff218e3 100644
--- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml
@@ -29,6 +29,10 @@ steps:
         command: |
           cd x-pack/packetbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -45,6 +49,10 @@ steps:
         command: |
           cd x-pack/packetbeat
           mage systemTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -61,6 +69,10 @@ steps:
         command: |
           cd x-pack/packetbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9_X86_64}"
@@ -77,6 +89,10 @@ steps:
           Set-Location -Path x-pack/packetbeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -95,6 +111,10 @@ steps:
           Set-Location -Path x-pack/packetbeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -114,6 +134,10 @@ steps:
         command: |
           Set-Location -Path x-pack/packetbeat
           mage systemTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -136,6 +160,10 @@ steps:
           Set-Location -Path x-pack/packetbeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -154,6 +182,10 @@ steps:
           Set-Location -Path x-pack/packetbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -172,6 +204,10 @@ steps:
           Set-Location -Path x-pack/packetbeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -191,6 +227,10 @@ steps:
         command: |
           Set-Location -Path x-pack/packetbeat
           mage systemTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -214,6 +254,10 @@ steps:
           cd x-pack/packetbeat
           mage build unitTest
         if: build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
@@ -236,6 +280,10 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/packetbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -253,6 +301,10 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/packetbeat
           mage build unitTest
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml
index c07e537adf09..76c7ad30d87a 100644
--- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml
@@ -29,6 +29,10 @@ steps:
           mage build unitTest
         env:
           MODULE: $MODULE
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -47,6 +51,10 @@ steps:
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -65,6 +73,10 @@ steps:
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -88,6 +100,10 @@ steps:
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -106,6 +122,10 @@ steps:
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -124,6 +144,10 @@ steps:
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"