From 02ea29d8cf4078ce30c0acb507ffada149101a9b Mon Sep 17 00:00:00 2001
From: Olga Naydyonock <olga.naidjonoka@elastic.co>
Date: Wed, 1 May 2024 23:04:45 +0300
Subject: [PATCH] Enabling retries for Beats flaky tests (#39174)

* added retries for auditbeat flaky tests

* added retries for filebeat flaky tests

* added retries

* test exitcode

* checged exit status code for retries

* set larger timeout for pytestOpts

* restored timeout for pytests
---
 .buildkite/auditbeat/auditbeat-pipeline.yml   | 33 ++++++++++++++++
 .buildkite/filebeat/filebeat-pipeline.yml     | 33 ++++++++++++++++
 .buildkite/heartbeat/heartbeat-pipeline.yml   | 36 +++++++++++++++++
 .buildkite/libbeat/pipeline.libbeat.yml       | 18 +++++++++
 .buildkite/metricbeat/pipeline.yml            | 33 ++++++++++++++++
 .buildkite/packetbeat/pipeline.packetbeat.yml | 30 ++++++++++++++
 .buildkite/winlogbeat/pipeline.winlogbeat.yml | 18 +++++++++
 .../x-pack/pipeline.xpack.auditbeat.yml       | 30 ++++++++++++++
 .../x-pack/pipeline.xpack.dockerlogbeat.yml   |  6 +++
 .buildkite/x-pack/pipeline.xpack.filebeat.yml | 33 ++++++++++++++++
 .../x-pack/pipeline.xpack.heartbeat.yml       | 29 +++++++++++++-
 .buildkite/x-pack/pipeline.xpack.libbeat.yml  | 27 +++++++++++++
 .../x-pack/pipeline.xpack.metricbeat.yml      | 30 ++++++++++++++
 .../x-pack/pipeline.xpack.osquerybeat.yml     | 27 +++++++++++++
 .../x-pack/pipeline.xpack.packetbeat.yml      | 39 +++++++++++++++++++
 .../x-pack/pipeline.xpack.winlogbeat.yml      | 18 +++++++++
 16 files changed, 439 insertions(+), 1 deletion(-)

diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
index 801768c271e..ed19c7d9164 100644
--- a/.buildkite/auditbeat/auditbeat-pipeline.yml
+++ b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -32,6 +32,9 @@ steps:
         command: |
           cd auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -47,6 +50,9 @@ steps:
         command: |
           cd auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9}"
@@ -62,6 +68,9 @@ steps:
         command: |
           Set-Location -Path auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -79,6 +88,9 @@ steps:
         command: |
           Set-Location -Path auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -97,6 +109,9 @@ steps:
           make -C auditbeat crosscompile
         env:
           GOX_FLAGS: "-arch amd64"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -115,6 +130,9 @@ steps:
           set -euo pipefail
           cd auditbeat
           mage unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
@@ -133,6 +151,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd auditbeat
           mage unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -147,6 +168,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd auditbeat
           mage unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
@@ -164,6 +188,9 @@ steps:
         command: |
           Set-Location -Path auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -180,6 +207,9 @@ steps:
         command: |
           Set-Location -Path auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -196,6 +226,9 @@ steps:
         command: |
           Set-Location -Path auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
index 7eedd9d76fb..053e8dbec41 100644
--- a/.buildkite/filebeat/filebeat-pipeline.yml
+++ b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -30,6 +30,9 @@ steps:
         command: |
           cd filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -45,6 +48,9 @@ steps:
         command: |
           cd filebeat
           mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -60,6 +66,9 @@ steps:
         command: |
           cd filebeat
           mage pythonIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -76,6 +85,9 @@ steps:
         command: |
           Set-Location -Path filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -94,6 +106,9 @@ steps:
         command: |
           Set-Location -Path filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -118,6 +133,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -136,6 +154,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
@@ -152,6 +173,9 @@ steps:
         command: |
           cd filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
@@ -172,6 +196,9 @@ steps:
         command: |
           Set-Location -Path filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -190,6 +217,9 @@ steps:
         command: |
           Set-Location -Path filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -208,6 +238,9 @@ steps:
         command: |
           Set-Location -Path filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
index 8091b2eead1..cadbcec1eca 100644
--- a/.buildkite/heartbeat/heartbeat-pipeline.yml
+++ b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -30,6 +30,9 @@ steps:
         command: |
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -45,6 +48,9 @@ steps:
         command: |
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9}"
@@ -61,6 +67,9 @@ steps:
         command: |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -78,6 +87,9 @@ steps:
         command: |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -94,6 +106,9 @@ steps:
         command:  |
           cd heartbeat
           mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -109,6 +124,9 @@ steps:
         command: |
           cd heartbeat
           mage pythonIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -130,6 +148,9 @@ steps:
         command: |
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
@@ -151,6 +172,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -168,6 +192,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
@@ -188,6 +215,9 @@ steps:
         command:  |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -205,6 +235,9 @@ steps:
         command: |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -222,6 +255,9 @@ steps:
         command: |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml
index 040ad9b1d66..bc77712c330 100644
--- a/.buildkite/libbeat/pipeline.libbeat.yml
+++ b/.buildkite/libbeat/pipeline.libbeat.yml
@@ -21,6 +21,9 @@ steps:
           set -euo pipefail
           cd libbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -38,6 +41,9 @@ steps:
           set -euo pipefail
           cd libbeat
           mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -55,6 +61,9 @@ steps:
           set -euo pipefail
           cd libbeat
           mage pythonIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -72,6 +81,9 @@ steps:
           set -euo pipefail
           cd libbeat
           make crosscompile
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -89,6 +101,9 @@ steps:
           set -euo pipefail
           cd libbeat
           make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' GOTEST_OUTPUT_OPTIONS=' | go-junit-report > libbeat-stress-test.xml' stress-tests
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -108,6 +123,9 @@ steps:
           set -euo pipefail
           cd libbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml
index 1fb6bfcc237..d15212d2ef3 100644
--- a/.buildkite/metricbeat/pipeline.yml
+++ b/.buildkite/metricbeat/pipeline.yml
@@ -32,6 +32,9 @@ steps:
       - label: ":linux: Ubuntu Unit Tests"
         key: "mandatory-linux-unit-test"
         command: "cd metricbeat && mage build unitTest"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -62,6 +65,9 @@ steps:
           echo "~~~ Running tests"
           export KUBECONFIG="$$PWD/kubecfg"
           cd metricbeat && mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -92,6 +98,9 @@ steps:
           echo "~~~ Running tests"
           export KUBECONFIG="$$PWD/kubecfg"
           cd metricbeat && mage pythonIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -106,6 +115,9 @@ steps:
       - label: ":negative_squared_cross_mark: Cross compile"
         key: "mandatory-cross-compile"
         command: "make -C metricbeat crosscompile"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -122,6 +134,9 @@ steps:
           Set-Location -Path metricbeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -140,6 +155,9 @@ steps:
           Set-Location -Path metricbeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -162,6 +180,9 @@ steps:
           Set-Location -Path metricbeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -180,6 +201,9 @@ steps:
           Set-Location -Path metricbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -198,6 +222,9 @@ steps:
           Set-Location -Path metricbeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -221,6 +248,9 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd metricbeat && mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -238,6 +268,9 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd metricbeat && mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml
index c0f5c1e1a73..d510107a89c 100644
--- a/.buildkite/packetbeat/pipeline.packetbeat.yml
+++ b/.buildkite/packetbeat/pipeline.packetbeat.yml
@@ -28,6 +28,9 @@ steps:
         command: |
           cd packetbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -43,6 +46,9 @@ steps:
         command: |
           cd packetbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9_X86_64}"
@@ -58,6 +64,9 @@ steps:
         command: |
           Set-Location -Path packetbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -75,6 +84,9 @@ steps:
         command: |
           Set-Location -Path packetbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -96,6 +108,9 @@ steps:
         command: |
           Set-Location -Path packetbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -114,6 +129,9 @@ steps:
           Set-Location -Path packetbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -132,6 +150,9 @@ steps:
           Set-Location -Path packetbeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -156,6 +177,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd packetbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -174,6 +198,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd packetbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
@@ -188,6 +215,9 @@ steps:
         key: "linux-arm64-unit-tests-extended"
         command: "cd packetbeat && mage build unitTest"
         if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml
index c71858b45b0..ff332791349 100644
--- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml
+++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml
@@ -24,6 +24,9 @@ steps:
       - label: ":ubuntu: Winlogbeat Crossccompile"
         key: "mandatory-cross-compile"
         command: "make -C winlogbeat crosscompile"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -40,6 +43,9 @@ steps:
           Set-Location -Path winlogbeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -58,6 +64,9 @@ steps:
           Set-Location -Path winlogbeat
           mage build unitTest
         key: "mandatory-win-2019-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -76,6 +85,9 @@ steps:
           Set-Location -Path winlogbeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -99,6 +111,9 @@ steps:
           Set-Location -Path winlogbeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -117,6 +132,9 @@ steps:
           Set-Location -Path winlogbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml
index 36fcb9bebd9..80c298c725d 100644
--- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml
@@ -36,6 +36,9 @@ steps:
           echo "~~~ Will run tests with env var MODULE=$$MODULE"
           cd x-pack/auditbeat
           mage update build test
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -52,6 +55,9 @@ steps:
         command: |
           cd x-pack/auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9_X86_64}"
@@ -68,6 +74,9 @@ steps:
           Set-Location -Path x-pack/auditbeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -86,6 +95,9 @@ steps:
           Set-Location -Path x-pack/auditbeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -108,6 +120,9 @@ steps:
           Set-Location -Path x-pack/auditbeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -126,6 +141,9 @@ steps:
           Set-Location -Path x-pack/auditbeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -144,6 +162,9 @@ steps:
           Set-Location -Path x-pack/auditbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -167,6 +188,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -183,6 +207,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
@@ -201,6 +228,9 @@ steps:
         command: |
           cd x-pack/auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml
index 05aee81e4d8..a64f7851913 100644
--- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml
@@ -28,6 +28,9 @@ steps:
       - label: ":ubuntu: Xpack/Dockerlogbeat Ubuntu Unit Tests"
         key: "mandatory-linux-unit-test"
         command: "cd x-pack/dockerlogbeat && mage build unitTest"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -44,6 +47,9 @@ steps:
         command: "cd x-pack/dockerlogbeat && mage goIntegTest"
         env:
           MODULE: $MODULE
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
index 795302bc2d9..b7e71e3c3c0 100644
--- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
@@ -30,6 +30,9 @@ steps:
         command: |
           cd x-pack/filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -50,6 +53,9 @@ steps:
           defineModuleFromTheChangeSet x-pack/filebeat
           echo "~~~ Will run tests with env var MODULE=$$MODULE"
           cd x-pack/filebeat && mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -70,6 +76,9 @@ steps:
           defineModuleFromTheChangeSet x-pack/filebeat
           echo "~~~ Running tests with env var MODULE=$$MODULE"
           cd x-pack/filebeat && mage pythonIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -86,6 +95,9 @@ steps:
           Set-Location -Path x-pack/filebeat
           mage build unitTest
         key: "x-pack-filebeat-mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -104,6 +116,9 @@ steps:
           Set-Location -Path x-pack/filebeat
           mage build unitTest
         key: "x-pack-filebeat-mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -122,6 +137,9 @@ steps:
         command: |
           cd x-pack/filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
@@ -142,6 +160,9 @@ steps:
           Set-Location -Path x-pack/filebeat
           mage build unitTest
         key: "x-pack-filebeat-extended-win-2019-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -160,6 +181,9 @@ steps:
           Set-Location -Path x-pack/filebeat
           mage build unitTest
         key: "x-pack-filebeat-extended-win-10-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -178,6 +202,9 @@ steps:
           Set-Location -Path x-pack/filebeat
           mage build unitTest
         key: "x-pack-filebeat-extended-win-11-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -200,6 +227,9 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/filebeat && mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -217,6 +247,9 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/filebeat && mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml
index 107dfa65f1b..136706e698c 100644
--- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml
@@ -39,6 +39,9 @@ steps:
           echo "~~~ Running tests"
           cd x-pack/heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -59,6 +62,9 @@ steps:
           echo "~~~ Running tests"
           cd x-pack/heartbeat
           mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -76,6 +82,9 @@ steps:
         command: |
           Set-Location -Path x-pack/heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -95,6 +104,9 @@ steps:
         command: |
           Set-Location -Path x-pack/heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -118,6 +130,9 @@ steps:
           Set-Location -Path x-pack/heartbeat
           mage build test
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -136,6 +151,9 @@ steps:
           Set-Location -Path x-pack/heartbeat
           mage build test
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -153,6 +171,9 @@ steps:
         command: |
           Set-Location -Path x-pack/heartbeat
           mage build test
+        retry:
+          automatic:
+           - limit: 3
         key: "extended-win-2019-unit-tests"
         agents:
           provider: "gcp"
@@ -166,7 +187,7 @@ steps:
         notify:
           - github_commit_status:
               context: "x-pack/heartbeat: Windows 2019 Unit Tests"
-  
+
   - group: "x-pack/heartbeat MacOS Extended Tests"
     key: "x-pack-heartbeat-extended-tests-macos"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
@@ -179,6 +200,9 @@ steps:
           installNodeJsDependencies
           cd x-pack/heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -197,6 +221,9 @@ steps:
           installNodeJsDependencies
           cd x-pack/heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml
index 14316a3ecd7..6bf456f6d83 100644
--- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml
@@ -26,6 +26,9 @@ steps:
         command: |
           cd x-pack/libbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -42,6 +45,9 @@ steps:
         command: |
           cd x-pack/libbeat
           mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -58,6 +64,9 @@ steps:
         command: |
           cd x-pack/libbeat
           mage pythonIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -74,6 +83,9 @@ steps:
           Set-Location -Path x-pack/libbeat
           mage -w reader\etw build goUnitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -92,6 +104,9 @@ steps:
           Set-Location -Path x-pack/libbeat
           mage -w reader\etw build goUnitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -114,6 +129,9 @@ steps:
           Set-Location -Path x-pack/libbeat
           mage -w reader\etw build goUnitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -132,6 +150,9 @@ steps:
           Set-Location -Path x-pack/libbeat
           mage -w reader\etw build goUnitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -150,6 +171,9 @@ steps:
           Set-Location -Path x-pack/libbeat
           mage -w reader\etw build goUnitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -172,6 +196,9 @@ steps:
         command: |
           cd x-pack/libbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
index 317b9069c55..4c1c31521f9 100644
--- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
@@ -30,6 +30,9 @@ steps:
         command: |
           cd x-pack/metricbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -50,6 +53,9 @@ steps:
           defineModuleFromTheChangeSet x-pack/metricbeat
           echo "~~~ Will run tests with env var MODULE=$$MODULE"
           cd x-pack/metricbeat && mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -70,6 +76,9 @@ steps:
           defineModuleFromTheChangeSet x-pack/metricbeat
           echo "~~~ Running tests with env var MODULE=$$MODULE"
           cd x-pack/metricbeat && mage pythonIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -86,6 +95,9 @@ steps:
           Set-Location -Path x-pack/metricbeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -104,6 +116,9 @@ steps:
           Set-Location -Path x-pack/metricbeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -126,6 +141,9 @@ steps:
           Set-Location -Path x-pack/metricbeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -144,6 +162,9 @@ steps:
           Set-Location -Path x-pack/metricbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -162,6 +183,9 @@ steps:
           Set-Location -Path x-pack/metricbeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -185,6 +209,9 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/metricbeat && mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -202,6 +229,9 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/metricbeat && mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
diff --git a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml
index 8c9137cb423..c8ecac79735 100644
--- a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml
@@ -30,6 +30,9 @@ steps:
         command: |
           cd x-pack/osquerybeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -46,6 +49,9 @@ steps:
         command: |
           cd x-pack/osquerybeat
           mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -62,6 +68,9 @@ steps:
           Set-Location -Path x-pack/osquerybeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -80,6 +89,9 @@ steps:
           Set-Location -Path x-pack/osquerybeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -102,6 +114,9 @@ steps:
           Set-Location -Path x-pack/osquerybeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -120,6 +135,9 @@ steps:
           Set-Location -Path x-pack/osquerybeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -138,6 +156,9 @@ steps:
           Set-Location -Path x-pack/osquerybeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -160,6 +181,9 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/osquerybeat && mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -175,6 +199,9 @@ steps:
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/osquerybeat && mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml
index 77fdf2af848..1ab71c30d7d 100644
--- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml
@@ -29,6 +29,9 @@ steps:
         command: |
           cd x-pack/packetbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -45,6 +48,9 @@ steps:
         command: |
           cd x-pack/packetbeat
           mage systemTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -61,6 +67,9 @@ steps:
         command: |
           cd x-pack/packetbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9_X86_64}"
@@ -77,6 +86,9 @@ steps:
           Set-Location -Path x-pack/packetbeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -95,6 +107,9 @@ steps:
           Set-Location -Path x-pack/packetbeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -114,6 +129,9 @@ steps:
         command: |
           Set-Location -Path x-pack/packetbeat
           mage systemTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -136,6 +154,9 @@ steps:
           Set-Location -Path x-pack/packetbeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -154,6 +175,9 @@ steps:
           Set-Location -Path x-pack/packetbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -172,6 +196,9 @@ steps:
           Set-Location -Path x-pack/packetbeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -191,6 +218,9 @@ steps:
         command: |
           Set-Location -Path x-pack/packetbeat
           mage systemTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -214,6 +244,9 @@ steps:
           cd x-pack/packetbeat
           mage build unitTest
         if: build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
@@ -236,6 +269,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/packetbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -253,6 +289,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/packetbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml
index c07e537adf0..c6b5a6f59fe 100644
--- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml
@@ -29,6 +29,9 @@ steps:
           mage build unitTest
         env:
           MODULE: $MODULE
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -47,6 +50,9 @@ steps:
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -65,6 +71,9 @@ steps:
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -88,6 +97,9 @@ steps:
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -106,6 +118,9 @@ steps:
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -124,6 +139,9 @@ steps:
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"