Remove hardcoded module definitions in CI (#39506)

For reasons described in [^1], we hard coded `aws` or `kubernetes` for the env var `MODULE`, when no file changes were detected under `modules/` per project.

This commit reverts this and allows all module tests to run if no changesets apply. As discussed in [^1], for the tests to run properly (and not go into an infinite loop)
if there are no module changes the `MODULE` env var should **not** be set, rather than set to an empty string.

This commit is an important fix as it covers a major delta in test coverage between Jenkins and Buildkite.

Closes elastic/ingest-dev#2993

[^1]: elastic/ingest-dev#2993 (comment)

.buildkite/metricbeat/pipeline.yml

@@ -57,7 +57,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet metricbeat
-          echo "~~~ Will run tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
 
           # TODO move this section to base image / pre-command hook
           echo "~~~ Installing kind"
@@ -90,7 +90,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet metricbeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
 
           # TODO move this section to base image / pre-command hook
           echo "~~~ Installing kind"

.buildkite/scripts/changesets.psm1

@@ -56,17 +56,9 @@ function DefineModuleFromTheChangeSet($projectPath) {
         }
     }
 
-    # TODO: remove this conditional when issue https://github.com/elastic/ingest-dev/issues/2993 gets resolved
-    if(!$changedModules) {
-        if($Env:BUILDKITE_PIPELINE_SLUG -eq 'beats-xpack-metricbeat') {
-            $Env:MODULE = "aws"
-        }
-        else {
-            $Env:MODULE = "kubernetes"
-        }
-    }
-    else {
-        # TODO: once https://github.com/elastic/ingest-dev/issues/2993 gets resolved, this should be the only thing we export
-        $Env:MODULE = $changedModules
+    if ($changedModules) {
+        $env:MODULE = $changedModules
+        Write-Output "~~~ Set env var MODULE to [$env:MODULE]"
+        Write-Output "~~~ Resuming commands"
     }
 }

.buildkite/scripts/changesets.sh

@@ -68,13 +68,10 @@ defineModuleFromTheChangeSet() {
     fi
   done
 
-  if [[ -z "$changed_modules" ]]; then # TODO: remove this conditional when issue https://github.com/elastic/ingest-dev/issues/2993 gets resolved
-    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
-      export MODULE="aws"
-    else
-      export MODULE="kubernetes"
-    fi
-  else
-    export MODULE="${changed_modules}"  # TODO: once https://github.com/elastic/ingest-dev/issues/2993 gets resolved, this should be the only thing we export
+  # export MODULE="" leads to an infinite loop https://github.com/elastic/ingest-dev/issues/2993
+  if [[ ! -z $changed_modules ]]; then 
+    export MODULE="${changed_modules}"
+    echo "~~~ Set env var MODULE to [$MODULE]"
+    echo "~~~ Resuming commands"
   fi
 }

.buildkite/x-pack/pipeline.xpack.auditbeat.yml

@@ -37,7 +37,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/auditbeat
-          echo "~~~ Will run tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
           cd x-pack/auditbeat
           mage update build test
         retry:

.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml

@@ -56,11 +56,9 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/dockerlogbeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
           cd x-pack/dockerlogbeat
           mage goIntegTest
-        env:
-          MODULE: $MODULE
         retry:
           automatic:
             - limit: 3

.buildkite/x-pack/pipeline.xpack.filebeat.yml

@@ -55,7 +55,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/filebeat
-          echo "~~~ Will run tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
           cd x-pack/filebeat && mage goIntegTest
         retry:
           automatic:
@@ -78,7 +78,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/filebeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
           cd x-pack/filebeat && mage pythonIntegTest
         retry:
           automatic:
@@ -272,7 +272,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/filebeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
 
           .buildkite/scripts/cloud_tests.sh
         env:
@@ -301,7 +301,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/filebeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
           .buildkite/scripts/cloud_tests.sh
         env:
           ASDF_TERRAFORM_VERSION: 1.0.2

.buildkite/x-pack/pipeline.xpack.metricbeat.yml

@@ -55,7 +55,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/metricbeat
-          echo "~~~ Will run tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
           cd x-pack/metricbeat && mage goIntegTest
         retry:
           automatic:
@@ -78,7 +78,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/metricbeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
           cd x-pack/metricbeat && mage pythonIntegTest
         retry:
           automatic:
@@ -257,7 +257,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/metricbeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
 
           .buildkite/scripts/cloud_tests.sh
         env:
@@ -288,7 +288,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/metricbeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
           .buildkite/scripts/cloud_tests.sh
         env:
           ASDF_TERRAFORM_VERSION: 1.0.2

.buildkite/x-pack/pipeline.xpack.winlogbeat.yml

@@ -27,11 +27,9 @@ steps:
         command: |
           Import-Module ./.buildkite/scripts/changesets.psm1
           defineModuleFromTheChangeSet 'x-pack/winlogbeat'
-          Write-Output "~~~ Will run tests with env var MODULE=$$Env:MODULE"
+          Write-Output "~~~ Running tests"
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
-        env:
-          MODULE: $MODULE
         retry:
           automatic:
             - limit: 3