Create different agent per each test execution

.buildkite/hooks/pre-command

@@ -46,13 +46,33 @@ export CREATED_DATE
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && ("$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives) ]]; then
+is_step_required_to_upload_safe_logs() {
+    if [[ "$BUILDKITE_PIPELINE_SLUG" != "elastic-package" ]]; then
+        return 1
+    fi
+    if [[ "$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives ]]; then
+        return 0
+    fi
+    return 1
+}
+
+if is_step_required_to_upload_safe_logs; then
     PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH} | jq -c)
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET
     export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp" ]]; then
+is_step_testing_gcp () {
+    if [[ "$BUILDKITE_PIPELINE_SLUG" != "elastic-package" ]]; then
+        return 1
+    fi
+    if [[ "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp-agent-false" || "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp-agent-true" ]]; then
+        return 0
+    fi
+    return 1
+}
+
+if is_step_testing_gcp; then
     ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectId ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
     export ELASTIC_PACKAGE_GCP_PROJECT_SECRET
     ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH} | jq -c)
@@ -63,7 +83,20 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" =
     export GCP_PROJECT_ID=${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && ("$BUILDKITE_STEP_KEY" == "integration-parallel-aws" || "$BUILDKITE_STEP_KEY" == "integration-parallel-aws_logs") ]]; then
+is_step_testing_aws () {
+    if [[ "$BUILDKITE_PIPELINE_SLUG" != "elastic-package" ]]; then
+        return 1
+    fi
+    if [[ "$BUILDKITE_STEP_KEY" == "integration-parallel-aws-agent-false" || "$BUILDKITE_STEP_KEY" == "integration-parallel-aws-agent-true" ]]; then
+        return 0
+    fi
+    if [[ "$BUILDKITE_STEP_KEY" == "integration-parallel-aws_logs-agent-false" || "$BUILDKITE_STEP_KEY" == "integration-parallel-aws_logs-agent-true" ]]; then
+        return 0
+    fi
+    return 1
+}
+
+if is_step_testing_aws; then
     ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
     export ELASTIC_PACKAGE_AWS_SECRET_KEY
     ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})

.buildkite/pipeline.trigger.integration.tests.sh

@@ -3,7 +3,6 @@
 # exit immediately on failure, or if an undefined variable is used
 set -eu
 
-
 # begin the pipeline.yml file
 echo "steps:"
 echo "  - group: \":terminal: Integration test suite\""
@@ -21,7 +20,7 @@ STACK_COMMAND_TESTS=(
 )
 
 for test in "${STACK_COMMAND_TESTS[@]}"; do
-    echo "      - label: \":go: Running integration test: ${test}\""
+    echo "      - label: \":go: Integration test: ${test}\""
     echo "        command: ./.buildkite/scripts/integration_tests.sh -t ${test}"
     echo "        agents:"
     echo "          provider: \"gcp\""
@@ -38,11 +37,14 @@ CHECK_PACKAGES_TESTS=(
     test-check-packages-benchmarks
     test-check-packages-with-logstash
 )
+for independent_agent in false true ; do
 for test in "${CHECK_PACKAGES_TESTS[@]}"; do
-    echo "      - label: \":go: Running integration test: ${test}\""
+    echo "      - label: \":go: Integration test: ${test} - independent_agent ${independent_agent}\""
     echo "        command: ./.buildkite/scripts/integration_tests.sh -t ${test}"
     echo "        agents:"
     echo "          provider: \"gcp\""
+    echo "        env:"
+    echo "          ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: ${independent_agent}"
     echo "        artifact_paths:"
     echo "          - build/test-results/*.xml"
     echo "          - build/elastic-stack-dump/check-*/logs/*.log"
@@ -52,11 +54,12 @@ for test in "${CHECK_PACKAGES_TESTS[@]}"; do
         echo "          - build/kubectl-dump.txt"
     fi
 done
+done
 
 pushd test/packages/false_positives > /dev/null
 for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     package_name=$(basename "${package}")
-    echo "      - label: \":go: Running integration test (false positive): ${package_name}\""
+    echo "      - label: \":go: Integration test (false positive): ${package_name}\""
     echo "        key: \"integration-false_positives-${package_name}\""
     echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-false-positives -p ${package_name}"
     echo "        env:"
@@ -71,50 +74,57 @@ done
  popd > /dev/null
 
 pushd test/packages/parallel > /dev/null
+for independent_agent in false true; do
 for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     package_name=$(basename "${package}")
-    echo "      - label: \":go: Running integration test: ${package_name}\""
-    echo "        key: \"integration-parallel-${package_name}\""
+    echo "      - label: \":go: Integration test: ${package_name} - independent_agent ${independent_agent}\""
+    echo "        key: \"integration-parallel-${package_name}-agent-${independent_agent}\""
     echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}"
     echo "        env:"
     echo "          UPLOAD_SAFE_LOGS: 1"
+    echo "          ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: ${independent_agent}"
     echo "        agents:"
     echo "          provider: \"gcp\""
     echo "        artifact_paths:"
     echo "          - build/test-results/*.xml"
     echo "          - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package
 done
+done
 
 popd > /dev/null
 
-echo "      - label: \":go: Running integration test: test-build-zip\""
+echo "      - label: \":go: Integration test: test-build-zip\""
 echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-build-zip"
 echo "        agents:"
 echo "          provider: \"gcp\""
 echo "        artifact_paths:"
 echo "          - build/elastic-stack-dump/build-zip/logs/*.log"
 echo "          - build/packages/*.sig"
 
-echo "      - label: \":go: Running integration test: test-install-zip\""
+echo "      - label: \":go: Integration test: test-install-zip\""
 echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-install-zip"
 echo "        agents:"
 echo "          provider: \"gcp\""
 echo "        artifact_paths:"
 echo "          - build/elastic-stack-dump/install-zip/logs/*.log"
 
-echo "      - label: \":go: Running integration test: test-install-zip-shellinit\""
+echo "      - label: \":go: Integration test: test-install-zip-shellinit\""
 echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-install-zip-shellinit"
 echo "        agents:"
 echo "          provider: \"gcp\""
 echo "        artifact_paths:"
 echo "          - build/elastic-stack-dump/install-zip-shellinit/logs/*.log"
 
-echo "      - label: \":go: Running integration test: test-system-test-flags\""
+for independent_agent in false true; do
+echo "      - label: \":go: Integration test: test-system-test-flags - independent_agent ${independent_agent}\""
 echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-system-test-flags"
 echo "        agents:"
 echo "          provider: \"gcp\""
+echo "        env:"
+echo "          ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: ${independent_agent}"
+done
 
-echo "      - label: \":go: Running integration test: test-profiles-command\""
+echo "      - label: \":go: Integration test: test-profiles-command\""
 echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-profiles-command"
 echo "        env:"
 echo "          DOCKER_COMPOSE_VERSION: \"false\""
@@ -124,7 +134,7 @@ echo "          image: \"${LINUX_AGENT_IMAGE}\""
 echo "          cpu: \"8\""
 echo "          memory: \"4G\""
 
-echo "      - label: \":go: Running integration test: test-check-update-version\""
+echo "      - label: \":go: Integration test: test-check-update-version\""
 echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-update-version"
 echo "        env:"
 echo "          DEFAULT_VERSION_TAG: v0.80.0"

.buildkite/scripts/integration_tests.sh

@@ -41,6 +41,7 @@ KIND_TARGET="test-check-packages-with-kind"
 SYSTEM_TEST_FLAGS_TARGET="test-system-test-flags"
 TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
 GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
+ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT=${ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT:-"false"}
 
 REPO_NAME=$(repo_name "${BUILDKITE_REPO}")
 REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
@@ -132,21 +133,25 @@ if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSIT
     set -e
 
     if [[ "${UPLOAD_SAFE_LOGS}" -eq 1 ]] ; then
+        package_folder="${PACKAGE}"
+        if [[ "${ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT}" != "false" ]]; then
+            package_folder="${package_folder}-independent_agent"
+        fi
         upload_safe_logs \
             "${JOB_GCS_BUCKET_INTERNAL}" \
             "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/*.*" \
-            "insecure-logs/${PACKAGE}/elastic-agent-logs/"
+            "insecure-logs/${package_folder}/elastic-agent-logs/"
 
         # required for <8.6.0
         upload_safe_logs \
             "${JOB_GCS_BUCKET_INTERNAL}" \
             "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/default/*" \
-            "insecure-logs/${PACKAGE}/elastic-agent-logs/default/"
+            "insecure-logs/${package_folder}/elastic-agent-logs/default/"
 
         upload_safe_logs \
             "${JOB_GCS_BUCKET_INTERNAL}" \
             "build/container-logs/*.log" \
-            "insecure-logs/${PACKAGE}/container-logs/"
+            "insecure-logs/${package_folder}/container-logs/"
     fi
 
     if [ $testReturnCode != 0 ]; then

cmd/testrunner.go

@@ -18,6 +18,7 @@ import (
 	"github.com/elastic/elastic-package/internal/cobraext"
 	"github.com/elastic/elastic-package/internal/common"
 	"github.com/elastic/elastic-package/internal/elasticsearch"
+	"github.com/elastic/elastic-package/internal/environment"
 	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/kibana"
 	"github.com/elastic/elastic-package/internal/logger"
@@ -52,6 +53,8 @@ These tests allow you to test a package's ability to ingest data end-to-end.
 
 For details on how to configure amd run system tests, review the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/system_testing.md).`
 
+var enableIndependentAgents = environment.WithElasticPackagePrefix("TEST_ENABLE_INDEPENDENT_AGENT")
+
 func setupTestCommand() *cobraext.Command {
 	var testTypeCmdActions []cobraext.CommandAction
 
@@ -186,6 +189,12 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command
 			return fmt.Errorf("cannot determine if package has data streams: %w", err)
 		}
 
+		runIndependentElasticAgent := false
+		v, ok := os.LookupEnv(enableIndependentAgents)
+		if ok && strings.ToLower(v) != "false" {
+			runIndependentElasticAgent = true
+		}
+
 		configFileFlag := ""
 		runSetup := false
 		runTearDown := false
@@ -340,20 +349,21 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command
 		var results []testrunner.TestResult
 		for _, folder := range testFolders {
 			r, err := testrunner.Run(ctx, testType, testrunner.TestOptions{
-				Profile:            profile,
-				TestFolder:         folder,
-				PackageRootPath:    packageRootPath,
-				GenerateTestResult: generateTestResult,
-				API:                esAPI,
-				KibanaClient:       kibanaClient,
-				DeferCleanup:       deferCleanup,
-				ServiceVariant:     variantFlag,
-				WithCoverage:       testCoverage,
-				CoverageType:       testCoverageFormat,
-				ConfigFilePath:     configFileFlag,
-				RunSetup:           runSetup,
-				RunTearDown:        runTearDown,
-				RunTestsOnly:       runTestsOnly,
+				Profile:                    profile,
+				TestFolder:                 folder,
+				PackageRootPath:            packageRootPath,
+				GenerateTestResult:         generateTestResult,
+				API:                        esAPI,
+				KibanaClient:               kibanaClient,
+				DeferCleanup:               deferCleanup,
+				ServiceVariant:             variantFlag,
+				WithCoverage:               testCoverage,
+				CoverageType:               testCoverageFormat,
+				ConfigFilePath:             configFileFlag,
+				RunSetup:                   runSetup,
+				RunTearDown:                runTearDown,
+				RunTestsOnly:               runTestsOnly,
+				RunIndependentElasticAgent: runIndependentElasticAgent,
 			})
 
 			results = append(results, r...)

internal/agentdeployer/_static/docker-agent-base.yml

@@ -0,0 +1,18 @@
+version: "2.3"
+services:
+  elastic-agent:
+    hostname: ${AGENT_HOSTNAME}
+    image: "${ELASTIC_AGENT_IMAGE_REF}"
+    healthcheck:
+      test: "elastic-agent status"
+      retries: 180
+      interval: 1s
+    environment:
+      - FLEET_ENROLL=1
+      - FLEET_URL=https://fleet-server:8220
+      - KIBANA_HOST=https://kibana:5601
+      - FLEET_TOKEN_POLICY_NAME=${FLEET_TOKEN_POLICY_NAME}
+    volumes:
+      - ${SERVICE_LOGS_DIR}:/tmp/service_logs/
+      - ${LOCAL_CA_CERT}:/etc/ssl/certs/elastic-package.pem
+