diff --git a/.buildkite/scripts/aws_cloud_tests.sh b/.buildkite/scripts/aws_cloud_tests.sh deleted file mode 100755 index 275dfa4ee222..000000000000 --- a/.buildkite/scripts/aws_cloud_tests.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - - -REPO_DIR=$(pwd) - -teardown() { - # reset the directory to the root of the project - cd $REPO_DIR - # Teardown resources after using them - echo "~~~ Terraform Cleanup" - tf_cleanup "${MODULE_DIR}" #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK - - echo "~~~ Docker Compose Cleanup" - docker-compose -f .ci/jobs/docker-compose.yml down -v #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK -} - -tf_cleanup() { - DIRECTORY=${1:-.} - - for tfstate in $(find $DIRECTORY -name terraform.tfstate); do - cd $(dirname $tfstate) - terraform init - if ! terraform destroy -auto-approve; then - echo "+++ Failed to Terraform destroy the resources" - fi - cd - - done -} - -trap 'teardown' EXIT - -# Prepare the cloud resources using Terraform -#startCloudTestEnv "${MODULE_DIR}" -echo "~~~ Loading creds" -set +o xtrace -export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY -export AWS_SECRET_ACCESS_KEY=$BEATS_AWS_SECRET_KEY -export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws" -set -o xtrace - -echo "~~~ Run docker-compose services for emulated cloud env" -docker-compose -f .ci/jobs/docker-compose.yml up -d #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK -echo "~~~ Initialize TF cloud resources" -cd "$MODULE_DIR" -export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g') -export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}" -export TF_VAR_CREATED_DATE=$(date +%s) -export TF_VAR_ENVIRONMENT="ci" -export TF_VAR_REPO="${REPO}" -terraform init && terraform apply -auto-approve -cd - - -# Run tests -echo "~~~ Run Cloud Tests for $BEATS_PROJECT_NAME" -cd "${BEATS_PROJECT_NAME}" -mage build test goIntegTest diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/setup_cloud_env.sh old mode 100755 new mode 100644 similarity index 92% rename from .buildkite/scripts/cloud_tests.sh rename to .buildkite/scripts/setup_cloud_env.sh index c8f3737e13e3..3c23d71573f1 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/setup_cloud_env.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash set -euo pipefail - REPO_DIR=$(pwd) teardown() { @@ -31,7 +30,6 @@ tf_cleanup() { trap 'teardown' EXIT # Prepare the cloud resources using Terraform -#startCloudTestEnv "${MODULE_DIR}" echo "~~~ Loading creds" set +o xtrace export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY @@ -50,8 +48,3 @@ export TF_VAR_ENVIRONMENT="ci" export TF_VAR_REPO="${REPO}" terraform init && terraform apply -auto-approve cd - - -# Run tests -echo "~~~ Run Cloud Tests for $BEATS_PROJECT_NAME" -cd "${BEATS_PROJECT_NAME}" -mage build test diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index a16e0b32c189..a658938ed9f9 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -2,7 +2,8 @@ name: "beats-xpack-filebeat" env: - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" @@ -10,7 +11,6 @@ env: IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" @@ -25,7 +25,7 @@ steps: - group: "x-pack/filebeat Mandatory Tests" key: "x-pack-filebeat-mandatory-tests" steps: - - label: ":ubuntu: Unit Tests" + - label: ":ubuntu: x-pack/filebeat Unit Tests" key: "x-pack-filebeat-mandatory-linux-unit-test" command: | cd x-pack/filebeat @@ -42,9 +42,9 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Ubuntu Unit Tests" + context: "x-pack/filebeat: Unit Tests" - - label: ":ubuntu: Go Integration Tests" + - label: ":ubuntu: x-pack/filebeat Go Integration Tests" key: "x-pack-filebeat-mandatory-int-test" command: | cd x-pack/filebeat @@ -63,7 +63,7 @@ steps: - github_commit_status: context: "x-pack/filebeat: Go Integration Tests" - - label: ":ubuntu: Python Integration Tests" + - label: ":ubuntu: x-pack/filebeat Python Integration Tests" key: "x-pack-filebeat-mandatory-python-int-test" command: | cd x-pack/filebeat @@ -82,7 +82,7 @@ steps: - github_commit_status: context: "x-pack/filebeat: Python Integration Tests" - - label: ":windows: Win-2022 Unit Tests" + - label: ":windows: x-pack/filebeat Win-2022 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -103,7 +103,7 @@ steps: - github_commit_status: context: "x-pack/filebeat: Win-2022 Unit Tests" - - label: ":windows: Win-2016 Unit Tests" + - label: ":windows: x-pack/filebeat Win-2016 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -129,7 +129,7 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ steps: - - label: ":windows: Win-2019 Unit Tests" + - label: ":windows: x-pack/filebeat Win-2019 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -150,7 +150,7 @@ steps: - github_commit_status: context: "x-pack/filebeat: Win-2019 Unit Tests" - - label: ":windows: Win 10 Unit Tests" + - label: ":windows: x-pack/filebeat Win 10 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -171,7 +171,7 @@ steps: - github_commit_status: context: "x-pack/filebeat: Win 10 Unit Tests" - - label: ":windows: Win 11 Unit Tests" + - label: ":windows: x-pack/filebeat Win 11 Unit Tests" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -197,7 +197,7 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm|aws).*/ steps: - - label: ":ubuntu: ARM Unit Tests" + - label: ":ubuntu: x-pack/filebeat Linux arm64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ key: "x-pack-filebeat-mandatory-linux-arm-unit-test" command: | @@ -205,16 +205,16 @@ steps: mage build unitTest agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: - "x-pack/filebeat/build/*.xml" - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Ubuntu ARM Unit Tests" + context: "x-pack/filebeat: Linux arm64 Unit Tests" - - label: ":mac: macOS x86_64 Unit Tests" + - label: ":mac: x-pack/filebeat macOS x86_64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/ command: | set -euo pipefail @@ -234,8 +234,9 @@ steps: - github_commit_status: context: "x-pack/filebeat: macOS x86_64 Unit Tests" - - label: ":mac: macOS arm64 Unit Tests" - skip: "https://github.com/elastic/beats/issues/33036" + - label: ":mac: x-pack/filebeat macOS arm64 Unit Tests" + skip: "Skipping due to elastic/beats#33036" + # https://github.com/elastic/beats/issues/33036 if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/ command: | set -euo pipefail @@ -255,7 +256,7 @@ steps: - github_commit_status: context: "x-pack/filebeat: macOS arm64 Unit Tests" - - label: ":ubuntu: Cloud (MODULE) Tests" + - label: ":ubuntu: x-pack/filebeat Cloud (MODULE) Tests" key: "x-pack-filebeat-extended-cloud-test" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | @@ -263,12 +264,14 @@ steps: # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/filebeat + echo "~~~ Running tests with env var MODULE=$$MODULE" - .buildkite/scripts/cloud_tests.sh + source .buildkite/scripts/setup_cloud_env.sh + cd x-pack/filebeat + mage build test env: ASDF_TERRAFORM_VERSION: 1.0.2 AWS_REGION: "eu-central-1" - BEATS_PROJECT_NAME: x-pack/filebeat MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform" REPO: beats agents: @@ -282,26 +285,29 @@ steps: - github_commit_status: context: "x-pack/filebeat: Cloud (MODULE) Tests" - - label: ":ubuntu: Cloud AWS (MODULE) Tests" + - label: ":ubuntu: x-pack/filebeat Cloud AWS (MODULE) Tests" key: "x-pack-filebeat-extended-cloud-test-aws" - skip: "Skip test until fixed https://github.com/elastic/beats/issues/36425" + skip: "Skipping due to elastic/beats#36425" + # https://github.com/elastic/beats/issues/36425 if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/filebeat + echo "~~~ Running tests with env var MODULE=$$MODULE" - .buildkite/scripts/aws_cloud_tests.sh + source .buildkite/scripts/setup_cloud_env.sh + cd x-pack/filebeat + mage build test goIntegTest env: ASDF_TERRAFORM_VERSION: 1.0.2 AWS_REGION: "eu-central-1" - BEATS_PROJECT_NAME: x-pack/filebeat MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform" REPO: beats agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: - "x-pack/filebeat/build/*.xml" @@ -321,7 +327,7 @@ steps: - group: "x-pack/filebeat Packaging" key: "x-pack-filebeat-packaging" steps: - - label: ":linux: Packaging Linux" + - label: ":linux: x-pack/filebeat Packaging Linux" key: "packaging-linux" command: | cd x-pack/filebeat @@ -338,18 +344,18 @@ steps: - github_commit_status: context: "x-pack/filebeat: Packaging Linux" - - label: ":linux: Packaging ARM" + - label: ":linux: x-pack/filebeat Packaging arm64" key: "packaging-arm" command: | cd x-pack/filebeat mage package agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: PLATFORMS: "linux/arm64" PACKAGES: "docker" notify: - github_commit_status: - context: "x-pack/filebeat: Packaging Linux ARM" + context: "x-pack/filebeat: Packaging Linux arm64" diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 4c1c31521f92..230c2c788127 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -2,7 +2,8 @@ name: "beats-xpack-metricbeat" env: - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" @@ -10,7 +11,6 @@ env: IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" @@ -223,7 +223,8 @@ steps: context: "x-pack/metricbeat: MacOS x86_64 Unit Tests" - label: ":mac: MacOS arm64 Unit Tests" - skip: "https://github.com/elastic/beats/issues/33036" + skip: "Skipping due to elastic/beats#33036" + # https://github.com/elastic/beats/issues/33036 if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.**/ command: | set -euo pipefail @@ -244,22 +245,22 @@ steps: - label: ":linux: Cloud (MODULE) Tests" key: "x-pack-metricbeat-extended-cloud-test" - skip: "doesn't belong in a stage in Jenkins, thus skipped" + skip: "Not a stage on Jenkins & elastic/ingest-dev#3170" + # https://github.com/elastic/ingest-dev/issues/3170 if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ - # see link in Jenkins: https://github.com/elastic/beats/blob/ccd7b135df70358f8a02393d9bd8b716428b8048/x-pack/metricbeat/Jenkinsfile.yml#L39 - # additionally skipping due to https://github.com/elastic/ingest-dev/issues/3170 command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/metricbeat - echo "~~~ Running tests with env var MODULE=$$MODULE" - .buildkite/scripts/cloud_tests.sh + echo "~~~ Running tests with env var MODULE=$$MODULE" + source .buildkite/scripts/setup_cloud_env.sh + cd x-pack/metricbeat + mage build test env: ASDF_TERRAFORM_VERSION: 1.0.2 AWS_REGION: "eu-central-1" - BEATS_PROJECT_NAME: x-pack/metricbeat MODULE_DIR: x-pack/metricbeat/module/aws REPO: beats agents: @@ -275,26 +276,27 @@ steps: - label: ":linux: Cloud AWS (MODULE) Tests" key: "x-pack-metricbeat-extended-cloud-test-aws" - skip: "https://github.com/elastic/beats/issues/36425" - # see commented out section in Jenkins: https://github.com/elastic/beats/blob/main/x-pack/metricbeat/Jenkinsfile.yml#L41-L52 - # additionally skipping due to https://github.com/elastic/ingest-dev/issues/3170 + skip: "Skipping due elastic/beats#36425 & elastic/ingest-dev#3170" + # https://github.com/elastic/beats/issues/36425 & https://github.com/elastic/ingest-dev/issues/3170 if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/metricbeat + echo "~~~ Running tests with env var MODULE=$$MODULE" - .buildkite/scripts/cloud_tests.sh + source .buildkite/scripts/setup_cloud_env.sh + cd x-pack/metricbeat + mage build test goIntegTest env: ASDF_TERRAFORM_VERSION: 1.0.2 AWS_REGION: "eu-central-1" - BEATS_PROJECT_NAME: x-pack/metricbeat MODULE_DIR: x-pack/metricbeat/module/aws REPO: beats agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: - "x-pack/metricbeat/build/*.xml" @@ -338,7 +340,7 @@ steps: mage package agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: PLATFORMS: "linux/arm64"