Skip to content

Commit

Permalink
Update xpack/filebeat & xpack/metricbeat pipelines to match Jenkins s…
Browse files Browse the repository at this point in the history
…teps (#39211)

Updated x-pack/filebeat and x-pack/metricbeat pipelines to match same steps & commands as in Jenkins
  • Loading branch information
oakrizan committed May 17, 2024
1 parent 6979849 commit bf13818
Show file tree
Hide file tree
Showing 3 changed files with 84 additions and 93 deletions.
7 changes: 0 additions & 7 deletions .buildkite/scripts/cloud_tests.sh → .buildkite/scripts/setup_cloud_env.sh
100755 → 100644
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
#!/usr/bin/env bash
set -euo pipefail


REPO_DIR=$(pwd)

teardown() {
Expand Down Expand Up @@ -31,7 +30,6 @@ tf_cleanup() {
trap 'teardown' EXIT

# Prepare the cloud resources using Terraform
#startCloudTestEnv "${MODULE_DIR}"
echo "~~~ Loading creds"
set +o xtrace
export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY
Expand All @@ -50,8 +48,3 @@ export TF_VAR_ENVIRONMENT="ci"
export TF_VAR_REPO="${REPO}"
terraform init && terraform apply -auto-approve
cd -

# Run tests
echo "~~~ Run Cloud Tests for $BEATS_PROJECT_NAME"
cd "${BEATS_PROJECT_NAME}"
mage build test
137 changes: 67 additions & 70 deletions .buildkite/x-pack/pipeline.xpack.filebeat.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@
name: "beats-xpack-filebeat"

env:
AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"

GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
GCP_WIN_MACHINE_TYPE: "n2-standard-8"

IMAGE_MACOS_ARM: "generic-13-ventura-arm"
IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
Expand All @@ -29,7 +29,7 @@ steps:
- group: "x-pack/filebeat Mandatory Tests"
key: "x-pack-filebeat-mandatory-tests"
steps:
- label: ":linux: Ubuntu Unit Tests"
- label: ":ubuntu: x-pack/filebeat Unit Tests"
key: "x-pack-filebeat-mandatory-linux-unit-test"
command: |
cd x-pack/filebeat
Expand All @@ -46,17 +46,13 @@ steps:
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: Ubuntu Unit Tests"
context: "x-pack/filebeat: Unit Tests"

- label: ":go: Go (MODULE) Integration Tests"
- label: ":ubuntu: x-pack/filebeat Go Integration Tests"
key: "x-pack-filebeat-mandatory-int-test"
command: |
set -euo pipefail
# defines the MODULE env var based on what's changed in a PR
source .buildkite/scripts/changesets.sh
defineModuleFromTheChangeSet x-pack/filebeat
echo "~~~ Running tests"
cd x-pack/filebeat && mage goIntegTest
cd x-pack/filebeat
mage goIntegTest
retry:
automatic:
- limit: 3
Expand All @@ -69,17 +65,13 @@ steps:
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: Go (MODULE) Integration Tests"
context: "x-pack/filebeat: Go Integration Tests"

- label: ":python: Python (MODULE) Integration Tests"
- label: ":ubuntu: x-pack/filebeat Python Integration Tests"
key: "x-pack-filebeat-mandatory-python-int-test"
command: |
set -euo pipefail
# defines the MODULE env var based on what's changed in a PR
source .buildkite/scripts/changesets.sh
defineModuleFromTheChangeSet x-pack/filebeat
echo "~~~ Running tests"
cd x-pack/filebeat && mage pythonIntegTest
cd x-pack/filebeat
mage pythonIntegTest
retry:
automatic:
- limit: 3
Expand All @@ -92,9 +84,9 @@ steps:
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: Python (MODULE) Integration Tests"
context: "x-pack/filebeat: Python Integration Tests"

- label: ":windows: Windows 2022 Unit Tests"
- label: ":windows: x-pack/filebeat Win-2022 Unit Tests"
command: |
Set-Location -Path x-pack/filebeat
mage build unitTest
Expand All @@ -113,9 +105,9 @@ steps:
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: Windows 2022 Unit Tests"
context: "x-pack/filebeat: Win-2022 Unit Tests"

- label: ":windows: Windows 2016 Unit Tests"
- label: ":windows: x-pack/filebeat Win-2016 Unit Tests"
command: |
Set-Location -Path x-pack/filebeat
mage build unitTest
Expand All @@ -134,32 +126,14 @@ steps:
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: Windows 2016 Unit Tests"

- label: ":linux: Ubuntu ARM Unit Tests"
key: "x-pack-filebeat-mandatory-linux-arm-unit-test"
command: |
cd x-pack/filebeat
mage build unitTest
retry:
automatic:
- limit: 3
agents:
provider: "aws"
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
artifact_paths:
- "x-pack/filebeat/build/*.xml"
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: Ubuntu ARM Unit Tests"
context: "x-pack/filebeat: Win-2016 Unit Tests"

- group: "x-pack/filebeat Extended Windows Tests"
key: "x-pack-filebeat-extended-win-tests"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/

steps:
- label: ":windows: Windows 2019 Unit Tests"
- label: ":windows: x-pack/filebeat Win-2019 Unit Tests"
command: |
Set-Location -Path x-pack/filebeat
mage build unitTest
Expand All @@ -178,9 +152,9 @@ steps:
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: Windows 2019 Unit Tests"
context: "x-pack/filebeat: Win-2019 Unit Tests"

- label: ":windows: Windows 10 Unit Tests"
- label: ":windows: x-pack/filebeat Win 10 Unit Tests"
command: |
Set-Location -Path x-pack/filebeat
mage build unitTest
Expand All @@ -199,9 +173,9 @@ steps:
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: Windows 10 Unit Tests"
context: "x-pack/filebeat: Win 10 Unit Tests"

- label: ":windows: Windows 11 Unit Tests"
- label: ":windows: x-pack/filebeat Win 11 Unit Tests"
command: |
Set-Location -Path x-pack/filebeat
mage build unitTest
Expand All @@ -220,17 +194,36 @@ steps:
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: Windows 11 Unit Tests"
context: "x-pack/filebeat: Win 11 Unit Tests"

- group: "x-pack/filebeat Extended Tests"
key: "x-pack-filebeat-extended-tests"

steps:
- label: ":mac: MacOS x86_64 Unit Tests"
- label: ":ubuntu: x-pack/filebeat Linux arm64 Unit Tests"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
key: "x-pack-filebeat-mandatory-linux-arm-unit-test"
command: |
cd x-pack/filebeat
mage build unitTest
agents:
provider: "aws"
imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
artifact_paths:
- "x-pack/filebeat/build/*.xml"
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: Linux arm64 Unit Tests"

- label: ":mac: x-pack/filebeat macOS x86_64 Unit Tests"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
command: |
set -euo pipefail
source .buildkite/scripts/install_macos_tools.sh
cd x-pack/filebeat && mage build unitTest
cd x-pack/filebeat
mage build unitTest
retry:
automatic:
- limit: 3
Expand All @@ -242,18 +235,20 @@ steps:
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: MacOS x86_64 Unit Tests"
context: "x-pack/filebeat: macOS x86_64 Unit Tests"

- label: ":mac: MacOS arm64 Unit Tests"
skip: "https://github.com/elastic/beats/issues/33036"
- label: ":mac: x-pack/filebeat macOS arm64 Unit Tests"
skip: "Skipping due to elastic/beats#33036"
# https://github.com/elastic/beats/issues/33036
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
command: |
set -euo pipefail
source .buildkite/scripts/install_macos_tools.sh
cd x-pack/filebeat && mage build unitTest
cd x-pack/filebeat
mage build unitTest
retry:
automatic:
- limit: 3
- limit: 3
agents:
provider: "orka"
imagePrefix: "${IMAGE_MACOS_ARM}"
Expand All @@ -262,9 +257,9 @@ steps:
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: MacOS arm64 Unit Tests"
context: "x-pack/filebeat: macOS arm64 Unit Tests"

- label: ":linux: Cloud (MODULE) Tests"
- label: ":ubuntu: x-pack/filebeat Cloud (MODULE) Tests"
key: "x-pack-filebeat-extended-cloud-test"
if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/
command: |
Expand All @@ -273,12 +268,12 @@ steps:
source .buildkite/scripts/changesets.sh
defineModuleFromTheChangeSet x-pack/filebeat
echo "~~~ Running tests"
.buildkite/scripts/cloud_tests.sh
source .buildkite/scripts/setup_cloud_env.sh
cd x-pack/filebeat
mage build test
env:
ASDF_TERRAFORM_VERSION: 1.0.2
AWS_REGION: "eu-central-1"
BEATS_PROJECT_NAME: x-pack/filebeat
MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform"
REPO: beats
agents:
Expand All @@ -292,26 +287,28 @@ steps:
- github_commit_status:
context: "x-pack/filebeat: Cloud (MODULE) Tests"

- label: ":linux: Cloud AWS (MODULE) Tests"
- label: ":ubuntu: x-pack/filebeat Cloud AWS (MODULE) Tests"
key: "x-pack-filebeat-extended-cloud-test-aws"
skip: "Skip test until fixed https://github.com/elastic/beats/issues/36425"
skip: "Skipping due to elastic/beats#36425"
# https://github.com/elastic/beats/issues/36425
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/
command: |
set -euo pipefail
# defines the MODULE env var based on what's changed in a PR
source .buildkite/scripts/changesets.sh
defineModuleFromTheChangeSet x-pack/filebeat
echo "~~~ Running tests"
.buildkite/scripts/cloud_tests.sh
source .buildkite/scripts/setup_cloud_env.sh
cd x-pack/filebeat
mage build test goIntegTest
env:
ASDF_TERRAFORM_VERSION: 1.0.2
AWS_REGION: "eu-central-1"
BEATS_PROJECT_NAME: x-pack/filebeat
MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform"
REPO: beats
agents:
provider: "aws"
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
artifact_paths:
- "x-pack/filebeat/build/*.xml"
Expand All @@ -331,7 +328,7 @@ steps:
- group: "x-pack/filebeat Packaging"
key: "x-pack-filebeat-packaging"
steps:
- label: ":linux: Packaging Linux"
- label: ":linux: x-pack/filebeat Packaging Linux"
key: "packaging-linux"
command: |
cd x-pack/filebeat
Expand All @@ -352,7 +349,7 @@ steps:
- github_commit_status:
context: "x-pack/filebeat: Packaging Linux"

- label: ":linux: Packaging ARM"
- label: ":linux: x-pack/filebeat Packaging arm64"
key: "packaging-arm"
command: |
cd x-pack/filebeat
Expand All @@ -363,11 +360,11 @@ steps:
timeout_in_minutes: 20
agents:
provider: "aws"
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
env:
PLATFORMS: "linux/arm64"
PACKAGES: "docker"
notify:
- github_commit_status:
context: "x-pack/filebeat: Packaging Linux ARM"
context: "x-pack/filebeat: Packaging Linux arm64"
Loading

0 comments on commit bf13818

Please sign in to comment.