From 59ade1d5ffc085afae492b51aff6c93ce8cdd574 Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Wed, 20 Mar 2024 09:45:33 +0200 Subject: [PATCH] Snapshot packaging stage migration (#37827) (cherry picked from commit 01f5a90209120459cfbbd090cb29fd803618b11a) # Conflicts: # .buildkite/filebeat/scripts/package-step.sh # .buildkite/heartbeat/scripts/package-step.sh # .buildkite/scripts/packaging/package-step.sh --- .buildkite/auditbeat/auditbeat-pipeline.yml | 28 +++++---- .buildkite/auditbeat/scripts/crosscompile.sh | 2 - .buildkite/auditbeat/scripts/package-step.sh | 51 ---------------- .buildkite/auditbeat/scripts/package.sh | 13 ----- .../auditbeat/scripts/unit-tests-win.ps1 | 51 ---------------- .buildkite/auditbeat/scripts/unit-tests.sh | 2 - .buildkite/env-scripts/env.sh | 42 ++++++++++++-- .buildkite/env-scripts/linux-env.sh | 45 -------------- .buildkite/env-scripts/util.sh | 26 ++++----- .buildkite/filebeat/filebeat-pipeline.yml | 31 +++++----- .../filebeat/scripts/integration-gotests.sh | 2 - .../filebeat/scripts/integration-pytests.sh | 2 - .buildkite/filebeat/scripts/package.sh | 10 ---- .../filebeat/scripts/unit-tests-win.ps1 | 51 ---------------- .buildkite/filebeat/scripts/unit-tests.sh | 2 - .buildkite/heartbeat/heartbeat-pipeline.yml | 28 +++++---- .../heartbeat/scripts/integration-gotests.sh | 3 - .../heartbeat/scripts/integration-pytests.sh | 3 - .buildkite/heartbeat/scripts/package.sh | 13 ----- .../heartbeat/scripts/unit-tests-win.ps1 | 51 ---------------- .buildkite/heartbeat/scripts/unit-tests.sh | 3 - .buildkite/hooks/post-checkout | 10 ++-- .buildkite/hooks/pre-command | 35 +++++++++-- .buildkite/hooks/pre-exit | 15 +++++ .buildkite/hooks/scripts/util.sh | 33 +++++++++++ .buildkite/scripts/packaging/package-step.sh | 58 +++++++++++++++++++ .buildkite/scripts/packaging/package-util.sh | 22 +++++++ .buildkite/scripts/packaging/package.sh | 51 ++++++++++++++++ .../module/file_integrity/metricset_test.go | 9 +++ 29 files changed, 329 insertions(+), 363 deletions(-) delete mode 100755 .buildkite/auditbeat/scripts/package-step.sh delete mode 100755 .buildkite/auditbeat/scripts/package.sh delete mode 100644 .buildkite/auditbeat/scripts/unit-tests-win.ps1 delete mode 100644 .buildkite/env-scripts/linux-env.sh delete mode 100755 .buildkite/filebeat/scripts/package.sh delete mode 100644 .buildkite/filebeat/scripts/unit-tests-win.ps1 delete mode 100755 .buildkite/heartbeat/scripts/package.sh delete mode 100644 .buildkite/heartbeat/scripts/unit-tests-win.ps1 create mode 100644 .buildkite/hooks/pre-exit create mode 100755 .buildkite/hooks/scripts/util.sh create mode 100755 .buildkite/scripts/packaging/package-step.sh create mode 100755 .buildkite/scripts/packaging/package-util.sh create mode 100755 .buildkite/scripts/packaging/package.sh diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 798939bbf32..be03d7843b2 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -1,13 +1,18 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + BEATS_PROJECT_NAME: "auditbeat" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64" IMAGE_WIN_2016: "family/core-windows-2016" IMAGE_WIN_2019: "family/core-windows-2019" IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_RHEL9: "family/core-rhel-9" + IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" steps: - group: "Auditbeat Mandatory Testing" @@ -24,6 +29,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: - "auditbeat/build/*.xml" - "auditbeat/build/*.json" @@ -37,19 +43,20 @@ steps: agents: provider: "gcp" image: "${IMAGE_RHEL9}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: - "auditbeat/build/*.xml" - "auditbeat/build/*.json" - label: ":windows:-{{matrix.image}} Unit Tests" - command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1" notify: - github_commit_status: context: "Auditbeat: windows/Unit Tests" agents: provider: "gcp" image: "{{matrix.image}}" - machine_type: "n2-standard-8" + machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 200 disk_type: "pd-ssd" matrix: @@ -72,6 +79,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - group: "Extended Testing" key: "extended-tests" @@ -88,8 +96,8 @@ steps: context: "Auditbeat/Extended: Unit Tests ARM" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.large" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: "auditbeat/build/*.xml" - label: ":mac: MacOS Unit Tests" @@ -112,14 +120,14 @@ steps: steps: - label: ":windows: Win 2019 Unit Tests" key: "win-extended-2019" - command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1" notify: - github_commit_status: context: "Auditbeat/Extended: Win-2019 Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" - machine_type: "n2-standard-8" + machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 200 disk_type: "pd-ssd" artifact_paths: @@ -134,4 +142,4 @@ steps: steps: - label: Package pipeline - commands: ".buildkite/auditbeat/scripts/package-step.sh" + commands: ".buildkite/scripts/packaging/package-step.sh" diff --git a/.buildkite/auditbeat/scripts/crosscompile.sh b/.buildkite/auditbeat/scripts/crosscompile.sh index 866d6be4223..da8452d5380 100755 --- a/.buildkite/auditbeat/scripts/crosscompile.sh +++ b/.buildkite/auditbeat/scripts/crosscompile.sh @@ -2,7 +2,5 @@ set -euo pipefail -source .buildkite/env-scripts/linux-env.sh - echo "--- Executing Crosscompile" make -C auditbeat crosscompile diff --git a/.buildkite/auditbeat/scripts/package-step.sh b/.buildkite/auditbeat/scripts/package-step.sh deleted file mode 100755 index 9c7958df58f..00000000000 --- a/.buildkite/auditbeat/scripts/package-step.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/env-scripts/util.sh - -changeset="^auditbeat/ -^go.mod -^pytest.ini -^dev-tools/ -^libbeat/ -^testing/ -^\.buildkite/auditbeat/" - -if are_files_changed "$changeset"; then - bk_pipeline=$(cat <<-YAML - steps: - - label: ":ubuntu: Packaging Linux X86" - key: "package-linux-x86" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" - command: - - ".buildkite/auditbeat/scripts/package.sh" - notify: - - github_commit_status: - context: "Auditbeat/Packaging: Linux X86" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - - - label: ":linux: Packaging Linux ARM" - key: "package-linux-arm" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" - command: - - ".buildkite/auditbeat/scripts/package.sh" - notify: - - github_commit_status: - context: "Auditbeat/Packaging: ARM" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.large" -YAML -) - echo "${bk_pipeline}" | buildkite-agent pipeline upload -else - buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning' - exit 0 -fi diff --git a/.buildkite/auditbeat/scripts/package.sh b/.buildkite/auditbeat/scripts/package.sh deleted file mode 100755 index 71872ca15a3..00000000000 --- a/.buildkite/auditbeat/scripts/package.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/env-scripts/linux-env.sh - -echo "--- Docker Version: $(docker --version)" - -echo "--- Start Packaging" -cd auditbeat -umask 0022 -mage package - diff --git a/.buildkite/auditbeat/scripts/unit-tests-win.ps1 b/.buildkite/auditbeat/scripts/unit-tests-win.ps1 deleted file mode 100644 index 200627d518f..00000000000 --- a/.buildkite/auditbeat/scripts/unit-tests-win.ps1 +++ /dev/null @@ -1,51 +0,0 @@ -$ErrorActionPreference = "Stop" # set -e -$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually - -# Forcing to checkout again all the files with a correct autocrlf. -# Doing this here because we cannot set git clone options before. -function fixCRLF() { - Write-Host "--- Fixing CRLF in git checkout --" - git config core.autocrlf false - git rm --quiet --cached -r . - git reset --quiet --hard -} - -function withGolang() { - Write-Host "--- Install golang $GoVersion --" - choco install golang -y --version $GoVersion - - $choco = Convert-Path "$((Get-Command choco).Path)\..\.." - Import-Module "$choco\helpers\chocolateyProfile.psm1" - refreshenv - go version - go env -} - -function installGoDependencies() { - $installPackages = @( - "github.com/magefile/mage" - "github.com/elastic/go-licenser" - "golang.org/x/tools/cmd/goimports" - "github.com/jstemmer/go-junit-report" - "github.com/tebeka/go2xunit" - ) - foreach ($pkg in $installPackages) { - go install "$pkg" - } -} - -fixCRLF - -$ErrorActionPreference = "Continue" # set +e - -Set-Location -Path auditbeat -New-Item -ItemType Directory -Force -Path "build" -withGolang -installGoDependencies - -mage build unitTest - -$EXITCODE=$LASTEXITCODE -$ErrorActionPreference = "Stop" - -Exit $EXITCODE diff --git a/.buildkite/auditbeat/scripts/unit-tests.sh b/.buildkite/auditbeat/scripts/unit-tests.sh index c1f5685c77f..8a7514d747a 100755 --- a/.buildkite/auditbeat/scripts/unit-tests.sh +++ b/.buildkite/auditbeat/scripts/unit-tests.sh @@ -2,8 +2,6 @@ set -euo pipefail -source .buildkite/env-scripts/linux-env.sh - echo "--- Running Unit Tests" sudo chmod -R go-w auditbeat/ diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh index 4dfc01bafc3..f28658a107d 100644 --- a/.buildkite/env-scripts/env.sh +++ b/.buildkite/env-scripts/env.sh @@ -1,19 +1,53 @@ #!/usr/bin/env bash -SETUP_GVM_VERSION="v0.5.1" +source .buildkite/env-scripts/util.sh + +DOCS_CHANGESET="^.*\.(asciidoc|md)$ +deploy/kubernetes/.*-kubernetes.yaml" +PACKAGING_CHANGESET="^dev-tools/packaging/ +^.go-version" + +REPO="beats" WORKSPACE="$(pwd)" BIN="${WORKSPACE}/bin" HW_TYPE="$(uname -m)" PLATFORM_TYPE="$(uname)" -REPO="beats" TMP_FOLDER="tmp.${REPO}" +SNAPSHOT="true" +PYTEST_ADDOPTS="" +OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" +XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" + +SETUP_GVM_VERSION="v0.5.1" +ASDF_MAGE_VERSION="1.14.0" +SETUP_WIN_PYTHON_VERSION="3.11.0" + +# Docker & DockerHub +DOCKER_COMPOSE_VERSION="1.21.0" DOCKER_REGISTRY="docker.elastic.co" -export SETUP_GVM_VERSION +ONLY_DOCS=$(changeset_applies "$DOCS_CHANGESET") +PACKAGING_CHANGES=$(changeset_applies "$PACKAGING_CHANGESET") +GO_MOD_CHANGES=$(changeset_applies "^go.mod") + +export REPO export WORKSPACE export BIN export HW_TYPE export PLATFORM_TYPE -export REPO export TMP_FOLDER +export SNAPSHOT +export PYTEST_ADDOPTS +export OSS_MODULE_PATTERN +export XPACK_MODULE_PATTERN + +export SETUP_GVM_VERSION +export ASDF_MAGE_VERSION +export SETUP_WIN_PYTHON_VERSION + +export DOCKER_COMPOSE_VERSION export DOCKER_REGISTRY + +export ONLY_DOCS +export PACKAGING_CHANGES +export GO_MOD_CHANGES diff --git a/.buildkite/env-scripts/linux-env.sh b/.buildkite/env-scripts/linux-env.sh deleted file mode 100644 index 1365aaace4a..00000000000 --- a/.buildkite/env-scripts/linux-env.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/env-scripts/util.sh - -DEBIAN_FRONTEND="noninteractive" - -sudo mkdir -p /etc/needrestart -echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null - -if [[ $PLATFORM_TYPE == "Linux" ]]; then - # Remove this code once beats specific agent is set up - if grep -q 'Ubuntu' /etc/*release; then - export DEBIAN_FRONTEND - - echo "--- Ubuntu - Installing libs" - sudo apt-get update - sudo apt-get install -y libsystemd-dev - sudo apt install -y python3-pip - sudo apt-get install -y python3-venv - fi - - # Remove this code once beats specific agent is set up - if grep -q 'Red Hat' /etc/*release; then - echo "--- RHL - Installing libs" - sudo yum update -y - sudo yum install -y systemd-devel - sudo yum install -y python3-pip - sudo yum install -y python3 - pip3 install virtualenv - fi -fi - -if [[ $PLATFORM_TYPE == Darwin* ]]; then - echo "--- Setting larger ulimit on MacOS" - # To bypass file descriptor errors like "Too many open files error" on MacOS - ulimit -Sn 50000 - echo "--- ULIMIT: $(ulimit -n)" -fi - -echo "--- Setting up environment" -add_bin_path -with_go -with_mage diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh index 6a5c36bcd04..68fd08a75df 100644 --- a/.buildkite/env-scripts/util.sh +++ b/.buildkite/env-scripts/util.sh @@ -9,11 +9,11 @@ add_bin_path() { } with_go() { - local go_version="${GOLANG_VERSION}" + local go_version="${GO_VERSION}" echo "Setting up the Go environment..." create_bin check_platform_architecture - retry 5 curl -sL -o ${BIN}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE}-${arch_type}" + retry_with_count 5 curl -sL -o ${BIN}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE}-${arch_type}" export PATH="${PATH}:${BIN}" chmod +x ${BIN}/gvm eval "$(gvm "$go_version")" @@ -60,7 +60,7 @@ check_platform_architecture() { esac } -retry() { +retry_with_count() { local retries=$1 shift local count=0 @@ -89,16 +89,16 @@ are_files_changed() { fi } -cleanup() { - echo "Deleting temporary files..." - rm -rf ${BIN}/${TMP_FOLDER}.* - echo "Done." +changeset_applies() { + local changeset=$1 + if are_files_changed "$changeset"; then + echo true + else + echo false + fi } -unset_secrets () { - for var in $(printenv | sed 's;=.*;;' | sort); do - if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then - unset "$var" - fi - done +set_git_config() { + git config user.name "${GITHUB_USERNAME_SECRET}" + git config user.email "${GITHUB_EMAIL_SECRET}" } diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index e811d286953..ae22629e6ec 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -1,12 +1,17 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + BEATS_PROJECT_NAME: "filebeat" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_WIN_2016: "family/core-windows-2016" IMAGE_WIN_2019: "family/core-windows-2019" IMAGE_WIN_2022: "family/core-windows-2022" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" steps: - group: "Filebeat Mandatory Testing" @@ -23,7 +28,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: - "filebeat/build/*.xml" - "filebeat/build/*.json" @@ -37,7 +42,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - "filebeat/build/*.xml" - "filebeat/build/*.json" @@ -49,22 +54,22 @@ steps: - github_commit_status: context: "Filebeat: Python Integration Tests" agents: - provider: "gcp" + provider: gcp image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - "filebeat/build/*.xml" - "filebeat/build/*.json" - label: ":windows:-{{matrix.image}} Unit Tests" - command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1" notify: - github_commit_status: context: "Filebeat: windows/Unit Tests" agents: provider: "gcp" image: "{{matrix.image}}" - machine_type: "n2-standard-8" + machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 200 disk_type: "pd-ssd" matrix: @@ -91,8 +96,8 @@ steps: context: "Filebeat/Extended: Unit Tests ARM" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.large" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: "filebeat/build/*.xml" - label: ":mac: MacOS Unit Tests" @@ -115,14 +120,14 @@ steps: steps: - label: ":windows: Win 2019 Unit Tests" key: "win-extended-2019" - command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1" notify: - github_commit_status: context: "Filebeat/Extended: Win-2019 Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" - machine_type: "n2-standard-8" + machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 200 disk_type: "pd-ssd" artifact_paths: @@ -137,4 +142,4 @@ steps: steps: - label: Package pipeline - commands: ".buildkite/filebeat/scripts/package-step.sh" + commands: ".buildkite/scripts/packaging/package-step.sh" diff --git a/.buildkite/filebeat/scripts/integration-gotests.sh b/.buildkite/filebeat/scripts/integration-gotests.sh index d64ce7c98eb..6de39ff8817 100755 --- a/.buildkite/filebeat/scripts/integration-gotests.sh +++ b/.buildkite/filebeat/scripts/integration-gotests.sh @@ -2,8 +2,6 @@ set -euo pipefail -source .buildkite/env-scripts/linux-env.sh - echo "--- Executing Integration Tests" sudo chmod -R go-w filebeat/ diff --git a/.buildkite/filebeat/scripts/integration-pytests.sh b/.buildkite/filebeat/scripts/integration-pytests.sh index b51e8ae18a6..9aff8695c35 100755 --- a/.buildkite/filebeat/scripts/integration-pytests.sh +++ b/.buildkite/filebeat/scripts/integration-pytests.sh @@ -2,8 +2,6 @@ set -euo pipefail -source .buildkite/env-scripts/linux-env.sh - echo "--- Executing Integration Tests" sudo chmod -R go-w filebeat/ diff --git a/.buildkite/filebeat/scripts/package.sh b/.buildkite/filebeat/scripts/package.sh deleted file mode 100755 index 0bb03250348..00000000000 --- a/.buildkite/filebeat/scripts/package.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/env-scripts/linux-env.sh - -echo "--- Start Packaging" -cd filebeat -umask 0022 -mage package diff --git a/.buildkite/filebeat/scripts/unit-tests-win.ps1 b/.buildkite/filebeat/scripts/unit-tests-win.ps1 deleted file mode 100644 index 8990eb30a09..00000000000 --- a/.buildkite/filebeat/scripts/unit-tests-win.ps1 +++ /dev/null @@ -1,51 +0,0 @@ -$ErrorActionPreference = "Stop" # set -e -$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually - -# Forcing to checkout again all the files with a correct autocrlf. -# Doing this here because we cannot set git clone options before. -function fixCRLF() { - Write-Host "-- Fixing CRLF in git checkout --" - git config core.autocrlf false - git rm --quiet --cached -r . - git reset --quiet --hard -} - -function withGolang() { - Write-Host "-- Install golang $GoVersion --" - choco install golang -y --version $GoVersion - - $choco = Convert-Path "$((Get-Command choco).Path)\..\.." - Import-Module "$choco\helpers\chocolateyProfile.psm1" - refreshenv - go version - go env -} - -function installGoDependencies() { - $installPackages = @( - "github.com/magefile/mage" - "github.com/elastic/go-licenser" - "golang.org/x/tools/cmd/goimports" - "github.com/jstemmer/go-junit-report" - "github.com/tebeka/go2xunit" - ) - foreach ($pkg in $installPackages) { - go install "$pkg" - } -} - -fixCRLF - -$ErrorActionPreference = "Continue" # set +e - -Set-Location -Path filebeat -New-Item -ItemType Directory -Force -Path "build" -withGolang -installGoDependencies - -mage build unitTest - -$EXITCODE=$LASTEXITCODE -$ErrorActionPreference = "Stop" - -Exit $EXITCODE diff --git a/.buildkite/filebeat/scripts/unit-tests.sh b/.buildkite/filebeat/scripts/unit-tests.sh index 08ce9d4ea1c..2efb6b1ff8e 100755 --- a/.buildkite/filebeat/scripts/unit-tests.sh +++ b/.buildkite/filebeat/scripts/unit-tests.sh @@ -2,8 +2,6 @@ set -euo pipefail -source .buildkite/env-scripts/linux-env.sh - echo "--- Executing Unit Tests" sudo chmod -R go-w filebeat/ diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index bf645a2b295..2b5f6195f45 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -1,13 +1,18 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + BEATS_PROJECT_NAME: "heartbeat" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64" IMAGE_WIN_2016: "family/core-windows-2016" IMAGE_WIN_2019: "family/core-windows-2019" IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_RHEL9: "family/core-rhel-9" + IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" steps: - group: "Heartbeat Mandatory Testing" @@ -24,6 +29,7 @@ steps: agents: provider: "gcp" image: "{{matrix.image}}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" matrix: setup: image: @@ -35,14 +41,14 @@ steps: - label: ":windows: Unit Tests / {{matrix.image}}" command: - - ".buildkite/heartbeat/scripts/unit-tests-win.ps1" + - ".buildkite/scripts/win_unit_tests.ps1" notify: - github_commit_status: context: "Heartbeat: windows/Unit Tests" agents: provider: "gcp" image: "{{matrix.image}}" - machine_type: "n2-standard-8" + machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_type: "pd-ssd" matrix: setup: @@ -62,6 +68,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" @@ -75,6 +82,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" @@ -94,8 +102,8 @@ steps: context: "Heartbeat/Extended: Unit Tests ARM" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.large" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: "heartbeat/build/*.xml" - label: ":mac: MacOS Unit Tests" @@ -118,14 +126,14 @@ steps: steps: - label: ":windows: Win 2019 Unit Tests" key: "win-extended-2019" - command: ".buildkite/heartbeat/scripts/unit-tests-win.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1" notify: - github_commit_status: context: "Heartbeat/Extended: Win-2019 Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" - machine_type: "n2-standard-8" + machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_type: "pd-ssd" artifact_paths: - "heartbeat/build/*.xml" @@ -139,4 +147,4 @@ steps: steps: - label: Package pipeline - commands: ".buildkite/heartbeat/scripts/package-step.sh" + commands: ".buildkite/scripts/packaging/package-step.sh" diff --git a/.buildkite/heartbeat/scripts/integration-gotests.sh b/.buildkite/heartbeat/scripts/integration-gotests.sh index 8eab0e8b5d8..c50dbf45347 100755 --- a/.buildkite/heartbeat/scripts/integration-gotests.sh +++ b/.buildkite/heartbeat/scripts/integration-gotests.sh @@ -2,9 +2,6 @@ set -euo pipefail -# Remove when custom image is set up -source .buildkite/env-scripts/linux-env.sh - echo "--- Executing Integration Tests" # Remove when custom image is set up sudo chmod -R go-w heartbeat/ diff --git a/.buildkite/heartbeat/scripts/integration-pytests.sh b/.buildkite/heartbeat/scripts/integration-pytests.sh index 729df5ae6f6..5875b5460ed 100755 --- a/.buildkite/heartbeat/scripts/integration-pytests.sh +++ b/.buildkite/heartbeat/scripts/integration-pytests.sh @@ -2,9 +2,6 @@ set -euo pipefail -# Remove when custom image is set up -source .buildkite/env-scripts/linux-env.sh - echo "--- Executing Integration Tests" # Remove when custom image is set up sudo chmod -R go-w heartbeat/ diff --git a/.buildkite/heartbeat/scripts/package.sh b/.buildkite/heartbeat/scripts/package.sh deleted file mode 100755 index 7f51a6b5ca1..00000000000 --- a/.buildkite/heartbeat/scripts/package.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/env-scripts/linux-env.sh - -echo "--- Docker Version: $(docker --version)" - -echo "--- Start Packaging" -cd heartbeat -umask 0022 -mage package - diff --git a/.buildkite/heartbeat/scripts/unit-tests-win.ps1 b/.buildkite/heartbeat/scripts/unit-tests-win.ps1 deleted file mode 100644 index 17282813e13..00000000000 --- a/.buildkite/heartbeat/scripts/unit-tests-win.ps1 +++ /dev/null @@ -1,51 +0,0 @@ -$ErrorActionPreference = "Stop" # set -e -$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually - -# Forcing to checkout again all the files with a correct autocrlf. -# Doing this here because we cannot set git clone options before. -function fixCRLF() { - Write-Host "--- Fixing CRLF in git checkout --" - git config core.autocrlf false - git rm --quiet --cached -r . - git reset --quiet --hard -} - -function withGolang() { - Write-Host "--- Install golang $GoVersion --" - choco install golang -y --version $GoVersion - - $choco = Convert-Path "$((Get-Command choco).Path)\..\.." - Import-Module "$choco\helpers\chocolateyProfile.psm1" - refreshenv - go version - go env -} - -function installGoDependencies() { - $installPackages = @( - "github.com/magefile/mage" - "github.com/elastic/go-licenser" - "golang.org/x/tools/cmd/goimports" - "github.com/jstemmer/go-junit-report" - "github.com/tebeka/go2xunit" - ) - foreach ($pkg in $installPackages) { - go install "$pkg" - } -} - -fixCRLF - -$ErrorActionPreference = "Continue" # set +e - -Set-Location -Path heartbeat -New-Item -ItemType Directory -Force -Path "build" -withGolang -installGoDependencies - -mage build unitTest - -$EXITCODE=$LASTEXITCODE -$ErrorActionPreference = "Stop" - -Exit $EXITCODE diff --git a/.buildkite/heartbeat/scripts/unit-tests.sh b/.buildkite/heartbeat/scripts/unit-tests.sh index 4b746da2d57..1d8de945788 100755 --- a/.buildkite/heartbeat/scripts/unit-tests.sh +++ b/.buildkite/heartbeat/scripts/unit-tests.sh @@ -2,9 +2,6 @@ set -euo pipefail -# Remove when custom image is set up -source .buildkite/env-scripts/linux-env.sh - echo "--- Running Unit Tests" # Remove when custom image is set up sudo chmod -R go-w heartbeat/ diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index b6cc7ad60bd..43881f6e2d8 100755 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -8,7 +8,7 @@ checkout_merge() { local merge_branch=$3 if [[ -z "${target_branch}" ]]; then - echo "No pull request target branch" + echo "--- No pull request target branch" exit 1 fi @@ -24,9 +24,9 @@ checkout_merge() { git config user.name "github-merged-pr-post-checkout" git config user.email "auto-merge@buildkite" - git merge --no-edit "${BUILDKITE_COMMIT}" || { + git merge --no-edit "${pr_commit}" || { local merge_result=$? - echo "Merge failed: ${merge_result}" + echo "--- Merge failed: ${merge_result}" git merge --abort exit ${merge_result} } @@ -35,7 +35,7 @@ checkout_merge() { pull_request="${BUILDKITE_PULL_REQUEST:-false}" if [[ "${pull_request}" == "false" ]]; then - echo "Not a pull request, skipping" + echo "--- Not a pull request, skipping" exit 0 fi @@ -46,7 +46,7 @@ MERGE_BRANCH="pr_merge_${PR_ID}" checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}" -echo "Commit information" +echo "--- Commit information" git --no-pager log --format=%B -n 1 # Ensure buildkite groups are rendered diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 7be1f965230..9df1bed50a7 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,8 +2,13 @@ set -euo pipefail +# Secrets must be redacted +# https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth" PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account" +DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" +PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials" +GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" retry() { local retries=$1 @@ -24,13 +29,35 @@ retry() { return 0 } -if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then source .buildkite/env-scripts/env.sh source .buildkite/env-scripts/util.sh - source .buildkite/env-scripts/win-env.sh - if [[ -z "${GOLANG_VERSION-""}" ]]; then - export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version") + if [[ -z "${GO_VERSION-""}" ]]; then + export GO_VERSION=$(cat "${WORKSPACE}/.go-version") + fi + + if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then + ulimit -Sn 30000 + + echo "--- Setting up environment" + add_bin_path + with_go + with_mage + fi + + if [[ "$BUILDKITE_STEP_KEY" == package* ]]; then + export DOCKER_USERNAME_SECRET=$(retry_with_count 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}") + export DOCKER_PASSWORD_SECRET=$(retry_with_count 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}") + export GITHUB_TOKEN_SECRET=$(retry_with_count 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH}) + + docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null + + github_username=$(retry_with_count 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH}) + github_email=$(retry_with_count 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH}) + + git config user.name "$github_username" + git config user.email "$github_email" fi fi diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit new file mode 100644 index 00000000000..d1ff6e0ac1c --- /dev/null +++ b/.buildkite/hooks/pre-exit @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/hooks/scripts/util.sh + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then + if [[ "$BUILDKITE_STEP_KEY" == package* ]]; then + docker logout "${DOCKER_REGISTRY}" + fi + + # Ensure that any temporal files created during any step are removed + cleanup + unset_secrets +fi diff --git a/.buildkite/hooks/scripts/util.sh b/.buildkite/hooks/scripts/util.sh new file mode 100755 index 00000000000..07ab6cf4c9a --- /dev/null +++ b/.buildkite/hooks/scripts/util.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -euo pipefail + +unset_secrets () { + for var in $(printenv | sed 's;=.*;;' | sort); do + if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then + unset "$var" + fi + done +} + +google_cloud_logout_active_account() { + local active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null) + if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then + echo "Logging out from GCP for active account" + gcloud auth revoke $active_account > /dev/null 2>&1 + else + echo "No active GCP accounts found." + fi + if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then + unset GOOGLE_APPLICATION_CREDENTIALS + cleanup + fi +} + +cleanup() { + echo "Deleting temporary files..." + if [[ -e "${BIN}/${TMP_FOLDER}" ]]; then + rm -rf "${BIN}/${TMP_FOLDER}.*" + fi + echo "Done." +} diff --git a/.buildkite/scripts/packaging/package-step.sh b/.buildkite/scripts/packaging/package-step.sh new file mode 100755 index 00000000000..c343a1043f9 --- /dev/null +++ b/.buildkite/scripts/packaging/package-step.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/util.sh + +changeset="^${BEATS_PROJECT_NAME}/ +^go.mod +^pytest.ini +^dev-tools/ +^libbeat/ +^testing/ +^\.buildkite/${BEATS_PROJECT_NAME}/" + +if are_files_changed "$changeset"; then + bk_pipeline=$(cat <<-YAML + steps: + - label: ":ubuntu: ${BEATS_PROJECT_NAME}/Packaging Linux X86" + key: "package-linux-x86" + env: +<<<<<<< HEAD:.buildkite/auditbeat/scripts/package-step.sh + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" + command: + - ".buildkite/auditbeat/scripts/package.sh" +======= + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + SNAPSHOT: true + command: ".buildkite/scripts/packaging/package.sh" +>>>>>>> 01f5a90209 (Snapshot packaging stage migration (#37827)):.buildkite/scripts/packaging/package-step.sh + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}/Packaging: Linux X86" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + + - label: ":linux: ${BEATS_PROJECT_NAME}/Packaging Linux ARM" + key: "package-linux-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + SNAPSHOT: true + command: ".buildkite/scripts/packaging/package.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}/Packaging: ARM" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" +YAML +) + echo "${bk_pipeline}" | buildkite-agent pipeline upload +else + buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning' + exit 0 +fi diff --git a/.buildkite/scripts/packaging/package-util.sh b/.buildkite/scripts/packaging/package-util.sh new file mode 100755 index 00000000000..4a50457cc9c --- /dev/null +++ b/.buildkite/scripts/packaging/package-util.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -euo pipefail + +is_pr() { + if [[ $BUILDKITE_PULL_REQUEST != false ]]; then + return 0 + else + return 1 + fi +} + +define_tags() { + aliasVersion="${VERSION%.*}${IMG_POSTFIX}" + tags=("${BUILDKITE_COMMIT}") + + if is_pr; then + tags+=("pr-${GITHUB_PR_NUMBER}") + else + tags+=("${SOURCE_TAG}" "${aliasVersion}") + fi +} diff --git a/.buildkite/scripts/packaging/package.sh b/.buildkite/scripts/packaging/package.sh new file mode 100755 index 00000000000..5744ee0776b --- /dev/null +++ b/.buildkite/scripts/packaging/package.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/packaging/package-util.sh + +IMG_POSTFIX="-SNAPSHOT" +VARIANTS=("" "-ubi" "-oss") +VERSION="$(make get-version)" +SOURCE_TAG+="${VERSION}${IMG_POSTFIX}" +TARGET="observability-ci/${BEATS_PROJECT_NAME}" + +echo "--- Creating package" +mage -d "${BEATS_PROJECT_NAME}" package + +echo "--- Distribution list" +dir="${BEATS_PROJECT_NAME}/build/distributions" +buildkite-agent artifact upload "$dir/*.tar.gz;$dir/*.tar.gz.sha512" + +echo "--- Docker image list" +docker images + +define_tags + +targetSuffix="" +if [[ ${HW_TYPE} == "aarch64" || ${HW_TYPE} == "arm64" ]]; then + targetSuffix="-arm64" +fi + +for variant in "${VARIANTS[@]}"; do + source="beats/${BEATS_PROJECT_NAME}${variant}" + + for tag in "${tags[@]}"; do + targetTag=$tag${targetSuffix} + + sourceName="${DOCKER_REGISTRY}/${source}:${SOURCE_TAG}" + targetName="${DOCKER_REGISTRY}/${TARGET}:${targetTag}" + #TODO Remove following line once beats fully migrated to Buildkite and Jenkins builds will be disabled + #Avoid clashing with the Jenkins produced images + targetName="${targetName}-buildkite" + + if docker image inspect "${sourceName}" &>/dev/null; then + echo "--- Tag & Push with target: $targetName" + echo "Source name: $sourceName" + docker tag "$sourceName" "$targetName" + docker push "$targetName" + else + echo "Docker image ${sourceName} does not exist" + fi + done +done diff --git a/auditbeat/module/file_integrity/metricset_test.go b/auditbeat/module/file_integrity/metricset_test.go index 13e27cf82ea..b4372aa8ba7 100644 --- a/auditbeat/module/file_integrity/metricset_test.go +++ b/auditbeat/module/file_integrity/metricset_test.go @@ -66,6 +66,7 @@ func TestData(t *testing.T) { func TestActions(t *testing.T) { skipOnCIForDarwinAMD64(t) + skipOnBuildkiteWindows(t) defer abtest.SetupDataDir(t)() @@ -176,6 +177,7 @@ func TestActions(t *testing.T) { func TestExcludedFiles(t *testing.T) { skipOnCIForDarwinAMD64(t) + skipOnBuildkiteWindows(t) defer abtest.SetupDataDir(t)() @@ -232,6 +234,7 @@ func TestExcludedFiles(t *testing.T) { func TestIncludedExcludedFiles(t *testing.T) { skipOnCIForDarwinAMD64(t) + skipOnBuildkiteWindows(t) defer abtest.SetupDataDir(t)() @@ -992,3 +995,9 @@ func skipOnCIForDarwinAMD64(t testing.TB) { t.Skip("Skip test on CI for darwin/amd64") } } + +func skipOnBuildkiteWindows(t testing.TB) { + if os.Getenv("BUILDKITE") == "true" && runtime.GOOS == "windows" { + t.Skip("Skip on Buildkite Windows: Shortened TMP problem") + } +}