From 5cb66c8d947e3a3d2934fc4e2aa8d6838e3bf35f Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 7 Mar 2023 16:00:29 +0100 Subject: [PATCH 01/11] Update safe logs to internal bucket --- .buildkite/hooks/pre-command | 5 ++ .../pipeline.trigger.integration.tests.sh | 2 + .buildkite/scripts/install_deps.sh | 5 ++ .buildkite/scripts/integration_tests.sh | 64 ++++++++++++++++--- .buildkite/scripts/signAndPublishPackage.sh | 9 +-- 5 files changed, 73 insertions(+), 12 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 44d9ebb2ff..28de6bd904 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -28,10 +28,15 @@ GITHUB_TOKEN_VAULT_PATH=kv/ci-shared/platform-ingest/github_token JENKINS_API_TOKEN_PATH=kv/ci-shared/platform-ingest/jenkins_api_tokens SIGNING_PACKAGES_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/signing_packages_gcs_artifacts_credentials PACKAGE_UPLOADER_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/package_storage_uploader +PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials # Secrets must be redacted # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables +if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" =~ ^integration-parallel ]]; then + export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) +fi + if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp" ]]; then export ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectId ${GCP_SERVICE_ACCOUNT_SECRET_PATH}) export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH}) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 165293f0b2..fcfb55a783 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -56,6 +56,8 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " - label: \":go: Running integration test: ${package_name}\"" echo " key: \"integration-parallel-${package_name}\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}" + echo " env:" + echo " UPLOAD_SAFE_LOGS: 1" echo " agents:" echo " provider: \"gcp\"" echo " artifact_paths:" diff --git a/.buildkite/scripts/install_deps.sh b/.buildkite/scripts/install_deps.sh index 20a1f8a9c2..e8de610df9 100755 --- a/.buildkite/scripts/install_deps.sh +++ b/.buildkite/scripts/install_deps.sh @@ -4,6 +4,10 @@ set -euo pipefail source .buildkite/scripts/tooling.sh +add_bin_path(){ + export PATH="${WORKSPACE}/bin:${PATH}" +} + with_kubernetes() { mkdir -p ${WORKSPACE}/bin retry 5 curl -sSLo ${WORKSPACE}/bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" @@ -25,6 +29,7 @@ with_go() { eval "$(gvm $(cat .go-version))" go version which go + export PATH="$(go env GOPATH)/bin:${PATH}" } with_docker_compose() { diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh index e94bba9307..27e19b0239 100755 --- a/.buildkite/scripts/integration_tests.sh +++ b/.buildkite/scripts/integration_tests.sh @@ -1,9 +1,13 @@ #!/bin/bash - set -euo pipefail -PARALLEL_TARGET="test-check-packages-parallel" -KIND_TARGET="test-check-packages-with-kind" +cleanup() { + echo "Deleting temporal files..." + cd ${WORKSPACE} + rm -rf "tmp.elastic-package.*" + echo "Done." +} +trap cleanup EXIT usage() { echo "$0 [-t ] [-h]" @@ -14,6 +18,18 @@ usage() { } source .buildkite/scripts/install_deps.sh +source .buildkite/scripts/tooling.sh + +WORKSPACE="$(pwd)" +PARALLEL_TARGET="test-check-packages-parallel" +KIND_TARGET="test-check-packages-with-kind" +TEMPLATE_TEMP_FOLDER="tmp.elastic-package.XXXXXXXXX" +GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json" + +JOB_GCS_BUCKET_INTERNAL="fleet-ci-temp-internal" + +REPO_NAME=$(repoName "${BUILDKITE_REPO}") +REPO_BUILD_TAG="${REPO_NAME}/${BUILDKITE_BUILD_NUMBER}" TARGET="" PACKAGE="" @@ -48,14 +64,34 @@ if [[ "${TARGET}" == "" ]]; then exit 1 fi -echo "Current path: $(pwd)" -WORKSPACE="$(pwd)" -export PATH="${WORKSPACE}/bin:${PATH}" -echo "Path: $PATH" +google_cloud_auth_safe_logs() { + local gsUtilLocation=$(mktemp -d -p . -t ${TEMPLATE_TEMP_FOLDER}) + local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME} + + echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation} + + google_cloud_auth "${secretFileLocation}" + + echo "${gsUtilLocation}" +} + +upload_safe_logs() { + local bucket="$1" + local source="$2" + local target="$3" + + local gsUtilLocation=$(google_cloud_auth_safe_logs) + + gsutil cp "${source}" "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}" + + rm -rf "${gsUtilLocation}" + unset GOOGLE_APPLICATIONS_CREDENTIALS +} + +add_bin_path echo "--- install go" with_go -export PATH="$(go env GOPATH)/bin:${PATH}" echo "--- install docker-compose" with_docker_compose @@ -69,6 +105,18 @@ echo "--- Run integration test ${TARGET}" if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then make install make PACKAGE_UNDER_TEST=${PACKAGE} ${TARGET} + + if [[ "${UPLOAD_SAFE_LOGS}" -eq 1 ]] ; then + upload_safe_logs \ + "${JOB_GCS_BUCKET_INTERNAL}" \ + "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/*" \ + "insecure-logs/${PACKAGE}/" + + upload_safe_logs \ + "${JOB_GCS_BUCKET_INTERNAL}" \ + "build/container-logs/*.log" \ + "insecure-logs/${PACKAGE}/" + fi exit 0 fi diff --git a/.buildkite/scripts/signAndPublishPackage.sh b/.buildkite/scripts/signAndPublishPackage.sh index a7eccb81bb..72c2ce9236 100755 --- a/.buildkite/scripts/signAndPublishPackage.sh +++ b/.buildkite/scripts/signAndPublishPackage.sh @@ -4,6 +4,9 @@ set -euo pipefail WORKSPACE="$(pwd)" TMP_FOLDER_TEMPLATE_BASE="tmp.elastic-package" +source .buildkite/scripts/install_deps.sh +source .buildkite/scripts/tooling.sh + cleanup() { echo "Deleting temporal files..." cd ${WORKSPACE} @@ -13,7 +16,8 @@ cleanup() { trap cleanup EXIT -export PATH="${WORKSPACE}/bin:${PATH}" +add_bin_path + echo "Checking gsutil command..." if ! command -v gsutil &> /dev/null ; then @@ -23,9 +27,6 @@ else echo "✅ gsutil is installed" fi -source .buildkite/scripts/install_deps.sh -source .buildkite/scripts/tooling.sh - isAlreadyPublished() { local packageZip=$1 From 34298b59659dbabaf5c1d9a1fac2af1ff0aad05a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 15 Mar 2023 03:43:31 +0100 Subject: [PATCH 02/11] Run add_bin_path after loading tooling.sh --- .buildkite/scripts/signAndPublishPackage.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.buildkite/scripts/signAndPublishPackage.sh b/.buildkite/scripts/signAndPublishPackage.sh index 72c2ce9236..5f539cc52c 100755 --- a/.buildkite/scripts/signAndPublishPackage.sh +++ b/.buildkite/scripts/signAndPublishPackage.sh @@ -18,15 +18,6 @@ trap cleanup EXIT add_bin_path - -echo "Checking gsutil command..." -if ! command -v gsutil &> /dev/null ; then - echo "⚠️ gsutil is not installed" - exit 1 -else - echo "✅ gsutil is installed" -fi - isAlreadyPublished() { local packageZip=$1 @@ -38,6 +29,15 @@ isAlreadyPublished() { return 1 } +echo "Checking gsutil command..." +if ! command -v gsutil &> /dev/null ; then + echo "⚠️ gsutil is not installed" + exit 1 +else + echo "✅ gsutil is installed" +fi + + REPO_NAME=$(repoName "${BUILDKITE_REPO}") BUILD_TAG="buildkite-${BUILDKITE_PIPELINE_SLUG}-${BUILDKITE_BUILD_NUMBER}" @@ -143,6 +143,8 @@ publishPackage() { rm -r "${gsUtilLocation}" } +add_bin_path + # Required to trigger Jenkins job with_go From 2c00b2a5d8a96fd7f2a82a2070e18666632e363b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 15 Mar 2023 04:22:57 +0100 Subject: [PATCH 03/11] Remove double quotes --- .buildkite/scripts/integration_tests.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh index 27e19b0239..e531b36399 100755 --- a/.buildkite/scripts/integration_tests.sh +++ b/.buildkite/scripts/integration_tests.sh @@ -82,7 +82,7 @@ upload_safe_logs() { local gsUtilLocation=$(google_cloud_auth_safe_logs) - gsutil cp "${source}" "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}" + gsutil cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}" rm -rf "${gsUtilLocation}" unset GOOGLE_APPLICATIONS_CREDENTIALS @@ -117,6 +117,7 @@ if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then "build/container-logs/*.log" \ "insecure-logs/${PACKAGE}/" fi + make check-git-clean exit 0 fi From 1d75290883b564237beb88b6086edc009a84e505 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 15 Mar 2023 04:31:02 +0100 Subject: [PATCH 04/11] Check artifacts exist --- .buildkite/scripts/integration_tests.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh index e531b36399..0e2f78966c 100755 --- a/.buildkite/scripts/integration_tests.sh +++ b/.buildkite/scripts/integration_tests.sh @@ -80,6 +80,11 @@ upload_safe_logs() { local source="$2" local target="$3" + if ! ls ${source} 2>&1 > /dev/null ; then + echo "upload_safe_logs: artifacts files not found, nothing will be archived" + return + fi + local gsUtilLocation=$(google_cloud_auth_safe_logs) gsutil cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}" From 4068dd123610351ac80b91df3bb143b570c7dabb Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 15 Mar 2023 05:19:25 +0100 Subject: [PATCH 05/11] Add subfolder --- .buildkite/scripts/integration_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh index 0e2f78966c..10823cf045 100755 --- a/.buildkite/scripts/integration_tests.sh +++ b/.buildkite/scripts/integration_tests.sh @@ -120,7 +120,7 @@ if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then upload_safe_logs \ "${JOB_GCS_BUCKET_INTERNAL}" \ "build/container-logs/*.log" \ - "insecure-logs/${PACKAGE}/" + "insecure-logs/${PACKAGE}/container-logs/" fi make check-git-clean exit 0 From c2995d5dd26b6ab8c687ee7af480db742492c8d5 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 15 Mar 2023 05:21:44 +0100 Subject: [PATCH 06/11] Restore Jenkinsfile to compare against Buildkite --- .ci/Jenkinsfile | 232 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 219 insertions(+), 13 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index cbb3c7063c..46aeffd4e4 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -12,8 +12,16 @@ pipeline { BASE_DIR="src/github.com/elastic/elastic-package" JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba" PIPELINE_LOG_LEVEL='INFO' + AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' HOME = "${env.WORKSPACE}" - BUILDKITE_MERGE_COMMIT = "79ee1738578deac17f8f5caf8d925db6244ab6df" + KIND_VERSION = 'v0.17.0' + K8S_VERSION = 'v1.26.0' + JOB_GCS_BUCKET = 'fleet-ci-temp' + JOB_GCS_BUCKET_INTERNAL = 'fleet-ci-temp-internal' + JOB_GCS_CREDENTIALS = 'fleet-ci-gcs-plugin' + JOB_GCS_EXT_CREDENTIALS = 'fleet-ci-gcs-plugin-file-credentials' + ELASTIC_PACKAGE_GCP_SECRET = 'secret/observability-team/ci/service-account/elastic-package-gcp' + ELASTIC_OBSERVABILITY_PROJECT_ID = 'elastic-observability' } options { timeout(time: 1, unit: 'HOURS') @@ -43,22 +51,66 @@ pipeline { stash allowEmpty: true, name: 'source', useDefaultExcludes: false } } - stage('Check if the PR contains Buildkite pipeline') { - when { - changeRequest() + /** + Check the source code. + */ + stage('Check static') { + steps { + cleanup() + dir("${BASE_DIR}"){ + withMageEnv(){ + sh(label: 'Check',script: 'make check-static') + } + } } + } + /** + Run the unit tests suite + */ + stage('Unit tests') { steps { cleanup() dir("${BASE_DIR}"){ - script { - githubEnv() - r = sh(label: 'Check', returnStatus: true, script: ''' - echo "Base commit: ${GIT_BASE_COMMIT}" - git log --format=%H "${GIT_BASE_COMMIT}" | grep "${BUILDKITE_MERGE_COMMIT}" - ''') - if (r != 0) { - echo("You should update your branch to include latest buildkite pipelines") - sh(label: 'Exit', script: 'exit 1') + withMageEnv(){ + sh(label: 'Check',script: 'make test-go-ci') + } + } + } + post { + always { + dir("${BASE_DIR}") { + archiveArtifacts(allowEmptyArchive: true, artifacts: 'build/test-results/*.xml') + junit(allowEmptyResults: false, keepLongStdio: true, testResults: "build/test-results/*.xml") + stashCoverageReport() + } + } + } + } + stage('Integration tests') { + failFast true + options { skipDefaultCheckout() } + steps { + withGithubNotify(context: 'Integration tests', tab: 'tests') { + cleanup() + dir("${BASE_DIR}") { + script { + def basicTasks = [ + 'stack-command-default': generateTestCommandStage(command: 'test-stack-command-default', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), + 'stack-command-oldest': generateTestCommandStage(command: 'test-stack-command-oldest', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), + 'stack-command-7x': generateTestCommandStage(command: 'test-stack-command-7x', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), + 'stack-command-86': generateTestCommandStage(command: 'test-stack-command-86', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), + 'stack-command-8x': generateTestCommandStage(command: 'test-stack-command-8x', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), + 'check-packages-with-kind': generateTestCommandStage(command: 'test-check-packages-with-kind', artifacts: ['build/test-results/*.xml', 'build/kubectl-dump.txt', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true), + 'check-packages-other': generateTestCommandStage(command: 'test-check-packages-other', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true), + 'check-packages-with-custom-agent': generateTestCommandStage(command: 'test-check-packages-with-custom-agent', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true), + 'check-packages-benchmarks': generateTestCommandStage(command: 'test-check-packages-benchmarks', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: false), + 'build-zip': generateTestCommandStage(command: 'test-build-zip', artifacts: ['build/elastic-stack-dump/build-zip/logs/*.log', 'build/packages/*.sig']), + 'profiles-command': generateTestCommandStage(command: 'test-profiles-command') + ] + + def checkSinglePackageTasks = generateTestCheckSinglePackageStage() + def tasks = basicTasks + checkSinglePackageTasks + parallel tasks } } } @@ -75,3 +127,157 @@ def cleanup(){ } unstash 'source' } + +def generateTestCheckSinglePackageStage(Map args = [:]) { + def artifacts = ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'] + + def integrations = [:] + dir("test/packages/parallel") { + // Include hack to skip temporary files with "@tmp" suffix. + // For reference: https://issues.jenkins.io/browse/JENKINS-52750 + findFiles()?.findAll{ !it.name.endsWith('@tmp') }?.collect{ it.name }?.sort()?.each { + integrations["check-package-${it}"] = { + withNode(labels: "ubuntu-20 && immutable", sleepMax: 20, forceWorkspace: true) { + cleanup() + try { + dir("${BASE_DIR}"){ + withMageEnv(){ + withCloudTestEnv() { + sh(label: 'Install elastic-package',script: "make install") + sh(label: 'Build elastic-package',script: "make PACKAGE_UNDER_TEST=${it} test-check-packages-parallel") + } + } + } + } finally { + dir("${BASE_DIR}") { + artifacts.each { artifact -> + archiveArtifacts(allowEmptyArchive: true, artifacts: "${artifact}") + } + archiveArtifactsSafe("insecure-logs/${it}", "build/elastic-stack-dump/check-${it}/logs/elastic-agent-internal/*") + archiveArtifactsSafe("insecure-logs/${it}/container-logs", "build/container-logs/*.log") + + junit(allowEmptyResults: true, keepLongStdio: true, testResults: "build/test-results/*.xml") + stashCoverageReport() + } + } + } + } + } + } + return integrations +} + +def generateTestCommandStage(Map args = [:]){ + def command = args.get('command') + def artifacts = args.get('artifacts') ? args.get('artifacts') : [] + def junitArtifacts = args.get('junitArtifacts') ? args.get('junitArtifacts') : false + def publishCoverage = args.get('publishCoverage') ? args.get('publishCoverage') : false + + return { + withNode(labels: "ubuntu-20 && immutable", sleepMax: 20, forceWorkspace: true) { + cleanup() + try { + dir("${BASE_DIR}"){ + withMageEnv(){ + withKubernetes() { + withCloudTestEnv() { + sh(label: 'Check',script: "make install ${command} check-git-clean") + } + } + } + } + } finally { + dir("${BASE_DIR}") { + artifacts.each { artifact -> + archiveArtifacts(allowEmptyArchive: true, artifacts: "${artifact}") + } + + if (junitArtifacts) { + junit(allowEmptyResults: true, + keepLongStdio: true, + testResults: "build/test-results/*.xml") + } + + if (publishCoverage) { + stashCoverageReport() + } + } + } + } + } +} + +def getCoverageBucketURI() { + return "gs://${JOB_GCS_BUCKET}/" + getCoveragePathPrefix() +} + +def getCoveragePathPrefix() { + return "${env.JOB_NAME}-${env.BUILD_ID}/test-coverage/" +} + +def publishCoverageReports() { + stage('Publish coverage reports') { + dir("${BASE_DIR}") { + def bucketUri = getCoverageBucketURI() + "*.xml" + googleStorageDownload(bucketUri: bucketUri, credentialsId: "${JOB_GCS_CREDENTIALS}", localDirectory: 'build/test-coverage', pathPrefix: getCoveragePathPrefix()) + coverageReport('build/test-coverage') + } + } +} + +def stashCoverageReport() { + r = sh(label: "isCoverageReportPresent", script: "ls build/test-coverage/*.xml", returnStatus: true) + if (r != 0) { + echo "isCoverageReportPresent: coverage files not found, report won't be stashed" + return + } + + googleStorageUploadExt(bucket: getCoverageBucketURI(), credentialsId: "${JOB_GCS_EXT_CREDENTIALS}", pattern: "build/test-coverage/*.xml") +} + +def withKubernetes(Closure body) { + retryWithSleep(retries: 2, seconds: 5, backoff: true) { sh(label: "Install kind", script: ''' + mkdir -p ${HOME}/bin + curl -sSLo ${HOME}/bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" + chmod +x ${HOME}/bin/kind + kind version + ''') } + retryWithSleep(retries: 2, seconds: 5, backoff: true) { sh(label: "Install kubectl", script: ''' + mkdir -p ${HOME}/bin + curl -sSLo ${HOME}/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl" + chmod +x ${HOME}/bin/kubectl + kubectl version --client + ''') } + withEnv(["PATH+TOOLS=${HOME}/bin"]) { + body() + } +} + +def withCloudTestEnv(Closure body) { + def maskedVars = [] + // GCP + withGCPEnv(secret: env.ELASTIC_PACKAGE_GCP_SECRET) { + maskedVars.add([var: 'GOOGLE_CREDENTIALS', password: readFile(file: env.GOOGLE_APPLICATION_CREDENTIALS)]); + maskedVars.add([var: 'GCP_PROJECT_ID', password: env.ELASTIC_OBSERVABILITY_PROJECT_ID]) + } + // Masking + withEnvMask(vars: maskedVars) { + withSecretVault(secret: "${AWS_ACCOUNT_SECRET}", + data: ['access_key': 'AWS_ACCESS_KEY_ID', 'secret_key': 'AWS_SECRET_ACCESS_KEY']) { + body() + } + } +} + +def archiveArtifactsSafe(remotePath, artifacts) { + r = sh(label: "areArtifactsPresent", script: "ls ${artifacts}", returnStatus: true) + if (r != 0) { + echo "areArtifactsPresent: artifacts files not found, nothing will be archived" + return + } + + googleStorageUploadExt( + bucket: "gs://${JOB_GCS_BUCKET_INTERNAL}/${env.JOB_NAME}-${env.BUILD_ID}/${remotePath}", + credentialsId: "${JOB_GCS_EXT_CREDENTIALS}", + pattern: artifacts) +} From 9f00c78e99b134295e105b1f73b7e29c503af84b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 15 Mar 2023 05:32:49 +0100 Subject: [PATCH 07/11] Avoid failing job in Jenkins unit tests --- .ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 46aeffd4e4..d7c8f3f320 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -72,7 +72,7 @@ pipeline { cleanup() dir("${BASE_DIR}"){ withMageEnv(){ - sh(label: 'Check',script: 'make test-go-ci') + sh(label: 'Check',script: 'curl --version; make test-go-ci || true') } } } From 9cd281bf929f6fa004c57e62c5c447ff53b69cc4 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 16 Mar 2023 05:36:56 +0100 Subject: [PATCH 08/11] Add env var for temporal base folder --- .buildkite/scripts/integration_tests.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh index 10823cf045..a7c0476788 100755 --- a/.buildkite/scripts/integration_tests.sh +++ b/.buildkite/scripts/integration_tests.sh @@ -1,10 +1,13 @@ #!/bin/bash set -euo pipefail +WORKSPACE="$(pwd)" +TMP_FOLDER_TEMPLATE_BASE="tmp.elastic-package" + cleanup() { echo "Deleting temporal files..." cd ${WORKSPACE} - rm -rf "tmp.elastic-package.*" + rm -rf "${TMP_FOLDER_TEMPLATE_BASE}.*" echo "Done." } trap cleanup EXIT @@ -20,10 +23,9 @@ usage() { source .buildkite/scripts/install_deps.sh source .buildkite/scripts/tooling.sh -WORKSPACE="$(pwd)" PARALLEL_TARGET="test-check-packages-parallel" KIND_TARGET="test-check-packages-with-kind" -TEMPLATE_TEMP_FOLDER="tmp.elastic-package.XXXXXXXXX" +TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX" GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json" JOB_GCS_BUCKET_INTERNAL="fleet-ci-temp-internal" @@ -65,7 +67,7 @@ if [[ "${TARGET}" == "" ]]; then fi google_cloud_auth_safe_logs() { - local gsUtilLocation=$(mktemp -d -p . -t ${TEMPLATE_TEMP_FOLDER}) + local gsUtilLocation=$(mktemp -d -p . -t ${TMP_FOLDER_TEMPLATE}) local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME} echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation} From 9ed0e7d3b27ed227bdaf2e58fca9aa0433c963c3 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 16 Mar 2023 22:05:23 +0100 Subject: [PATCH 09/11] Restore Jenkinsfile as in main branch --- .ci/Jenkinsfile | 232 +++--------------------------------------------- 1 file changed, 13 insertions(+), 219 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index d7c8f3f320..cbb3c7063c 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -12,16 +12,8 @@ pipeline { BASE_DIR="src/github.com/elastic/elastic-package" JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba" PIPELINE_LOG_LEVEL='INFO' - AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' HOME = "${env.WORKSPACE}" - KIND_VERSION = 'v0.17.0' - K8S_VERSION = 'v1.26.0' - JOB_GCS_BUCKET = 'fleet-ci-temp' - JOB_GCS_BUCKET_INTERNAL = 'fleet-ci-temp-internal' - JOB_GCS_CREDENTIALS = 'fleet-ci-gcs-plugin' - JOB_GCS_EXT_CREDENTIALS = 'fleet-ci-gcs-plugin-file-credentials' - ELASTIC_PACKAGE_GCP_SECRET = 'secret/observability-team/ci/service-account/elastic-package-gcp' - ELASTIC_OBSERVABILITY_PROJECT_ID = 'elastic-observability' + BUILDKITE_MERGE_COMMIT = "79ee1738578deac17f8f5caf8d925db6244ab6df" } options { timeout(time: 1, unit: 'HOURS') @@ -51,66 +43,22 @@ pipeline { stash allowEmpty: true, name: 'source', useDefaultExcludes: false } } - /** - Check the source code. - */ - stage('Check static') { - steps { - cleanup() - dir("${BASE_DIR}"){ - withMageEnv(){ - sh(label: 'Check',script: 'make check-static') - } - } + stage('Check if the PR contains Buildkite pipeline') { + when { + changeRequest() } - } - /** - Run the unit tests suite - */ - stage('Unit tests') { steps { cleanup() dir("${BASE_DIR}"){ - withMageEnv(){ - sh(label: 'Check',script: 'curl --version; make test-go-ci || true') - } - } - } - post { - always { - dir("${BASE_DIR}") { - archiveArtifacts(allowEmptyArchive: true, artifacts: 'build/test-results/*.xml') - junit(allowEmptyResults: false, keepLongStdio: true, testResults: "build/test-results/*.xml") - stashCoverageReport() - } - } - } - } - stage('Integration tests') { - failFast true - options { skipDefaultCheckout() } - steps { - withGithubNotify(context: 'Integration tests', tab: 'tests') { - cleanup() - dir("${BASE_DIR}") { - script { - def basicTasks = [ - 'stack-command-default': generateTestCommandStage(command: 'test-stack-command-default', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), - 'stack-command-oldest': generateTestCommandStage(command: 'test-stack-command-oldest', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), - 'stack-command-7x': generateTestCommandStage(command: 'test-stack-command-7x', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), - 'stack-command-86': generateTestCommandStage(command: 'test-stack-command-86', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), - 'stack-command-8x': generateTestCommandStage(command: 'test-stack-command-8x', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), - 'check-packages-with-kind': generateTestCommandStage(command: 'test-check-packages-with-kind', artifacts: ['build/test-results/*.xml', 'build/kubectl-dump.txt', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true), - 'check-packages-other': generateTestCommandStage(command: 'test-check-packages-other', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true), - 'check-packages-with-custom-agent': generateTestCommandStage(command: 'test-check-packages-with-custom-agent', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true), - 'check-packages-benchmarks': generateTestCommandStage(command: 'test-check-packages-benchmarks', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: false), - 'build-zip': generateTestCommandStage(command: 'test-build-zip', artifacts: ['build/elastic-stack-dump/build-zip/logs/*.log', 'build/packages/*.sig']), - 'profiles-command': generateTestCommandStage(command: 'test-profiles-command') - ] - - def checkSinglePackageTasks = generateTestCheckSinglePackageStage() - def tasks = basicTasks + checkSinglePackageTasks - parallel tasks + script { + githubEnv() + r = sh(label: 'Check', returnStatus: true, script: ''' + echo "Base commit: ${GIT_BASE_COMMIT}" + git log --format=%H "${GIT_BASE_COMMIT}" | grep "${BUILDKITE_MERGE_COMMIT}" + ''') + if (r != 0) { + echo("You should update your branch to include latest buildkite pipelines") + sh(label: 'Exit', script: 'exit 1') } } } @@ -127,157 +75,3 @@ def cleanup(){ } unstash 'source' } - -def generateTestCheckSinglePackageStage(Map args = [:]) { - def artifacts = ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'] - - def integrations = [:] - dir("test/packages/parallel") { - // Include hack to skip temporary files with "@tmp" suffix. - // For reference: https://issues.jenkins.io/browse/JENKINS-52750 - findFiles()?.findAll{ !it.name.endsWith('@tmp') }?.collect{ it.name }?.sort()?.each { - integrations["check-package-${it}"] = { - withNode(labels: "ubuntu-20 && immutable", sleepMax: 20, forceWorkspace: true) { - cleanup() - try { - dir("${BASE_DIR}"){ - withMageEnv(){ - withCloudTestEnv() { - sh(label: 'Install elastic-package',script: "make install") - sh(label: 'Build elastic-package',script: "make PACKAGE_UNDER_TEST=${it} test-check-packages-parallel") - } - } - } - } finally { - dir("${BASE_DIR}") { - artifacts.each { artifact -> - archiveArtifacts(allowEmptyArchive: true, artifacts: "${artifact}") - } - archiveArtifactsSafe("insecure-logs/${it}", "build/elastic-stack-dump/check-${it}/logs/elastic-agent-internal/*") - archiveArtifactsSafe("insecure-logs/${it}/container-logs", "build/container-logs/*.log") - - junit(allowEmptyResults: true, keepLongStdio: true, testResults: "build/test-results/*.xml") - stashCoverageReport() - } - } - } - } - } - } - return integrations -} - -def generateTestCommandStage(Map args = [:]){ - def command = args.get('command') - def artifacts = args.get('artifacts') ? args.get('artifacts') : [] - def junitArtifacts = args.get('junitArtifacts') ? args.get('junitArtifacts') : false - def publishCoverage = args.get('publishCoverage') ? args.get('publishCoverage') : false - - return { - withNode(labels: "ubuntu-20 && immutable", sleepMax: 20, forceWorkspace: true) { - cleanup() - try { - dir("${BASE_DIR}"){ - withMageEnv(){ - withKubernetes() { - withCloudTestEnv() { - sh(label: 'Check',script: "make install ${command} check-git-clean") - } - } - } - } - } finally { - dir("${BASE_DIR}") { - artifacts.each { artifact -> - archiveArtifacts(allowEmptyArchive: true, artifacts: "${artifact}") - } - - if (junitArtifacts) { - junit(allowEmptyResults: true, - keepLongStdio: true, - testResults: "build/test-results/*.xml") - } - - if (publishCoverage) { - stashCoverageReport() - } - } - } - } - } -} - -def getCoverageBucketURI() { - return "gs://${JOB_GCS_BUCKET}/" + getCoveragePathPrefix() -} - -def getCoveragePathPrefix() { - return "${env.JOB_NAME}-${env.BUILD_ID}/test-coverage/" -} - -def publishCoverageReports() { - stage('Publish coverage reports') { - dir("${BASE_DIR}") { - def bucketUri = getCoverageBucketURI() + "*.xml" - googleStorageDownload(bucketUri: bucketUri, credentialsId: "${JOB_GCS_CREDENTIALS}", localDirectory: 'build/test-coverage', pathPrefix: getCoveragePathPrefix()) - coverageReport('build/test-coverage') - } - } -} - -def stashCoverageReport() { - r = sh(label: "isCoverageReportPresent", script: "ls build/test-coverage/*.xml", returnStatus: true) - if (r != 0) { - echo "isCoverageReportPresent: coverage files not found, report won't be stashed" - return - } - - googleStorageUploadExt(bucket: getCoverageBucketURI(), credentialsId: "${JOB_GCS_EXT_CREDENTIALS}", pattern: "build/test-coverage/*.xml") -} - -def withKubernetes(Closure body) { - retryWithSleep(retries: 2, seconds: 5, backoff: true) { sh(label: "Install kind", script: ''' - mkdir -p ${HOME}/bin - curl -sSLo ${HOME}/bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" - chmod +x ${HOME}/bin/kind - kind version - ''') } - retryWithSleep(retries: 2, seconds: 5, backoff: true) { sh(label: "Install kubectl", script: ''' - mkdir -p ${HOME}/bin - curl -sSLo ${HOME}/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl" - chmod +x ${HOME}/bin/kubectl - kubectl version --client - ''') } - withEnv(["PATH+TOOLS=${HOME}/bin"]) { - body() - } -} - -def withCloudTestEnv(Closure body) { - def maskedVars = [] - // GCP - withGCPEnv(secret: env.ELASTIC_PACKAGE_GCP_SECRET) { - maskedVars.add([var: 'GOOGLE_CREDENTIALS', password: readFile(file: env.GOOGLE_APPLICATION_CREDENTIALS)]); - maskedVars.add([var: 'GCP_PROJECT_ID', password: env.ELASTIC_OBSERVABILITY_PROJECT_ID]) - } - // Masking - withEnvMask(vars: maskedVars) { - withSecretVault(secret: "${AWS_ACCOUNT_SECRET}", - data: ['access_key': 'AWS_ACCESS_KEY_ID', 'secret_key': 'AWS_SECRET_ACCESS_KEY']) { - body() - } - } -} - -def archiveArtifactsSafe(remotePath, artifacts) { - r = sh(label: "areArtifactsPresent", script: "ls ${artifacts}", returnStatus: true) - if (r != 0) { - echo "areArtifactsPresent: artifacts files not found, nothing will be archived" - return - } - - googleStorageUploadExt( - bucket: "gs://${JOB_GCS_BUCKET_INTERNAL}/${env.JOB_NAME}-${env.BUILD_ID}/${remotePath}", - credentialsId: "${JOB_GCS_EXT_CREDENTIALS}", - pattern: artifacts) -} From df9907c3052ac7bee3d61827b576d540445dd89b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 21 Mar 2023 11:41:00 +0100 Subject: [PATCH 10/11] Remove duplicated add_bin_path call --- .buildkite/scripts/signAndPublishPackage.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.buildkite/scripts/signAndPublishPackage.sh b/.buildkite/scripts/signAndPublishPackage.sh index 5f539cc52c..1fce10a016 100755 --- a/.buildkite/scripts/signAndPublishPackage.sh +++ b/.buildkite/scripts/signAndPublishPackage.sh @@ -16,8 +16,6 @@ cleanup() { trap cleanup EXIT -add_bin_path - isAlreadyPublished() { local packageZip=$1 From 5a3510881b3385abf3ca7f1ebc185df9c1ac2b5c Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 21 Mar 2023 11:51:01 +0100 Subject: [PATCH 11/11] Remove message if gsutil is installed --- .buildkite/scripts/signAndPublishPackage.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.buildkite/scripts/signAndPublishPackage.sh b/.buildkite/scripts/signAndPublishPackage.sh index 1fce10a016..98992dd91a 100755 --- a/.buildkite/scripts/signAndPublishPackage.sh +++ b/.buildkite/scripts/signAndPublishPackage.sh @@ -31,8 +31,6 @@ echo "Checking gsutil command..." if ! command -v gsutil &> /dev/null ; then echo "⚠️ gsutil is not installed" exit 1 -else - echo "✅ gsutil is installed" fi