From a89fb9b2fbb73742808fa419aaf98486c31d921d Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 12 Mar 2024 16:31:26 +0100 Subject: [PATCH] [Ops] Refactor env loading & fix agent-targeting rewrite (#178320) ## Summary This PR refactors a bit of the pre-command env setup, separating parts, so they can be individually skipped. Then it removes the setup-avoidance based on agent types, as this won't be useful after the migration. Also, it fixes a missed bit in the agent-targeting rewrite used for the migration, where the `provider: 'gcp'` was missing, and adds an optional targeting for the script. - add gcp as provider to all rewritten agent targeting rules - add option to target specific pipelines - refactor env-var loading to a separated file - refactor node installs so it can be switched by a flag - skip node installing in (some) jobs that don't require it --- .buildkite/hooks/post-command | 6 +- .buildkite/hooks/pre-command | 7 +- .buildkite/pipelines/emergency_release.yml | 1 + .../quality-gates/pipeline.kibana-tests.yaml | 1 + .buildkite/pipelines/sonarqube.yml | 4 + .buildkite/scripts/common/env.sh | 2 + .../scripts/common/setup_buildkite_deps.sh | 22 +++ .buildkite/scripts/common/setup_job_env.sh | 181 +++++++++++++++++ .buildkite/scripts/lifecycle/post_command.sh | 8 +- .buildkite/scripts/lifecycle/pre_command.sh | 185 +----------------- .../rewrite_buildkite_agent_rules.ts | 22 ++- 11 files changed, 247 insertions(+), 192 deletions(-) create mode 100644 .buildkite/scripts/common/setup_buildkite_deps.sh create mode 100644 .buildkite/scripts/common/setup_job_env.sh diff --git a/.buildkite/hooks/post-command b/.buildkite/hooks/post-command index 13e0f4d795828..21a4326498fc9 100755 --- a/.buildkite/hooks/post-command +++ b/.buildkite/hooks/post-command @@ -1,7 +1,3 @@ #!/usr/bin/env bash -if [[ "$BUILDKITE_AGENT_NAME" =~ ^bk-agent ]]; then - echo "Pipeline file triggered from outside the kibana executors, skipping post_command" -else - .buildkite/scripts/lifecycle/post_command.sh -fi +source .buildkite/scripts/lifecycle/post_command.sh diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 725591fdf19b4..58a2c5f0b499d 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -1,8 +1,3 @@ #!/usr/bin/env bash -if [[ "$BUILDKITE_AGENT_NAME" =~ ^bk-agent ]]; then - echo "Pipeline file triggered from outside the kibana executors, skipping .buildkite/scripts/lifecycle/pre_command.sh" - export SONAR_LOGIN=$(vault read -field=token secret/ci/elastic-kibana/sonarqube) -else - source .buildkite/scripts/lifecycle/pre_command.sh -fi +source .buildkite/scripts/lifecycle/pre_command.sh diff --git a/.buildkite/pipelines/emergency_release.yml b/.buildkite/pipelines/emergency_release.yml index 07ab52417f9a0..df9b1dea3cfb7 100644 --- a/.buildkite/pipelines/emergency_release.yml +++ b/.buildkite/pipelines/emergency_release.yml @@ -1,4 +1,5 @@ env: + SKIP_NODE_SETUP: true VERSION: ${BUILDKITE_COMMIT:0:12} EMERGENCY_RELEASE_REMOTE_SERVICE_CONFIG: https://raw.githubusercontent.com/elastic/serverless-gitops/main/gen/gpctl/kibana/emergency.yaml diff --git a/.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml b/.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml index 1217853275914..c24a1d504cd82 100644 --- a/.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml +++ b/.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml @@ -16,6 +16,7 @@ agents: memory: "8G" env: + SKIP_NODE_SETUP: true TEAM_CHANNEL: "#kibana-mission-control" ENVIRONMENT: ${ENVIRONMENT?} diff --git a/.buildkite/pipelines/sonarqube.yml b/.buildkite/pipelines/sonarqube.yml index 827c52815c1fa..61e275011140b 100644 --- a/.buildkite/pipelines/sonarqube.yml +++ b/.buildkite/pipelines/sonarqube.yml @@ -1,3 +1,7 @@ +env: + SKIP_NODE_SETUP: true + +steps: - label: ":sonarqube: Continuous Code Inspection" agents: image: docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest diff --git a/.buildkite/scripts/common/env.sh b/.buildkite/scripts/common/env.sh index 5823490ccab5e..0cdd9451171e8 100755 --- a/.buildkite/scripts/common/env.sh +++ b/.buildkite/scripts/common/env.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +echo '--- Setup environment vars' + export CI=true KIBANA_DIR=$(pwd) diff --git a/.buildkite/scripts/common/setup_buildkite_deps.sh b/.buildkite/scripts/common/setup_buildkite_deps.sh new file mode 100644 index 0000000000000..3d826d274694c --- /dev/null +++ b/.buildkite/scripts/common/setup_buildkite_deps.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -euo pipefail + +echo '--- Install/build buildkite dependencies' + +if [[ "$(type -t retry)" != "function" ]]; then + source "$(dirname "${BASH_SOURCE[0]}")/util.sh" +fi + +# `rm -rf ; npm install -g ts-node` will cause ts-node bin files to be messed up +# but literally just calling `npm install -g ts-node` a second time fixes it +# this is only on newer versions of npm +npm_install_global ts-node +if ! ts-node --version; then + npm_install_global ts-node + ts-node --version; +fi + +cd '.buildkite' +retry 5 15 npm ci +cd - diff --git a/.buildkite/scripts/common/setup_job_env.sh b/.buildkite/scripts/common/setup_job_env.sh new file mode 100644 index 0000000000000..6b07beebcfa9c --- /dev/null +++ b/.buildkite/scripts/common/setup_job_env.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash + +set -euo pipefail + +echo '--- Job Environment Setup' + +if [[ "$(type -t vault_get)" != "function" ]]; then + source .buildkite/scripts/common/vault_fns.sh +fi + +# Set up general-purpose tokens and credentials +{ + BUILDKITE_TOKEN="$(vault_get buildkite-ci buildkite_token_all_jobs)" + export BUILDKITE_TOKEN + + GITHUB_TOKEN=$(vault_get kibanamachine github_token) + export GITHUB_TOKEN + + KIBANA_CI_GITHUB_TOKEN=$(vault_get kibana-ci-github github_token) + export KIBANA_CI_GITHUB_TOKEN + + KIBANA_DOCKER_USERNAME="$(vault_get container-registry username)" + export KIBANA_DOCKER_USERNAME + + KIBANA_DOCKER_PASSWORD="$(vault_get container-registry password)" + export KIBANA_DOCKER_PASSWORD +} + +# Set up a custom ES Snapshot Manifest if one has been specified for this build +{ + ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST --default '')} + export ES_SNAPSHOT_MANIFEST + + if [[ "${ES_SNAPSHOT_MANIFEST:-}" ]]; then + cat << EOF | buildkite-agent annotate --style "info" --context es-snapshot-manifest + This build is running using a custom Elasticsearch snapshot. + + ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST + + To use this locally, simply prefix your commands with: + + \`\`\` + ES_SNAPSHOT_MANIFEST="$ES_SNAPSHOT_MANIFEST" + \`\`\` + + e.g. + + \`\`\` + ES_SNAPSHOT_MANIFEST="$ES_SNAPSHOT_MANIFEST" node scripts/functional_tests_server.js + \`\`\` +EOF + fi +} + +# If a custom manifest isn't specified, then use the default one that we resolve earlier in the build +{ + if [[ ! "${ES_SNAPSHOT_MANIFEST:-}" ]]; then + ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST_DEFAULT --default '')} + export ES_SNAPSHOT_MANIFEST + echo "Using default ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST" + fi +} + +# Setup CI Stats +{ + CI_STATS_BUILD_ID="$(buildkite-agent meta-data get ci_stats_build_id --default '')" + export CI_STATS_BUILD_ID + + CI_STATS_TOKEN="$(vault_get kibana_ci_stats api_token)" + export CI_STATS_TOKEN + + CI_STATS_HOST="$(vault_get kibana_ci_stats api_host)" + export CI_STATS_HOST + + if [[ "$CI_STATS_BUILD_ID" ]]; then + echo "CI Stats Build ID: $CI_STATS_BUILD_ID" + + KIBANA_CI_STATS_CONFIG=$(jq -n \ + --arg buildId "$CI_STATS_BUILD_ID" \ + --arg apiUrl "https://$CI_STATS_HOST" \ + --arg apiToken "$CI_STATS_TOKEN" \ + '{buildId: $buildId, apiUrl: $apiUrl, apiToken: $apiToken}' \ + ) + export KIBANA_CI_STATS_CONFIG + fi +} + +# Set up misc keys +{ + KIBANA_CI_REPORTER_KEY=$(vault_get kibanamachine-reporter value) + export KIBANA_CI_REPORTER_KEY + + EC_API_KEY="$(vault_get kibana-ci-cloud-deploy pr_deploy_api_key)" + export EC_API_KEY + + PROJECT_API_KEY="$(vault_get kibana-ci-project-deploy pr_deploy_api_key)" + export PROJECT_API_KEY + + PROJECT_API_DOMAIN="$(vault_get kibana-ci-project-deploy pr_deploy_domain)" + export PROJECT_API_DOMAIN + + SYNTHETICS_SERVICE_USERNAME="$(vault_get kibana-ci-synthetics-credentials username)" + export SYNTHETICS_SERVICE_USERNAME + + SYNTHETICS_SERVICE_PASSWORD="$(vault_get kibana-ci-synthetics-credentials password)" + export SYNTHETICS_SERVICE_PASSWORD + + SYNTHETICS_SERVICE_MANIFEST="$(vault_get kibana-ci-synthetics-credentials manifest)" + export SYNTHETICS_SERVICE_MANIFEST + + SYNTHETICS_REMOTE_KIBANA_USERNAME="$(vault_get kibana-ci-synthetics-remote-credentials username)" + export SYNTHETICS_REMOTE_KIBANA_USERNAME + + SYNTHETICS_REMOTE_KIBANA_PASSWORD="$(vault_get kibana-ci-synthetics-remote-credentials password)" + export SYNTHETICS_REMOTE_KIBANA_PASSWORD + + SYNTHETICS_REMOTE_KIBANA_URL=${SYNTHETICS_REMOTE_KIBANA_URL-"$(vault_get kibana-ci-synthetics-remote-credentials url)"} + export SYNTHETICS_REMOTE_KIBANA_URL + + DEPLOY_TAGGER_SLACK_WEBHOOK_URL=${DEPLOY_TAGGER_SLACK_WEBHOOK_URL:-"$(vault_get kibana-serverless-release-tools DEPLOY_TAGGER_SLACK_WEBHOOK_URL)"} + export DEPLOY_TAGGER_SLACK_WEBHOOK_URL + + SONAR_LOGIN=$(vault_get sonarqube token) + export SONAR_LOGIN + + ELASTIC_APM_SERVER_URL=$(vault_get project-kibana-ci-apm apm_server_url) + export ELASTIC_APM_SERVER_URL + + ELASTIC_APM_API_KEY=$(vault_get project-kibana-ci-apm apm_server_api_key) + export ELASTIC_APM_API_KEY +} + +# Set up GCS Service Account for CDN +{ + GCS_SA_CDN_KEY="$(vault_get gcs-sa-cdn-prod key)" + export GCS_SA_CDN_KEY + + GCS_SA_CDN_EMAIL="$(vault_get gcs-sa-cdn-prod email)" + export GCS_SA_CDN_EMAIL + + GCS_SA_CDN_BUCKET="$(vault_get gcs-sa-cdn-prod bucket)" + export GCS_SA_CDN_BUCKET + + GCS_SA_CDN_URL="$(vault_get gcs-sa-cdn-prod cdn)" + export GCS_SA_CDN_URL +} + +# Setup Failed Test Reporter Elasticsearch credentials +{ + TEST_FAILURES_ES_CLOUD_ID=$(vault_get failed_tests_reporter_es cloud_id) + export TEST_FAILURES_ES_CLOUD_ID + + TEST_FAILURES_ES_USERNAME=$(vault_get failed_tests_reporter_es username) + export TEST_FAILURES_ES_USERNAME + + TEST_FAILURES_ES_PASSWORD=$(vault_get failed_tests_reporter_es password) + export TEST_FAILURES_ES_PASSWORD +} + +# Setup Bazel Remote/Local Cache Credentials +{ + BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-local-dev.json" + export BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE + vault_get kibana-ci-bazel-remote-cache-local-dev service_account_json > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE" + + BAZEL_REMOTE_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-gcs.json" + export BAZEL_REMOTE_CACHE_CREDENTIALS_FILE + vault_get kibana-ci-bazel-remote-cache-sa-key key | base64 -d > "$BAZEL_REMOTE_CACHE_CREDENTIALS_FILE" +} + +# Setup GCS Service Account Proxy for CI +{ + KIBANA_SERVICE_ACCOUNT_PROXY_KEY="$(mktemp -d)/kibana-gcloud-service-account.json" + export KIBANA_SERVICE_ACCOUNT_PROXY_KEY + vault_get kibana-ci-sa-proxy-key key | base64 -d > "$KIBANA_SERVICE_ACCOUNT_PROXY_KEY" +} + +PIPELINE_PRE_COMMAND=${PIPELINE_PRE_COMMAND:-".buildkite/scripts/lifecycle/pipelines/$BUILDKITE_PIPELINE_SLUG/pre_command.sh"} +if [[ -f "$PIPELINE_PRE_COMMAND" ]]; then + source "$PIPELINE_PRE_COMMAND" +fi diff --git a/.buildkite/scripts/lifecycle/post_command.sh b/.buildkite/scripts/lifecycle/post_command.sh index 91854422855a4..218184d36f65a 100755 --- a/.buildkite/scripts/lifecycle/post_command.sh +++ b/.buildkite/scripts/lifecycle/post_command.sh @@ -6,8 +6,12 @@ echo '--- Log out of gcloud' ./.buildkite/scripts/common/activate_service_account.sh --unset-impersonation || echo "Failed to unset impersonation" ./.buildkite/scripts/common/activate_service_account.sh --logout-gcloud || echo "Failed to log out of gcloud" -echo '--- Agent Debug Info' -ts-node .buildkite/scripts/lifecycle/print_agent_links.ts || true +if [[ "${SKIP_NODE_SETUP:-}" =~ ^(1|true)$ ]]; then + echo '--- Skipping Agent Debug Info' +else + echo '--- Agent Debug Info' + ts-node .buildkite/scripts/lifecycle/print_agent_links.ts || true +fi IS_TEST_EXECUTION_STEP="$(buildkite-agent meta-data get "${BUILDKITE_JOB_ID}_is_test_execution_step" --default '')" diff --git a/.buildkite/scripts/lifecycle/pre_command.sh b/.buildkite/scripts/lifecycle/pre_command.sh index 1e77fa11303ce..0ac12e62a7b0d 100755 --- a/.buildkite/scripts/lifecycle/pre_command.sh +++ b/.buildkite/scripts/lifecycle/pre_command.sh @@ -3,191 +3,22 @@ set -euo pipefail source .buildkite/scripts/common/util.sh - -echo '--- Setup environment vars' source .buildkite/scripts/common/env.sh -source .buildkite/scripts/common/setup_node.sh - -BUILDKITE_TOKEN="$(vault_get buildkite-ci buildkite_token_all_jobs)" -export BUILDKITE_TOKEN +source .buildkite/scripts/common/setup_job_env.sh -echo '--- Install/build buildkite dependencies' +if [[ "${SKIP_NODE_SETUP:-}" =~ ^(1|true)$ ]]; then + echo "Skipping node setup (SKIP_NODE_SETUP=$SKIP_NODE_SETUP)" +else + source .buildkite/scripts/common/setup_node.sh + source .buildkite/scripts/common/setup_buildkite_deps.sh -# `rm -rf ; npm install -g ts-node` will cause ts-node bin files to be messed up -# but literally just calling `npm install -g ts-node` a second time fixes it -# this is only on newer versions of npm -npm_install_global ts-node -if ! ts-node --version; then - npm_install_global ts-node - ts-node --version; + echo '--- Agent Debug/SSH Info' + ts-node .buildkite/scripts/lifecycle/print_agent_links.ts || true fi -cd '.buildkite' -retry 5 15 npm ci -cd .. - -echo '--- Agent Debug/SSH Info' -ts-node .buildkite/scripts/lifecycle/print_agent_links.ts || true - if [[ "$(curl -is metadata.google.internal || true)" ]]; then echo "" echo "To SSH into this agent, run:" echo "gcloud compute ssh --tunnel-through-iap --project elastic-kibana-ci --zone \"$(curl -sH Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/zone)\" \"$(curl -sH Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/name)\"" echo "" fi - -echo '--- Job Environment Setup' - -# Set up a custom ES Snapshot Manifest if one has been specified for this build -{ - ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST --default '')} - export ES_SNAPSHOT_MANIFEST - - if [[ "${ES_SNAPSHOT_MANIFEST:-}" ]]; then - cat << EOF | buildkite-agent annotate --style "info" --context es-snapshot-manifest - This build is running using a custom Elasticsearch snapshot. - - ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST - - To use this locally, simply prefix your commands with: - - \`\`\` - ES_SNAPSHOT_MANIFEST="$ES_SNAPSHOT_MANIFEST" - \`\`\` - - e.g. - - \`\`\` - ES_SNAPSHOT_MANIFEST="$ES_SNAPSHOT_MANIFEST" node scripts/functional_tests_server.js - \`\`\` -EOF - fi -} - -# If a custom manifest isn't specified, then use the default one that we resolve earlier in the build -{ - if [[ ! "${ES_SNAPSHOT_MANIFEST:-}" ]]; then - ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST_DEFAULT --default '')} - export ES_SNAPSHOT_MANIFEST - echo "Using default ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST" - fi -} - -# Setup CI Stats -{ - CI_STATS_BUILD_ID="$(buildkite-agent meta-data get ci_stats_build_id --default '')" - export CI_STATS_BUILD_ID - - CI_STATS_TOKEN="$(vault_get kibana_ci_stats api_token)" - export CI_STATS_TOKEN - - CI_STATS_HOST="$(vault_get kibana_ci_stats api_host)" - export CI_STATS_HOST - - if [[ "$CI_STATS_BUILD_ID" ]]; then - echo "CI Stats Build ID: $CI_STATS_BUILD_ID" - - KIBANA_CI_STATS_CONFIG=$(jq -n \ - --arg buildId "$CI_STATS_BUILD_ID" \ - --arg apiUrl "https://$CI_STATS_HOST" \ - --arg apiToken "$CI_STATS_TOKEN" \ - '{buildId: $buildId, apiUrl: $apiUrl, apiToken: $apiToken}' \ - ) - export KIBANA_CI_STATS_CONFIG - fi -} - -GITHUB_TOKEN=$(vault_get kibanamachine github_token) -export GITHUB_TOKEN - -KIBANA_CI_GITHUB_TOKEN=$(vault_get kibana-ci-github github_token) -export KIBANA_CI_GITHUB_TOKEN - -KIBANA_CI_REPORTER_KEY=$(vault_get kibanamachine-reporter value) -export KIBANA_CI_REPORTER_KEY - -KIBANA_DOCKER_USERNAME="$(vault_get container-registry username)" -export KIBANA_DOCKER_USERNAME - -KIBANA_DOCKER_PASSWORD="$(vault_get container-registry password)" -export KIBANA_DOCKER_PASSWORD - -EC_API_KEY="$(vault_get kibana-ci-cloud-deploy pr_deploy_api_key)" -export EC_API_KEY - -PROJECT_API_KEY="$(vault_get kibana-ci-project-deploy pr_deploy_api_key)" -export PROJECT_API_KEY - -PROJECT_API_DOMAIN="$(vault_get kibana-ci-project-deploy pr_deploy_domain)" -export PROJECT_API_DOMAIN - -SYNTHETICS_SERVICE_USERNAME="$(vault_get kibana-ci-synthetics-credentials username)" -export SYNTHETICS_SERVICE_USERNAME - -SYNTHETICS_SERVICE_PASSWORD="$(vault_get kibana-ci-synthetics-credentials password)" -export SYNTHETICS_SERVICE_PASSWORD - -SYNTHETICS_SERVICE_MANIFEST="$(vault_get kibana-ci-synthetics-credentials manifest)" -export SYNTHETICS_SERVICE_MANIFEST - -SYNTHETICS_REMOTE_KIBANA_USERNAME="$(vault_get kibana-ci-synthetics-remote-credentials username)" -export SYNTHETICS_REMOTE_KIBANA_USERNAME - -SYNTHETICS_REMOTE_KIBANA_PASSWORD="$(vault_get kibana-ci-synthetics-remote-credentials password)" -export SYNTHETICS_REMOTE_KIBANA_PASSWORD - -SYNTHETICS_REMOTE_KIBANA_URL=${SYNTHETICS_REMOTE_KIBANA_URL-"$(vault_get kibana-ci-synthetics-remote-credentials url)"} -export SYNTHETICS_REMOTE_KIBANA_URL - -DEPLOY_TAGGER_SLACK_WEBHOOK_URL=${DEPLOY_TAGGER_SLACK_WEBHOOK_URL:-"$(vault_get kibana-serverless-release-tools DEPLOY_TAGGER_SLACK_WEBHOOK_URL)"} -export DEPLOY_TAGGER_SLACK_WEBHOOK_URL - -GCS_SA_CDN_KEY="$(vault_get gcs-sa-cdn-prod key)" -export GCS_SA_CDN_KEY - -GCS_SA_CDN_EMAIL="$(vault_get gcs-sa-cdn-prod email)" -export GCS_SA_CDN_EMAIL - -GCS_SA_CDN_BUCKET="$(vault_get gcs-sa-cdn-prod bucket)" -export GCS_SA_CDN_BUCKET - -GCS_SA_CDN_URL="$(vault_get gcs-sa-cdn-prod cdn)" -export GCS_SA_CDN_URL - - -ELASTIC_APM_SERVER_URL=$(vault_get project-kibana-ci-apm apm_server_url) -export ELASTIC_APM_SERVER_URL - -ELASTIC_APM_API_KEY=$(vault_get project-kibana-ci-apm apm_server_api_key) -export ELASTIC_APM_API_KEY - -# Setup Failed Test Reporter Elasticsearch credentials -{ - TEST_FAILURES_ES_CLOUD_ID=$(vault_get failed_tests_reporter_es cloud_id) - export TEST_FAILURES_ES_CLOUD_ID - - TEST_FAILURES_ES_USERNAME=$(vault_get failed_tests_reporter_es username) - export TEST_FAILURES_ES_USERNAME - - TEST_FAILURES_ES_PASSWORD=$(vault_get failed_tests_reporter_es password) - export TEST_FAILURES_ES_PASSWORD -} - -BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-local-dev.json" -export BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE -vault_get kibana-ci-bazel-remote-cache-local-dev service_account_json > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE" - -# Export key for accessing bazel remote cache's GCS bucket -BAZEL_REMOTE_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-gcs.json" -export BAZEL_REMOTE_CACHE_CREDENTIALS_FILE -vault_get kibana-ci-bazel-remote-cache-sa-key key | base64 -d > "$BAZEL_REMOTE_CACHE_CREDENTIALS_FILE" - -# Setup GCS Service Account Proxy for CI -KIBANA_SERVICE_ACCOUNT_PROXY_KEY="$(mktemp -d)/kibana-gcloud-service-account.json" -export KIBANA_SERVICE_ACCOUNT_PROXY_KEY -vault_get kibana-ci-sa-proxy-key key | base64 -d > "$KIBANA_SERVICE_ACCOUNT_PROXY_KEY" - -PIPELINE_PRE_COMMAND=${PIPELINE_PRE_COMMAND:-".buildkite/scripts/lifecycle/pipelines/$BUILDKITE_PIPELINE_SLUG/pre_command.sh"} -if [[ -f "$PIPELINE_PRE_COMMAND" ]]; then - source "$PIPELINE_PRE_COMMAND" -fi diff --git a/src/dev/buildkite_migration/rewrite_buildkite_agent_rules.ts b/src/dev/buildkite_migration/rewrite_buildkite_agent_rules.ts index 00843a474d510..64d94bda5a970 100644 --- a/src/dev/buildkite_migration/rewrite_buildkite_agent_rules.ts +++ b/src/dev/buildkite_migration/rewrite_buildkite_agent_rules.ts @@ -50,6 +50,7 @@ interface GobldGCPConfig { enableSecureBoot?: boolean; enableNestedVirtualization?: boolean; image: string; + provider: 'gcp'; localSsds?: number; localSsdInterface?: string; machineType: string; @@ -75,16 +76,32 @@ if (!fs.existsSync('data/agents.json')) { * rewrites all agent targeting rules from the shorthands to the full targeting syntax */ run( - async ({ log, flags }) => { + async ({ log, flags, flagsReader }) => { + const filterExpressions = flagsReader.getPositionals(); + const paths = await globby('.buildkite/**/*.yml', { cwd: REPO_ROOT, onlyFiles: true, gitignore: true, }); + const pathsFiltered = + filterExpressions.length === 0 + ? paths + : paths.filter((path) => { + return filterExpressions.some((expression) => path.includes(expression)); + }); + + if (pathsFiltered.length === 0) { + log.warning('No .yml files found to rewrite after filtering.'); + return; + } + + log.info('Applying rewrite to the following paths: \n', pathsFiltered.join('\n')); + const failedRewrites: Array<{ path: string; error: Error }> = []; - const rewritePromises: Array> = paths.map((ymlPath) => { + const rewritePromises: Array> = pathsFiltered.map((ymlPath) => { return rewriteFile(ymlPath, log).catch((e) => { // eslint-disable-next-line no-console console.error('Failed to rewrite: ' + ymlPath, e); @@ -193,6 +210,7 @@ function getFullAgentTargetingRule(queue: string): GobldGCPConfig { return removeNullish({ image: 'family/kibana-ubuntu-2004', imageProject: 'elastic-images-qa', + provider: 'gcp', assignExternalIP: agent.disableExternalIp === true ? false : undefined, diskSizeGb: agent.diskSizeGb, diskType: agent.diskType,