diff --git a/.buildkite/hooks/post-command b/.buildkite/hooks/post-command index d9a8ad668da0b5..21a4326498fc9d 100755 --- a/.buildkite/hooks/post-command +++ b/.buildkite/hooks/post-command @@ -1,3 +1,3 @@ #!/usr/bin/env bash -.buildkite/scripts/lifecycle/post_command.sh +source .buildkite/scripts/lifecycle/post_command.sh diff --git a/.buildkite/scripts/common/activate_service_account.sh b/.buildkite/scripts/common/activate_service_account.sh new file mode 100644 index 00000000000000..5569367fd281b7 --- /dev/null +++ b/.buildkite/scripts/common/activate_service_account.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/vault_fns.sh" + +CALL_ARGUMENT="${1:-}" +GCLOUD_EMAIL_POSTFIX="elastic-kibana-ci.iam.gserviceaccount.com" +GCLOUD_SA_PROXY_EMAIL="kibana-ci-sa-proxy@$GCLOUD_EMAIL_POSTFIX" + +if [[ -z "$CALL_ARGUMENT" ]]; then + echo "Usage: $0 " + exit 1 +elif [[ "$CALL_ARGUMENT" == "--unset-impersonation" ]]; then + echo "Unsetting impersonation" + gcloud config unset auth/impersonate_service_account + exit 0 +elif [[ "$CALL_ARGUMENT" == "--logout-gcloud" ]]; then + echo "Logging out of gcloud" + if [[ -x "$(command -v gcloud)" ]] && [[ "$(gcloud auth list 2>/dev/null | grep $GCLOUD_SA_PROXY_EMAIL)" != "" ]]; then + gcloud auth revoke $GCLOUD_SA_PROXY_EMAIL --no-user-output-enabled + fi + exit 0 +fi + +CURRENT_GCLOUD_USER=$(gcloud auth list --filter="status=ACTIVE" --format="value(account)") + +# Verify that the service account proxy is activated +if [[ "$CURRENT_GCLOUD_USER" != "$GCLOUD_SA_PROXY_EMAIL" ]]; then + if [[ -x "$(command -v gcloud)" ]]; then + if [[ -z "${KIBANA_SERVICE_ACCOUNT_PROXY_KEY:-}" ]]; then + echo "KIBANA_SERVICE_ACCOUNT_PROXY_KEY is not set, cannot activate service account $GCLOUD_SA_PROXY_EMAIL." + exit 1 + fi + + AUTH_RESULT=$(gcloud auth activate-service-account --key-file="$KIBANA_SERVICE_ACCOUNT_PROXY_KEY" || "FAILURE") + if [[ "$AUTH_RESULT" == "FAILURE" ]]; then + echo "Failed to activate service account $GCLOUD_SA_PROXY_EMAIL." + exit 1 + else + echo "Activated service account $GCLOUD_SA_PROXY_EMAIL" + fi + else + echo "gcloud is not installed, cannot activate service account $GCLOUD_SA_PROXY_EMAIL." + exit 1 + fi +fi + +# Check if the arg is a service account e-mail or a bucket name +EMAIL="" +if [[ "$CALL_ARGUMENT" =~ ^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$ ]]; then + EMAIL="$CALL_ARGUMENT" +elif [[ "$CALL_ARGUMENT" =~ ^gs://* ]]; then + BUCKET_NAME="${CALL_ARGUMENT:5}" +else + BUCKET_NAME="$CALL_ARGUMENT" +fi + +if [[ -z "$EMAIL" ]]; then + case "$BUCKET_NAME" in + "elastic-kibana-coverage-live") + EMAIL="kibana-ci-access-coverage@$GCLOUD_EMAIL_POSTFIX" + ;; + "kibana-ci-es-snapshots-daily") + EMAIL="kibana-ci-access-es-daily@$GCLOUD_EMAIL_POSTFIX" + ;; + "kibana-ci-es-snapshots-permanent") + EMAIL="kibana-ci-access-es-permanent@$GCLOUD_EMAIL_POSTFIX" + ;; + "kibana-so-types-snapshots") + EMAIL="kibana-ci-access-so-snapshots@$GCLOUD_EMAIL_POSTFIX" + ;; + "kibana-performance") + EMAIL="kibana-ci-access-perf-stats@$GCLOUD_EMAIL_POSTFIX" + ;; + "ci-artifacts.kibana.dev") + EMAIL="kibana-ci-access-artifacts@$GCLOUD_EMAIL_POSTFIX" + ;; + *) + EMAIL="$BUCKET_NAME@$GCLOUD_EMAIL_POSTFIX" + ;; + esac +fi + +# Activate the service account +echo "Impersonating $EMAIL" +gcloud config set auth/impersonate_service_account "$EMAIL" +echo "Activated service account $EMAIL" diff --git a/.buildkite/scripts/common/env.sh b/.buildkite/scripts/common/env.sh index f375dda90751b9..657578f0e93f6a 100755 --- a/.buildkite/scripts/common/env.sh +++ b/.buildkite/scripts/common/env.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +echo '--- Setup environment vars' + export CI=true KIBANA_DIR=$(pwd) diff --git a/.buildkite/scripts/common/setup_buildkite_deps.sh b/.buildkite/scripts/common/setup_buildkite_deps.sh new file mode 100644 index 00000000000000..1e7fd65fb69096 --- /dev/null +++ b/.buildkite/scripts/common/setup_buildkite_deps.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -euo pipefail + +echo '--- Install/build buildkite dependencies' + +cd '.buildkite' +retry 5 15 npm ci +cd - diff --git a/.buildkite/scripts/common/setup_job_env.sh b/.buildkite/scripts/common/setup_job_env.sh new file mode 100644 index 00000000000000..64666d022c976b --- /dev/null +++ b/.buildkite/scripts/common/setup_job_env.sh @@ -0,0 +1,175 @@ +#!/usr/bin/env bash + +set -euo pipefail + +echo '--- Job Environment Setup' + +if [[ "$(type -t vault_get)" != "function" ]]; then + source .buildkite/scripts/common/vault_fns.sh +fi + +# Set up general-purpose tokens and credentials +{ + BUILDKITE_TOKEN="$(vault_get buildkite-ci buildkite_token_all_jobs)" + export BUILDKITE_TOKEN + + GITHUB_TOKEN=$(vault_get kibanamachine github_token) + export GITHUB_TOKEN + + KIBANA_CI_GITHUB_TOKEN=$(vault_get kibana-ci-github github_token) + export KIBANA_CI_GITHUB_TOKEN + + KIBANA_DOCKER_USERNAME="$(vault_get container-registry username)" + export KIBANA_DOCKER_USERNAME + + KIBANA_DOCKER_PASSWORD="$(vault_get container-registry password)" + export KIBANA_DOCKER_PASSWORD +} + +# Set up a custom ES Snapshot Manifest if one has been specified for this build +{ + ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST --default '')} + export ES_SNAPSHOT_MANIFEST + + if [[ "${ES_SNAPSHOT_MANIFEST:-}" ]]; then + cat << EOF | buildkite-agent annotate --style "info" --context es-snapshot-manifest + This build is running using a custom Elasticsearch snapshot. + + ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST + + To use this locally, simply prefix your commands with: + + \`\`\` + ES_SNAPSHOT_MANIFEST="$ES_SNAPSHOT_MANIFEST" + \`\`\` + + e.g. + + \`\`\` + ES_SNAPSHOT_MANIFEST="$ES_SNAPSHOT_MANIFEST" node scripts/functional_tests_server.js + \`\`\` +EOF + fi +} + +# If a custom manifest isn't specified, then use the default one that we resolve earlier in the build +{ + if [[ ! "${ES_SNAPSHOT_MANIFEST:-}" ]]; then + ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST_DEFAULT --default '')} + export ES_SNAPSHOT_MANIFEST + echo "Using default ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST" + fi +} + +# Setup CI Stats +{ + CI_STATS_BUILD_ID="$(buildkite-agent meta-data get ci_stats_build_id --default '')" + export CI_STATS_BUILD_ID + + CI_STATS_TOKEN="$(vault_get kibana_ci_stats api_token)" + export CI_STATS_TOKEN + + CI_STATS_HOST="$(vault_get kibana_ci_stats api_host)" + export CI_STATS_HOST + + if [[ "$CI_STATS_BUILD_ID" ]]; then + echo "CI Stats Build ID: $CI_STATS_BUILD_ID" + + KIBANA_CI_STATS_CONFIG=$(jq -n \ + --arg buildId "$CI_STATS_BUILD_ID" \ + --arg apiUrl "https://$CI_STATS_HOST" \ + --arg apiToken "$CI_STATS_TOKEN" \ + '{buildId: $buildId, apiUrl: $apiUrl, apiToken: $apiToken}' \ + ) + export KIBANA_CI_STATS_CONFIG + fi +} + +# Set up misc keys +{ + KIBANA_CI_REPORTER_KEY=$(vault_get kibanamachine-reporter value) + export KIBANA_CI_REPORTER_KEY + + EC_API_KEY="$(vault_get kibana-ci-cloud-deploy pr_deploy_api_key)" + export EC_API_KEY + + PROJECT_API_KEY="$(vault_get kibana-ci-project-deploy pr_deploy_api_key)" + export PROJECT_API_KEY + + PROJECT_API_DOMAIN="$(vault_get kibana-ci-project-deploy pr_deploy_domain)" + export PROJECT_API_DOMAIN + + SYNTHETICS_SERVICE_USERNAME="$(vault_get kibana-ci-synthetics-credentials username)" + export SYNTHETICS_SERVICE_USERNAME + + SYNTHETICS_SERVICE_PASSWORD="$(vault_get kibana-ci-synthetics-credentials password)" + export SYNTHETICS_SERVICE_PASSWORD + + SYNTHETICS_SERVICE_MANIFEST="$(vault_get kibana-ci-synthetics-credentials manifest)" + export SYNTHETICS_SERVICE_MANIFEST + + SYNTHETICS_REMOTE_KIBANA_USERNAME="$(vault_get kibana-ci-synthetics-remote-credentials username)" + export SYNTHETICS_REMOTE_KIBANA_USERNAME + + SYNTHETICS_REMOTE_KIBANA_PASSWORD="$(vault_get kibana-ci-synthetics-remote-credentials password)" + export SYNTHETICS_REMOTE_KIBANA_PASSWORD + + SYNTHETICS_REMOTE_KIBANA_URL=${SYNTHETICS_REMOTE_KIBANA_URL-"$(vault_get kibana-ci-synthetics-remote-credentials url)"} + export SYNTHETICS_REMOTE_KIBANA_URL + + DEPLOY_TAGGER_SLACK_WEBHOOK_URL=${DEPLOY_TAGGER_SLACK_WEBHOOK_URL:-"$(vault_get kibana-serverless-release-tools DEPLOY_TAGGER_SLACK_WEBHOOK_URL)"} + export DEPLOY_TAGGER_SLACK_WEBHOOK_URL + + SONAR_LOGIN=$(vault_get sonarqube token) + export SONAR_LOGIN +} + +# Set up GCS Service Account for CDN +{ + GCS_SA_CDN_KEY="$(vault_get gcs-sa-cdn-prod key)" + export GCS_SA_CDN_KEY + + GCS_SA_CDN_EMAIL="$(vault_get gcs-sa-cdn-prod email)" + export GCS_SA_CDN_EMAIL + + GCS_SA_CDN_BUCKET="$(vault_get gcs-sa-cdn-prod bucket)" + export GCS_SA_CDN_BUCKET + + GCS_SA_CDN_URL="$(vault_get gcs-sa-cdn-prod cdn)" + export GCS_SA_CDN_URL +} + +# Setup Failed Test Reporter Elasticsearch credentials +{ + TEST_FAILURES_ES_CLOUD_ID=$(vault_get failed_tests_reporter_es cloud_id) + export TEST_FAILURES_ES_CLOUD_ID + + TEST_FAILURES_ES_USERNAME=$(vault_get failed_tests_reporter_es username) + export TEST_FAILURES_ES_USERNAME + + TEST_FAILURES_ES_PASSWORD=$(vault_get failed_tests_reporter_es password) + export TEST_FAILURES_ES_PASSWORD +} + +# Setup Bazel Remote/Local Cache Credentials +{ + BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-local-dev.json" + export BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE + vault_get kibana-ci-bazel-remote-cache-local-dev service_account_json > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE" + + BAZEL_REMOTE_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-gcs.json" + export BAZEL_REMOTE_CACHE_CREDENTIALS_FILE + vault_get kibana-ci-bazel-remote-cache-sa-key key | base64 -d > "$BAZEL_REMOTE_CACHE_CREDENTIALS_FILE" +} + +# Setup GCS Service Account Proxy for CI +{ + KIBANA_SERVICE_ACCOUNT_PROXY_KEY="$(mktemp -d)/kibana-gcloud-service-account.json" + export KIBANA_SERVICE_ACCOUNT_PROXY_KEY + vault_get kibana-ci-sa-proxy-key key | base64 -d > "$KIBANA_SERVICE_ACCOUNT_PROXY_KEY" +} + +PIPELINE_PRE_COMMAND=${PIPELINE_PRE_COMMAND:-".buildkite/scripts/lifecycle/pipelines/$BUILDKITE_PIPELINE_SLUG/pre_command.sh"} +if [[ -f "$PIPELINE_PRE_COMMAND" ]]; then + source "$PIPELINE_PRE_COMMAND" +fi diff --git a/.buildkite/scripts/common/util.sh b/.buildkite/scripts/common/util.sh index 18fa1b1d79000e..ca169f2adc0a8a 100755 --- a/.buildkite/scripts/common/util.sh +++ b/.buildkite/scripts/common/util.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +source "$(dirname "${BASH_SOURCE[0]}")/vault_fns.sh" + checks-reporter-with-killswitch() { if [ "$CHECKS_REPORTER_ACTIVE" == "true" ] ; then yarn run github-checks-reporter "$@" diff --git a/.buildkite/scripts/common/vault_fns.sh b/.buildkite/scripts/common/vault_fns.sh new file mode 100644 index 00000000000000..a7b92a4b05d6d9 --- /dev/null +++ b/.buildkite/scripts/common/vault_fns.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done +if [[ "${VAULT_ADDR:-}" == *"secrets.elastic.co"* ]]; then + VAULT_PATH_PREFIX="secret/kibana-issues/dev" + VAULT_KV_PREFIX="secret/kibana-issues/dev" + IS_LEGACY_VAULT_ADDR=true +else + VAULT_PATH_PREFIX="secret/ci/elastic-kibana" + VAULT_KV_PREFIX="kv/ci-shared/kibana-deployments" + IS_LEGACY_VAULT_ADDR=false +fi +export IS_LEGACY_VAULT_ADDR + +retry() { + local retries=$1; shift + local delay=$1; shift + local attempts=1 + + until "$@"; do + retry_exit_status=$? + echo "Exited with $retry_exit_status" >&2 + if (( retries == "0" )); then + return $retry_exit_status + elif (( attempts == retries )); then + echo "Failed $attempts retries" >&2 + return $retry_exit_status + else + echo "Retrying $((retries - attempts)) more times..." >&2 + attempts=$((attempts + 1)) + sleep "$delay" + fi + done +} + +vault_get() { + key_path=${1:-} + field=${2:-} + + fullPath="$VAULT_PATH_PREFIX/$key_path" + + if [[ -z "$field" || "$field" =~ ^-.* ]]; then + retry 5 5 vault read "$fullPath" "${@:2}" + else + retry 5 5 vault read -field="$field" "$fullPath" "${@:3}" + fi +} + +vault_set() { + key_path=$1 + shift + fields=("$@") + + + fullPath="$VAULT_PATH_PREFIX/$key_path" + + # shellcheck disable=SC2068 + retry 5 5 vault write "$fullPath" ${fields[@]} +} + +vault_kv_set() { + kv_path=$1 + shift + fields=("$@") + + vault kv put "$VAULT_KV_PREFIX/$kv_path" "${fields[@]}" +} diff --git a/.buildkite/scripts/lifecycle/post_command.sh b/.buildkite/scripts/lifecycle/post_command.sh index f111b3807745aa..e5bc6b5e3a63c4 100755 --- a/.buildkite/scripts/lifecycle/post_command.sh +++ b/.buildkite/scripts/lifecycle/post_command.sh @@ -2,6 +2,10 @@ set -euo pipefail +echo '--- Log out of gcloud' +./.buildkite/scripts/common/activate_service_account.sh --unset-impersonation || echo "Failed to unset impersonation" +./.buildkite/scripts/common/activate_service_account.sh --logout-gcloud || echo "Failed to log out of gcloud" + echo '--- Agent Debug Info' node .buildkite/scripts/lifecycle/print_agent_links.js || true diff --git a/.buildkite/scripts/lifecycle/pre_command.sh b/.buildkite/scripts/lifecycle/pre_command.sh index 2e5f692c354564..dd5a23cee14b97 100755 --- a/.buildkite/scripts/lifecycle/pre_command.sh +++ b/.buildkite/scripts/lifecycle/pre_command.sh @@ -3,24 +3,16 @@ set -euo pipefail source .buildkite/scripts/common/util.sh - -# By default, all steps should set up these things to get a full environment before running -# It can be skipped for pipeline upload steps though, to make job start time a little faster -if [[ "${SKIP_CI_SETUP:-}" != "true" ]]; then - if [[ -d .buildkite/scripts && "${BUILDKITE_COMMAND:-}" != "buildkite-agent pipeline upload"* ]]; then - source .buildkite/scripts/common/env.sh - source .buildkite/scripts/common/setup_node.sh - fi +source .buildkite/scripts/common/env.sh +source .buildkite/scripts/common/setup_job_env.sh + +if [[ "${SKIP_CI_SETUP:-}" == "true" || "${SKIP_NODE_SETUP:-}" =~ ^(1|true)$ ]]; then + echo "Skipping node setup (SKIP_NODE_SETUP=$SKIP_NODE_SETUP)" +else + source .buildkite/scripts/common/setup_node.sh + source .buildkite/scripts/common/setup_buildkite_deps.sh fi -BUILDKITE_TOKEN="$(retry 5 5 vault read -field=buildkite_token_all_jobs secret/kibana-issues/dev/buildkite-ci)" -export BUILDKITE_TOKEN - -echo '--- Install buildkite dependencies' -cd '.buildkite' -retry 5 15 npm ci -cd .. - echo '--- Agent Debug/SSH Info' node .buildkite/scripts/lifecycle/print_agent_links.js || true @@ -30,99 +22,3 @@ if [[ "$(curl -is metadata.google.internal || true)" ]]; then echo "gcloud compute ssh --tunnel-through-iap --project elastic-kibana-ci --zone \"$(curl -sH Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/zone)\" \"$(curl -sH Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/name)\"" echo "" fi - - -echo '--- Job Environment Setup' - -# Set up a custom ES Snapshot Manifest if one has been specified for this build -{ - ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST --default '')} - export ES_SNAPSHOT_MANIFEST - - if [[ "${ES_SNAPSHOT_MANIFEST:-}" ]]; then - cat << EOF | buildkite-agent annotate --style "info" --context es-snapshot-manifest - This build is running using a custom Elasticsearch snapshot. - - ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST - - To use this locally, simply prefix your commands with: - - \`\`\` - ES_SNAPSHOT_MANIFEST="$ES_SNAPSHOT_MANIFEST" - \`\`\` - - e.g. - - \`\`\` - ES_SNAPSHOT_MANIFEST="$ES_SNAPSHOT_MANIFEST" node scripts/functional_tests_server.js - \`\`\` -EOF - fi -} - -# If a custom manifest isn't specified, then use the default one that we resolve earlier in the build -{ - if [[ ! "${ES_SNAPSHOT_MANIFEST:-}" ]]; then - ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST_DEFAULT --default '')} - export ES_SNAPSHOT_MANIFEST - echo "Using default ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST" - fi -} - -# Setup CI Stats -{ - CI_STATS_BUILD_ID="$(buildkite-agent meta-data get ci_stats_build_id --default '')" - export CI_STATS_BUILD_ID - - CI_STATS_TOKEN="$(retry 5 5 vault read -field=api_token secret/kibana-issues/dev/kibana_ci_stats)" - export CI_STATS_TOKEN - - CI_STATS_HOST="$(retry 5 5 vault read -field=api_host secret/kibana-issues/dev/kibana_ci_stats)" - export CI_STATS_HOST - - if [[ "$CI_STATS_BUILD_ID" ]]; then - echo "CI Stats Build ID: $CI_STATS_BUILD_ID" - - KIBANA_CI_STATS_CONFIG=$(jq -n \ - --arg buildId "$CI_STATS_BUILD_ID" \ - --arg apiUrl "https://$CI_STATS_HOST" \ - --arg apiToken "$CI_STATS_TOKEN" \ - '{buildId: $buildId, apiUrl: $apiUrl, apiToken: $apiToken}' \ - ) - export KIBANA_CI_STATS_CONFIG - fi -} - -GITHUB_TOKEN=$(retry 5 5 vault read -field=github_token secret/kibana-issues/dev/kibanamachine) -export GITHUB_TOKEN - -KIBANA_DOCKER_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/container-registry)" -export KIBANA_DOCKER_USERNAME - -KIBANA_DOCKER_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/container-registry)" -export KIBANA_DOCKER_PASSWORD - -KIBANA_CI_REPORTER_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibanamachine-reporter) -export KIBANA_CI_REPORTER_KEY - - -BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-local-dev.json" -export BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE -retry 5 5 vault read -field=service_account_json secret/kibana-issues/dev/kibana-ci-bazel-remote-cache-local-dev > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE" - -# Setup Failed Test Reporter Elasticsearch credentials -{ - TEST_FAILURES_ES_CLOUD_ID=$(retry 5 5 vault read -field=cloud_id secret/kibana-issues/dev/failed_tests_reporter_es) - export TEST_FAILURES_ES_CLOUD_ID - - TEST_FAILURES_ES_USERNAME=$(retry 5 5 vault read -field=username secret/kibana-issues/dev/failed_tests_reporter_es) - export TEST_FAILURES_ES_USERNAME - - TEST_FAILURES_ES_PASSWORD=$(retry 5 5 vault read -field=password secret/kibana-issues/dev/failed_tests_reporter_es) - export TEST_FAILURES_ES_PASSWORD -} - -PIPELINE_PRE_COMMAND=${PIPELINE_PRE_COMMAND:-".buildkite/scripts/lifecycle/pipelines/$BUILDKITE_PIPELINE_SLUG/pre_command.sh"} -if [[ -f "$PIPELINE_PRE_COMMAND" ]]; then - source "$PIPELINE_PRE_COMMAND" -fi