[Buildkite] Revamp buildkite scripts

.buildkite/hooks/pre-command

@@ -3,7 +3,8 @@
 source .buildkite/scripts/tooling.sh
 set -euo pipefail
 
-export GO_VERSION=$(cat .go-version)
+GO_VERSION=$(cat .go-version)
+export GO_VERSION
 
 GCP_SERVICE_ACCOUNT_SECRET_PATH=secret/ci/elastic-elastic-package/gcp-service-account
 AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_account_auth
@@ -17,45 +18,59 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/private_ci_artifact
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && ("$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives) ]]; then
-    export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
+    PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
+    export PRIVATE_CI_GCS_CREDENTIALS_SECRET
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp" ]]; then
-    export ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectId ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
-    export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
+    ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectId ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_GCP_PROJECT_SECRET
+    ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET
 
     # Environment variables required by the service deployer
     export GOOGLE_CREDENTIALS=${ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET}
     export GCP_PROJECT_ID=${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && ("$BUILDKITE_STEP_KEY" == "integration-parallel-aws" || "$BUILDKITE_STEP_KEY" == "integration-parallel-aws_logs") ]]; then
-    export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
-    export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+    ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_AWS_SECRET_KEY
+    ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+    export ELASTIC_PACKAGE_AWS_ACCESS_KEY
 
     # Environment variables required by the service deployer
     export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
     export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "release" ]]; then
-    export GITHUB_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
+    GITHUB_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
+    export GITHUB_TOKEN
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-package-storage-publish" && "$BUILDKITE_STEP_KEY" == "sign-publish" ]]; then
-    export JENKINS_USERNAME_SECRET=$(retry 5 vault kv get -field username ${JENKINS_API_TOKEN_PATH})
-    export JENKINS_HOST_SECRET=$(retry 5 vault kv get -field internal_ci_host ${JENKINS_API_TOKEN_PATH})
-    export JENKINS_TOKEN=$(retry 5 vault kv get -field internal_ci ${JENKINS_API_TOKEN_PATH})
+    JENKINS_USERNAME_SECRET=$(retry 5 vault kv get -field username ${JENKINS_API_TOKEN_PATH})
+    export JENKINS_USERNAME_SECRET
+    JENKINS_HOST_SECRET=$(retry 5 vault kv get -field internal_ci_host ${JENKINS_API_TOKEN_PATH})
+    export JENKINS_HOST_SECRET
+    JENKINS_TOKEN=$(retry 5 vault kv get -field internal_ci ${JENKINS_API_TOKEN_PATH})
+    export JENKINS_TOKEN
 
     # signing job
-    export SIGNING_PACKAGES_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field value ${SIGNING_PACKAGES_GCS_CREDENTIALS_PATH})
+    SIGNING_PACKAGES_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field value ${SIGNING_PACKAGES_GCS_CREDENTIALS_PATH})
+    export SIGNING_PACKAGES_GCS_CREDENTIALS_SECRET
 
     # publishing job
-    export PACKAGE_UPLOADER_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field value ${PACKAGE_UPLOADER_GCS_CREDENTIALS_PATH})
+    PACKAGE_UPLOADER_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field value ${PACKAGE_UPLOADER_GCS_CREDENTIALS_PATH})
+    export PACKAGE_UPLOADER_GCS_CREDENTIALS_SECRET
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-test-with-integrations" && "$BUILDKITE_STEP_KEY" == "pr-integrations" ]]; then
-    export GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
-    export GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
-    export GITHUB_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
+    GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
+    export GITHUB_USERNAME_SECRET
+    GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
+    export GITHUB_EMAIL_SECRET
+    GITHUB_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
+    export GITHUB_TOKEN
 fi

.buildkite/scripts/install_deps.sh

@@ -1,61 +1,65 @@
 #!/bin/bash
 
+source .buildkite/scripts/tooling.sh
+
 set -euo pipefail
 
-source .buildkite/scripts/tooling.sh
+create_bin_folder() {
+    mkdir -p "${WORKSPACE}/bin"
+}
 
 add_bin_path(){
-    mkdir -p ${WORKSPACE}/bin
+    create_bin_folder
     export PATH="${WORKSPACE}/bin:${PATH}"
 }
 
 with_kubernetes() {
-    mkdir -p ${WORKSPACE}/bin
-    retry 5 curl -sSLo ${WORKSPACE}/bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
-    chmod +x ${WORKSPACE}/bin/kind
+    create_bin_folder
+    retry 5 curl -sSLo "${WORKSPACE}/bin/kind" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
+    chmod +x "${WORKSPACE}/bin/kind"
     kind version
     which kind
 
-    mkdir -p ${WORKSPACE}/bin
-    retry 5 curl -sSLo ${WORKSPACE}/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
-    chmod +x ${WORKSPACE}/bin/kubectl
+    retry 5 curl -sSLo "${WORKSPACE}/bin/kubectl" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
+    chmod +x "${WORKSPACE}/bin/kubectl"
     kubectl version --client
     which kubectl
 }
 
 with_go() {
-    mkdir -p ${WORKSPACE}/bin
-    retry 5 curl -sL -o ${WORKSPACE}/bin/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-linux-amd64"
-    chmod +x ${WORKSPACE}/bin/gvm
-    eval "$(gvm $(cat .go-version))"
+    create_bin_folder
+    retry 5 curl -sL -o "${WORKSPACE}/bin/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-linux-amd64"
+    chmod +x "${WORKSPACE}/bin/gvm"
+    eval "$(gvm "$(cat .go-version)")"
     go version
     which go
-    export PATH="${PATH}:$(go env GOPATH)/bin"
+    PATH="${PATH}:$(go env GOPATH)/bin"
+    export PATH
 }
 
 with_github_cli() {
-    mkdir -p ${WORKSPACE}/bin
-    mkdir -p ${WORKSPACE}/tmp
+    create_bin_folder
+    mkdir -p "${WORKSPACE}/tmp"
 
     local gh_filename="gh_${GH_CLI_VERSION}_linux_amd64"
     local gh_tar_file="${gh_filename}.tar.gz"
     local gh_tar_full_path="${WORKSPACE}/tmp/${gh_tar_file}"
 
-    retry 5 curl -sL -o ${gh_tar_full_path} "https://github.com/cli/cli/releases/download/v${GH_CLI_VERSION}/${gh_tar_file}"
+    retry 5 curl -sL -o "${gh_tar_full_path}" "https://github.com/cli/cli/releases/download/v${GH_CLI_VERSION}/${gh_tar_file}"
 
     # just extract the binary file from the tar.gz
-    tar -C ${WORKSPACE}/bin -xpf ${gh_tar_full_path} ${gh_filename}/bin/gh --strip-components=2
+    tar -C "${WORKSPACE}/bin" -xpf "${gh_tar_full_path}" "${gh_filename}/bin/gh" --strip-components=2
 
-    chmod +x ${WORKSPACE}/bin/gh
-    rm -rf ${WORKSPACE}/tmp
+    chmod +x "${WORKSPACE}/bin/gh"
+    rm -rf "${WORKSPACE}/tmp"
 
     gh version
 }
 
 with_jq() {
-    mkdir -p ${WORKSPACE}/bin
-    retry 5 curl -sL -o ${WORKSPACE}/bin/jq "https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64"
+    create_bin_folder
+    retry 5 curl -sL -o "${WORKSPACE}/bin/jq" "https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64"
 
-    chmod +x ${WORKSPACE}/bin/jq
+    chmod +x "${WORKSPACE}/bin/jq"
     jq --version
 }

.buildkite/scripts/integration_tests.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+
+source .buildkite/scripts/install_deps.sh
+source .buildkite/scripts/tooling.sh
+
 set -euo pipefail
 
 WORKSPACE="$(pwd)"
@@ -15,7 +19,7 @@ cleanup() {
     fi
 
     echo "Deleting temporal files..."
-    cd ${WORKSPACE}
+    cd "${WORKSPACE}"
     rm -rf "${TMP_FOLDER_TEMPLATE_BASE}.*"
     echo "Done."
 
@@ -31,9 +35,6 @@ usage() {
     echo -e "\t-h: Show this message"
 }
 
-source .buildkite/scripts/install_deps.sh
-source .buildkite/scripts/tooling.sh
-
 PARALLEL_TARGET="test-check-packages-parallel"
 FALSE_POSITIVES_TARGET="test-check-packages-false-positives"
 KIND_TARGET="test-check-packages-with-kind"
@@ -78,10 +79,12 @@ if [[ "${TARGET}" == "" ]]; then
 fi
 
 google_cloud_auth_safe_logs() {
-    local gsUtilLocation=$(mktemp -d -p ${WORKSPACE} -t ${TMP_FOLDER_TEMPLATE})
+    local gsUtilLocation=""
+    gsUtilLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE}")
+
     local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
 
-    echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
+    echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > "${secretFileLocation}"
 
     google_cloud_auth "${secretFileLocation}"
 }
@@ -119,7 +122,7 @@ if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSIT
 
     # allow to fail this command, to be able to upload safe logs
     set +e
-    make PACKAGE_UNDER_TEST=${PACKAGE} ${TARGET}
+    make PACKAGE_UNDER_TEST="${PACKAGE}" "${TARGET}"
     testReturnCode=$?
     set -e
 
@@ -150,4 +153,4 @@ if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSIT
     exit 0
 fi
 
-make install ${TARGET} check-git-clean
+make install "${TARGET}" check-git-clean

.buildkite/scripts/release.sh

@@ -3,7 +3,7 @@
 set -euo pipefail
 
 cleanup() {
-    rm -rf ${WORKSPACE}
+    rm -rf "${WORKSPACE}"
 }
 trap cleanup exit
 

.buildkite/scripts/signAndPublishPackage.sh

@@ -1,12 +1,12 @@
 #!/bin/bash
+source .buildkite/scripts/install_deps.sh
+source .buildkite/scripts/tooling.sh
+
 set -euo pipefail
 
 WORKSPACE="$(pwd)"
 TMP_FOLDER_TEMPLATE_BASE="tmp.elastic-package"
 
-source .buildkite/scripts/install_deps.sh
-source .buildkite/scripts/tooling.sh
-
 cleanup() {
     local error_code=$?
 
@@ -18,7 +18,7 @@ cleanup() {
     fi
 
     echo "Deleting temporal files..."
-    cd ${WORKSPACE}
+    cd "${WORKSPACE}"
     rm -rf ${TMP_FOLDER_TEMPLATE_BASE}.*
     echo "Done."
 
@@ -30,7 +30,7 @@ trap cleanup EXIT
 is_already_published() {
     local packageZip=$1
 
-    if curl -s --head https://package-storage.elastic.co/artifacts/packages/${packageZip} | grep -q "HTTP/2 200" ; then
+    if curl -s --head "https://package-storage.elastic.co/artifacts/packages/${packageZip}" | grep -q "HTTP/2 200" ; then
         echo "- Already published ${packageZip}"
         return 0
     fi
@@ -66,39 +66,42 @@ PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH="gs://elastic-bekitzur-pac
 
 
 google_cloud_auth_signing() {
-    local gsUtilLocation=$(mktemp -d -p ${WORKSPACE} -t ${TMP_FOLDER_TEMPLATE})
+    local gsUtilLocation
+    gsUtilLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE}")
 
     local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
-    echo "${SIGNING_PACKAGES_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
+    echo "${SIGNING_PACKAGES_GCS_CREDENTIALS_SECRET}" > "${secretFileLocation}"
 
     google_cloud_auth "${secretFileLocation}"
 }
 
 google_cloud_auth_publishing() {
-    local gsUtilLocation=$(mktemp -d -p ${WORKSPACE} -t ${TMP_FOLDER_TEMPLATE})
+    local gsUtilLocation
+    gsUtilLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE}")
 
     local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
-    echo "${PACKAGE_UPLOADER_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
+    echo "${PACKAGE_UPLOADER_GCS_CREDENTIALS_SECRET}" > "${secretFileLocation}"
 
     google_cloud_auth "${secretFileLocation}"
 }
 
 sign_package() {
     local package=${1}
-    local packageZip=$(basename ${package})
+    local packageZip
+    packageZip=$(basename "${package}")
 
     google_cloud_auth_signing
 
     # upload zip package (trailing forward slashes are required)
     echo "Upload package .zip file for signing ${package} to ${INFRA_SIGNING_BUCKET_ARTIFACTS_PATH}"
-    gsutil cp ${package} "${INFRA_SIGNING_BUCKET_ARTIFACTS_PATH}/"
+    gsutil cp "${package}" "${INFRA_SIGNING_BUCKET_ARTIFACTS_PATH}/"
 
     echo "Trigger Jenkins job for signing package ${packageZip}"
     pushd ${JENKINS_TRIGGER_PATH} > /dev/null
 
     go run main.go \
         --jenkins-job sign \
-        --folder ${INFRA_SIGNING_BUCKET_ARTIFACTS_PATH}
+        --folder "${INFRA_SIGNING_BUCKET_ARTIFACTS_PATH}"
 
     popd > /dev/null
 
@@ -117,19 +120,20 @@ sign_package() {
 
 publish_package() {
     local package=$1
-    local packageZip=$(basename ${package})
+    local packageZip
+    packageZip=$(basename "${package}")
 
     # create file with credentials
     google_cloud_auth_publishing
 
     # upload files (trailing forward slashes are required)
     echo "Upload package .zip file ${package} to ${PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}"
-    gsutil cp ${package} "${PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/"
+    gsutil cp "${package}" "${PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/"
     echo "Upload package .sig file ${package}.sig to ${PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}"
-    gsutil cp ${package}.sig "${PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/"
+    gsutil cp "${package}.sig" "${PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/"
 
     echo "Trigger Jenkins job for publishing package ${packageZip}"
-    pushd ${JENKINS_TRIGGER_PATH} > /dev/null
+    pushd "${JENKINS_TRIGGER_PATH}" > /dev/null
 
     go run main.go \
         --jenkins-job publish \