From 5a3cc807904cc7d9bb5eb0990c7fc6761fb8913d Mon Sep 17 00:00:00 2001 From: William Bezuidenhout Date: Thu, 12 Jun 2025 10:18:24 +0200 Subject: [PATCH] chore(ci): login with docker (#1218) Ultimate goal: login with docker to get higher pull limits What it involved: - Rework how the `buildkite-agent` vm is built - https://github.com/sourcegraph/infrastructure/pull/6848 - https://github.com/sourcegraph/infrastructure/pull/6849 - https://github.com/sourcegraph/infrastructure/pull/6850 - https://github.com/sourcegraph/infrastructure/pull/6851 - https://github.com/sourcegraph/infrastructure/pull/6852 - https://github.com/sourcegraph/infrastructure/pull/6854 - https://github.com/sourcegraph/infrastructure/pull/6855 - https://github.com/sourcegraph/infrastructure/pull/6856 - Transition to mise - Use a specific version of vagrant (2.4.1) otherwise the an older incompatible version of `vagrant-google` (2.2.0) gets installed, 2.7.0 is the latest and correct one - Fix vagrant issues ### Test plan CI --- .buildkite/ci-checkov.sh | 8 ++------ .buildkite/hooks/pre-command | 7 ++++++- .buildkite/vagrant-run.sh | 18 ++++++++++++------ .tool-versions | 12 ++++++------ .use_mise | 6 ++++++ test/Vagrantfile | 2 ++ test/smoke-test.sh | 3 +++ 7 files changed, 37 insertions(+), 19 deletions(-) create mode 100644 .use_mise diff --git a/.buildkite/ci-checkov.sh b/.buildkite/ci-checkov.sh index a9ee4ebd8..a4d1a9372 100755 --- a/.buildkite/ci-checkov.sh +++ b/.buildkite/ci-checkov.sh @@ -1,11 +1,7 @@ #!/usr/bin/env bash -# Set this to fail on the install +# Set this to fail on the install set -euxo pipefail -# Install and run the plugin for checkov -# Use the full path to run pip3.10 -pip3 install checkov - # List of checks we do not want to run here # This is a living list and will see additions and mostly removals over time. SKIP_CHECKS="CKV_GCP_22,CKV_GCP_66,CKV_GCP_13,CKV_GCP_71,CKV_GCP_61,CKV_GCP_21,CKV_GCP_65,CKV_GCP_67,CKV_GCP_20,CKV_GCP_69,CKV_GCP_12,CKV_GCP_24,CKV_GCP_25,CKV_GCP_64,CKV_GCP_68,CKV2_AWS_5,CKV2_GCP_3,CKV2_GCP_5,CKV_AWS_23,CKV_GCP_70,CKV_GCP_62,CKV_GCP_62,CKV_GCP_62,CKV_GCP_62,CKV_GCP_29,CKV_GCP_39" @@ -19,7 +15,7 @@ echo "========================================================================== # Set not to fail on non-zero exit code set +e # Run checkov -python3 -m checkov.main --skip-check $SKIP_CHECKS --quiet --framework terraform --compact -d . +checkov --skip-check $SKIP_CHECKS --quiet --framework terraform --compact -d . # Options # --quiet: Only show failing tests diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index cbdcbd37c..7f4a682ec 100755 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -27,4 +27,9 @@ trap remove_pidfile EXIT echo $$ > "$PIDFILE" echo "Installing asdf dependencies as defined in '${WORKDIR}/.tool-versions':" -asdf install +if [ ! -f ".use_mise" ]; then + asdf install +else + mise install + eval "$(mise activate)" +fi diff --git a/.buildkite/vagrant-run.sh b/.buildkite/vagrant-run.sh index 13d2c8a2e..06355a430 100755 --- a/.buildkite/vagrant-run.sh +++ b/.buildkite/vagrant-run.sh @@ -13,14 +13,20 @@ cleanup() { } echo --- ":vagrant: installing plugins" -plugins=(vagrant-google vagrant-env vagrant-scp) -for i in "${plugins[@]}"; do - if ! vagrant plugin list --no-tty | grep "$i"; then - vagrant plugin install "$i" - fi -done +vagrant --version +vagrant plugin install vagrant-google --plugin-version '2.7.0' +vagrant plugin install vagrant-env +vagrant plugin install vagrant-scp trap cleanup EXIT + +echo --- ":lock: builder account key" +KEY_PATH="/tmp/e2e-builder.json" +if [ ! -f ${KEY_PATH} ]; then + gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project=sourcegraph-ci > "${KEY_PATH}" +fi +export GOOGLE_JSON_KEY_LOCATION="${KEY_PATH}" + echo --- ":vagrant: starting box $box" vagrant up "$box" --provider=google || exit_code=$? diff --git a/.tool-versions b/.tool-versions index 0577dd159..4df9eb887 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,6 +1,6 @@ -nodejs 16.7.0 -yarn 1.22.4 -shellcheck 0.7.1 -golang 1.19.8 -github-cli 2.46.0 -python system +nodejs 16.7.0 +yarn 1.22.4 +shellcheck 0.7.1 +golang 1.19.8 +github-cli 2.46.0 +asdf:bosmak/asdf-checkov latest diff --git a/.use_mise b/.use_mise new file mode 100644 index 000000000..a4d863b6a --- /dev/null +++ b/.use_mise @@ -0,0 +1,6 @@ +Buildkite Agent CI use the presence of this file to determine whether it should install tools with mise or install them with ASDF. +Thus if you delete this file, CI will use ASDF to install tools and not mise. + +The file is only meant to be here while we transition to using mise completely. + +For more information you can reach out to the dev-infra team on #discuss-dev-infra. diff --git a/test/Vagrantfile b/test/Vagrantfile index 5630cdfb9..34db323a6 100644 --- a/test/Vagrantfile +++ b/test/Vagrantfile @@ -62,6 +62,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| cat << EOF >> /root/.profile export GIT_BRANCH=#{ENV['BUILDKITE_BRANCH']} export TEST_TYPE=#{ENV['TEST_TYPE']} +export DOCKER_USERNAME=#{ENV['DOCKER_USERNAME']} +export DOCKER_PASSWORD=#{ENV['DOCKER_PASSWORD']} EOF SHELL diff --git a/test/smoke-test.sh b/test/smoke-test.sh index b8621e34d..6526aa8e1 100755 --- a/test/smoke-test.sh +++ b/test/smoke-test.sh @@ -2,6 +2,9 @@ set -euxfo pipefail configure_docker() { + if [ -n "${DOCKER_USERNAME}" ] && [ -n "${DOCKER_PASSWORD}" ]; then + docker login -u "${DOCKER_USERNAME}" --password-stdin <<<"$DOCKER_PASSWORD" + fi gcloud auth configure-docker gcloud auth configure-docker us-central1-docker.pkg.dev }