ci: add bazel roachtest weekly job

Closes cockroachdb#67155.

Release note: None

build/teamcity-weekly-roachtest.sh

@@ -44,6 +44,9 @@ chmod o+rwx "${artifacts}"
 # NB(2): We specify --zones below so that nodes are created in us-central1-b
 # by default. This reserves us-east1-b (the roachprod default zone) for use
 # by manually created clusters.
+#
+# NB(3): If you make changes here, you should probably make the same change in
+# build/teamcity/cockroach/nightlies/roachtest_weekly_impl.sh.
 exit_status=0
 timeout -s INT $((7800*60)) bin/roachtest run \
   tag:weekly \

build/teamcity/cockroach/nightlies/roachtest_compile_bits.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# Builds all bits needed for roachtests, stages them in bin/ and lib.docker_amd64/.
+
+bazel build --config crosslinux --config ci --config with_ui -c opt \
+      //pkg/cmd/cockroach //pkg/cmd/workload //pkg/cmd/roachtest \
+      //pkg/cmd/roachprod //c-deps:libgeos
+BAZEL_BIN=$(bazel info bazel-bin --config crosslinux --config ci --config with_ui -c opt)
+# Move this stuff to bin for simplicity.
+mkdir -p bin
+chmod o+rwx bin
+cp $BAZEL_BIN/pkg/cmd/cockroach/cockroach_/cockroach bin
+cp $BAZEL_BIN/pkg/cmd/roachprod/roachprod_/roachprod bin
+cp $BAZEL_BIN/pkg/cmd/roachtest/roachtest_/roachtest bin
+cp $BAZEL_BIN/pkg/cmd/workload/workload_/workload    bin
+chmod a+w bin/cockroach bin/roachprod bin/roachtest bin/workload
+# Stage the geos libs in the appropriate spot.
+mkdir -p lib.docker_amd64
+chmod o+rwx lib.docker_amd64
+cp $BAZEL_BIN/c-deps/libgeos/lib/libgeos.so   lib.docker_amd64
+cp $BAZEL_BIN/c-deps/libgeos/lib/libgeos_c.so lib.docker_amd64
+chmod a+w lib.docker_amd64/libgeos.so lib.docker_amd64/libgeos_c.so
+

build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh

@@ -10,24 +10,7 @@ if [[ ! -f ~/.ssh/id_rsa.pub ]]; then
   ssh-keygen -q -C "roachtest-nightly-bazel $(date)" -N "" -f ~/.ssh/id_rsa
 fi
 
-bazel build --config crosslinux --config ci --config with_ui -c opt \
-      //pkg/cmd/cockroach //pkg/cmd/workload //pkg/cmd/roachtest \
-      //pkg/cmd/roachprod //c-deps:libgeos
-BAZEL_BIN=$(bazel info bazel-bin --config crosslinux --config ci --config with_ui -c opt)
-# Move this stuff to bin for simplicity.
-mkdir -p bin
-chmod o+rwx bin
-cp $BAZEL_BIN/pkg/cmd/cockroach/cockroach_/cockroach bin
-cp $BAZEL_BIN/pkg/cmd/roachprod/roachprod_/roachprod bin
-cp $BAZEL_BIN/pkg/cmd/roachtest/roachtest_/roachtest bin
-cp $BAZEL_BIN/pkg/cmd/workload/workload_/workload    bin
-chmod a+w bin/cockroach bin/roachprod bin/roachtest bin/workload
-# Stage the geos libs in the appropriate spot.
-mkdir -p lib.docker_amd64
-chmod o+rwx lib.docker_amd64
-cp $BAZEL_BIN/c-deps/libgeos/lib/libgeos.so   lib.docker_amd64
-cp $BAZEL_BIN/c-deps/libgeos/lib/libgeos_c.so lib.docker_amd64
-chmod a+w lib.docker_amd64/libgeos.so lib.docker_amd64/libgeos_c.so
+source $root/build/teamcity/cockroach/nightlies/roachtest_compile_bits.sh
 
 artifacts=/artifacts
 source $root/build/teamcity/util/roachtest_util.sh

build/teamcity/cockroach/nightlies/roachtest_weekly.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
+
+source "$dir/teamcity-support.sh"  # For $root
+source "$dir/teamcity-bazel-support.sh"  # For run_bazel
+
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e CLOUD -e COCKROACH_DEV_LICENSE -e COUNT -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILD_BRANCH -e TC_BUILD_ID" \
+			       run_bazel build/teamcity/cockroach/nightlies/roachtest_weekly_impl.sh

build/teamcity/cockroach/nightlies/roachtest_weekly_impl.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+google_credentials="$GOOGLE_EPHEMERAL_CREDENTIALS"
+source "$(dirname "${0}")/teamcity-support.sh"
+log_into_gcloud
+export ROACHPROD_USER=teamcity
+
+set -x
+
+generate_ssh_key
+
+export PATH=$PATH:$(go env GOPATH)/bin
+
+build_tag=$(git describe --abbrev=0 --tags --match=v[0-9]*)
+git checkout master
+git pull origin master
+
+git rev-parse HEAD
+
+source $root/build/teamcity/cockroach/nightlies/roachtest_compile_bits.sh
+
+artifacts_subdir=bazel-$(date +"%Y%m%d")-${TC_BUILD_ID}
+artifacts=$PWD/artifacts/$artifacts_subdir
+mkdir -p "$artifacts"
+chmod o+rwx "${artifacts}"
+
+# NB: Teamcity has a 7920 minute timeout that, when reached,
+# kills the process without a stack trace (probably SIGKILL).
+# We'd love to see a stack trace though, so after 7800 minutes,
+# kill with SIGINT which will allow roachtest to fail tests and
+# cleanup.
+#
+# NB(2): We specify --zones below so that nodes are created in us-central1-b
+# by default. This reserves us-east1-b (the roachprod default zone) for use
+# by manually created clusters.
+#
+# NB(3): If you make changes here, you should probably make the same change in
+# build/teamcity-weekly-roachtest.sh
+exit_status=0
+timeout -s INT $((7800*60)) bin/roachtest run \
+  tag:weekly \
+  --build-tag "${build_tag}" \
+  --cluster-id "${TC_BUILD_ID}" \
+  --zones "us-central1-b,us-west1-b,europe-west2-b" \
+  --cockroach "$PWD/bin/cockroach" \
+  --roachprod "$PWD/bin/roachprod" \
+  --workload "$PWD/bin/workload" \
+  --artifacts "/artifacts/${artifacts_subdir}" \
+  --artifacts-literal="${artifacts}" \
+  --parallelism 5 \
+  --encrypt=random \
+  --teamcity || exit_status=$?
+
+if [[ ${exit_status} -eq 10 ]]; then
+  # Exit code 10 indicates that some tests failed, but that roachtest
+  # as a whole passed. We want to exit zero in this case so that we
+  # can let TeamCity report failing tests without also failing the
+  # build. That way, build failures can be used to notify about serious
+  # problems that prevent tests from being invoked in the first place.
+  exit_status=0
+fi
+
+# Upload any stats.json files to the cockroach-nightly bucket.
+for file in $(find ${artifacts#${PWD}/} -name stats.json); do
+    gsutil cp ${file} gs://cockroach-nightly/${file}
+done
+
+exit "$exit_status"