From 04d9b30f72ebcf3f8c069f653cb80bf7da4fa0ea Mon Sep 17 00:00:00 2001 From: Joey Perrott Date: Tue, 13 Aug 2019 10:47:11 -0700 Subject: [PATCH] ci: move bazel saucelabs execution to script to be used across all Angular repos --- .bazelrc | 16 -- .circleci/config.yml | 30 ++-- .circleci/env.sh | 1 + scripts/saucelabs/run-bazel-via-tunnel.sh | 204 ++++++++++++++++++++++ 4 files changed, 218 insertions(+), 33 deletions(-) create mode 100755 scripts/saucelabs/run-bazel-via-tunnel.sh diff --git a/.bazelrc b/.bazelrc index 485c35ce840bc..3a63f5459c8a8 100644 --- a/.bazelrc +++ b/.bazelrc @@ -36,22 +36,6 @@ build --incompatible_strict_action_env run --incompatible_strict_action_env test --incompatible_strict_action_env -############################### -# Saucelabs support # -# Turn on these settings with # -# --config=saucelabs # -############################### - -# Expose SauceLabs environment to actions -# These environment variables are needed by -# web_test_karma to run on Saucelabs -test:saucelabs --action_env=SAUCE_USERNAME -test:saucelabs --action_env=SAUCE_ACCESS_KEY -test:saucelabs --action_env=SAUCE_READY_FILE -test:saucelabs --action_env=SAUCE_PID_FILE -test:saucelabs --action_env=SAUCE_TUNNEL_IDENTIFIER -test:saucelabs --define=KARMA_WEB_TEST_MODE=SL_REQUIRED - ############################### # Release support # # Turn on these settings with # diff --git a/.circleci/config.yml b/.circleci/config.yml index cf2fc37332c85..c66591873ad6b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -256,23 +256,19 @@ jobs: - *init_environment - *setup_circleci_bazel_config - run: - name: Preparing environment for running tests on Saucelabs. - command: setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev) - - run: - name: Starting Saucelabs tunnel - command: ./scripts/saucelabs/start-tunnel.sh - background: true - # Waits for the Saucelabs tunnel to be ready. This ensures that we don't run tests - # too early without Saucelabs not being ready. - - run: ./scripts/saucelabs/wait-for-tunnel.sh - # All web tests are contained within a single //:test_web_all target for Saucelabs - # as running each set of tests as a separate target will attempt to acquire too - # many browsers on Saucelabs (7 per target currently) and some tests will always - # fail to acquire browsers. For example: - # 14 02 2019 19:52:33.170:WARN [launcher]: chrome beta on SauceLabs have not captured in 180000 ms, killing. - # //packages/forms/test:web_test_sauce TIMEOUT in 315.0s - - run: yarn bazel test --config=saucelabs //:test_web_all - - run: ./scripts/saucelabs/stop-tunnel.sh + name: Run Bazel tests in saucelabs + # All web tests are contained within a single //:test_web_all target for Saucelabs + # as running each set of tests as a separate target will attempt to acquire too + # many browsers on Saucelabs (7 per target currently) and some tests will always + # fail to acquire browsers. For example: + # 14 02 2019 19:52:33.170:WARN [launcher]: chrome beta on SauceLabs have not captured in 180000 ms, killing. + # //packages/forms/test:web_test_sauce TIMEOUT in 315.0s + command: | + ./scripts/saucelabs/run-bazel-via-tunnel.sh \ + --tunnel-id angular-${CIRCLE_BUILD_NUM}-${CIRCLE_NODE_INDEX} \ + --username $SAUCE_USERNAME \ + --key $(echo $SAUCE_ACCESS_KEY | rev) \ + yarn bazel test //:test_web_all - *notify_dev_infra_on_fail test_aio: diff --git a/.circleci/env.sh b/.circleci/env.sh index aa2b1d1829f86..e994b5779ba7d 100755 --- a/.circleci/env.sh +++ b/.circleci/env.sh @@ -61,6 +61,7 @@ else setPublicVar SAUCE_USERNAME "angular-ci"; setSecretVar SAUCE_ACCESS_KEY "9b988f434ff8-fbca-8aa4-4ae3-35442987"; fi +# TODO(josephperrott): Remove environment variables once all saucelabs tests are via bazel method. setPublicVar SAUCE_LOG_FILE /tmp/angular/sauce-connect.log setPublicVar SAUCE_READY_FILE /tmp/angular/sauce-connect-ready-file.lock setPublicVar SAUCE_PID_FILE /tmp/angular/sauce-connect-pid-file.lock diff --git a/scripts/saucelabs/run-bazel-via-tunnel.sh b/scripts/saucelabs/run-bazel-via-tunnel.sh new file mode 100755 index 0000000000000..349a5772a93e2 --- /dev/null +++ b/scripts/saucelabs/run-bazel-via-tunnel.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash + +set -u -e -o pipefail + +# Prints out usage information for the script. +function printUsage { + echo -e "\e[1mrun-bazel-via-tunnel.sh\e[0m - Runs a bazel command using a saucelabs tunnel + + \e[1mUsage:\e[0m $0 --tunnel-id= \\ + --username= --key= + + \e[1mExample:\e[0m ./run-bazel-via-tunnel.sh --tunnel-id= \\ + --username= --key= \\ + yarn bazel test //src:everything + + Flags: + --username: The saucelabs username + --key: The saucelabs access key + --tunnel-id: An identifier for the saucelabs tunnel"; +} + +# Ensures a file is created, creating directories for the full path as needed. +function touch-safe { + for f in "$@"; do + [ -d $f:h ] || mkdir -p $f:h && command touch $f + done +} + +# The root directory of the git project the script is running in. +readonly GIT_ROOT_DIR=$(git rev-parse --show-toplevel 2> /dev/null) +# Location for the saucelabs log file. +readonly SAUCE_LOG_FILE=/tmp/angular/sauce-connect.log +# Location for the saucelabs ready to connect lock file. +readonly SAUCE_READY_FILE=/tmp/angular/sauce-connect-ready-file.lock +# Location for the saucelabs ready to connection process id lock file. +readonly SAUCE_PID_FILE=/tmp/angular/sauce-connect-pid-file.lock +# Amount of seconds we wait for sauceconnect to establish a tunnel instance. In order to not +# acquire CircleCI instances for too long if sauceconnect failed, we need a connect timeout. +readonly SAUCE_READY_FILE_TIMEOUT=120 + +# Create saucelabs log file if it doesn't already exist. +touch-safe $SAUCE_LOG_FILE; + +# Handle configuration of script from command line flags and arguments +OPTIONS=$(getopt -u -l tunnel-id:,username:,key:,help --options "" -- "$@") +# Exit if flag parsing fails. +if [ $? != 0 ] ; then echo "Failed to parse flags, exiting" && printUsage >&2 ; exit 1 ; fi +set -- $OPTIONS +while true; do + case "$1" in + --tunnel-id) + shift + SAUCE_TUNNEL_IDENTIFIER=$1 + ;; + --username) + shift + SAUCE_USERNAME=$1 + ;; + --key) + shift + SAUCE_ACCESS_KEY=$1 + ;; + --help) + printUsage + exit 2 + ;; + --) + shift + USER_COMMAND=$@ + break + ;; + *) + shift + ;; + esac +done + +# Check each required flag and parameter +if [[ -z ${SAUCE_TUNNEL_IDENTIFIER+x} ]]; then + echo "Missing required flag: --tunnel-id" + badCommandSyntax=1 +fi +if [[ -z ${SAUCE_USERNAME+x} ]]; then + echo "Missing required flag: --username" + badCommandSyntax=1 +fi +if [[ -z ${SAUCE_ACCESS_KEY+x} ]]; then + echo "Missing required flag: --key" + badCommandSyntax=1 +fi +if [[ "${USER_COMMAND}" == "" ]]; then + echo "Missing required bazel command: Bazel command for running in saucelabs tunnel" + badCommandSyntax=1 +elif [[ ! $USER_COMMAND =~ ^(yarn bazel) ]]; then + echo "The command provided must be a bazel command run via yarn, beginning with \"yarn bazel\"" + badCommandSyntax=1 +fi + +# If any required flag or parameter were found to be missing or incorrect, exit the script. +if [[ ${badCommandSyntax+x} ]]; then + echo + printUsage + exit 1 +fi + + +# Command arguments that will be passed to sauce-connect. +# By default we disable SSL bumping for all requests. This is because SSL bumping is +# not needed for our test setup and in order to perform the SSL bumping, Saucelabs +# intercepts all HTTP requests in the tunnel VM and modifies them. This can cause +# flakiness as it makes all requests dependent on the SSL bumping middleware. +# See: https://wiki.saucelabs.com/display/DOCS/Troubleshooting+Sauce+Connect#TroubleshootingSauceConnect-DisablingSSLBumping +sauceArgs="--no-ssl-bump-domains all" +sauceArgs="${sauceArgs} --logfile ${SAUCE_LOG_FILE}" +sauceArgs="${sauceArgs} --readyfile ${SAUCE_READY_FILE}" +sauceArgs="${sauceArgs} --pidfile ${SAUCE_PID_FILE}" +sauceArgs="${sauceArgs} --tunnel-identifier ${SAUCE_TUNNEL_IDENTIFIER}" +sauceArgs="${sauceArgs} -u ${SAUCE_USERNAME}" + +######################### +# Open saucelabs tunnel # +######################### + + +${GIT_ROOT_DIR}/node_modules/sauce-connect/bin/sc -k $SAUCE_ACCESS_KEY ${sauceArgs} & + + +######################################## +# Wait for saucelabs tunnel to connect # +######################################## +counter=0 + +while [[ ! -f ${SAUCE_READY_FILE} ]]; do + counter=$((counter + 1)) + + # Counter needs to be multiplied by two because the while loop only sleeps a half second. + # This has been made in favor of better progress logging (printing dots every half second) + if [ $counter -gt $[${SAUCE_READY_FILE_TIMEOUT} * 2] ]; then + echo "Timed out after ${SAUCE_READY_FILE_TIMEOUT} seconds waiting for tunnel ready file." + echo "Printing logfile output:" + echo "" + cat ${SAUCE_LOG_FILE} + exit 5 + fi + + printf "." + sleep 0.5 +done + +######################### +# Execute Bazel command # +######################### + +# Prevent immediate exit for Bazel test failures +set +e + +( + cd $GIT_ROOT_DIR && \ + # Run bazel command with saucelabs specific environment variables passed to the action + # The KARMA_WEB_TEST_MODE and SAUCE_TUNNEL_IDENTIFIER environment variables provide + # envirnment variables to be read in the karma configuration file to set correct + # configurations for karma saucelabs and browser configs. + # Usage of these envirnment variables can be seen in this repo in + # /karma-js.conf.js and /browser-providers.conf.js + eval "$USER_COMMAND --define=KARMA_WEB_TEST_MODE=SL_REQUIRED \ + --action_env=SAUCE_USERNAME=$SAUCE_USERNAME \ + --action_env=SAUCE_ACCESS_KEY=$SAUCE_ACCESS_KEY \ + --action_env=SAUCE_READY_FILE=$SAUCE_READY_FILE \ + --action_env=SAUCE_PID_FILE=$SAUCE_PID_FILE \ + --action_env=SAUCE_TUNNEL_IDENTIFIER=$SAUCE_TUNNEL_IDENTIFIER" +) +BAZEL_EXIT_CODE=$? +echo "Exit code for bazel command was: $BAZEL_EXIT_CODE" + +# Reenable immediate exit for failure exit code +set -e + +############################## +# Close the saucelabs tunnel # +############################## + +if [[ ! -f ${SAUCE_PID_FILE} ]]; then + echo "Could not find Saucelabs tunnel PID file. Cannot stop tunnel.." + exit 1 +fi + +echo "Shutting down Sauce Connect tunnel" + +# The process id for the sauce-connect instance is stored inside of the pidfile. +tunnelProcessId=$(cat ${SAUCE_PID_FILE}) + +# Kill the process by using the PID that has been read from the pidfile. Note that +# we cannot use killall because CircleCI base container images don't have it installed. +kill ${tunnelProcessId} + +while (ps -p ${tunnelProcessId} &> /dev/null); do + printf "." + sleep .5 +done + +echo "" +echo "Sauce Connect tunnel has been shut down" + +exit $BAZEL_EXIT_CODE