diff --git a/README.md b/README.md index 72e1af0..cd35332 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,7 @@ files within it's directory.
  1. die: print message and exit 1
  2. spushd/spopd: Safe verisons of pushd & popd that call die if the push/pop fails, they also drop stdout.
  3. +
  4. retry: Retry a command until it succeeds up to a user specified maximum number of attempts. Escalating delay between attempts.
diff --git a/filehandling/lib b/filehandling/lib index 4cd3f47..f06c153 100644 --- a/filehandling/lib +++ b/filehandling/lib @@ -1,24 +1,28 @@ #!/bin/bash : "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}" -. "${BASH_LIB_DIR}/helpers/lib" #https://stackoverflow.com/a/23002317 function abs_path() { # generate absolute path from relative path - # $1 : relative filename + # path : relative filename # return : absolute path - if [ -d "$1" ]; then + if [[ -z "${1:-}" ]]; then + path="." + else + path="${1}" + fi + if [ -d "${path}" ]; then # dir - (spushd "$1"; pwd) - elif [ -f "$1" ]; then + (spushd "${path}"; pwd) + elif [ -f "${path}" ]; then # file - if [[ $1 = /* ]]; then - echo "$1" - elif [[ $1 == */* ]]; then - echo "$(spushd "${1%/*}"; pwd)/${1##*/}" + if [[ ${path} = /* ]]; then + echo "${path}" + elif [[ ${path} == */* ]]; then + echo "$(spushd "${path%/*}"; pwd)/${path##*/}" else - echo "$(pwd)/$1" + echo "$(pwd)/${path}" fi fi -} \ No newline at end of file +} diff --git a/git/lib b/git/lib index 65ea05a..b5922fe 100644 --- a/git/lib +++ b/git/lib @@ -1,7 +1,6 @@ #!/bin/bash : "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}" -. "${BASH_LIB_DIR}/helpers/lib" # Get the top level of a git repo function repo_root(){ @@ -76,4 +75,4 @@ space seperated with three fields: subtree_path renmote_url remote_name" function tracked_files_excluding_subtrees(){ subtrees="$(cat_gittrees | awk '{print $1}' | paste -sd '|' -)" all_files_in_repo | grep -E -v "${subtrees}" -} \ No newline at end of file +} diff --git a/helpers/lib b/helpers/lib index f4cf58e..6bea6e4 100644 --- a/helpers/lib +++ b/helpers/lib @@ -17,4 +17,58 @@ function spushd(){ #safe popd function spopd(){ popd >/dev/null || die "popd failed :(" -} \ No newline at end of file +} + +# Retry a command multiple times until it succeeds, with escalating +# delay between attempts. +# Delay is 2 * n + random up to 30s, then 30s + random after that. +# For large numbers of retries the max delay is effectively the retry +# in minutes. +# Based on: +# https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746 +# but now quite heavily modified. +function retry { + # Maxiumum amount of fixed delay between attempts + # a random value will still be added. + local -r MAX_BACKOFF=30 + + if [[ ${#} -lt 2 ]]; then + echo "retry usage: retry " + exit 1 + fi + + local retries=$1 + shift + + if ! [[ ${retries} =~ ^[0-9\.]*$ ]]; then + echo "Invalid number of retries: ${retries} for command '${*}'". + exit 1 + fi + + local count=0 + until "$@"; do + # Command failed, otherwise until would have skipped the loop + + # Store return code so it can be reported to the user + exit=$? + count=$((count + 1)) + if [ "${count}" -lt "${retries}" ]; then + # There are still retries left, calculate delay and notify user. + backoff=$((2 * count)) + if [[ "${backoff}" -gt "${MAX_BACKOFF}" ]]; then + backoff=${MAX_BACKOFF} + fi; + + # Add a random amount to the delay to prevent competing processes + # from re-colliding. + wait=$(( backoff + (RANDOM % count) )) + echo "'${*}' Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + # Out of retries :( + echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} diff --git a/init b/init index c55fd63..e514598 100644 --- a/init +++ b/init @@ -20,18 +20,18 @@ BASH_LIB_DIR="${BASH_LIB_DIR_RELATIVE}" # Load the filehandling module for the abspath # function -. "${BASH_LIB_DIR_RELATIVE}/filehandling/lib" +for lib in helpers logging filehandling git k8s test-utils; do + . "${BASH_LIB_DIR_RELATIVE}/${lib}/lib" +done # Export the absolute path # shellcheck disable=SC2086 BASH_LIB_DIR="$(abs_path ${BASH_LIB_DIR_RELATIVE})" export BASH_LIB_DIR -. "${BASH_LIB_DIR}/helpers/lib" - # Update Submodules spushd "${BASH_LIB_DIR}" git submodule update --init --recursive spopd -export BATS_CMD="${BASH_LIB_DIR}/test-utils/bats/bin/bats" \ No newline at end of file +export BATS_CMD="${BASH_LIB_DIR}/test-utils/bats/bin/bats" diff --git a/k8s/lib b/k8s/lib index 6bea766..8e63900 100644 --- a/k8s/lib +++ b/k8s/lib @@ -1,7 +1,6 @@ #!/bin/bash : "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}" -. "${BASH_LIB_DIR}/helpers/lib" # Sets additional required environment variables that aren't available in the # secrets.yml file, and performs other preparatory steps @@ -52,4 +51,4 @@ function run_docker_gke_command() { /scripts/platform_login ${1} " -} \ No newline at end of file +} diff --git a/run-tests b/run-tests index f099ee4..3fea7ae 100755 --- a/run-tests +++ b/run-tests @@ -4,7 +4,6 @@ # is not assumed to have been run # shellcheck disable=SC2086 . "$(dirname ${BASH_SOURCE[0]})/init" -. "${BASH_LIB_DIR}/helpers/lib" # Run BATS Tests "${BASH_LIB_DIR}/tests-for-this-repo/run-bats-tests" diff --git a/test-utils/lib b/test-utils/lib index 5eab41f..31c49e0 100644 --- a/test-utils/lib +++ b/test-utils/lib @@ -1,11 +1,9 @@ #!/bin/bash : "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}" -. "${BASH_LIB_DIR}/git/lib" -. "${BASH_LIB_DIR}/helpers/lib" -readonly SHELLCHECK_IMAGE="${SHELLCHECK_IMAGE:-koalaman/shellcheck}" -readonly SHELLCHECK_TAG="${SHELLCHECK_TAG:-v0.6.0}" +SHELLCHECK_IMAGE="${SHELLCHECK_IMAGE:-koalaman/shellcheck}" +SHELLCHECK_TAG="${SHELLCHECK_TAG:-v0.6.0}" # Check a single shell script for syntax # and common errors. diff --git a/tests-for-this-repo/filehandling.bats b/tests-for-this-repo/filehandling.bats index 1b309fb..4492078 100644 --- a/tests-for-this-repo/filehandling.bats +++ b/tests-for-this-repo/filehandling.bats @@ -1,7 +1,7 @@ . "${BASH_LIB_DIR}/test-utils/bats-support/load.bash" . "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash" -. "${BASH_LIB_DIR}/filehandling/lib" +. "${BASH_LIB_DIR}/init" @test "abs_path returns absolute path for PWD" { run abs_path . @@ -9,8 +9,14 @@ assert_success } +@test "abs_path returns PWD when no arg specified" { + run abs_path + assert_output $PWD + assert_success +} + @test "abs_path returns same path when already absolute" { run abs_path /tmp assert_output /tmp assert_success -} \ No newline at end of file +} diff --git a/tests-for-this-repo/git.bats b/tests-for-this-repo/git.bats index 7ba2a1c..9c18031 100644 --- a/tests-for-this-repo/git.bats +++ b/tests-for-this-repo/git.bats @@ -1,7 +1,7 @@ . "${BASH_LIB_DIR}/test-utils/bats-support/load.bash" . "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash" -. "${BASH_LIB_DIR}/git/lib" +. "${BASH_LIB_DIR}/init" # run before every test setup(){ @@ -137,4 +137,4 @@ EOF assert_success assert_output --partial a_file assert_success -} \ No newline at end of file +} diff --git a/tests-for-this-repo/helpers.bats b/tests-for-this-repo/helpers.bats index af61b94..bbea6f8 100644 --- a/tests-for-this-repo/helpers.bats +++ b/tests-for-this-repo/helpers.bats @@ -1,35 +1,114 @@ . "${BASH_LIB_DIR}/test-utils/bats-support/load.bash" . "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash" -. "${BASH_LIB_DIR}/helpers/lib" +. "${BASH_LIB_DIR}/init" + +# run before every test +setup(){ + temp_dir="${BATS_TMPDIR}/testtemp" + mkdir "${temp_dir}" + afile="${temp_dir}/appendfile" +} + +teardown(){ + temp_dir="${BATS_TMPDIR}/testtemp" + rm -rf "${temp_dir}" +} @test "die exits and prints message" { - run bash -c ". ${BASH_LIB_DIR}/helpers/lib; die msg" + run bash -c ". ${BASH_LIB_DIR}/init; die msg" assert_output msg assert_failure } @test "spushd is quiet on stdout" { run spushd /tmp - refute_output + assert_output "" assert_success } @test "spopd is quiet on stdout" { pushd . run spopd - refute_output + assert_output "" assert_success } @test "spushd dies on failure" { - run bash -c ". ${BASH_LIB_DIR}/helpers/lib; spushd /this-doesnt-exist" + run bash -c ". ${BASH_LIB_DIR}/init; spushd /this-doesnt-exist" assert_output --partial "No such file or directory" assert_failure } @test "spopd dies on failure" { - run bash -c ". ${BASH_LIB_DIR}/helpers/lib; spopd" + run bash -c ". ${BASH_LIB_DIR}/init; spopd" assert_output --partial "stack empty" assert_failure -} \ No newline at end of file +} + +@test "retry runs command only once if it succeeds the first time" { + retryme(){ + date >> ${afile} + } + run retry 3 retryme + assert_success + assert_equal $(wc -l <${afile}) 1 +} + +@test "retry doesn't introduce delay when the command succeeds first time" { + retryme(){ + date >> ${afile} + } + start=$(date +%s) + run retry 3 retryme + end=$(date +%s) + assert [ "$(( start + 1 ))" -ge "${end}" ] + assert_success +} + +@test "retry runs n times on consecutive failure and waits between attempts" { + retryme(){ + date >> ${afile} + false + } + start=$(date +%s) + run retry 2 retryme + end=$(date +%s) + # introduces at least a two second delay between attempts + assert [ "$(( start + 2 ))" -le "${end}" ] + assert_failure + assert_equal $(wc -l <${afile}) 2 +} + +@test "retry returns after first success" { + retryme(){ + date >> "${afile}" + case $(wc -l < ${afile}) in + *1) + return 1 + ;; + *) + return 0 + ;; + esac + } + run retry 3 retryme + assert_success + assert_equal $(wc -l <${afile}) 2 +} + +@test "retry fails with less than two arguments" { + run retry 3 + assert_failure + assert_output --partial usage + assert [ ! -e "${temp_dir}/appendfile" ] +} + +@test "retry fails with non-integer retry count" { + run retry "this" date + assert_failure + assert_output --partial number + assert [ ! -e "${temp_dir}/appendfile" ] +} + + diff --git a/tests-for-this-repo/k8s.bats b/tests-for-this-repo/k8s.bats index 48e1bc2..92d3b71 100755 --- a/tests-for-this-repo/k8s.bats +++ b/tests-for-this-repo/k8s.bats @@ -1,7 +1,7 @@ . "${BASH_LIB_DIR}/test-utils/bats-support/load.bash" . "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash" -. "${BASH_LIB_DIR}/k8s/lib" +. "${BASH_LIB_DIR}/init" @test "gke-utils image builds" { run build_gke_image @@ -24,4 +24,4 @@ " run delete_gke_image "${image}" assert_success -} \ No newline at end of file +} diff --git a/tests-for-this-repo/lint.bats b/tests-for-this-repo/lint.bats index dab5d97..bd2291d 100755 --- a/tests-for-this-repo/lint.bats +++ b/tests-for-this-repo/lint.bats @@ -1,6 +1,5 @@ -. ${BASH_LIB_DIR}/git/lib -. ${BASH_LIB_DIR}/test-utils/lib -. ${BASH_LIB_DIR}/helpers/lib + +. "${BASH_LIB_DIR}/init" setup() { spushd ${BASH_LIB_DIR} @@ -66,4 +65,4 @@ setup() { fi done return ${rc} -} \ No newline at end of file +} diff --git a/tests-for-this-repo/logging.bats b/tests-for-this-repo/logging.bats index 47bc684..ccdb6d4 100644 --- a/tests-for-this-repo/logging.bats +++ b/tests-for-this-repo/logging.bats @@ -1,10 +1,9 @@ . "${BASH_LIB_DIR}/test-utils/bats-support/load.bash" . "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash" - -. "${BASH_LIB_DIR}/logging/lib" +. "${BASH_LIB_DIR}/init" @test "announce prints all arguments" { run announce one two one two assert_output --partial "one two one two" assert_success -} \ No newline at end of file +} diff --git a/tests-for-this-repo/run-bats-tests b/tests-for-this-repo/run-bats-tests index 455319e..38026eb 100755 --- a/tests-for-this-repo/run-bats-tests +++ b/tests-for-this-repo/run-bats-tests @@ -8,8 +8,7 @@ set -euo pipefail # is not assumed to have been run # shellcheck disable=SC2086,SC2046 . $(dirname ${BASH_SOURCE[0]})/../init -. "${BASH_LIB_DIR}/test-utils/lib" -. "${BASH_LIB_DIR}/helpers/lib" +set +eu # Check vital tools are installed @@ -66,4 +65,4 @@ if [[ "${BATS_OUTPUT_FORMAT}" == junit ]]; then echo "Junit output written to ${JUNIT_FILE}" fi -exit ${rc} \ No newline at end of file +exit ${rc} diff --git a/tests-for-this-repo/run-gitleaks b/tests-for-this-repo/run-gitleaks index 0faf3fd..652085f 100755 --- a/tests-for-this-repo/run-gitleaks +++ b/tests-for-this-repo/run-gitleaks @@ -6,8 +6,6 @@ # is not assumed to have been run # shellcheck disable=SC2086,SC2046 . $(dirname "${BASH_SOURCE[0]}")/../init -. "${BASH_LIB_DIR}/test-utils/lib" -. "${BASH_LIB_DIR}/helpers/lib" docker run \ --rm \ diff --git a/tests-for-this-repo/run-python-lint b/tests-for-this-repo/run-python-lint index c14e96b..c2ee68f 100755 --- a/tests-for-this-repo/run-python-lint +++ b/tests-for-this-repo/run-python-lint @@ -5,7 +5,6 @@ # shellcheck disable=SC2086,SC2046 . $(dirname ${BASH_SOURCE[0]})/../init -. "${BASH_LIB_DIR}/helpers/lib" rc=0 @@ -17,4 +16,4 @@ spushd ${BASH_LIB_DIR}/tests-for-this-repo/python-lint mv "${BASH_LIB_DIR}/junit.xml" "${BASH_LIB_DIR}/python-lint-junit.xml" spopd -exit ${rc} \ No newline at end of file +exit ${rc} diff --git a/tests-for-this-repo/test-utils.bats b/tests-for-this-repo/test-utils.bats index c723c34..28ee32c 100644 --- a/tests-for-this-repo/test-utils.bats +++ b/tests-for-this-repo/test-utils.bats @@ -1,8 +1,6 @@ . "${BASH_LIB_DIR}/test-utils/bats-support/load.bash" . "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash" - -. "${BASH_LIB_DIR}/helpers/lib" -. "${BASH_LIB_DIR}/test-utils/lib" +. "${BASH_LIB_DIR}/init" docker_safe_tmp(){ # neither mktemp -d not $BATS_TMPDIR @@ -77,4 +75,4 @@ docker_safe_tmp(){ rc=${?} assert_equal "${stdout}" "$(cat ${fdir}/tap2junit.out)" assert_equal "${rc}" "0" -} \ No newline at end of file +}