diff --git a/README.md b/README.md
index 72e1af0..cd35332 100644
--- a/README.md
+++ b/README.md
@@ -136,6 +136,7 @@ files within it's directory.
- die: print message and exit 1
- spushd/spopd: Safe verisons of pushd & popd that call die if the push/pop fails, they also drop stdout.
+ - retry: Retry a command until it succeeds up to a user specified maximum number of attempts. Escalating delay between attempts.
diff --git a/filehandling/lib b/filehandling/lib
index 4cd3f47..f06c153 100644
--- a/filehandling/lib
+++ b/filehandling/lib
@@ -1,24 +1,28 @@
#!/bin/bash
: "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}"
-. "${BASH_LIB_DIR}/helpers/lib"
#https://stackoverflow.com/a/23002317
function abs_path() {
# generate absolute path from relative path
- # $1 : relative filename
+ # path : relative filename
# return : absolute path
- if [ -d "$1" ]; then
+ if [[ -z "${1:-}" ]]; then
+ path="."
+ else
+ path="${1}"
+ fi
+ if [ -d "${path}" ]; then
# dir
- (spushd "$1"; pwd)
- elif [ -f "$1" ]; then
+ (spushd "${path}"; pwd)
+ elif [ -f "${path}" ]; then
# file
- if [[ $1 = /* ]]; then
- echo "$1"
- elif [[ $1 == */* ]]; then
- echo "$(spushd "${1%/*}"; pwd)/${1##*/}"
+ if [[ ${path} = /* ]]; then
+ echo "${path}"
+ elif [[ ${path} == */* ]]; then
+ echo "$(spushd "${path%/*}"; pwd)/${path##*/}"
else
- echo "$(pwd)/$1"
+ echo "$(pwd)/${path}"
fi
fi
-}
\ No newline at end of file
+}
diff --git a/git/lib b/git/lib
index 65ea05a..b5922fe 100644
--- a/git/lib
+++ b/git/lib
@@ -1,7 +1,6 @@
#!/bin/bash
: "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}"
-. "${BASH_LIB_DIR}/helpers/lib"
# Get the top level of a git repo
function repo_root(){
@@ -76,4 +75,4 @@ space seperated with three fields: subtree_path renmote_url remote_name"
function tracked_files_excluding_subtrees(){
subtrees="$(cat_gittrees | awk '{print $1}' | paste -sd '|' -)"
all_files_in_repo | grep -E -v "${subtrees}"
-}
\ No newline at end of file
+}
diff --git a/helpers/lib b/helpers/lib
index f4cf58e..6bea6e4 100644
--- a/helpers/lib
+++ b/helpers/lib
@@ -17,4 +17,58 @@ function spushd(){
#safe popd
function spopd(){
popd >/dev/null || die "popd failed :("
-}
\ No newline at end of file
+}
+
+# Retry a command multiple times until it succeeds, with escalating
+# delay between attempts.
+# Delay is 2 * n + random up to 30s, then 30s + random after that.
+# For large numbers of retries the max delay is effectively the retry
+# in minutes.
+# Based on:
+# https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746
+# but now quite heavily modified.
+function retry {
+ # Maxiumum amount of fixed delay between attempts
+ # a random value will still be added.
+ local -r MAX_BACKOFF=30
+
+ if [[ ${#} -lt 2 ]]; then
+ echo "retry usage: retry "
+ exit 1
+ fi
+
+ local retries=$1
+ shift
+
+ if ! [[ ${retries} =~ ^[0-9\.]*$ ]]; then
+ echo "Invalid number of retries: ${retries} for command '${*}'".
+ exit 1
+ fi
+
+ local count=0
+ until "$@"; do
+ # Command failed, otherwise until would have skipped the loop
+
+ # Store return code so it can be reported to the user
+ exit=$?
+ count=$((count + 1))
+ if [ "${count}" -lt "${retries}" ]; then
+ # There are still retries left, calculate delay and notify user.
+ backoff=$((2 * count))
+ if [[ "${backoff}" -gt "${MAX_BACKOFF}" ]]; then
+ backoff=${MAX_BACKOFF}
+ fi;
+
+ # Add a random amount to the delay to prevent competing processes
+ # from re-colliding.
+ wait=$(( backoff + (RANDOM % count) ))
+ echo "'${*}' Retry $count/$retries exited $exit, retrying in $wait seconds..."
+ sleep $wait
+ else
+ # Out of retries :(
+ echo "Retry $count/$retries exited $exit, no more retries left."
+ return $exit
+ fi
+ done
+ return 0
+}
diff --git a/init b/init
index c55fd63..e514598 100644
--- a/init
+++ b/init
@@ -20,18 +20,18 @@ BASH_LIB_DIR="${BASH_LIB_DIR_RELATIVE}"
# Load the filehandling module for the abspath
# function
-. "${BASH_LIB_DIR_RELATIVE}/filehandling/lib"
+for lib in helpers logging filehandling git k8s test-utils; do
+ . "${BASH_LIB_DIR_RELATIVE}/${lib}/lib"
+done
# Export the absolute path
# shellcheck disable=SC2086
BASH_LIB_DIR="$(abs_path ${BASH_LIB_DIR_RELATIVE})"
export BASH_LIB_DIR
-. "${BASH_LIB_DIR}/helpers/lib"
-
# Update Submodules
spushd "${BASH_LIB_DIR}"
git submodule update --init --recursive
spopd
-export BATS_CMD="${BASH_LIB_DIR}/test-utils/bats/bin/bats"
\ No newline at end of file
+export BATS_CMD="${BASH_LIB_DIR}/test-utils/bats/bin/bats"
diff --git a/k8s/lib b/k8s/lib
index 6bea766..8e63900 100644
--- a/k8s/lib
+++ b/k8s/lib
@@ -1,7 +1,6 @@
#!/bin/bash
: "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}"
-. "${BASH_LIB_DIR}/helpers/lib"
# Sets additional required environment variables that aren't available in the
# secrets.yml file, and performs other preparatory steps
@@ -52,4 +51,4 @@ function run_docker_gke_command() {
/scripts/platform_login
${1}
"
-}
\ No newline at end of file
+}
diff --git a/run-tests b/run-tests
index f099ee4..3fea7ae 100755
--- a/run-tests
+++ b/run-tests
@@ -4,7 +4,6 @@
# is not assumed to have been run
# shellcheck disable=SC2086
. "$(dirname ${BASH_SOURCE[0]})/init"
-. "${BASH_LIB_DIR}/helpers/lib"
# Run BATS Tests
"${BASH_LIB_DIR}/tests-for-this-repo/run-bats-tests"
diff --git a/test-utils/lib b/test-utils/lib
index 5eab41f..31c49e0 100644
--- a/test-utils/lib
+++ b/test-utils/lib
@@ -1,11 +1,9 @@
#!/bin/bash
: "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}"
-. "${BASH_LIB_DIR}/git/lib"
-. "${BASH_LIB_DIR}/helpers/lib"
-readonly SHELLCHECK_IMAGE="${SHELLCHECK_IMAGE:-koalaman/shellcheck}"
-readonly SHELLCHECK_TAG="${SHELLCHECK_TAG:-v0.6.0}"
+SHELLCHECK_IMAGE="${SHELLCHECK_IMAGE:-koalaman/shellcheck}"
+SHELLCHECK_TAG="${SHELLCHECK_TAG:-v0.6.0}"
# Check a single shell script for syntax
# and common errors.
diff --git a/tests-for-this-repo/filehandling.bats b/tests-for-this-repo/filehandling.bats
index 1b309fb..4492078 100644
--- a/tests-for-this-repo/filehandling.bats
+++ b/tests-for-this-repo/filehandling.bats
@@ -1,7 +1,7 @@
. "${BASH_LIB_DIR}/test-utils/bats-support/load.bash"
. "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash"
-. "${BASH_LIB_DIR}/filehandling/lib"
+. "${BASH_LIB_DIR}/init"
@test "abs_path returns absolute path for PWD" {
run abs_path .
@@ -9,8 +9,14 @@
assert_success
}
+@test "abs_path returns PWD when no arg specified" {
+ run abs_path
+ assert_output $PWD
+ assert_success
+}
+
@test "abs_path returns same path when already absolute" {
run abs_path /tmp
assert_output /tmp
assert_success
-}
\ No newline at end of file
+}
diff --git a/tests-for-this-repo/git.bats b/tests-for-this-repo/git.bats
index 7ba2a1c..9c18031 100644
--- a/tests-for-this-repo/git.bats
+++ b/tests-for-this-repo/git.bats
@@ -1,7 +1,7 @@
. "${BASH_LIB_DIR}/test-utils/bats-support/load.bash"
. "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash"
-. "${BASH_LIB_DIR}/git/lib"
+. "${BASH_LIB_DIR}/init"
# run before every test
setup(){
@@ -137,4 +137,4 @@ EOF
assert_success
assert_output --partial a_file
assert_success
-}
\ No newline at end of file
+}
diff --git a/tests-for-this-repo/helpers.bats b/tests-for-this-repo/helpers.bats
index af61b94..bbea6f8 100644
--- a/tests-for-this-repo/helpers.bats
+++ b/tests-for-this-repo/helpers.bats
@@ -1,35 +1,114 @@
. "${BASH_LIB_DIR}/test-utils/bats-support/load.bash"
. "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash"
-. "${BASH_LIB_DIR}/helpers/lib"
+. "${BASH_LIB_DIR}/init"
+
+# run before every test
+setup(){
+ temp_dir="${BATS_TMPDIR}/testtemp"
+ mkdir "${temp_dir}"
+ afile="${temp_dir}/appendfile"
+}
+
+teardown(){
+ temp_dir="${BATS_TMPDIR}/testtemp"
+ rm -rf "${temp_dir}"
+}
@test "die exits and prints message" {
- run bash -c ". ${BASH_LIB_DIR}/helpers/lib; die msg"
+ run bash -c ". ${BASH_LIB_DIR}/init; die msg"
assert_output msg
assert_failure
}
@test "spushd is quiet on stdout" {
run spushd /tmp
- refute_output
+ assert_output ""
assert_success
}
@test "spopd is quiet on stdout" {
pushd .
run spopd
- refute_output
+ assert_output ""
assert_success
}
@test "spushd dies on failure" {
- run bash -c ". ${BASH_LIB_DIR}/helpers/lib; spushd /this-doesnt-exist"
+ run bash -c ". ${BASH_LIB_DIR}/init; spushd /this-doesnt-exist"
assert_output --partial "No such file or directory"
assert_failure
}
@test "spopd dies on failure" {
- run bash -c ". ${BASH_LIB_DIR}/helpers/lib; spopd"
+ run bash -c ". ${BASH_LIB_DIR}/init; spopd"
assert_output --partial "stack empty"
assert_failure
-}
\ No newline at end of file
+}
+
+@test "retry runs command only once if it succeeds the first time" {
+ retryme(){
+ date >> ${afile}
+ }
+ run retry 3 retryme
+ assert_success
+ assert_equal $(wc -l <${afile}) 1
+}
+
+@test "retry doesn't introduce delay when the command succeeds first time" {
+ retryme(){
+ date >> ${afile}
+ }
+ start=$(date +%s)
+ run retry 3 retryme
+ end=$(date +%s)
+ assert [ "$(( start + 1 ))" -ge "${end}" ]
+ assert_success
+}
+
+@test "retry runs n times on consecutive failure and waits between attempts" {
+ retryme(){
+ date >> ${afile}
+ false
+ }
+ start=$(date +%s)
+ run retry 2 retryme
+ end=$(date +%s)
+ # introduces at least a two second delay between attempts
+ assert [ "$(( start + 2 ))" -le "${end}" ]
+ assert_failure
+ assert_equal $(wc -l <${afile}) 2
+}
+
+@test "retry returns after first success" {
+ retryme(){
+ date >> "${afile}"
+ case $(wc -l < ${afile}) in
+ *1)
+ return 1
+ ;;
+ *)
+ return 0
+ ;;
+ esac
+ }
+ run retry 3 retryme
+ assert_success
+ assert_equal $(wc -l <${afile}) 2
+}
+
+@test "retry fails with less than two arguments" {
+ run retry 3
+ assert_failure
+ assert_output --partial usage
+ assert [ ! -e "${temp_dir}/appendfile" ]
+}
+
+@test "retry fails with non-integer retry count" {
+ run retry "this" date
+ assert_failure
+ assert_output --partial number
+ assert [ ! -e "${temp_dir}/appendfile" ]
+}
+
+
diff --git a/tests-for-this-repo/k8s.bats b/tests-for-this-repo/k8s.bats
index 48e1bc2..92d3b71 100755
--- a/tests-for-this-repo/k8s.bats
+++ b/tests-for-this-repo/k8s.bats
@@ -1,7 +1,7 @@
. "${BASH_LIB_DIR}/test-utils/bats-support/load.bash"
. "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash"
-. "${BASH_LIB_DIR}/k8s/lib"
+. "${BASH_LIB_DIR}/init"
@test "gke-utils image builds" {
run build_gke_image
@@ -24,4 +24,4 @@
"
run delete_gke_image "${image}"
assert_success
-}
\ No newline at end of file
+}
diff --git a/tests-for-this-repo/lint.bats b/tests-for-this-repo/lint.bats
index dab5d97..bd2291d 100755
--- a/tests-for-this-repo/lint.bats
+++ b/tests-for-this-repo/lint.bats
@@ -1,6 +1,5 @@
-. ${BASH_LIB_DIR}/git/lib
-. ${BASH_LIB_DIR}/test-utils/lib
-. ${BASH_LIB_DIR}/helpers/lib
+
+. "${BASH_LIB_DIR}/init"
setup() {
spushd ${BASH_LIB_DIR}
@@ -66,4 +65,4 @@ setup() {
fi
done
return ${rc}
-}
\ No newline at end of file
+}
diff --git a/tests-for-this-repo/logging.bats b/tests-for-this-repo/logging.bats
index 47bc684..ccdb6d4 100644
--- a/tests-for-this-repo/logging.bats
+++ b/tests-for-this-repo/logging.bats
@@ -1,10 +1,9 @@
. "${BASH_LIB_DIR}/test-utils/bats-support/load.bash"
. "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash"
-
-. "${BASH_LIB_DIR}/logging/lib"
+. "${BASH_LIB_DIR}/init"
@test "announce prints all arguments" {
run announce one two one two
assert_output --partial "one two one two"
assert_success
-}
\ No newline at end of file
+}
diff --git a/tests-for-this-repo/run-bats-tests b/tests-for-this-repo/run-bats-tests
index 455319e..38026eb 100755
--- a/tests-for-this-repo/run-bats-tests
+++ b/tests-for-this-repo/run-bats-tests
@@ -8,8 +8,7 @@ set -euo pipefail
# is not assumed to have been run
# shellcheck disable=SC2086,SC2046
. $(dirname ${BASH_SOURCE[0]})/../init
-. "${BASH_LIB_DIR}/test-utils/lib"
-. "${BASH_LIB_DIR}/helpers/lib"
+set +eu
# Check vital tools are installed
@@ -66,4 +65,4 @@ if [[ "${BATS_OUTPUT_FORMAT}" == junit ]]; then
echo "Junit output written to ${JUNIT_FILE}"
fi
-exit ${rc}
\ No newline at end of file
+exit ${rc}
diff --git a/tests-for-this-repo/run-gitleaks b/tests-for-this-repo/run-gitleaks
index 0faf3fd..652085f 100755
--- a/tests-for-this-repo/run-gitleaks
+++ b/tests-for-this-repo/run-gitleaks
@@ -6,8 +6,6 @@
# is not assumed to have been run
# shellcheck disable=SC2086,SC2046
. $(dirname "${BASH_SOURCE[0]}")/../init
-. "${BASH_LIB_DIR}/test-utils/lib"
-. "${BASH_LIB_DIR}/helpers/lib"
docker run \
--rm \
diff --git a/tests-for-this-repo/run-python-lint b/tests-for-this-repo/run-python-lint
index c14e96b..c2ee68f 100755
--- a/tests-for-this-repo/run-python-lint
+++ b/tests-for-this-repo/run-python-lint
@@ -5,7 +5,6 @@
# shellcheck disable=SC2086,SC2046
. $(dirname ${BASH_SOURCE[0]})/../init
-. "${BASH_LIB_DIR}/helpers/lib"
rc=0
@@ -17,4 +16,4 @@ spushd ${BASH_LIB_DIR}/tests-for-this-repo/python-lint
mv "${BASH_LIB_DIR}/junit.xml" "${BASH_LIB_DIR}/python-lint-junit.xml"
spopd
-exit ${rc}
\ No newline at end of file
+exit ${rc}
diff --git a/tests-for-this-repo/test-utils.bats b/tests-for-this-repo/test-utils.bats
index c723c34..28ee32c 100644
--- a/tests-for-this-repo/test-utils.bats
+++ b/tests-for-this-repo/test-utils.bats
@@ -1,8 +1,6 @@
. "${BASH_LIB_DIR}/test-utils/bats-support/load.bash"
. "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash"
-
-. "${BASH_LIB_DIR}/helpers/lib"
-. "${BASH_LIB_DIR}/test-utils/lib"
+. "${BASH_LIB_DIR}/init"
docker_safe_tmp(){
# neither mktemp -d not $BATS_TMPDIR
@@ -77,4 +75,4 @@ docker_safe_tmp(){
rc=${?}
assert_equal "${stdout}" "$(cat ${fdir}/tap2junit.out)"
assert_equal "${rc}" "0"
-}
\ No newline at end of file
+}