All libraries are loaded by init script & retry function added

README.md

@@ -136,6 +136,7 @@ files within it's directory.
         <ol>
           <li><b>die</b>: print message and exit 1</li>
           <li><b>spushd/spopd</b>: Safe verisons of pushd & popd that call die if the push/pop fails, they also drop stdout. </li>
+          <li><b>retry</b>: Retry a command until it succeeds up to a user specified maximum number of attempts. Escalating delay between attempts.</li>
         </ol>
       </td>
     </tr>

filehandling/lib

@@ -1,24 +1,28 @@
 #!/bin/bash
 
 : "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}"
-. "${BASH_LIB_DIR}/helpers/lib"
 
 #https://stackoverflow.com/a/23002317
 function abs_path() {
     # generate absolute path from relative path
-    # $1     : relative filename
+    # path     : relative filename
     # return : absolute path
-    if [ -d "$1" ]; then
+    if [[ -z "${1:-}" ]]; then
+        path="."
+    else
+        path="${1}"
+    fi
+    if [ -d "${path}" ]; then
         # dir
-        (spushd "$1"; pwd)
-    elif [ -f "$1" ]; then
+        (spushd "${path}"; pwd)
+    elif [ -f "${path}" ]; then
         # file
-        if [[ $1 = /* ]]; then
-            echo "$1"
-        elif [[ $1 == */* ]]; then
-            echo "$(spushd "${1%/*}"; pwd)/${1##*/}"
+        if [[ ${path} = /* ]]; then
+            echo "${path}"
+        elif [[ ${path} == */* ]]; then
+            echo "$(spushd "${path%/*}"; pwd)/${path##*/}"
         else
-            echo "$(pwd)/$1"
+            echo "$(pwd)/${path}"
         fi
     fi
-}
+}

git/lib

@@ -1,7 +1,6 @@
 #!/bin/bash
 
 : "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}"
-. "${BASH_LIB_DIR}/helpers/lib"
 
 # Get the top level of a git repo
 function repo_root(){
@@ -76,4 +75,4 @@ space seperated with three fields: subtree_path renmote_url remote_name"
 function tracked_files_excluding_subtrees(){
     subtrees="$(cat_gittrees | awk '{print $1}' | paste -sd '|' -)"
     all_files_in_repo | grep -E -v "${subtrees}"
-}
+}

helpers/lib

@@ -17,4 +17,58 @@ function spushd(){
 #safe popd
 function spopd(){
     popd >/dev/null || die "popd failed :("
-}
+}
+
+# Retry a command multiple times until it succeeds, with escalating
+# delay between attempts.
+# Delay is 2 * n + random up to 30s, then 30s + random after that.
+# For large numbers of retries the max delay is effectively the retry
+# in minutes.
+# Based on:
+# https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746
+# but now quite heavily modified.
+function retry {
+    # Maxiumum amount of fixed delay between attempts
+    # a random value will still be added.
+    local -r MAX_BACKOFF=30
+
+    if [[ ${#} -lt 2 ]]; then
+        echo "retry usage: retry <retries> <command>"
+        exit 1
+    fi
+
+    local retries=$1
+    shift
+
+    if ! [[ ${retries} =~ ^[0-9\.]*$ ]]; then
+        echo "Invalid number of retries: ${retries} for command '${*}'".
+        exit 1
+    fi
+
+    local count=0
+    until "$@"; do
+        # Command failed, otherwise until would have skipped the loop
+
+        # Store return code so it can be reported to the user
+        exit=$?
+        count=$((count + 1))
+        if [ "${count}" -lt "${retries}" ]; then
+            # There are still retries left, calculate delay and notify user.
+            backoff=$((2 * count))
+            if [[ "${backoff}" -gt "${MAX_BACKOFF}" ]]; then
+                backoff=${MAX_BACKOFF}
+            fi;
+
+            # Add a random amount to the delay to prevent competing processes
+            # from re-colliding.
+            wait=$(( backoff + (RANDOM % count) ))
+            echo "'${*}' Retry $count/$retries exited $exit, retrying in $wait seconds..."
+            sleep $wait
+        else
+            # Out of retries :(
+            echo "Retry $count/$retries exited $exit, no more retries left."
+            return $exit
+        fi
+    done
+    return 0
+}

init

@@ -20,18 +20,18 @@ BASH_LIB_DIR="${BASH_LIB_DIR_RELATIVE}"
 
 # Load the filehandling module for the abspath
 # function
-. "${BASH_LIB_DIR_RELATIVE}/filehandling/lib"
+for lib in helpers logging filehandling git k8s test-utils; do
+    . "${BASH_LIB_DIR_RELATIVE}/${lib}/lib"
+done
 
 # Export the absolute path
 # shellcheck disable=SC2086
 BASH_LIB_DIR="$(abs_path ${BASH_LIB_DIR_RELATIVE})"
 export BASH_LIB_DIR
 
-. "${BASH_LIB_DIR}/helpers/lib"
-
 # Update Submodules
 spushd "${BASH_LIB_DIR}"
     git submodule update --init --recursive
 spopd
 
-export BATS_CMD="${BASH_LIB_DIR}/test-utils/bats/bin/bats"
+export BATS_CMD="${BASH_LIB_DIR}/test-utils/bats/bin/bats"

k8s/lib

@@ -1,7 +1,6 @@
 #!/bin/bash
 
 : "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}"
-. "${BASH_LIB_DIR}/helpers/lib"
 
 # Sets additional required environment variables that aren't available in the
 # secrets.yml file, and performs other preparatory steps
@@ -52,4 +51,4 @@ function run_docker_gke_command() {
       /scripts/platform_login
       ${1}
     "
-}
+}

run-tests

@@ -4,7 +4,6 @@
 # is not assumed to have been run
 # shellcheck disable=SC2086
 . "$(dirname ${BASH_SOURCE[0]})/init"
-. "${BASH_LIB_DIR}/helpers/lib"
 
 # Run BATS Tests
 "${BASH_LIB_DIR}/tests-for-this-repo/run-bats-tests"

test-utils/lib

@@ -1,11 +1,9 @@
 #!/bin/bash
 
 : "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}"
-. "${BASH_LIB_DIR}/git/lib"
-. "${BASH_LIB_DIR}/helpers/lib"
 
-readonly SHELLCHECK_IMAGE="${SHELLCHECK_IMAGE:-koalaman/shellcheck}"
-readonly SHELLCHECK_TAG="${SHELLCHECK_TAG:-v0.6.0}"
+SHELLCHECK_IMAGE="${SHELLCHECK_IMAGE:-koalaman/shellcheck}"
+SHELLCHECK_TAG="${SHELLCHECK_TAG:-v0.6.0}"
 
 # Check a single shell script for syntax
 # and common errors.

tests-for-this-repo/filehandling.bats

@@ -1,16 +1,22 @@
 . "${BASH_LIB_DIR}/test-utils/bats-support/load.bash"
 . "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash"
 
-. "${BASH_LIB_DIR}/filehandling/lib"
+. "${BASH_LIB_DIR}/init"
 
 @test "abs_path returns absolute path for PWD" {
     run abs_path .
     assert_output $PWD
     assert_success
 }
 
+@test "abs_path returns PWD when no arg specified" {
+    run abs_path
+    assert_output $PWD
+    assert_success
+}
+
 @test "abs_path returns same path when already absolute" {
     run abs_path /tmp
     assert_output /tmp
     assert_success
-}
+}

tests-for-this-repo/git.bats

@@ -1,7 +1,7 @@
 . "${BASH_LIB_DIR}/test-utils/bats-support/load.bash"
 . "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash"
 
-. "${BASH_LIB_DIR}/git/lib"
+. "${BASH_LIB_DIR}/init"
 
 # run before every test
 setup(){
@@ -137,4 +137,4 @@ EOF
     assert_success
     assert_output --partial a_file
     assert_success
-}
+}

tests-for-this-repo/helpers.bats

@@ -1,35 +1,114 @@
 . "${BASH_LIB_DIR}/test-utils/bats-support/load.bash"
 . "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash"
 
-. "${BASH_LIB_DIR}/helpers/lib"
+. "${BASH_LIB_DIR}/init"
+
+# run before every test
+setup(){
+    temp_dir="${BATS_TMPDIR}/testtemp"
+    mkdir "${temp_dir}"
+    afile="${temp_dir}/appendfile"
+}
+
+teardown(){
+    temp_dir="${BATS_TMPDIR}/testtemp"
+    rm -rf "${temp_dir}"
+}
 
 @test "die exits and prints message" {
-    run bash -c ". ${BASH_LIB_DIR}/helpers/lib; die msg"
+    run bash -c ". ${BASH_LIB_DIR}/init; die msg"
     assert_output msg
     assert_failure
 }
 
 @test "spushd is quiet on stdout" {
     run spushd /tmp
-    refute_output
+    assert_output ""
     assert_success
 }
 
 @test "spopd is quiet on stdout" {
     pushd .
     run spopd
-    refute_output
+    assert_output ""
     assert_success
 }
 
 @test "spushd dies on failure" {
-    run bash -c ". ${BASH_LIB_DIR}/helpers/lib; spushd /this-doesnt-exist"
+    run bash -c ". ${BASH_LIB_DIR}/init; spushd /this-doesnt-exist"
     assert_output --partial "No such file or directory"
     assert_failure
 }
 
 @test "spopd dies on failure" {
-    run bash -c ". ${BASH_LIB_DIR}/helpers/lib; spopd"
+    run bash -c ". ${BASH_LIB_DIR}/init; spopd"
     assert_output --partial "stack empty"
     assert_failure
-}
+}
+
+@test "retry runs command only once if it succeeds the first time" {
+    retryme(){
+        date >> ${afile}
+    }
+    run retry 3 retryme
+    assert_success
+    assert_equal $(wc -l <${afile}) 1
+}
+
+@test "retry doesn't introduce delay when the command succeeds first time" {
+    retryme(){
+        date >> ${afile}
+    }
+    start=$(date +%s)
+    run retry 3 retryme
+    end=$(date +%s)
+    assert [ "$(( start + 1 ))" -ge "${end}" ]
+    assert_success
+}
+
+@test "retry runs n times on consecutive failure and waits between attempts" {
+    retryme(){
+        date >> ${afile}
+        false
+    }
+    start=$(date +%s)
+    run retry 2 retryme
+    end=$(date +%s)
+    # introduces at least a two second delay between attempts
+    assert [ "$(( start + 2 ))" -le "${end}" ]
+    assert_failure
+    assert_equal $(wc -l <${afile}) 2
+}
+
+@test "retry returns after first success" {
+    retryme(){
+        date >> "${afile}"
+        case $(wc -l < ${afile}) in
+            *1)
+                return 1
+            ;;
+            *)
+                return 0
+            ;;
+        esac
+    }
+    run retry 3 retryme
+    assert_success
+    assert_equal $(wc -l <${afile}) 2
+}
+
+@test "retry fails with less than two arguments" {
+    run retry 3
+    assert_failure
+    assert_output --partial usage
+    assert [ ! -e "${temp_dir}/appendfile" ]
+}
+
+@test "retry fails with non-integer retry count" {
+    run retry "this" date
+    assert_failure
+    assert_output --partial number
+    assert [ ! -e "${temp_dir}/appendfile" ]
+}
+
+

tests-for-this-repo/k8s.bats

@@ -1,7 +1,7 @@
 . "${BASH_LIB_DIR}/test-utils/bats-support/load.bash"
 . "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash"
 
-. "${BASH_LIB_DIR}/k8s/lib"
+. "${BASH_LIB_DIR}/init"
 
 @test "gke-utils image builds" {
     run build_gke_image
@@ -24,4 +24,4 @@
     "
     run delete_gke_image "${image}"
     assert_success
-}
+}

tests-for-this-repo/lint.bats

@@ -1,6 +1,5 @@
-. ${BASH_LIB_DIR}/git/lib
-. ${BASH_LIB_DIR}/test-utils/lib
-. ${BASH_LIB_DIR}/helpers/lib
+
+. "${BASH_LIB_DIR}/init"
 
 setup() {
     spushd ${BASH_LIB_DIR}
@@ -66,4 +65,4 @@ setup() {
         fi
     done
     return ${rc}
-}
+}

tests-for-this-repo/logging.bats

@@ -1,10 +1,9 @@
 . "${BASH_LIB_DIR}/test-utils/bats-support/load.bash"
 . "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash"
-
-. "${BASH_LIB_DIR}/logging/lib"
+. "${BASH_LIB_DIR}/init"
 
 @test "announce prints all arguments" {
     run announce one two one two
     assert_output --partial "one two one two"
     assert_success
-}
+}