Merge pull request #172 from cyphar/bats-parallel

bats: support parallel execution of tests

Dockerfile

@@ -2,8 +2,12 @@ ARG bashver=latest
 
 FROM bash:${bashver}
 
-RUN ln -s /opt/bats/bin/bats /usr/sbin/bats
+# Install parallel and accept the citation notice (we aren't using this in a
+# context where it make sense to cite GNU Parallel).
+RUN apk add --no-cache parallel && \
+    mkdir -p ~/.parallel && touch ~/.parallel/will-cite
 
+RUN ln -s /opt/bats/bin/bats /usr/sbin/bats
 COPY . /opt/bats/
 
 ENTRYPOINT ["bash", "/usr/sbin/bats"]

README.md

@@ -190,7 +190,7 @@ supports:
 
 ```
 Bats x.y.z
-Usage: bats [-cr] [-f <regex>] [-p | -t] <test>...
+Usage: bats [-cr] [-f <regex>] [-j <jobs>] [-p | -t] <test>...
        bats [-h | -v]
 
   <test> is the path to a Bats test file, or the path to a directory
@@ -199,6 +199,7 @@ Usage: bats [-cr] [-f <regex>] [-p | -t] <test>...
   -c, --count      Count the number of test cases without running any tests
   -f, --filter     Filter test cases by names matching the regular expression
   -h, --help       Display this help message
+  -j, --jobs       Number of parallel jobs to run (requires GNU parallel)
   -p, --pretty     Show results in pretty format (default for terminals)
   -r, --recursive  Include tests in subdirectories
   -t, --tap        Show results in TAP format
@@ -237,6 +238,21 @@ option.
     ok 1 addition using bc
     ok 2 addition using dc
 
+### Parallel Execution
+
+By default, Bats will execute your tests serially. However, Bats supports
+parallel execution of tests (provided you have [GNU parallel][gnu-parallel] or
+a compatible replacement installed) using the `--jobs` parameter. This can
+result in your tests completing faster (depending on your tests and the testing
+hardware).
+
+Ordering of parallised tests is not guaranteed, so this mode may break suites
+with dependencies between tests (or tests that write to shared locations). When
+enabling `--jobs` for the first time be sure to re-run bats multiple times to
+identify any inter-test dependencies or non-deterministic test behaviour.
+
+[gnu-parallel]: https://www.gnu.org/software/parallel/
+
 ## Writing tests
 
 Each Bats test file is evaluated _n+1_ times, where _n_ is the number of

libexec/bats-core/bats

@@ -20,7 +20,7 @@ usage() {
   while IFS= read -r line; do
     printf '%s\n' "$line"
   done <<END_OF_HELP_TEXT
-Usage: $cmd [-cr] [-f <regex>] [-p | -t] <test>...
+Usage: $cmd [-cr] [-f <regex>] [-j <jobs>] [-p | -t] <test>...
        $cmd [-h | -v]
 
   <test> is the path to a Bats test file, or the path to a directory
@@ -29,6 +29,7 @@ Usage: $cmd [-cr] [-f <regex>] [-p | -t] <test>...
   -c, --count      Count the number of test cases without running any tests
   -f, --filter     Filter test cases by names matching the regular expression
   -h, --help       Display this help message
+  -j, --jobs       Number of parallel jobs to run (requires GNU parallel)
   -p, --pretty     Show results in pretty format (default for terminals)
   -r, --recursive  Include tests in subdirectories
   -t, --tap        Show results in TAP format
@@ -39,6 +40,11 @@ Usage: $cmd [-cr] [-f <regex>] [-p | -t] <test>...
 END_OF_HELP_TEXT
 }
 
+expand_link() {
+  readlink="$(type -p greadlink readlink | head -1)"
+  "$readlink" -f "$1"
+}
+
 expand_path() {
   local path="${1%/}"
   local dirname="${path%/*}"
@@ -54,10 +60,11 @@ expand_path() {
   printf -v "$result" '%s/%s' "$dirname" "${path##*/}"
 }
 
+BATS_LIBEXEC="$(dirname "$(expand_link "$BASH_SOURCE")")"
 export BATS_CWD="$PWD"
 export BATS_TEST_PATTERN="^[[:blank:]]*@test[[:blank:]]+(.*[^[:blank:]])[[:blank:]]+\{(.*)\$"
 export BATS_TEST_FILTER=
-export PATH="$BATS_ROOT/libexec/bats-core:$PATH"
+export PATH="$BATS_LIBEXEC:$PATH"
 
 arguments=()
 
@@ -106,6 +113,10 @@ while [[ "$#" -ne 0 ]]; do
     shift
     flags+=('-f' "$1")
     ;;
+  -j|--jobs)
+    shift
+    flags+=('-j' "$1")
+    ;;
   -r|--recursive)
     recursive=1
     ;;
@@ -150,15 +161,11 @@ for filename in "${arguments[@]}"; do
   fi
 done
 
-if [[ "${#filenames[@]}" -eq 1 ]]; then
-  command='bats-exec-test'
-else
-  command='bats-exec-suite'
+formatter="cat"
+if [[ -n "$pretty" ]]; then
+  flags+=("-x")
+  formatter="bats-format-tap-stream"
 fi
 
 set -o pipefail execfail
-if [[ -z "$pretty" ]]; then
-  exec "$command" "${flags[@]}" "${filenames[@]}"
-else
-  exec "$command" -x "${flags[@]}" "${filenames[@]}" | bats-format-tap-stream
-fi
+exec bats-exec-suite "${flags[@]}" "${filenames[@]}" | "$formatter"

libexec/bats-core/bats-exec-suite

@@ -4,6 +4,8 @@ set -e
 count_only_flag=''
 extended_syntax_flag=''
 filter=''
+num_jobs=1
+have_gnu_parallel=
 flags=()
 
 while [[ "$#" -ne 0 ]]; do
@@ -12,9 +14,13 @@ while [[ "$#" -ne 0 ]]; do
     count_only_flag=1
     ;;
   -f)
-    filter="$2"
+    shift
+    filter="$1"
     flags+=('-f' "$filter")
+    ;;
+  -j)
     shift
+    num_jobs="$1"
     ;;
   -x)
     extended_syntax_flag='-x'
@@ -27,55 +33,69 @@ while [[ "$#" -ne 0 ]]; do
   shift
 done
 
+if ( type -p parallel &>/dev/null ); then
+  have_gnu_parallel=1
+elif [[ "$num_jobs" != 1 ]]; then
+  printf 'bats: cannot execute "%s" jobs without GNU parallel\n' "$num_jobs" >&2
+  exit 1
+fi
+
 trap 'kill 0; exit 1' int
 
-count=0
+all_tests=()
 for filename in "$@"; do
-  while IFS= read -r line; do
-    if [[ "$line" =~ $BATS_TEST_PATTERN ]]; then
-      test_name="${BASH_REMATCH[1]#[\'\"]}"
-      test_name="${test_name%[\'\"]}"
-      if [[ -z "$filter" || "$test_name" =~ $filter ]]; then
-        ((++count))
-      fi
+  if  [[ ! -f "$filename" ]]; then
+    printf 'bats: %s does not exist\n' "$filename" >&2
+    exit 1
+  fi
+
+  test_names=()
+  test_dupes=()
+  while read -r line; do
+    if [[ ! "$line" =~ ^bats_test_function\  ]]; then
+      continue
+    fi
+    line="${line%$'\r'}"
+    line="${line#* }"
+
+    all_tests+=( "$(printf "%s\t%s" "$filename" "$line")" )
+    if [[ " ${test_names[*]} " == *" $line "* ]]; then
+      test_dupes+=("$line")
+      continue
     fi
-  done <"$filename"
+    test_names+=("$line")
+  done < <(BATS_TEST_FILTER="$filter" bats-preprocess "$filename")
+
+  if [[ "${#test_dupes[@]}" -ne 0 ]]; then
+    printf 'bats warning: duplicate test name(s) in %s: %s\n' "$filename" "${test_dupes[*]}" >&2
+  fi
 done
 
 if [[ -n "$count_only_flag" ]]; then
-  printf '%d\n' "$count"
+  printf '%d\n' "${#all_tests[@]}"
   exit
 fi
 
-printf '1..%d\n' "$count"
 status=0
-offset=0
-for filename in "$@"; do
-  index=0
-  {
-    IFS= read -r # 1..n
-    while IFS= read -r line; do
-      case "$line" in
-      'begin '* )
-        ((++index))
-        printf '%s\n' "${line/ $index / $(($offset + $index)) }"
-        ;;
-      'ok '* | 'not ok '* )
-        if [[ -z "$extended_syntax_flag" ]]; then
-          ((++index))
-        fi
-        printf '%s\n' "${line/ $index / $(($offset + $index)) }"
-        if [[ "${line:0:6}" == 'not ok' ]]; then
-          status=1
-        fi
-        ;;
-      * )
-        printf '%s\n' "$line"
-        ;;
-      esac
-    done
-  } < <( bats-exec-test "${flags[@]}" "$filename" )
-  offset=$(($offset + $index))
-done
+printf '1..%d\n' "${#all_tests[@]}"
 
+# No point on continuing if there's no tests.
+if [[ "${#all_tests[@]}" == 0 ]]; then
+  exit
+fi
+
+if [[ "$num_jobs" != 1 ]]; then
+  # Only use GNU parallel when we want parallel execution -- there is a small
+  # amount of overhead using it over a simple loop in the serial case.
+  set -o pipefail
+  printf '%s\n' "${all_tests[@]}" | grep -v '^$' | \
+    parallel -qk -j "$num_jobs" --colsep="\t" -- bats-exec-test "${flags[@]}" '{1}' '{2}' '{#}' || status=1
+else
+  # Just do it serially.
+  test_number=1
+  while IFS=$'\t' read -r filename test_name; do
+    bats-exec-test "${flags[@]}" "$filename" "$test_name" "$test_number" || status=1
+    ((++test_number))
+  done < <(printf '%s\n' "${all_tests[@]}" | grep -v '^$')
+fi
 exit "$status"