Merge pull request #367 from edsantiago/run_extended

run: add optional exit-status test

docs/source/writing-tests.md

@@ -35,14 +35,30 @@ The `$status` variable contains the status code of the command, and the
 `$output` variable contains the combined contents of the command's standard
 output and standard error streams.
 
+If invoked with one of the following as the first argument, `run`
+will perform an implicit check on the exit status of the invoked command:
+
+```pre
+    =N  expect exit status N (0-255), fail if otherwise
+    ! expect nonzero exit status (1-255), fail if command succeeds
+```
+
+We can then write the above more elegantly as:
+
+```bash
+@test "invoking foo with a nonexistent file prints an error" {
+  run =1 foo nonexistent_filename
+  [ "$output" = "foo: no such file 'nonexistent_filename'" ]
+}
+```
+
 A third special variable, the `$lines` array, is available for easily accessing
 individual lines of output. For example, if you want to test that invoking `foo`
 without any arguments prints usage information on the first line:
 
 ```bash
 @test "invoking foo without arguments prints usage" {
-  run foo
-  [ "$status" -eq 1 ]
+  run =1 foo
   [ "${lines[0]}" = "usage: foo <filename>" ]
 }
 ```
@@ -51,58 +67,6 @@ __Note:__ The `run` helper executes its argument(s) in a subshell, so if
 writing tests against environmental side-effects like a variable's value
 being changed, these changes will not persist after `run` completes.
 
-### When not to use `run`
-
-In some cases, using `run` is redundant and results in a longer and less readable code.
-Here are a few examples.
-
-#### 1. In case you only need to check the command succeeded, it is better to not use run, since
-
-```bash
-run command args ...
-echo "$output"
-[ "$status" -eq 0 ]
-```
-
-is equivalent to
-
-```bash
-command args ...
-```
-
-since bats sets `set -e` for all tests.
-
-#### 2. In case you want to hide the command output (which `run` does), use output redirection instead
-
-This
-
-```bash
-run command ...
-[ "$status" -eq 0 ]
-```
-
-is equivalent to
-
-```bash
-command ... >/dev/null
-```
-
-Note that the output is only shown if the test case fails.
-
-#### 3. In case you need to assign command output to a variable (and maybe check the command exit status), it is better to not use run, since
-
-```bash
-run command args ...
-[ "$status" -eq 0 ]
-var="$output"
-```
-
-is equivalent to
-
-```bash
-var=$(command args ...)
-```
-
 ## Comment syntax
 
 External tools (like `shellcheck`, `shfmt`, and various IDE's) may not support

lib/bats-core/test_functions.bash

@@ -63,13 +63,27 @@ bats_separate_lines() { # <output-array> <input-var>
   fi
 }
 
-run() { # [--keep-empty-lines] [--output merged|separate|stderr|stdout] [--] <command to run...>
+run() { # [!|=N] [--keep-empty-lines] [--output merged|separate|stderr|stdout] [--] <command to run...>
   trap bats_interrupt_trap_in_run INT
+  local expected_rc=
   local keep_empty_lines=
   local output_case=merged
   # parse options starting with -
-  while [[ $# -gt 0 && $1 == -* ]]; do
+  while [[ $# -gt 0 ]] && [[ $1 == -* || $1 == '!' || $1 == '='* ]]; do
     case "$1" in
+      '!')
+        expected_rc=-1
+      ;;
+      '='*)
+        expected_rc=${1#=}
+        if [[ $expected_rc =~ [^0-9] ]]; then
+          printf "Usage error: run: '=NNN' requires numeric NNN (got: %s)\n" "$expected_rc" >&2
+          return 1
+        elif [[ $expected_rc -gt 255 ]]; then
+          printf "Usage error: run: '=NNN': NNN must be <= 255 (got: %d)\n" "$expected_rc" >&2
+          return 1
+        fi
+      ;;
       --keep-empty-lines)
         keep_empty_lines=1
       ;;
@@ -111,20 +125,17 @@ run() { # [--keep-empty-lines] [--output merged|separate|stderr|stdout] [--] <co
   local origFlags="$-"
   set +eET
   local origIFS="$IFS"
+  status=0
   if [[ $keep_empty_lines ]]; then
     # 'output', 'status', 'lines' are global variables available to tests.
     # preserve trailing newlines by appending . and removing it later
     # shellcheck disable=SC2034
-    output="$($pre_command "$@"; status=$?; printf .; exit $status)"
-    # shellcheck disable=SC2034
-    status="$?"
+    output="$($pre_command "$@"; status=$?; printf .; exit $status)" || status="$?"    
     output="${output%.}"
   else
     # 'output', 'status', 'lines' are global variables available to tests.
     # shellcheck disable=SC2034
-    output="$($pre_command "$@")"
-    # shellcheck disable=SC2034
-    status="$?"
+    output="$($pre_command "$@")" || status="$?"
   fi
 
   bats_separate_lines lines output
@@ -146,6 +157,21 @@ run() { # [--keep-empty-lines] [--output merged|separate|stderr|stdout] [--] <co
 
   IFS="$origIFS"
   set "-$origFlags"
+
+  if [[ -n "$expected_rc" ]]; then
+    if [[ "$expected_rc" = "-1" ]]; then
+      if [[ "$status" -eq 0 ]]; then
+        bats_capture_stack_trace # fix backtrace
+        BATS_ERROR_SUFFIX=", expected nonzero exit code!"
+        return 1
+      fi
+    elif [ "$status" -ne "$expected_rc" ]; then
+      bats_capture_stack_trace # fix backtrace
+      # shellcheck disable=SC2034
+      BATS_ERROR_SUFFIX=", expected exit code $expected_rc, got $status"
+      return 1
+    fi
+  fi
 }
 
 setup() {

lib/bats-core/tracing.bash

@@ -5,6 +5,13 @@ bats_capture_stack_trace() {
 	local funcname
 	local i
 
+	# The last entry in the stack trace is not useful when en error occured:
+	# It is either duplicated (kinda correct) or has wrong line number (Bash < 4.4)
+	# Therefore we capture the stacktrace but use it only after the next debug
+	# trap fired.
+	# Expansion is required for empty arrays which otherwise error
+	BATS_CURRENT_STACK_TRACE=("${BATS_STACK_TRACE[@]+"${BATS_STACK_TRACE[@]}"}")
+
 	BATS_STACK_TRACE=()
 
 	for ((i = 2; i != ${#FUNCNAME[@]}; ++i)); do
@@ -80,9 +87,9 @@ bats_print_failed_command() {
 	printf '%s' "#   \`${failed_command}' "
 
 	if [[ "$BATS_ERROR_STATUS" -eq 1 ]]; then
-		printf 'failed\n'
+		printf 'failed%s\n' "$BATS_ERROR_SUFFIX"
 	else
-		printf 'failed with status %d\n' "$BATS_ERROR_STATUS"
+		printf 'failed with status %d%s\n' "$BATS_ERROR_STATUS" "$BATS_ERROR_SUFFIX"
 	fi
 }
 
@@ -151,12 +158,6 @@ bats_debug_trap() {
 	if [[ "$NORMALIZED_INPUT" != $NORMALIZED_BATS_ROOT/lib/* && 
 		  "$NORMALIZED_INPUT" != $NORMALIZED_BATS_ROOT/libexec/* &&
 		  "${BATS_INTERRUPTED-NOTSET}" == NOTSET ]]; then
-		# The last entry in the stack trace is not useful when en error occured:
-		# It is either duplicated (kinda correct) or has wrong line number (Bash < 4.4)
-		# Therefore we capture the stacktrace but use it only after the next debug
-		# trap fired.
-		# Expansion is required for empty arrays which otherwise error
-		BATS_CURRENT_STACK_TRACE=("${BATS_STACK_TRACE[@]+"${BATS_STACK_TRACE[@]}"}")
 		bats_capture_stack_trace
 	fi
 }

libexec/bats-core/bats-exec-test

@@ -162,6 +162,7 @@ bats_perform_test() {
   BATS_TEST_SKIPPED=
   BATS_TEARDOWN_COMPLETED=
   BATS_ERROR_STATUS=
+  BATS_ERROR_SUFFIX=
   trap 'bats_debug_trap "$BASH_SOURCE"' DEBUG
   trap 'bats_error_trap' ERR
   # mark this call as trap call

man/bats.7.ronn

@@ -34,19 +34,31 @@ THE RUN HELPER
 Many Bats tests need to run a command and then make assertions about
 its exit status and output. Bats includes a `run` helper that invokes
 its arguments as a command, saves the exit status and output into
-special global variables, and then returns with a `0` status code so
-you can continue to make assertions in your test case.
+special global variables, and (optionally) checks exit status against
+a given expected value. If successful, `run` returns with a `0` status
+code so you can continue to make assertions in your test case.
 
 For example, let's say you're testing that the `foo` command, when
 passed a nonexistent filename, exits with a `1` status code and prints
 an error message.
 
     @test "invoking foo with a nonexistent file prints an error" {
-      run foo nonexistent_filename
-      [ "$status" -eq 1 ]
+      run =1 foo nonexistent_filename
       [ "$output" = "foo: no such file 'nonexistent_filename'" ]
     }
 
+The `=1` as first argument tells `run` to expect 1 as an exit
+status, and to fail if the command exits with any other value.
+On failure, both actual and expected values will be displayed,
+along with the invoked command and its output:
+
+    (in test file test.bats, line 2)
+     `run =1 foo nonexistent_filename' failed, expected exit code 1, got 127
+
+This error indicates a possible problem with the installation or
+configuration of `foo`; note that a simple `[ $status != 0 ]`
+test would not have caught this kind of failure.
+
 The `$status` variable contains the status code of the command, and
 the `$output` variable contains the combined contents of the command's
 standard output and standard error streams.
@@ -57,8 +69,7 @@ that invoking `foo` without any arguments prints usage information on
 the first line:
 
     @test "invoking foo without arguments prints usage" {
-      run foo
-      [ "$status" -eq 1 ]
+      run =1 foo
       [ "${lines[0]}" = "usage: foo <filename>" ]
     }
 
@@ -86,16 +97,14 @@ test you wish to skip.
 
     @test "A test I don't want to execute for now" {
       skip
-      run foo
-      [ "$status" -eq 0 ]
+      run =0 foo
     }
 
 Optionally, you may include a reason for skipping:
 
     @test "A test I don't want to execute for now" {
       skip "This command will return zero soon, but not now"
-      run foo
-      [ "$status" -eq 0 ]
+      run =0 foo
     }
 
 Or you can skip conditionally:
@@ -105,8 +114,7 @@ Or you can skip conditionally:
         skip "foo isn't bar"
       fi
 
-      run foo
-      [ "$status" -eq 0 ]
+      run =0 foo
     }
 
 

test/fixtures/run/failing.bats

@@ -0,0 +1,22 @@
+@test "run =0 false" {
+  run =0 false
+}
+
+@test "run =1 echo hi" {
+  run =1 echo hi
+}
+
+@test "run =2 exit 3" {
+  run =2 exit 3
+}
+
+@test "run ! true" {
+  run ! true
+}
+
+@test "run multiple pass/fails" {
+  run ! false
+  run =0 echo hi
+  run =127 /no/such/cmd
+  run =1 /etc
+}

test/fixtures/run/invalid.bats

@@ -0,0 +1,7 @@
+@test "run =4evah echo hi" {
+  run =4evah echo hi
+}
+
+@test "run =256 echo hi" {
+  run =256 echo hi
+}

test/run.bats

@@ -1,3 +1,6 @@
+load test_helper
+fixtures run
+
 @test "run --keep-empty-lines preserves leading empty lines" {
     run --keep-empty-lines -- echo -n $'\na'
     printf "'%s'\n" "${lines[@]}"
@@ -80,4 +83,52 @@ print-stderr-stdout() {
     run true
     echo -- "$-" == "$old_flags"
     [ "$-" == "$old_flags" ]
-}
+}
+
+# Positive assertions: each of these should succeed
+@test "basic return-code checking" {
+  run      true
+  run =0   true
+  run '!'  false
+  run =1   false
+  run =3   exit 3
+  run =5   exit 5
+  run =111 exit 111
+  run =255 exit 255
+  run =127 /no/such/command
+}
+
+@test "run exit code check output " {
+  run ! bats --tap "${FIXTURE_ROOT}/failing.bats"
+  echo "$output"
+  [ "${lines[0]}" == 1..5 ]
+  [ "${lines[1]}" == "not ok 1 run =0 false" ]
+  [ "${lines[2]}" == "# (in test file ${RELATIVE_FIXTURE_ROOT}/failing.bats, line 2)" ]
+  [ "${lines[3]}" == "#   \`run =0 false' failed, expected exit code 0, got 1" ]
+  [ "${lines[4]}" == "not ok 2 run =1 echo hi" ]
+  [ "${lines[5]}" == "# (in test file ${RELATIVE_FIXTURE_ROOT}/failing.bats, line 6)" ]
+  [ "${lines[6]}" == "#   \`run =1 echo hi' failed, expected exit code 1, got 0" ]
+  [ "${lines[7]}" == "not ok 3 run =2 exit 3" ]
+  [ "${lines[8]}" == "# (in test file ${RELATIVE_FIXTURE_ROOT}/failing.bats, line 10)" ]
+  [ "${lines[9]}" == "#   \`run =2 exit 3' failed, expected exit code 2, got 3" ]
+  [ "${lines[10]}" == "not ok 4 run ! true" ]
+  [ "${lines[11]}" == "# (in test file ${RELATIVE_FIXTURE_ROOT}/failing.bats, line 14)" ]
+  [ "${lines[12]}" == "#   \`run ! true' failed, expected nonzero exit code!" ]
+  [ "${lines[13]}" == "not ok 5 run multiple pass/fails" ]
+  [ "${lines[14]}" == "# (in test file ${RELATIVE_FIXTURE_ROOT}/failing.bats, line 21)" ]
+  [ "${lines[15]}" == "#   \`run =1 /etc' failed, expected exit code 1, got 126" ]
+}
+
+@test "run invalid exit code check error message" {
+  run ! bats --tap "${FIXTURE_ROOT}/invalid.bats"
+  echo "$output"
+  [ "${lines[0]}" == 1..2 ]
+  [ "${lines[1]}" == "not ok 1 run =4evah echo hi" ]
+  [ "${lines[2]}" == "# (in test file ${RELATIVE_FIXTURE_ROOT}/invalid.bats, line 2)" ]
+  [ "${lines[3]}" == "#   \`run =4evah echo hi' failed" ]
+  [ "${lines[4]}" == "# Usage error: run: '=NNN' requires numeric NNN (got: 4evah)" ]
+  [ "${lines[5]}" == "not ok 2 run =256 echo hi" ]
+  [ "${lines[6]}" == "# (in test file ${RELATIVE_FIXTURE_ROOT}/invalid.bats, line 6)" ]
+  [ "${lines[7]}" == "#   \`run =256 echo hi' failed" ]
+  [ "${lines[8]}" == "# Usage error: run: '=NNN': NNN must be <= 255 (got: 256)" ]
+}