Merge pull request #100 from shellspec/improve_error_handling_for_bef…

…ore_after_all_hooks

Improve error handling for BeforeAll/AfterAll hooks

docs/reporter.md

@@ -2,7 +2,6 @@
 
 ## Reporting protocol
 
-```
 | pair   | type      | number of times                 |
 | ------ | --------- | ------------------------------- |
 |        | meta      | 1                               |
@@ -12,7 +11,8 @@
 | :  +-- | result    | same number of times as example |
 | +----- | end       | same number of times as begin   |
 |        | finished  | 1                               |
-```
+|        |           |                                 |
+|        | error     | 0..N (can occur at any time)    |
 
 ### types
 
@@ -21,10 +21,11 @@
 | meta      | shell, shell_type, shell_version, info                       |
 | begin     | specfile                                                     |
 | example   | id, block_no, example_no, focused, description, lineno_range |
-| statement | tag, lineno                                                  |
-| result    |                                                              |
+| statement | tag, lineno, [statement type fields...]                      |
+| result    | [result type fields...]                                      |
 | end       |                                                              |
 | finished  |                                                              |
+| error     | lineno, note, message, failure_message                       |
 
 ### statement type
 

lib/core/dsl.sh

@@ -26,6 +26,7 @@ SHELLSPEC_SKIP_ID=''
 SHELLSPEC_SKIP_REASON=''
 SHELLSPEC_PENDING_REASON=''
 SHELLSPEC_SHELL_OPTIONS=''
+SHELLSPEC_HOOK_ERROR=''
 
 shellspec_group_id() {
   # shellcheck disable=SC2034
@@ -82,12 +83,17 @@ shellspec_before_first_block() {
   shellspec_mark_group "$SHELLSPEC_BLOCK_NO" ""
   [ "$SHELLSPEC_DRYRUN" ] && return 0
   shellspec_if SKIP && return 0
-  shellspec_call_before_hooks ALL
+  if ! shellspec_call_before_hooks ALL; then
+    shellspec_output BEFORE_ALL_ERROR
+    SHELLSPEC_HOOK_ERROR=1
+  fi
   shellspec_mark_group "$SHELLSPEC_BLOCK_NO" 1
 }
 
 shellspec_after_last_block() {
-  shellspec_call_after_hooks ALL
+  if ! shellspec_call_after_hooks ALL; then
+    shellspec_output AFTER_ALL_ERROR
+  fi
 }
 
 shellspec_after_block() {
@@ -193,6 +199,11 @@ shellspec_invoke_example() {
 
   # Output SKIP message if skipped in outer group.
   if ! shellspec_output_if SKIP; then
+    if [ "$SHELLSPEC_HOOK_ERROR" ]; then
+      shellspec_output HOOK_ERROR "$SHELLSPEC_HOOK_ERROR"
+      shellspec_output FAILED
+      return 0
+    fi
     if ! shellspec_call_before_hooks EACH; then
       shellspec_output BEFORE_EACH_ERROR
       shellspec_output FAILED

lib/core/output.sh

@@ -58,6 +58,10 @@ shellspec_output_result() {
   shellspec_output_raw type:result "$@"
 }
 
+shellspec_output_error() {
+  shellspec_output_raw type:error "$@"
+}
+
 shellspec_output_if() {
   shellspec_if "$1" || return 1
   shellspec_output "$@"

lib/core/outputs.sh

@@ -104,26 +104,53 @@ shellspec_output_ASSERT_ERROR() {
     "(exit status: $1)"
 }
 
+shellspec_output_BEFORE_ALL_ERROR() {
+  shellspec_readfile SHELLSPEC_BEFORE_ALL_ERROR "$SHELLSPEC_ERROR_FILE"
+  shellspec_chomp SHELLSPEC_BEFORE_ALL_ERROR
+  set -- "$SHELLSPEC_HOOK" "$SHELLSPEC_BEFORE_ALL_ERROR" \
+    "$SHELLSPEC_HOOK_LINENO" "$SHELLSPEC_HOOK_STATUS"
+  shellspec_output_error "note:ERROR" "lineno:$SHELLSPEC_HOOK_LINENO" \
+    "message:An error occurred in before all hook '$1' (exit status: $4)"
+  shellspec_output_raw_append "failure_message:${2:-<no error>}"
+}
+
+shellspec_output_AFTER_ALL_ERROR() {
+  shellspec_readfile SHELLSPEC_AFTER_ALL_ERROR "$SHELLSPEC_ERROR_FILE"
+  shellspec_chomp SHELLSPEC_AFTER_ALL_ERROR
+  set -- "$SHELLSPEC_HOOK" "$SHELLSPEC_AFTER_ALL_ERROR" \
+    "$SHELLSPEC_HOOK_LINENO" "$SHELLSPEC_HOOK_STATUS"
+  shellspec_output_error "note:ERROR" "lineno:$SHELLSPEC_HOOK_LINENO" \
+    "message:An error occurred in after hook '$1' (exit status: $4)"
+  shellspec_output_raw_append "failure_message:${2:-<no error>}"
+}
+
 shellspec_output_BEFORE_EACH_ERROR() {
   shellspec_readfile SHELLSPEC_BEFORE_EACH_ERROR "$SHELLSPEC_ERROR_FILE"
+  shellspec_chomp SHELLSPEC_BEFORE_EACH_ERROR
   set -- "$SHELLSPEC_HOOK" "$SHELLSPEC_BEFORE_EACH_ERROR" \
     "$SHELLSPEC_HOOK_LINENO" "$SHELLSPEC_HOOK_STATUS"
   SHELLSPEC_LINENO=$SHELLSPEC_LINENO_BEGIN-$SHELLSPEC_LINENO_END
-  shellspec_output_statement "tag:bad" "note:" "fail:y" \
+  shellspec_output_statement "tag:bad" "note:" "fail:y" "evaluation:" \
     "message:An error occurred in before hook '$1' (line: $3, exit status: $4)"
   shellspec_output_raw_append "failure_message:${2:-<no error>}"
 }
 
 shellspec_output_AFTER_EACH_ERROR() {
   shellspec_readfile SHELLSPEC_AFTER_EACH_ERROR "$SHELLSPEC_ERROR_FILE"
+  shellspec_chomp SHELLSPEC_AFTER_EACH_ERROR
   set -- "$SHELLSPEC_HOOK" "$SHELLSPEC_AFTER_EACH_ERROR" \
     "$SHELLSPEC_HOOK_LINENO" "$SHELLSPEC_HOOK_STATUS"
   SHELLSPEC_LINENO=$SHELLSPEC_LINENO_BEGIN-$SHELLSPEC_LINENO_END
-  shellspec_output_statement "tag:bad" "note:" "fail:y" \
+  shellspec_output_statement "tag:bad" "note:" "fail:y" "evaluation:" \
     "message:An error occurred in after hook '$1' (line: $3, exit status: $4)"
   shellspec_output_raw_append "failure_message:${2:-<no error>}"
 }
 
+shellspec_output_HOOK_ERROR() {
+  shellspec_output_statement "tag:bad" "note:" "fail:y" "failure_message:" \
+    "message:Treat as a failure due to a preceding hook error"
+}
+
 shellspec_output_MATCHED() {
   shellspec_if PENDING && shellspec_output_MATCHED_FIXED && return 0
   shellspec_output_statement "tag:good" "note:" "fail:" \

lib/general.sh

@@ -268,6 +268,7 @@ shellspec_padding_() {
 }
 
 shellspec_wrap() {
+  [ ! "$2" ] && eval "$1=" && return 0
   case $2 in
     *"$SHELLSPEC_LF") set -- "$1" "$2" "${3:-}" "${4:-}" "" "" ;;
     *) set -- "$1" "$2${SHELLSPEC_LF}" "${3:-}" "${4:-}" "" x ;;

lib/libexec/reporter/color_schema.sh

@@ -1,6 +1,4 @@
-#shellcheck shell=sh
-
-: "${field_type:-} ${field_tag:-} ${field_pending:-} ${field_color:-}"
+#shellcheck shell=sh disable=SC2154
 
 color_constants() {
   if [ "$1" ]; then
@@ -35,8 +33,8 @@ color_schema() {
         good      ) field_color=${GREEN} ;;
         warn      ) field_color=${YELLOW} ;;
         pending   ) field_color=${MAGENTA} ;;
-        bad       ) [ "$field_pending" ] && field_color=${YELLOW} || \
-                       field_color=${RED};;
+        bad       )
+          [ "$field_pending" ] && field_color=${YELLOW} || field_color=${RED} ;;
         *         ) field_color=${WHITE} ;;
       esac ;;
     result)
@@ -47,6 +45,9 @@ color_schema() {
         todo     ) field_color=${BOLD}${YELLOW} ;;
         fixed    ) field_color=${BOLD}${GREEN} ;;
         skipped  ) field_color=${BOLD}${MAGENTA} ;;
-      esac
+      esac ;;
+    error        )
+      # shellcheck disable=SC2034
+      field_color=${BOLD}${RED} ;;
   esac
 }

lib/libexec/reporter/conclusion_formatter.sh

@@ -1,12 +1,8 @@
-#shellcheck shell=sh
+#shellcheck shell=sh disable=SC2154
 
-: "${example_index:-} ${detail_index:-}"
-: "${field_specfile:-} ${field_lineno:-} ${field_color:-} ${field_type:-}"
-: "${field_tag:-} ${field_note:-} ${field_evaluation:-}"
-: "${field_message:-} ${field_failure_message:-} ${field_pending:-}"
-
-conclusion_evaluation=''
+conclusion_evaluation='' \
 conclusion_last_example_index='' conclusion_detail_index=''
+
 create_buffers conclusion
 
 conclusion_each() {
@@ -40,17 +36,31 @@ conclusion_each() {
       _label="${_indent}${example_index}.${conclusion_detail_index}) "
       padding _indent ' ' ${#_label}
 
-      _message="${field_note:+[}$field_note${field_note:+] }$field_message"
+      _message="${field_note}${field_note:+: }${field_message}"
       conclusion '+=' "${_label}${field_color}${_message}${RESET}${LF}${LF}"
 
       case $field_tag in (warn|bad)
         _message='' _before="  ${_indent}${field_color}" _after="${RESET}"
         wrap _message "$field_failure_message" "$_before" "$_after"
-        conclusion '+=' "${_message}${LF}"
+        [ "$_message" ] && conclusion '+=' "${_message}${LF}"
       esac
 
       conclusion '+=' "${_indent}${CYAN}#" \
         "${field_specfile}:${field_lineno}${RESET}${LF}${LF}"
+      ;;
+    error)
+      padding _indent ' ' ${#error_index}
+      _before="      ${_indent}${field_color}" _after="${RESET}"
+      conclusion '|=' "Examples:${LF}"
+
+      _message="${field_color}${field_note}${field_note:+: }${field_message}"
+      conclusion '+=' "  $error_index) ${_message}${RESET}${LF}${LF}"
+
+      wrap _message "$field_failure_message" "$_before" "$_after"
+      conclusion '+=' "${field_color}${_message}${RESET}${LF}"
+      conclusion '+=' "    ${_indent}${CYAN}#" \
+        "${field_specfile}:${field_lineno}${RESET}${LF}${LF}"
+      ;;
   esac
 }
 

lib/libexec/reporter/debug_formatter.sh

@@ -1,6 +1,4 @@
-#shellcheck shell=sh
-
-: "${field_type:-} ${field_color:-}"
+#shellcheck shell=sh disable=SC2154
 
 create_buffers debug
 
@@ -15,6 +13,7 @@ debug_each() {
     result   ) _mark='=' ;;
     end      ) _mark='$' ;;
     finished ) _mark='&' ;;
+    error    ) _mark='-' ;;
     *        ) _mark='?' ;;
   esac
 

lib/libexec/reporter/documentation_formatter.sh

@@ -1,9 +1,7 @@
-#shellcheck shell=sh
-
-: "${example_index:-} ${field_type:-} ${field_note:-}"
-: "${field_description:-} ${field_color:-} ${field_id:-}"
+#shellcheck shell=sh disable=SC2154
 
 documentation_last_id=''
+
 create_buffers documentation
 
 require_formatters methods conclusion finished summary references profiler

lib/libexec/reporter/failures_formatter.sh

@@ -1,17 +1,18 @@
-#shellcheck shell=sh disable=SC2004
-
-: "${field_type:-} ${field_tag:-} ${field_fail:-}"
-: "${field_note:-} ${field_specfile:-} ${field_lineno_range:-}"
-: "${temporary_skip:-}"
+#shellcheck shell=sh disable=SC2154
 
 create_buffers failures
 
 failures_each() {
-  case $field_type in (result)
-    [ "$field_tag" = "succeeded" ] && return 0
-    [ "$field_tag" = "skipped" ] && [ "$temporary_skip" -eq 0 ] && return 0
-    failures '=' "./$field_specfile:${field_lineno_range%-*}:${field_note}"
-    failures '+=' ":$(field_description)${LF}"
+  case $field_type in
+    result)
+      [ "$field_tag" = "succeeded" ] && return 0
+      [ "$field_tag" = "skipped" ] && [ "$temporary_skip" -eq 0 ] && return 0
+      failures '=' "./${field_specfile}:${field_lineno_range%-*}:${field_note}"
+      failures '+=' ":$(field_description)${LF}"
+      ;;
+    error)
+      failures '=' "./${field_specfile}:${field_lineno}:${field_note}"
+      failures '+=' ":${field_message}${LF}"
   esac
 }
 

lib/libexec/reporter/finished_formatter.sh

@@ -1,4 +1,4 @@
-#shellcheck shell=sh
+#shellcheck shell=sh disable=SC2154
 
 create_buffers finished
 

lib/libexec/reporter/junit_formatter.sh

@@ -1,15 +1,11 @@
-#shellcheck shell=sh disable=SC2004
+#shellcheck shell=sh disable=SC2154
 
-: "${field_type:-} ${field_fail:-} ${field_tag:-} ${field_description:-}"
-: "${field_lineno:-} ${field_specfile:-} ${field_message:-}"
-: "${field_failure_message:-} ${time_real:-}"
+junit_testsuite=0 junit_tests=0 junit_failures=0 junit_skipped=0 \
+junit_tests_total=0 junit_failures_total=0 junit_system_error='' \
+junit_errors=0 junit_errors_total=0 junit_output="results_junit.xml"
 
-junit_testsuite=0 junit_tests=0 junit_failures=0 junit_skipped=0
-junit_tests_total=0 junit_failures_total=0
 create_buffers junit
 
-junit_output="results_junit.xml"
-
 junit_begin() {
   _attrs=''
   xmlattrs _attrs "name=$SHELLSPEC_PROJECT_NAME"
@@ -19,17 +15,19 @@ junit_begin() {
 }
 
 junit_each() {
-  _text='' _attrs='' _testsuite=$junit_testsuite
-  _tests=$junit_tests _failures=$junit_failures _skipped=$junit_skipped
-  _tests_total=$junit_tests_total _failures_total=$junit_failures_total
+  _text='' _attrs='' _testsuite=$junit_testsuite \
+  _tests=$junit_tests _failures=$junit_failures _skipped=$junit_skipped \
+  _tests_total=$junit_tests_total _failures_total=$junit_failures_total \
+  _errors=$junit_errors _errors_total=$junit_errors_total \
+  _system_error=$junit_system_error
 
   case $field_type in
     meta) junit '=' ;;
     begin)
       xmlattrs _attrs "name=$field_specfile" "hostname=$SHELLSPEC_HOSTNAME" \
         "timestamp=$(date -u '+%Y-%m-%dT%H:%M:%S')"
       junit '=' "  <testsuite $_attrs>${LF}"
-      _tests=0 _failures=0 _skipped=0
+      _tests=0 _errors=0 _failures=0 _skipped=0
       ;;
     example)
       xmlattrs _attrs "classname=$field_specfile" "name=$(field_description)"
@@ -58,19 +56,34 @@ junit_each() {
       junit '=' "</testcase>${LF}"
       ;;
     end)
-      junit '=' "  </testsuite>${LF}"
+      junit '='
+      if [ "$_system_error" ]; then
+        junit '+=' "    <system-error>$_system_error</system-error>${LF}"
+        _system_error=''
+      fi
+      junit '+=' "  </testsuite>${LF}"
       eval "junit_tests_${_testsuite}=\$_tests"
+      eval "junit_errors_${_testsuite}=\$_errors"
       eval "junit_failures_${_testsuite}=\$_failures"
       eval "junit_skipped_${_testsuite}=\$_skipped"
       inc _testsuite
       ;;
+    error)
+      inc _errors _errors_total
+      _message="${field_note}${field_note:+: }${field_message}"
+      _failure=$field_failure_message _address="$field_specfile:$field_lineno"
+      wrap _failure "${_failure%${LF}}${LF}" "  "
+      _system_error="${_system_error}${_message}${LF}${_failure}"
+      _system_error="${_system_error}# ${_address}${LF}${LF}"
+      ;;
     finished) junit '=' ;;
   esac
   junit '>>>' >> "$SHELLSPEC_TMPBASE/${junit_output}.work"
 
-  junit_testsuite=$_testsuite
-  junit_tests=$_tests junit_failures=$_failures junit_skipped=$_skipped
-  junit_tests_total=$_tests_total junit_failures_total=$_failures_total
+  junit_testsuite=$_testsuite junit_system_error=$_system_error \
+  junit_tests=$_tests junit_failures=$_failures junit_skipped=$_skipped \
+  junit_tests_total=$_tests_total junit_failures_total=$_failures_total \
+  junit_errors=$_errors junit_errors_total=$_errors_total
 }
 
 junit_end() {
@@ -85,14 +98,15 @@ junit_output() {
       case $_line in
         *\<testsuites\ *)
           _before=${_line%%<testsuites\ *} _after=${_line#*<testsuites\ }
-          xmlattrs _attrs "tests=$junit_tests_total" \
-            "failures=$junit_failures_total" "time=$time_real"
+          xmlattrs _attrs "tests=$junit_tests_total" "time=$time_real" \
+            "errors=$junit_errors_total" "failures=$junit_failures_total"
           putsn "$_before<testsuites $_attrs $_after"
           ;;
         *\<testsuite\ *)
           _before=${_line%%<testsuite\ *} _after=${_line#*<testsuite\ }
           eval "xmlattrs _attrs id=$_id tests=\$junit_tests_${_id} \
-            failures=\$junit_failures_${_id} skipped=\$junit_skipped_${_id}"
+            errors=\$junit_errors_${_id} failures=\$junit_failures_${_id} \
+            skipped=\$junit_skipped_${_id}"
           putsn "$_before<testsuite $_attrs $_after"
           inc _id
           ;;