Support scenario hook-errors with JUnitReporter (related to: #466)

 FIXES #466

CHANGES.rst

@@ -61,6 +61,7 @@ REPORTERS:
 
   * junit: Add timestamp and hostname attributes to testsuite XML element.
   * junit: Support to tweak output with userdata (experimental).
+  * junit: Support scenario hook-errors with JUnitReporter (related to: #466)
 
 CHANGES:
 
@@ -70,6 +71,7 @@ FIXED:
 
   * pull  #476: scenario.status when scenario without steps is skipped (provided by: ar45, jenisys)
   * issue #455: Restore backward compatibility to Cucumber style RegexMatcher (submitted by:  avabramov)
+  * issue #446: after_scenario HOOK-ERROR asserts with jUnit reporter (submitted by: lagin)
   * issue #416: JUnit report messages cut off (submitted by: remcowesterhoud, provided by: bittner)
   * issue #414: Support for Jython 2.7 (submitted by: gabtwi...)
   * issue #384: Active Tags fail with ScenarioOutline (submitted by: BRevzin)

behave/model.py

@@ -1182,6 +1182,8 @@ class Step(BasicStatement, Replayable):
        If the step failed then this will hold any error information, as a
        single string. It will otherwise be None.
 
+       .. versionmodified:: 1.2.6 (moved to base class)
+
     .. attribute:: filename
 
        The file name (or "<string>") of the *feature file* where the step was
@@ -1205,22 +1207,13 @@ def __init__(self, filename, line, keyword, step_type, name, text=None,
         self.status = "untested"
         self.hook_failed = False
         self.duration = 0
-        self.exception = None
-        self.exc_traceback = None
-        self.error_message = None
 
     def reset(self):
         """Reset temporary runtime data to reach clean state again."""
+        super(Step, self).reset()
         self.status = "untested"
         self.hook_failed = False
         self.duration = 0
-        self.exception = None
-        self.exc_traceback = None
-        self.error_message = None
-
-    def store_exception_context(self, exception):
-        self.exception = exception
-        self.exc_traceback = sys.exc_info()[2]
 
     def __repr__(self):
         return '<%s "%s">' % (self.step_type, self.name)
@@ -1250,9 +1243,10 @@ def set_values(self, table_row):
     def run(self, runner, quiet=False, capture=True):
         # pylint: disable=too-many-branches, too-many-statements
         # -- RESET: Run-time information.
-        self.exception = self.exc_traceback = self.error_message = None
-        self.status = "untested"
-        self.hook_failed = False
+        # self.exception = self.exc_traceback = self.error_message = None
+        # self.status = "untested"
+        # self.hook_failed = False
+        self.reset()
 
         match = runner.step_registry.find_match(self)
         if match is None:

behave/model_core.py

@@ -5,9 +5,11 @@
 """
 
 import os.path
+import sys
 import six
 from behave.textutil import text as _text
 
+
 # -----------------------------------------------------------------------------
 # GENERIC MODEL CLASSES:
 # -----------------------------------------------------------------------------
@@ -190,6 +192,10 @@ def __init__(self, filename, line, keyword, name):
         assert isinstance(name, six.text_type)
         self.keyword = keyword
         self.name = name
+        # -- ERROR CONTEXT INFO:
+        self.exception = None
+        self.exc_traceback = None
+        self.error_message = None
 
     @property
     def filename(self):
@@ -200,10 +206,15 @@ def filename(self):
     def line(self):
         return self.location.line
 
-    # @property
-    # def location(self):
-    #     p = os.path.relpath(self.filename, os.getcwd())
-    #     return '%s:%d' % (p, self.line)
+    def reset(self):
+        # -- RESET: ERROR CONTEXT INFO
+        self.exception = None
+        self.exc_traceback = None
+        self.error_message = None
+
+    def store_exception_context(self, exception):
+        self.exception = exception
+        self.exc_traceback = sys.exc_info()[2]
 
     def __hash__(self):
         # -- NEEDED-FOR: PYTHON3
@@ -304,13 +315,12 @@ def replay(self, formatter):
 # -----------------------------------------------------------------------------
 # UTILITY FUNCTIONS:
 # -----------------------------------------------------------------------------
-def unwrap_function(func, max=10):
+def unwrap_function(func, max_depth=10):
     """Unwraps a function that is wrapped with :func:`functools.partial()`"""
     iteration = 0
-    wrapped =  getattr(func, "__wrapped__", None)
-    while wrapped and iteration < max:
+    wrapped = getattr(func, "__wrapped__", None)
+    while wrapped and iteration < max_depth:
         func = wrapped
         wrapped = getattr(func, "__wrapped__", None)
         iteration += 1
     return func
-

behave/reporter/junit.py

@@ -72,6 +72,7 @@
 import os.path
 import codecs
 import sys
+import traceback
 from xml.etree import ElementTree
 from datetime import datetime
 from behave.reporter.base import Reporter
@@ -377,24 +378,35 @@ def _process_scenario(self, scenario, report):
                 step = self.select_step_with_status(status, scenario)
                 if step:
                     break
-            assert step, "OOPS: No failed step found in scenario: %s" % scenario.name
-            assert step.status in ('failed', 'undefined')
+            # -- NOTE: Scenario may fail now due to hook-errors.
             element_name = 'failure'
-            if isinstance(step.exception, (AssertionError, type(None))):
+            if step and isinstance(step.exception, (AssertionError, type(None))):
                 # -- FAILURE: AssertionError
+                assert step.status in ('failed', 'undefined')
                 report.counts_failed += 1
             else:
                 # -- UNEXPECTED RUNTIME-ERROR:
                 report.counts_errors += 1
                 element_name = 'error'
             # -- COMMON-PART:
             failure = ElementTree.Element(element_name)
-            step_text = self.describe_step(step).rstrip()
-            text = u"\nFailing step: %s\nLocation: %s\n" % (step_text, step.location)
-            message = _text(step.exception)
-            failure.set(u'type', step.exception.__class__.__name__)
-            failure.set(u'message', message)
-            text += _text(step.error_message)
+            if step:
+                step_text = self.describe_step(step).rstrip()
+                text = u"\nFailing step: %s\nLocation: %s\n" % (step_text, step.location)
+                message = _text(step.exception)
+                failure.set(u'type', step.exception.__class__.__name__)
+                failure.set(u'message', message)
+                text += _text(step.error_message)
+            else:
+                # -- MAYBE: Hook failure before any step is executed.
+                failure_type = "UnknownError"
+                if scenario.exception:
+                    failure_type = scenario.exception.__class__.__name__
+                failure.set(u'type', failure_type)
+                failure.set(u'message', scenario.error_message or "")
+                traceback_lines = traceback.format_tb(scenario.exc_traceback)
+                traceback_lines.insert(0, u"Traceback:\n")
+                text = _text(u"".join(traceback_lines))
             failure.append(CDATA(text))
             case.append(failure)
         elif scenario.status in ("skipped", "untested") and self.show_skipped:

behave/runner.py

@@ -2,6 +2,7 @@
 """
 This module provides Runner class to run behave feature files (or model elements).
 """
+
 from __future__ import absolute_import, print_function, with_statement
 import contextlib
 import os.path
@@ -467,24 +468,26 @@ def run_hook(self, name, context, *args):
                 if "tag" in name:
                     extra = "(tag=%s)" % args[0]
 
-                error_text = ExceptionUtil.describe(e, use_traceback)
-                print(u"HOOK-ERROR in %s%s: %s" % (name, extra, error_text))
+                error_text = ExceptionUtil.describe(e, use_traceback).rstrip()
+                error_message = u"HOOK-ERROR in %s%s: %s" % (name, extra, error_text)
+                print(error_message)
                 self.hook_failures += 1
-                if "step" in name:
-                    step = args[0]
-                    step.hook_failed = True
-                elif "tag" in name:
-                    # -- FEATURE or SCENARIO => Use Feature as collector.
-                    context.feature.hook_failed = True
-                elif "scenario" in name:
-                    scenario = args[0]
-                    scenario.hook_failed = True
-                elif "feature" in name:
-                    feature = args[0]
-                    feature.hook_failed = True
+                if "tag" in name:
+                    # -- SCENARIO or FEATURE
+                    statement = getattr(context, "scenario", context.feature)
                 elif "all" in name:
                     # -- ABORT EXECUTION: For before_all/after_all
                     self.aborted = True
+                    statement = None
+                else:
+                    # -- CASE: feature, scenario, step
+                    statement = args[0]
+
+                if statement:
+                    # -- CASE: feature, scenario, step
+                    statement.hook_failed = True
+                    statement.store_exception_context(e)
+                    statement.error_message = error_message
 
     def setup_capture(self):
         if not self.context:

features/runner.hook_errors.feature

@@ -246,7 +246,11 @@ Feature: Hooks processing in case of errors (exceptions)
           """
 
     @skipped.hook.after_feature
-    Scenario: Skipped feature with hook error
+    Scenario: Skipped feature with potential hook error (hooks are not run)
+
+      This goes unnoticed because hooks are not run for a skipped feature/scenario.
+      NOTE: Except if before_feature(), before_scenario() hook skips the feature/scenario. 
+
       When I run "behave -f plain -D HOOK_ERROR_LOC=after_feature -t ~@foo features/passing.feature"
       Then it should pass with:
           """

issue.features/issue0446.feature

@@ -0,0 +1,114 @@
+@issue
+@junit
+Feature: Issue #446 -- Support scenario hook-errors with JUnitReporter
+
+  . Currently, when a hook error occurs in:
+  .
+  .   * before_scenario()
+  .   * after_scenario()
+  .
+  . a sanity check in the JUnitReporter prevents sane JUnit XML output.
+
+    @setup
+    Scenario: Skip scenario without steps
+      Given a new working directory
+      And a file named "features/steps/pass_steps.py" with:
+        """
+        from behave import step
+
+        @step('{word:w} step passes')
+        def step_passes(context, word):
+            pass
+        """
+      And a file named "features/before_scenario_failure.feature" with:
+        """
+        Feature: Alice
+          @hook_failure.before_scenario
+          Scenario: A1
+            Given a step passes
+        """
+      And a file named "features/after_scenario_failure.feature" with:
+        """
+        Feature: Bob
+          @hook_failure.after_scenario
+          Scenario: B1
+            Given another step passes
+        """
+      And a file named "features/environment.py" with:
+        """
+        def cause_hook_failure():
+            1 / 0   # CAUSE: ZeroDivisionError
+
+        def before_scenario(context, scenario):
+            if "hook_failure.before_scenario" in scenario.tags:
+                cause_hook_failure()
+
+        def after_scenario(context, scenario):
+            if "hook_failure.after_scenario" in scenario.tags:
+                cause_hook_failure()
+        """
+      And a file named "behave.ini" with:
+        """
+        [behave.userdata]
+        behave.reporter.junit.show_timestamp = False
+        behave.reporter.junit.show_hostname = False
+        """
+
+    Scenario: Hook error in before_scenario()
+      When I run "behave -f plain --junit features/before_scenario_failure.feature"
+      Then it should fail with:
+        """
+        0 scenarios passed, 1 failed, 0 skipped
+        """
+      And the command output should contain:
+        """
+          Scenario: A1
+        HOOK-ERROR in before_scenario: ZeroDivisionError: integer division or modulo by zero
+        """
+      And the file "reports/TESTS-before_scenario_failure.xml" should contain:
+        """
+        <testsuite errors="1" failures="0" name="before_scenario_failure.Alice" skipped="0" tests="1"
+        """
+      And the file "reports/TESTS-before_scenario_failure.xml" should contain:
+        """
+        <error message="HOOK-ERROR in before_scenario: ZeroDivisionError: integer division or modulo by zero" type="ZeroDivisionError">
+        """
+      And the file "reports/TESTS-before_scenario_failure.xml" should contain:
+        """
+        File "features/environment.py", line 6, in before_scenario
+          cause_hook_failure()
+        File "features/environment.py", line 2, in cause_hook_failure
+          1 / 0   # CAUSE: ZeroDivisionError
+        """
+      And note that "the traceback is contained in the XML element <error/>"
+
+
+    Scenario: Hook error in after_scenario()
+      When I run "behave -f plain --junit features/after_scenario_failure.feature"
+      Then it should fail with:
+        """
+        0 scenarios passed, 1 failed, 0 skipped
+        """
+      And the command output should contain:
+        """
+          Scenario: B1
+            Given another step passes ... passed
+        HOOK-ERROR in after_scenario: ZeroDivisionError: integer division or modulo by zero
+        """
+      And the file "reports/TESTS-after_scenario_failure.xml" should contain:
+        """
+        <testsuite errors="1" failures="0" name="after_scenario_failure.Bob" skipped="0" tests="1"
+        """
+      And the file "reports/TESTS-after_scenario_failure.xml" should contain:
+        """
+        <error message="HOOK-ERROR in after_scenario: ZeroDivisionError: integer division or modulo by zero" type="ZeroDivisionError">
+        """
+      And the file "reports/TESTS-after_scenario_failure.xml" should contain:
+        """
+        File "features/environment.py", line 10, in after_scenario
+          cause_hook_failure()
+        File "features/environment.py", line 2, in cause_hook_failure
+          1 / 0   # CAUSE: ZeroDivisionError
+        """
+      And note that "the traceback is contained in the XML element <error/>"
+