Merge pull request #71 from cjwatson/layer-failure

Improve output when layer setUp or tearDown fails

CHANGES.rst

@@ -5,7 +5,8 @@
 5.1 (unreleased)
 ================
 
-- Nothing changed yet.
+- Recover more gracefully when layer setUp or tearDown fails, producing
+  useful subunit output.
 
 
 5.0 (2019-03-19)

src/zope/testrunner/formatter.py

@@ -885,6 +885,15 @@ def _emit_error(self, error_id, tag, exc_info, runnable=False):
             self._subunit.addError(test, exc_info)
             self._subunit.stopTest(test)
 
+    def _emit_failure(self, failure_id, tag, exc_info):
+        """Emit an failure to the subunit stream.
+
+        Use this to pass on information about failures that occur outside of
+        tests.
+        """
+        test = FakeTest(failure_id)
+        self._subunit.addFailure(test, exc_info)
+
     def _enter_layer(self, layer_name):
         """Tell subunit that we are entering a layer."""
         self._subunit.tags(['zope:layer:%s' % (layer_name,)], [])
@@ -1089,6 +1098,11 @@ def stop_set_up(self, seconds):
             self._subunit.stopTest(test)
         self._enter_layer(layer_name)
 
+    def layer_failure(self, failure_type, exc_info):
+        layer_name, start_time = self._last_layer
+        self._emit_failure(
+            '%s:%s' % (layer_name, failure_type), self.TAG_LAYER, exc_info)
+
     def start_tear_down(self, layer_name):
         """Report that we're tearing down a layer.
 

src/zope/testrunner/runner.py

@@ -317,11 +317,22 @@ def run_tests(self):
         if setup_layers:
             if self.options.resume_layer is None:
                 self.options.output.info("Tearing down left over layers:")
-            tear_down_unneeded(self.options, (), setup_layers, True)
+            tear_down_unneeded(
+                self.options, (), setup_layers, self.errors, optional=True)
 
         self.failed = bool(self.import_errors or self.failures or self.errors)
 
 
+def handle_layer_failure(failure_type, output, errors):
+    if hasattr(output, 'layer_failure'):
+        output.layer_failure(failure_type.subunit_label, sys.exc_info())
+    else:
+        f = StringIO()
+        traceback.print_exc(file=f)
+        output.error(f.getvalue())
+    errors.append((failure_type, sys.exc_info()))
+
+
 def run_tests(options, tests, name, failures, errors, skipped, import_errors):
     repeat = options.repeat or 1
     repeat_range = iter(range(repeat))
@@ -442,7 +453,7 @@ def run_layer(options, layer_name, layer, tests, setup_layers,
     needed = dict([(l, 1) for l in gathered])
     if options.resume_number != 0:
         output.info("Running %s tests:" % layer_name)
-    tear_down_unneeded(options, needed, setup_layers)
+    tear_down_unneeded(options, needed, setup_layers, errors)
 
     if options.resume_layer is not None:
         output.info_suboptimal("  Running in a subprocess.")
@@ -451,11 +462,10 @@ def run_layer(options, layer_name, layer, tests, setup_layers,
         setup_layer(options, layer, setup_layers)
     except zope.testrunner.interfaces.EndRun:
         raise
+    except MemoryError:
+        raise
     except Exception:
-        f = StringIO()
-        traceback.print_exc(file=f)
-        output.error(f.getvalue())
-        errors.append((SetUpLayerFailure(layer), sys.exc_info()))
+        handle_layer_failure(SetUpLayerFailure(layer), output, errors)
         return 0
     else:
         return run_tests(options, tests, layer_name, failures, errors, skipped,
@@ -464,6 +474,8 @@ def run_layer(options, layer_name, layer, tests, setup_layers,
 
 class SetUpLayerFailure(unittest.TestCase):
 
+    subunit_label = 'setUp'
+
     def __init__(self, layer):
         super(SetUpLayerFailure, self).__init__()
         self.layer = layer
@@ -472,9 +484,23 @@ def runTest(self):
         pass
 
     def __str__(self):
-        return "Layer: %s" % (name_from_layer(self.layer))
+        return "Layer: %s.setUp" % (name_from_layer(self.layer))
 
 
+class TearDownLayerFailure(unittest.TestCase):
+
+    subunit_label = 'tearDown'
+
+    def __init__(self, layer):
+        super(TearDownLayerFailure, self).__init__()
+        self.layer = layer
+
+    def runTest(self):
+        pass
+
+    def __str__(self):
+        return "Layer: %s.tearDown" % (name_from_layer(self.layer))
+
 
 def spawn_layer_in_subprocess(result, script_parts, options, features,
                               layer_name, layer, failures, errors, skipped,
@@ -748,7 +774,7 @@ def resume_tests(script_parts, options, features, layers, failures, errors,
     return sum(r.num_ran for r in results)
 
 
-def tear_down_unneeded(options, needed, setup_layers, optional=False):
+def tear_down_unneeded(options, needed, setup_layers, errors, optional=False):
     # Tear down any layers not needed for these tests. The unneeded layers
     # might interfere.
     unneeded = [l for l in setup_layers if l not in needed]
@@ -766,6 +792,10 @@ def tear_down_unneeded(options, needed, setup_layers, optional=False):
                 output.tear_down_not_supported()
                 if not optional:
                     raise CanNotTearDown(l)
+            except MemoryError:
+                raise
+            except Exception:
+                handle_layer_failure(TearDownLayerFailure(l), output, errors)
             else:
                 output.stop_tear_down(time.time() - t)
         finally:
@@ -790,6 +820,8 @@ def setup_layer(options, layer, setup_layers):
         if hasattr(layer, 'setUp'):
             try:
                 layer.setUp()
+            except MemoryError:
+                raise
             except Exception:
                 if options.post_mortem:
                     if options.resume_layer:

src/zope/testrunner/tests/testrunner-debugging-layer-setup.rst

@@ -131,7 +131,7 @@ a subprocess:
     <BLANKLINE>
     <BLANKLINE>
     Tests with errors:
-       Layer: tests2.Layer2
+       Layer: tests2.Layer2.setUp
     Total: 1 tests, 0 failures, 1 errors and 0 skipped in 0.210 seconds.
     True
 

src/zope/testrunner/tests/testrunner-ex/brokenlayer.py

@@ -0,0 +1,61 @@
+##############################################################################
+#
+# Copyright (c) 2012-2018 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Sample tests with layers that have broken set up and tear down."""
+
+import unittest
+
+
+class BrokenSetUpLayer:
+
+    @classmethod
+    def setUp(cls):
+        raise ValueError('No value is good enough for me!')
+
+    @classmethod
+    def tearDown(cls):
+        pass
+
+
+class BrokenTearDownLayer:
+
+    @classmethod
+    def setUp(cls):
+        pass
+
+    @classmethod
+    def tearDown(cls):
+        raise TypeError('You are not my type.  No-one is my type!')
+
+
+class TestSomething1(unittest.TestCase):
+
+    layer = BrokenSetUpLayer
+
+    def test_something(self):
+        pass
+
+
+class TestSomething2(unittest.TestCase):
+
+    layer = BrokenTearDownLayer
+
+    def test_something(self):
+        pass
+
+
+def test_suite():
+    suite = unittest.TestSuite()
+    suite.addTest(unittest.makeSuite(TestSomething1))
+    suite.addTest(unittest.makeSuite(TestSomething2))
+    return suite

src/zope/testrunner/tests/testrunner-subunit-v2.rst

@@ -314,6 +314,34 @@ the case, the subunit stream will say that the layer skipped its tearDown.
     False
 
 
+Layer failures
+--------------
+
+If a layer's setUp or tearDown method fails in some other way, this is shown
+in the subunit stream.
+
+    >>> sys.argv = 'test --tests-pattern ^brokenlayer$'.split()
+    >>> subunit_summarize(testrunner.run_internal, defaults)
+    id=brokenlayer.BrokenSetUpLayer:setUp status=inprogress !runnable
+    id=brokenlayer.BrokenSetUpLayer:setUp
+    traceback (text/x-traceback...)
+    Traceback (most recent call last):
+    ...
+    ValueError: No value is good enough for me!
+    <BLANKLINE>
+    id=brokenlayer.BrokenSetUpLayer:setUp status=fail tags=(zope:layer)
+    ...
+    id=brokenlayer.BrokenTearDownLayer:tearDown status=inprogress !runnable
+    id=brokenlayer.BrokenTearDownLayer:tearDown
+    traceback (text/x-traceback...)
+    Traceback (most recent call last):
+    ...
+    TypeError: You are not my type.  No-one is my type!
+    <BLANKLINE>
+    id=brokenlayer.BrokenTearDownLayer:tearDown status=fail tags=(zope:layer)
+    True
+
+
 Module import errors
 --------------------
 

src/zope/testrunner/tests/testrunner-subunit.rst

@@ -312,6 +312,33 @@ the case, the subunit stream will say that the layer skipped its tearDown.
     False
 
 
+Layer failures
+--------------
+
+If a layer's setUp or tearDown method fails in some other way, this is shown
+in the subunit stream.
+
+    >>> sys.argv = 'test --tests-pattern ^brokenlayer$'.split()
+    >>> testrunner.run_internal(defaults)
+    time: ...
+    test: brokenlayer.BrokenSetUpLayer:setUp
+    tags: zope:layer
+    failure: brokenlayer.BrokenSetUpLayer:setUp [
+    Traceback (most recent call last):
+    ...
+    ValueError: No value is good enough for me!
+    ]
+    time: ...
+    test: brokenlayer.BrokenTearDownLayer:tearDown
+    tags: zope:layer
+    failure: brokenlayer.BrokenTearDownLayer:tearDown [
+    Traceback (most recent call last):
+    ...
+    TypeError: You are not my type.  No-one is my type!
+    ]
+    True
+
+
 Module import errors
 --------------------