Improve output when layer setUp or tearDown fails

In particular, the subunit output is now much more precise in this case.

This is based on work by Graham Binns, Gary Poster, and Francesco
Banconi in Launchpad's zope.testing fork.

CHANGES.rst

@@ -5,7 +5,8 @@
 5.1 (unreleased)
 ================
 
-- Nothing changed yet.
+- Recover more gracefully when layer setUp or tearDown fails, producing
+  useful subunit output.
 
 
 5.0 (2019-03-19)

src/zope/testrunner/formatter.py

@@ -885,6 +885,15 @@ def _emit_error(self, error_id, tag, exc_info, runnable=False):
             self._subunit.addError(test, exc_info)
             self._subunit.stopTest(test)
 
+    def _emit_failure(self, failure_id, tag, exc_info):
+        """Emit an failure to the subunit stream.
+
+        Use this to pass on information about failures that occur outside of
+        tests.
+        """
+        test = FakeTest(failure_id)
+        self._subunit.addFailure(test, exc_info)
+
     def _enter_layer(self, layer_name):
         """Tell subunit that we are entering a layer."""
         self._subunit.tags(['zope:layer:%s' % (layer_name,)], [])
@@ -1089,6 +1098,11 @@ def stop_set_up(self, seconds):
             self._subunit.stopTest(test)
         self._enter_layer(layer_name)
 
+    def layer_failure(self, failure_type, exc_info):
+        layer_name, start_time = self._last_layer
+        self._emit_failure(
+            '%s:%s' % (layer_name, failure_type), self.TAG_LAYER, exc_info)
+
     def start_tear_down(self, layer_name):
         """Report that we're tearing down a layer.
 

src/zope/testrunner/runner.py

@@ -64,6 +64,7 @@ class UnexpectedSuccess(Exception):
     pass
 
 
+EXPLOSIVE_ERRORS = (MemoryError, KeyboardInterrupt, SystemExit)
 PYREFCOUNT_PATTERN = re.compile(r'\[[0-9]+ refs\]')
 
 is_jython = sys.platform.startswith('java')
@@ -317,11 +318,22 @@ def run_tests(self):
         if setup_layers:
             if self.options.resume_layer is None:
                 self.options.output.info("Tearing down left over layers:")
-            tear_down_unneeded(self.options, (), setup_layers, True)
+            tear_down_unneeded(
+                self.options, (), setup_layers, self.errors, optional=True)
 
         self.failed = bool(self.import_errors or self.failures or self.errors)
 
 
+def handle_layer_failure(failure_type, output, errors):
+    if hasattr(output, 'layer_failure'):
+        output.layer_failure(failure_type.subunit_label, sys.exc_info())
+    else:
+        f = StringIO()
+        traceback.print_exc(file=f)
+        output.error(f.getvalue())
+    errors.append((failure_type, sys.exc_info()))
+
+
 def run_tests(options, tests, name, failures, errors, skipped, import_errors):
     repeat = options.repeat or 1
     repeat_range = iter(range(repeat))
@@ -442,7 +454,7 @@ def run_layer(options, layer_name, layer, tests, setup_layers,
     needed = dict([(l, 1) for l in gathered])
     if options.resume_number != 0:
         output.info("Running %s tests:" % layer_name)
-    tear_down_unneeded(options, needed, setup_layers)
+    tear_down_unneeded(options, needed, setup_layers, errors)
 
     if options.resume_layer is not None:
         output.info_suboptimal("  Running in a subprocess.")
@@ -451,11 +463,10 @@ def run_layer(options, layer_name, layer, tests, setup_layers,
         setup_layer(options, layer, setup_layers)
     except zope.testrunner.interfaces.EndRun:
         raise
+    except EXPLOSIVE_ERRORS:
+        raise
     except Exception:
-        f = StringIO()
-        traceback.print_exc(file=f)
-        output.error(f.getvalue())
-        errors.append((SetUpLayerFailure(layer), sys.exc_info()))
+        handle_layer_failure(SetUpLayerFailure(layer), output, errors)
         return 0
     else:
         return run_tests(options, tests, layer_name, failures, errors, skipped,
@@ -464,6 +475,8 @@ def run_layer(options, layer_name, layer, tests, setup_layers,
 
 class SetUpLayerFailure(unittest.TestCase):
 
+    subunit_label = 'setUp'
+
     def __init__(self, layer):
         super(SetUpLayerFailure, self).__init__()
         self.layer = layer
@@ -472,9 +485,23 @@ def runTest(self):
         pass
 
     def __str__(self):
-        return "Layer: %s" % (name_from_layer(self.layer))
+        return "Layer: %s.setUp" % (name_from_layer(self.layer))
 
 
+class TearDownLayerFailure(unittest.TestCase):
+
+    subunit_label = 'tearDown'
+
+    def __init__(self, layer):
+        super(TearDownLayerFailure, self).__init__()
+        self.layer = layer
+
+    def runTest(self):
+        pass
+
+    def __str__(self):
+        return "Layer: %s.tearDown" % (name_from_layer(self.layer))
+
 
 def spawn_layer_in_subprocess(result, script_parts, options, features,
                               layer_name, layer, failures, errors, skipped,
@@ -748,7 +775,7 @@ def resume_tests(script_parts, options, features, layers, failures, errors,
     return sum(r.num_ran for r in results)
 
 
-def tear_down_unneeded(options, needed, setup_layers, optional=False):
+def tear_down_unneeded(options, needed, setup_layers, errors, optional=False):
     # Tear down any layers not needed for these tests. The unneeded layers
     # might interfere.
     unneeded = [l for l in setup_layers if l not in needed]
@@ -766,6 +793,10 @@ def tear_down_unneeded(options, needed, setup_layers, optional=False):
                 output.tear_down_not_supported()
                 if not optional:
                     raise CanNotTearDown(l)
+            except EXPLOSIVE_ERRORS:
+                raise
+            except Exception:
+                handle_layer_failure(TearDownLayerFailure(l), output, errors)
             else:
                 output.stop_tear_down(time.time() - t)
         finally:
@@ -790,6 +821,8 @@ def setup_layer(options, layer, setup_layers):
         if hasattr(layer, 'setUp'):
             try:
                 layer.setUp()
+            except EXPLOSIVE_ERRORS:
+                raise
             except Exception:
                 if options.post_mortem:
                     if options.resume_layer:

src/zope/testrunner/tests/testrunner-debugging-layer-setup.rst

@@ -131,7 +131,7 @@ a subprocess:
     <BLANKLINE>
     <BLANKLINE>
     Tests with errors:
-       Layer: tests2.Layer2
+       Layer: tests2.Layer2.setUp
     Total: 1 tests, 0 failures, 1 errors and 0 skipped in 0.210 seconds.
     True
 

src/zope/testrunner/tests/testrunner-ex/brokenlayer.py

@@ -0,0 +1,61 @@
+##############################################################################
+#
+# Copyright (c) 2012-2018 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Sample tests with layers that have broken set up and tear down."""
+
+import unittest
+
+
+class BrokenSetUpLayer:
+
+    @classmethod
+    def setUp(cls):
+        raise ValueError('No value is good enough for me!')
+
+    @classmethod
+    def tearDown(cls):
+        pass
+
+
+class BrokenTearDownLayer:
+
+    @classmethod
+    def setUp(cls):
+        pass
+
+    @classmethod
+    def tearDown(cls):
+        raise TypeError('You are not my type.  No-one is my type!')
+
+
+class TestSomething1(unittest.TestCase):
+
+    layer = BrokenSetUpLayer
+
+    def test_something(self):
+        pass
+
+
+class TestSomething2(unittest.TestCase):
+
+    layer = BrokenTearDownLayer
+
+    def test_something(self):
+        pass
+
+
+def test_suite():
+    suite = unittest.TestSuite()
+    suite.addTest(unittest.makeSuite(TestSomething1))
+    suite.addTest(unittest.makeSuite(TestSomething2))
+    return suite

src/zope/testrunner/tests/testrunner-subunit-v2.rst

@@ -291,8 +291,8 @@ https://bugs.launchpad.net/subunit/+bug/1740158.)
     True
 
 
-Layers that can't be torn down
-------------------------------
+Layer failures
+--------------
 
 A layer can have a tearDown method that raises NotImplementedError. If this is
 the case, the subunit stream will say that the layer skipped its tearDown.
@@ -313,6 +313,30 @@ the case, the subunit stream will say that the layer skipped its tearDown.
       !runnable
     False
 
+If a layer's setUp or tearDown method fails in some other way, this is shown
+in the subunit stream.
+
+    >>> sys.argv = 'test --tests-pattern ^brokenlayer$'.split()
+    >>> subunit_summarize(testrunner.run_internal, defaults)
+    id=brokenlayer.BrokenSetUpLayer:setUp status=inprogress !runnable
+    id=brokenlayer.BrokenSetUpLayer:setUp
+    traceback (text/x-traceback...)
+    Traceback (most recent call last):
+    ...
+    ValueError: No value is good enough for me!
+    <BLANKLINE>
+    id=brokenlayer.BrokenSetUpLayer:setUp status=fail tags=(zope:layer)
+    ...
+    id=brokenlayer.BrokenTearDownLayer:tearDown status=inprogress !runnable
+    id=brokenlayer.BrokenTearDownLayer:tearDown
+    traceback (text/x-traceback...)
+    Traceback (most recent call last):
+    ...
+    TypeError: You are not my type.  No-one is my type!
+    <BLANKLINE>
+    id=brokenlayer.BrokenTearDownLayer:tearDown status=fail tags=(zope:layer)
+    True
+
 
 Module import errors
 --------------------

src/zope/testrunner/tests/testrunner-subunit.rst

@@ -283,8 +283,8 @@ Errors are recorded in the subunit stream as MIME-encoded chunks of text.
     True
 
 
-Layers that can't be torn down
-------------------------------
+Layer failures
+--------------
 
 A layer can have a tearDown method that raises NotImplementedError. If this is
 the case, the subunit stream will say that the layer skipped its tearDown.
@@ -311,6 +311,29 @@ the case, the subunit stream will say that the layer skipped its tearDown.
     ]
     False
 
+If a layer's setUp or tearDown method fails in some other way, this is shown
+in the subunit stream.
+
+    >>> sys.argv = 'test --tests-pattern ^brokenlayer$'.split()
+    >>> testrunner.run_internal(defaults)
+    time: ...
+    test: brokenlayer.BrokenSetUpLayer:setUp
+    tags: zope:layer
+    failure: brokenlayer.BrokenSetUpLayer:setUp [
+    Traceback (most recent call last):
+    ...
+    ValueError: No value is good enough for me!
+    ]
+    time: ...
+    test: brokenlayer.BrokenTearDownLayer:tearDown
+    tags: zope:layer
+    failure: brokenlayer.BrokenTearDownLayer:tearDown [
+    Traceback (most recent call last):
+    ...
+    TypeError: You are not my type.  No-one is my type!
+    ]
+    True
+
 
 Module import errors
 --------------------