Merge pull request #17 from zopefoundation/unittests_first

Test Unittests first
zopefoundation · Dec 26, 2014 · 3ab8988 · 3ab8988
2 parents 2f4b3ec + c2c71bd
commit 3ab8988
Show file tree

Hide file tree

Showing 16 changed files with 183 additions and 124 deletions.
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -6,6 +6,9 @@ zope.testrunner Changelog
 
 - When looking for the right location of test code, start with longest
   location paths first. This fixes problems with nested code locations.
+- sort related layers close to each other to reduce the number of unnecessary teardowns (fixes #14)
+- run the unit test layer first (fixes https://bugs.launchpad.net/zope.testrunner/+bug/497871)
+
 
 
 4.4.3 (2014-03-19)

diff --git a/src/zope/testrunner/runner.py b/src/zope/testrunner/runner.py
@@ -18,6 +18,7 @@
 import subprocess
 import errno
 import gc
+import inspect
 import re
 import sys
 import threading
@@ -31,8 +32,9 @@
 from six import StringIO
 from zope.testrunner.find import import_name
 from zope.testrunner.find import name_from_layer, _layer_name_cache
-from zope.testrunner.refcount import TrackRefs
+from zope.testrunner.layer import UnitTests
 from zope.testrunner.options import get_options
+from zope.testrunner.refcount import TrackRefs
 import zope.testrunner
 import zope.testrunner.coverage
 import zope.testrunner._doctest
@@ -882,12 +884,17 @@ def layer_from_name(layer_name):
 
 def order_by_bases(layers):
     """Order the layers from least to most specific (bottom to top)
+       Group layers with common bases and put unittests first.
     """
-    named_layers = [(name_from_layer(layer), layer) for layer in layers]
-    named_layers.sort()
-    named_layers.reverse()
+    getmro = inspect.getmro
+
+    def layer_sortkey(layer):
+        return tuple((c.__module__, c.__name__) for c in getmro(layer)[::-1]
+                      if c not in (object, UnitTests))
+
+    layers = sorted(layers, key=layer_sortkey, reverse=True)
     gathered = []
-    for name, layer in named_layers:
+    for layer in layers:
         gather_layers(layer, gathered)
     gathered.reverse()
     seen = {}

diff --git a/src/zope/testrunner/tests/test_doctest.py b/src/zope/testrunner/tests/test_doctest.py
@@ -204,12 +204,13 @@ def test_suite():
         'testrunner-debugging.txt',
         'testrunner-edge-cases.txt',
         'testrunner-errors.txt',
-        'testrunner-layers-buff.txt',
-        'testrunner-layers-ntd.txt',
-        'testrunner-layers.txt',
         'testrunner-layers-api.txt',
+        'testrunner-layers-buff.txt',
         'testrunner-layers-cantfind.txt',
         'testrunner-layers-cwd.txt',
+        'testrunner-layers-ntd.txt',
+        'testrunner-layers-topological-sort.txt',
+        'testrunner-layers.txt',
         'testrunner-progress.txt',
         'testrunner-colors.txt',
         'testrunner-simple.txt',

diff --git a/src/zope/testrunner/tests/testrunner-colors.txt b/src/zope/testrunner/tests/testrunner-colors.txt
@@ -72,37 +72,7 @@ A failed test run highlights the failures in red:
 
     >>> sys.argv = 'test -c --tests-pattern ^sampletests(f|_e|_f)?$ '.split()
     >>> testrunner.run_internal(defaults)
-    {normal}Running samplelayers.Layer1 tests:{normal}
-      Set up samplelayers.Layer1 in {green}0.000{normal} seconds.
-    {normal}  Ran {green}9{normal} tests with {green}0{normal} failures, {green}0{normal} errors, {green}0{normal} skipped in {green}0.001{normal} seconds.{normal}
-    {normal}Running samplelayers.Layer11 tests:{normal}
-      Set up samplelayers.Layer11 in {green}0.000{normal} seconds.
-    {normal}  Ran {green}26{normal} tests with {green}0{normal} failures, {green}0{normal} errors, {green}0{normal} skipped in {green}0.007{normal} seconds.{normal}
-    {normal}Running samplelayers.Layer111 tests:{normal}
-      Set up samplelayers.Layerx in {green}0.000{normal} seconds.
-      Set up samplelayers.Layer111 in {green}0.000{normal} seconds.
-    {normal}  Ran {green}26{normal} tests with {green}0{normal} failures, {green}0{normal} errors, {green}0{normal} skipped in {green}0.008{normal} seconds.{normal}
-    {normal}Running samplelayers.Layer112 tests:{normal}
-      Tear down samplelayers.Layer111 in {green}0.000{normal} seconds.
-      Set up samplelayers.Layer112 in {green}0.000{normal} seconds.
-    {normal}  Ran {green}26{normal} tests with {green}0{normal} failures, {green}0{normal} errors, {green}0{normal} skipped in {green}0.008{normal} seconds.{normal}
-    {normal}Running samplelayers.Layer12 tests:{normal}
-      Tear down samplelayers.Layer112 in {green}0.000{normal} seconds.
-      Tear down samplelayers.Layerx in {green}0.000{normal} seconds.
-      Tear down samplelayers.Layer11 in {green}0.000{normal} seconds.
-      Set up samplelayers.Layer12 in {green}0.000{normal} seconds.
-    {normal}  Ran {green}26{normal} tests with {green}0{normal} failures, {green}0{normal} errors, {green}0{normal} skipped in {green}0.007{normal} seconds.{normal}
-    {normal}Running samplelayers.Layer121 tests:{normal}
-      Set up samplelayers.Layer121 in {green}0.000{normal} seconds.
-    {normal}  Ran {green}26{normal} tests with {green}0{normal} failures, {green}0{normal} errors, {green}0{normal} skipped in {green}0.007{normal} seconds.{normal}
-    {normal}Running samplelayers.Layer122 tests:{normal}
-      Tear down samplelayers.Layer121 in {green}0.000{normal} seconds.
-      Set up samplelayers.Layer122 in {green}0.000{normal} seconds.
-    {normal}  Ran {green}26{normal} tests with {green}0{normal} failures, {green}0{normal} errors, {green}0{normal} skipped in {green}0.008{normal} seconds.{normal}
     {normal}Running zope.testrunner.layer.UnitTests tests:{normal}
-      Tear down samplelayers.Layer122 in {green}0.000{normal} seconds.
-      Tear down samplelayers.Layer12 in {green}0.000{normal} seconds.
-      Tear down samplelayers.Layer1 in {green}0.000{normal} seconds.
       Set up zope.testrunner.layer.UnitTests in {green}N.NNN{normal} seconds.
     <BLANKLINE>
     <BLANKLINE>
@@ -175,9 +145,8 @@ A failed test run highlights the failures in red:
     {red}AssertionError: 1 != 0{normal}
     <BLANKLINE>
     {normal}  Ran {green}164{normal} tests with {boldred}3{normal} failures, {boldred}1{normal} errors, {green}0{normal} skipped in {green}0.045{normal} seconds.{normal}
-    {normal}Tearing down left over layers:{normal}
-      Tear down zope.testrunner.layer.UnitTests in {green}N.NNN{normal} seconds.
-    {normal}Total: {green}329{normal} tests, {boldred}3{normal} failures, {boldred}1{normal} errors, {green}0{normal} skipped in {green}0.023{normal} seconds.{normal}
+    ...
+    {normal}Total: {green}329{normal} tests, {boldred}3{normal} failures, {boldred}1{normal} errors, {green}0{normal} skipped in {green}N.NNN{normal} seconds.{normal}
     True
 
 

diff --git a/src/zope/testrunner/tests/testrunner-coverage.txt b/src/zope/testrunner/tests/testrunner-coverage.txt
@@ -19,7 +19,11 @@ directory name given as the parameter will be used to hold the reports.
 
     >>> from zope import testrunner
     >>> testrunner.run_internal(defaults)
+    Running zope.testrunner.layer.UnitTests tests:
+      Set up zope.testrunner.layer.UnitTests in 0.000 seconds.
+      Ran 156 tests with 0 failures, 0 errors and 0 skipped in 0.687 seconds.
     Running samplelayers.Layer1 tests:
+      Tear down zope.testrunner.layer.UnitTests in 0.000 seconds.
       Set up samplelayers.Layer1 in 0.000 seconds.
       Ran 9 tests with 0 failures, 0 errors and 0 skipped in 0.000 seconds.
     Running samplelayers.Layer11 tests:
@@ -46,14 +50,10 @@ directory name given as the parameter will be used to hold the reports.
       Tear down samplelayers.Layer121 in 0.000 seconds.
       Set up samplelayers.Layer122 in 0.000 seconds.
       Ran 26 tests with 0 failures, 0 errors and 0 skipped in 0.125 seconds.
-    Running zope.testrunner.layer.UnitTests tests:
+    Tearing down left over layers:
       Tear down samplelayers.Layer122 in 0.000 seconds.
       Tear down samplelayers.Layer12 in 0.000 seconds.
       Tear down samplelayers.Layer1 in 0.000 seconds.
-      Set up zope.testrunner.layer.UnitTests in 0.000 seconds.
-      Ran 156 tests with 0 failures, 0 errors and 0 skipped in 0.687 seconds.
-    Tearing down left over layers:
-      Tear down zope.testrunner.layer.UnitTests in 0.000 seconds.
     lines   cov%   module   (path)
     ...
        48   100%   sampletests.test1   (testrunner-ex/sampletests/test1.py)

diff --git a/src/zope/testrunner/tests/testrunner-edge-cases.txt b/src/zope/testrunner/tests/testrunner-edge-cases.txt
@@ -36,18 +36,18 @@ affecting the Python path:
     >>> sys.argv = ['test']
     >>> testrunner.run_internal(defaults)
     ... # doctest: +ELLIPSIS
+    Running zope.testrunner.layer.UnitTests tests:
+      Set up zope.testrunner.layer.UnitTests in N.NNN seconds.
+      Ran 156 tests with 0 failures, 0 errors and 0 skipped in N.NNN seconds.
     Running samplelayers.Layer1 tests:
+      Tear down zope.testrunner.layer.UnitTests in N.NNN seconds.
       Set up samplelayers.Layer1 in 0.000 seconds.
       Ran 9 tests with 0 failures, 0 errors and 0 skipped in 0.000 seconds.
     ...
-    Running zope.testrunner.layer.UnitTests tests:
+    Tearing down left over layers:
       Tear down samplelayers.Layer122 in N.NNN seconds.
       Tear down samplelayers.Layer12 in N.NNN seconds.
       Tear down samplelayers.Layer1 in N.NNN seconds.
-      Set up zope.testrunner.layer.UnitTests in N.NNN seconds.
-      Ran 156 tests with 0 failures, 0 errors and 0 skipped in N.NNN seconds.
-    Tearing down left over layers:
-      Tear down zope.testrunner.layer.UnitTests in N.NNN seconds.
     Total: 321 tests, 0 failures, 0 errors and 0 skipped in N.NNN seconds.
     False
 

diff --git a/src/zope/testrunner/tests/testrunner-errors.txt b/src/zope/testrunner/tests/testrunner-errors.txt
@@ -26,8 +26,6 @@ be read only):
     >>> sys.argv = 'test --tests-pattern ^sampletests(f|_e|_f)?$ '.split()
     >>> testrunner.run_internal(defaults)
     ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
-    Running samplelayers.Layer1 tests:
-    ...
     Running zope.testrunner.layer.UnitTests tests:
     ...
     <BLANKLINE>
@@ -97,8 +95,7 @@ be read only):
     AssertionError: 1 != 0
     <BLANKLINE>
       Ran 164 tests with 3 failures, 1 errors and 0 skipped in N.NNN seconds.
-    Tearing down left over layers:
-      Tear down zope.testrunner.layer.UnitTests in N.NNN seconds.
+    ...
     Total: 329 tests, 3 failures, 1 errors and 0 skipped in N.NNN seconds.
     True
 
@@ -835,16 +832,8 @@ Passing tests exit with code 0 according to UNIX practices:
     ... else:
     ...     print('sys.exit was not called')
     ... # doctest: +ELLIPSIS
-    Running samplelayers.Layer11 tests:
-    ...
     Running zope.testrunner.layer.UnitTests tests:
-      Tear down samplelayers.Layer122 in N.NNN seconds.
-      Tear down samplelayers.Layer12 in N.NNN seconds.
-      Tear down samplelayers.Layer1 in N.NNN seconds.
-      Set up zope.testrunner.layer.UnitTests in N.NNN seconds.
-      Ran 130 tests with 0 failures, 0 errors and 0 skipped in N.NNN seconds.
-    Tearing down left over layers:
-      Tear down zope.testrunner.layer.UnitTests in N.NNN seconds.
+    ...
     Total: 286 tests, 0 failures, 0 errors and 0 skipped in N.NNN seconds.
     exited with code 0
 

diff --git a/src/zope/testrunner/tests/testrunner-layers-api.txt b/src/zope/testrunner/tests/testrunner-layers-api.txt
@@ -114,15 +114,15 @@ TestSpecifyingBaseLayer tests.
 >>> from zope.testrunner.runner import Runner
 >>> runner = Runner(options=fresh_options(), args=[], found_suites=[umbrella_suite])
 >>> succeeded = runner.run()
-Running ...BaseLayer tests:
-  Set up ...BaseLayer in N.NNN seconds.
-  Ran 2 tests with 0 failures, 0 errors and 0 skipped in N.NNN seconds.
 Running zope.testrunner.layer.UnitTests tests:
-  Tear down ...BaseLayer in N.NNN seconds.
   Set up zope.testrunner.layer.UnitTests in N.NNN seconds.
   Ran 2 tests with 0 failures, 0 errors and 0 skipped in N.NNN seconds.
-Tearing down left over layers:
+Running ...BaseLayer tests:
   Tear down zope.testrunner.layer.UnitTests in N.NNN seconds.
+  Set up ...BaseLayer in N.NNN seconds.
+  Ran 2 tests with 0 failures, 0 errors and 0 skipped in N.NNN seconds.
+Tearing down left over layers:
+  Tear down ...BaseLayer in N.NNN seconds.
 Total: 4 tests, 0 failures, 0 errors and 0 skipped in N.NNN seconds.
 
 

diff --git a/src/zope/testrunner/tests/testrunner-layers-topological-sort.txt b/src/zope/testrunner/tests/testrunner-layers-topological-sort.txt
@@ -0,0 +1,90 @@
+Now lets stack a few more layers to ensure that our setUp and tearDown
+methods are called in the correct order.
+
+>>> from zope.testing.loggingsupport import InstalledHandler
+>>> from zope.testrunner.find import name_from_layer
+>>> from zope.testrunner import options
+>>> from zope.testrunner.runner import Runner
+>>> import logging
+>>> import unittest
+
+>>> def fresh_options():
+...     opts = options.get_options(['--test-filter', '.*'])
+...     opts.resume_layer = None
+...     opts.resume_number = 0
+...     return opts
+>>> def log(msg):
+...     logging.getLogger('zope.testrunner.tests').info(msg)
+>>> log_handler = InstalledHandler('zope.testrunner.tests')
+
+>>> class A(object):
+...     def setUp(cls):
+...         log('%s.setUp' % name_from_layer(cls))
+...     setUp = classmethod(setUp)
+...
+...     def tearDown(cls):
+...         log('%s.tearDown' % name_from_layer(cls))
+...     tearDown = classmethod(tearDown)
+...
+...     def testSetUp(cls):
+...         log('%s.testSetUp' % name_from_layer(cls))
+...     testSetUp = classmethod(testSetUp)
+...
+...     def testTearDown(cls):
+...         log('%s.testTearDown' % name_from_layer(cls))
+...     testTearDown = classmethod(testTearDown)
+...         
+>>> class AB(A): pass
+>>> class AC(A): pass
+>>> class AAAABD(AB): pass
+>>> class ZZZABE(AB): pass
+>>> class MMMACF(AC): pass
+
+>>> class DeepTest1(unittest.TestCase):
+...     layer = AAAABD
+...     def test(self):
+...         pass
+>>> class DeepTest2(unittest.TestCase):
+...     layer = MMMACF
+...     def test(self):
+...         pass
+>>> class DeepTest3(unittest.TestCase):
+...     layer = ZZZABE
+...     def test(self):
+...         pass
+>>> class QuickTests(unittest.TestCase):
+...     def test(self):
+...         pass
+>>> suite = unittest.TestSuite()
+>>> suite.addTest(unittest.makeSuite(DeepTest1))
+>>> suite.addTest(unittest.makeSuite(DeepTest2))
+>>> suite.addTest(unittest.makeSuite(DeepTest3))
+>>> suite.addTest(unittest.makeSuite(QuickTests))
+>>> log_handler.clear()
+>>> runner = Runner(options=fresh_options(), args=[], found_suites=[suite])
+>>> succeeded = runner.run() #doctest: +ELLIPSIS
+Running zope.testrunner.layer.UnitTests tests:
+  Set up zope.testrunner.layer.UnitTests in N.NNN seconds.
+  Ran 1 tests with 0 failures, 0 errors and 0 skipped in N.NNN seconds.
+Running ...AAAABD tests:
+  Tear down zope.testrunner.layer.UnitTests in N.NNN seconds.
+  Set up ...A in N.NNN seconds.
+  Set up ...AB in N.NNN seconds.
+  Set up ...AAAABD in N.NNN seconds.
+  Ran 1 tests with 0 failures, 0 errors and 0 skipped in N.NNN seconds.
+Running ...ZZZABE tests:
+  Tear down ...AAAABD in N.NNN seconds.
+  Set up ...ZZZABE in N.NNN seconds.
+  Ran 1 tests with 0 failures, 0 errors and 0 skipped in N.NNN seconds.
+Running ...MMMACF tests:
+  Tear down ...ZZZABE in N.NNN seconds.
+  Tear down ...AB in N.NNN seconds.
+  Set up ...AC in N.NNN seconds.
+  Set up ...MMMACF in N.NNN seconds.
+  Ran 1 tests with 0 failures, 0 errors and 0 skipped in N.NNN seconds.
+Tearing down left over layers:
+  Tear down ...MMMACF in N.NNN seconds.
+  Tear down ...AC in N.NNN seconds.
+  Tear down ...A in N.NNN seconds.
+Total: 4 tests, 0 failures, 0 errors and 0 skipped in N.NNN seconds.
+