Merge pull request #478 from stefanholek/334-subtest-support

Subtest support

nose2/events.py

@@ -4,7 +4,9 @@
 # unittest2 is Copyright (c) 2001-2010 Python Software Foundation; All
 # Rights Reserved. See: http://docs.python.org/license.html
 
+import sys
 import logging
+import unittest
 
 import argparse
 import six
@@ -348,10 +350,15 @@ def _format(self):
                           for k in self._attrs])
 
     def __getstate__(self):
-        state = self.__dict__
+        state = self.__dict__.copy()
         # FIXME fails for loadTestsFailure
         if 'test' in state:
-            state['test'] = util.test_name(state['test'])
+            test = state['test']
+            state['test'] = util.test_name(test)
+            # subtest support
+            if sys.version_info >= (3, 4):
+                if isinstance(test, unittest.case._SubTest):
+                    state['metadata']['subtest'] = (test._message, test.params)
         if 'executeTests' in state:
             state['executeTests'] = None
         if 'exc_info' in state and state['exc_info'] is not None:
@@ -704,7 +711,7 @@ class TestOutcomeEvent(Event):
     .. attribute :: outcome
 
        Description of test outcome. Typically will be one of 'error',
-       'failed', 'skipped', or 'passed'.
+       'failed', 'skipped', 'passed', or 'subtest'.
 
     .. attribute :: exc_info
 

nose2/plugins/failfast.py

@@ -19,6 +19,11 @@ class FailFast(events.Plugin):
     commandLineSwitch = (
         'F', 'fail-fast', 'Stop the test run after the first error or failure')
 
+    def resultCreated(self, event):
+        """Mark new result"""
+        if hasattr(event.result, 'failfast'):
+            event.result.failfast = True
+
     def testOutcome(self, event):
         """Stop on unexpected error or failure"""
         if event.exc_info and not event.expected:

nose2/plugins/junitxml.py

@@ -143,6 +143,10 @@ def testOutcome(self, event):
             test_args = ':'.join(testid_lines[1:])
             method = '%s (%s)' % (method, test_args)
 
+        # subtests do not report success
+        if event.outcome == result.SUBTEST and event.exc_info is None:
+            return
+
         testcase = ET.SubElement(self.tree, 'testcase')
         testcase.set('time', "%.6f" % self._time())
         if not classname:
@@ -186,6 +190,17 @@ def testOutcome(self, event):
             skipped = ET.SubElement(testcase, 'skipped')
             skipped.set('message', 'expected test failure')
             skipped.text = msg
+        elif event.outcome == result.SUBTEST:
+            if issubclass(event.exc_info[0], event.test.failureException):
+                self.failed += 1
+                failure = ET.SubElement(testcase, 'failure')
+                failure.set('message', 'test failure')
+                failure.text = msg
+            else:
+                self.errors += 1
+                error = ET.SubElement(testcase, 'error')
+                error.set('message', 'test failure')
+                error.text = msg
 
         system_out = ET.SubElement(testcase, 'system-out')
         system_out.text = string_cleanup(

nose2/plugins/mp.py

@@ -274,6 +274,14 @@ def _localize(self, event):
                     'test_not_found',
                     RuntimeError("Unable to locate test case for %s in "
                                  "main process" % event.test))._tests[0]
+            # subtest support
+            if 'subtest' in event.metadata:
+                message, params = event.metadata.pop('subtest')
+                # XXX the sentinel value does not survive the pickle
+                # round-trip and must be reset by hand
+                if type(message) == type(object()):
+                    message = unittest.case._subtest_msg_sentinel
+                event.test = unittest.case._SubTest(event.test, message, params)
 
     def _exportSession(self):
         """

nose2/plugins/result.py

@@ -90,6 +90,15 @@ def testOutcome(self, event):
             else:
                 self.reportCategories['unexpectedSuccesses'].append(event)
                 self._reportUnexpectedSuccess(event)
+        elif event.outcome == result.SUBTEST:
+            # subtests do not report success
+            if event.exc_info is not None:
+                if issubclass(event.exc_info[0], event.test.failureException):
+                    self.reportCategories['failures'].append(event)
+                    self._reportFailure(event)
+                else:
+                    self.reportCategories['errors'].append(event)
+                    self._reportError(event)
         else:
             # generic outcome handling
             self.reportCategories.setdefault(event.outcome, []).append(event)
@@ -115,7 +124,8 @@ def wasSuccessful(self, event):
         for name, events in self.reportCategories.items():
             for e in events:
                 if (e.outcome == result.ERROR or
-                    (e.outcome == result.FAIL and not e.expected)):
+                    (e.outcome == result.FAIL and not e.expected) or
+                    e.outcome == result.SUBTEST):
                     event.success = False
                     break
 

nose2/result.py

@@ -7,6 +7,7 @@
 FAIL = 'failed'
 SKIP = 'skipped'
 PASS = 'passed'
+SUBTEST = 'subtest'
 __unittest = True
 
 
@@ -29,6 +30,8 @@ class PluggableTestResult(object):
     def __init__(self, session):
         self.session = session
         self.shouldStop = False
+        # XXX TestCase.subTest expects a result.failfast attribute
+        self.failfast = False
 
     def startTest(self, test):
         """Start a test case.
@@ -68,6 +71,16 @@ def addFailure(self, test, err):
         self.session.hooks.setTestOutcome(event)
         self.session.hooks.testOutcome(event)
 
+    def addSubTest(self, test, subtest, err):
+        """Called at the end of a subtest.
+
+        Fires :func:`setTestOutcome` and :func:`testOutcome` hooks.
+
+        """
+        event = events.TestOutcomeEvent(subtest, self, SUBTEST, err)
+        self.session.hooks.setTestOutcome(event)
+        self.session.hooks.testOutcome(event)
+
     def addSuccess(self, test):
         """Test case resulted in success.
 

nose2/tests/functional/support/scenario/subtests/test_subtests.py

@@ -0,0 +1,30 @@
+import unittest
+
+
+class Case(unittest.TestCase):
+
+    def test_subtest_success(self):
+        for i in range(3):
+            with self.subTest(i=i):
+                self.assertTrue(i < 3)
+
+    def test_subtest_failure(self):
+        for i in range(6):
+            with self.subTest(i=i):
+                self.assertEqual(i%2, 0)
+
+    def test_subtest_error(self):
+        for i in range(3):
+            with self.subTest(i=i):
+                raise RuntimeError(i)
+
+    @unittest.expectedFailure
+    def test_subtest_expected_failure(self):
+        for i in range(6):
+            with self.subTest(i=i):
+                self.assertEqual(i%2, 0)
+
+    def test_subtest_message(self):
+        for i in range(6):
+            with self.subTest('msg', i=i):
+                self.assertEqual(i%2, 0)