Skip to content

Commit

Permalink
bpo-25894: Always report skipped and failed subtests separately (GH-2…
Browse files Browse the repository at this point in the history 
…8082)

* In default mode output separate characters for skipped and failed subtests.
* In verbose mode output separate lines (including description) for skipped
   and failed subtests.
* In verbose mode output test description for errors in test cleanup.
  • Loading branch information
@serhiy-storchaka
serhiy-storchaka committed Sep 10, 2021
1 parent ab327f2 commit f0f29f3
Show file tree
Hide file tree
Showing 3 changed files with 189 additions and 49 deletions.
38 changes: 34 additions & 4 deletions Lib/unittest/runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import warnings

from . import result
from .case import _SubTest
from .signals import registerResult

__unittest = True
Expand Down Expand Up @@ -40,6 +41,7 @@ def __init__(self, stream, descriptions, verbosity):
self.showAll = verbosity > 1
self.dots = verbosity == 1
self.descriptions = descriptions
self._newline = True

def getDescription(self, test):
doc_first_line = test.shortDescription()
Expand All @@ -54,35 +56,63 @@ def startTest(self, test):
self.stream.write(self.getDescription(test))
self.stream.write(" ... ")
self.stream.flush()
self._newline = False

def _write_status(self, test, status):
is_subtest = isinstance(test, _SubTest)
if is_subtest or self._newline:
if not self._newline:
self.stream.writeln()
if is_subtest:
self.stream.write(" ")
self.stream.write(self.getDescription(test))
self.stream.write(" ... ")
self.stream.writeln(status)
self._newline = True

def addSubTest(self, test, subtest, err):
if err is not None:
if self.showAll:
if issubclass(err[0], subtest.failureException):
self._write_status(subtest, "FAIL")
else:
self._write_status(subtest, "ERROR")
elif self.dots:
if issubclass(err[0], subtest.failureException):
self.stream.write('F')
else:
self.stream.write('E')
self.stream.flush()
super(TextTestResult, self).addSubTest(test, subtest, err)

def addSuccess(self, test):
super(TextTestResult, self).addSuccess(test)
if self.showAll:
self.stream.writeln("ok")
self._write_status(test, "ok")
elif self.dots:
self.stream.write('.')
self.stream.flush()

def addError(self, test, err):
super(TextTestResult, self).addError(test, err)
if self.showAll:
self.stream.writeln("ERROR")
self._write_status(test, "ERROR")
elif self.dots:
self.stream.write('E')
self.stream.flush()

def addFailure(self, test, err):
super(TextTestResult, self).addFailure(test, err)
if self.showAll:
self.stream.writeln("FAIL")
self._write_status(test, "FAIL")
elif self.dots:
self.stream.write('F')
self.stream.flush()

def addSkip(self, test, reason):
super(TextTestResult, self).addSkip(test, reason)
if self.showAll:
self.stream.writeln("skipped {0!r}".format(reason))
self._write_status(test, "skipped {0!r}".format(reason))
elif self.dots:
self.stream.write("s")
self.stream.flush()
Expand Down
196 changes: 151 additions & 45 deletions Lib/unittest/test/test_result.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,37 +305,76 @@ def test_1(self):
self.assertIs(test_case, subtest)
self.assertIn("some recognizable failure", formatted_exc)

def testStackFrameTrimming(self):
class Frame(object):
class tb_frame(object):
f_globals = {}
result = unittest.TestResult()
self.assertFalse(result._is_relevant_tb_level(Frame))

Frame.tb_frame.f_globals['__unittest'] = True
self.assertTrue(result._is_relevant_tb_level(Frame))

def testFailFast(self):
result = unittest.TestResult()
result._exc_info_to_string = lambda *_: ''
result.failfast = True
result.addError(None, None)
self.assertTrue(result.shouldStop)

result = unittest.TestResult()
result._exc_info_to_string = lambda *_: ''
result.failfast = True
result.addFailure(None, None)
self.assertTrue(result.shouldStop)

result = unittest.TestResult()
result._exc_info_to_string = lambda *_: ''
result.failfast = True
result.addUnexpectedSuccess(None)
self.assertTrue(result.shouldStop)

def testFailFastSetByRunner(self):
runner = unittest.TextTestRunner(stream=io.StringIO(), failfast=True)
def test(result):
self.assertTrue(result.failfast)
result = runner.run(test)


class Test_TextTestResult(unittest.TestCase):
maxDiff = None

def testGetDescriptionWithoutDocstring(self):
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self),
'testGetDescriptionWithoutDocstring (' + __name__ +
'.Test_TestResult)')
'.Test_TextTestResult)')

def testGetSubTestDescriptionWithoutDocstring(self):
with self.subTest(foo=1, bar=2):
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self._subtest),
'testGetSubTestDescriptionWithoutDocstring (' + __name__ +
'.Test_TestResult) (foo=1, bar=2)')
'.Test_TextTestResult) (foo=1, bar=2)')
with self.subTest('some message'):
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self._subtest),
'testGetSubTestDescriptionWithoutDocstring (' + __name__ +
'.Test_TestResult) [some message]')
'.Test_TextTestResult) [some message]')

def testGetSubTestDescriptionWithoutDocstringAndParams(self):
with self.subTest():
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self._subtest),
'testGetSubTestDescriptionWithoutDocstringAndParams '
'(' + __name__ + '.Test_TestResult) (<subtest>)')
'(' + __name__ + '.Test_TextTestResult) (<subtest>)')

def testGetSubTestDescriptionForFalsyValues(self):
expected = 'testGetSubTestDescriptionForFalsyValues (%s.Test_TestResult) [%s]'
expected = 'testGetSubTestDescriptionForFalsyValues (%s.Test_TextTestResult) [%s]'
result = unittest.TextTestResult(None, True, 1)
for arg in [0, None, []]:
with self.subTest(arg):
Expand All @@ -351,7 +390,7 @@ def testGetNestedSubTestDescriptionWithoutDocstring(self):
self.assertEqual(
result.getDescription(self._subtest),
'testGetNestedSubTestDescriptionWithoutDocstring '
'(' + __name__ + '.Test_TestResult) (baz=2, bar=3, foo=1)')
'(' + __name__ + '.Test_TextTestResult) (baz=2, bar=3, foo=1)')

def testGetDuplicatedNestedSubTestDescriptionWithoutDocstring(self):
with self.subTest(foo=1, bar=2):
Expand All @@ -360,7 +399,7 @@ def testGetDuplicatedNestedSubTestDescriptionWithoutDocstring(self):
self.assertEqual(
result.getDescription(self._subtest),
'testGetDuplicatedNestedSubTestDescriptionWithoutDocstring '
'(' + __name__ + '.Test_TestResult) (baz=3, bar=4, foo=1)')
'(' + __name__ + '.Test_TextTestResult) (baz=3, bar=4, foo=1)')

@unittest.skipIf(sys.flags.optimize >= 2,
"Docstrings are omitted with -O2 and above")
Expand All @@ -370,7 +409,7 @@ def testGetDescriptionWithOneLineDocstring(self):
self.assertEqual(
result.getDescription(self),
('testGetDescriptionWithOneLineDocstring '
'(' + __name__ + '.Test_TestResult)\n'
'(' + __name__ + '.Test_TextTestResult)\n'
'Tests getDescription() for a method with a docstring.'))

@unittest.skipIf(sys.flags.optimize >= 2,
Expand All @@ -382,7 +421,7 @@ def testGetSubTestDescriptionWithOneLineDocstring(self):
self.assertEqual(
result.getDescription(self._subtest),
('testGetSubTestDescriptionWithOneLineDocstring '
'(' + __name__ + '.Test_TestResult) (foo=1, bar=2)\n'
'(' + __name__ + '.Test_TextTestResult) (foo=1, bar=2)\n'
'Tests getDescription() for a method with a docstring.'))

@unittest.skipIf(sys.flags.optimize >= 2,
Expand All @@ -395,7 +434,7 @@ def testGetDescriptionWithMultiLineDocstring(self):
self.assertEqual(
result.getDescription(self),
('testGetDescriptionWithMultiLineDocstring '
'(' + __name__ + '.Test_TestResult)\n'
'(' + __name__ + '.Test_TextTestResult)\n'
'Tests getDescription() for a method with a longer '
'docstring.'))

Expand All @@ -410,44 +449,111 @@ def testGetSubTestDescriptionWithMultiLineDocstring(self):
self.assertEqual(
result.getDescription(self._subtest),
('testGetSubTestDescriptionWithMultiLineDocstring '
'(' + __name__ + '.Test_TestResult) (foo=1, bar=2)\n'
'(' + __name__ + '.Test_TextTestResult) (foo=1, bar=2)\n'
'Tests getDescription() for a method with a longer '
'docstring.'))

def testStackFrameTrimming(self):
class Frame(object):
class tb_frame(object):
f_globals = {}
result = unittest.TestResult()
self.assertFalse(result._is_relevant_tb_level(Frame))

Frame.tb_frame.f_globals['__unittest'] = True
self.assertTrue(result._is_relevant_tb_level(Frame))

def testFailFast(self):
result = unittest.TestResult()
result._exc_info_to_string = lambda *_: ''
result.failfast = True
result.addError(None, None)
self.assertTrue(result.shouldStop)

result = unittest.TestResult()
result._exc_info_to_string = lambda *_: ''
result.failfast = True
result.addFailure(None, None)
self.assertTrue(result.shouldStop)

result = unittest.TestResult()
result._exc_info_to_string = lambda *_: ''
result.failfast = True
result.addUnexpectedSuccess(None)
self.assertTrue(result.shouldStop)

def testFailFastSetByRunner(self):
runner = unittest.TextTestRunner(stream=io.StringIO(), failfast=True)
def test(result):
self.assertTrue(result.failfast)
result = runner.run(test)
class Test(unittest.TestCase):
def testSuccess(self):
pass
def testSkip(self):
self.skipTest('skip')
def testFail(self):
self.fail('fail')
def testError(self):
raise Exception('error')
def testSubTestSuccess(self):
with self.subTest('one', a=1):
pass
with self.subTest('two', b=2):
pass
def testSubTestMixed(self):
with self.subTest('success', a=1):
pass
with self.subTest('skip', b=2):
self.skipTest('skip')
with self.subTest('fail', c=3):
self.fail('fail')
with self.subTest('error', d=4):
raise Exception('error')

tearDownError = None
def tearDown(self):
if self.tearDownError is not None:
raise self.tearDownError

def _run_test(self, test_name, verbosity, tearDownError=None):
stream = io.StringIO()
stream = unittest.runner._WritelnDecorator(stream)
result = unittest.TextTestResult(stream, True, verbosity)
test = self.Test(test_name)
test.tearDownError = tearDownError
test.run(result)
return stream.getvalue()

def testDotsOutput(self):
self.assertEqual(self._run_test('testSuccess', 1), '.')
self.assertEqual(self._run_test('testSkip', 1), 's')
self.assertEqual(self._run_test('testFail', 1), 'F')
self.assertEqual(self._run_test('testError', 1), 'E')

def testLongOutput(self):
classname = f'{__name__}.{self.Test.__qualname__}'
self.assertEqual(self._run_test('testSuccess', 2),
f'testSuccess ({classname}) ... ok\n')
self.assertEqual(self._run_test('testSkip', 2),
f"testSkip ({classname}) ... skipped 'skip'\n")
self.assertEqual(self._run_test('testFail', 2),
f'testFail ({classname}) ... FAIL\n')
self.assertEqual(self._run_test('testError', 2),
f'testError ({classname}) ... ERROR\n')

def testDotsOutputSubTestSuccess(self):
self.assertEqual(self._run_test('testSubTestSuccess', 1), '.')

def testLongOutputSubTestSuccess(self):
classname = f'{__name__}.{self.Test.__qualname__}'
self.assertEqual(self._run_test('testSubTestSuccess', 2),
f'testSubTestSuccess ({classname}) ... ok\n')

def testDotsOutputSubTestMixed(self):
self.assertEqual(self._run_test('testSubTestMixed', 1), 'sFE')

def testLongOutputSubTestMixed(self):
classname = f'{__name__}.{self.Test.__qualname__}'
self.assertEqual(self._run_test('testSubTestMixed', 2),
f'testSubTestMixed ({classname}) ... \n'
f" testSubTestMixed ({classname}) [skip] (b=2) ... skipped 'skip'\n"
f' testSubTestMixed ({classname}) [fail] (c=3) ... FAIL\n'
f' testSubTestMixed ({classname}) [error] (d=4) ... ERROR\n')

def testDotsOutputTearDownFail(self):
out = self._run_test('testSuccess', 1, AssertionError('fail'))
self.assertEqual(out, 'F')
out = self._run_test('testError', 1, AssertionError('fail'))
self.assertEqual(out, 'EF')
out = self._run_test('testFail', 1, Exception('error'))
self.assertEqual(out, 'FE')
out = self._run_test('testSkip', 1, AssertionError('fail'))
self.assertEqual(out, 'sF')

def testLongOutputTearDownFail(self):
classname = f'{__name__}.{self.Test.__qualname__}'
out = self._run_test('testSuccess', 2, AssertionError('fail'))
self.assertEqual(out,
f'testSuccess ({classname}) ... FAIL\n')
out = self._run_test('testError', 2, AssertionError('fail'))
self.assertEqual(out,
f'testError ({classname}) ... ERROR\n'
f'testError ({classname}) ... FAIL\n')
out = self._run_test('testFail', 2, Exception('error'))
self.assertEqual(out,
f'testFail ({classname}) ... FAIL\n'
f'testFail ({classname}) ... ERROR\n')
out = self._run_test('testSkip', 2, AssertionError('fail'))
self.assertEqual(out,
f"testSkip ({classname}) ... skipped 'skip'\n"
f'testSkip ({classname}) ... FAIL\n')


classDict = dict(unittest.TestResult.__dict__)
Expand Down
4 changes: 4 additions & 0 deletions Misc/NEWS.d/next/Library/2021-09-05-13-15-08.bpo-25894.zjbi2f.rst
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
:mod:`unittest` now always reports skipped and failed subtests separately:
separate characters in default mode and separate lines in verbose mode. Also
the test description is now output for errors in test method, class and
module cleanups.

0 comments on commit f0f29f3

Please sign in to comment.