From 23f40382c67e82d24db95dfa672d2fdc87812964 Mon Sep 17 00:00:00 2001 From: Dan Villiom Podlaski Christiansen Date: Thu, 14 Mar 2019 17:19:50 -0700 Subject: [PATCH 1/4] always include output and info for testcases --- xmlrunner/result.py | 55 +++++++++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/xmlrunner/result.py b/xmlrunner/result.py index e931337..2749820 100644 --- a/xmlrunner/result.py +++ b/xmlrunner/result.py @@ -130,6 +130,13 @@ class _TestInfo(object): # Possible test outcomes (SUCCESS, FAILURE, ERROR, SKIP) = range(4) + OUTCOME_ELEMENTS = { + SUCCESS: None, + FAILURE: 'failure', + ERROR: 'error', + SKIP: 'skipped', + } + def __init__(self, test_result, test_method, outcome=SUCCESS, err=None, subTest=None): self.test_result = test_result self.outcome = outcome @@ -531,25 +538,39 @@ def _report_testcase(test_result, xml_testsuite, xml_document): testcase.setAttribute('time', '%.3f' % test_result.elapsed_time) testcase.setAttribute('timestamp', test_result.timestamp) - if (test_result.outcome != test_result.SUCCESS): - elem_name = ('failure', 'error', 'skipped')[test_result.outcome-1] - failure = xml_document.createElement(elem_name) - testcase.appendChild(failure) - if test_result.outcome != test_result.SKIP: - failure.setAttribute( - 'type', - test_result.test_exception_name - ) - failure.setAttribute( - 'message', - test_result.test_exception_message - ) + if test_result.stdout: + systemout = xml_document.createElement('system-out') + testcase.appendChild(systemout) + + _XMLTestResult._createCDATAsections(xml_document, systemout, + test_result.stdout) + + if test_result.stderr: + systemerr = xml_document.createElement('system-err') + testcase.appendChild(systemerr) + + _XMLTestResult._createCDATAsections(xml_document, systemerr, + test_result.stderr) + + + result_elem_name = test_result.OUTCOME_ELEMENTS[test_result.outcome] + + if result_elem_name: + result_elem = xml_document.createElement(result_elem_name) + testcase.appendChild(result_elem) + + result_elem.setAttribute( + 'type', + test_result.test_exception_name + ) + result_elem.setAttribute( + 'message', + test_result.test_exception_message + ) + if test_result.get_error_info(): error_info = safe_unicode(test_result.get_error_info()) _XMLTestResult._createCDATAsections( - xml_document, failure, error_info) - else: - failure.setAttribute('type', 'skip') - failure.setAttribute('message', test_result.test_exception_message) + xml_document, result_elem, error_info) if test_result.stdout: systemout = xml_document.createElement('system-out') From 476b2f0bc405da79097821e21e391efc5f6a9bc0 Mon Sep 17 00:00:00 2001 From: Dan Villiom Podlaski Christiansen Date: Fri, 15 Mar 2019 10:09:12 +0100 Subject: [PATCH 2/4] adjust output to correspond to unittest Generally speaking, expected occurrences are lowercase whereas anything exceptional and worthy of notice is uppercase. Examples of the former include tests passing, skips and expected failures; examples of the latter include failures, errors and unexpected successes. --- xmlrunner/result.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/xmlrunner/result.py b/xmlrunner/result.py index 2749820..1609f58 100644 --- a/xmlrunner/result.py +++ b/xmlrunner/result.py @@ -313,7 +313,7 @@ def addSuccess(self, test): """ self._save_output_data() self._prepare_callback( - self.infoclass(self, test), self.successes, 'OK', '.' + self.infoclass(self, test), self.successes, 'ok', '.' ) @failfast @@ -380,8 +380,10 @@ def addSkip(self, test, reason): self._save_output_data() testinfo = self.infoclass( self, test, self.infoclass.SKIP, reason) + testinfo.test_exception_name = 'skip' + testinfo.test_exception_message = reason self.skipped.append((testinfo, reason)) - self._prepare_callback(testinfo, [], 'SKIP', 'S') + self._prepare_callback(testinfo, [], 'skip', 's') def addExpectedFailure(self, test, err): """ @@ -390,11 +392,11 @@ def addExpectedFailure(self, test, err): self._save_output_data() testinfo = self.infoclass(self, test, self.infoclass.ERROR, err) - testinfo.test_exception_name = 'ExpectedFailure' - testinfo.test_exception_message = 'EXPECTED FAILURE: {}'.format(testinfo.test_exception_message) + testinfo.test_exception_name = 'XFAIL' + testinfo.test_exception_message = 'expected failure: {}'.format(testinfo.test_exception_message) self.expectedFailures.append((testinfo, self._exc_info_to_string(err, test))) - self._prepare_callback(testinfo, [], 'EXPECTED FAILURE', 'X') + self._prepare_callback(testinfo, [], 'expected failure', 'x') @failfast def addUnexpectedSuccess(self, test): @@ -407,11 +409,11 @@ def addUnexpectedSuccess(self, test): testinfo.outcome = self.infoclass.ERROR # But since we want to have error outcome, we need to provide additional fields: testinfo.test_exception_name = 'UnexpectedSuccess' - testinfo.test_exception_message = ('UNEXPECTED SUCCESS: This test was marked as expected failure but passed, ' + testinfo.test_exception_message = ('Unexpected success: This test was marked as expected failure but passed, ' 'please review it') - self.unexpectedSuccesses.append(testinfo) - self._prepare_callback(testinfo, [], 'UNEXPECTED SUCCESS', 'U') + self.unexpectedSuccesses.append((testinfo, 'unexpected success')) + self._prepare_callback(testinfo, [], 'unexpected success', 'u') def printErrorList(self, flavour, errors): """ From b3b8fc26d8ab4173ab9af90e5e7d1718bf7d9a30 Mon Sep 17 00:00:00 2001 From: Dan Villiom Podlaski Christiansen Date: Fri, 15 Mar 2019 10:09:13 +0100 Subject: [PATCH 3/4] _test_xmlrunner: allow passing a BytesIO as outdir --- tests/testsuite.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/testsuite.py b/tests/testsuite.py index b208c30..f49e556 100755 --- a/tests/testsuite.py +++ b/tests/testsuite.py @@ -185,8 +185,9 @@ def setUp(self): self.runner_kwargs = {} self.addCleanup(rmtree, self.outdir) - def _test_xmlrunner(self, suite, runner=None): - outdir = self.outdir + def _test_xmlrunner(self, suite, runner=None, outdir=None): + if outdir is None: + outdir = self.outdir stream = self.stream verbosity = self.verbosity runner_kwargs = self.runner_kwargs @@ -194,9 +195,15 @@ def _test_xmlrunner(self, suite, runner=None): runner = xmlrunner.XMLTestRunner( stream=stream, output=outdir, verbosity=verbosity, **runner_kwargs) - self.assertEqual(0, len(glob(os.path.join(outdir, '*xml')))) + if isinstance(outdir, BytesIO): + self.assertFalse(outdir.getvalue()) + else: + self.assertEqual(0, len(glob(os.path.join(outdir, '*xml')))) runner.run(suite) - self.assertEqual(1, len(glob(os.path.join(outdir, '*xml')))) + if isinstance(outdir, BytesIO): + self.assertTrue(outdir.getvalue()) + else: + self.assertEqual(1, len(glob(os.path.join(outdir, '*xml')))) return runner def test_basic_unittest_constructs(self): From 7c4d6a298971ee529abef62b0d9a6a93766ac42f Mon Sep 17 00:00:00 2001 From: Dan Villiom Podlaski Christiansen Date: Thu, 14 Mar 2019 11:33:23 +0100 Subject: [PATCH 4/4] properly include expected failures & unexpected successes in output --- tests/testsuite.py | 36 ++++++++++++++++++++++++++++++++++++ xmlrunner/result.py | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/tests/testsuite.py b/tests/testsuite.py index f49e556..212a220 100755 --- a/tests/testsuite.py +++ b/tests/testsuite.py @@ -237,6 +237,42 @@ def test_classnames(self): r'classname="tests\.testsuite\.(XMLTestRunnerTestCase\.)?' r'DummySubTest" name="test_subTest_pass"'.encode('utf8')) + def test_expected_failure(self): + suite = unittest.TestSuite() + suite.addTest(self.DummyTest('test_expected_failure')) + outdir = BytesIO() + + self._test_xmlrunner(suite, outdir=outdir) + + self.assertNotIn(b'