diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 92da6133f9bf09..6e49f965e34190 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -1409,6 +1409,28 @@ DocTestParser objects identifying this string, and is only used for error messages. +TestResults objects +^^^^^^^^^^^^^^^^^^^ + + +.. class:: TestResults(failed, attempted) + + .. attribute:: failed + + Number of failed tests. + + .. attribute:: attempted + + Number of attempted tests. + + .. attribute:: skipped + + Number of skipped tests. + + .. versionchanged:: 3.13 + Add :attr:`skipped` attribute. + + .. _doctest-doctestrunner: DocTestRunner objects @@ -1500,7 +1522,7 @@ DocTestRunner objects .. method:: run(test, compileflags=None, out=None, clear_globs=True) Run the examples in *test* (a :class:`DocTest` object), and display the - results using the writer function *out*. + results using the writer function *out*. Return a :class:`TestResults`. The examples are run in the namespace ``test.globs``. If *clear_globs* is true (the default), then this namespace will be cleared after the test runs, @@ -1519,7 +1541,7 @@ DocTestRunner objects .. method:: summarize(verbose=None) Print a summary of all the test cases that have been run by this DocTestRunner, - and return a :term:`named tuple` ``TestResults(failed, attempted)``. + and return a :class:`TestResults`. The optional *verbose* argument controls how detailed the summary is. If the verbosity is not specified, then the :class:`DocTestRunner`'s verbosity is diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index d35d20ebb25293..b254d3d1828640 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -122,6 +122,13 @@ dbm from the database. (Contributed by Dong-hee Na in :gh:`107122`.) +doctest +------- + +* :meth:`doctest.DocTestRunner.run` method now counts the number of skipped + tests. Add :attr:`doctest.TestResults.skipped` attribute. + (Contributed by Victor Stinner in :gh:`108794`.) + io -- diff --git a/Lib/doctest.py b/Lib/doctest.py index a63df46a112e64..2d9ea0abf0c37e 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -105,7 +105,21 @@ def _test(): from io import StringIO, IncrementalNewlineDecoder from collections import namedtuple -TestResults = namedtuple('TestResults', 'failed attempted') + +class TestResults(namedtuple('TestResults', 'failed attempted')): + def __new__(cls, failed, attempted, *, skipped=0): + results = super().__new__(cls, failed, attempted) + results.skipped = skipped + return results + + def __repr__(self): + if self.skipped: + return (f'TestResults(failed={self.failed}, ' + f'attempted={self.attempted}, ' + f'skipped={self.skipped})') + else: + return super().__repr__() + # There are 4 basic classes: # - Example: a pair, plus an intra-docstring line number. @@ -1150,8 +1164,7 @@ class DocTestRunner: """ A class used to run DocTest test cases, and accumulate statistics. The `run` method is used to process a single DocTest case. It - returns a tuple `(f, t)`, where `t` is the number of test cases - tried, and `f` is the number of test cases that failed. + returns a TestResults. >>> tests = DocTestFinder().find(_TestClass) >>> runner = DocTestRunner(verbose=False) @@ -1164,8 +1177,7 @@ class DocTestRunner: _TestClass.square -> TestResults(failed=0, attempted=1) The `summarize` method prints a summary of all the test cases that - have been run by the runner, and returns an aggregated `(f, t)` - tuple: + have been run by the runner, and returns an aggregated TestResults: >>> runner.summarize(verbose=1) 4 items passed all tests: @@ -1233,7 +1245,8 @@ def __init__(self, checker=None, verbose=None, optionflags=0): # Keep track of the examples we've run. self.tries = 0 self.failures = 0 - self._name2ft = {} + self.skipped = 0 + self._stats = {} # Create a fake output target for capturing doctest output. self._fakeout = _SpoofOut() @@ -1302,13 +1315,11 @@ def __run(self, test, compileflags, out): Run the examples in `test`. Write the outcome of each example with one of the `DocTestRunner.report_*` methods, using the writer function `out`. `compileflags` is the set of compiler - flags that should be used to execute examples. Return a tuple - `(f, t)`, where `t` is the number of examples tried, and `f` - is the number of examples that failed. The examples are run - in the namespace `test.globs`. + flags that should be used to execute examples. Return a TestResults. + The examples are run in the namespace `test.globs`. """ - # Keep track of the number of failures and tries. - failures = tries = 0 + # Keep track of the number of failures, attempted and skipped. + failures = attempted = skipped = 0 # Save the option flags (since option directives can be used # to modify them). @@ -1320,6 +1331,7 @@ def __run(self, test, compileflags, out): # Process each example. for examplenum, example in enumerate(test.examples): + attempted += 1 # If REPORT_ONLY_FIRST_FAILURE is set, then suppress # reporting after the first failure. @@ -1337,10 +1349,10 @@ def __run(self, test, compileflags, out): # If 'SKIP' is set, then skip this example. if self.optionflags & SKIP: + skipped += 1 continue # Record that we started this example. - tries += 1 if not quiet: self.report_start(out, test, example) @@ -1418,19 +1430,22 @@ def __run(self, test, compileflags, out): # Restore the option flags (in case they were modified) self.optionflags = original_optionflags - # Record and return the number of failures and tries. - self.__record_outcome(test, failures, tries) - return TestResults(failures, tries) + # Record and return the number of failures and attempted. + self.__record_outcome(test, failures, attempted, skipped) + return TestResults(failures, attempted, skipped=skipped) - def __record_outcome(self, test, f, t): + def __record_outcome(self, test, failures, tries, skipped): """ - Record the fact that the given DocTest (`test`) generated `f` - failures out of `t` tried examples. + Record the fact that the given DocTest (`test`) generated `failures` + failures out of `tries` tried examples. """ - f2, t2 = self._name2ft.get(test.name, (0,0)) - self._name2ft[test.name] = (f+f2, t+t2) - self.failures += f - self.tries += t + failures2, tries2, skipped2 = self._stats.get(test.name, (0, 0, 0)) + self._stats[test.name] = (failures + failures2, + tries + tries2, + skipped + skipped2) + self.failures += failures + self.tries += tries + self.skipped += skipped __LINECACHE_FILENAME_RE = re.compile(r'.+)' @@ -1519,9 +1534,7 @@ def out(s): def summarize(self, verbose=None): """ Print a summary of all the test cases that have been run by - this DocTestRunner, and return a tuple `(f, t)`, where `f` is - the total number of failed examples, and `t` is the total - number of tried examples. + this DocTestRunner, and return a TestResults. The optional `verbose` argument controls how detailed the summary is. If the verbosity is not specified, then the @@ -1532,59 +1545,61 @@ def summarize(self, verbose=None): notests = [] passed = [] failed = [] - totalt = totalf = 0 - for x in self._name2ft.items(): - name, (f, t) = x - assert f <= t - totalt += t - totalf += f - if t == 0: + total_tries = total_failures = total_skipped = 0 + for item in self._stats.items(): + name, (failures, tries, skipped) = item + assert failures <= tries + total_tries += tries + total_failures += failures + total_skipped += skipped + if tries == 0: notests.append(name) - elif f == 0: - passed.append( (name, t) ) + elif failures == 0: + passed.append((name, tries)) else: - failed.append(x) + failed.append(item) if verbose: if notests: - print(len(notests), "items had no tests:") + print(f"{len(notests)} items had no tests:") notests.sort() - for thing in notests: - print(" ", thing) + for name in notests: + print(f" {name}") if passed: - print(len(passed), "items passed all tests:") + print(f"{len(passed)} items passed all tests:") passed.sort() - for thing, count in passed: - print(" %3d tests in %s" % (count, thing)) + for name, count in passed: + print(f" {count:3d} tests in {name}") if failed: print(self.DIVIDER) - print(len(failed), "items had failures:") + print(f"{len(failed)} items had failures:") failed.sort() - for thing, (f, t) in failed: - print(" %3d of %3d in %s" % (f, t, thing)) + for name, (failures, tries, skipped) in failed: + print(f" {failures:3d} of {tries:3d} in {name}") if verbose: - print(totalt, "tests in", len(self._name2ft), "items.") - print(totalt - totalf, "passed and", totalf, "failed.") - if totalf: - print("***Test Failed***", totalf, "failures.") + print(f"{total_tries} tests in {len(self._stats)} items.") + print(f"{total_tries - total_failures} passed and {total_failures} failed.") + if total_failures: + msg = f"***Test Failed*** {total_failures} failures" + if total_skipped: + msg = f"{msg} and {total_skipped} skipped tests" + print(f"{msg}.") elif verbose: print("Test passed.") - return TestResults(totalf, totalt) + return TestResults(total_failures, total_tries, skipped=total_skipped) #///////////////////////////////////////////////////////////////// # Backward compatibility cruft to maintain doctest.master. #///////////////////////////////////////////////////////////////// def merge(self, other): - d = self._name2ft - for name, (f, t) in other._name2ft.items(): + d = self._stats + for name, (failures, tries, skipped) in other._stats.items(): if name in d: - # Don't print here by default, since doing - # so breaks some of the buildbots - #print("*** DocTestRunner.merge: '" + name + "' in both" \ - # " testers; summing outcomes.") - f2, t2 = d[name] - f = f + f2 - t = t + t2 - d[name] = f, t + failures2, tries2, skipped2 = d[name] + failures = failures + failures2 + tries = tries + tries2 + skipped = skipped + skipped2 + d[name] = (failures, tries, skipped) + class OutputChecker: """ @@ -1984,7 +1999,8 @@ class doctest.Tester, then merges the results into (or creates) else: master.merge(runner) - return TestResults(runner.failures, runner.tries) + return TestResults(runner.failures, runner.tries, skipped=runner.skipped) + def testfile(filename, module_relative=True, name=None, package=None, globs=None, verbose=None, report=True, optionflags=0, @@ -2107,7 +2123,8 @@ class doctest.Tester, then merges the results into (or creates) else: master.merge(runner) - return TestResults(runner.failures, runner.tries) + return TestResults(runner.failures, runner.tries, skipped=runner.skipped) + def run_docstring_examples(f, globs, verbose=False, name="NoName", compileflags=None, optionflags=0): diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index c364e81a6b1495..9cc460c8b913f6 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -748,6 +748,38 @@ def non_Python_modules(): r""" """ +class TestDocTest(unittest.TestCase): + + def test_run(self): + test = ''' + >>> 1 + 1 + 11 + >>> 2 + 3 # doctest: +SKIP + "23" + >>> 5 + 7 + 57 + ''' + + def myfunc(): + pass + myfunc.__doc__ = test + + # test DocTestFinder.run() + test = doctest.DocTestFinder().find(myfunc)[0] + with support.captured_stdout(): + with support.captured_stderr(): + results = doctest.DocTestRunner(verbose=False).run(test) + + # test TestResults + self.assertIsInstance(results, doctest.TestResults) + self.assertEqual(results.failed, 2) + self.assertEqual(results.attempted, 3) + self.assertEqual(results.skipped, 1) + self.assertEqual(tuple(results), (2, 3)) + x, y = results + self.assertEqual((x, y), (2, 3)) + + class TestDocTestFinder(unittest.TestCase): def test_issue35753(self): diff --git a/Misc/NEWS.d/next/Tests/2023-09-02-05-13-38.gh-issue-108794.tGHXBt.rst b/Misc/NEWS.d/next/Tests/2023-09-02-05-13-38.gh-issue-108794.tGHXBt.rst new file mode 100644 index 00000000000000..aa339292a729b8 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-02-05-13-38.gh-issue-108794.tGHXBt.rst @@ -0,0 +1,3 @@ +:meth:`doctest.DocTestRunner.run` method now counts the number of skipped +tests. Add :attr:`doctest.TestResults.skipped` attribute. Patch by Victor +Stinner.