Skip to content

Commit

Permalink
Fixed #32591 -- Made DiscoverRunner order _FailedTest objects first.
Browse files Browse the repository at this point in the history
Failures detected when loading tests are ordered before all of the
above for quicker feedback. This includes things like test modules that
couldn't be found or that couldn't be loaded due to syntax errors.
  • Loading branch information
cjerdonek authored and felixxm committed Mar 26, 2021
1 parent cecdec9 commit dcb06c2
Show file tree
Hide file tree
Showing 3 changed files with 26 additions and 1 deletion.
6 changes: 5 additions & 1 deletion django/test/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -635,7 +635,11 @@ def build_suite(self, test_labels=None, extra_tests=None, **kwargs):
print('Excluding test tag(s): %s.' % ', '.join(sorted(self.exclude_tags)))
all_tests = filter_tests_by_tags(all_tests, self.tags, self.exclude_tags)

all_tests = reorder_tests(all_tests, self.reorder_by, self.reverse)
# Put the failures detected at load time first for quicker feedback.
# _FailedTest objects include things like test modules that couldn't be
# found or that couldn't be loaded due to syntax errors.
test_types = (unittest.loader._FailedTest, *self.reorder_by)
all_tests = reorder_tests(all_tests, test_types, self.reverse)
suite = self.test_suite(all_tests)

if self.parallel > 1:
Expand Down
11 changes: 11 additions & 0 deletions docs/topics/testing/overview.txt
Original file line number Diff line number Diff line change
Expand Up @@ -229,10 +229,21 @@ the Django test runner reorders tests in the following way:
database by a given :class:`~django.test.TransactionTestCase` test, they
must be updated to be able to run independently.

.. note::

Failures detected when loading tests are ordered before all of the above
for quicker feedback. This includes things like test modules that couldn't
be found or that couldn't be loaded due to syntax errors.

You may reverse the execution order inside groups using the :option:`test
--reverse` option. This can help with ensuring your tests are independent from
each other.

.. versionchanged:: 4.0

In older versions, failures detected when loading tests were not ordered
first.

.. _test-case-serialized-rollback:

Rollback emulation
Expand Down
10 changes: 10 additions & 0 deletions tests/test_runner/test_discover_runner.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import os
import unittest.loader
from argparse import ArgumentParser
from contextlib import contextmanager
from unittest import TestSuite, TextTestRunner, defaultTestLoader, mock
Expand Down Expand Up @@ -209,6 +210,15 @@ def test_reverse(self):
self.assertIn('test_2', suite[9].id(),
msg="Methods of unittest cases should be reversed.")

def test_build_suite_failed_tests_first(self):
# The "doesnotexist" label results in a _FailedTest instance.
suite = DiscoverRunner().build_suite(
test_labels=['test_runner_apps.sample', 'doesnotexist'],
)
tests = list(suite)
self.assertIsInstance(tests[0], unittest.loader._FailedTest)
self.assertNotIsInstance(tests[-1], unittest.loader._FailedTest)

def test_overridable_get_test_runner_kwargs(self):
self.assertIsInstance(DiscoverRunner().get_test_runner_kwargs(), dict)

Expand Down

0 comments on commit dcb06c2

Please sign in to comment.