Terminate the test loop if shouldfail or shouldstop is set (#1026)

Author: bbrown1867

changelog/1024.bugfix

@@ -0,0 +1,2 @@
+Added proper handling of ``shouldstop`` (such as set by ``--max-fail``) and ``shouldfail`` conditions in workers.
+Previously, a worker might have continued executing further tests before the controller could terminate the session.

src/xdist/dsession.py

@@ -182,9 +182,17 @@ def worker_workerfinished(self, node):
             self.shouldstop = f"{node} received keyboard-interrupt"
             self.worker_errordown(node, "keyboard-interrupt")
             return
-        if node in self.sched.nodes:
-            crashitem = self.sched.remove_node(node)
-            assert not crashitem, (crashitem, node)
+        shouldfail = node.workeroutput["shouldfail"]
+        shouldstop = node.workeroutput["shouldstop"]
+        for shouldx in [shouldfail, shouldstop]:
+            if shouldx:
+                if not self.shouldstop:
+                    self.shouldstop = shouldx
+                break
+        else:
+            if node in self.sched.nodes:
+                crashitem = self.sched.remove_node(node)
+                assert not crashitem, (crashitem, node)
         self._active_nodes.remove(node)
 
     def worker_internal_error(self, node, formatted_error):

src/xdist/remote.py

@@ -103,6 +103,8 @@ def pytest_sessionstart(self, session):
     def pytest_sessionfinish(self, exitstatus):
         # in pytest 5.0+, exitstatus is an IntEnum object
         self.config.workeroutput["exitstatus"] = int(exitstatus)
+        self.config.workeroutput["shouldfail"] = self.session.shouldfail
+        self.config.workeroutput["shouldstop"] = self.session.shouldstop
         yield
         self.sendevent("workerfinished", workeroutput=self.config.workeroutput)
 
@@ -155,6 +157,8 @@ def pytest_runtestloop(self, session):
         self.nextitem_index = self._get_next_item_index()
         while self.nextitem_index is not self.SHUTDOWN_MARK:
             self.run_one_test()
+            if session.shouldfail or session.shouldstop:
+                break
         return True
 
     def run_one_test(self):

testing/acceptance_test.py

@@ -109,12 +109,12 @@ def test_skip():
         )
         assert result.ret == 1
 
-    def test_exitfail_waits_for_workers_to_finish(
+    def test_exitfirst_waits_for_workers_to_finish(
         self, pytester: pytest.Pytester
     ) -> None:
         """The DSession waits for workers before exiting early on failure.
 
-        When -x/--exitfail is set, the DSession wait for the workers to finish
+        When -x/--exitfirst is set, the DSession wait for all workers to finish
         before raising an Interrupt exception. This prevents reports from the
         faiing test and other tests from being discarded.
         """
@@ -138,15 +138,14 @@ def test_fail6():
                 time.sleep(0.3)
         """
         )
+        # Two workers are used
         result = pytester.runpytest(p1, "-x", "-rA", "-v", "-n2")
         assert result.ret == 2
-        result.stdout.re_match_lines([".*Interrupted: stopping.*[12].*"])
-        m = re.search(r"== (\d+) failed, (\d+) passed in ", str(result.stdout))
-        assert m
-        n_failed, n_passed = (int(s) for s in m.groups())
-        assert 1 <= n_failed <= 2
-        assert 1 <= n_passed <= 3
-        assert (n_passed + n_failed) < 6
+        # DSession should stop when the first failure is reached. Two failures
+        # may actually occur, due to timing.
+        outcomes = result.parseoutcomes()
+        assert "failed" in outcomes, "Expected at least one failure"
+        assert 1 <= outcomes["failed"] <= 2, "Expected no more than 2 failures"
 
     def test_basetemp_in_subprocesses(self, pytester: pytest.Pytester) -> None:
         p1 = pytester.makepyfile(
@@ -1180,6 +1179,23 @@ def test_aaa1(crasher):
     assert "INTERNALERROR" not in result.stderr.str()
 
 
+def test_maxfail_causes_early_termination(pytester: pytest.Pytester) -> None:
+    """
+    Ensure subsequent tests on a worker aren't run when using --maxfail (#1024).
+    """
+    pytester.makepyfile(
+        """
+        def test1():
+            assert False
+
+        def test2():
+            pass
+    """
+    )
+    result = pytester.runpytest_subprocess("--maxfail=1", "-n 1")
+    result.assert_outcomes(failed=1)
+
+
 def test_internal_errors_propagate_to_controller(pytester: pytest.Pytester) -> None:
     pytester.makeconftest(
         """