diff --git a/reframe/frontend/executors/policies.py b/reframe/frontend/executors/policies.py index 9feafbcf1b..deec7b9fe1 100644 --- a/reframe/frontend/executors/policies.py +++ b/reframe/frontend/executors/policies.py @@ -290,7 +290,9 @@ def runcase(self, case): self.printer.status('HOLD', task.check.info(), just='right') except TaskExit: if not task.failed: - self._reschedule(task) + with contextlib.suppress(TaskExit): + self._reschedule(task) + return except ABORT_REASONS as e: if not task.failed: @@ -409,7 +411,8 @@ def exit(self): time.sleep(t) except TaskExit: - self._reschedule_all() + with contextlib.suppress(TaskExit): + self._reschedule_all() except ABORT_REASONS as e: self._failall(e) raise diff --git a/unittests/resources/checks/frontend_checks.py b/unittests/resources/checks/frontend_checks.py index 1e7b77d2c2..3922dd0053 100644 --- a/unittests/resources/checks/frontend_checks.py +++ b/unittests/resources/checks/frontend_checks.py @@ -220,3 +220,13 @@ def run(self): super().run() time.sleep(0.5) os.kill(os.getpid(), signal.SIGTERM) + + +class CompileFailureCheck(rfm.RegressionTest): + def __init__(self): + self.valid_systems = ['*'] + self.valid_prog_environs = ['*'] + self.sanity_patterns = sn.assert_found(r'hello', self.stdout) + self.sourcesdir = None + self.sourcepath = 'x.c' + self.prebuild_cmd = ['echo foo > x.c'] diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 9093183a3f..770ca21016 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -21,6 +21,7 @@ from unittests.resources.checks.frontend_checks import ( BadSetupCheck, BadSetupCheckEarly, + CompileFailureCheck, KeyboardInterruptCheck, RetriesCheck, SelfKillCheck, @@ -557,8 +558,8 @@ def test_kbd_interrupt_in_setup_with_limited_concurrency( assert_interrupted_run(runner) -def test_poll_fails_in_main_loop(async_runner, make_cases, - make_async_exec_ctx): +def test_poll_fails_main_loop(async_runner, make_cases, + make_async_exec_ctx): ctx = make_async_exec_ctx(1) next(ctx) @@ -573,8 +574,8 @@ def test_poll_fails_in_main_loop(async_runner, make_cases, assert num_checks == len(stats.failures()) -def test_poll_fails_in_busy_loop(async_runner, make_cases, - make_async_exec_ctx): +def test_poll_fails_busy_loop(async_runner, make_cases, + make_async_exec_ctx): ctx = make_async_exec_ctx(1) next(ctx) @@ -587,3 +588,34 @@ def test_poll_fails_in_busy_loop(async_runner, make_cases, assert num_checks == stats.num_cases() assert_runall(runner) assert num_checks == len(stats.failures()) + + +def test_compile_fail_reschedule_main_loop(async_runner, make_cases, + make_async_exec_ctx): + ctx = make_async_exec_ctx(1) + next(ctx) + + runner, _ = async_runner + num_checks = 2 + runner.runall(make_cases([SleepCheckPollFail(.1), CompileFailureCheck()])) + + stats = runner.stats + assert num_checks == stats.num_cases() + assert_runall(runner) + assert num_checks == len(stats.failures()) + + +def test_compile_fail_reschedule_busy_loop(async_runner, make_cases, + make_async_exec_ctx): + ctx = make_async_exec_ctx(1) + next(ctx) + + runner, _ = async_runner + num_checks = 2 + runner.runall( + make_cases([SleepCheckPollFailLate(1.5), CompileFailureCheck()]) + ) + stats = runner.stats + assert num_checks == stats.num_cases() + assert_runall(runner) + assert num_checks == len(stats.failures())