Skip to content

Commit

Permalink
Fix timing issue in ctrl_mpexec unit test (DM-26974)
Browse files Browse the repository at this point in the history
EXpect timeouts also for tasks that are supposed to finish successfully.
Adjusted timings to make this situation less frequent.
  • Loading branch information
andy-slac committed Sep 29, 2020
1 parent 0620665 commit 038c61b
Showing 1 changed file with 10 additions and 3 deletions.
13 changes: 10 additions & 3 deletions tests/test_executors.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import time
from types import SimpleNamespace
import unittest
import warnings

from lsst.ctrl.mpexec import MPGraphExecutor, MPGraphExecutorError, MPTimeoutError, QuantumExecutor
from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId
Expand Down Expand Up @@ -141,7 +142,7 @@ class TaskMockSleep:

def runQuantum(self):
_LOG.debug("TaskMockSleep.runQuantum")
time.sleep(3.)
time.sleep(5.)


class TaskMockNoMP:
Expand Down Expand Up @@ -251,10 +252,16 @@ def test_mpexec_timeout(self):

# with failFast=False exception happens after last task finishes
qexec = QuantumExecutorMock(mp=True)
mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=False)
mpexec = MPGraphExecutor(numProc=3, timeout=3, quantumExecutor=qexec, failFast=False)
with self.assertRaises(MPTimeoutError):
mpexec.execute(qgraph, butler=None)
self.assertCountEqual(qexec.getDataIds("detector"), [0, 2])
# We expect two tasks (0 and 2) to finish successfully and one task to
# timeout. Unfortunately on busy CPU there is no guarantee that tasks
# finish on time, so expect more timeouts and issue a warning.
detectorIds = set(qexec.getDataIds("detector"))
self.assertLess(len(detectorIds), 3)
if detectorIds != {0, 2}:
warnings.warn(f"Possibly timed out tasks, expected [0, 2], received {detectorIds}")

def test_mpexec_failure(self):
"""Failure in one task should not stop other tasks"""
Expand Down

0 comments on commit 038c61b

Please sign in to comment.