Merge b1d4e2b into 6a33334

Makefile

@@ -4,7 +4,7 @@ PYTHON_PKG_BASE?=$(HOME)/.local
 test: check check_check
 	coverage run --include "WDL/*" --omit WDL/CLI.py -m unittest -v
 	coverage report -m
-	prove -v tests/check.t tests/cromwell.t tests/runner.t
+	prove -v tests/*.t
 
 # fail fast
 qtest:

WDL/runtime/task.py

@@ -6,9 +6,11 @@
 import copy
 import traceback
 import glob
+import signal
 from datetime import datetime
 from abc import ABC, abstractmethod
 from typing import Tuple, List, Dict, Optional, Callable, BinaryIO
+from types import FrameType
 from requests.exceptions import ReadTimeout
 import docker
 import WDL
@@ -117,16 +119,36 @@ def run(self, logger: logging.Logger, command: str) -> None:
 
         The container is torn down in any case, including SIGTERM/SIGHUP signal which is trapped.
         """
-        assert not self._running
+        # container-specific logic should be in _run(). this wrapper traps SIGTERM/SIGHUP
+        # and sets self._terminate
+
+        assert not (self._running or self._terminate)
         if command.strip():  # if the command is empty then don't bother with any of this
+            signals = [signal.SIGTERM, signal.SIGINT, signal.SIGHUP, signal.SIGPIPE, signal.SIGALRM]
+
+            def handle_signal(
+                signal: int,
+                frame: FrameType,
+                self: TaskContainer = self,
+                logger: logging.Logger = logger,
+            ) -> None:
+                logger.critical("received termination signal {}".format(signal))
+                self._terminate = True
+
+            restore_signal_handlers = dict(
+                (sig, signal.signal(sig, handle_signal)) for sig in signals
+            )
+
             self._running = True
-            # container-specific logic should be in _run(). this wrapper traps SIGTERM/SIGHUP
-            # and sets self._terminate
             try:
                 exit_status = self._run(logger, command)
             finally:
                 self._running = False
+                for sig, handler in restore_signal_handlers.items():
+                    signal.signal(sig, handler)
 
+            if self._terminate:
+                raise Terminated()
             if exit_status != 0:
                 raise CommandError("command exit status = " + str(exit_status))
 
@@ -256,7 +278,7 @@ def _run(self, logger: logging.Logger, command: str) -> int:
                         # workaround for docker-py not throwing the exception class
                         # it's supposed to
                         s_exn = str(exn)
-                        if "Read timed out" not in s_exn and "Timeout" not in s_exn:
+                        if "timed out" not in s_exn and "Timeout" not in s_exn:
                             raise
                 logger.info("container exit info = " + str(exit_info))
             except:
@@ -268,6 +290,7 @@ def _run(self, logger: logging.Logger, command: str) -> int:
                         container.remove(force=True)
                     except Exception as exn:
                         logger.error("failed to remove docker container: " + str(exn))
+                    logger.info("force-removed docker container")
                 raise
 
             # retrieve and check container exit status
@@ -350,7 +373,10 @@ def run_local_task(
     except Exception as exn:
         logger.debug(traceback.format_exc())
         wrapper = TaskFailure(task.name, task_id)
-        logger.error("{}: {}, {}".format(str(wrapper), exn.__class__.__name__, str(exn)))
+        msg = "{}: {}".format(str(wrapper), exn.__class__.__name__)
+        if str(exn):
+            msg += ", " + str(exn)
+        logger.error(msg)
         raise wrapper from exn
 
 

tests/runner.t

@@ -11,7 +11,7 @@ source tests/bash-tap/bash-tap-bootstrap
 export PYTHONPATH="$SOURCE_DIR:$PYTHONPATH"
 miniwdl="python3 -m WDL"
 
-plan tests 19
+plan tests 20
 
 DN=$(mktemp -d --tmpdir miniwdl_runner_tests_XXXXXX)
 cd $DN
@@ -75,3 +75,23 @@ is "$(basename $f1)" "fox" "task product fox"
 is "$(ls $f1)" "$f1" "task product fox file"
 is "$(ls taskrun/outputs/echo.out_f/2)" "fox" "task product fox link"
 
+cat << 'EOF' > sleep.wdl
+version 1.0
+task sleep {
+    input {
+        Int seconds
+    }
+
+    command {
+        sleep ~{seconds}
+    }
+}
+EOF
+
+t0=$(date +%s)
+$miniwdl run sleep.wdl seconds=10 & pid=$!
+sleep 3
+kill $pid
+wait $pid || true
+t1=$(date +%s)
+is "$(( t1 - t0 < 5 ))" "1" "task SIGTERM"

tests/test_4taskrun.py

@@ -3,6 +3,8 @@
 import tempfile
 import os
 import docker
+import signal
+import time
 from .context import WDL
 
 class TestTaskRunner(unittest.TestCase):
@@ -467,3 +469,17 @@ def test_topsort(self):
         """
         self.assertEqual(self._test_task(txt, {"x": 22})["yy"], 42)
         self.assertEqual(self._test_task(txt, {"x": 22, "y": 99})["yy"], 99)
+
+    def test_signal(self):
+        signal.alarm(3)
+        t0 = time.time()
+        self._test_task(R"""
+        version 1.0
+        task t {
+            command {
+                sleep 10
+            }
+        }
+        """, expected_exception=WDL.runtime.task.Terminated)
+        t1 = time.time()
+        self.assertLess(t1 - t0, 5)