Merge 98da27d into 0463f5a

WDL/runtime/backend/cli_subprocess.py

@@ -129,6 +129,7 @@ def _run(self, logger: logging.Logger, terminating: Callable[[], bool], command:
     def cli_name(self) -> str:
         pass
 
+    @property
     def cli_exe(self) -> List[str]:
         return [self.cli_name]
 

WDL/runtime/backend/docker_swarm.py

@@ -210,6 +210,9 @@ def _run(self, logger: logging.Logger, terminating: Callable[[], bool], command:
                 "container_labels": {"miniwdl_run_id": self.run_id},
                 "env": [f"{k}={v}" for (k, v) in self.runtime_values.get("env", {}).items()],
             }
+            if self.runtime_values.get("privileged", False) is True:
+                logger.warning("runtime.privileged enabled (security & portability warning)")
+                kwargs["cap_add"] = ["ALL"]
             kwargs.update(self.create_service_kwargs or {})
             logger.debug(_("docker create service kwargs", **kwargs))
             svc = client.services.create(image_tag, **kwargs)

WDL/runtime/backend/podman.py

@@ -85,6 +85,10 @@ def _run_invocation(self, logger: logging.Logger, cleanup: ExitStack, image: str
                 )
             ans += ["--user", f"{os.geteuid()}:{os.getegid()}"]
 
+        if self.runtime_values.get("privileged", False) is True:
+            logger.warning("runtime.privileged enabled (security & portability warning)")
+            ans.append("--privileged")
+
         mounts = self.prepare_mounts()
         logger.info(
             _(

WDL/runtime/backend/singularity.py

@@ -68,6 +68,9 @@ def _run_invocation(self, logger: logging.Logger, cleanup: ExitStack, image: str
             "--pwd",
             os.path.join(self.container_dir, "work"),
         ]
+        if self.runtime_values.get("privileged", False) is True:
+            logger.warning("runtime.privileged enabled (security & portability warning)")
+            ans += ["--add-caps", "all"]
         ans += self.cfg.get_list("singularity", "run_options")
 
         mounts = self.prepare_mounts()

WDL/runtime/config.py

@@ -322,6 +322,8 @@ def _parse_dict(v: str) -> Dict[str, Any]:
 
 
 def _parse_list(v: str) -> List[Any]:
+    if not v.startswith("["):
+        return [v]
     ans = json.loads(v)
     assert isinstance(ans, list)
     return ans

WDL/runtime/config_templates/default.cfg

@@ -110,6 +110,9 @@ placeholder_regex = (.|\n)*
 # WDL engines and/or compute platforms. Explicit WDL task inputs are usually better, except for a
 # few cases like auth tokens for platform-specific tasks.
 env = {}
+# If true, recognize `privileged: true` in task runtime sections and add restricted capabilities to
+# respective containers. Not recommended, for security & portability reasons. (New in v1.4.2)
+allow_privileged = false
 
 
 [download_cache]

WDL/runtime/task.py

@@ -458,6 +458,8 @@ def _eval_task_runtime(
             runtime_values[key] = Value.String(v)
         elif isinstance(v, int):
             runtime_values[key] = Value.Int(v)
+        elif isinstance(v, bool):
+            runtime_values[key] = Value.Boolean(v)
         else:
             raise Error.InputError(f"invalid default runtime setting {key} = {v}")
     for key, expr in task.runtime.items():  # evaluate expressions in source code
@@ -483,6 +485,18 @@ def _eval_task_runtime(
             docker_value = docker_value.value[0]
         ans["docker"] = docker_value.coerce(Type.String()).value
 
+    if (
+        isinstance(runtime_values.get("privileged", None), Value.Boolean)
+        and runtime_values["privileged"].value is True
+    ):
+        if cfg.get_bool("task_runtime", "allow_privileged"):
+            ans["privileged"] = True
+        else:
+            logger.warning(
+                "runtime.privileged ignored; to enable, set configuration"
+                " [task_runtime] allow_privileged = true (security+portability warning)"
+            )
+
     host_limits = container.__class__.detect_resource_limits(cfg, logger)
     if "cpu" in runtime_values:
         cpu_value = runtime_values["cpu"].coerce(Type.Int()).value

docs/runner_reference.md

@@ -45,6 +45,8 @@ The default local scheduler observes these task `runtime {}` attributes:
   * Automatically rounds down to all host memory, if less
   * The memory reservation informs scheduling, but isn't an enforced limit unless the configuration option `[task_runtime] memory_limit_multiplier` is set
 * `maxRetries` (Int): retry failing tasks up to this many additional attempts (after the first)
+* `returnCodes` (Int|Array[Int]|"*"): consider the given non-zero exit code(s) to indicate command success
+* `privileged` (Boolean): if true, *and* configuration option `[task_runtime] allow_privileged = true`, then run task containers with privileged capabilities. (Not recommended, for security & portability reasons.)
 
 ## File & Directory URI downloads
 

tests/test_4taskrun.py

@@ -1152,6 +1152,30 @@ def my_plugin(cfg, logger, task, run_id, run_dir, **recv):
         except WDL.runtime.error.CommandFailed as exn:
             self.assertEqual(exn.exit_status, 42)
 
+    def test_runtime_privileged(self):
+        txt = R"""
+        version 1.0
+        task xxx {
+            input {
+                Boolean privileged
+            }
+            command {
+                dmesg > /dev/null
+            }
+            output {
+            }
+            runtime {
+                privileged: privileged
+            }
+        }
+        """
+        self._test_task(txt, {"privileged": False}, expected_exception=WDL.runtime.CommandFailed)
+        self._test_task(txt, {"privileged": True}, expected_exception=WDL.runtime.CommandFailed)
+        cfg = WDL.runtime.config.Loader(logging.getLogger(self.id()), [])
+        cfg.override({"task_runtime": {"allow_privileged": True}})
+        self._test_task(txt, {"privileged": False}, cfg=cfg, expected_exception=WDL.runtime.CommandFailed)
+        self._test_task(txt, {"privileged": True}, cfg=cfg)
+
 
 class TestConfigLoader(unittest.TestCase):
     @classmethod