set env vars in {podman,singularity,udocker}

chanzuckerberg · Nov 25, 2022 · 2480420 · 2480420
1 parent e5b04f4
commit 2480420
Show file tree

Hide file tree

Showing 6 changed files with 88 additions and 5 deletions.
diff --git a/WDL/runtime/backend/cli_subprocess.py b/WDL/runtime/backend/cli_subprocess.py
@@ -1,5 +1,6 @@
 import os
 import time
+import shlex
 import psutil
 import logging
 import threading
@@ -81,8 +82,13 @@ def _run(self, logger: logging.Logger, terminating: Callable[[], bool], command:
                 )
             )
 
-            # prepare command
+            # prepare command & environment
+            # we set the environment variables at the beginning of the command script because:
+            # 1) --env is subject to command line length limitations
+            # 2) --env-file isn't implemented consistently wrt quoting, escaping, etc.
             with open(os.path.join(self.host_dir, "command"), "w") as outfile:
+                for k, v in self.runtime_values.get("env", {}).items():
+                    outfile.write(f"export {k}={shlex.quote(v)}\n")
                 outfile.write(command)
 
             # start subprocess

diff --git a/tests/multi_line_strings.t b/tests/multi_line_strings.t
@@ -11,7 +11,7 @@ source tests/bash-tap/bash-tap-bootstrap
 export PYTHONPATH="$SOURCE_DIR:$PYTHONPATH"
 miniwdl="python3 -m WDL"
 
-plan tests 1
+plan tests 2
 
 if [[ -z $TMPDIR ]]; then
     TMPDIR=/tmp
@@ -22,3 +22,6 @@ cd $DN
 
 $miniwdl run "$SOURCE_DIR/tests/multi_line_strings.wdl" --verbose
 is $? "0"
+
+$miniwdl run $SOURCE_DIR/tests/task_env_inputs.wdl --dir test_env
+is "$?" "0" "env input escaping"
diff --git a/tests/podman.t b/tests/podman.t
@@ -19,13 +19,16 @@ DN=$(realpath "$DN")
 cd $DN
 echo "$DN"
 
-plan tests 3
+plan tests 4
 
 export MINIWDL__SCHEDULER__CONTAINER_BACKEND=podman
 
 $miniwdl run_self_test --dir "$DN"
 is "$?" "0" "run_self_test"
 
+$miniwdl run $SOURCE_DIR/tests/task_env_inputs.wdl --dir "$DN"
+is "$?" "0" "env input escaping"
+
 git clone --depth=1 https://github.com/broadinstitute/viral-pipelines.git
 cd viral-pipelines
 

diff --git a/tests/singularity.t b/tests/singularity.t
@@ -19,7 +19,7 @@ DN=$(realpath "$DN")
 cd $DN
 echo "$DN"
 
-plan tests 6
+plan tests 7
 
 export MINIWDL__SCHEDULER__CONTAINER_BACKEND=singularity
 
@@ -38,6 +38,9 @@ is "$?" "0" "run_self_test with image cache"
 grep 'SIF found in image cache directory' $(find "$DN/use_cache" -name workflow.log)
 is "$?" "0" "singularity image used from cache"
 
+$miniwdl run $SOURCE_DIR/tests/task_env_inputs.wdl --dir "$DN"
+is "$?" "0" "env input escaping"
+
 git clone --depth=1 https://github.com/broadinstitute/viral-pipelines.git
 cd viral-pipelines
 

diff --git a/tests/task_env_inputs.wdl b/tests/task_env_inputs.wdl
@@ -0,0 +1,65 @@
+version development
+
+# This WDL exercises proper escaping of environment variables passed to task commands, especially
+# for different container runtimes (docker, podman, singularity, udocker).
+
+struct Person {
+    String name
+    Int age
+}
+
+workflow w {
+    String s = <<<
+            Let's go!
+            # not a comment
+            $pwd\t${pwd}\\\\
+            CNN is working frantically to find their "source."
+        >>>
+    call t {
+        input:
+        s, p = Person { name: 'Alyssa', age: 42 }
+    }
+
+    String p_expected = '{"name": "Alyssa", "age": 42}'
+    scatter (pair in [(t.s_out, s), (t.p_out, p_expected)]) {
+        if (pair.left != pair.right) {
+            call fail { input: lhs = pair.left, rhs = pair.right}
+        }
+    }
+
+    output {
+        String s_out = t.s_out
+        String p_out = t.p_out
+    }
+}
+
+task t {
+    input {
+        env String s
+        env Person p
+    }
+
+    command <<<
+        echo "$s" > s
+        echo "$p" > p
+    >>>
+
+    output {
+        String s_out = read_string("s")
+        String p_out = read_string("p")
+    }
+}
+
+task fail {
+    input {
+        String lhs
+        String rhs
+    }
+
+    command {
+        >&2 echo "$(cat ~{write_json(lhs)}) != $(cat ~{write_json(rhs)})"
+        exit 1
+    }
+
+    output {}
+}
diff --git a/tests/udocker.t b/tests/udocker.t
@@ -19,13 +19,16 @@ DN=$(realpath "$DN")
 cd $DN
 echo "$DN"
 
-plan tests 3
+plan tests 4
 
 export MINIWDL__SCHEDULER__CONTAINER_BACKEND=udocker
 
 $miniwdl run_self_test --dir "$DN"
 is "$?" "0" "run_self_test"
 
+$miniwdl run $SOURCE_DIR/tests/task_env_inputs.wdl --dir "$DN"
+is "$?" "0" "env input escaping"
+
 git clone --depth=1 https://github.com/broadinstitute/viral-pipelines.git
 cd viral-pipelines