ensure glob() returns in-container paths

chanzuckerberg · Jul 13, 2019 · 7ea468b · 7ea468b
1 parent fe20a1d
commit 7ea468b
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 8 deletions.
diff --git a/WDL/Value.py b/WDL/Value.py
@@ -114,7 +114,7 @@ def __init__(self, value: str) -> None:
 
     def coerce(self, desired_type: Optional[Type.Base] = None) -> Base:
         ""
-        if isinstance(desired_type, Type.File):
+        if isinstance(desired_type, Type.File) and not isinstance(self, File):
             return File(self.value)
         return super().coerce(desired_type)
 

diff --git a/WDL/runtime/task.py b/WDL/runtime/task.py
@@ -456,7 +456,7 @@ def _eval_task_outputs(
     def map_files(v: Value.Base) -> Value.Base:
         if isinstance(v, Value.File):
             host_file = container.host_file(v.value)
-            logger.debug("File {} -> {}".format(v.value, host_file))
+            logger.debug("container output file %s -> host %s", v.value, host_file)
             v.value = host_file
         for ch in v.children:
             map_files(ch)
@@ -670,11 +670,17 @@ def _glob(pattern: Value.String, lib: OutputStdLib = self) -> Value.Array:
                 raise OutputError("glob() pattern must not use .. uplevels")
             if pat.startswith("./"):
                 pat = pat[2:]
+            # glob the host directory
             pat = os.path.join(lib.container.host_dir, "work", pat)
-            return Value.Array(
-                Type.Array(Type.File()),
-                [Value.String(fn) for fn in sorted(glob.glob(pat)) if os.path.isfile(fn)],
-            )
+            host_files = sorted(fn for fn in glob.glob(pat) if os.path.isfile(fn))
+            # convert the host filenames to in-container filenames
+            container_files = []
+            for hf in host_files:
+                dstrip = lib.container.host_dir
+                dstrip += "" if dstrip.endswith("/") else "/"
+                assert hf.startswith(dstrip)
+                container_files.append(os.path.join(lib.container.container_dir, hf[len(dstrip) :]))
+            return Value.Array(Type.Array(Type.File()), [Value.File(fn) for fn in container_files])
 
         self._override_static("glob", _glob)
 

diff --git a/tests/test_5stdlib.py b/tests/test_5stdlib.py
@@ -350,6 +350,8 @@ def test_glob(self):
                     glob("*/*"),
                     glob("bogus")
                 ]
+                File f1 = glob("stuff/foo")[0]
+                String s1 = read_string(f1)
             }
         }
         """)
@@ -363,9 +365,10 @@ def test_glob(self):
         self.assertEqual(len(outputs["globs"][2]), 5)
         self.assertTrue(outputs["globs"][2][4].endswith("/stuff/foo"))
         self.assertEqual(len(outputs["globs"][3]), 0)
-        for g in outputs["globs"]:
+        for g in outputs["globs"] + [[outputs["f1"]]]:
             for fn in g:
-                self.assertTrue(os.path.isfile(fn))
+                assert os.path.isfile(fn), fn
+        self.assertTrue(outputs["f1"].endswith("/stuff/foo"))
 
         self._test_task(R"""
         version 1.0
@@ -512,6 +515,7 @@ def test_write(self):
                 File o_json = json
                 Array[Array[String]] o_tsv = read_tsv(tsv)
                 Map[String,String] o_map = read_map(map)
+                File whynot = write_lines(["foo","bar","baz"])
             }
         }
         """)