Merge pull request #2900 from broadinstitute/cjl_globs

Mr Globtastic's Globtacular Globbing Globs

backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala

@@ -36,7 +36,7 @@ trait GlobFunctions extends IoFunctionSet {
   /**
     * Returns a list of path from the glob.
     *
-    * The paths are currently read from a list file based on the pattern, and the path parameter is not used.
+    * The paths are read from a list file based on the pattern.
     *
     * @param pattern The pattern of the glob. This is the same "glob" passed to globPath().
     * @return The paths that match the pattern.

centaur/src/main/resources/standardTestCases/dontglobinputs.test

@@ -1,4 +1,3 @@
-ignore: true
 name: dontglobinputs
 testFormat: workflowsuccess
 

centaur/src/main/resources/standardTestCases/globbingBehavior.test

@@ -1,4 +1,3 @@
-ignore: true
 name: globbingBehavior
 testFormat: workflowsuccess
 

centaur/src/main/resources/standardTestCases/globbingscatter.test

@@ -1,7 +1,12 @@
-ignore: true
 name: globbingscatter
 testFormat: workflowsuccess
 
 files {
   wdl: globbingscatter/globbingscatter.wdl
 }
+
+metadata {
+  workflowName: globbingscatter
+  "outputs.globbingscatter.combiner1_result": "globbing is my number 1 best hobby\nglobbing is my number 2 best hobby\nglobbing is my number 3 best hobby\nglobbing is my number 4 best hobby\nglobbing is my number 5 best hobby"
+  "outputs.globbingscatter.combiner2_result": "globbing is my number 1 best hobby\nglobbing is my number 2 best hobby\nglobbing is my number 3 best hobby\nglobbing is my number 4 best hobby\nglobbing is my number 5 best hobby"
+}

centaur/src/main/resources/standardTestCases/globbingscatter/globbingscatter.wdl

@@ -44,6 +44,7 @@ workflow globbingscatter {
   }
   call combiner as combiner2 { input: in_file=catter2.result }
   output {
-     combiner2.result
+     String combiner1_result = combiner1.result
+     String combiner2_result = combiner2.result
   }
 }

centaur/src/main/resources/standardTestCases/lots_of_inputs.test

@@ -1,8 +1,17 @@
-ignore: true
+# This test makes sure that:
+# - 400 output files are all found and collected by the glob() method
+# - 400 input files to a task doesn't make anything explode inappropriately
 name: lots_of_inputs
 testFormat: workflowsuccess
 tags: [ big_metadata ]
 
 files {
   wdl: lots_of_inputs/lots_of_inputs.wdl
 }
+
+metadata {
+    workflowName: lots_of_inputs
+    status: Succeeded
+    "outputs.lots_of_inputs.out_count": "400"
+    "outputs.lots_of_inputs.nothing_out": "no-op"
+}

centaur/src/main/resources/standardTestCases/lots_of_inputs/lots_of_inputs.wdl

@@ -32,6 +32,11 @@ task make_array {
 }
 
 workflow lots_of_inputs {
-  call make_array {input: n=400}
-  call do_nothing {input: f=make_array.a}
+  call make_array { input: n = 400 }
+  call do_nothing { input: f = make_array.a }
+
+  output {
+    Int out_count = length(make_array.a)
+    String nothing_out = do_nothing.o
+  }
 }

centaur/src/main/resources/standardTestCases/space.test

@@ -1,4 +1,3 @@
-ignore: true
 name: space
 testFormat: workflowsuccess
 
@@ -9,4 +8,10 @@ files {
 metadata {
     workflowName: space
     status: Succeeded
+    "outputs.space.o1": "Hello world"
+    "outputs.space.o2": "Hello world"
+    "outputs.space.o3.0": "Hello world"
+    "outputs.space.o3.1": "Hello world"
+    "outputs.space.o3.2": "Hello world"
+    "outputs.space.o3.3": "Hello world"
 }

centaur/src/main/resources/standardTestCases/space/space.wdl

@@ -20,16 +20,16 @@ task goodbye {
     Array[File] files
     Array[File] files2
     command {
-        echo "Goodbye ${f}"
-        echo "Goodbye ${f2}"
-        echo "Goodbye ${sep = " " files}"
-        echo "Goodbye ${sep = " " files2}"
+        cat "${f}"
+        cat "${f2}"
+        cat "${sep = " " files}"
+        cat "${sep = " " files2}"
     }
     runtime {
             docker: "ubuntu:latest"
     }
     output {
-        String out = read_string(stdout())
+        Array[String] out = read_lines(stdout())
     }
 }
 
@@ -48,6 +48,6 @@ workflow space {
     output {
         String o1 = s1
         String o2 = s2
-        String o3 = goodbye.out
+        Array[String] o3 = goodbye.out
     }
 }

wdl/src/main/scala/wdl/WdlExpression.scala

@@ -227,7 +227,7 @@ final case class WdlWomExpression(wdlExpression: WdlExpression, from: Option[Sco
 
       override def stderr(params: Seq[Try[WomValue]]): Try[WomFile] = ioFunctionSet.stderr(params)
 
-      override def glob(path: String, pattern: String): Seq[String] = ioFunctionSet.glob(pattern)
+      override def globHelper(pattern: String): Seq[String] = ioFunctionSet.glob(pattern)
 
       override def size(params: Seq[Try[WomValue]]): Try[WomFloat] = ioFunctionSet.size(params)
     }

wdl/src/main/scala/wdl/expression/WdlFunctions.scala

@@ -28,9 +28,4 @@ trait WdlFunctions[T] {
     * Path where to write files created by standard functions (write_*).
    */
   def tempFilePath: String = throw new NotImplementedError("write_* functions are not supported by this implementation")
-
-  /**
-    * Path where to glob from when the glob standard function evaluates.
-   */
-  def globPath(glob: String): String = throw new NotImplementedError("glob function is not supported by this implementation")
 }

wdl/src/main/scala/wdl/expression/WdlStandardLibraryFunctions.scala

@@ -23,8 +23,6 @@ trait WdlStandardLibraryFunctions extends WdlFunctions[WomValue] {
 
   def stderr(params: Seq[Try[WomValue]]): Try[WomFile]
 
-  def glob(path: String, pattern: String): Seq[String]
-
   def size(params: Seq[Try[WomValue]]): Try[WomFloat]
 
   private def writeContent(baseName: String, content: String): Try[WomFile] = writeFile(s"${baseName}_${content.md5Sum}.tmp", content)
@@ -90,13 +88,15 @@ trait WdlStandardLibraryFunctions extends WdlFunctions[WomValue] {
     read_string(params) map { s => WomBoolean(java.lang.Boolean.parseBoolean(s.value.trim.toLowerCase)) }
   }
 
-  def glob(params: Seq[Try[WomValue]]): Try[WomArray] = {
+  def globHelper(pattern: String): Seq[String]
+
+  final def glob(params: Seq[Try[WomValue]]): Try[WomArray] = {
     for {
-      singleArgument <- extractSingleArgument("glob", params)
-      globVal = singleArgument.valueString
-      files = glob(globPath(globVal), globVal)
-      wdlFiles = files map { WomFile(_, isGlob = false) }
-    } yield WomArray(WomArrayType(WomFileType), wdlFiles)
+      pattern <- extractSingleArgument("glob", params)
+      womString <- WomStringType.coerceRawValue(pattern)
+      patternString = womString.valueString
+      filePaths <- Try(globHelper(patternString))
+    } yield WomArray(WomArrayType(WomFileType), filePaths.map(WomSingleFile))
   }
 
   def basename(params: Seq[Try[WomValue]]): Try[WomString] = {
@@ -309,7 +309,7 @@ object WdlStandardLibraryFunctions {
 
     override def stderr(params: Seq[Try[WomValue]]): Try[WomFile] = ioFunctionSet.stderr(params)
 
-    override def glob(path: String, pattern: String): Seq[String] = ioFunctionSet.glob(pattern)
+    override def globHelper(pattern: String): Seq[String] = ioFunctionSet.glob(pattern)
 
     override def size(params: Seq[Try[WomValue]]): Try[WomFloat] = ioFunctionSet.size(params)
   }
@@ -349,7 +349,7 @@ trait PureStandardLibraryFunctionsLike extends WdlStandardLibraryFunctions {
   override def size(params: Seq[Try[WomValue]]): Try[WomFloat] = Failure(new NotImplementedError(s"size not available in $className."))
   override def write_tsv(params: Seq[Try[WomValue]]): Try[WomFile] = Failure(new NotImplementedError(s"write_tsv not available in $className."))
   override def stdout(params: Seq[Try[WomValue]]): Try[WomFile] = Failure(new NotImplementedError(s"stdout not available in $className."))
-  override def glob(path: String, pattern: String): Seq[String] = throw new NotImplementedError(s"glob not available in $className.")
+  override def globHelper(pattern: String): Seq[String] = throw new NotImplementedError(s"glob not available in $className.")
   override def stderr(params: Seq[Try[WomValue]]): Try[WomFile] = Failure(new NotImplementedError(s"stderr not available in $className."))
 }
 
@@ -443,7 +443,7 @@ class WdlStandardLibraryFunctionsType extends WdlFunctions[WomType] {
 }
 
 case object NoFunctions extends WdlStandardLibraryFunctions {
-  override def glob(path: String, pattern: String): Seq[String] = throw new NotImplementedError()
+  override def globHelper(pattern: String): Seq[String] = throw new NotImplementedError()
   override def readFile(path: String): String = throw new NotImplementedError()
   override def writeFile(path: String, content: String): Try[WomFile] = throw new NotImplementedError()
   override def stdout(params: Seq[Try[WomValue]]): Try[WomFile] = Failure(new NotImplementedError())

wdl/src/test/scala/wdl/expression/ValueEvaluatorSpec.scala

@@ -53,7 +53,7 @@ class ValueEvaluatorSpec extends FlatSpec with Matchers {
   def identifierTypeLookup(name: String): WomType = identifierLookup(name).womType
 
   class TestValueFunctions extends WdlStandardLibraryFunctions {
-    override def glob(path: String, pattern: String): Seq[String] = throw new NotImplementedError()
+    override def globHelper(pattern: String): Seq[String] = throw new NotImplementedError()
 
     override def readFile(path: String): String = throw new NotImplementedError()
 

wdl/src/test/scala/wdl/expression/WdlStandardLibraryFunctionsSpec.scala

@@ -61,7 +61,7 @@ case object TestableFunctions extends WdlStandardLibraryFunctions {
   override def transpose(params: Seq[Try[WomValue]]): Try[WomArray] = throw new NotImplementedError
   override def write_tsv(params: Seq[Try[WomValue]]): Try[WomFile] = throw new NotImplementedError
   override def stdout(params: Seq[Try[WomValue]]): Try[WomFile] = throw new NotImplementedError
-  override def glob(path: String, pattern: String): Seq[String] = throw new NotImplementedError
+  override def globHelper(pattern: String): Seq[String] = throw new NotImplementedError
   override def stderr(params: Seq[Try[WomValue]]): Try[WomFile] = throw new NotImplementedError
 }