Adding expressionLib support

centaur/src/main/resources/standardTestCases/expressionLib.test

@@ -0,0 +1,14 @@
+name: expression_lib_cwl
+cwlVersion: 1.0
+testFormat: workflowsuccess
+workflowType: CWL
+workflowTypeVersion: v1.0
+
+files {
+  wdl: expressionLib/expressionLib.cwl
+}
+
+metadata {
+  "submittedFiles.workflowType": CWL
+  "submittedFiles.workflowTypeVersion": v1.0
+}

centaur/src/main/resources/standardTestCases/expressionLib/expressionLib.cwl

@@ -0,0 +1,14 @@
+class: CommandLineTool
+cwlVersion: v1.0
+hints:
+  DockerRequirement:
+    dockerPull: "ubuntu:latest"
+requirements:
+  InlineJavascriptRequirement:
+    expressionLib:
+      - "function foo() { return 2; }"
+inputs: []
+outputs:
+  out: stdout
+arguments: [echo, $(foo())]
+stdout: whatever.txt

cwl/src/main/scala/cwl/ArgumentToCommandPart.scala

@@ -5,19 +5,22 @@ import shapeless._
 import wom.CommandPart
 
 object ArgumentToCommandPart extends Poly1 {
-  implicit def caseStringOrExpression: Case.Aux[StringOrExpression, CommandPart] = at {
+  type ExpressionToCommandPart = ExpressionLib => CommandPart
+  implicit def caseStringOrExpression: Case.Aux[StringOrExpression, ExpressionToCommandPart] = at {
     _.fold(this)
   }
 
-  implicit def caseExpression: Case.Aux[Expression, CommandPart] = at {
+  implicit def caseExpression: Case.Aux[Expression, ExpressionToCommandPart] = at {
     CwlExpressionCommandPart.apply
   }
 
-  implicit def caseString: Case.Aux[String, CommandPart] = at {
-    StringCommandPart.apply
+  implicit def caseString: Case.Aux[String, ExpressionToCommandPart] = at {
+    string =>
+      _ =>
+        StringCommandPart(string)
   }
 
-  implicit def caseCommandLineBinding: Case.Aux[ArgumentCommandLineBinding, CommandPart] = at {
+  implicit def caseCommandLineBinding: Case.Aux[ArgumentCommandLineBinding, ExpressionToCommandPart] = at {
     ArgumentCommandLineBindingCommandPart.apply
   }
 }

cwl/src/main/scala/cwl/CommandLineTool.scala

@@ -18,7 +18,7 @@ import cwl.command.ParentName
 import cwl.requirement.RequirementToAttributeMap
 import eu.timepit.refined.W
 import shapeless.syntax.singleton._
-import shapeless.{:+:, CNil, Coproduct, Poly1, Witness}
+import shapeless.{:+:, CNil, Coproduct, Inl, Poly1, Witness}
 import wom.callable.Callable.{InputDefinitionWithDefault, OutputDefinition, RequiredInputDefinition}
 import wom.callable.{Callable, CallableTaskDefinition}
 import wom.executable.Executable
@@ -63,7 +63,7 @@ case class CommandLineTool private(
 
   /** Builds an `Executable` directly from a `CommandLineTool` CWL with no parent workflow. */
   def womExecutable(validator: RequirementsValidator, inputFile: Option[String] = None): Checked[Executable] = {
-    val taskDefinition = buildTaskDefinition(validator)
+    val taskDefinition = buildTaskDefinition(validator, Vector.empty)
     CwlExecutableValidation.buildWomExecutable(taskDefinition, inputFile)
   }
 
@@ -88,8 +88,8 @@ case class CommandLineTool private(
     }
   }
 
-  private def processRequirement(requirement: Requirement): Map[String, WomExpression] = {
-    requirement.fold(RequirementToAttributeMap).apply(inputNames)
+  private def processRequirement(requirement: Requirement, expressionLib: ExpressionLib): Map[String, WomExpression] = {
+    requirement.fold(RequirementToAttributeMap).apply(inputNames, expressionLib)
   }
 
   /*
@@ -99,7 +99,7 @@ case class CommandLineTool private(
    * - Finally the inputs are folded one by one into a CommandPartsList
    * - arguments and inputs CommandParts are sorted according to their sort key
    */
-  private [cwl] def buildCommandTemplate(inputValues: WomEvaluatedCallInputs): ErrorOr[List[CommandPart]] = {
+  private [cwl] def buildCommandTemplate(expressionLib: ExpressionLib)(inputValues: WomEvaluatedCallInputs): ErrorOr[List[CommandPart]] = {
     import cats.instances.list._
     import cats.syntax.traverse._
 
@@ -111,7 +111,7 @@ case class CommandLineTool private(
         // zip the index because we need it in the sorting key
         .zipWithIndex.foldLeft(CommandPartsList.empty)({
         case (commandPartsList, (argument, index)) =>
-          val part = argument.fold(ArgumentToCommandPart)
+          val part = argument.fold(ArgumentToCommandPart).apply(expressionLib)
           // Get the position from the binding if there is one
           val position = argument.select[ArgumentCommandLineBinding].flatMap(_.position)
             .map(Coproduct[StringOrInt](_)).getOrElse(DefaultPosition)
@@ -138,8 +138,8 @@ case class CommandLineTool private(
               // See http://www.commonwl.org/v1.0/CommandLineTool.html#Input_binding
               lazy val initialKey = CommandBindingSortingKey.empty
                 .append(inputParameter.inputBinding, Coproduct[StringOrInt](parsedName))
-    
-              inputParameter.`type`.toList.flatMap(_.fold(MyriadInputTypeToSortedCommandParts).apply(inputParameter.inputBinding, value, initialKey.asNewKey)).validNel
+
+              inputParameter.`type`.toList.flatMap(_.fold(MyriadInputTypeToSortedCommandParts).apply(inputParameter.inputBinding, value, initialKey.asNewKey, expressionLib)).validNel
            case Some(Invalid(errors)) => Invalid(errors)
            case None => s"Could not find an input value for input $parsedName in ${inputValues.prettyString}".invalidNel
         }
@@ -150,16 +150,19 @@ case class CommandLineTool private(
     }
   }
 
-  def buildTaskDefinition(validator: RequirementsValidator): ErrorOr[CallableTaskDefinition] = {
-    validateRequirementsAndHints(validator) map { requirementsAndHints =>
+  def buildTaskDefinition(validator: RequirementsValidator, parentExpressionLib: ExpressionLib): ErrorOr[CallableTaskDefinition] = {
+    validateRequirementsAndHints(validator) map { requirementsAndHints: Seq[cwl.Requirement] =>
       val id = this.id
 
+      val expressionLib: ExpressionLib =
+        parentExpressionLib ++ inlineJavascriptRequirements(requirementsAndHints)
+
       // This is basically doing a `foldMap` but can't actually be a `foldMap` because:
       // - There is no monoid instance for `WomExpression`s.
       // - We want to fold from the right so the hints and requirements with the lowest precedence are processed first
       //   and later overridden if there are duplicate hints or requirements of the same type with higher precedence.
-      val finalAttributesMap = (requirementsAndHints ++ DefaultDockerRequirement).foldRight(Map.empty[String, WomExpression])({
-        case (requirement, attributesMap) => attributesMap ++ processRequirement(requirement)
+      val finalAttributesMap: Map[String, WomExpression] = (requirementsAndHints ++ DefaultDockerRequirement).foldRight(Map.empty[String, WomExpression])({
+        case (requirement, attributesMap) => attributesMap ++ processRequirement(requirement, expressionLib)
       })
 
       val runtimeAttributes: RuntimeAttributes = RuntimeAttributes(finalAttributesMap)
@@ -176,7 +179,7 @@ case class CommandLineTool private(
       val outputs: List[Callable.OutputDefinition] = this.outputs.map {
         case p @ CommandOutputParameter(cop_id, _, _, _, _, _, _, Some(tpe)) =>
           val womType = tpe.fold(MyriadOutputTypeToWomType)
-          OutputDefinition(FullyQualifiedName(cop_id).id, womType, CommandOutputParameterExpression(p, womType, inputNames))
+          OutputDefinition(FullyQualifiedName(cop_id).id, womType, CommandOutputParameterExpression(p, womType, inputNames, expressionLib))
         case other => throw new NotImplementedError(s"Command output parameters such as $other are not yet supported")
       }.toList
 
@@ -207,11 +210,11 @@ case class CommandLineTool private(
         requirements <- requirements.getOrElse(Array.empty[Requirement])
         initialWorkDirRequirement <- requirements.select[InitialWorkDirRequirement].toArray
         listing <- initialWorkDirRequirement.listings
-      } yield InitialWorkDirFileGeneratorExpression(listing)).toSet[WomExpression]
+      } yield InitialWorkDirFileGeneratorExpression(listing, expressionLib)).toSet[WomExpression]
 
       CallableTaskDefinition(
         taskName,
-        buildCommandTemplate,
+        buildCommandTemplate(expressionLib),
         runtimeAttributes,
         meta,
         parameterMeta,
@@ -295,6 +298,14 @@ object CommandLineTool {
   // Ordering for a CommandPartSortMapping: order by sorting key
   implicit val SortKeyAndCommandPartOrdering: Ordering[SortKeyAndCommandPart] = Ordering.by(_.sortingKey)
 
+  def inlineJavascriptRequirements(allRequirementsAndHints: Seq[Requirement]): Vector[String] = {
+    val inlineJavscriptRequirements: Seq[InlineJavascriptRequirement] = allRequirementsAndHints.toList.collect {
+      case Inl(ijr:InlineJavascriptRequirement) => ijr
+    }
+
+    inlineJavscriptRequirements.flatMap(_.expressionLib.toList.flatten).toVector
+  }
+
   def apply(inputs: Array[CommandInputParameter] = Array.empty,
             outputs: Array[CommandOutputParameter] = Array.empty,
             id: String,
@@ -430,17 +441,18 @@ object CommandLineTool {
   object CommandOutputParameter {
 
     def format(formatOption: Option[StringOrExpression],
-               parameterContext: ParameterContext): ErrorOr[Option[String]] = {
+               parameterContext: ParameterContext,
+               expressionLib: ExpressionLib): ErrorOr[Option[String]] = {
       formatOption.traverse[ErrorOr, String] {
-        format(_, parameterContext)
+        format(_, parameterContext, expressionLib)
       }
     }
 
-    def format(format: StringOrExpression, parameterContext: ParameterContext): ErrorOr[String] = {
-      format.fold(CommandLineTool.CommandOutputParameter.FormatPoly).apply(parameterContext)
+    def format(format: StringOrExpression, parameterContext: ParameterContext, expressionLib: ExpressionLib): ErrorOr[String] = {
+      format.fold(CommandLineTool.CommandOutputParameter.FormatPoly).apply(parameterContext, expressionLib)
     }
 
-    type FormatFunction = ParameterContext => ErrorOr[String]
+    type FormatFunction = (ParameterContext, ExpressionLib) => ErrorOr[String]
 
     object FormatPoly extends Poly1 {
       implicit def caseStringOrExpression: Case.Aux[StringOrExpression, FormatFunction] = {
@@ -452,26 +464,27 @@ object CommandLineTool {
       implicit def caseExpression: Case.Aux[Expression, FormatFunction] = {
         at {
           expression =>
-            parameterContext =>
-              val result: ErrorOr[WomValue] = ExpressionEvaluator.eval(expression, parameterContext)
+            (parameterContext, expressionLib) =>
+              val result: ErrorOr[WomValue] = ExpressionEvaluator.eval(expression, parameterContext, expressionLib)
               result flatMap {
                 case womString: WomString => womString.value.valid
                 case other => s"Not a valid file format: $other".invalidNel
               }
         }
       }
 
-      implicit def caseString: Case.Aux[String, FormatFunction] = at { string => _ => string.valid }
+      implicit def caseString: Case.Aux[String, FormatFunction] = at { string => (_,_) => string.valid }
     }
 
     /**
       * Returns the list of secondary files for the primary file.
       */
     def secondaryFiles(primaryWomFile: WomFile,
                        secondaryFilesOption: Option[SecondaryFiles],
-                       parameterContext: ParameterContext): ErrorOr[List[WomFile]] = {
+                       parameterContext: ParameterContext,
+                       expressionLib: ExpressionLib): ErrorOr[List[WomFile]] = {
       secondaryFilesOption
-        .map(secondaryFiles(primaryWomFile, _, parameterContext))
+        .map(secondaryFiles(primaryWomFile, _, parameterContext, expressionLib))
         .getOrElse(Nil.valid)
     }
 
@@ -480,13 +493,14 @@ object CommandLineTool {
       */
     def secondaryFiles(primaryWomFile: WomFile,
                        secondaryFiles: SecondaryFiles,
-                       parameterContext: ParameterContext): ErrorOr[List[WomFile]] = {
+                       parameterContext: ParameterContext,
+                       expressionLib: ExpressionLib): ErrorOr[List[WomFile]] = {
       secondaryFiles
         .fold(CommandLineTool.CommandOutputParameter.SecondaryFilesPoly)
-        .apply(primaryWomFile, parameterContext)
+        .apply(primaryWomFile, parameterContext, expressionLib)
     }
 
-    type SecondaryFilesFunction = (WomFile, ParameterContext) => ErrorOr[List[WomFile]]
+    type SecondaryFilesFunction = (WomFile, ParameterContext, ExpressionLib) => ErrorOr[List[WomFile]]
 
     object SecondaryFilesPoly extends Poly1 {
       implicit def caseStringOrExpression: Case.Aux[StringOrExpression, SecondaryFilesFunction] = {
@@ -498,25 +512,25 @@ object CommandLineTool {
       implicit def caseExpression: Case.Aux[Expression, SecondaryFilesFunction] = {
         at {
           expression =>
-            (primaryWomFile, parameterContext) =>
-              File.secondaryExpressionFiles(primaryWomFile, expression, parameterContext)
+            (primaryWomFile, parameterContext, expressionLib) =>
+              File.secondaryExpressionFiles(primaryWomFile, expression, parameterContext, expressionLib)
         }
       }
 
       implicit def caseString: Case.Aux[String, SecondaryFilesFunction] = {
         at {
           string =>
-            (primaryWomFile, _) =>
+            (primaryWomFile, _, _) =>
               File.secondaryStringFile(primaryWomFile, string).map(List(_))
         }
       }
 
       implicit def caseArray: Case.Aux[Array[StringOrExpression], SecondaryFilesFunction] = {
         at {
           array =>
-            (primaryWomFile, parameterContext) =>
+            (primaryWomFile, parameterContext, expressionLib) =>
               val functions: List[SecondaryFilesFunction] = array.toList.map(_.fold(this))
-              functions.flatTraverse(_ (primaryWomFile, parameterContext))
+              functions.flatTraverse(_ (primaryWomFile, parameterContext, expressionLib))
         }
       }
     }

cwl/src/main/scala/cwl/CommandOutputBinding.scala

@@ -27,7 +27,8 @@ object CommandOutputBinding {
   def getOutputWomFiles(inputValues: Map[String, WomValue],
                         outputWomType: WomType,
                         commandOutputBinding: CommandOutputBinding,
-                        secondaryFilesOption: Option[SecondaryFiles]): ErrorOr[Set[WomFile]] = {
+                        secondaryFilesOption: Option[SecondaryFiles],
+                        expressionLib: ExpressionLib): ErrorOr[Set[WomFile]] = {
     val parameterContext = ParameterContext(inputs = inputValues)
 
     /*
@@ -47,14 +48,14 @@ object CommandOutputBinding {
     }
 
     for {
-      primaryPaths <- GlobEvaluator.globs(commandOutputBinding.glob, parameterContext)
+      primaryPaths <- GlobEvaluator.globs(commandOutputBinding.glob, parameterContext, expressionLib)
       primaryWomFiles <- outputWomFlatType match {
         case WomGlobFileType => primaryPaths.map(WomGlobFile).valid
         case WomUnlistedDirectoryType => primaryPaths.map(WomUnlistedDirectory).valid
         case other => s"Program error: $other type was not expected".invalidNel
       }
       secondaryWomFiles <- primaryWomFiles.flatTraverse[ErrorOr, WomFile] {
-        CommandLineTool.CommandOutputParameter.secondaryFiles(_, secondaryFilesOption, parameterContext)
+        CommandLineTool.CommandOutputParameter.secondaryFiles(_, secondaryFilesOption, parameterContext, expressionLib)
       }
     } yield (primaryWomFiles ++ secondaryWomFiles).toSet
   }
@@ -75,7 +76,8 @@ object CommandOutputBinding {
                              outputWomType: WomType,
                              commandOutputBinding: CommandOutputBinding,
                              secondaryFilesCoproduct: Option[SecondaryFiles],
-                             formatCoproduct: Option[StringOrExpression]): ErrorOr[WomValue] = {
+                             formatCoproduct: Option[StringOrExpression],
+                             expressionLib: ExpressionLib): ErrorOr[WomValue] = {
     val parameterContext = ParameterContext(inputs = inputValues)
 
     // 3. outputEval: pass in the primary files to an expression to generate our return value
@@ -84,14 +86,14 @@ object CommandOutputBinding {
         case Some(StringOrExpression.String(string)) => WomString(string).valid
         case Some(StringOrExpression.Expression(expression)) =>
           val outputEvalParameterContext = parameterContext.copy(self = womFilesArray)
-          ExpressionEvaluator.eval(expression, outputEvalParameterContext)
+          ExpressionEvaluator.eval(expression, outputEvalParameterContext, expressionLib)
         case None =>
           womFilesArray.valid
       }
     }
 
     // Used to retrieve the file format to be injected into a file result.
-    def formatOptionErrorOr = CommandLineTool.CommandOutputParameter.format(formatCoproduct, parameterContext)
+    def formatOptionErrorOr = CommandLineTool.CommandOutputParameter.format(formatCoproduct, parameterContext, expressionLib)
 
     // 4. secondaryFiles: just before returning the value, fill in the secondary files on the return value
     def populateSecondaryFiles(evaluatedWomValue: WomValue): ErrorOr[WomValue] = {
@@ -100,7 +102,8 @@ object CommandOutputBinding {
           val secondaryFilesErrorOr = CommandLineTool.CommandOutputParameter.secondaryFiles(
             womMaybePopulatedFile,
             secondaryFilesCoproduct,
-            parameterContext
+            parameterContext,
+            expressionLib
           )
 
           (secondaryFilesErrorOr, formatOptionErrorOr) mapN { (secondaryFiles, formatOption) =>
@@ -132,7 +135,7 @@ object CommandOutputBinding {
 
     for {
       // 1. glob: get a list the globbed files as our primary files
-      primaryPaths <- GlobEvaluator.globs(commandOutputBinding.glob, parameterContext)
+      primaryPaths <- GlobEvaluator.globs(commandOutputBinding.glob, parameterContext, expressionLib)
 
       // 2. loadContents: load the contents of the primary files
       primaryAsDirectoryOrFiles <- primaryPaths.flatTraverse[ErrorOr, WomFile] {

cwl/src/main/scala/cwl/CommandOutputParameterExpression.scala

@@ -8,8 +8,9 @@ import wom.types._
 import wom.values.{WomFile, WomValue}
 
 case class CommandOutputParameterExpression(parameter: CommandOutputParameter,
-                                   override val cwlExpressionType: WomType,
-                                   override val inputs: Set[String]) extends CwlWomExpression {
+                                            override val cwlExpressionType: WomType,
+                                            override val inputs: Set[String],
+                                            override val expressionLib: ExpressionLib) extends CwlWomExpression {
 
   override def sourceString = parameter.toString
 
@@ -25,7 +26,8 @@ case class CommandOutputParameterExpression(parameter: CommandOutputParameter,
       cwlExpressionType,
       outputBinding,
       secondaryFilesOption,
-      formatOption
+      formatOption,
+      expressionLib
     )
   }
 
@@ -38,7 +40,8 @@ case class CommandOutputParameterExpression(parameter: CommandOutputParameter,
       inputValues,
       coerceTo,
       outputBinding,
-      secondaryFilesOption
+      secondaryFilesOption,
+      expressionLib
     )
   }
 

cwl/src/main/scala/cwl/CwlCodecs.scala

@@ -22,5 +22,5 @@ object CwlCodecs {
   implicit val cltD = implicitly[Decoder[CommandLineTool]]
   implicit val etD = implicitly[Decoder[ExpressionTool]]
 
-  def decodeCwl(in: String): Checked[CwlFile] = decodeAccumulating[CwlFile](in).leftMap(_.map(_.getMessage)).toEither
+  def decodeCwl(in: String): Checked[CwlFile] = decodeAccumulating[CwlFile](in).leftMap(_.map(_.getMessage).map(s"error parsing: $in" + _)).toEither
 }