InputRecordSchema and Command line sorting (#3112)

* InputRecordSchema and Command template generation

.travis.yml

@@ -36,7 +36,7 @@ env:
     - BUILD_TYPE=centaurLocal
     - BUILD_TYPE=centaurTes
     - BUILD_TYPE=centaurCwlConformanceLocal
-    - BUILD_TYPE=centaurCwlConformancePAPI
+#    - BUILD_TYPE=centaurCwlConformancePAPI
 script:
   - sudo apt-get install procps
   - pip install 'requests[security]'

centaurCwlRunner/src/main/scala/centaur/cwl/OutputManipulator.scala

@@ -1,7 +1,7 @@
 package centaur.cwl
 
 import io.circe.Json
-import spray.json.{JsArray, JsNumber, JsString, JsValue}
+import spray.json.{JsArray, JsNumber, JsObject, JsString, JsValue}
 import cwl.{MyriadOutputType, File => CwlFile}
 import shapeless.{Inl, Poly1}
 import cwl.{CwlType, MyriadOutputType, File => CwlFile}
@@ -12,9 +12,9 @@ import io.circe.refined._
 import io.circe.shapes._
 import io.circe.syntax._
 import shapeless.{Inl, Poly1}
-import spray.json.{JsNumber, JsString, JsValue}
 import _root_.cwl._
 import cromwell.core.path.PathBuilder
+import cwl.command.ParentName
 
 //Take cromwell's outputs and format them as expected by the spec
 object OutputManipulator extends Poly1 {
@@ -48,7 +48,21 @@ object OutputManipulator extends Poly1 {
           innerType <- items.select[MyriadOutputInnerType]
           outputJson = metadata.map(m => resolveOutputViaInnerType(innerType)(m, pathBuilder)).asJson
         } yield outputJson).getOrElse(throw new RuntimeException(s"We currently do not support output arrays with ${tpe.select[OutputArraySchema].get.items} inner type"))
-      case (json, tpe) => throw new RuntimeException(s" we currently do not support outputs of $json and type $tpe")
+      case (JsObject(metadata), tpe) if tpe.select[OutputRecordSchema].isDefined =>
+        def processField(field: OutputRecordField) = {
+          val parsedName = FullyQualifiedName(field.name)(ParentName.empty).id
+          field.`type`.select[MyriadOutputInnerType] map { parsedName -> _ }
+        }
+
+        (for {
+          schema <- tpe.select[OutputRecordSchema]
+          fields <- schema.fields
+          typeMap = fields.flatMap(processField).toMap
+          outputJson = metadata.map({
+            case (k, v) => k -> resolveOutputViaInnerType(typeMap(k))(v, pathBuilder)
+          }).asJson
+        } yield outputJson).getOrElse(throw new RuntimeException(s"We currently do not support output arrays with ${tpe.select[OutputArraySchema].get.items} inner type"))
+      case (json, tpe) => throw new RuntimeException(s"We currently do not support outputs (${json.getClass.getSimpleName}) of $json and type $tpe")
     }
   }
 

core/src/main/scala/cromwell/core/simpleton/WomValueBuilder.scala

@@ -115,6 +115,16 @@ object WomValueBuilder {
         // map keys are guaranteed by the WDL spec to be primitives, so the "coerceRawValue(..).get" is safe.
         val map: Map[String, WomValue] = groupedByMapKey map { case (k, ss) => k -> toWomValue(WomAnyType, ss) }
         WomObject(map)
+      case composite: WomCompositeType =>
+        val groupedByMapKey: Map[String, Traversable[SimpletonComponent]] = group(components map descendIntoMap)
+        // map keys are guaranteed by the WDL spec to be primitives, so the "coerceRawValue(..).get" is safe.
+        val map: Map[String, WomValue] = groupedByMapKey map { case (k, ss) => 
+          val valueType = composite
+            .typeMap
+            .getOrElse(k, throw new RuntimeException(s"Field $k is not a declared field of composite type $composite. Cannot build a WomValue from the simpletons."))
+          k -> toWomValue(valueType, ss) 
+        }
+        WomObject.withType(map, composite)
       case WomAnyType =>
         // Ok, we're going to have to guess, but the keys should give us some clues:
         if (components forall { component => MapElementPattern.findFirstMatchIn(component.path).isDefined }) {

core/src/main/scala/cromwell/core/simpleton/WomValueSimpleton.scala

@@ -31,7 +31,7 @@ object WomValueSimpleton {
       case WomArray(_, arrayValue) => arrayValue.zipWithIndex flatMap { case (arrayItem, index) => arrayItem.simplify(s"$name[$index]") }
       case WomMap(_, mapValue) => mapValue flatMap { case (key, value) => value.simplify(s"$name:${key.valueString.escapeMeta}") }
       case WomPair(left, right) => left.simplify(s"$name:left") ++ right.simplify(s"$name:right")
-      case womObjectLike: WomObjectLike => womObjectLike.value flatMap {
+      case womObjectLike: WomObjectLike => womObjectLike.values flatMap {
         case (key, value) => value.simplify(s"$name:${key.escapeMeta}")
       }
       // TODO: WOM: WOMFILE: Better simplification of listed dirs / populated files

core/src/main/scala/cromwell/util/JsonFormatting/WomValueJsonFormatter.scala

@@ -12,7 +12,7 @@ object WomValueJsonFormatter extends DefaultJsonProtocol {
       case f: WomFloat => JsNumber(f.value)
       case b: WomBoolean => JsBoolean(b.value)
       case f: WomSingleFile => JsString(f.value)
-      case o: WomObject => new JsObject(o.value map {case(k, v) => k -> write(v)})
+      case o: WomObjectLike => new JsObject(o.values map {case(k, v) => k -> write(v)})
       case a: WomArray => new JsArray(a.value.map(write).toVector)
       case m: WomMap => new JsObject(m.value map {case(k,v) => k.valueString -> write(v)})
       case q: WomPair => new JsObject(Map("left" -> write(q.left), "right" -> write(q.right)))

core/src/test/scala/cromwell/core/simpleton/WomValueBuilderSpec.scala

@@ -211,8 +211,8 @@ class WomValueBuilderSpec extends FlatSpec with Matchers with Mockito {
     val rebuiltObject = rebuiltValues.head._2
     rebuiltObject match {
       case o: WomObject =>
-        aMap.womType.coerceRawValue(o.value("a")) should be(Success(aMap))
-        bMap.womType.coerceRawValue(o.value("b")) should be(Success(bMap))
+        aMap.womType.coerceRawValue(o.values("a")) should be(Success(aMap))
+        bMap.womType.coerceRawValue(o.values("b")) should be(Success(bMap))
       case other => fail(s"Expected reconstruction to Object but got ${other.womType.toDisplayString}")
     }
   }

core/src/test/scala/cromwell/util/WomMocks.scala

@@ -1,5 +1,6 @@
 package cromwell.util
 
+import cats.syntax.validated._
 import cromwell.core.CallOutputs
 import wom.RuntimeAttributes
 import wom.callable.Callable.OutputDefinition
@@ -10,7 +11,7 @@ import wom.types.{WomStringType, WomType}
 import wom.values.WomValue
 
 object WomMocks {
-  val EmptyTaskDefinition = CallableTaskDefinition("emptyTask", List.empty, RuntimeAttributes(Map.empty),
+  val EmptyTaskDefinition = CallableTaskDefinition("emptyTask", Function.const(List.empty.validNel), RuntimeAttributes(Map.empty),
     Map.empty, Map.empty, List.empty, List.empty, Set.empty)
 
   val EmptyWorkflowDefinition = mockWorkflowDefinition("emptyWorkflow")
@@ -28,7 +29,7 @@ object WomMocks {
   }
 
   def mockTaskDefinition(name: String) = {
-    CallableTaskDefinition(name, List.empty, RuntimeAttributes(Map.empty),
+    CallableTaskDefinition(name, Function.const(List.empty.validNel), RuntimeAttributes(Map.empty),
       Map.empty, Map.empty, List.empty, List.empty, Set.empty)
   }
 

cwl/src/main/scala/cwl/ArgumentToCommandPart.scala

@@ -17,7 +17,7 @@ object ArgumentToCommandPart extends Poly1 {
     StringCommandPart.apply
   }
 
-  implicit def caseCommandLineBinding: Case.Aux[CommandLineBinding, CommandPart] = at {
-    CommandLineBindingCommandPart.apply
+  implicit def caseCommandLineBinding: Case.Aux[ArgumentCommandLineBinding, CommandPart] = at {
+    ArgumentCommandLineBindingCommandPart.apply
   }
 }