Merge 4c29340 into b8f4d65

.gitmodules

@@ -46,6 +46,9 @@
 [submodule "test_corpi/biowdl/tasks"]
 	path = test_corpi/biowdl/tasks
 	url = https://github.com/biowdl/tasks.git
+[submodule "test_corpi/biowdl/aligning"]
+	path = test_corpi/biowdl/aligning
+	url = https://github.com/biowdl/aligning
 [submodule "test_corpi/biowdl/expression-quantification"]
 	path = test_corpi/biowdl/expression-quantification
 	url = https://github.com/biowdl/expression-quantification.git

WDL/Error.py

@@ -236,7 +236,8 @@ def maybe_raise(self) -> None:
         if len(self._exceptions) == 1:
             raise self._exceptions[0]
         if self._exceptions:
-            raise MultipleValidationErrors(*self._exceptions)  # pyre-ignore
+            # pyre-ignore
+            raise MultipleValidationErrors(*self._exceptions) from self._exceptions[0]
 
 
 @contextmanager

WDL/Expr.py

@@ -465,9 +465,25 @@ def _infer_type(self, type_env: Env.Types) -> T.Base:
             else:
                 v.typecheck(vty)
         if kty is None:
-            return T.Map((T.Any(), T.Any()))
+            return T.Map((T.Any(), T.Any()), literal_keys=set())
         assert vty is not None
-        return T.Map((kty, vty))
+        literal_keys = None
+        if kty == T.String():
+            # If the keys are string constants, record them in the Type object
+            # for potential later use in struct coercion. (Normally the Type
+            # encodes the common type of the keys, but not the keys themselves)
+            literal_keys = set()
+            for k, _ in self.items:
+                if (
+                    literal_keys is not None
+                    and isinstance(k, String)
+                    and len(k.parts) == 3
+                    and isinstance(k.parts[1], str)
+                ):
+                    literal_keys.add(k.parts[1])
+                else:
+                    literal_keys = None
+        return T.Map((kty, vty), literal_keys=literal_keys)
 
     def _eval(self, env: Env.Values, stdlib: "Optional[WDL.StdLib.Base]" = None) -> V.Base:
         ""

WDL/Tree.py

@@ -113,20 +113,21 @@ def children(self) -> Iterable[SourceNode]:
             yield self.expr
 
     def add_to_type_env(
-        self, struct_typedefs: Env.StructTypeDefs, type_env: Env.Types
+        self, struct_typedefs: Env.StructTypeDefs, type_env: Env.Types, collision_ok: bool = False
     ) -> Env.Types:
         # Add an appropriate binding in the type env, after checking for name
         # collision.
-        try:
-            Env.resolve(type_env, [], self.name)
-            raise Err.MultipleDefinitions(self, "Multiple declarations of " + self.name)
-        except KeyError:
-            pass
-        try:
-            Env.resolve_namespace(type_env, [self.name])
-            raise Err.MultipleDefinitions(self, "Value/call name collision on " + self.name)
-        except KeyError:
-            pass
+        if not collision_ok:
+            try:
+                Env.resolve(type_env, [], self.name)
+                raise Err.MultipleDefinitions(self, "Multiple declarations of " + self.name)
+            except KeyError:
+                pass
+            try:
+                Env.resolve_namespace(type_env, [self.name])
+                raise Err.MultipleDefinitions(self, "Value/call name collision on " + self.name)
+            except KeyError:
+                pass
         _resolve_struct_typedefs(self.pos, self.type, struct_typedefs)
         if isinstance(self.type, T.StructInstance):
             return _add_struct_instance_to_type_env([self.name], self.type, type_env, ctx=self)
@@ -835,13 +836,24 @@ def typecheck(self, doc: TVDocument, check_quant: bool) -> None:
                                 output, "multiple workflow outputs named " + output.name
                             )
                         )
+                    output_names.add(output.name)
+                    # tricky sequence here: we need to call Decl.add_to_type_env to resolve
+                    # potential struct type, but:
+                    # 1. we don't want it to check for name collision in the usual way in order to
+                    #    handle a quirk of draft-2 workflow output style, where an output may take
+                    #    the name of another decl in the workflow. Instead we've tracked and
+                    #    rejected any duplicate names among the workflow outputs.
+                    # 2. we still want to typecheck the output expression againsnt the 'old' type
+                    #    environment
+                    output_type_env2 = output.add_to_type_env(
+                        doc.struct_typedefs, output_type_env, collision_ok=True
+                    )
                     errors.try1(
                         lambda output=output: output.typecheck(
                             output_type_env, check_quant=check_quant
                         )
                     )
-                    output_names.add(output.name)
-                    output_type_env = Env.bind(output_type_env, [], output.name, output.type)
+                    output_type_env = output_type_env2
         # 6. check for cyclic dependencies
         WDL._util.detect_cycles(_dependency_matrix(_decls_and_calls(self)))  # pyre-fixme
 

WDL/Type.py

@@ -39,7 +39,7 @@
    :top-classes: WDL.Type.Base
 """
 from abc import ABC
-from typing import Optional, Tuple, Dict, Iterable
+from typing import Optional, Tuple, Dict, Iterable, Set
 import copy
 
 
@@ -250,11 +250,23 @@ class Map(Base):
     The key and value types may be ``Any`` when not known statically, such as in a literal empty map ``{}``.
     """
 
-    def __init__(self, item_type: Tuple[Base, Base], optional: bool = False) -> None:
+    literal_keys: Optional[Set[str]]
+    ""
+    # Special use: Map[String,_] literal stores the key names here for potential use in
+    # struct coercions where we need them. (Normally the Map type would record the common
+    # type of the keys but not the keys themselves.)
+
+    def __init__(
+        self,
+        item_type: Tuple[Base, Base],
+        optional: bool = False,
+        literal_keys: Optional[Set[str]] = None,
+    ) -> None:
         self._optional = optional
         if item_type is None:
             item_type = (Any(), Any())
         self.item_type = item_type
+        self.literal_keys = literal_keys
 
     def __str__(self) -> str:
         return (
@@ -281,6 +293,21 @@ def coerces(self, rhs: Base, check_quant: bool = True) -> bool:
                 and self.item_type[1].coerces(rhs.item_type[1], check_quant)
                 and self._check_optional(rhs, check_quant)
             )
+        if isinstance(rhs, StructInstance) and self.literal_keys is not None:
+            # struct assignment from map literal: the map literal must contain all non-optional
+            # struct members, and the value type must be coercible to those member types
+            rhs_members = rhs.members
+            assert rhs_members is not None
+            rhs_keys = set(rhs_members.keys())
+            if self.literal_keys - rhs_keys:
+                return False
+            for k in self.literal_keys:
+                if not self.item_type[1].coerces(rhs_members[k], check_quant):
+                    return False
+            for opt_k in rhs_keys - self.literal_keys:
+                if not rhs_members[opt_k].optional:
+                    return False
+            return True
         if isinstance(rhs, Any):
             return self._check_optional(rhs, check_quant)
         return False
@@ -360,8 +387,9 @@ def __init__(self, type_name: str, optional: bool = False) -> None:
         self.members = None
 
     def __str__(self) -> str:
-        assert self.members
-        return _struct_type_id(self.members) + ("?" if self.optional else "")
+        return (_struct_type_id(self.members) if self.members else self.type_name) + (
+            "?" if self.optional else ""
+        )
 
     def coerces(self, rhs: Base, check_quant: bool = True) -> bool:
         ""

WDL/Value.py

@@ -194,6 +194,14 @@ def coerce(self, desired_type: Optional[T.Base] = None) -> Base:
                     for (k, v) in self.value
                 ],
             )
+        if isinstance(desired_type, T.StructInstance):
+            assert desired_type.members
+            ans = {}
+            for k, v in self.value:
+                k = k.coerce(T.String()).value
+                assert k in desired_type.members
+                ans[k] = v
+            return Struct(desired_type, ans)
         return super().coerce(desired_type)
 
 
@@ -265,21 +273,20 @@ class Struct(Base):
 
     def __init__(self, type: Union[T.Object, T.StructInstance], value: Dict[str, Base]) -> None:
         super().__init__(type, value)
-        self.value = value
+        self.value = dict(value)
+        if isinstance(type, T.StructInstance):
+            # if initializer (map or object literal) omits optional members,
+            # fill them in with null
+            assert type.members
+            for k in type.members:
+                if k not in self.value:
+                    assert type.members[k].optional
+                    self.value[k] = Null()
 
     def coerce(self, desired_type: Optional[T.Base] = None) -> Base:
         ""
         if isinstance(self.type, T.Object) and isinstance(desired_type, T.StructInstance):
-            ans = dict(self.value)
-            # object literal may omit optional struct members; fill them out with null
-            # TODO: this should be done in the eval() of struct literals when there's
-            # a new syntax specifying the struct type in the literal
-            assert desired_type.members
-            for k in desired_type.members:
-                if k not in self.value:
-                    assert desired_type.members[k].optional  # typechecker should ensure
-                    ans[k] = Null()
-            return Struct(desired_type, ans)
+            return Struct(desired_type, self.value)
         return self
 
     def __str__(self) -> str:

tests/test_3corpi.py

@@ -209,6 +209,14 @@ class Contrived2(unittest.TestCase):
 class BioWDLTasks(unittest.TestCase):
     pass
 
+@test_corpus(
+    ["test_corpi/biowdl/aligning/**"],
+    expected_lint={'UnusedImport': 12, 'OptionalCoercion': 12, 'StringCoercion': 14, 'UnusedDeclaration': 12, 'UnnecessaryQuantifier': 41, 'NonemptyCoercion': 1, 'NameCollision': 1},
+    check_quant=False,
+)
+class BioWDLAligning(unittest.TestCase):
+    pass
+
 @test_corpus(
     ["test_corpi/biowdl/expression-quantification/**"],
     expected_lint={'UnusedImport': 12, 'OptionalCoercion': 11, 'StringCoercion': 14, 'UnusedDeclaration': 12, 'UnnecessaryQuantifier': 41, 'NonemptyCoercion': 3, 'NameCollision': 1},

tests/test_4taskrun.py

@@ -365,7 +365,7 @@ def test_coercion(self):
         version 1.0
         struct Car {
             String model
-            Int year
+            Int? year
             Int? mileage
         }
         task t {
@@ -375,10 +375,14 @@ def test_coercion(self):
                     model: "Mazda",
                     year: 2017
                 }
+                Car car2 = {
+                    "model": "Toyota"
+                }
             }
         }
         """)
         self.assertEqual(outputs["car"], {"model": "Mazda", "year": 2017, "mileage": None})
+        self.assertEqual(outputs["car2"], {"model": "Toyota", "year": None, "mileage": None})
 
     def test_errors(self):
         self._test_task(R"""