Merge 291696d into a45880e

WDL/Error.py

@@ -41,6 +41,10 @@ class NotAnArray(Base):
     def __init__(self, node : SourceNode) -> None:
         super().__init__(node, "Not an array")
 
+class NotAPair(Base):
+    def __init__(self, node : SourceNode) -> None:
+        super().__init__(node, "Not a pair (taking left or right)")
+
 class StaticTypeMismatch(Base):
     def __init__(self, node : SourceNode, expected : T.Base, actual : T.Base, message : Optional[str] = None) -> None:
         msg = "Expected {} instead of {}".format(str(expected), str(actual))

WDL/Expr.py

@@ -125,8 +125,8 @@ def _infer_type(self, type_env : Env.Types) -> T.Base:
         if isinstance(self.expr.type, T.Array):
             if 'sep' not in self.options:
                 raise Error.StaticTypeMismatch(self, T.Array(None), self.expr.type, "array command placeholder must have 'sep'")
-            if sum(1 for t in [T.Int, T.Float, T.Boolean, T.String, T.File] if isinstance(self.expr.type.item_type, t)) == 0:
-                raise Error.StaticTypeMismatch(self, T.Array(None), self.expr.type, "cannot use array of complex types for command placeholder")
+            #if sum(1 for t in [T.Int, T.Float, T.Boolean, T.String, T.File] if isinstance(self.expr.type.item_type, t)) == 0:
+            #    raise Error.StaticTypeMismatch(self, T.Array(None), self.expr.type, "cannot use array of complex types for command placeholder")
         elif 'sep' in self.options:
                 raise Error.StaticTypeMismatch(self, T.Array(None), self.expr.type, "command placeholder has 'sep' option for non-Array expression")
         if ('true' in self.options or 'false' in self.options):
@@ -144,7 +144,7 @@ def eval(self, env : Env.Values) -> V.String:
         if isinstance(v, V.String):
             return v
         if isinstance(v, V.Array):
-            return V.String(self.options['sep'].join(str(item.value) for item in v.value)) # pyre-ignore
+            return V.String(self.options['sep'].join(str(item.value) for item in v.value))
         if v == V.Boolean(True) and 'true' in self.options:
             return V.String(self.options['true'])
         if v == V.Boolean(False) and 'false' in self.options:
@@ -321,15 +321,61 @@ def __init__(self, pos : SourcePosition, parts : List[str]) -> None:
         self.namespace = parts[:-1]
 
     def _infer_type(self, type_env : Env.Types) -> T.Base:
+        if len(self.namespace) > 0 and (self.name in ['left', 'right']):
+            # Special case for pair access, IDENT.left or IDENT.right
+            # Pair access through non-identifier expressions goes a different
+            # path, through the get_left and get_right terminals.
+            # TODO: avoid having two paths by ensuring .left and .right can't
+            #       parse as Ident
+            pair_name = self.namespace[-1]
+            pair_namespace = self.namespace[:-1]
+            try:
+                ans : T.Base = Env.resolve(type_env, pair_namespace, pair_name)
+            except KeyError:
+                pass
+            if isinstance(ans, T.Pair):
+                return ans.left_type if self.name == 'left' else ans.right_type
         try:
             ans : T.Base = Env.resolve(type_env, self.namespace, self.name)
             return ans
         except KeyError:
             raise Error.UnknownIdentifier(self) from None
 
     def eval(self, env : Env.Values) -> V.Base:
+        if len(self.namespace) > 0 and (self.name in ['left', 'right']):
+            pair_name = self.namespace[-1]
+            pair_namespace = self.namespace[:-1]
+            try:
+                ans : V.Base = Env.resolve(env, pair_namespace, pair_name)
+                if isinstance(ans, V.Pair):
+                    assert ans.value is not None
+                    return ans.value[0] if self.name == 'left' else ans.value[1]
+            except KeyError:
+                pass
         try:
             ans : V.Base = Env.resolve(env, self.namespace, self.name)
             return ans
         except KeyError:
             raise Error.UnknownIdentifier(self) from None
+
+# Pair literal
+
+class Pair(Base):
+    left : Base
+    right : Base
+
+    def __init__(self, pos : SourcePosition, left : Base, right : Base) -> None:
+        super().__init__(pos)
+        self.left = left
+        self.right = right
+
+    def _infer_type(self, type_env : Env.Types) -> T.Base:
+        self.left.infer_type(type_env)
+        self.right.infer_type(type_env)
+        return T.Pair(self.left.type, self.right.type)
+
+    def eval(self, env : Env.Values) -> V.Base:
+        assert isinstance(self.type, T.Pair)
+        lv = self.left.eval(env)
+        rv = self.right.eval(env)
+        return V.Pair(self.type, (lv,rv))

WDL/StdLib.py

@@ -33,6 +33,26 @@ def __call__(self, expr : E.Apply, env : E.Env) -> V.Base:
         return arr.value[idx] # pyre-ignore
 E._stdlib["_get"] = _ArrayGet()
 
+# Pair get (EXPR.left/EXPR.right)
+# The special case where EXPR is an identifier goes a different path, through
+# Expr.Ident.
+class _PairGet(E._Function):
+    left : bool
+    def __init__(self, left : bool) -> None:
+        self.left = left
+    def infer_type(self, expr : E.Apply) -> T.Base:
+        assert len(expr.arguments) == 1
+        if not isinstance(expr.arguments[0].type, T.Pair):
+            raise Error.NotAPair(expr.arguments[0])
+        return expr.arguments[0].type.left_type if self.left else expr.arguments[0].type.right_type
+    def __call__(self, expr : E.Apply, env : E.Env) -> V.Base:
+        assert len(expr.arguments) == 1
+        pair = expr.arguments[0].eval(env)
+        assert isinstance(pair.type, T.Pair)
+        assert isinstance(pair.value, tuple)
+        return pair.value[0] if self.left else pair.value[1]
+E._stdlib["_get_left"] = _PairGet(True)
+E._stdlib["_get_right"] = _PairGet(False)
 
 # _Function helper for simple functions with fixed argument and return types
 class _StaticFunction(E._Function):
@@ -73,7 +93,15 @@ def __call__(self, expr : E.Apply, env : E.Env) -> V.Base:
     ("ceil", [T.Float()], T.Int(), lambda x: exec('raise NotImplementedError()')),
     ("glob", [T.String()], T.Array(T.File()), lambda pattern: exec('raise NotImplementedError()')),
     ("read_int", [T.String()], T.Int(), lambda pattern: exec('raise NotImplementedError()')),
-    ("range", [T.Int()], T.Array(T.Int()), lambda high: exec('raise NotImplementedError()'))
+    ("read_boolean", [T.String()], T.Boolean(), lambda pattern: exec('raise NotImplementedError()')),
+    ("read_string", [T.String()], T.String(), lambda pattern: exec('raise NotImplementedError()')),
+    ("read_float", [T.String()], T.Float(), lambda pattern: exec('raise NotImplementedError()')),
+    ("read_array", [T.String()], T.Array(None), lambda pattern: exec('raise NotImplementedError()')),
+    ("read_map", [T.String()], T.Map(None), lambda pattern: exec('raise NotImplementedError()')),
+    ("read_lines", [T.String()], T.Array(None), lambda pattern: exec('raise NotImplementedError()')),
+    ("read_tsv", [T.String()], T.Array(T.Array(T.String())), lambda pattern: exec('raise NotImplementedError()')),
+    ("write_map", [T.Map(None)], T.String(), lambda pattern: exec('raise NotImplementedError()')),
+    ("range", [T.Int()], T.Array(T.Int()), lambda high: exec('raise NotImplementedError()')),
 ]
 for name, argument_types, return_type, F in _static_functions:
     E._stdlib[name] = _StaticFunction(name, argument_types, return_type, F)
@@ -212,9 +240,9 @@ def infer_type(self, expr : E.Apply) -> T.Base:
         if len(expr.arguments) != 1:
             raise Error.WrongArity(expr, 1)
         if not isinstance(expr.arguments[0].type, T.Array):
-            raise Error.StaticTypeMismatch(expr, T.Array(None), expr.arguments[0].type)
+            raise Error.StaticTypeMismatch(expr.arguments[0], T.Array(None), expr.arguments[0].type)
         if expr.arguments[0].type.item_type is None:
-            raise Error.EmptyArray(expr.arguments[0])
+            raise Error.EmptyArray(expr.arguments[0]) # TODO: error for 'indeterminate type'
         ty = copy.copy(expr.arguments[0].type.item_type)
         assert isinstance(ty, T.Base)
         ty.optional = False
@@ -223,3 +251,22 @@ def infer_type(self, expr : E.Apply) -> T.Base:
     def __call__(self, expr : E.Apply, env : Env.Values) -> V.Base:
         raise NotImplementedError()
 E._stdlib["select_first"] = _SelectFirst()
+
+class _Zip(E._Function):
+    # 'a array -> 'b array -> ('a,'b) array
+    def infer_type(self, expr : E.Apply) -> T.Base:
+        if len(expr.arguments) != 2:
+            raise Error.WrongArity(expr, 2)
+        if not isinstance(expr.arguments[0].type, T.Array):
+            raise Error.StaticTypeMismatch(expr.arguments[0], T.Array(None), expr.arguments[0].type)
+        if expr.arguments[0].type.item_type is None:
+            raise Error.EmptyArray(expr.arguments[0]) # TODO: error for 'indeterminate type'
+        if not isinstance(expr.arguments[1].type, T.Array):
+            raise Error.StaticTypeMismatch(expr.arguments[1], T.Array(None), expr.arguments[0].type)
+        if expr.arguments[1].type.item_type is None:
+            raise Error.EmptyArray(expr.arguments[1]) # TODO: error for 'indeterminate type'
+        return T.Array(T.Pair(expr.arguments[0].type.item_type, expr.arguments[1].type.item_type))
+
+    def __call__(self, expr : E.Apply, env : Env.Values) -> V.Base:
+        raise NotImplementedError()
+E._stdlib["zip"] = _Zip()

WDL/Type.py

@@ -23,7 +23,7 @@
 ``WDL.Type.Array(WDL.Type.String()) != WDL.Type.Array(WDL.Type.Float())``.
 """
 from abc import ABC, abstractmethod
-from typing import Optional, TypeVar
+from typing import Optional, TypeVar, Tuple
 
 TVBase = TypeVar("TVBase", bound="Base")
 class Base(ABC):
@@ -91,10 +91,10 @@ class Array(Base):
     """
     Array type, parameterized by the type of the constituent items.
 
-    ``item_type`` may be None to represent the type of the literal empty array
-    ``[]``, which is considered compatible with any array type (lacking the
-    nonempty quantifier). This special case should be considered explicitly
-    when comparing array types.
+    ``item_type`` may be None to represent an array whose item type isn't
+    known statically, such as a literal empty array ``[]``, or the result
+    of the ``read_array()`` standard library function. This is considered
+    statically coercible to any array type (but may fail at runtime)
     """
     item_type : Optional[Base]
     nonempty : bool
@@ -109,11 +109,47 @@ def __str__(self) -> str:
         ans = "Array[" + (str(self.item_type) if self.item_type is not None else "") + "]"
         return ans
     def coerces(self, rhs : Base) -> bool:
-        if self.item_type is None and isinstance(rhs, Array) and not rhs.nonempty:
-            return True
         if isinstance(rhs, Array):
-            if self.item_type is None:
-                return (not rhs.nonempty)
+            if self.item_type is None or rhs.item_type is None:
+                return True
             else:
                 return self.item_type.coerces(rhs.item_type)
         return super().coerces(rhs)
+
+class Map(Base):
+    """
+    Map type, parameterized by the (key,value) item type.
+
+    ``item_type`` may be None to represent a map whose type isn't known
+    statically, such as a literal empty may ``{}``, or the result of the
+    ``read_map()`` standard library function. This is considered statically
+    coercible to any map type (but may fail at runtime)
+    """
+    item_type : Optional[Tuple[Base,Base]]
+
+    def __init__(self, item_type : Optional[Tuple[Base,Base]], optional : bool = False) -> None:
+        self.optional = optional
+        self.item_type = item_type
+    def __str__(self) -> str:
+        return "Map[" + (str(self.item_type[0]) + "," + str(self.item_type[1]) if self.item_type is not None else "") + "]" # pyre-fixme
+    def coerces(self, rhs : Base) -> bool:
+        if isinstance(rhs, Map):
+            if self.item_type is None or rhs.item_type is None:
+                return True
+            else:
+                return self.item_type[0].coerces(rhs.item_type[0]) and self.item_type[1].coerces(rhs.item_type[1]) # pyre-fixme
+        return super().coerces(rhs)
+
+class Pair(Base):
+    """
+    Pair type, parameterized by the left and right item types.
+    """
+    left_type : Base
+    right_type : Base
+
+    def __init__(self, left_type : Base, right_type : Base, optional : bool = False) -> None:
+        self.optional = optional
+        self.left_type = left_type
+        self.right_type = right_type
+    def __str__(self) -> str:
+        return "Pair[" + (str(self.left_type) + "," + str(self.right_type)) + "]"

WDL/Value.py

@@ -6,7 +6,7 @@
 ``WDL.Value.Base``.
 """
 from abc import ABC, abstractmethod
-from typing import Any, List, Optional, TypeVar
+from typing import Any, List, Optional, TypeVar, Tuple
 import WDL.Type as T
 import json
 
@@ -75,12 +75,27 @@ def __str__(self) -> str:
 
 class Array(Base):
     """``value`` is a Python ``list`` of other ``WDL.Value`` instances"""
-    value : List[Any] = []
-    def __init__(self, type : T.Array, value : List[Any]) -> None:
+    value : List[Base] = []
+    def __init__(self, type : T.Array, value : List[Base]) -> None:
         super().__init__(type, value)
     def __str__(self) -> str:
         return "[" + ", ".join([str(item) for item in self.value]) + "]"
 
+class Map(Base):
+    value : List[Tuple[Base,Base]] = []
+    def __init__(self, type : T.Map, value : List[Tuple[Base,Base]]) -> None:
+        super().__init__(type, value)
+    def __str__(self) -> str:
+        raise NotImplementedError() # TODO
+
+class Pair(Base):
+    value : Optional[Tuple[Base,Base]] = None
+    def __init__(self, type : T.Pair, value : Tuple[Base,Base]) -> None:
+        super().__init__(type, value)
+    def __str__(self) -> str:
+        assert isinstance(self.value, tuple)
+        return "(" + str(self.value[0]) + "," + str(self.value[1]) + ")" # pyre-fixme
+
 class Null(Base):
     """Represents the missing value which optional inputs may take. ``type`` and ``value`` are both None."""
     type : Optional[Any] # pyre-ignore

WDL/__init__.py

@@ -69,6 +69,14 @@ def negate(self, items, meta) -> E.Base:
     def get(self, items, meta) -> E.Base:
         return E.Apply(sp(self.filename, meta), "_get", items)
 
+    def pair(self, items, meta) -> E.Base:
+        assert len(items) == 2
+        return E.Pair(sp(self.filename, meta), items[0], items[1])
+    def get_left(self, items, meta) -> E.Base:
+        return E.Apply(sp(self.filename, meta), "_get_left", items)
+    def get_right(self, items, meta) -> E.Base:
+        return E.Apply(sp(self.filename, meta), "_get_right", items)
+
     def ifthenelse(self, items, meta) -> E.Base:
         assert len(items) == 3
         return E.IfThenElse(sp(self.filename, meta), *items)
@@ -124,6 +132,24 @@ def array_type(self, items, meta):
             if items[1].value == "+":
                 nonempty = True
         return T.Array(items[0], optional, nonempty)
+    def map_type(self, items, meta):
+        assert len(items) >= 2
+        assert isinstance(items[0], WDL.Type.Base)
+        assert isinstance(items[1], WDL.Type.Base)
+        optional = False
+        if len(items) > 2:
+            if items[2].value == "?":
+                optional = True
+        return T.Map((items[0], items[1]), optional)
+    def pair_type(self, items, meta):
+        assert len(items) >= 2
+        assert isinstance(items[0], WDL.Type.Base)
+        assert isinstance(items[1], WDL.Type.Base)
+        optional = False
+        if len(items) > 2:
+            if items[2].value == "?":
+                optional = True
+        return T.Pair(items[0], items[1], optional)
 
 class _DocTransformer(_ExprTransformer, _TypeTransformer):
     def __init__(self, file : str) -> None:
@@ -303,6 +329,8 @@ def parse_document(txt : str, uri : str = '') -> D.Document:
         return _DocTransformer(uri).transform(WDL._parser.parse(txt, "document"))
     except lark.exceptions.UnexpectedCharacters as exn:
         raise Err.ParserError(uri if uri != '' else '(in buffer)') from exn
+    except lark.exceptions.UnexpectedToken as exn:
+        raise Err.ParserError(uri if uri != '' else '(in buffer)') from exn
 
 def load(uri : str, path : List[str] = []) -> D.Document:
     """

WDL/_parser.py

@@ -42,6 +42,10 @@
           | "[" [expr ("," expr)*] "]" -> array
           | expr_core "[" expr "]" -> get
 
+          | "(" expr "," expr ")" -> pair
+          | expr_core "." _LEFT -> get_left
+          | expr_core "." _RIGHT -> get_right
+
           | "if" expr "then" expr "else" expr -> ifthenelse
 
           | ident
@@ -55,13 +59,13 @@
         | SIGNED_FLOAT -> float
 
 // string (single-quoted)
-STRING1_CHAR: "\\'" | /[^'$]/ | /\$[^{]/
+STRING1_CHAR: "\\'" | /[^'$]/ | /\$[^{']/
 STRING1_END: STRING1_CHAR* "$"? "'"
 STRING1_FRAGMENT: STRING1_CHAR* "${"
 string1: /'/ [(STRING1_FRAGMENT expr "}")*] STRING1_END -> string
 
 // string (double-quoted)
-STRING2_CHAR: "\\\"" | /[^"$]/ | /\$[^{]/
+STRING2_CHAR: "\\\"" | /[^"$]/ | /\$[^{"]/
 STRING2_END: STRING2_CHAR* "$"? /"/
 STRING2_FRAGMENT: STRING2_CHAR* "${"
 string2: /"/ [(STRING2_FRAGMENT expr "}")*] STRING2_END -> string
@@ -72,6 +76,8 @@
 ESCAPED_STRING1: "'" STRING_INNER1* "'"
 string_literal: ESCAPED_STRING | ESCAPED_STRING1
 
+_LEFT.2: "left"
+_RIGHT.2: "right"
 ident: [CNAME ("." CNAME)*]
 
 // WDL types and declarations
@@ -81,12 +87,16 @@
     | _STRING QUANT? -> string_type
     | _FILE QUANT? -> file_type
     | _ARRAY "[" type "]" ARRAY_QUANT? -> array_type
+    | _MAP "[" type "," type "]" QUANT? -> map_type
+    | _PAIR "[" type "," type "]" QUANT? -> pair_type
 _INT.2: "Int"           // .2 ensures higher priority than CNAME
 _FLOAT.2: "Float"
 _BOOLEAN.2: "Boolean"
 _STRING.2: "String"
 _FILE.2: "File"
 _ARRAY.2: "Array"
+_MAP.2: "Map"
+_PAIR.2: "Pair"
 QUANT: "?"
 ARRAY_QUANT: "?" | "+"