Skip to content

Commit

Permalink
Merge 21ab482 into 2cea7d0
Browse files Browse the repository at this point in the history
  • Loading branch information
@mlin
mlin committed Apr 2, 2020
2 parents 2cea7d0 + 21ab482 commit 300b7cc
Show file tree
Hide file tree
Showing 6 changed files with 117 additions and 108 deletions.
104 changes: 21 additions & 83 deletions WDL/Expr.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -412,13 +412,11 @@ def children(self) -> Iterable[SourceNode]:
def _infer_type(self, type_env: Env.Bindings[Type.Base]) -> Type.Base:
if not self.items:
return Type.Array(Type.Any())
item_type = _unify_types(
[item.type for item in self.items],
self._check_quant,
self,
"(unable to unify array item types)",
all_stringifiable=True,
item_type = Type.unify(
[item.type for item in self.items], check_quant=self._check_quant, force_string=True
)
if isinstance(item_type, Type.Any):
raise Error.IndeterminateType(self, "unable to unify array item types")
return Type.Array(item_type, optional=False, nonempty=True)

def typecheck(self, expected: Optional[Type.Base]) -> Base:
Expand Down Expand Up @@ -514,19 +512,14 @@ def children(self) -> Iterable[SourceNode]:
def _infer_type(self, type_env: Env.Bindings[Type.Base]) -> Type.Base:
if not self.items:
return Type.Map((Type.Any(), Type.Any()), literal_keys=set())
kty = _unify_types(
[k.type for (k, _) in self.items],
self._check_quant,
self,
"(unable to unify map key types)",
)
vty = _unify_types(
[v.type for (_, v) in self.items],
self._check_quant,
self,
"(unable to unify map value types)",
all_stringifiable=True,
kty = Type.unify([k.type for (k, _) in self.items], check_quant=self._check_quant)
if isinstance(kty, Type.Any):
raise Error.IndeterminateType(self, "unable to unify map key types")
vty = Type.unify(
[v.type for (_, v) in self.items], check_quant=self._check_quant, force_string=True
)
if isinstance(vty, Type.Any):
raise Error.IndeterminateType(self, "unable to unify map value types")
literal_keys = None
if kty == Type.String():
# If the keys are string constants, record them in the Type object
Expand Down Expand Up @@ -656,12 +649,17 @@ def _infer_type(self, type_env: Env.Bindings[Type.Base]) -> Type.Base:
raise Error.StaticTypeMismatch(
self, Type.Boolean(), self.condition.type, "in if condition"
)
return _unify_types(
[self.consequent.type, self.alternative.type],
self._check_quant,
self,
"(unable to unify consequent & alternative types)",
ty = Type.unify(
[self.consequent.type, self.alternative.type], check_quant=self._check_quant
)
if isinstance(ty, Type.Any):
raise Error.StaticTypeMismatch(
self,
self.consequent.type,
self.alternative.type,
"(unable to unify consequent & alternative types)",
)
return ty

def _eval(
self, env: Env.Bindings[Value.Base], stdlib: "Optional[StdLib.Base]" = None
Expand Down Expand Up @@ -1011,63 +1009,3 @@ def _eval(
f = getattr(stdlib, self.function_name, None)
assert isinstance(f, StdLib.Function)
return f(self, env, stdlib)


def _unify_types(
types: List[Type.Base],
check_quant: bool,
node: SourceNode,
message: str,
all_stringifiable: bool = False,
) -> Type.Base:
"""
Given a nonempty list of types, compute a type to which they're all coercible (or raise
StaticTypeMismatch)
all_stringifiable: permit unification to String even if no item is a String (but all can be
coerced)
"""
assert types

# begin with first type; or if --no-quant-check, the first array type (as we can try to promote
# other T to Array[T])
t = types[0]
if not check_quant:
t = next((a for a in types if isinstance(a, Type.Array)), t)

# potentially promote/generalize t to other types seen
optional = False
all_nonempty = True
for t2 in types:
if isinstance(t, Type.Int) and isinstance(t2, Type.Float):
t = Type.Float()
if isinstance(t, Type.String) and isinstance(t2, Type.File):
t = Type.File()

if (
isinstance(t2, Type.String)
and not isinstance(t2, Type.File)
and not isinstance(t, Type.File)
and (not check_quant or not isinstance(t, Type.Array))
):
t = Type.String()
if not t2.coerces(Type.String(optional=True), check_quant=check_quant):
all_stringifiable = False

if t2.optional:
optional = True
if isinstance(t2, Type.Array) and not t2.nonempty:
all_nonempty = False

if isinstance(t, Type.Array):
t = t.copy(nonempty=all_nonempty)
t = t.copy(optional=optional)

# check all types are coercible to t
for t2 in types:
if not t2.coerces(t, check_quant=check_quant):
if all_stringifiable:
return Type.String(optional=optional)
raise Error.StaticTypeMismatch(node, t, t2, message)

return t
19 changes: 1 addition & 18 deletions WDL/StdLib.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -328,24 +328,7 @@ def _parse_map(s: str) -> Value.Map:


def _parse_json(s: str) -> Value.Base:
# TODO: handle nested map/array types...tricky as don't know the expected WDL type
j = json.loads(s)
if isinstance(j, dict):
ans = []
for k in j:
ans.append((Value.String(str(k)), Value.from_json(Type.Any(), j[k])))
return Value.Map((Type.String(), Type.Any()), ans)
if isinstance(j, list):
return Value.Array(Type.Any(), [Value.from_json(Type.Any(), v) for v in j])
if isinstance(j, bool):
return Value.Boolean(j)
if isinstance(j, int):
return Value.Int(j)
if isinstance(j, float):
return Value.Float(j)
if j is None:
return Value.Null()
raise Error.InputError("parse_json()")
return Value.from_json(Type.Any(), json.loads(s))


def _serialize_lines(array: Value.Array, outfile: BinaryIO) -> None:
Expand Down
58 changes: 57 additions & 1 deletion WDL/Type.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
"""
import copy
from abc import ABC
from typing import Optional, Tuple, Dict, Iterable, Set
from typing import Optional, Tuple, Dict, Iterable, Set, List


class Base(ABC):
Expand Down Expand Up @@ -481,3 +481,59 @@ def coerces(self, rhs: Base, check_quant: bool = True) -> bool:
if isinstance(rhs, Any):
return self._check_optional(rhs, check_quant)
return False


def unify(types: List[Base], check_quant: bool = True, force_string: bool = False,) -> Base:
"""
Given a list of types, compute a type to which they're all coercible, or :class:`WDL.Type.Any`
if no more-specific inference is possible.
:param force_string: permit last-resort unification to ``String`` even if no item is currently
a ``String``, but all can be coerced
"""
if not types:
return Any()

# begin with first type; or if --no-quant-check, the first array type (as we can try to promote
# other T to Array[T])
t = types[0]
if not check_quant:
t = next((a for a in types if isinstance(a, Array)), t)

# potentially promote/generalize t to other types seen
optional = False
all_nonempty = True
all_stringifiable = True
for t2 in types:
if isinstance(t, Int) and isinstance(t2, Float):
t = Float()
if isinstance(t, String) and isinstance(t2, File):
t = File()

if (
isinstance(t2, String)
and not isinstance(t2, File)
and not isinstance(t, File)
and (not check_quant or not isinstance(t, Array))
):
t = String()
if not t2.coerces(String(optional=True), check_quant=check_quant):
all_stringifiable = False

if t2.optional:
optional = True
if isinstance(t2, Array) and not t2.nonempty:
all_nonempty = False

if isinstance(t, Array):
t = t.copy(nonempty=all_nonempty)
t = t.copy(optional=optional)

# check all types are coercible to t
for t2 in types:
if not t2.coerces(t, check_quant=check_quant):
if all_stringifiable and force_string:
return String(optional=optional)
return Any()

return t
35 changes: 31 additions & 4 deletions WDL/Value.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -362,10 +362,17 @@ def children(self) -> Iterable[Base]:

def from_json(type: Type.Base, value: Any) -> Base:
"""
Instantiate a WDL value of the specified type from a parsed JSON value (str, int, float, list, dict, or null).
Instantiate a WDL value of the specified type from a parsed JSON value (str, int, float, list,
dict, or null).
If type is :class:`WDL.Type.Any`, attempts to infer a WDL type & value from the JSON's
intrinsic types. This isn't ideal; for example, Files can't be distinguished from Strings, and
JSON lists and dicts with heterogeneous item types may give undefined results.
:raise WDL.Error.InputError: if the given value isn't coercible to the specified type
"""
if isinstance(type, Type.Any):
return _infer_from_json(value)
if isinstance(type, (Type.Boolean, Type.Any)) and value in [True, False]:
return Boolean(value)
if isinstance(type, (Type.Int, Type.Any)) and isinstance(value, int):
Expand Down Expand Up @@ -401,6 +408,26 @@ def from_json(type: Type.Base, value: Any) -> Base:
return Struct(Type.Object(type.members), items)
if type.optional and value is None:
return Null()
raise Error.InputError(
"couldn't construct {} from input {}".format(str(type), json.dumps(value))
)
raise Error.InputError(f"couldn't construct {str(type)} from: {json.dumps(value)}")


def _infer_from_json(j: Any) -> Base:
if isinstance(j, str):
return String(j)
if isinstance(j, bool):
return Boolean(j)
if isinstance(j, int):
return Int(j)
if isinstance(j, float):
return Float(j)
if j is None:
return Null()
if isinstance(j, list):
items = [_infer_from_json(v) for v in j]
item_type = Type.unify([item.type for item in items])
return Array(item_type, [item.coerce(item_type) for item in items])
if isinstance(j, dict):
items = [(String(str(k)), _infer_from_json(j[k])) for k in j]
value_type = Type.unify([v.type for _, v in items])
return Map((Type.String(), value_type), [(k, v.coerce(value_type)) for k, v in items])
raise Error.InputError(f"couldn't construct value from: {json.dumps(j)}")
4 changes: 2 additions & 2 deletions tests/test_0eval.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,7 @@ def test_array(self):
("[]","[]", WDL.Type.Array(WDL.Type.Any())),
("[] == []","true"),
("[1, false]", '["1", "false"]', WDL.Type.Array(WDL.Type.String(), nonempty=True)),
("[1, {}]", "(Ln 1, Col 1) Expected Int instead of Boolean; inconsistent types within array", WDL.Error.StaticTypeMismatch),
("[1, {}]", "(Ln 1, Col 1) Expected Int instead of Boolean; inconsistent types within array", WDL.Error.IndeterminateType),
("1 + 2[3]", "(Ln 1, Col 5) Not an array", WDL.Error.NotAnArray),
("[1, 2, 3][true]", "(Ln 1, Col 11) Expected Int instead of Boolean; Array index", WDL.Error.StaticTypeMismatch),
("[1, 2, 3][4]", "(Ln 1, Col 11) Array index out of bounds", WDL.Error.OutOfBounds)
Expand Down Expand Up @@ -330,7 +330,7 @@ def test_map(self):
("{0: 1, 2: 3}[false]", "", WDL.Error.StaticTypeMismatch),
("{0: 1, 2: 3}['foo']", "", WDL.Error.EvalError),
("{'foo': 1, 'bar': 2}[3]", "", WDL.Error.OutOfBounds), # int coerces to string...
("{3: 1, false: 2}", "", WDL.Error.StaticTypeMismatch),
("{3: 1, false: 2}", "", WDL.Error.IndeterminateType),
("{'foo': true, 'bar': 0,}", '{"foo": true, "bar": 0}', WDL.Type.Map((WDL.Type.String(), WDL.Type.String())))
)

Expand Down
5 changes: 5 additions & 0 deletions tests/test_5stdlib.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -494,6 +494,7 @@ def test_read_json(self):
echo 3.14159 > float.json
echo true > bool.json
echo null > null.json
echo '{"out": ["Element 1", "Element 2"]}' > out.txt
>>>
output {
Map[String,String] map = read_json("object.json")
Expand All @@ -506,6 +507,8 @@ def test_read_json(self):
# issue #320
String baz1 = read_json("object.json")["bas"]
Int three = read_json("list.json")[2]
Array[String] out1 = read_json('out.txt')["out"]
String out2 = read_json('out.txt')["out"][1]
}
}
""")
Expand All @@ -517,6 +520,8 @@ def test_read_json(self):
self.assertEqual(outputs["null"], None)
self.assertEqual(outputs["baz1"], "baz")
self.assertEqual(outputs["three"], 3)
self.assertEqual(outputs["out1"], ["Element 1", "Element 2"])
self.assertEqual(outputs["out2"], "Element 2")

outputs = self._test_task(R"""
version 1.0
Expand Down

0 comments on commit 300b7cc

Please sign in to comment.