Merge e1ba483 into 78172bb

.pyre_configuration

@@ -5,5 +5,6 @@
   ],
   "search_path": [
     "."
-  ]
+  ],
+  "workers": 4
 }

Makefile

@@ -9,7 +9,7 @@ qtest:
 	python3 -m unittest -v -f
 
 check:
-	pylint --errors-only WDL
+	pylint -j `nproc` --errors-only WDL
 	pyre \
 		--search-path $(HOME)/.local/lib/python3.6/site-packages/lark \
 		--typeshed $(HOME)/.local/lib/pyre_check/typeshed \

WDL/Env.py

@@ -2,7 +2,7 @@
 """
 Environments, for identifier resolution during WDL typechecking and evaluation.
 """
-from typing import List, TypeVar, Generic
+from typing import List, TypeVar, Generic, Any
 import WDL.Type as T
 import WDL.Value as V
 
@@ -33,9 +33,13 @@ class Binding(Generic[R]):
     rhs: R
     """:type: Union[WDL.Type.Base,WDL.Value.Base]"""
 
-    def __init__(self, name: str, rhs: R) -> None:
+    ctx: Any
+    "Arbitrary, secondary context also associated with name"
+
+    def __init__(self, name: str, rhs: R, ctx: Any = None) -> None:
         self.name = name
         self.rhs = rhs
+        self.ctx = ctx
 
 
 class Namespace(Generic[R]):
@@ -63,9 +67,9 @@ def __init__(self, namespace: str, bindings: "Tree[R]") -> None:
 """Environment of values, an immutable list of bindings to values and/or namespaces"""
 
 
-def bind(name: str, rhs: R, tree: "Tree[R]") -> "Tree[R]":
+def bind(name: str, rhs: R, tree: "Tree[R]", ctx: Any = None) -> "Tree[R]":
     """Prepend a new binding to an environment"""
-    return [Binding(name, rhs)] + tree
+    return [Binding(name, rhs, ctx)] + tree
 
 
 def namespace(namespace: str, bindings: "Tree[R]", tree: "Tree[R]") -> "Tree[R]":
@@ -83,14 +87,29 @@ def resolve_namespace(tree: "Tree[R]", namespace: List[str]) -> R:
     raise KeyError()
 
 
-def resolve(tree: "Tree[R]", namespace: List[str], name: str) -> R:
-    """Resolve a name within an environment"""
+def resolve_binding(tree: "Tree[R]", namespace: List[str], name: str) -> Binding[R]:
+    """
+    Resolve a name within an environment to the corresponding Binding object
+    """
     ns = resolve_namespace(tree, namespace)
     for node in ns:
         if isinstance(node, Binding) and node.name == name:
-            return node.rhs
+            ans: Binding[R] = node
+            return ans
     raise KeyError()
 
 
+def resolve(tree: "Tree[R]", namespace: List[str], name: str) -> R:
+    """Resolve a name within an environment"""
+    ans: R = resolve_binding(tree, namespace, name).rhs
+    return ans
+
+
+def resolve_ctx(tree: "Tree[R]", namespace: List[str], name: str) -> Any:  # pyre-ignore
+    """Resolve a name to its secondary context value"""
+    ans: Any = resolve_binding(tree, namespace, name).ctx
+    return ans
+
+
 # print(arrayize([Binding('x',T.Int())])[0].rhs)
 # assert resolve([Binding('x',T.Int())], [], 'x') == T.Int()

WDL/Expr.py

@@ -9,7 +9,7 @@
 a suitable Env.
 """
 from abc import ABC, abstractmethod
-from typing import List, Optional, Dict, TypeVar, Tuple, Union
+from typing import List, Optional, Dict, TypeVar, Tuple, Union, Any
 import WDL.Type as T
 import WDL.Value as V
 import WDL.Env as Env
@@ -532,6 +532,16 @@ class Ident(Base):
     name: str
     ":type: str"
 
+    ctx: Any = None
+    """
+    After typechecking, stores context about the binding from the type
+    environment.
+
+    The ``Tree`` typechecker typically stores here a reference to a ``Decl``
+    (for value references in tasks and workflows), a ``Call`` (for references
+    to a call output), or a ``Scatter`` (for references to a scatter variable).
+    """
+
     def __init__(self, pos: SourcePosition, parts: List[str]) -> None:
         super().__init__(pos)
         assert parts
@@ -555,9 +565,10 @@ def _infer_type(self, type_env: Env.Types) -> T.Base:
                 return ans.left_type if self.name == "left" else ans.right_type
         try:
             ans: T.Base = Env.resolve(type_env, self.namespace, self.name)
-            return ans
         except KeyError:
             raise Error.UnknownIdentifier(self) from None
+        self.ctx = Env.resolve_ctx(type_env, self.namespace, self.name)
+        return ans
 
     def eval(self, env: Env.Values) -> V.Base:
         ""

WDL/Lint.py

@@ -299,3 +299,21 @@ def document(self, obj: WDL.Document) -> Any:
             if not any_called:
                 self.add(obj, "nothing used from the import " + namespace)
         super().document(obj)
+
+
+@a_linter
+class ForwardReference(Linter):
+    def expr(self, obj: WDL.Expr.Base) -> Any:
+        if (
+            isinstance(obj, WDL.Expr.Ident)
+            and isinstance(obj.ctx, (WDL.Decl, WDL.Call))
+            and (
+                obj.ctx.pos.line > obj.pos.line
+                or (obj.ctx.pos.line == obj.pos.line and obj.ctx.pos.column > obj.pos.column)
+            )
+        ):
+            msg = "identifier lexically precedes the referenced declaration"
+            if isinstance(obj.ctx, WDL.Call):
+                msg = "reference to call output lexically precedes the call"
+            self.add(getattr(obj, "parent"), msg, obj)
+        super().expr(obj)

WDL/Tree.py

@@ -61,7 +61,7 @@ def add_to_type_env(self, type_env: Env.Types) -> Env.Types:
         ty = self.type
         if isinstance(self.expr, (E.Int, E.Float, E.String, E.Array, E.Pair, E.Map)):
             ty = ty.copy(optional=False)
-        ans: Env.Types = Env.bind(self.name, ty, type_env)
+        ans: Env.Types = Env.bind(self.name, ty, type_env, ctx=self)
         return ans
 
     def typecheck(self, type_env: Env.Types) -> None:
@@ -255,7 +255,7 @@ def add_to_type_env(self, type_env: Env.Types) -> Env.Types:
             pass
         outputs_env = []
         for outp in self.callee.outputs:
-            outputs_env = Env.bind(outp.name, outp.type, outputs_env)
+            outputs_env = Env.bind(outp.name, outp.type, outputs_env, ctx=self)
         return Env.namespace(self.name, outputs_env, type_env)
 
     def typecheck_input(self, type_env: Env.Types, doc: TVDocument) -> None:
@@ -625,7 +625,7 @@ def _build_workflow_type_env(
             )
         except KeyError:
             pass
-        type_env = Env.bind(self.variable, self.expr.type.item_type, type_env)
+        type_env = Env.bind(self.variable, self.expr.type.item_type, type_env, ctx=self)
     elif isinstance(self, Conditional):
         # typecheck the condition
         self.expr.infer_type(type_env)

WDL/Walker.py

@@ -9,7 +9,7 @@ class Base:
     scatter, conditional, decl, task). The base implementations of these
     methods recurse into the node's "children." Overriding subclasses can thus
     invoke their super at the appropriate point for preorder or postorder
-    traversal.
+    traversal (or omit super to prevent further descent).
 
     ``
     class PrintUnconditionalCallNames(Walker.Base):

tests/test_3corpi.py

@@ -82,7 +82,7 @@ class gatk4_germline_snps_indels(unittest.TestCase):
 
 @test_corpus(
     ["test_corpi/gatk-workflows/gatk4-somatic-snvs-indels/**"],
-    expected_lint={'StringCoercion': 20},
+    expected_lint= {'ForwardReference': 3, 'StringCoercion': 20},
 )
 class gatk4_somatic_snvs_indels(unittest.TestCase):
     pass