From 9bb344bf83623559ef35dcfa1b259905a41d2342 Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Wed, 12 Jun 2019 15:07:39 -0700 Subject: [PATCH 1/4] add biowdl/tasks test corpus - parser: spike in newline at end of input, to accommodate last-line comments - UnusedDeclaration: make heuiristic index files test case-insensitive - file issue for preoutput declarations --- .gitmodules | 3 +++ WDL/Lint.py | 4 ++-- WDL/_parser.py | 2 +- test_corpi/biowdl/tasks | 1 + tests/test_3corpi.py | 15 +++++++++++++++ 5 files changed, 22 insertions(+), 3 deletions(-) create mode 160000 test_corpi/biowdl/tasks diff --git a/.gitmodules b/.gitmodules index 6eaaea4f..a1e9fe57 100644 --- a/.gitmodules +++ b/.gitmodules @@ -43,3 +43,6 @@ [submodule "tests/bash-tap"] path = tests/bash-tap url = https://github.com/illusori/bash-tap.git +[submodule "test_corpi/biowdl/tasks"] + path = test_corpi/biowdl/tasks + url = https://github.com/biowdl/tasks.git diff --git a/WDL/Lint.py b/WDL/Lint.py index e25b663d..df4071ce 100644 --- a/WDL/Lint.py +++ b/WDL/Lint.py @@ -654,12 +654,12 @@ def decl(self, obj: WDL.Tree.Decl) -> Any: if not ( ( isinstance(obj.type, WDL.Type.File) - and sum(1 for sfx in index_suffixes if obj.name.endswith(sfx)) + and sum(1 for sfx in index_suffixes if obj.name.lower().endswith(sfx)) ) or ( isinstance(obj.type, WDL.Type.Array) and isinstance(obj.type.item_type, WDL.Type.File) - and sum(1 for sfx in index_suffixes if obj.name.endswith(sfx)) + and sum(1 for sfx in index_suffixes if obj.name.lower().endswith(sfx)) ) or ( isinstance(pt, WDL.Tree.Task) diff --git a/WDL/_parser.py b/WDL/_parser.py index 04dde4ed..2bdfe83a 100644 --- a/WDL/_parser.py +++ b/WDL/_parser.py @@ -262,7 +262,7 @@ def parse(txt: str, start: str, version: Optional[str] = None) -> lark.Tree: _lark_cache[(version, start)] = lark.Lark( _grammar_for_version(version), start=start, parser="lalr", propagate_positions=True ) - return _lark_cache[(version, start)].parse(txt) + return _lark_cache[(version, start)].parse(txt + ("\n" if not txt.endswith("\n") else "")) def sp(filename, meta) -> SourcePosition: diff --git a/test_corpi/biowdl/tasks b/test_corpi/biowdl/tasks new file mode 160000 index 00000000..46385ddc --- /dev/null +++ b/test_corpi/biowdl/tasks @@ -0,0 +1 @@ +Subproject commit 46385ddc0eca2583dca3d9261fbc2788303f78d6 diff --git a/tests/test_3corpi.py b/tests/test_3corpi.py index fd6d71f1..e392f7cf 100644 --- a/tests/test_3corpi.py +++ b/tests/test_3corpi.py @@ -200,3 +200,18 @@ class Contrived(unittest.TestCase): ) class Contrived2(unittest.TestCase): pass + +@test_corpus( + ["test_corpi/biowdl/tasks/**"], + expected_lint={'OptionalCoercion': 3, 'NonemptyCoercion': 1, 'UnnecessaryQuantifier': 3, 'UnusedDeclaration': 6}, + check_quant=False, + blacklist=[ + # use Object + "common", "bamstats", "seqstat", "flash", "sampleconfig", "strelka", + "stringtie", "vardict", "manta", "somaticseq", "biopet", + # preoutput declarations (FIXME) + "gffcompare", "multiqc", + ], +) +class BioWDLTasks(unittest.TestCase): + pass From 0c38dbc03de2a4968b05cccc76c2bfdf2eed5f3b Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Wed, 12 Jun 2019 23:31:33 -0700 Subject: [PATCH 2/4] allow workflow outputs to reference previous workflow outputs --- WDL/Tree.py | 5 ++++- tests/test_1doc.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/WDL/Tree.py b/WDL/Tree.py index db37423f..5dd8be7e 100644 --- a/WDL/Tree.py +++ b/WDL/Tree.py @@ -825,6 +825,8 @@ def typecheck(self, doc: TVDocument, check_quant: bool) -> None: # 5. typecheck the output expressions if self.outputs: output_names = set() + output_type_env = self._type_env + assert output_type_env for output in self.outputs: assert output.expr if output.name in output_names: @@ -835,10 +837,11 @@ def typecheck(self, doc: TVDocument, check_quant: bool) -> None: ) errors.try1( lambda output=output: output.typecheck( - self._type_env, check_quant=check_quant + output_type_env, check_quant=check_quant ) ) output_names.add(output.name) + output_type_env = Env.bind(output_type_env, [], output.name, output.type) # 6. check for cyclic dependencies WDL._util.detect_cycles(_dependency_matrix(_decls_and_calls(self))) # pyre-fixme diff --git a/tests/test_1doc.py b/tests/test_1doc.py index f4b0fad5..a02ddfd7 100644 --- a/tests/test_1doc.py +++ b/tests/test_1doc.py @@ -1187,7 +1187,9 @@ def test_workflow_inputs(self): y = y } output { - Int z = z + Int z = z+1 + Int w = x+y + Array[Int] outs = [z,w] } } """ From 37d27f054e7863def6d34b35db61be01a3832681 Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Thu, 13 Jun 2019 12:00:26 -0700 Subject: [PATCH 3/4] allow task 'non-input' declarations to interleave anywhere among other sections --- WDL/_parser.py | 31 +++++++++++++++---------------- tests/test_3corpi.py | 4 +--- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/WDL/_parser.py b/WDL/_parser.py index 2bdfe83a..78334132 100644 --- a/WDL/_parser.py +++ b/WDL/_parser.py @@ -61,16 +61,12 @@ // WDL tasks input_decls: "input" "{" any_decl* "}" output_decls: "output" "{" bound_decl* "}" -?task_sections1: input_decls - | output_decls - | meta_section - | runtime_section - | any_decl+ -> noninput_decls -?task_sections2: input_decls - | output_decls - | meta_section - | runtime_section -task: "task" CNAME "{" task_sections1* command task_sections2* "}" +?task_section: input_decls + | output_decls + | meta_section + | runtime_section + | any_decl -> noninput_decl +task: "task" CNAME "{" task_section* command task_section* "}" tasks: task* @@ -484,8 +480,8 @@ def decl(self, items, meta): def input_decls(self, items, meta): return {"inputs": items} - def noninput_decls(self, items, meta): - return {"decls": items} + def noninput_decl(self, items, meta): + return {"noninput_decl": items[0]} def placeholder_option(self, items, meta): assert len(items) == 2 @@ -547,15 +543,18 @@ def runtime_section(self, items, meta): return {"runtime": d} def task(self, items, meta): - d = {} + d = {"noninput_decls": []} for item in items: if isinstance(item, dict): for k, v in item.items(): - if k in d: + if k == "noninput_decl": + d["noninput_decls"].append(v) + elif k in d: raise Err.MultipleDefinitions( sp(self.filename, meta), "redundant sections in task" ) - d[k] = v + else: + d[k] = v else: assert isinstance(item, str) assert "name" not in d @@ -565,7 +564,7 @@ def task(self, items, meta): sp(self.filename, meta), d["name"], d.get("inputs", None), - d.get("decls", []), + d["noninput_decls"], d["command"], d.get("outputs", []), d.get("parameter_meta", {}), diff --git a/tests/test_3corpi.py b/tests/test_3corpi.py index e392f7cf..53536a55 100644 --- a/tests/test_3corpi.py +++ b/tests/test_3corpi.py @@ -203,14 +203,12 @@ class Contrived2(unittest.TestCase): @test_corpus( ["test_corpi/biowdl/tasks/**"], - expected_lint={'OptionalCoercion': 3, 'NonemptyCoercion': 1, 'UnnecessaryQuantifier': 3, 'UnusedDeclaration': 6}, + expected_lint={'OptionalCoercion': 4, 'NonemptyCoercion': 1, 'UnnecessaryQuantifier': 3, 'UnusedDeclaration': 9}, check_quant=False, blacklist=[ # use Object "common", "bamstats", "seqstat", "flash", "sampleconfig", "strelka", "stringtie", "vardict", "manta", "somaticseq", "biopet", - # preoutput declarations (FIXME) - "gffcompare", "multiqc", ], ) class BioWDLTasks(unittest.TestCase): From c48fef3dfad4c87baeba72dc16e925f09e15a4b6 Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Thu, 13 Jun 2019 16:26:57 -0700 Subject: [PATCH 4/4] couple of error stack suppressions --- WDL/Tree.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WDL/Tree.py b/WDL/Tree.py index 5dd8be7e..7c72d8ab 100644 --- a/WDL/Tree.py +++ b/WDL/Tree.py @@ -1390,7 +1390,7 @@ def _resolve_struct_typedef( try: struct_typedef = Env.resolve(struct_typedefs, [], ty.type_name) except KeyError: - raise Err.InvalidType(pos, "Unknown type " + ty.type_name) + raise Err.InvalidType(pos, "Unknown type " + ty.type_name) from None ty.members = struct_typedef.members @@ -1421,7 +1421,7 @@ def _initialize_struct_typedefs(struct_typedefs: Env.StructTypeDefs): try: _resolve_struct_typedefs(b.rhs.pos, member_ty, struct_typedefs) except StopIteration: - raise Err.CircularDependencies(b.rhs) + raise Err.CircularDependencies(b.rhs) from None def _add_struct_instance_to_type_env(