lex string interpolation corner cases like $${x} and ~~{x}

chanzuckerberg · Feb 15, 2019 · 17511f4 · 17511f4
1 parent 5d43eba
commit 17511f4
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 20 deletions.
diff --git a/WDL/_parser.py b/WDL/_parser.py
@@ -179,22 +179,22 @@
 # - workflow outputs can be bare identifiers rather than complete decls
 productions_pre_1_0 = r"""
 // string (single-quoted)
-STRING1_CHAR: "\\'" | /[^'$]/ | /\$[^{']/
+STRING1_CHAR: "\\'" | /[^'$]/ | /\$[^{$']/
 STRING1_FRAGMENT: STRING1_CHAR+
-string1: /'/ (STRING1_FRAGMENT? "${" expr "}")* STRING1_FRAGMENT? /\$/? /'/ -> string
+string1: /'/ (STRING1_FRAGMENT? /\$/* "${" expr "}")* STRING1_FRAGMENT? /\$/* /'/ -> string
 
 // string (double-quoted)
-STRING2_CHAR: "\\\"" | /[^"$]/ | /\$[^{"]/
+STRING2_CHAR: "\\\"" | /[^"$]/ | /\$[^{$"]/
 STRING2_FRAGMENT: STRING2_CHAR+
-string2: /"/ (STRING2_FRAGMENT? "${" expr "}")* STRING2_FRAGMENT? /\$/? /"/ -> string
+string2: /"/ (STRING2_FRAGMENT? /\$/* "${" expr "}")* STRING2_FRAGMENT? /\$/* /"/ -> string
 
-COMMAND1_CHAR: /[^$}]/ | /\$[^{]/
+COMMAND1_CHAR: /[^$}]/ | /\$[^{$]/
 COMMAND1_FRAGMENT: COMMAND1_CHAR+
-command1: "command" "{" (COMMAND1_FRAGMENT? "${" placeholder "}")* COMMAND1_FRAGMENT? /\$/? "}" -> command
+command1: "command" "{" (COMMAND1_FRAGMENT? /\$/* "${" placeholder "}")* COMMAND1_FRAGMENT? /\$/* "}" -> command
 
-COMMAND2_CHAR: /[^$>]/ | /\$[^{]/ | />[^>]/ | />>[^>]/
+COMMAND2_CHAR: /[^$>]/ | /\$[^{$]/ | />[^>]/ | />>[^>]/
 COMMAND2_FRAGMENT: COMMAND2_CHAR+
-command2: "command" "<<<" (COMMAND2_FRAGMENT? "${" placeholder "}")* COMMAND2_FRAGMENT? /\$/? ">>>" -> command
+command2: "command" "<<<" (COMMAND2_FRAGMENT? /\$/* "${" placeholder "}")* COMMAND2_FRAGMENT? /\$/* ">>>" -> command
 
 ?workflow_outputs: "output" "{" workflow_output_decls "}"
 workflow_output_decls: workflow_output_decl*
@@ -207,25 +207,25 @@
 # - within <<< >>> commands, placeholders are delimited by ~{ } only
 # - workflow outputs are complete decls
 productions_1_0 = r"""
-_EITHER_DELIM: "${" | "~{"
+_EITHER_DELIM.2: "~{" | "${"
 
 // string (single-quoted)
-STRING1_CHAR: "\\'" | /[^'~$]/ | /\$[^{']/ | /~[^{']/
+STRING1_CHAR: "\\'" | /[^'~$]/ | /\$[^{$~']/ | /\~[^{$~']/
 STRING1_FRAGMENT: STRING1_CHAR+
-string1: /'/ (STRING1_FRAGMENT? _EITHER_DELIM expr "}")* STRING1_FRAGMENT? /\$/? /~/? /'/ -> string
+string1: /'/ (STRING1_FRAGMENT? /\$/* /\~/* _EITHER_DELIM expr "}")* STRING1_FRAGMENT? /\$/* /\~/* /'/ -> string
 
 // string (double-quoted)
-STRING2_CHAR: "\\\"" | /[^"~$]/ | /\$[^{"]/ | /~[^{"]/
+STRING2_CHAR: "\\\"" | /[^"~$]/ | /\$[^{$~"]/ | /~[^{$~"]/
 STRING2_FRAGMENT: STRING2_CHAR+
-string2: /"/ (STRING2_FRAGMENT? _EITHER_DELIM expr "}")* STRING2_FRAGMENT? /\$/? /~/? /"/ -> string
+string2: /"/ (STRING2_FRAGMENT? /\$/* /\~/* _EITHER_DELIM expr "}")* STRING2_FRAGMENT? /\$/* /\~/* /"/ -> string
 
-COMMAND1_CHAR: /[^~$}]/ | /\$[^{]/ | /~[^{]/
+COMMAND1_CHAR: /[^~$}]/ | /\$[^{$~]/ | /~[^{$~]/
 COMMAND1_FRAGMENT: COMMAND1_CHAR+
-command1: "command" "{" (COMMAND1_FRAGMENT? _EITHER_DELIM placeholder "}")* COMMAND1_FRAGMENT? /\$/? /~/? "}" -> command
+command1: "command" "{" (COMMAND1_FRAGMENT? /\$/* /\~/* _EITHER_DELIM placeholder "}")* COMMAND1_FRAGMENT? /\$/* /\~/* "}" -> command
 
-COMMAND2_CHAR: /[^~>]/ | /~[^{]/ | />[^>]/ | />>[^>]/
+COMMAND2_CHAR: /[^~>]/ | /~[^{~]/ | />[^>]/ | />>[^>]/
 COMMAND2_FRAGMENT: COMMAND2_CHAR+
-command2: "command" "<<<" (COMMAND2_FRAGMENT? "~{" placeholder "}")* COMMAND2_FRAGMENT? /~/? ">>>" -> command
+command2: "command" "<<<" (COMMAND2_FRAGMENT? /\~/? "~{" placeholder "}")* COMMAND2_FRAGMENT? /\~/* ">>>" -> command
 
 ?workflow_outputs: output_decls
 """

diff --git a/tests/test_0eval.py b/tests/test_0eval.py
@@ -55,8 +55,11 @@ def _test_tuples(self, *tuples):
                 with self.assertRaises(exn, msg=expected):
                     x = WDL.parse_expr(expr, version=version).infer_type(type_env).eval(env)
             else:
-                v = WDL.parse_expr(expr, version=version).infer_type(type_env).eval(env).expect(expected_type)
-                self.assertEqual(str(v), expected)
+                try:
+                    v = WDL.parse_expr(expr, version=version).infer_type(type_env).eval(env).expect(expected_type)
+                    self.assertEqual(str(v), expected)
+                except:
+                    assert False, str(expr)
 
     def test_logic(self):
         self._test_tuples(
@@ -232,6 +235,16 @@ def test_interpolation(self):
             ("'${pi} ~{pi}$'", '"3.14159 ~{pi}$"', env, "draft-2"),
             ('"${pi} ~{pi}$"', '"3.14159 3.14159$"', env, "1.0"),
             ("'${pi} ~{pi}~'", '"3.14159 3.14159~"', env, "1.0"),
+            ("'$${pi}$'", '"$3.14159$"', env, "draft-2"),
+            ('"$${pi}$$"', '"$3.14159$$"', env, "draft-2"),
+            ("'$${pi}$'", '"$3.14159$"', env, "1.0"),
+            ("'$${pi}$$'", '"$3.14159$$"', env, "1.0"),
+            ("'$$${pi}~'", '"$$3.14159~"', env, "1.0"),
+            ("'~~{pi}~'", '"~3.14159~"', env, "1.0"),
+            ('"~~{pi}~"', '"~3.14159~"', env, "1.0"),
+            ("'~~${pi}~'", '"~~3.14159~"', env, "1.0"),
+            ("'$~{pi}~~'", '"$3.14159~~"', env, "1.0"),
+            ("'$~${pi}~~'", '"$~3.14159~~"', env, "1.0"),
         )
 
     def test_pair(self):

diff --git a/tests/test_4corpi.py b/tests/test_4corpi.py
@@ -56,7 +56,7 @@ class HCAskylab_task(unittest.TestCase):
 @test_corpus(
     ["test_corpi/HumanCellAtlas/skylab/pipelines/**"],
     path=[["test_corpi/HumanCellAtlas/skylab/library/tasks"]],
-    expected_lint={'UnusedDeclaration': 15, 'NameCollision': 1, 'StringCoercion': 4, 'FileCoercion': 1}
+    expected_lint={'UnusedDeclaration': 10, 'NameCollision': 1, 'StringCoercion': 4, 'FileCoercion': 1}
 )
 class HCAskylab_workflow(unittest.TestCase):
     pass