Fixing tests, adding grammar file, updating generation script

.travis.yml

@@ -3,4 +3,4 @@ python:
   - "2.7"
   - "3.5"
 install: "python setup.py install"
-script: py.test
+script: py.test wdl/test

generate.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
-hermes generate ../grammar.hgr --name=wdl --directory=wdl --header
+hermes generate grammar.hgr --name=wdl --directory=wdl --header
 mv wdl/wdl_parser.py wdl/parser.py

grammar.hgr

@@ -0,0 +1,344 @@
+grammar {
+  lexer {
+    partials {
+      r'[a-zA-Z]' -> _identifier_start
+      r'[a-zA-Z0-9_]' -> _identifier_follow
+      r'(Array|Map|Object|Boolean|Int|Float|Uri|File|String)(?!{%_identifier_follow%})' -> _type
+      r'{%_identifier_start%}({%_identifier_follow%})*' -> _identifier
+    }
+
+    r'\s+' -> null
+
+    enum {
+      python: r'/\*(.*?)\*/' (DOTALL)
+      c: r'/\*(.*?)\*/' (PCRE_DOTALL)
+      java: r'/\*(.*?)\*/' (DOTALL)
+      javascript: r'/\*(.*?)\*/' (m)
+    } -> null
+
+    r'#.*' -> null
+    r'task(?!{%_identifier_follow%})' -> task(:task)
+    r'(call)\s+' -> :call[1] @task_fqn
+    r'workflow(?!{%_identifier_follow%})' -> workflow(:workflow)
+    r'import(?!{%_identifier_follow%})' -> :import
+    r'input(?!{%_identifier_follow%})' -> :input
+    r'output(?!{%_identifier_follow%})' -> output(:output)
+    r'as(?!{%_identifier_follow%})' -> :as
+    r'if(?!{%_identifier_follow%})' -> :if
+    r'while(?!{%_identifier_follow%})' -> :while
+    r'runtime(?!{%_identifier_follow%})' -> :runtime
+    r'scatter(?!{%_identifier_follow%})' -> :scatter @scatter
+    r'command\s*(?=<<<)' -> :raw_command  @raw_command2
+    r'command\s*(?=\{)' -> :raw_command  @raw_command
+    r'parameter_meta(?!{%_identifier_follow%})' -> :parameter_meta
+    r'meta(?!{%_identifier_follow%})' -> :meta
+    r'(true|false)(?!{%_identifier_follow%})' -> :boolean
+    r'(object)\s*(\{)' -> :object :lbrace
+    r'{%_type%}(?!{%_identifier_follow%})' -> :type
+    r'{%_identifier%}' -> :identifier
+
+    # TODO: these should probably be partials...
+    enum {
+      python: r'"([^\\\"\n]|\\[\\"\'nrbtfav\?]|\\[0-7]{1,3}|\\x[0-9a-fA-F]+|\\[uU]([0-9a-fA-F]{4})([0-9a-fA-F]{4})?)*"'
+      java: "\\\"(?>[^\\\\\\\"\\n]|\\\\[\\\"\\'nrbtfav\\\\?]|\\\\[0-7]{1,3}|\\\\x[0-9a-fA-F]+|\\\\[uU]([0-9a-fA-F]{4})([0-9a-fA-F]{4})?)*\\\""
+    } -> unescape(:string)
+    enum {
+      python: r'\'([^\\\'\n]|\\[\\"\'nrbtfav\?]|\\[0-7]{1,3}|\\x[0-9a-fA-F]+|\\[uU]([0-9a-fA-F]{4})([0-9a-fA-F]{4})?)*\''
+      java: "'(?>[^\\\\\\'\\n]|\\\\[\\\"\\'nrbtfav\\\\?]|\\\\[0-7]{1,3}|\\\\x[0-9a-fA-F]+|\\\\[uU]([0-9a-fA-F]{4})([0-9a-fA-F]{4})?)*'"
+    } -> unescape(:string)
+
+    r':' -> :colon
+    r',' -> :comma
+    r'==' -> :double_equal
+    r'\|\|' -> :double_pipe
+    r'\&\&' -> :double_ampersand
+    r'!=' -> :not_equal
+    r'=' -> :equal
+    r'\.' -> :dot
+    r'\{' -> :lbrace
+    r'\}' -> :rbrace
+    r'\(' -> :lparen
+    r'\)' -> :rparen
+    r'\[' -> :lsquare
+    r'\]' -> :rsquare
+    r'\+' -> :plus
+    r'\*' -> :asterisk
+    r'-' -> :dash
+    r'/' -> :slash
+    r'%' -> :percent
+    r'<=' -> :lteq
+    r'<' -> :lt
+    r'>=' -> :gteq
+    r'>' -> :gt
+    r'!' -> :not
+    enum {
+      python: r'\?'
+      java: "\\?"
+    } -> :qmark
+    r'-?[0-9]+\.[0-9]+' -> :float
+    r'[0-9]+' -> :integer
+    mode<wf_output> {
+      r'\s+' -> null
+      r'\{' -> :lbrace
+      r'\}' -> :rbrace %pop
+      r',' -> :comma
+      r'\.' -> :dot
+      r'\*' -> :asterisk
+      enum {
+        python: r'{%_identifier%}(\.{%_identifier%})*'
+        java: "{%_identifier%}(\\.{%_identifier%})*"
+      } -> :fqn
+    }
+    mode<task_fqn> {
+      r'\s+' -> null
+      enum {
+        python: r'{%_identifier%}(\.{%_identifier%})*'
+        java: "{%_identifier%}(\\.{%_identifier%})*"
+      } -> :fqn %pop
+    }
+    mode<scatter> {
+      r'\s+' -> null
+      r'\)' -> :rparen %pop
+      r'\(' -> :lparen
+      r'\.' -> :dot
+      r'\[' -> :lsquare
+      r'\]' -> :rsquare
+      r'in(?!{%_identifier_follow%})' -> :in
+      r'{%_identifier%}' -> :identifier
+    }
+    mode<raw_command> {
+      r'\{' -> :raw_cmd_start
+      r'\}' -> :raw_cmd_end %pop
+      r'\$\{' -> :cmd_param_start @cmd_param
+      enum {
+        python: r'(.*?)(?=\$\{|\})' (DOTALL)
+        c: r'(.*?)(?=\$\{|\})' (PCRE_DOTALL)
+        java: r'(.*?)(?=\$\{|\})' (DOTALL)
+        javascript: r'(.*?)(?=\$\{|\})' (m)
+      } -> :cmd_part
+    }
+    mode<raw_command2> {
+      r'<<<' -> :raw_cmd_start
+      r'>>>' -> :raw_cmd_end %pop
+      r'\$\{' -> :cmd_param_start @cmd_param
+      enum {
+        python: r'(.*?)(?=\$\{|>>>)' (DOTALL)
+        c: r'(.*?)(?=\$\{|>>>)' (PCRE_DOTALL)
+        java: r'(.*?)(?=\$\{|>>>)' (DOTALL)
+        javascript: r'(.*?)(?=\$\{|>>>)' (m)
+      } -> :cmd_part
+    }
+    mode<cmd_param> {
+      r'\s+' -> null
+      r'\}' -> :cmd_param_end %pop
+      r'\[' -> :lsquare
+      r'\]' -> :rsquare
+      r'=' -> :equal
+      r'\+' -> :plus
+      r'\*' -> :asterisk
+      r'[0-9]+' -> :integer
+      r'(true|false)(?!{%_identifier_follow%})' -> :boolean
+      r'{%_type%}(?!{%_identifier_follow%})' -> :type
+      r'{%_identifier%}(?=\s*=)' -> :cmd_attr_hint[] :identifier
+      r'{%_identifier%}' -> :identifier
+
+      # Expression tokens
+      r':' -> :colon
+      r',' -> :comma
+      r'\.' -> :dot
+      r'==' -> :double_equal
+      r'\|\|' -> :double_pipe
+      r'\&\&' -> :double_ampersand
+      r'!=' -> :not_equal
+      r'=' -> :equal
+      r'\.' -> :dot
+      r'\{' -> :lbrace
+      r'\(' -> :lparen
+      r'\)' -> :rparen
+      r'\[' -> :lsquare
+      r'\]' -> :rsquare
+      r'\+' -> :plus
+      r'\*' -> :asterisk
+      r'-' -> :dash
+      r'/' -> :slash
+      r'%' -> :percent
+      r'<=' -> :lteq
+      r'<' -> :lt
+      r'>=' -> :gteq
+      r'>' -> :gt
+      r'!' -> :not
+
+      # Literals
+      enum {
+        python: r'"([^\\\"\n]|\\[\\"\'nrbtfav\?]|\\[0-7]{1,3}|\\x[0-9a-fA-F]+|\\[uU]([0-9a-fA-F]{4})([0-9a-fA-F]{4})?)*"'
+        java: "\\\"(?>[^\\\\\\\"\\n]|\\\\[\\\"\\'nrbtfav\\\\?]|\\\\[0-7]{1,3}|\\\\x[0-9a-fA-F]+|\\\\[uU]([0-9a-fA-F]{4})([0-9a-fA-F]{4})?)*\\\""
+      } -> unescape(:string)
+      enum {
+        python: r'\'([^\\\'\n]|\\[\\"\'nrbtfav\?]|\\[0-7]{1,3}|\\x[0-9a-fA-F]+|\\[uU]([0-9a-fA-F]{4})([0-9a-fA-F]{4})?)*\''
+        java: "'(?>[^\\\\\\'\\n]|\\\\[\\\"\\'nrbtfav\\\\?]|\\\\[0-7]{1,3}|\\\\x[0-9a-fA-F]+|\\\\[uU]([0-9a-fA-F]{4})([0-9a-fA-F]{4})?)*'"
+      } -> unescape(:string)
+      r'-?[0-9]+\.[0-9]+' -> :float
+      r'[0-9]+' -> :integer
+    }
+
+    code<python> << PYTHON
+    def init():
+        return {
+            'context': None,
+            'replacements': {
+                re.compile(r"\\n"): 0x000A,
+                re.compile(r"\\r"): 0x000D,
+                re.compile(r"\\b"): 0x0008,
+                re.compile(r"\\t"): 0x0009,
+                re.compile(r"\\a"): 0x0007,
+                re.compile(r"\\v"): 0x000B,
+                re.compile(r'\\"'): 0x0022,
+                re.compile(r"\\'"): 0x0027,
+                re.compile(r"\\\?"): 0x003F
+            },
+            'escapes': {
+                re.compile(r'(\\([0-7]{1,3}))'): 8,
+                re.compile(r'(\\[xX]([0-9a-fA-F]{1,4}))'): 16,
+                re.compile(r'(\\[uU]([0-9a-fA-F]{4}))'): 16
+            }
+        }
+    def workflow(ctx, terminal, source_string, line, col):
+        ctx.user_context['context'] = 'workflow'
+        default_action(ctx, terminal, source_string, line, col)
+    def task(ctx, terminal, source_string, line, col):
+        ctx.user_context['context'] = 'task'
+        default_action(ctx, terminal, source_string, line, col)
+    def output(ctx, terminal, source_string, line, col):
+        if ctx.user_context['context'] == 'workflow':
+            ctx.stack.append('wf_output')
+        default_action(ctx, terminal, source_string, line, col)
+    def unescape(ctx, terminal, source_string, line, col):
+
+        for regex, c in ctx.user_context['replacements'].items():
+            source_string = regex.sub(chr(c), source_string)
+
+        source_string = source_string.replace("\\\\", "\\")
+
+        for regex, base in ctx.user_context['escapes'].items():
+            for escape_sequence, number in regex.findall(source_string):
+                source_string = source_string.replace(escape_sequence, unichr(int(number, base)))
+        default_action(ctx, terminal, source_string[1:-1], line, col)
+    PYTHON
+
+    code<java> << JAVA
+    private class WdlContext {
+        public String wf_or_task = null;
+    }
+    public Object init() {
+        return new WdlContext();
+    }
+    public void workflow(LexerContext ctx, TerminalIdentifier terminal, String source_string, int line, int col) {
+        ((WdlContext) ctx.context).wf_or_task = "workflow";
+        default_action(ctx, terminal, source_string, line, col);
+    }
+    public void task(LexerContext ctx, TerminalIdentifier terminal, String source_string, int line, int col) {
+        ((WdlContext) ctx.context).wf_or_task = "task";
+        default_action(ctx, terminal, source_string, line, col);
+    }
+    public void output(LexerContext ctx, TerminalIdentifier terminal, String source_string, int line, int col) {
+        WdlContext user_ctx = (WdlContext) ctx.context;
+        if (user_ctx.wf_or_task != null && user_ctx.wf_or_task.equals("workflow")) {
+            ctx.stack.push("wf_output");
+        }
+        default_action(ctx, terminal, source_string, line, col);
+    }
+    public void unescape(LexerContext ctx, TerminalIdentifier terminal, String source_string, int line, int col) {
+        default_action(ctx, terminal, StringEscapeUtils.unescapeJava(source_string.substring(1, source_string.length() - 1)), line, col);
+    }
+    JAVA
+  }
+  parser {
+    # Document: https://github.com/broadinstitute/wdl/blob/wdl2/SPEC.md#document
+    $document = list($import) list($workflow_or_task) -> Document(imports=$0, definitions=$1)
+    $workflow_or_task = $workflow | $task
+
+    # Import Statements: https://github.com/broadinstitute/wdl/blob/wdl2/SPEC.md#import-statements
+    $import = :import :string optional($import_namespace) -> Import(uri=$1, namespace=$2)
+    $import_namespace = :as :identifier -> $1
+
+    # Task Definition: https://github.com/broadinstitute/wdl/blob/wdl2/SPEC.md#task-definition
+    $task = :task :identifier :lbrace list($declaration) list($sections) :rbrace -> Task(name=$1, declarations=$3, sections=$4)
+    $sections = $command | $outputs | $runtime | $parameter_meta | $meta
+    $command = :raw_command :raw_cmd_start list($command_part) :raw_cmd_end -> RawCommand(parts=$2)
+    $command_part = :cmd_part | $cmd_param
+    $cmd_param = :cmd_param_start list($cmd_param_kv) $e :cmd_param_end -> CommandParameter(attributes=$1, expr=$2)
+    $cmd_param_kv = :cmd_attr_hint :identifier :equal $e -> CommandParameterAttr(key=$1, value=$3)
+    $outputs = :output :lbrace list($output_kv) :rbrace -> Outputs(attributes=$2)
+    $output_kv = $type_e :identifier :equal $e -> Output(type=$0, var=$1, expression=$3)
+    $runtime = :runtime $map -> Runtime(map=$1)
+    $parameter_meta = :parameter_meta $map -> ParameterMeta(map=$1)
+    $meta = :meta $map -> Meta(map=$1)
+    $map = :lbrace list($kv) :rbrace -> $1
+    $kv = :identifier :colon $e -> RuntimeAttribute(key=$0, value=$2)
+
+    # Declarations: https://github.com/broadinstitute/wdl/blob/wdl2/SPEC.md#declarations
+    $declaration = $type_e optional($postfix_quantifier) :identifier optional($setter) -> Declaration(type=$0, postfix=$1, name=$2, expression=$3)
+    $setter = :equal $e -> $1
+    $postfix_quantifier = :qmark | :plus
+
+    # Types: https://github.com/broadinstitute/wdl/blob/wdl2/SPEC.md#types
+    $type_e = parser<expression> {
+      (*:left) $type_e = :type <=> :lsquare list($type_e, :comma) :rsquare -> Type(name=$0, subtype=$2)
+      $type_e = :type
+    }
+
+    # Expressions: https://github.com/broadinstitute/wdl/blob/wdl2/SPEC.md#expressions
+    $e = parser<expression> {
+      (*:left) $e = $e :double_pipe $e -> LogicalOr(lhs=$0, rhs=$2)
+      (*:left) $e = $e :double_ampersand $e -> LogicalAnd(lhs=$0, rhs=$2)
+      (*:left) $e = $e :double_equal $e -> Equals(lhs=$0, rhs=$2)
+      (-:left) $e = $e :not_equal $e -> NotEquals(lhs=$0, rhs=$2)
+      (*:left) $e = $e :lt $e -> LessThan(lhs=$0, rhs=$2)
+      (-:left) $e = $e :lteq $e -> LessThanOrEqual(lhs=$0, rhs=$2)
+      (-:left) $e = $e :gt $e -> GreaterThan(lhs=$0, rhs=$2)
+      (-:left) $e = $e :gteq $e -> GreaterThanOrEqual(lhs=$0, rhs=$2)
+      (*:left) $e = $e :plus $e -> Add(lhs=$0, rhs=$2)
+      (-:left) $e = $e :dash $e -> Subtract(lhs=$0, rhs=$2)
+      (*:left) $e = $e :asterisk $e -> Multiply(lhs=$0, rhs=$2)
+      (-:left) $e = $e :slash $e -> Divide(lhs=$0, rhs=$2)
+      (-:left) $e = $e :percent $e -> Remainder(lhs=$0, rhs=$2)
+      (*:unary) $e = :not $e -> LogicalNot(expression=$1)
+      (-:unary) $e = :plus $e -> UnaryPlus(expression=$1)
+      (-:unary) $e = :dash $e -> UnaryNegation(expression=$1)
+      (*:left) $e = :identifier <=> :lparen list($e, :comma) :rparen -> FunctionCall(name=$0, params=$2)
+      (*:left) $e = :identifier <=> :lsquare $e :rsquare -> ArrayOrMapLookup(lhs=$0, rhs=$2)
+      (*:left) $e = :identifier <=> :dot :identifier -> MemberAccess(lhs=$0, rhs=$2)
+      # TODO: is there a better object literal syntax?
+      (*:left) $e = :object :lbrace list($object_kv, :comma) :rbrace -> ObjectLiteral(map=$2)
+      (*:left) $e = :lsquare list($e, :comma) :rsquare -> ArrayLiteral(values=$1)
+      (*:left) $e = :lbrace list($map_kv, :comma) :rbrace -> MapLiteral(map=$1)
+      (*:left) $e = :lparen $e :rparen -> $1
+      $e = :string
+      $e = :identifier
+      $e = :boolean
+      $e = :integer
+      $e = :float
+    }
+    $map_kv = $e :colon $e -> MapLiteralKv(key=$0, value=$2)
+
+    # Workflows: https://github.com/broadinstitute/wdl/blob/wdl2/SPEC.md#workflow-definition
+    $workflow = :workflow :identifier :lbrace list($wf_body_element) :rbrace -> Workflow(name=$1, body=$3)
+    $wf_body_element = $call | $declaration | $while_loop | $if_stmt | $scatter | $wf_outputs
+    $call = :call :fqn optional($alias) optional($call_body) -> Call(task=$1, alias=$2, body=$3)
+    $call_body = :lbrace list($declaration) list($call_input) :rbrace -> CallBody(declarations=$1, io=$2)
+    $call_input = :input :colon list($mapping, :comma) -> Inputs(map=$2)
+    $mapping = :identifier :equal $e -> IOMapping(key=$0, value=$2)
+    $alias = :as :identifier -> $1
+    $wf_outputs = :output :lbrace list($wf_output, :comma) :rbrace -> WorkflowOutputs(outputs=$2)
+    $wf_output = :fqn optional($wf_output_wildcard) -> WorkflowOutput(fqn=$0, wildcard=$1)
+    $wf_output_wildcard = :dot :asterisk -> $1
+
+    $while_loop = :while :lparen $e :rparen :lbrace list($wf_body_element) :rbrace -> WhileLoop(expression=$2, body=$5)
+    $if_stmt = :if :lparen $e :rparen :lbrace list($wf_body_element) :rbrace -> If(expression=$2, body=$5)
+
+    $scatter = :scatter :lparen :identifier :in $e :rparen :lbrace list($wf_body_element) :rbrace
+        -> Scatter(item=$2, collection=$4, body=$7)
+
+    $object_kv = :identifier :colon $e -> ObjectKV(key=$0, value=$2)
+  }
+}

wdl/binding.py

@@ -488,12 +488,12 @@ def eval(ast, lookup=lambda var: None, functions=None):
             array_or_map = eval(ast.attr('lhs'), lookup, functions)
             index = eval(ast.attr('rhs'), lookup, functions)
 
-	    if isinstance(array_or_map, WdlArray) and isinstance(index, WdlInteger):
-	    	return array_or_map.value[index.value]
-	    elif isinstance(array_or_map, WdlArray):
-	    	raise EvalException('Cannot index array {} with {}'.format(array_or_map, index))
-	    elif isinstance(array_or_map, WdlMap) and isinstance(index.type, WdlPrimitiveType):
-	    	return array_or_map.value[index]
+            if isinstance(array_or_map, WdlArray) and isinstance(index, WdlInteger):
+                return array_or_map.value[index.value]
+            elif isinstance(array_or_map, WdlArray):
+                raise EvalException('Cannot index array {} with {}'.format(array_or_map, index))
+            elif isinstance(array_or_map, WdlMap) and isinstance(index.type, WdlPrimitiveType):
+                return array_or_map.value[index]
             raise EvalException('ArrayOrMapLookup not implemented yet')
         if ast.name == 'MemberAccess':
             object = eval(ast.attr('lhs'), lookup, functions)