Merge pull request #1 from spyoungtech/rewrite

Rewrite parser

Grammar/AutoHotkey.gram

@@ -238,4 +238,7 @@ string : SINGLE_QUOTED_STRING
                                                                 # spaces are expressed here because it is needed for
                                                                 # disambiguation from lookups (see ``location :``)
 
+function_call_statement_arguments
+function_call_statement : NEWLINE [ WHITESPACE ] NAME [ WHITESPACE ] [ arguments ] terminator
+
 empty  :

ahk_ast/errors.py

@@ -42,7 +42,7 @@ def __reduce__(self):  # type: ignore
         return self.__class__, (self.msg, self.token)
 
 
-class ParsingException(AHKDecodeError):
+class AHKParsingException(AHKDecodeError):
     def __init__(self, msg: str, token: Any):
         lineno = getattr(token, 'lineno', 0)
         index = getattr(token, 'index', 0)
@@ -61,5 +61,5 @@ def __reduce__(self):  # type: ignore
         return self.__class__, (self.msg, self.token)
 
 
-class InvalidHotkeyException(ParsingException):
+class InvalidHotkeyException(AHKParsingException):
     ...

ahk_ast/model.py

@@ -271,12 +271,16 @@ def __init__(self, expression: Optional[Expression]):
 
 
 class FunctionCall(ExpressionStatement):
-    def __init__(self, name: str, arguments: Union[Sequence[Expression], None]):
-        assert isinstance(name, str)  # can functions be stored at locations?
+    def __init__(self, func_location: Location, arguments: Union[Sequence[Expression], None]):
+        assert isinstance(func_location, Location)  # can functions be stored at locations?
         assert arguments is None or isinstance(arguments, Iterable)
         arguments = tuple(arguments) if arguments else tuple()
         assert all(isinstance(arg, Expression) for arg in arguments)
-        super().__init__(name=name, arguments=arguments)
+        super().__init__(func_location=func_location, arguments=arguments)
+
+
+class FunctionCallStatement(FunctionCall):
+    pass  # distinguish `MsgBox "foo"` from `MsgBox("foo")`
 
 
 class Hotkey(Node):

ahk_ast/parser.py

@@ -1,4 +1,7 @@
+import sys
 from typing import Any
+from typing import Generator
+from typing import NoReturn
 from typing import Sequence
 from typing import Union
 
@@ -7,6 +10,8 @@
 from sly.yacc import YaccProduction  # type: ignore[import]
 
 from .errors import AHKAstBaseException
+from .errors import AHKDecodeError
+from .errors import AHKParsingException
 from .errors import InvalidHotkeyException
 from .model import *
 from .tokenizer import AHKLexer
@@ -17,6 +22,7 @@
 class AHKParser(Parser):
     debugfile = 'parser.out'
     tokens = AHKLexer.tokens
+    start = 'program'
 
     def __init__(self, *args: Any, **kwargs: Any):
         super().__init__(*args, **kwargs)
@@ -26,120 +32,69 @@ def __init__(self, *args: Any, **kwargs: Any):
         self.last_token = None
         self.seen_tokens: list[AHKToken]
         self.seen_tokens = []
-        self.expecting: list[AHKToken]
+        self.expecting: list[list[str]]
         self.expecting = []
 
-    @_('statements')
-    def program(self, p: YaccProduction) -> Any:
-        return p[0]
-
-    # statements : { statement }      # zero or more statements { }
-    @_('{ statement }')
-    def statements(self, p: YaccProduction) -> Any:
-        # p.statement   --- it's already a list of statement (by SLY)
-        return Program(*p.statement)
-
-    @_(
-        'assignment_statement',
-        # 'augmented_assignment_statement',
-        # 'if_statement',
-        # 'loop_statement',
-        # 'while_statement',
-        # 'class_definition',
-        # 'function_definition',
-        # 'hotkey_definition',
-        # 'for_statement',
-        # 'try_statement',
-        # 'variable_declaration',
-        'expression_statement',
-        'return_statement',
-    )
-    def statement(self, p: YaccProduction) -> Any:
+    @_('WHITESPACE', 'NEWLINE')
+    def wsc(self, p: YaccProduction) -> Any:
         return p[0]
 
-    @_('RETURN [ WHITESPACE expression ]')
-    def return_statement(self, p: YaccProduction) -> Any:
-        return ReturnStatement(expression=p.expression)
+    @_('{ wsc } statements { wsc }')
+    def program(self, p: YaccProduction) -> Any:
+        return Program(*p.statements)
 
-    @_(
-        # 'grouping'
-        # 'deref'
-        'function_call'
-        # 'tenary_expression'
-    )
-    def expression_statement(self, p: YaccProduction) -> Any:
-        return p[0]
+    @_('NEWLINE [ WHITESPACE ] [ statements ]')
+    def additional_statement(self, p: YaccProduction) -> Any:
+        print('ADDITIONAL SEEN')
+        return p.statements
 
-    @_(
-        'EXP',
-        'PLUS',
-        'MINUS',
-        'TIMES',
-        'DIVIDE',
-        'LT',
-        'LE',
-        'GT',
-        'GE',
-        'EQ',
-        'SEQ',
-        'NE',
-        'SNE',
-        'LAND',
-        'REMATCH',
-        'LOR',
-        'AND',
-        'OR',
-        'IN',
-        'IS',
-    )
-    def bin_operator(self, p: YaccProduction) -> Any:
+    @_('statement { additional_statement }')
+    def statements(self, p: YaccProduction) -> Any:
+        print(p.statement, p.additional_statement)
+        ret = [p.statement]
+        for stmts in p.additional_statement:
+            if stmts:
+                ret.extend(stmts)
+        return ret
+
+    @_('assignment_statement', 'function_call_statement')
+    def statement(self, p: YaccProduction) -> Any:
         return p[0]
 
-    @_(
-        'expression [ WHITESPACE ] bin_operator [ WHITESPACE ] expression',
-    )
-    def bin_op(self, p: YaccProduction) -> Any:
-        op = p.bin_operator
-        left = p.expression0
-        right = p.expression1
-        return BinOp(op=op, left=left, right=right)
-
     @_('NAME')
     def location(self, p: YaccProduction) -> Any:
-        return Identifier(name=p.NAME)
+        return Identifier(name=p[0])
 
-    @_('TRUE')
-    def expression(self) -> Any:
-        return Bool(True)
+    @_('')
+    def seen_ASSIGN(self, p: YaccProduction) -> Any:
+        self.expecting.append(['expression'])
 
-    @_('FALSE')
-    def expression(self) -> Any:
-        return Bool(False)
+    @_('location [ WHITESPACE ] ASSIGN seen_ASSIGN [ WHITESPACE ] expression')
+    def assignment_statement(self, p: YaccProduction) -> Assignment:
+        return Assignment(location=p.location, value=p.expression)
 
-    @_(
-        'expression_statement',
-        'bin_op',
-        'location',
-    )
+    @_('literal', 'location')
     def expression(self, p: YaccProduction) -> Any:
+        self.expecting.pop()
         return p[0]
 
-    @_('NAME LPAREN [ WHITESPACE ] RPAREN')
-    def function_call(self, p: YaccProduction) -> Any:
-        return FunctionCall(name=p.NAME, arguments=None)
+    @_('INTEGER')
+    def literal(self, p: YaccProduction) -> Integer:
+        return Integer(value=int(p[0]))
 
-    @_('NAME LPAREN [ WHITESPACE ] arguments RPAREN')
-    def function_call(self, p: YaccProduction) -> Any:
-        function_name = p.NAME
-        return FunctionCall(name=function_name, arguments=p.arguments)
+    @_('DOUBLE_QUOTED_STRING')
+    def string(self, p: YaccProduction) -> DoubleQuotedString:
+        # TODO: unescape value
+        return DoubleQuotedString(value=p[0][1:-1])
 
-    @_('INTEGER')
-    def expression(self, p: YaccProduction) -> Any:
-        return Integer(value=int(p.INTEGER))
+    @_('SINGLE_QUOTED_STRING')
+    def string(self, p: YaccProduction) -> SingleQuotedString:
+        # TODO: unescape value
+        return SingleQuotedString(value=p[0][1:-1])
 
-    @_('NEWLINE [ WHITESPACE ] NAME [ WHITESPACE ] [ arguments ] terminator')
-    def function_call(self, p: YaccProduction) -> FunctionCall:
-        return FunctionCall(name=p.NAME, arguments=p.arguments)
+    @_('string')
+    def literal(self, p: YaccProduction) -> Any:
+        return p[0]
 
     @_('COMMA [ WHITESPACE ] first_argument')
     def additional_arguments(self, p: YaccProduction) -> Any:
@@ -150,27 +105,83 @@ def first_argument(self, p: YaccProduction) -> Any:
         return p[0]
 
     @_('first_argument { additional_arguments }')
-    def arguments(self, p: YaccProduction) -> Any:
+    def function_call_statement_arguments(self, p: YaccProduction) -> Any:
         args = [p.first_argument]
         for a in p.additional_arguments:
             args.append(a)
         return args
 
-    @_('location [ WHITESPACE ] ASSIGN [ WHITESPACE ] expression terminator')
-    def assignment_statement(self, p: YaccProduction) -> Any:
-        return Assignment(location=p.location, value=p.expression)
+    @_('')
+    def seen_function_call_statement_start(self, p: YaccProduction) -> Any:
+        self.expecting.append(['expression'])
 
-    @_('NEWLINE')
-    def terminator(self, p: YaccProduction) -> Any:
-        return p[0]
+    @_(
+        'location [ WHITESPACE ] [ seen_function_call_statement_start function_call_statement_arguments ]'
+    )
+    def function_call_statement(self, p: YaccProduction) -> FunctionCallStatement:
+        return FunctionCallStatement(
+            func_location=p.location, arguments=p.function_call_statement_arguments
+        )
+
+    def error(self, token: Union[AHKToken, None]) -> NoReturn:
+        if token:
+            if self.expecting:
+                expected = self.expecting[-1]
+
+                message = f"Syntax Error. Was expecting {' or '.join(expected)}"
+            else:
+                message = 'Syntax Error'
+            raise AHKParsingException(message, token)
+
+        elif self.last_token:
+            doc = self.last_token.doc
+            pos = len(doc)
+            lineno = doc.count('\n', 0, pos) + 1
+            colno = pos - doc.rfind('\n', 0, pos)
+            message = f'Unexpected EOF at: ' f'line {lineno} column {colno} (char {pos})'
+            if self.expecting:
+                expected = self.expecting[-1]
+                message += f'. Was expecting {" or ".join(expected)}'
+            raise AHKParsingException(message, None)
+        else:
+            #  Empty file
+            raise AHKParsingException(
+                'Expecting at least one statement. Received unexpected EOF', None
+            )
+
+    def _token_gen(self, tokens: Iterable[AHKToken]) -> Generator[AHKToken, None, None]:
+        for tok in tokens:
+            # if self.last_token is None and tok.type != "NEWLINE":
+            #     class t(Token):
+            #         type = "NEWLINE"
+            #         index = 0
+            #         lineno = 0
+            #         value = '\n'
+            #     yield AHKToken(tok=t(), doc=tok.doc)
+            self.last_token = tok
+            self.seen_tokens.append(tok)
+            yield tok
+
+    def parse(self, tokens: Iterable[AHKToken]) -> Program:
+        tokens = self._token_gen(tokens)
+        model: Program
+        model = super().parse(tokens)
+        return model
 
 
 def parse_tokens(raw_tokens: Iterable['Token']) -> Node:
     parser = AHKParser()
-    return parser.parse(raw_tokens)  # type: ignore[no-any-return]
+    return parser.parse(raw_tokens)
 
 
 def parse(text: str) -> Node:
     tokens = tokenize(text)
     model = parse_tokens(tokens)
     return model
+
+
+if __name__ == '__main__':
+    fp = sys.argv[1]
+    with open(fp) as f:
+        text = f.read()
+    print(parse(text))

ahk_ast/tokenizer.py

@@ -40,16 +40,18 @@ def __init__(self, *args: Any, **kwargs: Any):
 
     regex_module = re
     reflags = re.MULTILINE
+
     tokens = {
+        EOF,
         ARROW,
         AMP,
         HASH,
         LBRACE,
         RBRACE,
         LBRACKET,
         RBRACKET,
-        DOUBLE_QUOTE_STRING,
-        SINGLE_QUOTE_STRING,
+        DOUBLE_QUOTED_STRING,
+        SINGLE_QUOTED_STRING,
         UNTERMINATED_DOUBLE_QUOTE_STRING,
         UNTERMINATED_SINGLE_QUOTE_STRING,
         NAME,
@@ -172,8 +174,8 @@ def LINE_COMMENT(self, tok: Token) -> Union[Token, None]:
     #     # r'\\\d{1,3}',
     #     # r'\\x[a-fA-F0-9]{1,2}',
     # ]
-    DOUBLE_QUOTE_STRING = r'"(?:[^"`]|`.)*"'
-    SINGLE_QUOTE_STRING = r"'(?:[^'`]|`.)*'"
+    DOUBLE_QUOTED_STRING = r'"(?:[^"`]|`.)*"'
+    SINGLE_QUOTED_STRING = r"'(?:[^'`]|`.)*'"
 
     # Specify tokens as regex rules
     DOLLAR = r'\$'
@@ -281,7 +283,7 @@ def WHITESPACE(self, tok: Token) -> Token:
     DCOLON = r'::'
     COLON = r':'
 
-    def tokenize(self, text: str, *args: Any, **kwargs: Any) -> Generator[Token, None, None]:
+    def tokenize(self, text: str, *args: Any, **kwargs: Any) -> Generator[AHKToken, None, None]:
         for tok in super().tokenize(text, *args, **kwargs):
             tok = AHKToken(tok, text)
             yield tok