[grammar][parser] add 'alert' expressions to grammar (#247)

* [grammars] add alerts with syntax ^`message`
* [contexts] register alerts as part of parseinfo
* [docs][syntax] document the alert expression

docs/syntax.rst

@@ -356,14 +356,29 @@ The expressions, in reverse order of operator precedence, can be:
 
 .. code:: python
 
-            eval(f'{repr(constant)}.format(**{ast})')
+    eval(f'{repr(constant)}.format(**{ast})')
 
 `````constant`````
 ^^^^^^^^^^^^^^^^^^
 
     A multiline version of ```constant```.
 
 
+^```constant``` and ^`````constant`````
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    An alert. There will be no token returned by the parser, but an alert will be registed in the parse context and added to the current node's ``parseinfo``.
+
+    The ``^`` character may appear more than once to indicate the alert level.
+
+
+.. code::
+
+    assignment = identifier '=' (
+        | value
+        | ->'&; ^^^`could not parse value in assignment to {identifier}`
+
+
 ``rulename``
 ^^^^^^^^^^^^
     Invoke the rule named ``rulename``. To help with lexical aspects of grammars, rules with names that begin with an uppercase letter will not advance the input over whitespace or comments.

grammar/tatsu.ebnf

@@ -348,7 +348,7 @@ skip_to::SkipTo
 
 atom
     =
-    cut | cut_deprecated | token | constant | call | pattern | eof
+    cut | cut_deprecated | token | alert | constant | call | pattern | eof
     ;
 
 
@@ -400,6 +400,12 @@ constant::Constant
     ;
 
 
+alert::Alert
+    =
+     level:/\^+/ message:constant
+    ;
+
+
 token::Token
     =
     string | raw_string

tatsu/bootstrap.py

@@ -685,11 +685,12 @@ def _term_(self):  # noqa
             self._error(
                 'expecting one of: '
                 "'!' '&' '(' '()' '->' '?(' '[' '{'"
-                '<atom> <call> <closure> <constant> <cut>'
-                '<cut_deprecated> <empty_closure> <eof>'
-                '<gather> <group> <join> <left_join>'
-                '<lookahead> <negative_lookahead>'
-                '<optional> <pattern> <positive_closure>'
+                '<alert> <atom> <call> <closure>'
+                '<constant> <cut> <cut_deprecated>'
+                '<empty_closure> <eof> <gather> <group>'
+                '<join> <left_join> <lookahead>'
+                '<negative_lookahead> <optional>'
+                '<pattern> <positive_closure>'
                 '<right_join> <separator> <skip_to>'
                 '<special> <token> <void>'
             )
@@ -980,6 +981,8 @@ def _atom_(self):  # noqa
                 self._cut_deprecated_()
             with self._option():
                 self._token_()
+            with self._option():
+                self._alert_()
             with self._option():
                 self._constant_()
             with self._option():
@@ -990,10 +993,10 @@ def _atom_(self):  # noqa
                 self._eof_()
             self._error(
                 'expecting one of: '
-                "'$' '>>' '`' '~' <call> <constant> <cut>"
-                '<cut_deprecated> <eof> <pattern>'
-                '<raw_string> <regexes> <string> <token>'
-                '<word>'
+                "'$' '>>' '`' '~' <alert> <call>"
+                '<constant> <cut> <cut_deprecated> <eof>'
+                '<pattern> <raw_string> <regexes>'
+                '<string> <token> <word> \\^+'
             )
 
     @tatsumasu('RuleRef')
@@ -1047,6 +1050,17 @@ def _constant_(self):  # noqa
                     "'`' '```' `(.*?)`"
                 )
 
+    @tatsumasu('Alert')
+    def _alert_(self):  # noqa
+        self._pattern('\\^+')
+        self.name_last_node('level')
+        self._constant_()
+        self.name_last_node('message')
+        self._define(
+            ['level', 'message'],
+            []
+        )
+
     @tatsumasu('Token')
     def _token_(self):  # noqa
         with self._choice():
@@ -1357,6 +1371,9 @@ def name(self, ast):  # noqa
     def constant(self, ast):  # noqa
         return ast
 
+    def alert(self, ast):  # noqa
+        return ast
+
     def token(self, ast):  # noqa
         return ast
 

tatsu/codegen/python.py

@@ -117,9 +117,19 @@ def render_fields(self, fields):
 
 class Constant(Base):
     def render_fields(self, fields):
-        fields.update(literal=repr(self.node.literal))
+        fields.update(constant=repr(self.node.literal))
 
-    template = "self._constant({literal})"
+    template = "self._constant({constant})"
+
+
+class Alert(Base):
+    def render_fields(self, fields):
+        fields.update(
+            literal=repr(self.node.literal),
+            level=self.node.level,
+        )
+
+    template = "self._alert({literal}, {level})"
 
 
 class Pattern(Base):

tatsu/contexts.py

@@ -21,6 +21,7 @@
 from .util import left_assoc, right_assoc
 from .tokenizing import Tokenizer
 from .infos import (
+    Alert,
     MemoKey,
     ParseInfo,
     RuleInfo,
@@ -549,12 +550,13 @@ def _fail(self):
     def _get_parseinfo(self, name, pos):
         endpos = self._pos
         return ParseInfo(
-            self.tokenizer,
-            name,
-            pos,
-            endpos,
-            self.tokenizer.posline(pos),
-            self.tokenizer.posline(endpos),
+            tokenizer=self.tokenizer,
+            rule=name,
+            pos=pos,
+            endpos=endpos,
+            line=self.tokenizer.posline(pos),
+            endline=self.tokenizer.posline(endpos),
+            alerts=self.state.alerts,
         )
 
     @property
@@ -730,6 +732,12 @@ def _constant(self, literal):
         self._append_cst(literal)
         return literal
 
+    def _alert(self, message, level):
+        self._next_token()
+        self._trace_match(f'{"^" * level}`{message}`', failed=True)
+        self.state.alerts.append(Alert(message=message, level=level))
+        return None
+
     def _pattern(self, pattern):
         token = self.tokenizer.matchre(pattern)
         if token is None:

tatsu/grammars.py

@@ -342,20 +342,34 @@ def parse(self, ctx):
             text = self.literal
             if '\n' in text:
                 text = trim(text)
-            return eval(f'{repr(text)}.format(**{ctx.ast})')  # pylint: disable=eval-used
+            return eval(f'{"f" + repr(text)}', {}, dict(ctx.ast))  # pylint: disable=eval-used
         else:
             return self.literal
 
     def _first(self, k, f):
         return {()}
 
     def _to_str(self, lean=False):
-        return '`%s`' % repr(self.literal)
+        return f'`{repr(self.literal)}`'
 
     def _nullable(self):
         return True
 
 
+class Alert(Constant):
+    def __post_init__(self):
+        super().__post_init__()
+        self.literal = self.ast.message.literal
+        self.level = len(self.ast.level)
+
+    def parse(self, ctx):
+        message = super().parse(ctx)
+        return message
+
+    def _to_str(self, lean=False):
+        return f'{"^" * self.level}{super()._to_str()}'
+
+
 class Pattern(Model):
     def __post_init__(self):
         super().__post_init__()

tatsu/infos.py

@@ -168,13 +168,19 @@ def new_comment():
         return CommentInfo([], [])
 
 
+class Alert(NamedTuple):
+    level: int = 1
+    message: str = ''
+
+
 class ParseInfo(NamedTuple):
     tokenizer: Any
     rule: str
     pos: int
     endpos: int
     line: int
     endline: int
+    alerts: list[Alert] = []
 
     def text_lines(self):
         return self.tokenizer.get_lines(self.line, self.endline)
@@ -225,3 +231,4 @@ class ParseState:
     pos: int = 0
     ast: AST = dataclasses.field(default_factory=AST)
     cst: Any = None
+    alerts: list[Alert] = dataclasses.field(default_factory=list)

test/grammar/alerts_test.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+from tatsu.tool import parse
+
+
+def test_alert_interpolation():
+    input = '42 69'
+    grammar = '''
+            start = a:number b: number i:^`"seen: {a}, {b}"` $ ;
+            number = /\d+/ ;
+    '''
+    assert parse(grammar, input) == {'a': '42', 'b': '69', 'i': 'seen: 42, 69'}