Skip to content

Commit

Permalink
1. Fix "Python type check / Format (pull request)" failure in test_la…
Browse files Browse the repository at this point in the history 
…rk_lark.py (trailing whitespace in test_06_*())

2. Remove what was left of "Literals can be one of: ..." under "Terminals" in grammar.md.

3. Address @erezsh's point about inlined terminals under "Terminals" in grammar.md.

4. Remove "lark.lark-ORIG" references in test_lark_lark.py and test_grammar_formal.py that shouldn't have been pushed.

5. Address @erezsh's point about f.readlines() in test_grammar_formal.py.

6. Address @erezsh's point commented-out tests in test_grammar_formal.py.
  • Loading branch information
@RossPatterson
RossPatterson committed Feb 1, 2024
1 parent db1a5a5 commit 9493f81
Show file tree
Hide file tree
Showing 3 changed files with 5 additions and 83 deletions.
6 changes: 1 addition & 5 deletions docs/grammar.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,7 @@ Terminals are used to match text into symbols. They can be defined as a combinat
<NAME> [. <priority>] : <items-to-match>
```

Terminal names must be uppercase. They must start with an underscore (`_`) or a letter (`A` through `Z`), and may be composed of letters, underscores, and digits (`0` through `9`). Terminal names that start with "_" will not be included in the parse tree, unless the `keep_all_tokens` option is specified.

Literals can be one of:

* Literal range: `"a".."z"`, `"1".."9"`, etc. - Each literal must be a single character, and the range represends all values between the two literals, inclusively.
Terminal names must be uppercase. They must start with an underscore (`_`) or a letter (`A` through `Z`), and may be composed of letters, underscores, and digits (`0` through `9`). Terminal names that start with "_" will not be included in the parse tree, unless the `keep_all_tokens` option is specified, or unless they are part of a containing terminal.

Each item is one of:

Expand Down
77 changes: 2 additions & 75 deletions tests/test_grammar_formal.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,14 @@
from lark.load_grammar import GrammarError


# Based on TestGrammar, with lots of tests that can't be run elided.
# Based on TestGrammar, with lots of tests that can't be run deleted.
class TestGrammarFormal(TestCase):
def setUp(self):
lark_path = os.path.join(os.path.dirname(lark.__file__), 'grammars/lark.lark')
# lark_path = os.path.join(os.path.dirname(lark.__file__), 'grammars/lark.lark-ORIG')
with open(lark_path, 'r') as f:
self.lark_grammar = "\n".join(f.readlines())
self.lark_grammar = f.read())

def test_errors(self):
# raise NotImplementedError("Doesn't work yet.")
l = Lark(self.lark_grammar, parser="lalr")

# This is an unrolled form of the test_grammar.py:GRAMMAR_ERRORS tests, because the lark.lark messages vary.
Expand Down Expand Up @@ -76,39 +74,6 @@ def test_errors(self):
# '%ignore expects a value', '%ignore %import\n'
self.assertRaisesRegex(UnexpectedToken, 'Unexpected token Token..__ANON_2., .%import..', l.parse, '%ignore %import\n')

# def test_empty_literal(self):
# raise NotImplementedError("Breaks tests/test_parser.py:_TestParser:test_backslash2().")

# def test_ignore_name(self):
# raise NotImplementedError("Can't parse using parsed grammar.")

# def test_override_rule_1(self):
# raise NotImplementedError("Can't parse using parsed grammar.")

# def test_override_rule_2(self):
# raise NotImplementedError("Can't test semantics of grammar, only syntax.")

# def test_override_rule_3(self):
# raise NotImplementedError("Can't test semantics of grammar, only syntax.")

# def test_override_terminal(self):
# raise NotImplementedError("Can't parse using parsed grammar.")

# def test_extend_rule_1(self):
# raise NotImplementedError("Can't parse using parsed grammar.")

# def test_extend_rule_2(self):
# raise NotImplementedError("Can't test semantics of grammar, only syntax.")

# def test_extend_term(self):
# raise NotImplementedError("Can't parse using parsed grammar.")

# def test_extend_twice(self):
# raise NotImplementedError("Can't parse using parsed grammar.")

# def test_undefined_ignore(self):
# raise NotImplementedError("Can't parse using parsed grammar.")

def test_alias_in_terminal(self):
l = Lark(self.lark_grammar, parser="lalr")
g = """start: TERM
Expand All @@ -117,53 +82,15 @@ def test_alias_in_terminal(self):
# self.assertRaisesRegex( GrammarError, "Aliasing not allowed in terminals", Lark, g)
self.assertRaisesRegex( UnexpectedToken, "Unexpected token Token.'__ANON_0', '->'.", l.parse, g)

# def test_undefined_rule(self):
# raise NotImplementedError("Can't test semantics of grammar, only syntax.")

# def test_undefined_term(self):
# raise NotImplementedError("Can't test semantics of grammar, only syntax.")

# def test_token_multiline_only_works_with_x_flag(self):
# raise NotImplementedError("Can't test regex flags in Lark grammar.")

# def test_import_custom_sources(self):
# raise NotImplementedError("Can't parse using parsed grammar.")

# def test_import_custom_sources2(self):
# raise NotImplementedError("Can't parse using parsed grammar.")

# def test_import_custom_sources3(self):
# raise NotImplementedError("Can't parse using parsed grammar.")

# def test_my_find_grammar_errors(self):
# raise NotImplementedError("Can't parse using parsed grammar.")

# def test_ranged_repeat_terms(self):
# raise NotImplementedError("Can't parse using parsed grammar.")

# def test_ranged_repeat_large(self):
# raise NotImplementedError("Can't parse using parsed grammar.")

# def test_large_terminal(self):
# raise NotImplementedError("Can't parse using parsed grammar.")

# def test_list_grammar_imports(self):
# raise NotImplementedError("Can't test semantics of grammar, only syntax.")

def test_inline_with_expand_single(self):
l = Lark(self.lark_grammar, parser="lalr")
grammar = r"""
start: _a
!?_a: "A"
"""
# self.assertRaisesRegex(GrammarError, "Inlined rules (_rule) cannot use the ?rule modifier.", l.parse, grammar)
# TODO Is this really catching the right problem?
self.assertRaisesRegex(UnexpectedToken, "Unexpected token Token.'OP', '?'.", l.parse, grammar)


# def test_line_breaks(self):
# raise NotImplementedError("Can't parse using parsed grammar.")


if __name__ == '__main__':
main()
5 changes: 2 additions & 3 deletions tests/test_lark_lark.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
class TestLarkLark(TestCase):
def setUp(self):
lark_path = os.path.join(os.path.dirname(lark.__file__), 'grammars/lark.lark')
# lark_path = os.path.join(os.path.dirname(lark.__file__), 'grammars/lark.lark-ORIG')
self.lark_parser = Lark.open(lark_path, parser="lalr")

def test_01_no_alias_in_terminal_lg(self):
Expand Down Expand Up @@ -91,14 +90,14 @@ def test_05_extend_term_ll(self):

def test_06_no_term_templates_lg(self):
g = """start: TERM
separated{x, sep}: x (sep x)*
separated{x, sep}: x (sep x)*
TERM: separated{"A", " "}
"""
self.assertRaises( AssertionError, Lark, g)

def test_06_no_term_templates_ll(self):
g = """start: TERM
separated{x, sep}: x (sep x)*
separated{x, sep}: x (sep x)*
TERM: separated{"A", " "}
"""
self.assertRaisesRegex( UnexpectedToken, "Unexpected token Token.'RULE', 'separated'.", self.lark_parser.parse, g)
Expand Down

0 comments on commit 9493f81

Please sign in to comment.