Skip to content

Commit

Permalink
Fix left-assoc ternary operator in infixNotation; fix White \u string…
Browse files Browse the repository at this point in the history 
… typos; backport pyparsing_test from 3.0
  • Loading branch information
@ptmcg
ptmcg committed Dec 24, 2019
1 parent c15af9d commit 0a6cef0
Show file tree
Hide file tree
Showing 3 changed files with 258 additions and 24 deletions.
28 changes: 28 additions & 0 deletions CHANGES
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,34 @@
Change Log
==========

Version 2.4.6 - December, 2019
------------------------------
- Fixed typos in White mapping of whitespace characters, to use
correct "\u" prefix instead of "u\".

- Fix bug in left-associative ternary operators defined using
infixNotation. First reported on StackOverflow by user Jeronimo.

- Backport of pyparsing_test namespace from 3.0.0, including
TestParseResultsAsserts mixin class defining unittest-helper
methods:
. def assertParseResultsEquals(
self, result, expected_list=None, expected_dict=None, msg=None)
. def assertParseAndCheckList(
self, expr, test_string, expected_list, msg=None, verbose=True)
. def assertParseAndCheckDict(
self, expr, test_string, expected_dict, msg=None, verbose=True)
. def assertRunTestResults(
self, run_tests_report, expected_parse_results=None, msg=None)
. def assertRaisesParseException(self, exc_type=ParseException, msg=None)

To use the methods in this mixin class, declare your unittest classes as:

from pyparsing import pyparsing_test as ppt
class MyParserTest(ppt.TestParseResultsAsserts, unittest.TestCase):
...


Version 2.4.5 - November, 2019
------------------------------
- Fixed encoding when setup.py reads README.rst to include the
Expand Down
225 changes: 204 additions & 21 deletions pyparsing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,8 @@
namespace class
"""

__version__ = "2.4.5"
__versionTime__ = "09 Nov 2019 23:03 UTC"
__version__ = "2.4.6"
__versionTime__ = "24 Dec 2019 04:27 UTC"
__author__ = "Paul McGuire <ptmcg@users.sourceforge.net>"

import string
Expand All @@ -114,6 +114,7 @@
from operator import itemgetter
import itertools
from functools import wraps
from contextlib import contextmanager

try:
# Python 3
Expand Down Expand Up @@ -184,6 +185,7 @@ class SimpleNamespace: pass
__diag__.warn_name_set_on_empty_Forward = False
__diag__.warn_on_multiple_string_args_to_oneof = False
__diag__.enable_debug_on_named_expressions = False
__diag__._all_names = [nm for nm in vars(__diag__) if nm.startswith("enable_") or nm.startswith("warn_")]

def _enable_all_warnings():
__diag__.warn_multiple_tokens_in_named_alternation = True
Expand Down Expand Up @@ -3630,24 +3632,24 @@ class White(Token):
'\n': '<LF>',
'\r': '<CR>',
'\f': '<FF>',
'u\00A0': '<NBSP>',
'u\1680': '<OGHAM_SPACE_MARK>',
'u\180E': '<MONGOLIAN_VOWEL_SEPARATOR>',
'u\2000': '<EN_QUAD>',
'u\2001': '<EM_QUAD>',
'u\2002': '<EN_SPACE>',
'u\2003': '<EM_SPACE>',
'u\2004': '<THREE-PER-EM_SPACE>',
'u\2005': '<FOUR-PER-EM_SPACE>',
'u\2006': '<SIX-PER-EM_SPACE>',
'u\2007': '<FIGURE_SPACE>',
'u\2008': '<PUNCTUATION_SPACE>',
'u\2009': '<THIN_SPACE>',
'u\200A': '<HAIR_SPACE>',
'u\200B': '<ZERO_WIDTH_SPACE>',
'u\202F': '<NNBSP>',
'u\205F': '<MMSP>',
'u\3000': '<IDEOGRAPHIC_SPACE>',
u'\u00A0': '<NBSP>',
u'\u1680': '<OGHAM_SPACE_MARK>',
u'\u180E': '<MONGOLIAN_VOWEL_SEPARATOR>',
u'\u2000': '<EN_QUAD>',
u'\u2001': '<EM_QUAD>',
u'\u2002': '<EN_SPACE>',
u'\u2003': '<EM_SPACE>',
u'\u2004': '<THREE-PER-EM_SPACE>',
u'\u2005': '<FOUR-PER-EM_SPACE>',
u'\u2006': '<SIX-PER-EM_SPACE>',
u'\u2007': '<FIGURE_SPACE>',
u'\u2008': '<PUNCTUATION_SPACE>',
u'\u2009': '<THIN_SPACE>',
u'\u200A': '<HAIR_SPACE>',
u'\u200B': '<ZERO_WIDTH_SPACE>',
u'\u202F': '<NNBSP>',
u'\u205F': '<MMSP>',
u'\u3000': '<IDEOGRAPHIC_SPACE>',
}
def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0):
super(White, self).__init__()
Expand Down Expand Up @@ -6064,7 +6066,7 @@ def parseImpl(self, instring, loc, doActions=True):
matchExpr = _FB(lastExpr + lastExpr) + Group(lastExpr + OneOrMore(lastExpr))
elif arity == 3:
matchExpr = (_FB(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr)
+ Group(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr))
+ Group(lastExpr + OneOrMore(opExpr1 + lastExpr + opExpr2 + lastExpr)))
else:
raise ValueError("operator must be unary (1), binary (2), or ternary (3)")
elif rightLeftAssoc == opAssoc.RIGHT:
Expand Down Expand Up @@ -6835,6 +6837,187 @@ class Devanagari(unicode_set):
setattr(pyparsing_unicode, u"देवनागरी", pyparsing_unicode.Devanagari)


class pyparsing_test:
"""
namespace class for classes useful in writing unit tests
"""

class reset_pyparsing_context:
"""
Context manager to be used when writing unit tests that modify pyparsing config values:
- packrat parsing
- default whitespace characters.
- default keyword characters
- literal string auto-conversion class
- __diag__ settings
Example:
with reset_pyparsing_context():
# test that literals used to construct a grammar are automatically suppressed
ParserElement.inlineLiteralsUsing(Suppress)
term = Word(alphas) | Word(nums)
group = Group('(' + term[...] + ')')
# assert that the '()' characters are not included in the parsed tokens
self.assertParseAndCheckLisst(group, "(abc 123 def)", ['abc', '123', 'def'])
# after exiting context manager, literals are converted to Literal expressions again
"""

def __init__(self):
self._save_context = {}

def save(self):
self._save_context["default_whitespace"] = ParserElement.DEFAULT_WHITE_CHARS
self._save_context["default_keyword_chars"] = Keyword.DEFAULT_KEYWORD_CHARS
self._save_context[
"literal_string_class"
] = ParserElement._literalStringClass
self._save_context["packrat_enabled"] = ParserElement._packratEnabled
self._save_context["packrat_parse"] = ParserElement._parse
self._save_context["__diag__"] = {
name: getattr(__diag__, name) for name in __diag__._all_names
}
self._save_context["__compat__"] = {
"collect_all_And_tokens": __compat__.collect_all_And_tokens
}
return self

def restore(self):
# reset pyparsing global state
if (
ParserElement.DEFAULT_WHITE_CHARS
!= self._save_context["default_whitespace"]
):
ParserElement.setDefaultWhitespaceChars(
self._save_context["default_whitespace"]
)
Keyword.DEFAULT_KEYWORD_CHARS = self._save_context["default_keyword_chars"]
ParserElement.inlineLiteralsUsing(
self._save_context["literal_string_class"]
)
for name, value in self._save_context["__diag__"].items():
setattr(__diag__, name, value)
ParserElement._packratEnabled = self._save_context["packrat_enabled"]
ParserElement._parse = self._save_context["packrat_parse"]
__compat__.collect_all_And_tokens = self._save_context["__compat__"]

def __enter__(self):
return self.save()

def __exit__(self, *args):
return self.restore()

class TestParseResultsAsserts:
"""
A mixin class to add parse results assertion methods to normal unittest.TestCase classes.
"""
def assertParseResultsEquals(
self, result, expected_list=None, expected_dict=None, msg=None
):
"""
Unit test assertion to compare a ParseResults object with an optional expected_list,
and compare any defined results names with an optional expected_dict.
"""
if expected_list is not None:
self.assertEqual(expected_list, result.asList(), msg=msg)
if expected_dict is not None:
self.assertEqual(expected_dict, result.asDict(), msg=msg)

def assertParseAndCheckList(
self, expr, test_string, expected_list, msg=None, verbose=True
):
"""
Convenience wrapper assert to test a parser element and input string, and assert that
the resulting ParseResults.asList() is equal to the expected_list.
"""
result = expr.parseString(test_string, parseAll=True)
if verbose:
print(result.dump())
self.assertParseResultsEquals(result, expected_list=expected_list, msg=msg)

def assertParseAndCheckDict(
self, expr, test_string, expected_dict, msg=None, verbose=True
):
"""
Convenience wrapper assert to test a parser element and input string, and assert that
the resulting ParseResults.asDict() is equal to the expected_dict.
"""
result = expr.parseString(test_string, parseAll=True)
if verbose:
print(result.dump())
self.assertParseResultsEquals(result, expected_dict=expected_dict, msg=msg)

def assertRunTestResults(
self, run_tests_report, expected_parse_results=None, msg=None
):
"""
Unit test assertion to evaluate output of ParserElement.runTests(). If a list of
list-dict tuples is given as the expected_parse_results argument, then these are zipped
with the report tuples returned by runTests and evaluated using assertParseResultsEquals.
Finally, asserts that the overall runTests() success value is True.
:param run_tests_report: tuple(bool, [tuple(str, ParseResults or Exception)]) returned from runTests
:param expected_parse_results (optional): [tuple(str, list, dict, Exception)]
"""
run_test_success, run_test_results = run_tests_report

if expected_parse_results is not None:
merged = [
(rpt[0], rpt[1], expected)
for rpt, expected in zip(run_test_results, expected_parse_results)
]
for test_string, result, expected in merged:
# expected should be a tuple containing a list and/or a dict or an exception,
# and optional failure message string
# an empty tuple will skip any result validation
fail_msg = next(
(exp for exp in expected if isinstance(exp, str)), None
)
expected_exception = next(
(
exp
for exp in expected
if isinstance(exp, type) and issubclass(exp, Exception)
),
None,
)
if expected_exception is not None:
with self.assertRaises(
expected_exception=expected_exception, msg=fail_msg or msg
):
if isinstance(result, Exception):
raise result
else:
expected_list = next(
(exp for exp in expected if isinstance(exp, list)), None
)
expected_dict = next(
(exp for exp in expected if isinstance(exp, dict)), None
)
if (expected_list, expected_dict) != (None, None):
self.assertParseResultsEquals(
result,
expected_list=expected_list,
expected_dict=expected_dict,
msg=fail_msg or msg,
)
else:
# warning here maybe?
print("no validation for {!r}".format(test_string))

# do this last, in case some specific test results can be reported instead
self.assertTrue(
run_test_success, msg=msg if msg is not None else "failed runTests"
)

@contextmanager
def assertRaisesParseException(self, exc_type=ParseException, msg=None):
with self.assertRaises(exc_type, msg=msg):
yield


if __name__ == "__main__":

selectToken = CaselessLiteral("select")
Expand Down
29 changes: 26 additions & 3 deletions unitTests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

from unittest import TestCase, TestSuite, TextTestRunner
import datetime
from pyparsing import ParseException
from pyparsing import ParseException, pyparsing_test as ppt
import pyparsing as pp

import sys
Expand Down Expand Up @@ -82,7 +82,7 @@ def __exit__(self, *args):

BUFFER_OUTPUT = True

class ParseTestCase(TestCase):
class ParseTestCase(ppt.TestParseResultsAsserts, TestCase):
def __init__(self):
super(ParseTestCase, self).__init__(methodName='_runTest')
self.expect_traceback = False
Expand All @@ -99,7 +99,8 @@ def _runTest(self):
sys.stdout = buffered_stdout
sys.stderr = buffered_stdout
print_(">>>> Starting test",str(self))
self.runTest()
with ppt.reset_pyparsing_context():
self.runTest()

finally:
print_("<<<< End of test",str(self))
Expand Down Expand Up @@ -4731,6 +4732,27 @@ def runTest(self):
""")


class ChainedTernaryOperator(ParseTestCase):
def runTest(self):
import pyparsing as pp

TERNARY_INFIX = pp.infixNotation(
pp.pyparsing_common.integer, [
(("?", ":"), 3, pp.opAssoc.LEFT),
])
self.assertParseAndCheckList(TERNARY_INFIX,
"1?1:0?1:0",
[[1, '?', 1, ':', 0, '?', 1, ':', 0]])

TERNARY_INFIX = pp.infixNotation(
pp.pyparsing_common.integer, [
(("?", ":"), 3, pp.opAssoc.RIGHT),
])
self.assertParseAndCheckList(TERNARY_INFIX,
"1?1:0?1:0",
[[1, '?', 1, ':', [0, '?', 1, ':', 0]]])


class MiscellaneousParserTests(ParseTestCase):
def runTest(self):
self.expect_warning = True
Expand Down Expand Up @@ -4981,6 +5003,7 @@ def makeTestSuiteTemp(classes):
# run specific tests by including them in this list, otherwise
# all tests will be run
testclasses = [
ChainedTernaryOperator
]

if not testclasses:
Expand Down

0 comments on commit 0a6cef0

Please sign in to comment.