diff --git a/peppercorn/__init__.py b/peppercorn/__init__.py index 8d988e6..217fa29 100644 --- a/peppercorn/__init__.py +++ b/peppercorn/__init__.py @@ -1,11 +1,3 @@ -import functools -from peppercorn.compat import next - -def data_type(value): - if ':' in value: - return [ x.strip() for x in value.rsplit(':', 1) ] - return ('', value.strip()) - START = '__start__' END = '__end__' SEQUENCE = 'sequence' @@ -15,43 +7,36 @@ def data_type(value): class ParseError(Exception): """ - An exception raised by :func:`parse` when the input is malformed. + An exception raised when the input is malformed. """ -def stream(next_token_gen, token): - """ - thanks to the effbot for - http://effbot.org/zone/simple-iterator-parser.htm +class RenameList(list): pass + + +_COLLECTION_TYPES = { + SEQUENCE: list, + MAPPING: dict, + RENAME: RenameList, +} + + +def data_type(marker): + """Extract the name and the data type from a start marker. + + Return the name and a collection instance. """ - op, data = token - if op == START: - name, typ = data_type(data) - out = [] - if typ in (SEQUENCE, MAPPING, RENAME): - if typ in (SEQUENCE, RENAME): - out = [] - add = lambda x, y: out.append(y) - else: - out = {} - add = out.__setitem__ - token = next_token_gen() - op, data = token - while op != END: - key, val = stream(next_token_gen, token) - add(key, val) - token = next_token_gen() - op, data = token - if typ == RENAME: - if out: - out = out[0] - else: - out = '' - return name, out - else: - raise ParseError('Unknown stream start marker %s' % repr(token)) + if ':' in marker: + name, typ = [ x.strip() for x in marker.rsplit(':', 1) ] else: - return op, data + name = '' + typ = marker.strip() + try: + collection = _COLLECTION_TYPES[typ]() + except KeyError: + raise ParseError('Unknown stream start marker %s' % marker) + return name, collection + def parse(fields): """ Infer a data structure from the ordered set of fields and @@ -59,12 +44,45 @@ def parse(fields): A :exc:`ParseError` is raised if a data structure can't be inferred. """ - fields = [(START, MAPPING)] + list(fields) + [(END,'')] - src = iter(fields) - try: - result = stream(functools.partial(next, src), next(src))[1] - except StopIteration: + stack = [{}] + + def add_item(name, value): + """Add an item to the last collection in the stack""" + current = stack[-1] + if isinstance(current, dict): + current[name] = value + else: + current.append(value) + + for op, data in fields: + if op == START: + name, collection = data_type(data) + add_item(name, collection) + # Make future calls to `add_item` work on this collection: + stack.append(collection) + elif op == END: + if len(stack): + # Replace all instances of RenameList with their first item. + collection = stack[-1] + if isinstance(collection, dict): + items = collection.items() + else: + items = enumerate(collection) + rename_info = [] + for key, value in items: + if isinstance(value, RenameList): + rename_info.append((key, value[0] if value else '')) + for key, value in rename_info: + collection[key] = value + + if len(stack) > 1: + stack.pop() + else: + break + else: + add_item(op, data) + + if len(stack) > 1: raise ParseError('Unclosed sequence') - except RuntimeError: - raise ParseError('Input too deeply nested') - return result + + return stack[0] diff --git a/peppercorn/compat.py b/peppercorn/compat.py index bd9b7f8..ee3cab1 100644 --- a/peppercorn/compat.py +++ b/peppercorn/compat.py @@ -1,10 +1,2 @@ import sys PY3 = sys.version_info[0] == 3 - -try: - next = next -except NameError: # pragma: no cover - # for Python 2.5 - def next(gen): - return gen.next() - diff --git a/peppercorn/tests.py b/peppercorn/tests.py index 43de668..f66ba2f 100644 --- a/peppercorn/tests.py +++ b/peppercorn/tests.py @@ -140,11 +140,20 @@ def test_unclosed_sequence(self): def test_deep_nesting(self): import sys - from peppercorn import START, END, MAPPING, ParseError + from peppercorn import START, END, MAPPING depth = sys.getrecursionlimit() # Create a valid input nested deeper than the recursion limit: fields = [(START, 'x:' + MAPPING)] * depth + [(END, '')] * depth - self.assertRaises(ParseError, self._callFUT, fields) + result = self._callFUT(fields) + + temp = data = {} + for _ in range(depth): + temp['x'] = temp = {} + + # Don't do self.assertEqual(result, data) here as + # this will exceed the recursion limit: + deep_ok = repr(result) == repr(data) + self.assertTrue(deep_ok, 'deep nesting failed') def test_spurios_initial_end(self): from peppercorn import END