Handle arbitrarily nested input

peppercorn/__init__.py

@@ -1,11 +1,3 @@
-import functools
-from peppercorn.compat import next
-
-def data_type(value):
-    if ':' in value:
-        return [ x.strip() for x in value.rsplit(':', 1) ]
-    return ('', value.strip())
-
 START = '__start__'
 END = '__end__'
 SEQUENCE = 'sequence'
@@ -15,56 +7,82 @@ def data_type(value):
 
 class ParseError(Exception):
     """
-    An exception raised by :func:`parse` when the input is malformed.
+    An exception raised when the input is malformed.
     """
 
 
-def stream(next_token_gen, token):
-    """
-    thanks to the effbot for
-    http://effbot.org/zone/simple-iterator-parser.htm
+class RenameList(list): pass
+
+
+_COLLECTION_TYPES = {
+    SEQUENCE: list,
+    MAPPING: dict,
+    RENAME: RenameList,
+}
+
+
+def data_type(marker):
+    """Extract the name and the data type from a start marker.
+
+    Return the name and a collection instance.
     """
-    op, data = token
-    if op == START:
-        name, typ = data_type(data)
-        out = []
-        if typ in (SEQUENCE, MAPPING, RENAME):
-            if typ in (SEQUENCE, RENAME):
-                out = []
-                add = lambda x, y: out.append(y)
-            else:
-                out = {}
-                add = out.__setitem__
-            token = next_token_gen()
-            op, data = token
-            while op != END:
-                key, val = stream(next_token_gen, token)
-                add(key, val)
-                token = next_token_gen()
-                op, data = token
-            if typ == RENAME:
-                if out:
-                    out = out[0]
-                else:
-                    out = ''
-            return name, out
-        else:
-            raise ParseError('Unknown stream start marker %s' % repr(token))
+    if ':' in marker:
+        name, typ = [ x.strip() for x in marker.rsplit(':', 1) ]
     else:
-        return op, data
+        name = ''
+        typ = marker.strip()
+    try:
+        collection = _COLLECTION_TYPES[typ]()
+    except KeyError:
+        raise ParseError('Unknown stream start marker %s' % marker)
+    return name, collection
+
 
 def parse(fields):
     """ Infer a data structure from the ordered set of fields and
     return it.
 
     A :exc:`ParseError` is raised if a data structure can't be inferred.
     """
-    fields = [(START, MAPPING)] + list(fields) + [(END,'')]
-    src = iter(fields)
-    try:
-        result = stream(functools.partial(next, src), next(src))[1]
-    except StopIteration:
+    stack = [{}]
+
+    def add_item(name, value):
+        """Add an item to the last collection in the stack"""
+        current = stack[-1]
+        if isinstance(current, dict):
+            current[name] = value
+        else:
+            current.append(value)
+
+    for op, data in fields:
+        if op == START:
+            name, collection = data_type(data)
+            add_item(name, collection)
+            # Make future calls to `add_item` work on this collection:
+            stack.append(collection)
+        elif op == END:
+            if len(stack):
+                # Replace all instances of RenameList with their first item.
+                collection = stack[-1]
+                if isinstance(collection, dict):
+                    items = collection.items()
+                else:
+                    items = enumerate(collection)
+                rename_info = []
+                for key, value in items:
+                    if isinstance(value, RenameList):
+                        rename_info.append((key, value[0] if value else ''))
+                for key, value in rename_info:
+                    collection[key] = value
+
+                if len(stack) > 1:
+                    stack.pop()
+                else:
+                    break
+        else:
+            add_item(op, data)
+
+    if len(stack) > 1:
         raise ParseError('Unclosed sequence')
-    except RuntimeError:
-        raise ParseError('Input too deeply nested')
-    return result
+
+    return stack[0]

peppercorn/compat.py

@@ -1,10 +1,2 @@
 import sys
 PY3 = sys.version_info[0] == 3
-
-try:
-    next = next
-except NameError: # pragma: no cover
-    # for Python 2.5
-    def next(gen):
-        return gen.next()
-

peppercorn/tests.py

@@ -140,11 +140,20 @@ def test_unclosed_sequence(self):
 
     def test_deep_nesting(self):
         import sys
-        from peppercorn import START, END, MAPPING, ParseError
+        from peppercorn import START, END, MAPPING
         depth = sys.getrecursionlimit()
         # Create a valid input nested deeper than the recursion limit:
         fields = [(START, 'x:' + MAPPING)] * depth + [(END, '')] * depth
-        self.assertRaises(ParseError, self._callFUT, fields)
+        result = self._callFUT(fields)
+
+        temp = data = {}
+        for _ in range(depth):
+            temp['x'] = temp = {}
+
+        # Don't do self.assertEqual(result, data) here as
+        # this will exceed the recursion limit:
+        deep_ok = repr(result) == repr(data)
+        self.assertTrue(deep_ok, 'deep nesting failed')
 
     def test_spurios_initial_end(self):
         from peppercorn import END