Added optional repetition to parser.

Any item can now be optionally repeated by adding the special sequence
"[...]" after it. This is can be used with Token (or any derived
version), AnyToken, Sequence or Alternation. Attempting to place it
after AnyTokenString or another Repeater will result in a parse error,
as these cases are nonsensical. As a result of the possiblity of
repetition, the "fields" dictionary is now populated with a list of
values in the order they were matched from the commandline.

Unit tests have been updated with basic functional tests for repetition
but, as always, more cases might be useful. Also, additional tracing
possibilities have been added for debugging parse errors.

cmdparser.py

@@ -1,7 +1,6 @@
 """cmdparser - A simple command parsing library."""
 
 
-import cmd
 import itertools
 import shlex
 
@@ -32,6 +31,11 @@ def __del__(self):
             self.trace.append("<<< " + self.name)
 
 
+    def fail(self, items):
+        if self.trace is not None:
+            self.trace.append("!! " + self.name + " [" + " ".join(items) + "]")
+
+
 
 class ParseItem(object):
     """Base class for all items in a command specification."""
@@ -57,6 +61,14 @@ def add(self, child):
         raise ParseError("children not allowed")
 
 
+    def pop(self):
+        """Called to recover and remove the most recently-added child item.
+
+        The default is to disallow children, derived classes can override.
+        """
+        raise ParseError("children not allowed")
+
+
     def add_alternate(self):
         """Called to add a new alternate option.
 
@@ -159,6 +171,15 @@ def add(self, child):
         self.items.append(child)
 
 
+    def pop(self):
+        """See ParseItem.pop()."""
+
+        try:
+            return self.items.pop()
+        except IndexError:
+            raise ParseError("no child item to pop")
+
+
     def match(self, compare_items, fields=None, completions=None, trace=None,
               context=None):
         """See ParseItem.match()."""
@@ -172,6 +193,54 @@ def match(self, compare_items, fields=None, completions=None, trace=None,
 
 
 
+class Repeater(ParseItem):
+    """Matches a single specified item one or more times."""
+
+    def __init__(self):
+        self.item = None
+
+
+    def __str__(self):
+        return str(self.item) + " [...]"
+
+
+    def finalise(self):
+        """See ParseItem.finalise()."""
+        if self.item is None:
+            raise ParseError("empty repeater")
+
+
+    def add(self, child):
+        """See ParseItem.add()."""
+
+        assert isinstance(child, ParseItem)
+        if isinstance(child, Repeater) or isinstance(child, AnyTokenString):
+            raise ParseError("repeater cannot accept a repeating child")
+        if self.item is not None:
+            raise ParseError("repeater may only have a single child")
+        self.item = child
+
+
+    def match(self, compare_items, fields=None, completions=None, trace=None,
+              context=None):
+
+        tracer = CallTracer(trace, "Repeater")
+        repeats = 0
+        while True:
+            try:
+                new_items = self.item.match(compare_items, fields=fields,
+                                            completions=completions,
+                                            trace=trace, context=context)
+                compare_items = new_items
+                repeats += 1
+            except MatchError, e:
+                if repeats == 0:
+                    tracer.fail(e.args[0])
+                    raise
+                return compare_items
+
+
+
 class Alternation(ParseItem):
     """Matches any of a list of alternative Sequence items.
 
@@ -213,6 +282,12 @@ def add(self, child):
         self.options[-1].add(child)
 
 
+    def pop(self):
+        """See ParseItem.pop()."""
+
+        return self.options[-1].pop()
+
+
     def add_alternate(self):
         """See ParseItem.add_alternate()."""
 
@@ -236,6 +311,7 @@ def match(self, compare_items, fields=None, completions=None, trace=None,
         if self.optional:
             return compare_items
         else:
+            tracer.fail(remaining)
             raise MatchError(remaining)
 
 
@@ -277,12 +353,14 @@ def match(self, compare_items, fields=None, completions=None, trace=None,
         if not compare_items:
             if completions is not None:
                 completions.update(self.get_values(context))
+            tracer.fail([])
             raise MatchError([])
         for value in self.get_values(context):
             if compare_items and compare_items[0] == value:
                 if fields is not None:
-                    fields[self.name] = value
+                    fields.setdefault(self.name, []).append(value)
                 return compare_items[1:]
+        tracer.fail(compare_items)
         raise MatchError(compare_items)
 
 
@@ -302,9 +380,10 @@ def match(self, compare_items, fields=None, completions=None, trace=None,
               context=None):
         tracer = CallTracer(trace, "AnyToken(%s)" % (self.name,))
         if not compare_items:
+            tracer.fail([])
             raise MatchError([])
         if fields is not None:
-            fields[self.name] = compare_items[0]
+            fields.setdefault(self.name, []).append(compare_items[0])
         return compare_items[1:]
 
 
@@ -324,9 +403,10 @@ def match(self, compare_items, fields=None, completions=None, trace=None,
               context=None):
         tracer = CallTracer(trace, "AnyTokenString(%s)" % (self.name,))
         if not compare_items:
+            tracer.fail([])
             raise MatchError([])
         if fields is not None:
-            fields[self.name] = " ".join(compare_items)
+            fields.setdefault(self.name, []).extend(compare_items)
         return []
 
 
@@ -337,8 +417,16 @@ def parse_spec(spec, ident_factory=None):
     token = ""
     name = None
     ident = False
+    skip_chars = 0
+
+    for num, chars in ((i+1, spec[i:]) for i in xrange(len(spec))):
 
-    for num, char in enumerate(spec, 1):
+        if skip_chars:
+            skip_chars -= 1
+            continue
+
+        # Most matching happens on only the first character.
+        char = chars[0]
 
         # Perform correctness checks.
         if ident and (char in ":()[]|<" or char.isspace()):
@@ -362,7 +450,20 @@ def parse_spec(spec, ident_factory=None):
         if char == "(":
             stack.append(Alternation())
         elif char == "[":
-            stack.append(Alternation(optional=True))
+            # String [...] is a special case meaning "optionally repeat last
+            # item". We recover the last item from the latest stack item
+            # and wrap it in a Repeater.
+            if chars[:5] == "[...]":
+                try:
+                    last_item = stack[-1].pop()
+                    repeater = Repeater()
+                    repeater.add(last_item)
+                    stack[-1].add(repeater)
+                    skip_chars = 4
+                except ParseError:
+                    raise ParseError("no token to repeat at char %d" % (num,))
+            else:
+                stack.append(Alternation(optional=True))
         elif char == "<":
             ident = True
         elif char == "|":

test_cmdparser.py

@@ -32,6 +32,18 @@ def test_parse_alternation(self):
             self.assertEqual(item.items[0].name, spec_item)
 
 
+    def test_parse_repeat_token(self):
+        spec = "one two [...]"
+        tree = cmdparser.parse_spec(spec)
+        self.assertIsInstance(tree, cmdparser.Sequence)
+        self.assertEqual(len(tree.items), 2)
+        self.assertIsInstance(tree.items[0], cmdparser.Token)
+        self.assertEqual(tree.items[0].name, "one")
+        self.assertIsInstance(tree.items[1], cmdparser.Repeater)
+        self.assertIsInstance(tree.items[1].item, cmdparser.Token)
+        self.assertEqual(tree.items[1].item.name, "two")
+
+
     def test_parse_optional(self):
         spec = "one [two] three"
         tree = cmdparser.parse_spec(spec)
@@ -52,7 +64,7 @@ def test_parse_optional(self):
 
     def test_parse_identifier(self):
         class XYZIdent(cmdparser.Token):
-            def get_values(self):
+            def get_values(self, context):
                 return ["x", "y", "z"]
         def ident_factory(ident):
             if ident == "three":
@@ -74,7 +86,7 @@ def ident_factory(ident):
 
     def test_parse_full(self):
         class XYZIdent(cmdparser.Token):
-            def get_values(self):
+            def get_values(self, context):
                 return ["x", "y", "z"]
         def ident_factory(ident):
             if ident == "five":
@@ -141,6 +153,34 @@ def test_match_alternation(self):
         self.assertEqual(tree.check_match([]), "")
 
 
+    def test_match_repeat_token(self):
+        spec = "one two [...] three"
+        tree = cmdparser.parse_spec(spec)
+        self.assertEqual(tree.check_match(("one", "two", "three")), None)
+        fields = {}
+        self.assertEqual(tree.check_match(("one", "two", "two", "three"),
+                                          fields=fields), None)
+        self.assertEqual(fields, {"one": ["one"], "two": ["two", "two"],
+                         "three": ["three"]})
+        self.assertEqual(tree.check_match(("one", "two", "two", "two",
+                                           "three")), None)
+        self.assertEqual(tree.check_match(("one",)), "")
+        self.assertEqual(tree.check_match(("one", "three")), "three")
+        self.assertEqual(tree.check_match(("one", "two", "three", "two")),
+                         "two")
+
+
+    def test_match_repeat_sequence(self):
+        spec = "(one two) [...] three"
+        tree = cmdparser.parse_spec(spec)
+        self.assertEqual(tree.check_match(("one", "two", "three")), None)
+        self.assertEqual(tree.check_match(("one", "two", "one", "two",
+                                           "three")), None)
+        self.assertEqual(tree.check_match(("one", "two", "one", "three")),
+                         "one") # Fails on "one" is correct!
+        self.assertEqual(tree.check_match(("one", "two", "two")), "two")
+
+
     def test_match_optional(self):
         spec = "one [two] three"
         tree = cmdparser.parse_spec(spec)
@@ -159,7 +199,7 @@ def test_match_optional(self):
 
     def test_match_identifier(self):
         class XYZIdent(cmdparser.Token):
-            def get_values(self):
+            def get_values(self, context):
                 return ["x", "y", "z"]
         def ident_factory(ident):
             if ident == "three":
@@ -170,21 +210,21 @@ def ident_factory(ident):
         fields = {}
         self.assertEqual(tree.check_match(("one", "foo", "x", "a", "b"),
                                           fields=fields), None)
-        self.assertEqual(fields, {"one": "one", "two": "foo", "three": "x",
-                                  "four": "a b"})
+        self.assertEqual(fields, {"one": ["one"], "two": ["foo"],
+                                  "three": ["x"], "four": ["a", "b"]})
         fields = {}
         self.assertEqual(tree.check_match(("one", "bar", "z", "baz"),
                                           fields=fields), None)
-        self.assertEqual(fields, {"one": "one", "two": "bar", "three": "z",
-                                  "four": "baz"})
+        self.assertEqual(fields, {"one": ["one"], "two": ["bar"],
+                                  "three": ["z"], "four": ["baz"]})
         self.assertEqual(tree.check_match(("one", "foo", "x")), "")
         self.assertEqual(tree.check_match(("one", "foo", "w", "a")), "w")
         self.assertEqual(tree.check_match(("one", "x", "a")), "a")
 
 
     def test_match_full(self):
         class XYZIdent(cmdparser.Token):
-            def get_values(self):
+            def get_values(self, context):
                 return ["x", "y", "z"]
         def ident_factory(ident):
             if ident == "five":
@@ -195,22 +235,24 @@ def ident_factory(ident):
         fields = {}
         self.assertEqual(tree.check_match(("one", "two", "three", "six",
                                            "foo", "bar"), fields=fields), None)
-        self.assertEqual(fields, {"one": "one", "two": "two", "three": "three",
-                                  "six": "six", "eight": "foo bar"})
+        self.assertEqual(fields, {"one": ["one"], "two": ["two"],
+                                  "three": ["three"], "six": ["six"],
+                                  "eight": ["foo", "bar"]})
         fields = {}
         self.assertEqual(tree.check_match(("one", "four", "seven", "foo"),
                                           fields=fields), None)
-        self.assertEqual(fields, {"one": "one", "four": "four",
-                                  "seven": "seven", "eight": "foo"})
+        self.assertEqual(fields, {"one": ["one"], "four": ["four"],
+                                  "seven": ["seven"], "eight": ["foo"]})
         fields = {}
         self.assertEqual(tree.check_match(("one", "four", "foo"),
                                           fields=fields), None)
-        self.assertEqual(fields, {"one": "one", "four": "four", "eight": "foo"})
+        self.assertEqual(fields, {"one": ["one"], "four": ["four"],
+                                  "eight": ["foo"]})
         fields = {}
         self.assertEqual(tree.check_match(("one", "four", "x", "foo", "bar"),
                                           fields=fields), None)
-        self.assertEqual(fields, {"one": "one", "four": "four", "five": "x",
-                                  "eight": "foo bar"})
+        self.assertEqual(fields, {"one": ["one"], "four": ["four"],
+                                  "five": ["x"], "eight": ["foo", "bar"]})
 
         self.assertEqual(tree.check_match(("one", "two", "foo")), "foo")
         self.assertEqual(tree.check_match(("one", "four", "x")), "")
@@ -245,6 +287,16 @@ def test_complete_alternation(self):
                          set())
 
 
+    def test_complete_repeat_token(self):
+        spec = "one two [...]"
+        tree = cmdparser.parse_spec(spec)
+        self.assertEqual(tree.get_completions(()), set(("one",)))
+        self.assertEqual(tree.get_completions(("one",)), set(("two",)))
+        self.assertEqual(tree.get_completions(("one", "two")), set(("two",)))
+        self.assertEqual(tree.get_completions(("one", "two", "two")),
+                         set(("two",)))
+
+
     def test_complete_optional(self):
         spec = "one [two] three"
         tree = cmdparser.parse_spec(spec)
@@ -261,7 +313,7 @@ def test_complete_optional(self):
 
     def test_complete_identifier(self):
         class XYZIdent(cmdparser.Token):
-            def get_values(self):
+            def get_values(self, context):
                 return ["x", "y", "z"]
         def ident_factory(ident):
             if ident == "three":
@@ -282,7 +334,7 @@ def ident_factory(ident):
 
     def test_complete_full(self):
         class XYZIdent(cmdparser.Token):
-            def get_values(self):
+            def get_values(self, context):
                 return ["x", "y", "z"]
         def ident_factory(ident):
             if ident == "five":