Finished unordered and ordered lists

src/org/pegdown/AstNodeType.java

@@ -38,29 +38,30 @@ public interface AstNodeType {
     // lists
     static int BULLET_LIST = 26;
     static int ORDERED_LIST = 27;
-    static int LIST_ITEM = 28;
+    static int TIGHT_LIST_ITEM = 28;
+    static int LOOSE_LIST_ITEM = 29;
 
     // links
-    static int AUTO_LINK = 29;
-    static int EXP_LINK = 30;
-    static int EXP_IMG_LINK = 31;
-    static int MAIL_LINK = 32;
-    static int REF_LINK = 33;
-    static int REF_IMG_LINK = 34;
+    static int AUTO_LINK = 30;
+    static int EXP_LINK = 31;
+    static int EXP_IMG_LINK = 32;
+    static int MAIL_LINK = 33;
+    static int REF_LINK = 34;
+    static int REF_IMG_LINK = 35;
 
-    static int LINK_LABEL = 35;
-    static int LINK_REF = 36;
-    static int LINK_TITLE = 37;
-    static int LINK_URL = 38;
-    static int REFERENCE = 39;
+    static int LINK_LABEL = 36;
+    static int LINK_REF = 37;
+    static int LINK_TITLE = 38;
+    static int LINK_URL = 39;
+    static int REFERENCE = 40;
 
     // reverse resolution
     static String[] TYPE_NAMES = new String[]{
             "DEFAULT",
             "APOSTROPHE", "ELLIPSIS", "EMDASH", "ENDASH", "HTML", "LINEBREAK", "SPACE", "SPECIAL", "TEXT",
             "CODE", "EMPH", "H1", "H2", "H3", "H4", "H5", "H6", "STRONG", "SINGLE_QUOTED", "DOUBLE_QUOTED",
             "BLOCKQUOTE", "HRULE", "HTMLBLOCK", "PARA", "VERBATIM",
-            "BULLET_LIST", "ORDERED_LIST", "LIST_ITEM",
+            "BULLET_LIST", "ORDERED_LIST", "TIGHT_LIST_ITEM", "LOOSE_LIST_ITEM",
             "AUTO_LINK", "EXP_LINK", "EXP_IMG_LINK", "MAIL_LINK", "REF_LINK", "REF_IMG_LINK",
             "LINK_LABEL", "LINK_REF", "LINK_TITLE", "LINK_URL", "REFERENCE"
     };

src/org/pegdown/MarkDownParser.java

@@ -1,9 +1,8 @@
 package org.pegdown;
 
-import org.parboiled.BaseParser;
-import org.parboiled.ReportingParseRunner;
-import org.parboiled.Rule;
+import org.parboiled.*;
 import org.parboiled.annotations.Cached;
+import org.parboiled.annotations.SuppressSubnodes;
 import org.parboiled.common.StringUtils;
 import org.parboiled.google.base.Function;
 import org.parboiled.support.ParsingResult;
@@ -70,18 +69,20 @@ Rule BlockQuote() {
     }
 
     Rule Verbatim() {
+        Var<String> text = new Var<String>("");
+        Var<String> temp = new Var<String>("");
         return Sequence(
-                create(VERBATIM, ""),
                 OneOrMore(
                         Sequence(
-                                create(DEFAULT, ""),
-                                ZeroOrMore(Sequence(BlankLine(), UP2(value().addText("\n")))),
-                                NonblankIndentedLine(), UP2(value().addText(DOWN2(value().text) + lastValue().text))
+                                ZeroOrMore(Sequence(BlankLine(), temp.set(prevText()))),
+                                NonblankIndentedLine(), text.set(text.get() + temp.get() + prevValue().text)
                         )
-                )
+                ),
+                create(VERBATIM, text.get())
         );
     }
 
+    @SuppressSubnodes
     Rule HorizontalRule() {
         return Sequence(
                 NonindentSpace(),
@@ -172,9 +173,9 @@ Rule OrderedList() {
 
     Rule ListTight() {
         return Sequence(
-                ListItem(), create(DEFAULT, lastValue()),
+                ListItem(TIGHT_LIST_ITEM), create(DEFAULT, lastValue()),
                 ZeroOrMore(
-                        Sequence(ListItem(), UP2(attach(lastValue())))
+                        Sequence(ListItem(TIGHT_LIST_ITEM), UP2(attach(lastValue())))
                 ),
                 ZeroOrMore(BlankLine()),
                 TestNot(FirstOf(Bullet(), Enumerator()))
@@ -183,41 +184,111 @@ Rule ListTight() {
 
     Rule ListLoose() {
         return Sequence(
-                ListItem(), create(DEFAULT, lastValue()),
+                ListItem(LOOSE_LIST_ITEM), create(DEFAULT, lastValue()),
                 ZeroOrMore(BlankLine()),
                 ZeroOrMore(
                         Sequence(
-                                ListItem(),
+                                ListItem(LOOSE_LIST_ITEM),
                                 UP2(attach(lastValue())),
                                 ZeroOrMore(BlankLine())
                         )
                 )
         );
     }
 
-    Rule ListItem() {
+    Rule ListItem(int type) {
+        // for a simpler parser design we use a recursive parsing strategy for list items:
+        // we collect the markdown source for an item, run a complete parsing cycle on this inner source and attach
+        // the root of the inner parsing results AST to the outer AST tree
+        // we use two helper variables in this ListItem rule, and since we cannot write to local variables from within
+        // a rule we have to use one level of indirection, org.parboiled.Var objects are the best solution for this,
+        // since they also provide for proper frame management in the case of rule recursions
+
+        Var<String> innerSource = new Var<String>();
+        Var<String> blanks = new Var<String>("");
+        Var<String> extraNLs = new Var<String>("");
+
         return Sequence(
                 FirstOf(Bullet(), Enumerator()),
-                create(LIST_ITEM, ""),
-                ListBlock(), value().addText(lastText()),
+
+                ListBlock(),
+                innerSource.set(lastNode().getValue().text), // set the inner source to the block text
+
                 ZeroOrMore(
                         Sequence(
-                                ZeroOrMore(BlankLine()),
+                                FirstOf(
+                                        // if we have blank lines append them to the inner source
+                                        Sequence(
+                                                OneOrMore(BlankLine()),
+                                                blanks.set(lastText())
+                                        ),
+
+                                        // if we do not have a blank line we need to trigger an inner parse
+                                        // so store the current inner source value in the helper string
+                                        // and reinitialize the inner source
+                                        blanks.set("\u0001")
+                                ),
                                 OneOrMore(
                                         Sequence(
-                                                Indent(),
-                                                ListBlock(),
-                                                UP4(value().addText(lastText()))
+                                                Indent(), ListBlock(),
+
+                                                // append potentially captured blanks and the block text
+                                                innerSource.set(act(innerSource.get(), blanks.exchange(""), lastNode()
+                                                        .getValue().text))
                                         )
-                                )
+                                ),
+                                extraNLs.set("\n\n")
                         )
                 ),
-                set(parseRawBlock(value().text).parseTreeRoot.getValue().setType(LIST_ITEM))
+
+                // finally, after having built the complete source we run an inner parse and attach its AST root
+                listAction(type, innerSource.get() + extraNLs.get())
         );
     }
 
+    String act(String innerSource, String blanks, String text) {
+        return innerSource + blanks + text;
+    }
+
+    boolean listAction(int type, String innerSource) {
+        int start = 0;
+        int end = innerSource.indexOf('\u0001', start);
+        if (end == -1) {
+            Context<AstNode> context = getContext();
+            AstNode astNode = parseRawBlock(type == LOOSE_LIST_ITEM ? innerSource + "\n\n" : innerSource).parseTreeRoot
+                    .getValue();
+            setContext(context);
+            return set(astNode.setType(type));
+        }
+
+        create(type);
+
+        while (true) {
+            end = innerSource.indexOf('\u0001', start);
+            if (end == -1) end = innerSource.length();
+            String sourcePart = innerSource.substring(start, end);
+
+            Context<AstNode> context = getContext();
+            AstNode astNode = parseRawBlock(type == LOOSE_LIST_ITEM ? sourcePart + "\n\n" : sourcePart).parseTreeRoot
+                    .getValue();
+            setContext(context);
+            attach(astNode.getChildren().get(0));
+
+            if (end == innerSource.length()) return true;
+            start = end + 1;
+        }
+    }
+
     Rule ListBlock() {
-        return Sequence(Line(), ZeroOrMore(ListBlockLine()));
+        Var<String> source = new Var<String>();
+        return Sequence(
+                Line(),
+                source.set(prevValue().text),
+                ZeroOrMore(
+                        Sequence(ListBlockLine(), source.set(source.get() + prevValue().text))
+                ),
+                create(DEFAULT, source.get())
+        );
     }
 
     Rule ListBlockLine() {
@@ -240,12 +311,13 @@ Rule Bullet() {
     //************* HTML BLOCK ****************
 
     Rule HtmlBlock() {
+        Var<String> blockSource = new Var<String>();
         return Sequence(
                 Sequence(
-                        FirstOf(HtmlBlockInTags(), HtmlComment(), HtmlBlockSelfClosing()), UP(put()),
+                        FirstOf(HtmlBlockInTags(), HtmlComment(), HtmlBlockSelfClosing()), blockSource.set(prevText()),
                         OneOrMore(BlankLine())
                 ),
-                create(HTMLBLOCK, text(get()))
+                create(HTMLBLOCK, blockSource.get())
         );
     }
 
@@ -430,16 +502,18 @@ Rule ExplicitLink() {
     }
 
     Rule ReferenceLink() {
+        Var<String> spaceSource = new Var<String>();
         return Sequence(
                 Label(), create(REF_LINK, lastValue()),
                 FirstOf(
                         // regular reference link
-                        Sequence(Spn1(), UP2(put()), TestNot("[]"), Label(), set(lastValue().setType(LINK_REF))),
+                        Sequence(Spn1(), spaceSource.set(prevText()), TestNot("[]"), Label(),
+                                set(lastValue().setType(LINK_REF))),
 
                         // implicit reference link
-                        Optional(Sequence(Spn1(), UP2(put()), "[]", create(LINK_REF, "")))
+                        Optional(Sequence(Spn1(), spaceSource.set(prevText()), "[]", create(LINK_REF, "")))
                 ),
-                attach(new AstNode().setType(SPACE).setText(text(get()))),
+                attach(new AstNode().setType(SPACE).setText(spaceSource.get())),
                 attach()
         );
     }
@@ -555,9 +629,8 @@ ticks, Sp(),
                                         FirstOf(Spacechar(), Sequence(Newline(), TestNot(BlankLine()))))
                         )
                 ),
-                put(),
-                Sp(), ticks,
-                create(CODE, text(get()))
+                create(CODE, prevText()),
+                Sp(), ticks
         );
     }
 
@@ -602,8 +675,9 @@ Rule NonblankIndentedLine() {
         return Sequence(TestNot(BlankLine()), IndentedLine());
     }
 
+    @SuppressSubnodes
     Rule BlankLine() {
-        return Sequence(Sp(), Newline());
+        return Sequence(Sp(), Newline(), create(DEFAULT, lastText()));
     }
 
     Rule IndentedLine() {
@@ -617,10 +691,13 @@ Rule OptionallyIndentedLine() {
     Rule Line() {
         return Sequence(
                 FirstOf(
-                        Sequence(ZeroOrMore(Sequence(TestNot('\r'), TestNot('\n'), Any())).label("line"), Newline()),
-                        Sequence(OneOrMore(Any()).label("line"), Eoi())
+                        Sequence(
+                                ZeroOrMore(Sequence(TestNot('\r'), TestNot('\n'), Any())).suppressSubnodes(),
+                                Newline()
+                        ),
+                        Sequence(OneOrMore(Any()).suppressSubnodes(), Test(Eoi()))
                 ),
-                create(DEFAULT, text(nodeByLabel("line")) + '\n')
+                create(DEFAULT, lastText())
         );
     }
 
@@ -669,7 +746,7 @@ Rule CharEntity() {
 
     Rule Str() {
         return Sequence(
-                Sequence(NormalChar(), ZeroOrMore(Sequence(ZeroOrMore('_'), NormalChar()))),
+                Sequence(NormalChar(), ZeroOrMore(Sequence(ZeroOrMore('_'), NormalChar()))).suppressSubnodes(),
                 create(TEXT, lastText())
         );
     }

src/org/pegdown/PegDownProcessor.java

@@ -158,8 +158,10 @@ private PegDownProcessor print(AstNode node) {
                 return printOnNL("<ul>").indent(+2).printChildren(node).indent(-2).printOnNL("</ul>");
             case ORDERED_LIST:
                 return printOnNL("<ol>").indent(+2).printChildren(node).indent(-2).printOnNL("</ol>");
-            case LIST_ITEM:
+            case TIGHT_LIST_ITEM:
                 return printOnNL("<li>").printChildren(node).print("</li>");
+            case LOOSE_LIST_ITEM:
+                return printOnNL("<li>").indent(+2).printChildren(node).indent(-2).printOnNL("</li>");
 
             // links
             case AUTO_LINK:

test-cases/Auto links.ast.text

@@ -13,17 +13,17 @@ DEFAULT
         SPACE: " "
         AUTO_LINK: "http://example.com/?foo=1&bar=2"
     BULLET_LIST
-        LIST_ITEM
+        TIGHT_LIST_ITEM
             DEFAULT
                 TEXT: "In"
                 SPACE: " "
                 TEXT: "a"
                 SPACE: " "
                 TEXT: "list?"
-        LIST_ITEM
+        TIGHT_LIST_ITEM
             DEFAULT
                 AUTO_LINK: "http://example.com/"
-        LIST_ITEM
+        TIGHT_LIST_ITEM
             DEFAULT
                 TEXT: "It"
                 SPACE: " "