add function call and $sep syntax

SYNTAX.md

@@ -128,6 +128,7 @@ Patterns define the structure of the input that matches a production rule.
  - `P*` : Zero or more repetitions of `P`
  - `P+` : One or more repetitions of `P`
  - `P?` : Zero or one repetition of `P`
+ - `$sep( P, P_separator, repetition )`: A repetition of `P` separated by `P_separator`. The `repetition` can be `*`, or `+` to indicate zero or more, or one or more repetitions respectively
  - `(P1 P2 P3)` : Grouping of patterns
  - `P / term` or `P / [term1 term_start-term_last]`: Pattern `P` followed by lookaheads. Lookaheads will not be consumed
  - `'a'` or `b'a'`: Single character literal or byte literal. This is only supported if the `%tokentype` is `char` or `u8`
@@ -233,6 +234,7 @@ For some regex patterns, the type of variable will be modified as follows:
  - `P*` : `Vec<P>`
  - `P+` : `Vec<P>`
  - `P?` : `Option<P>`
+ - `$sep(P, P_separator, repetetion)` : `Vec<P>`
 
 You can still access the `Vec` or `Option` by using the base name of the pattern.
 ```rust

example/json/src/parser.rs

@@ -35,12 +35,10 @@ Members: Member
 
 Member: WS String WS ':' Element;
 
-Array: '[' WS ']'
-     | '[' Elements ']'
+Array: '[' Elements ']'
      ;
 
-Elements: Element
-        | Element ',' Elements
+Elements: $sep(Element, ',', *)
         ;
 
 Element: WS Value WS;

rusty_lr_core/src/tree.rs

@@ -77,58 +77,113 @@ impl<Term, NonTerm> TreeNonTerminal<Term, NonTerm> {
             // remove left/right recursion, make it to flat array
             Some(NonTerminalType::PlusLeft) => {
                 let tree = TermTree::new(nonterm_name);
-                let tree = if self.tokens.len() == 1 {
-                    let child = self.tokens[0]
-                        .to_term_tree(term_to_display, nonterm_to_display)
-                        .into_iter()
-                        .next()
-                        .unwrap();
-                    tree.with_leaves([child])
-                } else {
-                    // 2
-                    let mut child_list = self.tokens[0]
-                        .to_term_tree(term_to_display, nonterm_to_display)
-                        .into_iter()
-                        .next()
-                        .unwrap()
-                        .leaves;
-                    let child = self.tokens[1]
-                        .to_term_tree(term_to_display, nonterm_to_display)
-                        .into_iter()
-                        .next()
-                        .unwrap();
-                    child_list.push(child);
-                    tree.with_leaves(child_list)
+                let tree = match self.tokens.len() {
+                    1 => {
+                        let child = self.tokens[0]
+                            .to_term_tree(term_to_display, nonterm_to_display)
+                            .into_iter()
+                            .next()
+                            .unwrap();
+                        tree.with_leaves([child])
+                    }
+                    2 => {
+                        let mut child_list = self.tokens[0]
+                            .to_term_tree(term_to_display, nonterm_to_display)
+                            .into_iter()
+                            .next()
+                            .unwrap()
+                            .leaves;
+                        let child = self.tokens[1]
+                            .to_term_tree(term_to_display, nonterm_to_display)
+                            .into_iter()
+                            .next()
+                            .unwrap();
+                        child_list.push(child);
+                        tree.with_leaves(child_list)
+                    }
+                    3 => {
+                        let mut child_list = self.tokens[0]
+                            .to_term_tree(term_to_display, nonterm_to_display)
+                            .into_iter()
+                            .next()
+                            .unwrap()
+                            .leaves;
+                        let separator = self.tokens[1]
+                            .to_term_tree(term_to_display, nonterm_to_display)
+                            .into_iter()
+                            .next()
+                            .unwrap();
+                        let child = self.tokens[2]
+                            .to_term_tree(term_to_display, nonterm_to_display)
+                            .into_iter()
+                            .next()
+                            .unwrap();
+                        child_list.push(separator);
+                        child_list.push(child);
+                        tree.with_leaves(child_list)
+                    }
+                    _ => {
+                        unreachable!("PlusLeft length of child: {}", self.tokens.len())
+                    }
                 };
                 vec![tree]
             }
             // remove left/right recursion, make it to flat array
             Some(NonTerminalType::PlusRight) => {
                 let tree = TermTree::new(nonterm_name);
-                let tree = if self.tokens.len() == 1 {
-                    let child = self.tokens[0]
-                        .to_term_tree(term_to_display, nonterm_to_display)
-                        .into_iter()
-                        .next()
-                        .unwrap();
-                    tree.with_leaves([child])
-                } else {
-                    // 2
-                    let child = self.tokens[0]
-                        .to_term_tree(term_to_display, nonterm_to_display)
-                        .into_iter()
-                        .next()
-                        .unwrap();
-                    let mut child_list = self.tokens[1]
-                        .to_term_tree(term_to_display, nonterm_to_display)
-                        .into_iter()
-                        .next()
-                        .unwrap()
-                        .leaves;
-                    let mut children = vec![child];
-                    children.append(&mut child_list);
+                let tree = match self.tokens.len() {
+                    1 => {
+                        let child = self.tokens[0]
+                            .to_term_tree(term_to_display, nonterm_to_display)
+                            .into_iter()
+                            .next()
+                            .unwrap();
+                        tree.with_leaves([child])
+                    }
+                    2 => {
+                        let child = self.tokens[0]
+                            .to_term_tree(term_to_display, nonterm_to_display)
+                            .into_iter()
+                            .next()
+                            .unwrap();
+                        let mut child_list = self.tokens[1]
+                            .to_term_tree(term_to_display, nonterm_to_display)
+                            .into_iter()
+                            .next()
+                            .unwrap()
+                            .leaves;
+                        let mut children = vec![child];
+                        children.append(&mut child_list);
+
+                        tree.with_leaves(children)
+                    }
+                    3 => {
+                        // with separator
+                        let child = self.tokens[0]
+                            .to_term_tree(term_to_display, nonterm_to_display)
+                            .into_iter()
+                            .next()
+                            .unwrap();
+                        let separator = self.tokens[1]
+                            .to_term_tree(term_to_display, nonterm_to_display)
+                            .into_iter()
+                            .next()
+                            .unwrap();
+                        let mut child_list = self.tokens[2]
+                            .to_term_tree(term_to_display, nonterm_to_display)
+                            .into_iter()
+                            .next()
+                            .unwrap()
+                            .leaves;
+                        let mut children = vec![child];
+                        children.push(separator);
+                        children.append(&mut child_list);
 
-                    tree.with_leaves(children)
+                        tree.with_leaves(children)
+                    }
+                    _ => {
+                        unreachable!("PlusRight length of child: {}", self.tokens.len())
+                    }
                 };
                 vec![tree]
             }

rusty_lr_parser/src/parser/args.rs

@@ -94,6 +94,8 @@ pub enum PatternArgs {
 
     /// Pattern - Terminals exclusion
     Minus(Box<PatternArgs>, Box<PatternArgs>),
+
+    Sep(Box<PatternArgs>, Box<PatternArgs>, bool, SpanPair),
 }
 
 impl std::fmt::Display for PatternArgs {
@@ -125,6 +127,9 @@ impl std::fmt::Display for PatternArgs {
             PatternArgs::Minus(base, terminal_set) => {
                 write!(f, "{}-{}", base, terminal_set)
             }
+            PatternArgs::Sep(base, del, one, _) => {
+                write!(f, "$sep({base}, {del}, {})", if *one { '+' } else { '*' })
+            }
         }
     }
 }
@@ -276,6 +281,15 @@ impl PatternArgs {
                     Ok(pattern)
                 }
             }
+            PatternArgs::Sep(base, del, one, _) => {
+                let base = base.into_pattern(grammar, put_exclamation)?;
+                let del = del.into_pattern(grammar, false)?;
+                let pattern = Pattern {
+                    internal: PatternInternal::Sep(Box::new(base), Box::new(del), one),
+                    pretty_name: pretty_name.clone(),
+                };
+                Ok(pattern)
+            }
         }
     }
 
@@ -346,6 +360,10 @@ impl PatternArgs {
                     (true, true) => Ok((false, rhs_set.difference(&lhs_set).copied().collect())),
                 }
             }
+            PatternArgs::Sep(_, _, _, span) => {
+                let (first, last) = span.pair.unwrap();
+                Err(ParseError::OnlyTerminalSet(first, last))
+            }
         }
     }
     pub fn span_pair(&self) -> (Span, Span) {
@@ -372,6 +390,7 @@ impl PatternArgs {
             PatternArgs::Minus(base, terminal_set) => {
                 (base.span_pair().0, terminal_set.span_pair().1)
             }
+            PatternArgs::Sep(_, _, _, span) => span.pair.unwrap(),
         }
     }
 
@@ -432,6 +451,11 @@ impl PatternArgs {
                 terminal_set.range_resolve(grammar)?;
                 Ok(())
             }
+            PatternArgs::Sep(base, del, _, _) => {
+                base.range_resolve(grammar)?;
+                del.range_resolve(grammar)?;
+                Ok(())
+            }
         }
     }
 }

rusty_lr_parser/src/parser/lexer.rs

@@ -30,6 +30,8 @@ pub enum Lexed {
     Exclamation(Punct),
     Slash(Punct),
     Dot(Punct),
+    Dollar(Punct),
+    Comma(Punct),
     OtherPunct(Punct),
 
     Literal(Literal),
@@ -84,6 +86,8 @@ impl Lexed {
             Lexed::Exclamation(punct) => stream.append(punct),
             Lexed::Slash(punct) => stream.append(punct),
             Lexed::Dot(punct) => stream.append(punct),
+            Lexed::Dollar(punct) => stream.append(punct),
+            Lexed::Comma(punct) => stream.append(punct),
             Lexed::OtherPunct(punct) => stream.append(punct),
 
             Lexed::Literal(lit) => stream.append(lit),
@@ -183,7 +187,9 @@ impl std::fmt::Display for Lexed {
             Lexed::Exclamation(_) => write!(f, "'!'"),
             Lexed::Slash(_) => write!(f, "'/'"),
             Lexed::Dot(_) => write!(f, "'.'"),
-            Lexed::OtherPunct(_) => write!(f, "<Punct>"),
+            Lexed::Dollar(_) => write!(f, "'$'"),
+            Lexed::Comma(_) => write!(f, "','"),
+            Lexed::OtherPunct(p) => write!(f, "'{}'", p.as_char()),
 
             Lexed::ParenGroup(_) => write!(f, "<ParenGroup>"),
             Lexed::BraceGroup(_) => write!(f, "<BraceGroup>"),
@@ -318,6 +324,12 @@ pub fn feed_recursive(
                 '%' => {
                     context.feed_location(parser, Lexed::Percent(punct), grammar_args, location)?
                 }
+                '$' => {
+                    context.feed_location(parser, Lexed::Dollar(punct), grammar_args, location)?
+                }
+                ',' => {
+                    context.feed_location(parser, Lexed::Comma(punct), grammar_args, location)?
+                }
                 _ => context.feed_location(
                     parser,
                     Lexed::OtherPunct(punct),