Make generic improvements

builder.go

@@ -5,28 +5,33 @@ import (
 	"log"
 )
 
-type ParsingError struct{}
+type ParsingError struct {
+	errString string
+}
 
 func (e *ParsingError) Error() string {
-	// TODO: Must return useful detail. ex. no tokens left
-	return "parsing error"
+	return e.errString
+}
+
+func newParsingError(errString string) *ParsingError {
+	return &ParsingError{errString: errString}
 }
 
 // Builder helps in building a recursive descent parser.
 // It stores a slice of tokens and an index to the current token.
-// It maintains a stack used in building the parse tree (check Tree()).
+// It maintains a stack used in building the parse tree (check ParseTree()).
 // It also builds a debug tree that helps in understanding the parsing
 // flow (check DebugTree()).
 // Enter/Exit methods are used in logging enter and exit of non-terminal
 // functions.
-// Add/Next/Match/Reset are used while working with terminals.
+// Add/Next/Match/Backtrack are used while working with terminals.
 type Builder struct {
 	tokens         []Token
 	current        int
 	stack          stack
 	finalEle       ele
 	debugStack     debugStack
-	finalDebugTree *debugTree
+	finalDebugTree *DebugTree
 	finalErr       *ParsingError
 }
 
@@ -44,17 +49,21 @@ func NewBuilder(tokens []Token) *Builder {
 // no tokens are left, else true.
 func (b *Builder) Next() (token Token, ok bool) {
 	b.mustEnter("Next")
+	return b.next()
+}
+
+func (b *Builder) next() (token Token, ok bool) {
 	if b.current == len(b.tokens)-1 {
 		return nil, false
 	}
 	b.current++
 	return b.tokens[b.current], true
 }
 
-// Reset resets the current index for the current non-terminal, and discards
+// Backtrack resets the current index for the current non-terminal, and discards
 // any matches done inside it.
-func (b *Builder) Reset() {
-	b.mustEnter("Reset")
+func (b *Builder) Backtrack() {
+	b.mustEnter("Backtrack")
 	e := b.stack.peek()
 	b.current = e.index
 	e.nonTerm.Subtrees = []*Tree{}
@@ -116,40 +125,44 @@ func (b *Builder) Exit(result *bool) {
 		panic("Exit result cannot be nil")
 	}
 	e := b.stack.pop()
-	if *result {
-		if b.stack.isEmpty() {
-			b.finalEle = e
+	if *result && b.stack.isEmpty() {
+		if _, ok := b.next(); ok {
+			b.finalErr = newParsingError("tokens left after parsing")
 		} else {
-			parent := b.stack.peek()
-			parent.nonTerm.Add(e.nonTerm)
+			b.finalEle = e
 		}
+	} else if *result {
+		parent := b.stack.peek()
+		parent.nonTerm.Add(e.nonTerm)
+	} else if b.stack.isEmpty() {
+		// TODO: improve error message
+		b.finalErr = newParsingError("parsing error")
+		b.current = e.index
 	} else {
 		b.current = e.index
 	}
+
 	dt := b.debugStack.pop()
 	dt.data += fmt.Sprintf("(%t)", *result)
 	if b.debugStack.isEmpty() {
 		b.finalDebugTree = dt
-		if !*result {
-			b.finalErr = &ParsingError{}
-		}
 	} else {
 		parent := b.debugStack.peek()
 		parent.add(dt)
 	}
 }
 
-// Tree returns the parse tree. It's set after the root non-terminal exits with
+// ParseTree returns the parse tree. It's set after the root non-terminal exits with
 // true result.
-func (b *Builder) Tree() *Tree {
+func (b *Builder) ParseTree() *Tree {
 	return b.finalEle.nonTerm
 }
 
 // DebugTree returns a tree that includes all matches and non-matches, and
 // non-terminal results (displayed in parentheses) captured throughout parsing.
 // Helps in understanding the parsing flow. It's set after the root non-terminal
 // exits. It has methods Print and Sprint.
-func (b *Builder) DebugTree() *debugTree {
+func (b *Builder) DebugTree() *DebugTree {
 	return b.finalDebugTree
 }
 

examples/arithmetic/backtrackingparser/parser.go

@@ -14,56 +14,54 @@ const Grammar = `
 	Factor = "(" Expr ")" | "-" Factor | Number
 `
 
-var (
-	numberRegex = regexp.MustCompile(`^(\d*\.\d+|\d+)$`)
-	b           *rd.Builder
-)
+var numberRegex = regexp.MustCompile(`^(\d*\.\d+|\d+)$`)
 
-func Expr() (ok bool) {
+func Expr(b *rd.Builder) (ok bool) {
 	b.Enter("Expr")
 	defer b.Exit(&ok)
 
-	if Term() && b.Match(Plus) && Expr() {
+	if Term(b) && b.Match(Plus) && Expr(b) {
 		return true
 	}
-	b.Reset()
-	if Term() && b.Match(Minus) && Expr() {
+	b.Backtrack()
+	if Term(b) && b.Match(Minus) && Expr(b) {
 		return true
 	}
-	b.Reset()
-	return Term()
+	b.Backtrack()
+	return Term(b)
 }
 
-func Term() (ok bool) {
+func Term(b *rd.Builder) (ok bool) {
 	b.Enter("Term")
 	defer b.Exit(&ok)
 
-	if Factor() && b.Match(Star) && Term() {
+	if Factor(b) && b.Match(Star) && Term(b) {
 		return true
 	}
-	b.Reset()
-	if Factor() && b.Match(Slash) && Term() {
+	b.Backtrack()
+	if Factor(b) && b.Match(Slash) && Term(b) {
 		return true
 	}
-	b.Reset()
-	return Factor()
+	b.Backtrack()
+	return Factor(b)
 }
 
-func Factor() (ok bool) {
+func Factor(b *rd.Builder) (ok bool) {
 	b.Enter("Factor")
 	defer b.Exit(&ok)
 
-	if b.Match(OpenParen) && Expr() && b.Match(CloseParen) {
+	if b.Match(OpenParen) && Expr(b) && b.Match(CloseParen) {
 		return true
 	}
-	b.Reset()
-	if b.Match(Minus) && Factor() {
+	b.Backtrack()
+	if b.Match(Minus) && Factor(b) {
 		return true
 	}
-	return Number()
+	b.Backtrack()
+	return Number(b)
 }
 
-func Number() (ok bool) {
+func Number(b *rd.Builder) (ok bool) {
 	b.Enter("Number")
 	defer b.Exit(&ok)
 
@@ -75,14 +73,15 @@ func Number() (ok bool) {
 		b.Add(token)
 		return true
 	}
-	b.Reset()
+	b.Backtrack()
 	return false
 }
 
-func Parse(tokens []rd.Token) (parseTree *rd.Tree, debugTree string, err error) {
-	b = rd.NewBuilder(tokens)
-	if ok := Expr(); !ok {
-		return nil, b.DebugTree().Sprint(), b.Err()
+func Parse(tokens []rd.Token) (parseTree *rd.Tree, debugTree *rd.DebugTree, err error) {
+	b := rd.NewBuilder(tokens)
+	ok := Expr(b)
+	if ok && b.Err() == nil {
+		return b.ParseTree(), b.DebugTree(), nil
 	}
-	return b.Tree(), b.DebugTree().Sprint(), nil
+	return nil, b.DebugTree(), b.Err()
 }

examples/arithmetic/lexer.go

@@ -28,12 +28,10 @@ func Lex(expr string) (tokens []rd.Token, err error) {
 		return nil, err
 	}
 	for _, token := range iter.Tokens() {
-		switch token.Type {
-		case chroma.Operator, chroma.Punctuation, chroma.NumberInteger, chroma.NumberFloat:
-			tokens = append(tokens, token.Value)
-		case chroma.Error:
+		if token.Type == chroma.Error {
 			return nil, fmt.Errorf("invalid token: %v", token)
 		}
+		tokens = append(tokens, token.Value)
 	}
 	return tokens, nil
 }

examples/arithmetic/main.go

@@ -9,6 +9,7 @@ import (
 	"github.com/shivamMg/rd/examples/arithmetic/parser"
 )
 
+
 func main() {
 	if len(os.Args) < 2 {
 		fmt.Println("invalid arguments. pass arithmetic expression as argument")
@@ -18,22 +19,19 @@ func main() {
 	expr := strings.Join(os.Args[1:], " ")
 	tokens, err := Lex(expr)
 	if err != nil {
-		fmt.Println("Lexing failed.", err)
-		os.Exit(1)
+		printExit("Lexing failed.", err)
 	}
 	printTokens(tokens)
 
-	fmt.Println("Grammar in EBNF:")
-	fmt.Println(parser.Grammar)
+	fmt.Print("Grammar:")
+	fmt.Print(parser.Grammar)
 
 	parseTree, debugTree, err := parser.Parse(tokens)
 	if err != nil {
-		fmt.Println("Parsing failed.", err)
-		fmt.Print("Debug Tree:\n", debugTree, "\n")
-		os.Exit(1)
+		fmt.Print("Debug Tree:\n\n", debugTree.Sprint())
+		printExit("Parsing failed.", err)
 	}
-	fmt.Print("Parse Tree:\n\n")
-	parseTree.Print()
+	fmt.Print("Parse Tree:\n\n", parseTree.Sprint())
 }
 
 func printTokens(tokens []rd.Token) {
@@ -44,3 +42,8 @@ func printTokens(tokens []rd.Token) {
 	}
 	fmt.Println(b.String())
 }
+
+func printExit(a ...interface{}) {
+	fmt.Fprintln(os.Stderr, a...)
+	os.Exit(1)
+}

examples/arithmetic/main_test.go

@@ -69,8 +69,9 @@ func TestArithmeticExpressionsGrammar(t *testing.T) {
          ├─ <no tokens left> ≠ +
          └─ <no tokens left> ≠ -
 `
-	if debugTree != expectedDebugTree {
-		t.Errorf("invalid debug tree. expected: %s\ngot: %s\n", expectedDebugTree, debugTree)
+	got := debugTree.Sprint()
+	if got != expectedDebugTree {
+		t.Errorf("invalid debug tree. expected: %s\ngot: %s\n", expectedDebugTree, got)
 	}
 
 	expectedParseTree := `Expr
@@ -116,7 +117,7 @@ func TestArithmeticExpressionsGrammar(t *testing.T) {
       └─ Expr'
          └─ ε
 `
-	got := parseTree.Sprint()
+	got = parseTree.Sprint()
 	if got != expectedParseTree {
 		t.Errorf("invalid parse tree. want: %s\ngot: %s\n", expectedParseTree, got)
 	}