add parse_test, add skipComment and basic parser structure to the env…

…ironment

neo/ast.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 )
 
+var textFormat = "%s" // Changed to "%q" in tests for better error messages.
+
 type NodeType int
 
 func (t NodeType) Type() NodeType {
@@ -61,3 +63,22 @@ func (l *ListNode) CopyList() *ListNode {
 func (l *ListNode) Copy() Node {
 	return l.CopyList()
 }
+
+// TextNode holds plain text.
+type TextNode struct {
+	NodeType
+	Pos
+	Text []byte // The text; may span newlines.
+}
+
+func newText(pos Pos, text string) *TextNode {
+	return &TextNode{NodeType: NodeText, Pos: pos, Text: []byte(text)}
+}
+
+func (t *TextNode) String() string {
+	return fmt.Sprintf(textFormat, t.Text)
+}
+
+func (t *TextNode) Copy() Node {
+	return &TextNode{NodeType: NodeText, Text: append([]byte{}, t.Text...)}
+}

neo/environment.go

@@ -106,3 +106,10 @@ func (e *Environment) lex(source, name, filename string) *lexer {
 	go l.run()
 	return l
 }
+
+// parse completely parses template source, returning the Node errors.
+func (e *Environment) parse(source, name, filename string) (*Tree, error) {
+	lex := e.lex(source, name, filename)
+	t := newTree(name)
+	return t.Parse(lex)
+}

neo/parse.go

@@ -102,7 +102,7 @@ func (t *Tree) peekNonSpace() (token item) {
 // Parsing.
 
 // New allocates a new parse tree with the given name.
-func newParser(name string) *Tree {
+func newTree(name string) *Tree {
 	return &Tree{
 		Name: name,
 	}
@@ -149,6 +149,11 @@ func (t *Tree) recover(errp *error) {
 	return
 }
 
+// unexpected complains about the token and terminates processing.
+func (t *Tree) unexpected(token item, context string) {
+	t.errorf("unexpected %s in %s", token, context)
+}
+
 // startParse initializes the parser, using the lexer.
 func (t *Tree) startParse(lex *lexer) {
 	t.Root = nil
@@ -173,28 +178,55 @@ func (t *Tree) Parse(lex *lexer) (tree *Tree, err error) {
 
 // -- parsing --
 
+// Here is where the code will depart quite a bit from text/template.
+// Starting at parse(), we must take care of all sorts of different
+// block types and tags, and we also have no concept of embedding
+// multiple named templates within one file/[]byte/string.
+
 // parse is the top-level parser for a template, essentially the same
 // as itemList except it also parses {{define}} actions.
 // It runs to EOF.
 func (t *Tree) parse() (next Node) {
 	t.Root = newList(t.peek().pos)
 	for t.peek().typ != tokenEOF {
-		if t.peek().typ == tokenBlockBegin {
-			/*
-				delim := t.next()
-				if t.nextNonSpace().typ == itemDefine {
-					newT := New("definition") // name will be updated once we know it.
-					newT.text = t.text
-					newT.ParseName = t.ParseName
-					newT.startParse(t.funcs, t.lex)
-					newT.parseDefinition(treeSet)
-					continue
-
-				}
-				t.backup2(delim)
-			*/
+		var n Node
+		switch t.peek().typ {
+		case tokenBlockBegin:
+			// the start of a {% .. %} tag block.
+			fmt.Println("Got BlockBegin")
+			continue
+		//	delim := t.next()
+
+		case tokenVariableBegin:
+			// the start of a {{ .. }} variable print block.
+			fmt.Println("Got VariableBegin")
+			continue
+		//	delim := t.next()
+
+		case tokenCommentBegin:
+			t.skipComment()
+			continue
+
+		case tokenText:
+			// this token is text, lets save it in a text node and continue
+			n = t.parseText()
 		}
+		t.Root.append(n)
+
 		/*
+			delim := t.next()
+			if t.nextNonSpace().typ == itemDefine {
+				newT := New("definition") // name will be updated once we know it.
+				newT.text = t.text
+				newT.ParseName = t.ParseName
+				newT.startParse(t.funcs, t.lex)
+				newT.parseDefinition(treeSet)
+				continue
+
+			}
+			t.backup2(delim)
+
+
 			n := t.textOrAction()
 			if n.Type() == nodeEnd {
 				t.errorf("unexpected %s", n)
@@ -204,3 +236,32 @@ func (t *Tree) parse() (next Node) {
 	}
 	return nil
 }
+
+func (t *Tree) parseText() Node {
+	switch token := t.next(); token.typ {
+	case tokenText:
+		return newText(token.pos, token.val)
+	default:
+		t.unexpected(token, "input")
+	}
+	return nil
+}
+
+// Skips over a comment;  commends are not represented in the final AST.
+func (t *Tree) skipComment() {
+	token := t.next()
+	if token.typ != tokenCommentBegin {
+		t.unexpected(token, "begin comment")
+	}
+	for {
+		token = t.nextNonSpace()
+		switch token.typ {
+		case tokenText:
+			continue
+		case tokenCommentEnd:
+		default:
+			t.unexpected(token, "end commend")
+		}
+		break
+	}
+}

neo/parse_test.go

@@ -0,0 +1,57 @@
+package jigo
+
+import (
+	"fmt"
+	"testing"
+)
+
+type parseTest struct {
+	isError bool
+	output  string
+}
+
+type parsetest struct {
+	*testing.T
+}
+
+func (p *parsetest) Test(input string, test parseTest) {
+	t := p.T
+	e := NewEnvironment()
+	tree, err := e.parse(input, "test", "test.jigo")
+
+	/*
+		if len(nodes) != len(tests) {
+			t.Errorf("Expected %d nodes, got %d\n", len(tests), len(nodes))
+		}
+		for i, tok := range nodes {
+			if i >= len(tests) {
+				return
+			}
+			test := tests[i]
+			if test.typ != tok.typ {
+				fmt.Printf("nodes: %v\ntests:  %v\n", nodes, tests)
+				t.Errorf("Expecting %d token type to be %d, got %d\n", i, test.typ, tok.typ)
+			}
+			if tok.typ != tokenWhitespace && test.val != tok.val {
+				fmt.Printf("nodes: %v\ntests:  %v\n", nodes, tests)
+				t.Errorf("Expecting %d token val to be `%s`, got `%s`\n", i, test.val, tok.val)
+			}
+		}
+	*/
+	if err != nil && !test.isError {
+		t.Errorf("Unexpected error: %s\n", err)
+	}
+
+	fmt.Printf("Tree: %#v\n", tree.Root)
+	fmt.Printf("Tree: %s\n", tree.Root)
+}
+
+func TestParser(t *testing.T) {
+	tester := parsetest{t}
+	fmt.Println(tester)
+
+	tester.Test(
+		`Hello, {# comment #}World`,
+		parseTest{isError: false},
+	)
+}