initial commit for lexer

anaray · Jun 13, 2016 · 025acad · 025acad
1 parent 7a1d18d
commit 025acad
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 98 deletions.
diff --git a/lexer.go b/lexer.go
diff --git a/lexer_test.go b/lexer_test.go
diff --git a/state_functions.go b/state_functions.go
diff --git a/token.go b/token.go
@@ -1,16 +1,30 @@
 package regnet_lex_parse
 
-type tokenType int
+type TokenType int
 
 const (
-	IILEGAL tokenType = iota
+	IILEGAL TokenType = iota
 	KEY
 	EQUALS
 	PERCENT
 	START_BRACE
 	END_BRACE
 	REG_TEXT
 	WHITE_SPACE
+	EOF
+	SOME_TEXT_UNMARKED
 )
 
 const eof = rune(0)
+
+type Position int
+
+type Token struct {
+  Type TokenType
+	Pos Position
+  Text string
+}
+
+func (t Token) String() string {
+		return t.Text
+}