Permalink
Browse files

Lex keywords.

  • Loading branch information...
umitanuki committed Jan 18, 2013
1 parent 1e1a15b commit 200d5b84a456e9f7e959a676c92b97fdbf9203b6
Showing with 100 additions and 21 deletions.
  1. +4 −1 src/bigpot/parser/Makefile
  2. +3 −0 src/bigpot/parser/gram.y
  3. +38 −0 src/bigpot/parser/kwlist.go
  4. +18 −6 src/bigpot/parser/scan.l
  5. +37 −14 src/bigpot/parser/scan_test.go
@@ -11,5 +11,8 @@ gram.go: gram.y
test: scan.go gram.go
go test bigpot/parser
+coverage: scan.go gram.go
+ gocov test bigpot/parser | gocov report
+
clean:
- rm -f scan.go gram.go
+ rm -f scan.go gram.go y.output
View
@@ -16,6 +16,7 @@ type Node struct {
node *Node
str string
ival int
+ keyword string
}
%token
@@ -39,6 +40,8 @@ type Node struct {
%token <ival> ICONST PARAM
%token TYPECAST DOT_DOT COLON_EQUALS
+%token <keyword> SELECT
+
%%
statements: /* empty */
| statements ';' statement
@@ -0,0 +1,38 @@
+package parser
+
+import (
+ "errors"
+ "strings"
+)
+
+const (
+ ReservedKeyword = iota
+ TypeFuncNameKeyword
+ ColNameKeyword
+ UnreservedKeyword
+)
+
+type keyword struct {
+ name string
+ token int
+ kwtype int
+}
+
+/*
+ * We'd like to avoid dupliate definitions between gram.y and here,
+ * but because we need our keyword type definition while yacc needs to know
+ * the set of keywords at compile time.
+ */
+var keywordList = []keyword{
+ {"select", SELECT, ReservedKeyword},
+}
+
+func findKeyword(name string) (*keyword, error) {
+ name = strings.ToLower(name)
+ for _, kw := range keywordList {
+ if kw.name == name {
+ return &kw, nil
+ }
+ }
+ return nil, errors.New("keyword not found")
+}
View
@@ -562,11 +562,11 @@ other .
{param}
l.SET_YYLLOC()
- if val, err := strconv.Atoi(l.getBuf()); err == nil {
+ if val, err := strconv.Atoi(l.getBuf()[1:]); err == nil {
lval.ival = val
return PARAM
}
- l.Error("value ouf of range for param")
+ l.Error("value out of range for param")
{integer}
l.SET_YYLLOC()
@@ -605,13 +605,25 @@ other .
return FCONST
{identifier}
- /* TODO: keyword, to_lower() */
- ident := l.getBuf()
- lval.str = ident
+ yystr := l.getBuf()
+ /*
+ * If this is a keyword, return the token number instead of IDENT
+ */
+ if keyword, err := findKeyword(yystr); err == nil {
+ lval.keyword = keyword.name
+ return keyword.token
+ }
+ lval.str = strings.ToLower(yystr)
+ lval.str = yystr
return IDENT
{other}
- return 1
+ l.SET_YYLLOC()
+ return int(l.buf[0])
+
+\0
+ l.SET_YYLLOC()
+ return yyEofCode
%%
@@ -3,7 +3,6 @@ package parser
import (
"fmt"
"strings"
-// "testing"
)
func strLexer(input string) *lexer {
@@ -14,62 +13,86 @@ func strLexer(input string) *lexer {
func (l *lexer) lexPrintExpect(expected int) {
lval := &yySymType{}
if token := l.Lex(lval); token != expected {
-// panic(fmt.Sprintf("not expected token: %d != %d", token, expected))
+ fmt.Printf("not expected token: %d != %d", token, expected)
+ return
}
switch expected {
default:
if expected > 0 && expected < 127 {
fmt.Printf(" %c", expected)
+ } else if expected > COLON_EQUALS { /* too hacky... */
+ fmt.Printf(" %s", lval.keyword)
} else {
fmt.Printf(" %s", lval.str)
}
- case ICONST:
+ case ICONST, PARAM:
fmt.Printf(" %d", lval.ival)
}
}
+func recoverPanic() {
+ if err := recover(); err != nil {
+ fmt.Println("failed:", err)
+ }
+}
+
func ExampleLex_1() {
lexer := strLexer("select 1")
- lexer.lexPrintExpect(IDENT)
+ lexer.lexPrintExpect(SELECT)
// Output: select
}
func ExampleLex_2() {
lexer := strLexer("select 'foo' /* comment */ bar")
- lexer.lexPrintExpect(IDENT)
+ lexer.lexPrintExpect(SELECT)
lexer.lexPrintExpect(SCONST)
lexer.lexPrintExpect(IDENT)
// Output: select foo bar
}
func ExampleLex_numbers() {
- lexer := strLexer("10 0.1e 1.53e-1")
+ lexer := strLexer("10 0.1e 1.53e-1 1. 0001.999 9999999999999999999")
lexer.lexPrintExpect(ICONST)
lexer.lexPrintExpect(FCONST)
lexer.lexPrintExpect(IDENT)
lexer.lexPrintExpect(FCONST)
- // Output: 10 0.1 e 1.53e-1
+ lexer.lexPrintExpect(FCONST)
+ lexer.lexPrintExpect(FCONST)
+ lexer.lexPrintExpect(FCONST)
+ // Output: 10 0.1 e 1.53e-1 1. 0001.999 9999999999999999999
}
func ExampleLex_operators() {
- lexer := strLexer("1 % 2 -/* foo*/10")
+ lexer := strLexer("1 % 2 -/* foo*/10 <> !=")
lexer.lexPrintExpect(ICONST)
lexer.lexPrintExpect(int('%'))
lexer.lexPrintExpect(ICONST)
lexer.lexPrintExpect(int('-'))
lexer.lexPrintExpect(ICONST)
- // Output: 1 % 2 - 10
+ lexer.lexPrintExpect(Op)
+ lexer.lexPrintExpect(Op)
+ // Output: 1 % 2 - 10 <> <>
+}
+
+func ExampleLex_params() {
+ lexer := strLexer("$1 $0")
+ lexer.lexPrintExpect(PARAM)
+ lexer.lexPrintExpect(PARAM)
+ // Output: 1 0
}
func ExampleLex_negative1() {
- defer func() {
- if err := recover(); err != nil {
- fmt.Println("failed:", err)
- }
- } ()
+ defer recoverPanic()
lexer := strLexer("select /* comment")
lval := &yySymType{}
lexer.Lex(lval)
lexer.Lex(lval)
// Output: failed: unterminated /* comment
}
+
+func ExampleLex_params_negative() {
+ defer recoverPanic()
+ lexer := strLexer("$99999999999999999999999999999")
+ lexer.lexPrintExpect(PARAM)
+ // Output: failed: value out of range for param
+}

0 comments on commit 200d5b8

Please sign in to comment.