Skip to content
Permalink
Browse files

Get started on writing a lexer for GraphQL

  • Loading branch information...
manishrjain committed Oct 27, 2015
1 parent 88a3e1b commit 73b0e41e2254c6b8cb15e47f521066224de69243
Showing with 255 additions and 0 deletions.
  1. +127 −0 gql/lexer/lexer.go
  2. +23 −0 gql/lexer/lexer_test.go
  3. +105 −0 gql/lexer/state.go
@@ -0,0 +1,127 @@
package gqlex

import (
"fmt"
"strings"
"unicode/utf8"
)

type itemType int

const (
itemError itemType = iota
itemEOF
itemLeftCurl // left curly bracket
itemRightCurl // right curly bracket
itemString // quoted string
itemText // plain text
itemIdentifier // variables
)

const EOF = -1

type item struct {
typ itemType
val string
}

func (i item) String() string {
switch i.typ {
case itemEOF:
return "EOF"
case itemError:
return i.val
case itemIdentifier:
return fmt.Sprintf("var: [%v]", i.val)
}
/*
if len(i.val) > 10 {
return fmt.Sprintf("%.10q...", i.val)
}
*/
return fmt.Sprintf("%q", i.val)
}

type lexer struct {
// NOTE: Using a text scanner wouldn't work because it's designed for parsing
// Golang. It won't keep track of start position, or allow us to retrieve
// slice from [start:pos]. Better to just use normal string.
input string // string being scanned.
start int // start position of this item.
pos int // current position of this item.
width int // width of last rune read from input.
items chan item // channel of scanned items.
depth int // nesting of {}
}

func newLexer(input string) *lexer {
l := &lexer{
input: input,
items: make(chan item),
}
go l.run()
return l
}

func (l *lexer) errorf(format string,
args ...interface{}) stateFn {
l.items <- item{
typ: itemError,
val: fmt.Sprintf(format, args...),
}
return nil
}

func (l *lexer) emit(t itemType) {
l.items <- item{
typ: t,
val: l.input[l.start:l.pos],
}
l.start = l.pos
}

func (l *lexer) run() {
for state := lexText; state != nil; {
state = state(l)
}
close(l.items) // No more tokens.
}

func (l *lexer) next() (result rune) {
if l.pos >= len(l.input) {
l.width = 0
return EOF
}
r, w := utf8.DecodeRuneInString(l.input[l.pos:])
l.width = w
l.pos += l.width
return r
}

func (l *lexer) backup() {
l.pos -= l.width
}

func (l *lexer) peek() rune {
r := l.next()
l.backup()
return r
}

func (l *lexer) ignore() {
l.start = l.pos
}

func (l *lexer) accept(valid string) bool {
if strings.IndexRune(valid, l.next()) >= 0 {
return true
}
l.backup()
return false
}

func (l *lexer) acceptRun(valid string) {
for strings.IndexRune(valid, l.next()) >= 0 {
}
l.backup()
}
@@ -0,0 +1,23 @@
package gqlex

import (
"fmt"
"testing"
)

func TestNewLexer(t *testing.T) {
input := `
{
me {
name
city
friends {
name
}
}
}`
l := newLexer(input)
for item := range l.items {
fmt.Println(item.String())
}
}
@@ -0,0 +1,105 @@
package gqlex

import (
"strings"
"unicode"
)

const (
leftCurl = "{"
rightCurl = "}"
)

// stateFn represents the state of the scanner as a function that
// returns the next state.
type stateFn func(*lexer) stateFn

func lexText(l *lexer) stateFn {
for {
if strings.HasPrefix(l.input[l.pos:], leftCurl) {
if l.pos > l.start {
l.emit(itemText)
}
return lexLeftCurl
}
if strings.HasPrefix(l.input[l.pos:], rightCurl) {
return l.errorf("Too many right brackets")
}
if l.next() == EOF {
break
}
}
// Correctly reached EOF.
if l.pos > l.start {
l.emit(itemText)
}
l.emit(itemEOF)
return nil // Stop the run loop.
}

func lexLeftCurl(l *lexer) stateFn {
l.pos += len(leftCurl)
l.depth += 1
l.emit(itemLeftCurl)
return lexInside(l)
}

func lexRightCurl(l *lexer) stateFn {
l.pos += len(rightCurl)
l.depth -= 1
l.emit(itemRightCurl)

if l.depth == 0 {
return lexText
} else {
return lexInside
}
}

func lexInside(l *lexer) stateFn {
for {
if strings.HasPrefix(l.input[l.pos:], rightCurl) {
return lexRightCurl
}
if strings.HasPrefix(l.input[l.pos:], leftCurl) {
return lexLeftCurl
}

switch r := l.next(); {
case r == EOF:
return l.errorf("unclosed action")
case isSpace(r) || isEndOfLine(r):
l.ignore()
case isAlphaNumeric(r):
l.backup()
return lexIdentifier
}
}
}

func lexIdentifier(l *lexer) stateFn {
Loop:
for {
switch r := l.next(); {
case isAlphaNumeric(r):
// absorb.
default:
l.backup()
l.emit(itemIdentifier)
break Loop
}
}
return lexInside
}

func isSpace(r rune) bool {
return r == ' ' || r == '\t'
}

func isEndOfLine(r rune) bool {
return r == '\r' || r == '\n'
}

func isAlphaNumeric(r rune) bool {
return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
}

0 comments on commit 73b0e41

Please sign in to comment.
You can’t perform that action at this time.