query_scan.go

package main

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"strings"
	"unicode"
)

type tokenTypeEvaluator func(char rune) (isTokenType bool)

// QueryTokenType represents a type of query token
type QueryTokenType int

// The set of token types that can be generated by the scanner
const (
	QtkInvalid QueryTokenType = iota

	QtkWhiteSpace
	QtkEOF

	QtkIdentifier
	QtkNumber
	QtkString

	QtkAnd
	QtkOr

	QtkNot

	QtkCmpEq
	QtkCmpNe
	QtkCmpGt
	QtkCmpGe
	QtkCmpLt
	QtkCmpLe

	QtkCmpGlob
	QtkCmpRegexp

	QtkLparen
	QtkRparen
)

// QueryScannerPos is the position in the query input stream
type QueryScannerPos struct {
	line uint
	col  uint
}

// QueryToken is a token and metadata parsed from the input stream
type QueryToken struct {
	tokenType QueryTokenType
	value     string
	startPos  QueryScannerPos
	endPos    QueryScannerPos
	err       error
}

// QueryScanner tokenises an input query into a sequence of tokens
type QueryScanner struct {
	reader          *bufio.Reader
	pos             QueryScannerPos
	lastCharLineEnd bool
	lastLineEndCol  uint
}

// Equal returns true if this token is equal to the provided token
func (token *QueryToken) Equal(other *QueryToken) bool {
	if other == nil {
		return false
	}

	return token.tokenType == other.tokenType &&
		token.value == other.value &&
		token.startPos == other.startPos &&
		token.endPos == other.endPos &&
		((token.err == nil && other.err == nil) ||
			(token.err != nil && other.err != nil &&
				token.err.Error() == other.err.Error()))
}

// Value returns the string value of this token
func (token *QueryToken) Value() string {
	if token.tokenType == QtkEOF {
		return "EOF"
	}

	return token.value
}

// NewQueryScanner creates a new instance
func NewQueryScanner(reader io.Reader) *QueryScanner {
	return &QueryScanner{
		reader: bufio.NewReader(reader),
		pos: QueryScannerPos{
			line: 1,
			col:  0,
		},
	}
}

func (scanner *QueryScanner) read() (char rune, eof bool, err error) {
	char, _, err = scanner.reader.ReadRune()

	if err == io.EOF {
		eof = true
		err = nil

		if scanner.pos.col == 0 {
			scanner.pos.col = 1
		}
	} else if err == nil {
		if scanner.lastCharLineEnd {
			scanner.lastLineEndCol = scanner.pos.col
			scanner.pos.line++
			scanner.pos.col = 1
		} else {
			scanner.pos.col++
		}

		scanner.lastCharLineEnd = (char == '\n')
	}

	return
}

func (scanner *QueryScanner) unread() (err error) {
	if err = scanner.reader.UnreadRune(); err != nil {
		return
	}

	if scanner.pos.line > 1 && scanner.pos.col == 1 {
		scanner.pos.line--
		scanner.pos.col = scanner.lastLineEndCol
		scanner.lastCharLineEnd = true
	} else {
		scanner.pos.col--
		scanner.lastCharLineEnd = false
	}

	return
}

// Scan returns the next token from the input stream
func (scanner *QueryScanner) Scan() (token *QueryToken, err error) {
	char, eof, err := scanner.read()
	startPos := scanner.pos

	switch {
	case err != nil:
	case eof:
		token = &QueryToken{
			tokenType: QtkEOF,
			endPos:    scanner.pos,
		}
	case unicode.IsSpace(char):
		if err = scanner.unread(); err != nil {
			break
		}

		token, err = scanner.scanWhiteSpace()
	case unicode.IsLetter(char):
		if err = scanner.unread(); err != nil {
			break
		}

		token, err = scanner.scanIdentifier()

		if err != nil || token.err != nil {
			break
		}

		switch strings.ToUpper(token.value) {
		case "AND":
			token.tokenType = QtkAnd
		case "OR":
			token.tokenType = QtkOr
		case "NOT":
			token.tokenType = QtkNot
		case "GLOB":
			token.tokenType = QtkCmpGlob
		case "REGEXP":
			token.tokenType = QtkCmpRegexp
		}
	case char == '"':
		if err = scanner.unread(); err != nil {
			break
		}

		token, err = scanner.scanString()
	case char == '-' || char == '.' || unicode.IsNumber(char):
		if err = scanner.unread(); err != nil {
			break
		}

		token, err = scanner.scanNumber()
	case char == '=':
		token = &QueryToken{
			tokenType: QtkCmpEq,
			value:     "=",
			endPos:    scanner.pos,
		}
	case char == '!':
		char, _, err = scanner.read()

		if err != nil {
			break
		}

		if char == '=' {
			token = &QueryToken{
				tokenType: QtkCmpNe,
				value:     "!=",
				endPos:    scanner.pos,
			}
		} else {
			token = &QueryToken{
				tokenType: QtkInvalid,
				value:     fmt.Sprintf("!%c", char),
				endPos:    scanner.pos,
				err:       errors.New("Expected '=' character after '!'"),
			}
		}
	case char == '>' || char == '<':
		nextChar, eof, err := scanner.read()

		if err != nil {
			break
		}

		var tokenType QueryTokenType
		var tokenValue string

		if nextChar == '=' {
			if char == '>' {
				tokenType = QtkCmpGe
			} else {
				tokenType = QtkCmpLe
			}

			tokenValue = fmt.Sprintf("%c=", char)
		} else {
			if char == '>' {
				tokenType = QtkCmpGt
			} else {
				tokenType = QtkCmpLt
			}

			tokenValue = fmt.Sprintf("%c", char)

			if !eof {
				if err = scanner.unread(); err != nil {
					break
				}
			}
		}

		token = &QueryToken{
			tokenType: tokenType,
			value:     tokenValue,
			endPos:    scanner.pos,
		}
	case char == '(':
		token = &QueryToken{
			tokenType: QtkLparen,
			value:     "(",
			endPos:    scanner.pos,
		}
	case char == ')':
		token = &QueryToken{
			tokenType: QtkRparen,
			value:     ")",
			endPos:    scanner.pos,
		}
	default:
		token = &QueryToken{
			tokenType: QtkInvalid,
			value:     fmt.Sprintf("%c", char),
			endPos:    scanner.pos,
			err:       fmt.Errorf("Unexpected character %c", char),
		}
	}

	if token != nil {
		token.startPos = startPos
	}

	return
}

func (scanner *QueryScanner) scanToken(tokenType QueryTokenType, evaluator tokenTypeEvaluator) (token *QueryToken, err error) {
	var buffer bytes.Buffer
	var char rune
	var eof bool

OuterLoop:
	for {
		char, eof, err = scanner.read()

		switch {
		case err != nil:
			return
		case eof:
			break OuterLoop
		case !evaluator(char):
			if err = scanner.unread(); err != nil {
				return
			}

			break OuterLoop
		default:
			if _, err = buffer.WriteRune(char); err != nil {
				return
			}
		}
	}

	token = &QueryToken{
		tokenType: tokenType,
		value:     buffer.String(),
		endPos:    scanner.pos,
	}

	return
}

func isLetterOrNumber(char rune) bool {
	return unicode.IsLetter(char) || unicode.IsNumber(char)
}

func (scanner *QueryScanner) scanWhiteSpace() (token *QueryToken, err error) {
	return scanner.scanToken(QtkWhiteSpace, unicode.IsSpace)
}

func (scanner *QueryScanner) scanIdentifier() (token *QueryToken, err error) {
	return scanner.scanToken(QtkIdentifier, isLetterOrNumber)
}

func (scanner *QueryScanner) scanNumber() (token *QueryToken, err error) {
	var buffer bytes.Buffer
	var char rune
	var eof bool
	dotSeen := false

OuterLoop:
	for {
		char, eof, err = scanner.read()

		switch {
		case err != nil:
			return
		case eof:
			break OuterLoop
		case char == '-':
			offset := buffer.Len()

			if _, err = buffer.WriteRune(char); err != nil {
				return
			}

			if offset != 0 {
				token = &QueryToken{
					tokenType: QtkInvalid,
					value:     buffer.String(),
					endPos:    scanner.pos,
					err:       errors.New("Unexpected '-' character in number"),
				}

				return
			}
		case char == '.':
			if _, err = buffer.WriteRune(char); err != nil {
				return
			}

			if dotSeen {
				token = &QueryToken{
					tokenType: QtkInvalid,
					value:     buffer.String(),
					endPos:    scanner.pos,
					err:       errors.New("Unexpected '.' character in number"),
				}

				return
			}

			dotSeen = true
		case !unicode.IsNumber(char):
			if err = scanner.unread(); err != nil {
				return
			}

			break OuterLoop
		default:
			if _, err = buffer.WriteRune(char); err != nil {
				return
			}
		}
	}

	token = &QueryToken{
		tokenType: QtkNumber,
		value:     buffer.String(),
		endPos:    scanner.pos,
	}

	return
}

func (scanner *QueryScanner) scanString() (token *QueryToken, err error) {
	var buffer bytes.Buffer
	var char rune
	var eof bool

	char, eof, err = scanner.read()
	if err != nil || eof {
		return
	}

	if _, err = buffer.WriteRune(char); err != nil {
		return
	}

	closingQuoteFound := false
	escape := false

OuterLoop:
	for {
		char, eof, err = scanner.read()

		switch {
		case err != nil:
			return
		case eof:
			break OuterLoop
		case char == '\\':
			if _, err = buffer.WriteRune(char); err != nil {
				return
			}

			if !escape {
				escape = true
				continue
			}
		case char == '"':
			if _, err = buffer.WriteRune(char); err != nil {
				return
			}

			if !escape {
				closingQuoteFound = true
				break OuterLoop
			}
		default:
			if _, err = buffer.WriteRune(char); err != nil {
				return
			}
		}

		escape = false
	}

	if closingQuoteFound {
		var word string
		word, err = scanner.processString(buffer.String())
		if err != nil {
			return
		}

		token = &QueryToken{
			tokenType: QtkString,
			value:     word,
			endPos:    scanner.pos,
		}
	} else {
		token = &QueryToken{
			tokenType: QtkInvalid,
			value:     buffer.String(),
			endPos:    scanner.pos,
			err:       errors.New("Unterminated string"),
		}
	}

	return
}

func (scanner *QueryScanner) processString(str string) (string, error) {
	var buffer bytes.Buffer
	chars := []rune(str)

	if len(chars) < 2 || chars[0] != '"' || chars[len(chars)-1] != '"' {
		return str, fmt.Errorf("Invalid string: %v", str)
	}

	chars = chars[1 : len(chars)-1]
	escape := false

	for _, char := range chars {
		switch {
		case escape:
			switch char {
			case 'n':
				buffer.WriteRune('\n')
			case 't':
				buffer.WriteRune('\t')
			default:
				buffer.WriteRune(char)
			}

			escape = false
		case char == '\\':
			escape = true
		default:
			buffer.WriteRune(char)
		}
	}

	return buffer.String(), nil
}