lex.go

// Copyright 2017 Baliance. All rights reserved.
//
// Use of this source code is governed by the terms of the Affero GNU General
// Public License version 3.0 as published by the Free Software Foundation and
// appearing in the file LICENSE included in the packaging of this file. A
// commercial license can be purchased by contacting sales@baliance.com.

package formula

// Based off of http://ieeexplore.ieee.org/document/7335408/

import (
	"fmt"
	"io"
	"strings"
	"sync"
)

var debugLex = false

type tokenType int

func (t tokenType) String() string {
	return yyTokname(int(t))
}

type node struct {
	token tokenType
	val   string
}

func printable(s string) string {
	s = strings.Replace(s, "\n", "\\n", -1)
	s = strings.Replace(s, "\r", "\\r", -1)
	s = strings.Replace(s, "\t", "\\t", -1)
	return s
}
func (n node) String() string {
	return fmt.Sprintf("{%s %s}", n.token, printable(string(n.val)))
}

//go:generate ragel -G2 -Z lexer.rl
//go:generate goimports -w lexer.go
type Lexer struct {
	nodes    chan *node
	lock     sync.Mutex
	injected []chan *node
	peeked   []*node
}

func NewLexer() *Lexer {
	return &Lexer{nodes: make(chan *node)}
}

func LexReader(r io.Reader) chan *node {
	l := NewLexer()
	go l.lex(r)
	return l.nodes
}

func (l *Lexer) emit(typ tokenType, val []byte) {
	if debugLex {
		fmt.Println("emit", typ, printable(string(val)))
	}
	l.nodes <- &node{typ, string(val)}
}

func (l *Lexer) nextRaw() *node {
	for len(l.injected) != 0 {
		n := <-l.injected[len(l.injected)-1]
		if n != nil {
			return n
		}
		l.injected = l.injected[0 : len(l.injected)-1]
	}
	return <-l.nodes
}

func (l *Lexer) Next() *node {
	l.lock.Lock()
	defer l.lock.Unlock()
	if len(l.peeked) > 0 {
		n := l.peeked[0]
		l.peeked = l.peeked[1:]
		return n
	}
	return l.nextRaw()
}