token.go

package svn

import (
	"bufio"
	"fmt"
	"io"
)

// A TokenType is the type of a Token.
type TokenType int

const (
	// ErrorToken means that an error occurred during tokenization.
	ErrorToken TokenType = iota
	WordToken
	NumberToken
	StringToken
	LeftParenToken
	RightParenToken
)

// A Tokenizer returns a stream of SVN Tokens.
type Tokenizer struct {
	r     *bufio.Reader
	token Token
	err   error
	done  bool
}

// A Token describes a token in a SVN conversation.
// There are only 5 types of tokens: word, number, string, left and right parenthesis.
type Token struct {
	Type   TokenType
	Number uint
	Text   string
}

// NewTokenizer returns a new SVN Tokenizer for the given Reader.
func NewTokenizer(r io.Reader) *Tokenizer {
	z := &Tokenizer{
		r: bufio.NewReader(r),
	}
	return z
}

func (t *Tokenizer) readByte() byte {
	b, err := t.r.ReadByte()
	if err != nil {
		t.done = true
		t.err = err
	}
	return b
}

// Scan advances the Tokenizer to the next token, which will then be available through the Token method.
// It returns false when the scan stops, either by reaching the end of the input or an error.
// After Scan returns false, the Err method will return any error that occurred during scanning.
func (t *Tokenizer) Scan() bool {
	if t.done {
		return false
	}
	t.token.Type = ErrorToken

	var b byte
	for b = t.readByte(); isspace(b); b = t.readByte() {
	}
	if t.err != nil {
		return false
	}

	switch {
	case isnum(b):
		var number uint
		for isnum(b) {
			number *= 10
			number += uint(b - '0')
			b = t.readByte()
		}
		t.token.Type = NumberToken
		t.token.Number = number
		if b == ':' {
			t.token.Type = StringToken
			octets := make([]byte, number)
			_, err := io.ReadFull(t.r, octets)
			if err != nil {
				t.token.Type = ErrorToken
				t.done = true
				t.err = err
				return false
			}
			t.token.Text = string(octets)
			b = t.readByte()
		}
	case b == '(':
		t.token.Type = LeftParenToken
		b = t.readByte()
	case b == ')':
		t.token.Type = RightParenToken
		b = t.readByte()
	case isalpha(b):
		t.token.Type = WordToken
		t.token.Text = ""
		for isalnum(b) || b == '-' {
			t.token.Text += string(b)
			b = t.readByte()
		}
	default:
		t.token.Type = ErrorToken
		t.err = fmt.Errorf("syntax error: unexpected \"%c\"", b)
		t.done = true
		return false
	}

	if !isspace(b) {
		t.err = fmt.Errorf("syntax error: expected space after %q", t.token)
		t.token.Type = ErrorToken
		t.done = true
		return false
	}
	return true
}

// Token returns the most recent token generated by a call to Scan.
func (t *Tokenizer) Token() Token {
	return t.token
}

// Err returns the first error that was encountered by the Tokenizer.
func (t *Tokenizer) Err() error {
	return t.err
}

// String returns a string representation of the Token.
func (t Token) String() string {
	s := ""
	switch t.Type {
	case ErrorToken:
		s = "**ERROR**"
	case WordToken:
		s = t.Text
	case NumberToken:
		s = fmt.Sprint(t.Number)
	case StringToken:
		s = fmt.Sprintf("%d:%s", len(t.Text), t.Text)
	case LeftParenToken:
		s = "("
	case RightParenToken:
		s = ")"
	}
	return s
}