Permalink
Browse files

internal/go/token: Make Token a type alias of stdlib token.Token.

The signature of highlight_go.TokenKind is:

	func TokenKind(tok token.Token, lit string) syntaxhighlight.Kind

We need to make the token.Token it uses compatible with stdlib
token.Token. Otherwise, it's not possible for users of highlight_go
to call highlight_go.TokenKind, because of the following error:

	cannot use tok (type "go/token".Token) as type "github.com/shurcooL/highlight_go/internal/go/token".Token in argument to highlight_go.TokenKind
  • Loading branch information...
dmitshur committed Dec 15, 2018
1 parent 98c3abb commit 9d8641ddf2e1517d286cedf0f4f4a6fb41ef5043
Showing with 3 additions and 56 deletions.
  1. +3 −56 internal/go/token/token.go
@@ -3,12 +3,13 @@
// license that can be found in the LICENSE file.

// Package token is a copy of go/token from Go 1.11.1.
// It makes its Token a type alias of "go/token".Token.
package token

import "strconv"
import "go/token"

// Token is the set of lexical tokens of the Go programming language.
type Token int
type Token = token.Token

// The list of tokens.
const (
@@ -221,23 +222,6 @@ var tokens = [...]string{
VAR: "var",
}

// String returns the string corresponding to the token tok.
// For operators, delimiters, and keywords the string is the actual
// token character sequence (e.g., for the token ADD, the string is
// "+"). For all other tokens the string corresponds to the token
// constant name (e.g. for the token IDENT, the string is "IDENT").
//
func (tok Token) String() string {
s := ""
if 0 <= tok && tok < Token(len(tokens)) {
s = tokens[tok]
}
if s == "" {
s = "token(" + strconv.Itoa(int(tok)) + ")"
}
return s
}

// A set of constants for precedence-based expression parsing.
// Non-operators have lowest precedence, followed by operators
// starting with precedence 1 up to unary operators. The highest
@@ -250,26 +234,6 @@ const (
HighestPrec = 7
)

// Precedence returns the operator precedence of the binary
// operator op. If op is not a binary operator, the result
// is LowestPrecedence.
//
func (op Token) Precedence() int {
switch op {
case LOR:
return 1
case LAND:
return 2
case EQL, NEQ, LSS, LEQ, GTR, GEQ:
return 3
case ADD, SUB, OR, XOR:
return 4
case MUL, QUO, REM, SHL, SHR, AND, AND_NOT:
return 5
}
return LowestPrec
}

var keywords map[string]Token

func init() {
@@ -287,20 +251,3 @@ func Lookup(ident string) Token {
}
return IDENT
}

// Predicates

// IsLiteral returns true for tokens corresponding to identifiers
// and basic type literals; it returns false otherwise.
//
func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_end }

// IsOperator returns true for tokens corresponding to operators and
// delimiters; it returns false otherwise.
//
func (tok Token) IsOperator() bool { return operator_beg < tok && tok < operator_end }

// IsKeyword returns true for tokens corresponding to keywords;
// it returns false otherwise.
//
func (tok Token) IsKeyword() bool { return keyword_beg < tok && tok < keyword_end }

0 comments on commit 9d8641d

Please sign in to comment.