change Tokens to operate on a rune level

otherwise all utf8 characters have a "common prefix" which allowed one
route to return a different one somehow.

adds an explicit test for matching unicode, too. having this mixed in
with the other test made things pretty difficult to debug

package/router/lex/tokens.go

@@ -3,6 +3,7 @@ package lex
 import (
 	"fmt"
 	"strings"
+	"unicode/utf8"
 )
 
 // token type
@@ -41,9 +42,10 @@ func (tokens Tokens) At(i int) string {
 	for _, token := range tokens {
 		switch token.Type {
 		case PathToken, SlashToken:
-			for j := 0; j < len(token.Value); j++ {
+			chars := []rune(token.Value)
+			for j := 0; j < len(chars); j++ {
 				if i == 0 {
-					return string(token.Value[j])
+					return string(chars[j])
 				}
 				i--
 			}
@@ -63,9 +65,7 @@ func (tokens Tokens) Size() (n int) {
 	for _, token := range tokens {
 		switch token.Type {
 		case PathToken, SlashToken:
-			for j := 0; j < len(token.Value); j++ {
-				n++
-			}
+			n += utf8.RuneCountInString(token.Value)
 		case SlotToken, QuestionToken, StarToken:
 			n++
 		}
@@ -79,23 +79,24 @@ func (tokens Tokens) Split(at int) []Tokens {
 	for i, token := range tokens {
 		switch token.Type {
 		case PathToken, SlashToken:
-			for j := 0; j < len(token.Value); j++ {
+			runes := []rune(token.Value)
+			for j := 0; j < len(runes); j++ {
 				if at != 0 {
 					at--
 					continue
 				}
-				left, right := token.Value[:j], token.Value[j:]
+				left, right := runes[:j], runes[j:]
 				// At the edge
-				if left == "" || right == "" {
+				if len(left) == 0 || len(right) == 0 {
 					if i > 0 && i < len(tokens) {
 						return []Tokens{tokens[:i], tokens[i:]}
 					}
 					return []Tokens{tokens}
 				}
-				newToken := Token{token.Type, left}
+				newToken := Token{token.Type, string(left)}
 				leftTokens := append(append(Tokens{}, tokens[:i]...), newToken)
 				rightTokens := append(Tokens{}, tokens[i:]...)
-				rightTokens[0].Value = right
+				rightTokens[0].Value = string(right)
 				return []Tokens{leftTokens, rightTokens}
 			}
 		case SlotToken, QuestionToken, StarToken:

package/router/lex/tokens_test.go

@@ -32,6 +32,10 @@ func TestAt(t *testing.T) {
 	is.Equal(toks.At(1), "h")
 	is.Equal(toks.At(2), "i")
 	is.Equal(toks.At(3), "")
+	toks = tokens(t, "/α")
+	is.Equal(toks.At(0), "/")
+	is.Equal(toks.At(1), "α")
+	is.Equal(toks.At(2), "")
 	toks = tokens(t, "/:a")
 	is.Equal(toks.At(0), "/")
 	is.Equal(toks.At(1), ":a")
@@ -64,6 +68,8 @@ func TestSize(t *testing.T) {
 	is.Equal(toks.Size(), 1)
 	toks = tokens(t, "/hi")
 	is.Equal(toks.Size(), 3)
+	toks = tokens(t, "/α")
+	is.Equal(toks.Size(), 1)
 	toks = tokens(t, "/:a")
 	is.Equal(toks.Size(), 2)
 	toks = tokens(t, "/users/:id")
@@ -108,6 +114,33 @@ func TestSplit(t *testing.T) {
 	is.Equal(parts[1].Size(), 1)
 	is.Equal(parts[1].At(0), "i")
 
+	toks = tokens(t, "/α")
+	is.Equal(len(toks), 2)
+	is.Equal(toks.Size(), 2)
+	parts = toks.Split(1)
+	is.Equal(len(parts), 2)
+	is.Equal(parts[0].Size(), 1)
+	is.Equal(parts[0].At(0), "/")
+	is.Equal(parts[0].At(1), "")
+	is.Equal(parts[1].Size(), 1)
+	is.Equal(parts[1].At(0), "α")
+	is.Equal(parts[1].At(1), "")
+
+	toks = tokens(t, "/αβ")
+	is.Equal(len(toks), 2)
+	is.Equal(toks.Size(), 2)
+	parts = toks.Split(2)
+	is.Equal(len(parts), 3)
+	is.Equal(parts[0].Size(), 1)
+	is.Equal(parts[0].At(0), "/")
+	is.Equal(parts[0].At(1), "")
+	is.Equal(parts[1].Size(), 1)
+	is.Equal(parts[1].At(0), "α")
+	is.Equal(parts[1].At(1), "")
+	is.Equal(parts[1].Size(), 1)
+	is.Equal(parts[1].At(0), "β")
+	is.Equal(parts[1].At(1), "")
+
 	toks = tokens(t, "/:a")
 	is.Equal(len(toks), 2)
 	is.Equal(toks.Size(), 2)

package/router/radix/tree_test.go

@@ -274,6 +274,20 @@ func TestMatch(t *testing.T) {
 	})
 }
 
+func TestMatchUnicode(t *testing.T) {
+	ok(t, &test{
+		inserts: []*insert{
+			{route: "/α"},
+			{route: "/β"},
+		},
+		requests: []*request{
+			{path: "/α", route: "/α"},
+			{path: "/β", route: "/β"},
+			{path: "/αβ", nomatch: true},
+		},
+	})
+}
+
 func TestOptional(t *testing.T) {
 	okp(t, &test{
 		inserts: []*insert{