Skip to content

Commit

Permalink
Use utf8.RuneCountInString() rather than len() :(
Browse files Browse the repository at this point in the history
Fixes #10. Thanks @curio77.
  • Loading branch information
alecthomas committed Sep 20, 2017
1 parent 3f941dd commit 36ead72
Showing 1 changed file with 2 additions and 1 deletion.
3 changes: 2 additions & 1 deletion regexp.go
Expand Up @@ -5,6 +5,7 @@ import (
"regexp"
"strings"
"sync"
"unicode/utf8"

"github.com/dlclark/regexp2"
)
Expand Down Expand Up @@ -221,7 +222,7 @@ func (r *RegexLexer) Tokenise(options *TokeniseOptions, text string, out func(*T
state.Rule = ruleIndex

state.Groups = groups
state.Pos += len(groups[0])
state.Pos += utf8.RuneCountInString(groups[0])
if rule.Mutator != nil {
if err := rule.Mutator.Mutate(state); err != nil {
return err
Expand Down

0 comments on commit 36ead72

Please sign in to comment.