From 3c231b4ff3b419f8fc02f001daa313c157129acb Mon Sep 17 00:00:00 2001
From: Joey Hain <hain.joseph@gmail.com>
Date: Mon, 22 Sep 2025 16:11:07 -0700
Subject: [PATCH] fix token with range to account for multi-byte characters

---
 src/tokenizer.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index ba16cb508..428bb148b 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -769,7 +769,7 @@ impl State<'_> {
         match self.peekable.next() {
             None => None,
             Some(s) => {
-                self.pos += 1;
+                self.pos += s.len_utf8();
                 if s == '\n' {
                     self.line += 1;
                     self.col = 1;