potential out of bounds read

ggerganov · Mar 18, 2023 · 448f398 · 448f398
1 parent 5ae5e2c
commit 448f398
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/utils.cpp b/utils.cpp
@@ -323,7 +323,7 @@ struct llama_tokenizer {
         int index = 0;
         while (!text.empty()) {
             llama_sp_symbol sym;
-            size_t char_len = utf8_len(text.data()[0]);
+            size_t char_len = std::min(text.size(), utf8_len(text.data()[0]));
             sym.text = std::string_view(text.data(), char_len);
             sym.prev = index - 1;
             text.remove_prefix(char_len);