decode fix

IlyaGusev · May 16, 2023 · 5b31a0b · 5b31a0b
1 parent 15c9889
commit 5b31a0b
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/self_instruct/src/interact_llamacpp.py b/self_instruct/src/interact_llamacpp.py
@@ -61,7 +61,7 @@ def interact(
             repeat_penalty=repeat_penalty
         )
         for token in generator:
-            token_str = model.detokenize([token]).decode("utf-8")
+            token_str = model.detokenize([token]).decode("utf-8", errors="ignore")
             tokens.append(token)
             if token == model.token_eos():
                 break