Generate: change flush CJK characters to print

huggingface · Apr 8, 2023 · 58a4dfe · 58a4dfe
1 parent 2fa45c4
commit 58a4dfe
Showing 1 changed file with 2 additions and 3 deletions.
diff --git a/src/transformers/generation/streamers.py b/src/transformers/generation/streamers.py
@@ -101,11 +101,10 @@ def put(self, value):
             printable_text = text[self.print_len :]
             self.token_cache = []
             self.print_len = 0
-        # If the last token is a CJK character, we flush the cache.
+        # If the last token is a CJK character, we print the characters.
         elif len(text) > 0 and self._is_chinese_char(ord(text[-1])):
             printable_text = text[self.print_len :]
-            self.token_cache = []
-            self.print_len = 0
+            self.print_len += len(printable_text)
         # Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words,
         # which may change with the subsequent token -- there are probably smarter ways to do this!)
         else: