修复解码时出现的一些乱码

ztxz16 · Jul 6, 2024 · 677bae9 · 677bae9
1 parent 860bace
commit 677bae9
Showing 1 changed file with 10 additions and 4 deletions.
diff --git a/tools/fastllm_pytools/llm.py b/tools/fastllm_pytools/llm.py
@@ -617,8 +617,11 @@ def stream_response(self,
                                       stop_token_ids = stop_token_ids):
                 if one_by_one:
                     ret = cur[0][lastlen:]
-                    lastlen = len(cur[0])
-                    yield ret
+                    if (ret.encode().find(b'\xef\xbf\xbd') == -1):
+                        lastlen = len(cur[0])
+                        yield ret
+                    else:
+                        yield ""
                 else:
                     yield cur[0]
         else:
@@ -670,8 +673,11 @@ async def stream_response_async(self,
                                       stop_token_ids = stop_token_ids):
                 if one_by_one:
                     ret = cur[0][lastlen:]
-                    lastlen = len(cur[0])
-                    yield ret
+                    if (ret.encode().find(b'\xef\xbf\xbd') == -1):
+                        lastlen = len(cur[0])
+                        yield ret
+                    else:
+                        yield ""
                 else:
                     yield cur[0]
         else: