Fix bytearray

CyberZHG · May 29, 2019 · 4f019d2 · 4f019d2
1 parent ddfcad7
commit 4f019d2
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/keras_gpt_2/bpe.py b/keras_gpt_2/bpe.py
@@ -76,7 +76,8 @@ def get_bpe(self, token):
     def encode(self, text):
         indices = []
         for token in re.findall(self.token_pattern, text):
-            chars = ''.join(self.byte_encoder[code] for code in token.encode('utf-8'))
+            token = bytearray(token.encode('utf-8'))
+            chars = ''.join(self.byte_encoder[code] for code in token)
             indices += [self.token_dict[token] for token in self.get_bpe(chars)]
         return indices
 

diff --git a/setup.py b/setup.py
@@ -12,7 +12,7 @@
 
 setup(
     name='keras-gpt-2',
-    version='0.9.0',
+    version='0.10.0',
     packages=find_packages(),
     url='https://github.com/CyberZHG/keras-gpt-2',
     license='MIT',