Support single token decode for CodeGenTokenizer (#28628)

convert token id to list in .decode()
huggingface · May 14, 2024 · a81e4c8 · a81e4c8
1 parent f946b1e
commit a81e4c8
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/src/transformers/models/codegen/tokenization_codegen.py b/src/transformers/models/codegen/tokenization_codegen.py
@@ -23,7 +23,7 @@
 import numpy as np
 import regex as re
 
-from ...utils import is_tf_available, is_torch_available, logging
+from ...utils import is_tf_available, is_torch_available, logging, to_py_obj
 
 
 if TYPE_CHECKING:
@@ -352,6 +352,9 @@ def decode(
         Returns:
             `str`: The decoded sentence.
         """
+
+        token_ids = to_py_obj(token_ids)
+
         decoded_text = super()._decode(
             token_ids=token_ids,
             skip_special_tokens=skip_special_tokens,