Fix fast tokenization problems

src/transformers/models/albert/tokenization_albert.py

@@ -143,7 +143,11 @@ def __init__(
         **kwargs
     ) -> None:
         # Mask token behave like a normal word, i.e. include the space before it
-        mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token
+        mask_token = (
+            AddedToken(mask_token, lstrip=True, rstrip=False, normalized=False)
+            if isinstance(mask_token, str)
+            else mask_token
+        )
 
         self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
 

src/transformers/models/albert/tokenization_albert_fast.py

@@ -20,7 +20,6 @@
 from typing import List, Optional, Tuple
 
 from ...file_utils import is_sentencepiece_available
-from ...tokenization_utils import AddedToken
 from ...tokenization_utils_fast import PreTrainedTokenizerFast
 from ...utils import logging
 
@@ -135,8 +134,6 @@ def __init__(
         mask_token="[MASK]",
         **kwargs
     ):
-        # Mask token behave like a normal word, i.e. include the space before it
-        mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token
 
         super().__init__(
             vocab_file,

src/transformers/tokenization_utils_fast.py

@@ -132,6 +132,10 @@ def __init__(self, *args, **kwargs):
         # We call this after having initialized the backend tokenizer because we update it.
         super().__init__(**kwargs)
 
+        # Ensure special tokens directly specified from kwargs (not from pretrained) are sanitized.
+        if "name_or_path" not in kwargs:
+            self.sanitize_special_tokens()
+
     @property
     def is_fast(self) -> bool:
         return True