diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 65a3885a1c46..c830fee86987 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -442,6 +442,15 @@ "GPT2TokenizerFast" if is_tokenizers_available() else None, ), ), + ( + "ministral", + ( + "MistralCommonTokenizer" + if is_mistral_common_available() + else ("LlamaTokenizer" if is_sentencepiece_available() else None), + "LlamaTokenizerFast" if is_tokenizers_available() and not is_mistral_common_available() else None, + ), + ), ( "mistral", ( @@ -451,6 +460,15 @@ "LlamaTokenizerFast" if is_tokenizers_available() and not is_mistral_common_available() else None, ), ), + ( + "mistral3", + ( + "MistralCommonTokenizer" + if is_mistral_common_available() + else ("LlamaTokenizer" if is_sentencepiece_available() else None), + "LlamaTokenizerFast" if is_tokenizers_available() and not is_mistral_common_available() else None, + ), + ), ( "mixtral", (