From d6bb62223d0d0268c11d2d7e70b854309dd2372a Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Fri, 14 Nov 2025 00:39:59 +0100 Subject: [PATCH 1/2] WIP --- src/transformers/models/auto/tokenization_auto.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 65a3885a1c46..ed475d823183 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -451,6 +451,15 @@ "LlamaTokenizerFast" if is_tokenizers_available() and not is_mistral_common_available() else None, ), ), + ( + "mistral3", + ( + "MistralCommonTokenizer" + if is_mistral_common_available() + else ("LlamaTokenizer" if is_sentencepiece_available() else None), + "LlamaTokenizerFast" if is_tokenizers_available() and not is_mistral_common_available() else None, + ), + ), ( "mixtral", ( From 4b30918ca3cad7f16f5a81afd4e489ba9569d241 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Fri, 14 Nov 2025 00:43:29 +0100 Subject: [PATCH 2/2] WIP --- src/transformers/models/auto/tokenization_auto.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index ed475d823183..c830fee86987 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -442,6 +442,15 @@ "GPT2TokenizerFast" if is_tokenizers_available() else None, ), ), + ( + "ministral", + ( + "MistralCommonTokenizer" + if is_mistral_common_available() + else ("LlamaTokenizer" if is_sentencepiece_available() else None), + "LlamaTokenizerFast" if is_tokenizers_available() and not is_mistral_common_available() else None, + ), + ), ( "mistral", (