Skip to content

mistral common backend fix #46667

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ArthurZucker merged 5 commits into from
Jun 15, 2026
Merged

mistral common backend fix #46667

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
69a6cb6
mistral common backend fix
itazap Jun 15, 2026
d48db2a
testss
itazap Jun 15, 2026
33430a8
legacy mistral use tokenziersbackend (zephyr)
itazap Jun 15, 2026
c5604e1
fix
itazap Jun 15, 2026
4bbe2a3
fix_mistral_Regex arg
itazap Jun 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 57 additions & 8 deletions src/transformers/models/auto/tokenization_auto.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
is_tokenizers_available,
logging,
)
from ...utils.hub import cached_file
from ...utils.hub import cached_file, has_file
from ..encoder_decoder import EncoderDecoderConfig
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
Expand Down Expand Up @@ -416,6 +416,25 @@ def load_merges(merges_file):
return merges


def _has_tekken_tokenizer_file(
pretrained_model_name_or_path: str | os.PathLike[str],
**kwargs,
) -> bool:
subfolder = kwargs.get("subfolder", "")
tekken_filename = os.path.join(subfolder, "tekken.json") if subfolder else "tekken.json"
try:
return has_file(
pretrained_model_name_or_path,
tekken_filename,
revision=kwargs.get("revision"),
token=kwargs.get("token"),
cache_dir=kwargs.get("cache_dir"),
local_files_only=kwargs.get("local_files_only", False),
)
except OSError:
return False


def tokenizer_class_from_name(class_name: str) -> type[Any] | None:
# Bloom tokenizer classes were removed but should map to the fast backend for BC
if class_name in {"BloomTokenizer", "BloomTokenizerFast"}:
Expand Down Expand Up @@ -701,6 +720,7 @@ def from_pretrained(
config = PreTrainedConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)

config_model_type = config.model_type
config_model_name = config.model_name if hasattr(config, "model_name") else None

# Next, let's try to use the tokenizer_config file to get the tokenizer class.
tokenizer_config = get_tokenizer_config(pretrained_model_name_or_path, **kwargs)
Expand All @@ -717,22 +737,32 @@ def from_pretrained(

# if there is a config, we can check that the tokenizer class != than model class.
# Use the config class if it's a specialized tokenizer, otherwise fall back to TokenizersBackend.
# Hub class should prioritize tokenizer_config.json or fallback to config.tokenizer_class.
_hub_class = tokenizer_config_class or getattr(config, "tokenizer_class", None)
if (
tokenizer_auto_map is None
and tokenizer_config_class is not None
and _hub_class is not None
and config_model_type is not None
and config_model_type != ""
and TOKENIZER_MAPPING_NAMES.get(config_model_type) is not None
and (TOKENIZER_MAPPING_NAMES.get(config_model_type).removesuffix("Fast"))
!= (tokenizer_config_class.removesuffix("Fast"))
!= (_hub_class.removesuffix("Fast"))
):
registered_class_name = TOKENIZER_MAPPING_NAMES.get(config_model_type).removesuffix("Fast")
if registered_class_name not in ("TokenizersBackend", "PythonBackend", "PreTrainedTokenizerFast"):
if registered_class_name not in (
"TokenizersBackend",
"PythonBackend",
"PreTrainedTokenizerFast",
"MistralCommonBackend",
):
# If the hub class is known incorrect for this model type, use the registered class; otherwise trust the hub.
class_name = (
registered_class_name
if config_model_type in MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS
else tokenizer_config_class
if (
config_model_type in MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS
or config_model_name in MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS
)
else _hub_class
)
tokenizer_class = tokenizer_class_from_name(class_name)
if tokenizer_class is not None and tokenizer_class.__name__ not in (
Expand All @@ -742,11 +772,21 @@ def from_pretrained(
):
return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)

if (
registered_class_name == "MistralCommonBackend"
and is_mistral_common_available()
and "fix_mistral_regex" not in kwargs
and _has_tekken_tokenizer_file(pretrained_model_name_or_path, **kwargs)
):
tokenizer_class = tokenizer_class_from_name("MistralCommonBackend")
if tokenizer_class is not None:
return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)

if TokenizersBackend is not None:
return TokenizersBackend.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)

raise ValueError(
f"Tokenizer class '{tokenizer_config_class}' specified in the tokenizer config was not found. "
f"Tokenizer class '{_hub_class}' specified in the tokenizer config was not found. "
f"The tokenizer may need to be converted or re-saved."
)

Expand Down Expand Up @@ -776,7 +816,11 @@ def from_pretrained(
)
)
# V5: Skip remote tokenizer for custom models with incorrect hub tokenizer class
if has_remote_code and config_model_type in MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS:
if (
has_remote_code
and config_model_type in MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS
and trust_remote_code is not True
):
has_remote_code = False
tokenizer_auto_map = None

Expand Down Expand Up @@ -839,6 +883,11 @@ def from_pretrained(
if model_type is not None:
tokenizer_class = TOKENIZER_MAPPING.get(type(config), TokenizersBackend)
if tokenizer_class is not None:
if getattr(tokenizer_class, "__name__", None) == "MistralCommonBackend" and (
"fix_mistral_regex" in kwargs
or not _has_tekken_tokenizer_file(pretrained_model_name_or_path, **kwargs)
):
tokenizer_class = TokenizersBackend
return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)

# Fallback: try tokenizer_class from tokenizer_config.json
Expand Down
38 changes: 38 additions & 0 deletions tests/models/auto/test_tokenization_auto.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
SMALL_MODEL_IDENTIFIER,
CaptureLogger,
RequestCounter,
require_mistral_common,
require_sentencepiece,
require_tokenizers,
slow,
Expand Down Expand Up @@ -236,6 +237,43 @@ def test_voxtral_tokenizer_converts_from_tekken(self):
self.assertTrue(tokenizer.is_fast)
self.assertGreater(len(tokenizer("Voxtral")["input_ids"]), 0)

@require_tokenizers
@require_mistral_common
def test_mistral_common_backend_skips_incorrect_hub_tokenizer_class(self):
"""Some Mistral checkpoint have tokenizer_class=LlamaTokenizer in its hub tokenizer_config.json.
When tekken.json is available, the wrong hub class should be ignored and MistralCommonBackend should be loaded."""
from transformers.tokenization_mistral_common import MistralCommonBackend

tokenizer = AutoTokenizer.from_pretrained("mistralai/Ministral-8B-Instruct-2410")
self.assertIsInstance(tokenizer, MistralCommonBackend)
self.assertGreater(len(tokenizer("Ministral")["input_ids"]), 0)

@require_tokenizers
def test_mistral_common_backend_skips_incorrect_hub_tokenizer_class_without_mistral_common(self):
"""Some Mistral checkpoint have tokenizer_class=LlamaTokenizer in its hub tokenizer_config.json.
When mistral-common is unavailable and tokenizer.json exists, TokenizersBackend should be loaded."""
repo_id = "mistralai/Ministral-8B-Instruct-2410"
with (
mock.patch(
"transformers.models.auto.tokenization_auto.is_mistral_common_available",
return_value=False,
),
mock.patch.dict(TOKENIZER_MAPPING_NAMES, {"ministral": "TokenizersBackend"}),
):
tokenizer = AutoTokenizer.from_pretrained(repo_id)

self.assertIsInstance(tokenizer, TokenizersBackend)
self.assertGreater(len(tokenizer("Ministral")["input_ids"]), 0)

@require_tokenizers
@require_mistral_common
def test_mistral_sentencepiece_models_use_tokenizers_backend(self):
"""Regression: legacy Mistral models with only tokenizer.model (no tekken.json) should keep
using TokenizersBackend even when mistral-common is installed."""
tokenizer = AutoTokenizer.from_pretrained("HuggingFaceH4/zephyr-7b-beta")
self.assertIsInstance(tokenizer, TokenizersBackend)
self.assertGreater(len(tokenizer("zephyr")["input_ids"]), 0)
Comment on lines +240 to +275

@ArthurZucker ArthurZucker Jun 15, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

that is very nice! 🤗


@require_tokenizers
def test_do_lower_case(self):
tokenizer = AutoTokenizer.from_pretrained("distilbert/distilbert-base-uncased", do_lower_case=False)
Expand Down
Loading