From 27c6a5771677ce7c9f13d69c52d78bdb574230bd Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Tue, 3 Oct 2023 03:51:11 +0800 Subject: [PATCH] [tokenizer] Allows import non-english model (#2797) --- extensions/tokenizers/src/main/python/huggingface_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/tokenizers/src/main/python/huggingface_models.py b/extensions/tokenizers/src/main/python/huggingface_models.py index 3418815d5c4..7e5ada55ea9 100644 --- a/extensions/tokenizers/src/main/python/huggingface_models.py +++ b/extensions/tokenizers/src/main/python/huggingface_models.py @@ -83,7 +83,7 @@ def list_models(self, args: Namespace) -> List[dict]: continue languages = get_lang_tags(model_info) - if "en" not in languages: + if "en" not in languages and not os.environ["HF_IMPORT_ALL"]: logging.warning(f"Skip non-English model: {model_id}.") continue