[tokenizer] Allows import non-english model (#2797)

deepjavalibrary · Oct 2, 2023 · 27c6a57 · 27c6a57
1 parent 8fd79db
commit 27c6a57
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/extensions/tokenizers/src/main/python/huggingface_models.py b/extensions/tokenizers/src/main/python/huggingface_models.py
@@ -83,7 +83,7 @@ def list_models(self, args: Namespace) -> List[dict]:
                 continue
 
             languages = get_lang_tags(model_info)
-            if "en" not in languages:
+            if "en" not in languages and not os.environ["HF_IMPORT_ALL"]:
                 logging.warning(f"Skip non-English model: {model_id}.")
                 continue