Merge pull request huggingface#37 from jamesthesnake/pop

Pop

docs/source/ko/_toctree.yml

@@ -26,8 +26,8 @@
   - sections:
     - local: in_translation
       title: (번역중) Create a custom architecture
-    - local: in_translation
-      title: (번역중) Sharing custom models
+    - local: custom_models
+      title: 사용자 정의 모델 공유하기
     - local: in_translation
       title: (번역중) Train with a script
     - local: sagemaker
@@ -59,8 +59,8 @@
         title: (번역중) Causal language modeling
       - local: in_translation
         title: (번역중) Masked language modeling
-      - local: in_translation
-        title: (번역중) Translation
+      - local: tasks/translation
+        title: 번역
       - local: in_translation
         title: (번역중) Summarization
       - local: in_translation

docs/source/ko/sagemaker.mdx

@@ -14,11 +14,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 -->
 
-# Amazon SageMaker에서 학습 실행하기
+# Amazon SageMaker에서 학습 실행하기[[run-training-on-amazon-sagemaker]]
 
 문서가 [hf.co/docs/sagemaker](https://huggingface.co/docs/sagemaker)로 이동되었습니다. 이 페이지는 `transformers` 5.0 에서 삭제될 예정입니다. 
 
-### 목차
+### 목차[[table-of-content]]
 
 - [Train Hugging Face models on Amazon SageMaker with the SageMaker Python SDK](https://huggingface.co/docs/sagemaker/train)
 - [Deploy Hugging Face models to Amazon SageMaker with the SageMaker Python SDK](https://huggingface.co/docs/sagemaker/inference)

examples/tensorflow/_tests_requirements.txt

@@ -4,7 +4,6 @@ scikit-learn
 seqeval
 psutil
 sacrebleu >= 1.4.12
-git+https://github.com/huggingface/accelerate@main#egg=accelerate
 rouge-score
 tensorflow_datasets
 matplotlib

examples/tensorflow/image-classification/run_image_classification.py

@@ -34,7 +34,7 @@
 
 import transformers
 from transformers import (
-    MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
+    TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
     AutoConfig,
     AutoImageProcessor,
     DefaultDataCollator,
@@ -58,7 +58,7 @@
 
 require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/image-classification/requirements.txt")
 
-MODEL_CONFIG_CLASSES = list(MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING.keys())
+MODEL_CONFIG_CLASSES = list(TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING.keys())
 MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
 
 
@@ -262,11 +262,6 @@ def main():
         transformers.utils.logging.set_verbosity_info()
         transformers.utils.logging.enable_default_handler()
         transformers.utils.logging.enable_explicit_format()
-    # Log on each process the small summary:
-    logger.warning(
-        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
-        + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
-    )
     logger.info(f"Training/evaluation parameters {training_args}")
 
     # region Dataset and labels

examples/tensorflow/token-classification/run_ner.py

@@ -14,8 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """
-Fine-tuning a 🤗 Transformers model on token classification tasks (NER, POS, CHUNKS) relying on the accelerate library
-without using a Trainer.
+Fine-tuning a 🤗 Transformers model on token classification tasks (NER, POS, CHUNKS)
 """
 
 import json

src/transformers/configuration_utils.py

@@ -667,6 +667,11 @@ def _get_config_dict(
         else:
             logger.info(f"loading configuration file {configuration_file} from cache at {resolved_config_file}")
 
+        if "auto_map" in config_dict and not is_local:
+            config_dict["auto_map"] = {
+                k: (f"{pretrained_model_name_or_path}--{v}" if "--" not in v else v)
+                for k, v in config_dict["auto_map"].items()
+            }
         return config_dict, kwargs
 
     @classmethod

src/transformers/dynamic_module_utils.py

@@ -29,6 +29,7 @@
     extract_commit_hash,
     is_offline_mode,
     logging,
+    try_to_load_from_cache,
 )
 
 
@@ -222,11 +223,16 @@ def get_cached_module_file(
 
     # Download and cache module_file from the repo `pretrained_model_name_or_path` of grab it if it's a local file.
     pretrained_model_name_or_path = str(pretrained_model_name_or_path)
-    if os.path.isdir(pretrained_model_name_or_path):
+    is_local = os.path.isdir(pretrained_model_name_or_path)
+    if is_local:
         submodule = pretrained_model_name_or_path.split(os.path.sep)[-1]
     else:
         submodule = pretrained_model_name_or_path.replace("/", os.path.sep)
+        cached_module = try_to_load_from_cache(
+            pretrained_model_name_or_path, module_file, cache_dir=cache_dir, revision=_commit_hash
+        )
 
+    new_files = []
     try:
         # Load from URL or cache if already cached
         resolved_module_file = cached_file(
@@ -241,6 +247,8 @@ def get_cached_module_file(
             revision=revision,
             _commit_hash=_commit_hash,
         )
+        if not is_local and cached_module != resolved_module_file:
+            new_files.append(module_file)
 
     except EnvironmentError:
         logger.error(f"Could not locate the {module_file} inside {pretrained_model_name_or_path}.")
@@ -284,7 +292,7 @@ def get_cached_module_file(
             importlib.invalidate_caches()
         # Make sure we also have every file with relative
         for module_needed in modules_needed:
-            if not (submodule_path / module_needed).exists():
+            if not (submodule_path / f"{module_needed}.py").exists():
                 get_cached_module_file(
                     pretrained_model_name_or_path,
                     f"{module_needed}.py",
@@ -295,14 +303,24 @@ def get_cached_module_file(
                     use_auth_token=use_auth_token,
                     revision=revision,
                     local_files_only=local_files_only,
+                    _commit_hash=commit_hash,
                 )
+                new_files.append(f"{module_needed}.py")
+
+    if len(new_files) > 0:
+        new_files = "\n".join([f"- {f}" for f in new_files])
+        logger.warning(
+            f"A new version of the following files was downloaded from {pretrained_model_name_or_path}:\n{new_files}"
+            "\n. Make sure to double-check they do not contain any added malicious code. To avoid downloading new "
+            "versions of the code file, you can pin a revision."
+        )
+
     return os.path.join(full_submodule, module_file)
 
 
 def get_class_from_dynamic_module(
+    class_reference: str,
     pretrained_model_name_or_path: Union[str, os.PathLike],
-    module_file: str,
-    class_name: str,
     cache_dir: Optional[Union[str, os.PathLike]] = None,
     force_download: bool = False,
     resume_download: bool = False,
@@ -323,6 +341,8 @@ def get_class_from_dynamic_module(
     </Tip>
 
     Args:
+        class_reference (`str`):
+            The full name of the class to load, including its module and optionally its repo.
         pretrained_model_name_or_path (`str` or `os.PathLike`):
             This can be either:
 
@@ -332,6 +352,7 @@ def get_class_from_dynamic_module(
             - a path to a *directory* containing a configuration file saved using the
               [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.
 
+            This is used when `class_reference` does not specify another repo.
         module_file (`str`):
             The name of the module file containing the class to look for.
         class_name (`str`):
@@ -371,12 +392,25 @@ def get_class_from_dynamic_module(
     ```python
     # Download module `modeling.py` from huggingface.co and cache then extract the class `MyBertModel` from this
     # module.
-    cls = get_class_from_dynamic_module("sgugger/my-bert-model", "modeling.py", "MyBertModel")
+    cls = get_class_from_dynamic_module("modeling.MyBertModel", "sgugger/my-bert-model")
+
+    # Download module `modeling.py` from a given repo and cache then extract the class `MyBertModel` from this
+    # module.
+    cls = get_class_from_dynamic_module("sgugger/my-bert-model--modeling.MyBertModel", "sgugger/another-bert-model")
     ```"""
+    # Catch the name of the repo if it's specified in `class_reference`
+    if "--" in class_reference:
+        repo_id, class_reference = class_reference.split("--")
+        # Invalidate revision since it's not relevant for this repo
+        revision = "main"
+    else:
+        repo_id = pretrained_model_name_or_path
+    module_file, class_name = class_reference.split(".")
+
     # And lastly we get the class inside our newly created module
     final_module = get_cached_module_file(
-        pretrained_model_name_or_path,
-        module_file,
+        repo_id,
+        module_file + ".py",
         cache_dir=cache_dir,
         force_download=force_download,
         resume_download=resume_download,

src/transformers/models/auto/auto_factory.py

@@ -403,8 +403,12 @@ def from_config(cls, config, **kwargs):
                     "no malicious code has been contributed in a newer revision."
                 )
             class_ref = config.auto_map[cls.__name__]
+            if "--" in class_ref:
+                repo_id, class_ref = class_ref.split("--")
+            else:
+                repo_id = config.name_or_path
             module_file, class_name = class_ref.split(".")
-            model_class = get_class_from_dynamic_module(config.name_or_path, module_file + ".py", class_name, **kwargs)
+            model_class = get_class_from_dynamic_module(repo_id, module_file + ".py", class_name, **kwargs)
             return model_class._from_config(config, **kwargs)
         elif type(config) in cls._model_mapping.keys():
             model_class = _get_model_class(config, cls._model_mapping)
@@ -452,17 +456,10 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
                     "on your local machine. Make sure you have read the code there to avoid malicious use, then set "
                     "the option `trust_remote_code=True` to remove this error."
                 )
-            if hub_kwargs.get("revision", None) is None:
-                logger.warning(
-                    "Explicitly passing a `revision` is encouraged when loading a model with custom code to ensure "
-                    "no malicious code has been contributed in a newer revision."
-                )
             class_ref = config.auto_map[cls.__name__]
-            module_file, class_name = class_ref.split(".")
             model_class = get_class_from_dynamic_module(
-                pretrained_model_name_or_path, module_file + ".py", class_name, **hub_kwargs, **kwargs
+                class_ref, pretrained_model_name_or_path, **hub_kwargs, **kwargs
             )
-            model_class.register_for_auto_class(cls.__name__)
             return model_class.from_pretrained(
                 pretrained_model_name_or_path, *model_args, config=config, **hub_kwargs, **kwargs
             )

src/transformers/models/auto/configuration_auto.py

@@ -921,17 +921,8 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
                     " repo on your local machine. Make sure you have read the code there to avoid malicious use, then"
                     " set the option `trust_remote_code=True` to remove this error."
                 )
-            if kwargs.get("revision", None) is None:
-                logger.warning(
-                    "Explicitly passing a `revision` is encouraged when loading a configuration with custom code to "
-                    "ensure no malicious code has been contributed in a newer revision."
-                )
             class_ref = config_dict["auto_map"]["AutoConfig"]
-            module_file, class_name = class_ref.split(".")
-            config_class = get_class_from_dynamic_module(
-                pretrained_model_name_or_path, module_file + ".py", class_name, **kwargs
-            )
-            config_class.register_for_auto_class()
+            config_class = get_class_from_dynamic_module(class_ref, pretrained_model_name_or_path, **kwargs)
             return config_class.from_pretrained(pretrained_model_name_or_path, **kwargs)
         elif "model_type" in config_dict:
             config_class = CONFIG_MAPPING[config_dict["model_type"]]

src/transformers/models/auto/feature_extraction_auto.py

@@ -333,17 +333,9 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
                         "in that repo on your local machine. Make sure you have read the code there to avoid "
                         "malicious use, then set the option `trust_remote_code=True` to remove this error."
                     )
-                if kwargs.get("revision", None) is None:
-                    logger.warning(
-                        "Explicitly passing a `revision` is encouraged when loading a feature extractor with custom "
-                        "code to ensure no malicious code has been contributed in a newer revision."
-                    )
-
-                module_file, class_name = feature_extractor_auto_map.split(".")
                 feature_extractor_class = get_class_from_dynamic_module(
-                    pretrained_model_name_or_path, module_file + ".py", class_name, **kwargs
+                    feature_extractor_auto_map, pretrained_model_name_or_path, **kwargs
                 )
-                feature_extractor_class.register_for_auto_class()
             else:
                 feature_extractor_class = feature_extractor_class_from_name(feature_extractor_class)
 

src/transformers/models/auto/image_processing_auto.py

@@ -355,17 +355,9 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
                         "in that repo on your local machine. Make sure you have read the code there to avoid "
                         "malicious use, then set the option `trust_remote_code=True` to remove this error."
                     )
-                if kwargs.get("revision", None) is None:
-                    logger.warning(
-                        "Explicitly passing a `revision` is encouraged when loading a image processor with custom "
-                        "code to ensure no malicious code has been contributed in a newer revision."
-                    )
-
-                module_file, class_name = image_processor_auto_map.split(".")
                 image_processor_class = get_class_from_dynamic_module(
-                    pretrained_model_name_or_path, module_file + ".py", class_name, **kwargs
+                    image_processor_auto_map, pretrained_model_name_or_path, **kwargs
                 )
-                image_processor_class.register_for_auto_class()
             else:
                 image_processor_class = image_processor_class_from_name(image_processor_class)
 

src/transformers/models/auto/processing_auto.py

@@ -254,17 +254,10 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
                         "in that repo on your local machine. Make sure you have read the code there to avoid "
                         "malicious use, then set the option `trust_remote_code=True` to remove this error."
                     )
-                if kwargs.get("revision", None) is None:
-                    logger.warning(
-                        "Explicitly passing a `revision` is encouraged when loading a feature extractor with custom "
-                        "code to ensure no malicious code has been contributed in a newer revision."
-                    )
 
-                module_file, class_name = processor_auto_map.split(".")
                 processor_class = get_class_from_dynamic_module(
-                    pretrained_model_name_or_path, module_file + ".py", class_name, **kwargs
+                    processor_auto_map, pretrained_model_name_or_path, **kwargs
                 )
-                processor_class.register_for_auto_class()
             else:
                 processor_class = processor_class_from_name(processor_class)
 

src/transformers/models/auto/tokenization_auto.py

@@ -671,22 +671,12 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
                         " repo on your local machine. Make sure you have read the code there to avoid malicious use,"
                         " then set the option `trust_remote_code=True` to remove this error."
                     )
-                if kwargs.get("revision", None) is None:
-                    logger.warning(
-                        "Explicitly passing a `revision` is encouraged when loading a model with custom code to ensure"
-                        " no malicious code has been contributed in a newer revision."
-                    )
 
                 if use_fast and tokenizer_auto_map[1] is not None:
                     class_ref = tokenizer_auto_map[1]
                 else:
                     class_ref = tokenizer_auto_map[0]
-
-                module_file, class_name = class_ref.split(".")
-                tokenizer_class = get_class_from_dynamic_module(
-                    pretrained_model_name_or_path, module_file + ".py", class_name, **kwargs
-                )
-                tokenizer_class.register_for_auto_class()
+                tokenizer_class = get_class_from_dynamic_module(class_ref, pretrained_model_name_or_path, **kwargs)
 
             elif use_fast and not config_tokenizer_class.endswith("Fast"):
                 tokenizer_class_candidate = f"{config_tokenizer_class}Fast"

src/transformers/models/llama/modeling_llama.py

@@ -131,10 +131,11 @@ def rotate_half(x):
 
 
 def apply_rotary_pos_emb(q, k, cos, sin, position_ids):
-    gather_indices = position_ids[:, None, :, None]  # [bs, 1, seq_len, 1]
-    gather_indices = gather_indices.repeat(1, cos.shape[1], 1, cos.shape[3])
-    cos = torch.gather(cos.repeat(gather_indices.shape[0], 1, 1, 1), 2, gather_indices)
-    sin = torch.gather(sin.repeat(gather_indices.shape[0], 1, 1, 1), 2, gather_indices)
+    # The first two dimensions of cos and sin are always 1, so we can `squeeze` them.
+    cos = cos.squeeze(1).squeeze(0)  # [seq_len, dim]
+    sin = sin.squeeze(1).squeeze(0)  # [seq_len, dim]
+    cos = cos[position_ids].unsqueeze(1)  # [bs, 1, seq_len, dim]
+    sin = sin[position_ids].unsqueeze(1)  # [bs, 1, seq_len, dim]
     q_embed = (q * cos) + (rotate_half(q) * sin)
     k_embed = (k * cos) + (rotate_half(k) * sin)
     return q_embed, k_embed

src/transformers/pipelines/__init__.py

@@ -727,9 +727,8 @@ def pipeline(
                     " set the option `trust_remote_code=True` to remove this error."
                 )
             class_ref = targeted_task["impl"]
-            module_file, class_name = class_ref.split(".")
             pipeline_class = get_class_from_dynamic_module(
-                model, module_file + ".py", class_name, revision=revision, use_auth_token=use_auth_token
+                class_ref, model, revision=revision, use_auth_token=use_auth_token
             )
     else:
         normalized_task, targeted_task, task_options = check_task(task)