adapt transformers 4.37 loading (#1606)

Signed-off-by: changwangss <chang1.wang@intel.com>
Signed-off-by: chensuyue <suyue.chen@intel.com>

.azure-pipelines/scripts/ut/env_setup.sh

@@ -84,7 +84,7 @@ fi
 # install special test env requirements
 # common deps
 pip install cmake
-pip install transformers==4.36.2
+pip install transformers
 
 if [[ $(echo "${test_case}" | grep -c "others") != 0 ]];then
     pip install tf_slim xgboost accelerate==0.21.0 peft

.azure-pipelines/scripts/ut/run_itrex.sh

@@ -12,7 +12,7 @@ bash /intel-extension-for-transformers/.github/workflows/script/install_binary.s
 
 # prepare test env
 # tmp install transformers for incompatible issue
-pip install transformers==4.36.2
+pip install transformers
 pip install -r /intel-extension-for-transformers/tests/requirements.txt
 LOG_DIR=/neural-compressor/log_dir
 mkdir -p ${LOG_DIR}

neural_compressor/utils/load_huggingface.py

@@ -122,13 +122,49 @@ def from_pretrained(cls, model_name_or_path: str, **kwargs) -> torch.nn.Module:
             else:  # pragma: no cover
                 model_class._keys_to_ignore_on_load_missing.extend(missing_keys_to_ignore_on_load)
 
+            if not os.path.isdir(model_name_or_path) and not os.path.isfile(model_name_or_path):  # pragma: no cover
+                from transformers.utils import cached_file
+
+                try:
+                    # Load from URL or cache if already cached
+                    resolved_weights_file = cached_file(
+                        model_name_or_path,
+                        filename=WEIGHTS_NAME,
+                        cache_dir=cache_dir,
+                        force_download=force_download,
+                        resume_download=resume_download,
+                        use_auth_token=use_auth_token,
+                    )
+                except EnvironmentError as err:  # pragma: no cover
+                    logger.error(err)
+                    msg = (
+                        f"Can't load weights for '{model_name_or_path}'. Make sure that:\n\n"
+                        f"- '{model_name_or_path}' is a correct model identifier "
+                        f"listed on 'https://huggingface.co/models'\n  (make sure "
+                        f"'{model_name_or_path}' is not a path to a local directory with "
+                        f"something else, in that case)\n\n- or '{model_name_or_path}' is "
+                        f"the correct path to a directory containing a file "
+                        f"named one of {WEIGHTS_NAME}\n\n"
+                    )
+                    if revision is not None:
+                        msg += (
+                            f"- or '{revision}' is a valid git identifier "
+                            f"(branch name, a tag name, or a commit id) that "
+                            f"exists for this model name as listed on its model "
+                            f"page on 'https://huggingface.co/models'\n\n"
+                        )
+                    raise EnvironmentError(msg)
+            else:
+                resolved_weights_file = os.path.join(model_name_or_path, WEIGHTS_NAME)
+            state_dict = torch.load(resolved_weights_file, {})
             model = model_class.from_pretrained(
                 model_name_or_path,
                 cache_dir=cache_dir,
                 force_download=force_download,
                 resume_download=resume_download,
                 use_auth_token=use_auth_token,
                 revision=revision,
+                state_dict=state_dict,
                 **kwargs,
             )