LlamaAttention has no attribute rotary_emb (4.50.0.dev0)

[efsotr]

System Info

pip install git+https://github.com/huggingface/transformers@v4.49.0-Gemma-3
transformers 4.50.0.dev0

Who can help?

@ArthurZucker

Information

• The official example scripts
• My own modified scripts

Tasks

• An officially supported task in the examples folder (such as GLUE/SQuAD, ...)
• My own task or dataset (give details below)

Reproduction

from transformers import AutoModelForCausalLM
model = AutoModelForCausalLM.from_pretrained(
    "./models/Llama-2-7b-hf",
    device_map="cpu"
)

error log

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
File ~/anaconda3/envs/profile/lib/python3.10/site-packages/transformers/modeling_utils.py:5906, in caching_allocator_warmup(model, expanded_device_map, dtype)
   5905 try:
-> 5906     param = getattr(model, param_name)
   5907 except AttributeError:

File ~/anaconda3/envs/profile/lib/python3.10/site-packages/torch/nn/modules/module.py:1931, in Module.__getattr__(self, name)
   1930         return modules[name]
-> 1931 raise AttributeError(
   1932     f"'{type(self).__name__}' object has no attribute '{name}'"
   1933 )

AttributeError: 'LlamaForCausalLM' object has no attribute 'model.layers.0.self_attn.rotary_emb.inv_freq'

During handling of the above exception, another exception occurred:

AttributeError                            Traceback (most recent call last)
Cell In[2], line 2
      1 from transformers import AutoModelForCausalLM
----> 2 model = AutoModelForCausalLM.from_pretrained(
      3     "/home/linli/Quant/Ours/models/Llama-2-7b-hf",
      4     device_map="cpu"
      5 )

File ~/anaconda3/envs/profile/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py:564, in _BaseAutoModelClass.from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
    562 elif type(config) in cls._model_mapping.keys():
    563     model_class = _get_model_class(config, cls._model_mapping)
--> 564     return model_class.from_pretrained(
    565         pretrained_model_name_or_path, *model_args, config=config, **hub_kwargs, **kwargs
    566     )
    567 raise ValueError(
    568     f"Unrecognized configuration class {config.__class__} for this kind of AutoModel: {cls.__name__}.\n"
    569     f"Model type should be one of {', '.join(c.__name__ for c in cls._model_mapping.keys())}."
    570 )

File ~/anaconda3/envs/profile/lib/python3.10/site-packages/transformers/modeling_utils.py:273, in restore_default_torch_dtype.<locals>._wrapper(*args, **kwargs)
    271 old_dtype = torch.get_default_dtype()
    272 try:
--> 273     return func(*args, **kwargs)
    274 finally:
    275     torch.set_default_dtype(old_dtype)

File ~/anaconda3/envs/profile/lib/python3.10/site-packages/transformers/modeling_utils.py:4451, in PreTrainedModel.from_pretrained(cls, pretrained_model_name_or_path, config, cache_dir, ignore_mismatched_sizes, force_download, local_files_only, token, revision, use_safetensors, weights_only, *model_args, **kwargs)
   4441     if dtype_orig is not None:
   4442         torch.set_default_dtype(dtype_orig)
   4444     (
   4445         model,
   4446         missing_keys,
   4447         unexpected_keys,
   4448         mismatched_keys,
   4449         offload_index,
   4450         error_msgs,
-> 4451     ) = cls._load_pretrained_model(
   4452         model,
   4453         state_dict,
   4454         loaded_state_dict_keys,  # XXX: rename?
   4455         resolved_archive_file or gguf_file,
   4456         pretrained_model_name_or_path,
   4457         ignore_mismatched_sizes=ignore_mismatched_sizes,
   4458         sharded_metadata=sharded_metadata,
   4459         _fast_init=_fast_init,
   4460         low_cpu_mem_usage=low_cpu_mem_usage,
   4461         device_map=device_map,
   4462         offload_folder=offload_folder,
   4463         offload_state_dict=offload_state_dict,
   4464         dtype=torch_dtype,
   4465         hf_quantizer=hf_quantizer,
   4466         keep_in_fp32_modules=keep_in_fp32_modules,
   4467         gguf_path=gguf_path,
   4468         weights_only=weights_only,
   4469         device_mesh=device_mesh,
   4470     )
   4472 # make sure token embedding weights are still tied if needed
   4473 model.tie_weights()

File ~/anaconda3/envs/profile/lib/python3.10/site-packages/transformers/modeling_utils.py:4894, in PreTrainedModel._load_pretrained_model(cls, model, state_dict, loaded_keys, resolved_archive_file, pretrained_model_name_or_path, ignore_mismatched_sizes, sharded_metadata, _fast_init, low_cpu_mem_usage, device_map, offload_folder, offload_state_dict, dtype, hf_quantizer, keep_in_fp32_modules, gguf_path, weights_only, device_mesh)
   4892     expanded_device_map = expand_device_map(device_map, original_loaded_keys, start_prefix)
   4893     if hf_quantizer is None:
-> 4894         caching_allocator_warmup(model_to_load, expanded_device_map, dtype)
   4896 if device_map is not None and is_safetensors:
   4897     param_device_map = expanded_device_map

File ~/anaconda3/envs/profile/lib/python3.10/site-packages/transformers/modeling_utils.py:5910, in caching_allocator_warmup(model, expanded_device_map, dtype)
   5908 if "." in param_name:
   5909     param_name, param_type = param_name.rsplit(".", 1)
-> 5910     param = getattr(model.get_submodule(param_name), param_type)
   5911 else:
   5912     param = model.get_buffer(param_name)

File ~/anaconda3/envs/profile/lib/python3.10/site-packages/torch/nn/modules/module.py:717, in Module.get_submodule(self, target)
    715 for item in atoms:
    716     if not hasattr(mod, item):
--> 717         raise AttributeError(
    718             mod._get_name() + " has no " "attribute `" + item + "`"
    719         )
    721     mod = getattr(mod, item)
    723     if not isinstance(mod, torch.nn.Module):

AttributeError: LlamaAttention has no attribute `rotary_emb`

Expected behavior

No error

[efsotr]

@Rocketknight1

[Zephyr271828]

@efsotr Hi, I just submitted a PR to fix this issue!

[Rocketknight1]

Hi @efsotr, branches like 4.49.0-Gemma3 are intended to rapidly add support for a new model like Gemma3. They are intended as temporary / not fully stable, and will be superceded once support for that model is added in the next full release, 4.50. We generally don't fix bugs in them, particularly when the bugs don't relate to the new model they support!

Closing for now, but feel free to reopen if this bug is occurring on the main branch as well