Lora loading for SDXL works on StableDiffusionXLPipeline but not on StablediffusionXLPipeline #4664
Description
Describe the bug
Hello,
Thank you for this useful library.
I have a small problem, I managed to use the code to generate images using SDXL with controlnet, then an Image with a Lora. But for some reasons not with both.
When I load the lora using the StableDiffusionXLPipeline It does work but not when I do so with this : StablediffusionXLPipeline
Thank you very much for your help.
And I hope that this will help improve the library :).
Reproduction
`import torch
from diffusers import StableDiffusionXLControlNetPipeline, ControlNetModel, AutoencoderKL, UniPCMultistepScheduler
controlnets = [
ControlNetModel.from_pretrained("diffusers/controlnet-canny-sdxl-1.0", torch_dtype=torch.float16, use_safetensors=True),
]
vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16, use_safetensors=True)
model_path = "stabilityai/stable-diffusion-xl-base-1.0"
pipe = StableDiffusionXLControlNetPipeline.from_pretrained(model_path, torch_dtype=torch.float16,controlnet=controlnets, vae=vae)
pipe.to("cuda")
pipe.load_lora_weights("nerijs/pixel-art-xl", weight_name="pixel-art-xl.safetensors")
`
Logs
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ /home/user/anaconda3/envs/sandbox/lib/python3.9/runpy.py:197 in _run_module_as_main │
│ │
│ 194 │ main_globals = sys.modules["__main__"].__dict__ │
│ 195 │ if alter_argv: │
│ 196 │ │ sys.argv[0] = mod_spec.origin │
│ ❱ 197 │ return _run_code(code, main_globals, None, │
│ 198 │ │ │ │ │ "__main__", mod_spec) │
│ 199 │
│ 200 def run_module(mod_name, init_globals=None, │
│ │
│ /home/user/anaconda3/envs/sandbox/lib/python3.9/runpy.py:87 in _run_code │
│ │
│ 84 │ │ │ │ │ __loader__ = loader, │
│ 85 │ │ │ │ │ __package__ = pkg_name, │
│ 86 │ │ │ │ │ __spec__ = mod_spec) │
│ ❱ 87 │ exec(code, run_globals) │
│ 88 │ return run_globals │
│ 89 │
│ 90 def _run_module_code(code, init_globals=None, │
│ │
│ /home/user/.vscode/extensions/ms-python.python-2023.14.0/pythonFiles/lib/python/debugpy/adapter │
│ /../../debugpy/launcher/../../debugpy/__main__.py:39 in <module> │
│ │
│ 36 │ │
│ 37 │ from debugpy.server import cli │
│ 38 │ │
│ ❱ 39 │ cli.main() │
│ 40 │
│ │
│ /home/user/.vscode/extensions/ms-python.python-2023.14.0/pythonFiles/lib/python/debugpy/adapter │
│ /../../debugpy/launcher/../../debugpy/../debugpy/server/cli.py:430 in main │
│ │
│ 427 │ │ │ "code": run_code, │
│ 428 │ │ │ "pid": attach_to_pid, │
│ 429 │ │ }[options.target_kind] │
│ ❱ 430 │ │ run() │
│ 431 │ except SystemExit as exc: │
│ 432 │ │ log.reraise_exception( │
│ 433 │ │ │ "Debuggee exited via SystemExit: {0!r}", exc.code, level="debug" │
│ │
│ /home/user/.vscode/extensions/ms-python.python-2023.14.0/pythonFiles/lib/python/debugpy/adapter │
│ /../../debugpy/launcher/../../debugpy/../debugpy/server/cli.py:284 in run_file │
│ │
│ 281 │ log.describe_environment("Pre-launch environment:") │
│ 282 │ │
│ 283 │ log.info("Running file {0!r}", target) │
│ ❱ 284 │ runpy.run_path(target, run_name="__main__") │
│ 285 │
│ 286 │
│ 287 def run_module(): │
│ │
│ /home/user/.vscode/extensions/ms-python.python-2023.14.0/pythonFiles/lib/python/debugpy/_vendor │
│ ed/pydevd/_pydevd_bundle/pydevd_runpy.py:321 in run_path │
│ │
│ 318 │ │ # Not a valid sys.path entry, so run the code directly │
│ 319 │ │ # execfile() doesn't help as we want to allow compiled files │
│ 320 │ │ code, fname = _get_code_from_file(run_name, path_name) │
│ ❱ 321 │ │ return _run_module_code(code, init_globals, run_name, │
│ 322 │ │ │ │ │ │ │ │ pkg_name=pkg_name, script_name=fname) │
│ 323 │ else: │
│ 324 │ │ # Finder is defined for path, so add it to │
│ │
│ /home/user/.vscode/extensions/ms-python.python-2023.14.0/pythonFiles/lib/python/debugpy/_vendor │
│ ed/pydevd/_pydevd_bundle/pydevd_runpy.py:135 in _run_module_code │
│ │
│ 132 │ fname = script_name if mod_spec is None else mod_spec.origin │
│ 133 │ with _TempModule(mod_name) as temp_module, _ModifiedArgv0(fname): │
│ 134 │ │ mod_globals = temp_module.module.__dict__ │
│ ❱ 135 │ │ _run_code(code, mod_globals, init_globals, │
│ 136 │ │ │ │ mod_name, mod_spec, pkg_name, script_name) │
│ 137 │ # Copy the globals of the temporary module, as they │
│ 138 │ # may be cleared when the temporary module goes away │
│ │
│ /home/user/.vscode/extensions/ms-python.python-2023.14.0/pythonFiles/lib/python/debugpy/_vendor │
│ ed/pydevd/_pydevd_bundle/pydevd_runpy.py:124 in _run_code │
│ │
│ 121 │ │ │ │ │ __loader__=loader, │
│ 122 │ │ │ │ │ __package__=pkg_name, │
│ 123 │ │ │ │ │ __spec__=mod_spec) │
│ ❱ 124 │ exec(code, run_globals) │
│ 125 │ return run_globals │
│ 126 │
│ 127 │
│ │
│ /mnt/bigStorage/projects/SDXL/aaa.py:13 in <module> │
│ │
│ 10 model_path = "stabilityai/stable-diffusion-xl-base-1.0" │
│ 11 pipe = StableDiffusionXLControlNetPipeline.from_pretrained(model_path, torch_dtype=torch │
│ 12 pipe.to("cuda") │
│ ❱ 13 pipe.load_lora_weights("nerijs/pixel-art-xl", weight_name="pixel-art-xl.safetensors") │
│ 14 │
│ │
│ /home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/diffusers/loaders.py:928 in │
│ load_lora_weights │
│ │
│ 925 │ │ │ │ See [`~loaders.LoraLoaderMixin.lora_state_dict`]. │
│ 926 │ │ """ │
│ 927 │ │ state_dict, network_alphas = self.lora_state_dict(pretrained_model_name_or_path_ │
│ ❱ 928 │ │ self.load_lora_into_unet(state_dict, network_alphas=network_alphas, unet=self.un │
│ 929 │ │ self.load_lora_into_text_encoder( │
│ 930 │ │ │ state_dict, │
│ 931 │ │ │ network_alphas=network_alphas, │
│ │
│ /home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/diffusers/loaders.py:1210 in │
│ load_lora_into_unet │
│ │
│ 1207 │ │ │ warnings.warn(warn_message) │
│ 1208 │ │ │
│ 1209 │ │ # load loras into unet │
│ ❱ 1210 │ │ unet.load_attn_procs(state_dict, network_alphas=network_alphas) │
│ 1211 │ │
│ 1212 │ @classmethod │
│ 1213 │ def load_lora_into_text_encoder(cls, state_dict, network_alphas, text_encoder, prefi │
│ │
│ /home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/diffusers/loaders.py:366 in │
│ load_attn_procs │
│ │
│ 363 │ │ │ for key, value_dict in lora_grouped_dict.items(): │
│ 364 │ │ │ │ attn_processor = self │
│ 365 │ │ │ │ for sub_key in key.split("."): │
│ ❱ 366 │ │ │ │ │ attn_processor = getattr(attn_processor, sub_key) │
│ 367 │ │ │ │ │
│ 368 │ │ │ │ # Process non-attention layers, which don't have to_{k,v,q,out_proj}_lor │
│ 369 │ │ │ │ # or add_{k,v,q,out_proj}_proj_lora layers. │
│ │
│ /home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/torch/nn/modules/module.py:1614 │
│ in __getattr__ │
│ │
│ 1611 │ │ │ modules = self.__dict__['_modules'] │
│ 1612 │ │ │ if name in modules: │
│ 1613 │ │ │ │ return modules[name] │
│ ❱ 1614 │ │ raise AttributeError("'{}' object has no attribute '{}'".format( │
│ 1615 │ │ │ type(self).__name__, name)) │
│ 1616 │ │
│ 1617 │ def __setattr__(self, name: str, value: Union[Tensor, 'Module']) -> None: │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
AttributeError: 'ModuleList' object has no attribute '4'System Info
diffusersversion: 0.20.0- Platform: Linux-5.19.0-50-generic-x86_64-with-glibc2.35
- Python version: 3.9.15
- PyTorch version (GPU?): 2.0.1+cu118 (True)
- Huggingface_hub version: 0.15.1
- Transformers version: 4.29.2
- Accelerate version: 0.19.0
- xFormers version: not installed
- Using GPU in script?: Yes
- Using distributed or parallel set-up in script?: NO
Who can help?
No response