T2I-Adapter with multiple adapters is broken

[vladmandic]

Describe the bug

pipelines StableDiffusionAdapterPipeline and StableDiffusionXLAdapterPipeline produce error when running with more than one T2IAdapter.
this can be a list of adapters or via using MultiAdapter

this impacts for SD15 and SDXL variants and its not specific to exact adapter model.

Reproduction

import torch
import diffusers
from PIL import Image
from rich import print

model_id = "runwayml/stable-diffusion-v1-5"
print(f'torch=={torch.__version__} diffusers=={diffusers.__version__}')

print(f'loading: {model_id}')
base = diffusers.StableDiffusionPipeline.from_pretrained(model_id, variant="fp16", cache_dir='/mnt/d/Models/Diffusers').to('cuda')
print('loaded')

txt2img = diffusers.AutoPipelineForText2Image.from_pipe(base)
output = txt2img(prompt='test', negative_prompt='test', num_inference_steps=10) # ok
print(f'txt2img: {output}')

img2img = diffusers.AutoPipelineForImage2Image.from_pipe(base)
image = Image.new('RGB', (512,512), 0) # input is irrelevant, so just creating blank image
output = img2img(prompt='test', negative_prompt='test', num_inference_steps=10, image=image) # ok
print(f'img2img: {output}')

adapter1 = diffusers.T2IAdapter.from_pretrained('TencentARC/t2iadapter_depth_sd15v2', cache_dir='/mnt/d/Models/Diffusers')
pipe = diffusers.StableDiffusionAdapterPipeline(
    vae=base.vae,
    text_encoder=base.text_encoder,
    tokenizer=base.tokenizer,
    unet=base.unet,
    scheduler=base.scheduler,
    requires_safety_checker=False,
    safety_checker=None,
    feature_extractor=None,
    adapter=adapter1,
).to('cuda')
output = pipe(prompt='test', negative_prompt='test', num_inference_steps=10, image=image) # ok
print(f'adapter: {output}')

adapter2 = diffusers.T2IAdapter.from_pretrained('TencentARC/t2iadapter_zoedepth_sd15v1', cache_dir='/mnt/d/Models/Diffusers')
pipe = diffusers.StableDiffusionAdapterPipeline(
    vae=base.vae,
    text_encoder=base.text_encoder,
    tokenizer=base.tokenizer,
    unet=base.unet,
    scheduler=base.scheduler,
    requires_safety_checker=False,
    safety_checker=None,
    feature_extractor=None,
    adapter=[adapter1, adapter2],
).to('cuda')
output = pipe(prompt='test', negative_prompt='test', num_inference_steps=10, image=[image, image]) # fails
print(f'adapter-list: {output}')

pipe = diffusers.StableDiffusionAdapterPipeline(
    vae=base.vae,
    text_encoder=base.text_encoder,
    tokenizer=base.tokenizer,
    unet=base.unet,
    scheduler=base.scheduler,
    requires_safety_checker=False,
    safety_checker=None,
    feature_extractor=None,
    adapter=diffusers.MultiAdapter([adapter1, adapter2])
).to('cuda')
output = pipe(prompt='test', negative_prompt='test', num_inference_steps=10, image=[image, image]) # also fails
print(f'multiadapter: {output}')

Logs

File "diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py", line 884, in __call__
    adapter_state = self.adapter(adapter_input, adapter_conditioning_scale)
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "diffusers/models/adapter.py", line 92, in forward
    for x, w, adapter in zip(xs, adapter_weights, self.adapters):
                         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "torch/_tensor.py", line 990, in __iter__
    raise TypeError("iteration over a 0-d tensor")

System Info

torch==2.1.2+cu121
diffusers==0.25.0.dev0

Who can help?

@sayakpaul @yiyixuxu @DN6 @patrickvonplaten

[sayakpaul]

Cc: @williamberman

[sayakpaul]

I was able to run the code from https://huggingface.co/docs/diffusers/main/en/api/pipelines/stable_diffusion/adapter#combining-multiple-adapters without any errors. I am on diffusers main.

from diffusers.utils import load_image, make_image_grid

cond_keypose = load_image(
    "https://huggingface.co/datasets/diffusers/docs-images/resolve/main/t2i-adapter/keypose_sample_input.png"
)
cond_depth = load_image(
    "https://huggingface.co/datasets/diffusers/docs-images/resolve/main/t2i-adapter/depth_sample_input.png"
)
cond = [cond_keypose, cond_depth]

prompt = ["A man walking in an office room with a nice view"]

Then:

import torch
from diffusers import StableDiffusionAdapterPipeline, MultiAdapter, T2IAdapter

adapters = MultiAdapter(
    [
        T2IAdapter.from_pretrained("TencentARC/t2iadapter_keypose_sd14v1"),
        T2IAdapter.from_pretrained("TencentARC/t2iadapter_depth_sd14v1"),
    ]
)
adapters = adapters.to(torch.float16)

pipe = StableDiffusionAdapterPipeline.from_pretrained(
    "CompVis/stable-diffusion-v1-4",
    torch_dtype=torch.float16,
    adapter=adapters,
).to("cuda")

image = pipe(prompt, cond, adapter_conditioning_scale=[0.8, 0.8]).images[0]
make_image_grid([cond_keypose, cond_depth, image], rows=1, cols=3)

[sayakpaul]

I am able to do:

from PIL import Image

adapter = [
    T2IAdapter.from_pretrained("TencentARC/t2iadapter_keypose_sd14v1"),
    T2IAdapter.from_pretrained("TencentARC/t2iadapter_zoedepth_sd15v1")
]

pipe = StableDiffusionAdapterPipeline.from_pretrained(
    "CompVis/stable-diffusion-v1-4",
    torch_dtype=torch.float16,
    adapter=adapters,
).to("cuda")

cond = [cond[0], Image.new('RGB', (512,512), 0)]

image = pipe(prompt, cond, adapter_conditioning_scale=[0.8, 0.8]).images[0]

as well.

What am I missing?

[vladmandic]

this one was my fault - recreating pipeline should restore feature_extractor
this fixes it:

feature_extractor=getattr(base, 'feature_extractor', None)

closing this issue.