unsupported operand type(s) for *: 'dict' and 'Tensor'

[DoosooChang]

When I ran InstantStyle in diffusers with,
scale = {
"up": {"block_0": [0.0, 1.0, 0.0]},
}
, i got the error in 'pipeline' as follows:

Traceback (most recent call last):
File "/media/user-pc16/DATA/Projects/mediaTF-illustration/src/InstantStyle/test.py", line 151, in
test_with_diffusers()
File "/media/user-pc16/DATA/Projects/mediaTF-illustration/src/InstantStyle/test.py", line 137, in test_with_diffusers
image = pipe(
File "/media/user-pc16/DATA/Projects/mediaTF-illustration/media-vtf/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
File "/media/user-pc16/DATA/Projects/mediaTF-illustration/media-vtf/lib/python3.10/site-packages/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py", line 1174, in call
noise_pred = self.unet(
File "/media/user-pc16/DATA/Projects/mediaTF-illustration/media-vtf/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/media/user-pc16/DATA/Projects/mediaTF-illustration/media-vtf/lib/python3.10/site-packages/diffusers/models/unets/unet_2d_condition.py", line 1216, in forward
sample, res_samples = downsample_block(
File "/media/user-pc16/DATA/Projects/mediaTF-illustration/media-vtf/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/media/user-pc16/DATA/Projects/mediaTF-illustration/media-vtf/lib/python3.10/site-packages/diffusers/models/unets/unet_2d_blocks.py", line 1279, in forward
hidden_states = attn(
File "/media/user-pc16/DATA/Projects/mediaTF-illustration/media-vtf/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/media/user-pc16/DATA/Projects/mediaTF-illustration/media-vtf/lib/python3.10/site-packages/diffusers/models/transformers/transformer_2d.py", line 397, in forward
hidden_states = block(
File "/media/user-pc16/DATA/Projects/mediaTF-illustration/media-vtf/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/media/user-pc16/DATA/Projects/mediaTF-illustration/media-vtf/lib/python3.10/site-packages/diffusers/models/attention.py", line 366, in forward
attn_output = self.attn2(
File "/media/user-pc16/DATA/Projects/mediaTF-illustration/media-vtf/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/media/user-pc16/DATA/Projects/mediaTF-illustration/media-vtf/lib/python3.10/site-packages/diffusers/models/attention_processor.py", line 522, in forward
return self.processor(
File "/media/user-pc16/DATA/Projects/mediaTF-illustration/media-vtf/lib/python3.10/site-packages/diffusers/models/attention_processor.py", line 2417, in call
hidden_states = hidden_states + scale * current_ip_hidden_states
TypeError: unsupported operand type(s) for *: 'dict' and 'Tensor'

---

here is my code:

def test_with_diffusers():

image_encoder = CLIPVisionModelWithProjection.from_pretrained(
    "h94/IP-Adapter", subfolder="models/image_encoder", torch_dtype=torch.float16
).to("cuda")

pipe = StableDiffusionXLPipeline.from_pretrained(
    "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16, image_encoder=image_encoder, variant="fp16",
).to("cuda")


pipe.load_ip_adapter(
    "h94/IP-Adapter",
    subfolder="sdxl_models",
    weight_name="ip-adapter_sdxl_vit-h.safetensors",
    image_encoder_folder=None,
    # target_blocks=["up_blocks.0.attentions.1"],
)
pipe.enable_vae_tiling()

# for style blocks only
scale = {
"up": {"block_0": [1.0, 1.0, 0.0]},
}
pipe.set_ip_adapter_scale(scale)


path = "test.png"
ref_img = Image.open(path)



generator = torch.Generator(device="cpu").manual_seed(123456789)
image = pipe(
    prompt="old town with many cars, masterpiece, best quality, high quality",
    negative_prompt= "text, watermark, lowres, low quality, worst quality, deformed, glitch, low contrast, noisy, saturation, blurry",
    ip_adapter_image=ref_img,
    guidance_scale=5,
    num_inference_steps=30,
    generator=generator,
    width=1152,
    height=768,
).images[0]
image.save("temp.png")

---

the code is based on the example in InstantStyle github.
transformers 4.35.0
diffusers 0.27.2
torch 2.0.1

[haofanwang]

Please install diffusers from source.