Add support for more WAN Loras

[asomoza]

There are more loras for WAN now, some really popular ones, but as right now they don't work with diffusers.

T2V Example with this lora

import torch

from diffusers import WanPipeline
from diffusers.utils import export_to_video


model_id = "Wan-AI/Wan2.1-T2V-14B-Diffusers"

pipe = WanPipeline.from_pretrained(model_id, torch_dtype=torch.bfloat16)
pipe.to("cuda:2")

pipe.load_lora_weights("studio_ghibli_wan14b_t2v_v01.safetensors")

prompt = "Studio Ghibli style. A girl with a playful smile leans forward on the ferry railing, holding onto her sunhat as the wind threatens to carry it away. She looks directly at the camera with bright, excited eyes, her laughter almost visible in her expression. The sun reflects off the sparkling water, and behind her, the coastline of a charming island town approaches. The camera slowly zooms out, capturing the anticipation of arrival."
negative_prompt = "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards"
num_frames = 33

frames = pipe(prompt=prompt, negative_prompt=negative_prompt, num_frames=num_frames).frames[0]
export_to_video(frames, "wan-t2v_720.mp4", fps=16)

I2V Example with this lora:

import numpy as np
import torch
from transformers import CLIPVisionModel

from diffusers import AutoencoderKLWan, WanImageToVideoPipeline
from diffusers.utils import export_to_video, load_image


model_id = "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers"
image_encoder = CLIPVisionModel.from_pretrained(
    model_id, subfolder="image_encoder", torch_dtype=torch.float32
)
vae = AutoencoderKLWan.from_pretrained(model_id, subfolder="vae", torch_dtype=torch.float32)
pipe = WanImageToVideoPipeline.from_pretrained(
    model_id, vae=vae, image_encoder=image_encoder, torch_dtype=torch.bfloat16
)
pipe.load_lora_weights("baozhatou.safetensors")
pipe.to("cuda:2")

image = load_image(
    "https://huggingface.co/datasets/OzzyGT/testing-resources/resolve/main/wan_i2v/charlesdeluvio-K4mSJ7kc0As-unsplash.jpg"
)
max_area = 480 * 832
aspect_ratio = image.height / image.width
mod_value = pipe.vae_scale_factor_spatial * pipe.transformer.config.patch_size[1]
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
image = image.resize((width, height))
prompt = (
    "baozhatou, A dog's hair swells and turns into a curly hair, creating a baozhatou effect while remaining still"
)
negative_prompt = ""

output = pipe(
    image=image,
    prompt=prompt,
    negative_prompt=negative_prompt,
    height=height,
    width=width,
    num_frames=81,
    guidance_scale=5.0,
).frames[0]
export_to_video(output, "output.mp4", fps=16)

Output:

No LoRA keys associated to WanTransformer3DModel found with the prefix='transformer'. This is safe to ignore if LoRA state dict didn't originally have any WanTransformer3DModel related params.

The inference works but there's no effect of the lora in the video generated.

@sayakpaul