feat: AnimateDiff SD 1.5 motion modules (v2 + v3)

CMakeLists.txt

@@ -312,7 +312,7 @@ set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
 
 if (NOT SD_USE_SYSTEM_GGML)
     # see https://github.com/ggerganov/ggml/pull/682
-    add_definitions(-DGGML_MAX_NAME=128)
+    add_definitions(-DGGML_MAX_NAME=160)
 endif()
 
 # deps

docs/animatediff.md

@@ -0,0 +1,152 @@
+# AnimateDiff (SD 1.5)
+
+AnimateDiff adds motion to a frozen Stable Diffusion 1.5 checkpoint by
+injecting a temporal-attention module at 20 UNet slots. The base SD 1.5
+model, VAE, and text encoder are unchanged; only the motion module produces
+the temporal residual that turns a batch of independent frames into a
+coherent animation. Reference: Guo et al., "AnimateDiff: Animate Your
+Personalized Text-to-Image Diffusion Models without Specific Tuning"
+(https://arxiv.org/abs/2307.04725).
+
+## Download weights
+
+- Motion module (v3, recommended)
+    - fp16 safetensors: https://huggingface.co/conrevo/AnimateDiff-A1111/resolve/main/motion_module/mm_sd15_v3.safetensors
+    - original checkpoint: https://huggingface.co/guoyww/animatediff/resolve/main/v3_sd15_mm.ckpt
+- SD 1.5 base model
+    - any SD 1.5 checkpoint works. `realisticVisionV60B1` and `toonyou_beta3`
+      are the ones used in guoyww's reference configs.
+- Domain Adapter LoRA (optional, v3 only, sharpens the base UNet's output
+  toward the motion module's trained distribution)
+    - ckpt: https://huggingface.co/guoyww/animatediff/resolve/main/v3_sd15_adapter.ckpt
+    - place under your `--lora-model-dir` and reference in the prompt as
+      `<lora:v3_sd15_adapter:1.0>`.
+
+The motion module is `~836 MB` and loads alongside the SD 1.5 UNet via
+`--motion-module`.
+
+## Motion module versions
+
+Per [animatediff.net/models](https://animatediff.net/models):
+
+| Module              | Base | Native res | Character |
+|---------------------|------|------------|-----------|
+| `mm_sd_v14.ckpt`    | 1.5  | 256x256    | earliest, more jittery |
+| `mm_sd_v15.ckpt`    | 1.5  | 256x256    | improved stability over v1.4 |
+| `mm_sd_v15_v2.ckpt` | 1.5  | 384x384    | significantly better motion dynamics |
+| `v3_sd15_mm.ckpt`   | 1.5  | 512x512    | smoothest, highest quality; pairs with a Domain Adapter LoRA |
+| `mm_sdxl_v10_beta`  | SDXL | 512x512    | experimental, not yet supported here |
+
+Match your `-H -W` to the module's native resolution for best results. v3 is
+trained at 512x512 - going smaller (e.g. 384x384) still works but the motion
+character is closer to v2.
+
+## Examples
+
+Generate an 8-frame animation at 512x512, seed 42, 20 steps. The sampler /
+scheduler / CFG values below match what mm_sd15_v3 was trained with; using
+SD 1.5 defaults (euler_a, low CFG) produces noise-like output.
+
+```
+.\bin\Release\sd-cli.exe -M vid_gen \
+    --model ..\models\checkpoints\realisticVisionV60B1.safetensors \
+    --motion-module ..\models\animatediff\mm_sd15_v3.safetensors \
+    --offload-to-cpu --diffusion-fa \
+    -p "a red apple on a wooden table" \
+    --cfg-scale 8.0 --sampling-method euler --scheduler discrete \
+    -H 512 -W 512 --video-frames 8 --fps 8 --steps 20 -s 42 \
+    -o out.avi
+```
+
+Generate at the motion module's native 16-frame context (recommended for
+best temporal quality). Needs more VRAM at 512x512, so drop to 384x384 or
+use layer streaming:
+
+```
+.\bin\Release\sd-cli.exe -M vid_gen \
+    --model ..\models\checkpoints\realisticVisionV60B1.safetensors \
+    --motion-module ..\models\animatediff\mm_sd15_v3.safetensors \
+    --offload-to-cpu --diffusion-fa \
+    -p "photo of coastline, rocks, storm weather, wind, waves, lightning" \
+    --cfg-scale 8.0 --sampling-method euler --scheduler discrete \
+    -H 384 -W 384 --video-frames 16 --fps 8 --steps 20 -s 42 \
+    -o out.avi
+```
+
+Low-VRAM streaming (verified with a 2 GiB cap on RTX 3060):
+
+```
+.\bin\Release\sd-cli.exe -M vid_gen \
+    --model ..\models\checkpoints\realisticVisionV60B1.safetensors \
+    --motion-module ..\models\animatediff\mm_sd15_v3.safetensors \
+    --max-vram 2.0 --stream-layers --diffusion-fa \
+    -p "photo of coastline, rocks, storm weather, wind, waves, lightning" \
+    --cfg-scale 8.0 --sampling-method euler --scheduler discrete \
+    -H 384 -W 384 --video-frames 8 --fps 8 --steps 20 -s 42 \
+    -o out.avi
+```
+
+## Reference-quality reproduction
+
+Using guoyww's official reference configs on this impl (RealisticVision v6.0
+base + `mm_sd15_v3` or `mm_sd_v15_v2` + native resolution + 16 frames + euler
++ 25 steps + CFG 8 + linear beta schedule) reproduces the reference
+AnimateDiff output style.
+
+### v3 (512x512, `mm_sd15_v3`)
+
+| Prompt                                | Sample |
+|---------------------------------------|--------|
+| B&W man on stormy coastline           | <img src="../assets/animatediff/v3_man.gif" width="256"/>    |
+| Close-up rabbit macro shot            | <img src="../assets/animatediff/v3_rabbit.gif" width="256"/> |
+| Coastline, storm, waves, lightning    | <img src="../assets/animatediff/v3_coast.gif" width="256"/>  |
+| Old house, storm, forest, night       | <img src="../assets/animatediff/v3_house.gif" width="256"/>  |
+
+### v2 (384x384, `mm_sd_v15_v2.ckpt`)
+
+| Prompt                                | Sample |
+|---------------------------------------|--------|
+| B&W man on stormy coastline           | <img src="../assets/animatediff/v2_man.gif" width="256"/>    |
+| Close-up rabbit macro shot            | <img src="../assets/animatediff/v2_rabbit.gif" width="256"/> |
+| Coastline, storm, waves, lightning    | <img src="../assets/animatediff/v2_coast.gif" width="256"/>  |
+| Old house, storm, forest, night       | <img src="../assets/animatediff/v2_house.gif" width="256"/>  |
+
+Motion is strong for scenes with motion cues in the prompt (storm/waves/wind)
+and subtle for static subjects (close-up macro), matching reference behavior.
+v2 has an additional motion module at the UNet middle block that v3 dropped;
+this impl auto-detects the topology from the checkpoint.
+
+### v3 + Domain Adapter LoRA
+
+Attaching the `v3_sd15_adapter` LoRA sharpens the base UNet output toward
+the training distribution the motion module was fine-tuned against. Same
+prompt, seed, config as above:
+
+<img src="../assets/animatediff/v3_rabbit_domain_lora.gif" width="256"/>
+
+Individual fur strands, glowing inner-ear, and richer forest detail become
+visible compared to the no-LoRA rendering.
+
+```
+sd-cli -M vid_gen --model realisticVisionV60B1.safetensors \
+       --motion-module mm_sd15_v3.safetensors \
+       --lora-model-dir ./loras \
+       -p "close up photo of a rabbit ...<lora:v3_sd15_adapter:1.0>" ...
+```
+
+## Notes
+
+- The motion module was trained at `video_length=16`. Running with
+  `--video-frames 16` gives the best coherence; F=8 works but shows a shorter
+  motion arc. Frame counts up to 32 are supported by the positional encoding
+  but exceed the trained regime and produce more static output.
+- At `--video-frames 1` the motion module is skipped entirely and the output
+  is bit-identical to `-M img_gen`. This avoids the single-token
+  temporal-attention degeneracy that would otherwise emit an untrained-magnitude
+  residual on a single-frame sample.
+- The base UNet is frozen, so character identity and style follow the SD 1.5
+  checkpoint you pass to `--model`. LoRAs and prompt weighting attach to the
+  base model in the usual way.
+- No mid_block motion module in v3. `mm_sdxl_v10_beta` (SDXL variant) is not
+  supported yet.
+- Output is written as MJPEG AVI. Use `--fps` to set playback speed.

examples/common/common.cpp

@@ -423,6 +423,11 @@ ArgOptions SDContextParams::get_options() {
          "path to control net model",
          0,
          &control_net_path},
+        {"",
+         "--motion-module",
+         "path to AnimateDiff motion module (SD 1.5); enables video generation on --video-frames > 1",
+         0,
+         &motion_module_path},
         {"",
          "--embd-dir",
          "embeddings directory",
@@ -672,7 +677,7 @@ ArgOptions SDContextParams::get_options() {
 }
 
 void SDContextParams::build_embedding_map() {
-    static const std::vector<std::string> valid_ext = {".gguf", ".safetensors", ".pt"};
+    static const std::vector<std::string> valid_ext = {".gguf", ".safetensors", ".pt", ".ckpt"};
 
     if (!fs::exists(embedding_dir) || !fs::is_directory(embedding_dir)) {
         return;
@@ -867,6 +872,7 @@ sd_ctx_params_t SDContextParams::to_sd_ctx_params_t(bool taesd_preview) {
     sd_ctx_params.audio_vae_path                  = audio_vae_path.c_str();
     sd_ctx_params.taesd_path                      = taesd_path.c_str();
     sd_ctx_params.control_net_path                = control_net_path.c_str();
+    sd_ctx_params.motion_module_path              = motion_module_path.c_str();
     sd_ctx_params.embeddings                      = embedding_vec.data();
     sd_ctx_params.embedding_count                 = static_cast<uint32_t>(embedding_vec.size());
     sd_ctx_params.photo_maker_path                = photo_maker_path.c_str();
@@ -2032,7 +2038,7 @@ void SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_d
         return;
     }
     static const std::regex re(R"(<lora:([^:>]+):([^>]+)>)");
-    static const std::vector<std::string> valid_ext = {".gguf", ".safetensors", ".pt"};
+    static const std::vector<std::string> valid_ext = {".gguf", ".safetensors", ".pt", ".ckpt"};
     std::smatch m;
 
     std::string tmp = prompt;

examples/common/common.h

@@ -132,6 +132,7 @@ struct SDContextParams {
     std::string taesd_path;
     std::string esrgan_path;
     std::string control_net_path;
+    std::string motion_module_path;
     std::string embedding_dir;
     std::string photo_maker_path;
     std::string pulid_weights_path;

include/stable-diffusion.h

@@ -199,6 +199,7 @@ typedef struct {
     const char* audio_vae_path;
     const char* taesd_path;
     const char* control_net_path;
+    const char* motion_module_path;
     const sd_embedding_t* embeddings;
     uint32_t embedding_count;
     const char* photo_maker_path;

src/model/diffusion/animatediff.hpp

@@ -0,0 +1,182 @@
+#ifndef __SD_MODEL_DIFFUSION_ANIMATEDIFF_HPP__
+#define __SD_MODEL_DIFFUSION_ANIMATEDIFF_HPP__
+
+#include "core/ggml_extend.hpp"
+#include "model/common/block.hpp"
+
+// AnimateDiff (https://arxiv.org/abs/2307.04725) SD 1.5 motion modules.
+namespace AnimateDiff {
+
+    struct MotionModuleConfig {
+        int max_frames                     = 32;
+        int64_t num_heads                  = 8;
+        int norm_num_groups                = 32;
+        std::vector<int64_t> down_channels = {320, 640, 1280, 1280};
+        std::vector<int64_t> up_channels   = {1280, 1280, 640, 320};
+        int num_down_motion_per_block      = 2;
+        int num_up_motion_per_block        = 3;
+        bool enable_mid_block              = false;
+        int64_t mid_channels               = 1280;
+    };
+
+    class TemporalAttention : public GGMLBlock {
+    protected:
+        int64_t channels;
+        int64_t num_heads;
+        int max_frames;
+
+        void init_params(ggml_context* ctx,
+                         const String2TensorStorage& tensor_storage_map = {},
+                         const std::string prefix                       = "") override {
+            params["pos_encoder.pe"] = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, channels, max_frames, 1);
+        }
+
+    public:
+        TemporalAttention(int64_t channels, int64_t num_heads, int max_frames)
+            : channels(channels), num_heads(num_heads), max_frames(max_frames) {
+            blocks["to_q"]     = std::shared_ptr<GGMLBlock>(new Linear(channels, channels, false));
+            blocks["to_k"]     = std::shared_ptr<GGMLBlock>(new Linear(channels, channels, false));
+            blocks["to_v"]     = std::shared_ptr<GGMLBlock>(new Linear(channels, channels, false));
+            blocks["to_out.0"] = std::shared_ptr<GGMLBlock>(new Linear(channels, channels, true));
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            auto to_q   = std::dynamic_pointer_cast<Linear>(blocks["to_q"]);
+            auto to_k   = std::dynamic_pointer_cast<Linear>(blocks["to_k"]);
+            auto to_v   = std::dynamic_pointer_cast<Linear>(blocks["to_v"]);
+            auto to_out = std::dynamic_pointer_cast<Linear>(blocks["to_out.0"]);
+
+            int64_t C = x->ne[0];
+            int64_t F = x->ne[1];
+
+            auto pe   = params["pos_encoder.pe"];
+            auto pe_f = (F == pe->ne[1])
+                            ? pe
+                            : ggml_view_3d(ctx->ggml_ctx, pe, C, F, 1, pe->nb[1], pe->nb[2], 0);
+            auto x_pe = ggml_add(ctx->ggml_ctx, x, ggml_repeat(ctx->ggml_ctx, pe_f, x));
+
+            auto q = to_q->forward(ctx, x_pe);
+            auto k = to_k->forward(ctx, x_pe);
+            auto v = to_v->forward(ctx, x_pe);
+
+            auto a = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, (int)num_heads, nullptr, false);
+            return to_out->forward(ctx, a);
+        }
+    };
+
+    class TemporalTransformerBlock : public GGMLBlock {
+    public:
+        TemporalTransformerBlock(int64_t channels, int64_t num_heads, int max_frames) {
+            blocks["attention_blocks.0"] = std::make_shared<TemporalAttention>(channels, num_heads, max_frames);
+            blocks["attention_blocks.1"] = std::make_shared<TemporalAttention>(channels, num_heads, max_frames);
+            blocks["norms.0"]            = std::shared_ptr<GGMLBlock>(new LayerNorm(channels));
+            blocks["norms.1"]            = std::shared_ptr<GGMLBlock>(new LayerNorm(channels));
+            blocks["ff"]                 = std::make_shared<FeedForward>(channels, channels, 4, FeedForward::Activation::GEGLU);
+            blocks["ff_norm"]            = std::shared_ptr<GGMLBlock>(new LayerNorm(channels));
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            auto attn0   = std::dynamic_pointer_cast<TemporalAttention>(blocks["attention_blocks.0"]);
+            auto attn1   = std::dynamic_pointer_cast<TemporalAttention>(blocks["attention_blocks.1"]);
+            auto norm0   = std::dynamic_pointer_cast<LayerNorm>(blocks["norms.0"]);
+            auto norm1   = std::dynamic_pointer_cast<LayerNorm>(blocks["norms.1"]);
+            auto ff      = std::dynamic_pointer_cast<FeedForward>(blocks["ff"]);
+            auto ff_norm = std::dynamic_pointer_cast<LayerNorm>(blocks["ff_norm"]);
+
+            auto r = x;
+            x      = ggml_add(ctx->ggml_ctx, attn0->forward(ctx, norm0->forward(ctx, x)), r);
+
+            r = x;
+            x = ggml_add(ctx->ggml_ctx, attn1->forward(ctx, norm1->forward(ctx, x)), r);
+
+            r = x;
+            x = ggml_add(ctx->ggml_ctx, ff->forward(ctx, ff_norm->forward(ctx, x)), r);
+
+            return x;
+        }
+    };
+
+    class TemporalTransformer : public GGMLBlock {
+    public:
+        TemporalTransformer(int64_t channels, int64_t num_heads, int norm_num_groups, int max_frames) {
+            blocks["norm"]                 = std::shared_ptr<GGMLBlock>(new GroupNorm(norm_num_groups, channels));
+            blocks["proj_in"]              = std::shared_ptr<GGMLBlock>(new Linear(channels, channels, true));
+            blocks["transformer_blocks.0"] = std::make_shared<TemporalTransformerBlock>(channels, num_heads, max_frames);
+            blocks["proj_out"]             = std::shared_ptr<GGMLBlock>(new Linear(channels, channels, true));
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, int64_t num_frames) {
+            auto norm     = std::dynamic_pointer_cast<GroupNorm>(blocks["norm"]);
+            auto proj_in  = std::dynamic_pointer_cast<Linear>(blocks["proj_in"]);
+            auto tb0      = std::dynamic_pointer_cast<TemporalTransformerBlock>(blocks["transformer_blocks.0"]);
+            auto proj_out = std::dynamic_pointer_cast<Linear>(blocks["proj_out"]);
+
+            int64_t W = x->ne[0];
+            int64_t H = x->ne[1];
+            int64_t C = x->ne[2];
+            GGML_ASSERT(x->ne[3] == num_frames);
+
+            auto residual = x;
+            auto h        = norm->forward(ctx, x);
+
+            h = ggml_ext_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, h, 2, 3, 0, 1));
+            h = ggml_reshape_3d(ctx->ggml_ctx, h, C, num_frames, W * H);
+            h = proj_in->forward(ctx, h);
+            h = tb0->forward(ctx, h);
+            h = proj_out->forward(ctx, h);
+            h = ggml_reshape_4d(ctx->ggml_ctx, h, C, num_frames, W, H);
+            h = ggml_ext_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, h, 2, 3, 0, 1));
+
+            return ggml_add(ctx->ggml_ctx, h, residual);
+        }
+    };
+
+    class MotionModule : public GGMLBlock {
+    public:
+        MotionModule(int64_t channels, int64_t num_heads, int norm_num_groups, int max_frames) {
+            blocks["temporal_transformer"] = std::make_shared<TemporalTransformer>(channels, num_heads, norm_num_groups, max_frames);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, int64_t num_frames) {
+            auto tt = std::dynamic_pointer_cast<TemporalTransformer>(blocks["temporal_transformer"]);
+            return tt->forward(ctx, x, num_frames);
+        }
+    };
+
+    class AnimateDiffModel : public GGMLBlock {
+    public:
+        MotionModuleConfig config;
+
+        AnimateDiffModel(const MotionModuleConfig& cfg)
+            : config(cfg) {
+            for (int i = 0; i < static_cast<int>(cfg.down_channels.size()); ++i) {
+                int64_t ch = cfg.down_channels[i];
+                for (int j = 0; j < cfg.num_down_motion_per_block; ++j) {
+                    blocks["down_blocks." + std::to_string(i) + ".motion_modules." + std::to_string(j)] =
+                        std::make_shared<MotionModule>(ch, cfg.num_heads, cfg.norm_num_groups, cfg.max_frames);
+                }
+            }
+            for (int i = 0; i < static_cast<int>(cfg.up_channels.size()); ++i) {
+                int64_t ch = cfg.up_channels[i];
+                for (int j = 0; j < cfg.num_up_motion_per_block; ++j) {
+                    blocks["up_blocks." + std::to_string(i) + ".motion_modules." + std::to_string(j)] =
+                        std::make_shared<MotionModule>(ch, cfg.num_heads, cfg.norm_num_groups, cfg.max_frames);
+                }
+            }
+            if (cfg.enable_mid_block) {
+                blocks["mid_block.motion_modules.0"] =
+                    std::make_shared<MotionModule>(cfg.mid_channels, cfg.num_heads, cfg.norm_num_groups, cfg.max_frames);
+            }
+        }
+
+        std::shared_ptr<MotionModule> motion(const std::string& key) {
+            auto it = blocks.find(key);
+            if (it == blocks.end())
+                return nullptr;
+            return std::dynamic_pointer_cast<MotionModule>(it->second);
+        }
+    };
+
+}  // namespace AnimateDiff
+
+#endif  // __SD_MODEL_DIFFUSION_ANIMATEDIFF_HPP__