Updated stablediffusion.cpp to 36ec16a

DarthAffe · Jan 29, 2024 · dd4c862 · dd4c862
1 parent 97f44b8
commit dd4c862
Show file tree

Hide file tree

Showing 7 changed files with 70 additions and 15 deletions.
diff --git a/README.md b/README.md
@@ -2,6 +2,9 @@
 
 Based on https://github.com/leejet/stable-diffusion.cpp
 
+> At least for me the current version of stable-diffusion.cpp has really bad tiling issues.   
+If you experience them too, I'd recommend using https://github.com/DarthAffe/StableDiffusion.NET/releases/tag/c6071fa until that's fixed.
+
 ## Usage
 ### Setup
 Run `build.bat` to build the native libs (modify params like CUDA-builds if needed)

diff --git a/StableDiffusion.NET.sln.DotSettings b/StableDiffusion.NET.sln.DotSettings
@@ -0,0 +1,2 @@
+<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=CPU/@EntryIndexedValue">CPU</s:String></wpf:ResourceDictionary>
diff --git a/StableDiffusion.NET/ModelParameter.cs b/StableDiffusion.NET/ModelParameter.cs
@@ -11,6 +11,9 @@ public class ModelParameter
     public string LoraModelDir { get; set; } = string.Empty;
     public RngType RngType { get; set; } = RngType.Standard;
     public string VaePath { get; set; } = string.Empty;
+    public string ControlNetPath { get; set; } = string.Empty;
+    public string EmbeddingsDirectory { get; set; } = string.Empty;
+    public bool KeepControlNetOnCPU { get; set; } = false;
 
     //TODO DarthAffe 01.01.2024: K-Quants doesn't seem to work so far
     public Quantization Quantization { get; set; } = Quantization.F16;

diff --git a/StableDiffusion.NET/Native.cs b/StableDiffusion.NET/Native.cs
@@ -51,14 +51,17 @@ internal struct sd_image_t
     internal static partial sd_ctx_t* new_sd_ctx([MarshalAs(UnmanagedType.LPStr)] string model_path,
                                                  [MarshalAs(UnmanagedType.LPStr)] string vae_path,
                                                  [MarshalAs(UnmanagedType.LPStr)] string taesd_path,
+                                                 [MarshalAs(UnmanagedType.LPStr)] string control_net_path_c_str,
                                                  [MarshalAs(UnmanagedType.LPStr)] string lora_model_dir,
+                                                 [MarshalAs(UnmanagedType.LPStr)] string embed_dir_c_str,
                                                  [MarshalAs(UnmanagedType.I1)] bool vae_decode_only,
                                                  [MarshalAs(UnmanagedType.I1)] bool vae_tiling,
                                                  [MarshalAs(UnmanagedType.I1)] bool free_params_immediately,
                                                  int n_threads,
                                                  sd_type_t wtype,
                                                  rng_type_t rng_type,
-                                                 schedule_t s);
+                                                 schedule_t s,
+                                                 [MarshalAs(UnmanagedType.I1)] bool keep_control_net_cpu);
 
     [LibraryImport(LIB_NAME, EntryPoint = "free_sd_ctx")]
     internal static partial void free_sd_ctx(sd_ctx_t* sd_ctx);
@@ -74,7 +77,9 @@ internal struct sd_image_t
                                                 sample_method_t sample_method,
                                                 int sample_steps,
                                                 long seed,
-                                                int batch_count);
+                                                int batch_count,
+                                                sd_image_t* control_cond,
+                                                float control_strength);
 
     [LibraryImport(LIB_NAME, EntryPoint = "img2img")]
     internal static partial sd_image_t* img2img(sd_ctx_t* sd_ctx,

diff --git a/StableDiffusion.NET/StableDiffusionModel.cs b/StableDiffusion.NET/StableDiffusionModel.cs
@@ -51,14 +51,17 @@ private void Initialize()
         _ctx = Native.new_sd_ctx(_modelPath,
                                  _parameter.VaePath,
                                  _parameter.TaesdPath,
+                                 _parameter.ControlNetPath,
                                  _parameter.LoraModelDir,
+                                 _parameter.EmbeddingsDirectory,
                                  _parameter.VaeDecodeOnly,
                                  _parameter.VaeTiling,
                                   false,
                                  _parameter.ThreadCount,
                                  _parameter.Quantization,
                                  _parameter.RngType,
-                                 _parameter.Schedule);
+                                 _parameter.Schedule,
+                                 _parameter.KeepControlNetOnCPU);
         if (_ctx == null) throw new NullReferenceException("Failed to initialize Stable Diffusion");
 
         if (_upscalerParameter != null)
@@ -74,23 +77,58 @@ public StableDiffusionImage TextToImage(string prompt, StableDiffusionParameter
     {
         ObjectDisposedException.ThrowIf(_disposed, this);
 
-        Native.sd_image_t* result = Native.txt2img(_ctx,
-                                                   prompt,
-                                                   parameter.NegativePrompt,
-                                                   parameter.ClipSkip,
-                                                   parameter.CfgScale,
-                                                   parameter.Width,
-                                                   parameter.Height,
-                                                   parameter.SampleMethod,
-                                                   parameter.SampleSteps,
-                                                   parameter.Seed,
-                                                   1);
+        Native.sd_image_t* result;
+        if ((parameter.ControlNetImage == null) || (parameter.ControlNetImage.Length == 0))
+        {
+            result = Native.txt2img(_ctx,
+                                    prompt,
+                                    parameter.NegativePrompt,
+                                    parameter.ClipSkip,
+                                    parameter.CfgScale,
+                                    parameter.Width,
+                                    parameter.Height,
+                                    parameter.SampleMethod,
+                                    parameter.SampleSteps,
+                                    parameter.Seed,
+                                    1,
+                                    null,
+                                    0);
+        }
+        else
+        {
+            fixed (byte* imagePtr = parameter.ControlNetImage)
+            {
+                Native.sd_image_t controlNetImage = new()
+                {
+                    width = (uint)parameter.Width,
+                    height = (uint)parameter.Height,
+                    channel = 3,
+                    data = imagePtr
+                };
+
+                result = Native.txt2img(_ctx,
+                                        prompt,
+                                        parameter.NegativePrompt,
+                                        parameter.ClipSkip,
+                                        parameter.CfgScale,
+                                        parameter.Width,
+                                        parameter.Height,
+                                        parameter.SampleMethod,
+                                        parameter.SampleSteps,
+                                        parameter.Seed,
+                                        1,
+                                        &controlNetImage,
+                                        parameter.ControlNetStrength);
+            }
+        }
 
         return new StableDiffusionImage(result);
     }
 
     public StableDiffusionImage ImageToImage(string prompt, in ReadOnlySpan<byte> image, StableDiffusionParameter parameter)
     {
+        ObjectDisposedException.ThrowIf(_disposed, this);
+
         fixed (byte* imagePtr = image)
         {
             Native.sd_image_t img = new()
@@ -110,6 +148,8 @@ public StableDiffusionImage ImageToImage(string prompt, StableDiffusionImage ima
 
     private StableDiffusionImage ImageToImage(string prompt, Native.sd_image_t image, StableDiffusionParameter parameter)
     {
+        ObjectDisposedException.ThrowIf(_disposed, this);
+
         Native.sd_image_t* result = Native.img2img(_ctx,
                                                    image,
                                                    prompt,

diff --git a/StableDiffusion.NET/StableDiffusionParameter.cs b/StableDiffusion.NET/StableDiffusionParameter.cs
@@ -13,6 +13,8 @@ public sealed class StableDiffusionParameter
     public long Seed { get; set; } = -1;
     public float Strength { get; set; } = 0.7f;
     public int ClipSkip { get; set; } = -1;
+    public byte[]? ControlNetImage { get; set; } = null;
+    public float ControlNetStrength { get; set; } = 0.9f;
 
     #endregion
 }
diff --git a/build.bat b/build.bat
@@ -4,7 +4,7 @@ if not exist stable-diffusion.cpp (
 
 cd stable-diffusion.cpp
 git fetch
-git checkout c6071fa82fb1d0e688f75c9a3d870fe71d3a7a1d
+git checkout 36ec16ac9937d393d0ba8939640352cee430efb7
 git submodule init
 git submodule update