From eda7db5f933e8fb94cf710c33613b819d53a48af Mon Sep 17 00:00:00 2001 From: mkshing <33302880+mkshing@users.noreply.github.com> Date: Tue, 18 Oct 2022 09:34:17 +0900 Subject: [PATCH 1/3] Support LMSDiscreteScheduler in LDMPipeline This is a small change to support all schedulers such as LMSDiscreteScheduler in LDMPipeline. What's changed ------- * Add the `scale_model_input` function before `step` to ensure correct denoising (L77) --- .../pipeline_latent_diffusion_uncond.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py b/src/diffusers/pipelines/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py index ef82abb7e6cb..3caa64ae3f69 100644 --- a/src/diffusers/pipelines/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py +++ b/src/diffusers/pipelines/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py @@ -74,8 +74,9 @@ def __call__( extra_kwargs["eta"] = eta for t in self.progress_bar(self.scheduler.timesteps): + latent_model_input = self.scheduler.scale_model_input(latents, t) # predict the noise residual - noise_prediction = self.unet(latents, t).sample + noise_prediction = self.unet(latent_model_input, t).sample # compute the previous noisy sample x_t -> x_t-1 latents = self.scheduler.step(noise_prediction, t, latents, **extra_kwargs).prev_sample From dacf922b5b8f5244a3d1b42a6f918faaa9570880 Mon Sep 17 00:00:00 2001 From: mkshing <33302880+mkshing@users.noreply.github.com> Date: Fri, 21 Oct 2022 21:08:55 +0900 Subject: [PATCH 2/3] Add "scale the initial noise by the standard deviation required by the scheduler" --- .../pipeline_latent_diffusion_uncond.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py b/src/diffusers/pipelines/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py index 3caa64ae3f69..331463c813b0 100644 --- a/src/diffusers/pipelines/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py +++ b/src/diffusers/pipelines/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py @@ -63,7 +63,10 @@ def __call__( generator=generator, ) latents = latents.to(self.device) - + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + self.scheduler.set_timesteps(num_inference_steps) # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature From aabc9d5140d6f0a31d2a9a6c6ee9f8ba55a032fa Mon Sep 17 00:00:00 2001 From: anton-l Date: Fri, 21 Oct 2022 15:37:25 +0200 Subject: [PATCH 3/3] run `make style` --- .../pipeline_latent_diffusion_uncond.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/diffusers/pipelines/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py b/src/diffusers/pipelines/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py index 331463c813b0..a7ffb4adc99a 100644 --- a/src/diffusers/pipelines/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py +++ b/src/diffusers/pipelines/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py @@ -63,10 +63,10 @@ def __call__( generator=generator, ) latents = latents.to(self.device) - + # scale the initial noise by the standard deviation required by the scheduler - latents = latents * self.scheduler.init_noise_sigma - + latents = latents * self.scheduler.init_noise_sigma + self.scheduler.set_timesteps(num_inference_steps) # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature