From bf863cba87ea7b951f6fcf023c81233de7a14154 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 25 Oct 2023 12:06:05 +0000 Subject: [PATCH 1/2] [Schedulers] Fix 2nd order other than heun --- .../controlnet/pipeline_controlnet_inpaint_sd_xl.py | 7 ++++--- .../pipeline_stable_diffusion_xl_img2img.py | 7 ++++--- .../pipeline_stable_diffusion_xl_inpaint.py | 7 ++++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py index 1423920191d7..f47b9fdef880 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py @@ -898,9 +898,10 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N ) num_inference_steps = (timesteps < discrete_timestep_cutoff).sum().item() - if self.scheduler.order == 2: - # if the scheduler is a 2nd order scheduler we ALWAYS have to do +1 - # because `num_inference_steps` will always be even given that every timestep + print(num_inference_steps) + if self.scheduler.order == 2 and num_inference_steps % 2 == 0: + # if the scheduler is a 2nd order scheduler we might have to do +1 + # because `num_inference_steps` might be even given that every timestep # (except the highest one) is duplicated. If `num_inference_steps` is even it would # mean that we cut the timesteps in the middle of the denoising step # (between 1st and 2nd devirative) which leads to incorrect results. By adding 1 diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py index ff9d669a802b..2ab630fcd2f8 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py @@ -555,9 +555,10 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N ) num_inference_steps = (timesteps < discrete_timestep_cutoff).sum().item() - if self.scheduler.order == 2: - # if the scheduler is a 2nd order scheduler we ALWAYS have to do +1 - # because `num_inference_steps` will always be even given that every timestep + print(num_inference_steps) + if self.scheduler.order == 2 and num_inference_steps % 2 == 0: + # if the scheduler is a 2nd order scheduler we might have to do +1 + # because `num_inference_steps` might be even given that every timestep # (except the highest one) is duplicated. If `num_inference_steps` is even it would # mean that we cut the timesteps in the middle of the denoising step # (between 1st and 2nd devirative) which leads to incorrect results. By adding 1 diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py index 200f5a7bf45a..fd4b9636b41a 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py @@ -840,9 +840,10 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N ) num_inference_steps = (timesteps < discrete_timestep_cutoff).sum().item() - if self.scheduler.order == 2: - # if the scheduler is a 2nd order scheduler we ALWAYS have to do +1 - # because `num_inference_steps` will always be even given that every timestep + print(num_inference_steps) + if self.scheduler.order == 2 and num_inference_steps % 2 == 0: + # if the scheduler is a 2nd order scheduler we might have to do +1 + # because `num_inference_steps` might be even given that every timestep # (except the highest one) is duplicated. If `num_inference_steps` is even it would # mean that we cut the timesteps in the middle of the denoising step # (between 1st and 2nd devirative) which leads to incorrect results. By adding 1 From a8259be8c63b698ebd8bc2f35b5326365e56c7ff Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 25 Oct 2023 14:06:37 +0200 Subject: [PATCH 2/2] Apply suggestions from code review --- .../pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py | 1 - .../stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py | 1 - .../stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py | 1 - 3 files changed, 3 deletions(-) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py index f47b9fdef880..46c9f25b6eb6 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py @@ -898,7 +898,6 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N ) num_inference_steps = (timesteps < discrete_timestep_cutoff).sum().item() - print(num_inference_steps) if self.scheduler.order == 2 and num_inference_steps % 2 == 0: # if the scheduler is a 2nd order scheduler we might have to do +1 # because `num_inference_steps` might be even given that every timestep diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py index 2ab630fcd2f8..57d00af82106 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py @@ -555,7 +555,6 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N ) num_inference_steps = (timesteps < discrete_timestep_cutoff).sum().item() - print(num_inference_steps) if self.scheduler.order == 2 and num_inference_steps % 2 == 0: # if the scheduler is a 2nd order scheduler we might have to do +1 # because `num_inference_steps` might be even given that every timestep diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py index fd4b9636b41a..11ae0a0d85f0 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py @@ -840,7 +840,6 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N ) num_inference_steps = (timesteps < discrete_timestep_cutoff).sum().item() - print(num_inference_steps) if self.scheduler.order == 2 and num_inference_steps % 2 == 0: # if the scheduler is a 2nd order scheduler we might have to do +1 # because `num_inference_steps` might be even given that every timestep