From 4b1e7d202bd4904b93e3980a11bc4a8c798783c2 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 24 Oct 2023 15:56:03 +0000 Subject: [PATCH 01/10] fix --- .../pipeline_stable_diffusion_xl_img2img.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py index f4d738fb2b03..43b072bd2137 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py @@ -515,8 +515,12 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N - (denoising_start * self.scheduler.config.num_train_timesteps) ) ) - timesteps = list(filter(lambda ts: ts < discrete_timestep_cutoff, timesteps)) - return torch.tensor(timesteps), len(timesteps) + + num_inference_steps = len(list(filter(lambda ts: ts < discrete_timestep_cutoff, timesteps))) + if self.scheduler.order == 2: + num_inference_steps = num_inference_steps + 1 + timesteps = timesteps[-num_inference_steps:] + return timesteps, num_inference_steps return timesteps, num_inference_steps - t_start From 8ea162b9121290155979ca245826e0dc4dd86d24 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 24 Oct 2023 16:53:54 +0000 Subject: [PATCH 02/10] fix copies --- .../controlnet/pipeline_controlnet_inpaint_sd_xl.py | 8 ++++++-- .../pipeline_stable_diffusion_xl_inpaint.py | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py index 118fc0230e46..ccd5dca11f48 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py @@ -867,8 +867,12 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N - (denoising_start * self.scheduler.config.num_train_timesteps) ) ) - timesteps = list(filter(lambda ts: ts < discrete_timestep_cutoff, timesteps)) - return torch.tensor(timesteps), len(timesteps) + + num_inference_steps = len(list(filter(lambda ts: ts < discrete_timestep_cutoff, timesteps))) + if self.scheduler.order == 2: + num_inference_steps = num_inference_steps + 1 + timesteps = timesteps[-num_inference_steps:] + return timesteps, num_inference_steps return timesteps, num_inference_steps - t_start diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py index fc942ca5227b..f795c275ff2b 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py @@ -799,8 +799,12 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N - (denoising_start * self.scheduler.config.num_train_timesteps) ) ) - timesteps = list(filter(lambda ts: ts < discrete_timestep_cutoff, timesteps)) - return torch.tensor(timesteps), len(timesteps) + + num_inference_steps = len(list(filter(lambda ts: ts < discrete_timestep_cutoff, timesteps))) + if self.scheduler.order == 2: + num_inference_steps = num_inference_steps + 1 + timesteps = timesteps[-num_inference_steps:] + return timesteps, num_inference_steps return timesteps, num_inference_steps - t_start From 5a0ac586a093823386513860985cd5d724a15c40 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 24 Oct 2023 18:16:13 +0000 Subject: [PATCH 03/10] remove heun from tests --- .../stable_diffusion_xl/test_stable_diffusion_xl_inpaint.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_inpaint.py b/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_inpaint.py index 7e3698d8ca16..779b270526fe 100644 --- a/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_inpaint.py +++ b/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_inpaint.py @@ -438,7 +438,6 @@ def new_step(self, *args, **kwargs): EulerDiscreteScheduler, DPMSolverMultistepScheduler, UniPCMultistepScheduler, - HeunDiscreteScheduler, ]: assert_run_mixture(steps, split_1, split_2, scheduler_cls) From 443aa6b6341ad274105a1b55fa09c606933fed66 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 24 Oct 2023 19:16:12 +0000 Subject: [PATCH 04/10] add back heun and fix the tests to include 2nd order --- .../test_stable_diffusion_xl_inpaint.py | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_inpaint.py b/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_inpaint.py index 779b270526fe..898fda0d7bb4 100644 --- a/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_inpaint.py +++ b/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_inpaint.py @@ -318,11 +318,14 @@ class scheduler_cls(scheduler_cls_orig): expected_steps = pipe_1.scheduler.timesteps.tolist() split_ts = num_train_timesteps - int(round(num_train_timesteps * split)) - expected_steps_1 = expected_steps[:split_ts] - expected_steps_2 = expected_steps[split_ts:] - expected_steps_1 = list(filter(lambda ts: ts >= split_ts, expected_steps)) - expected_steps_2 = list(filter(lambda ts: ts < split_ts, expected_steps)) + if pipe_1.scheduler.order == 2: + expected_steps_1 = list(filter(lambda ts: ts >= split_ts, expected_steps)) + expected_steps_2 = expected_steps_1[-1:] + list(filter(lambda ts: ts < split_ts, expected_steps)) + expected_steps = expected_steps_1 + expected_steps_2 + else: + expected_steps_1 = list(filter(lambda ts: ts >= split_ts, expected_steps)) + expected_steps_2 = list(filter(lambda ts: ts < split_ts, expected_steps)) # now we monkey patch step `done_steps` # list into the step function for testing @@ -389,13 +392,18 @@ class scheduler_cls(scheduler_cls_orig): split_1_ts = num_train_timesteps - int(round(num_train_timesteps * split_1)) split_2_ts = num_train_timesteps - int(round(num_train_timesteps * split_2)) - expected_steps_1 = expected_steps[:split_1_ts] - expected_steps_2 = expected_steps[split_1_ts:split_2_ts] - expected_steps_3 = expected_steps[split_2_ts:] - expected_steps_1 = list(filter(lambda ts: ts >= split_1_ts, expected_steps)) - expected_steps_2 = list(filter(lambda ts: ts >= split_2_ts and ts < split_1_ts, expected_steps)) - expected_steps_3 = list(filter(lambda ts: ts < split_2_ts, expected_steps)) + if pipe_1.scheduler.order == 2: + expected_steps_1 = list(filter(lambda ts: ts >= split_1_ts, expected_steps)) + expected_steps_2 = expected_steps_1[-1:] + list( + filter(lambda ts: ts >= split_2_ts and ts < split_1_ts, expected_steps) + ) + expected_steps_3 = expected_steps_2[-1:] + list(filter(lambda ts: ts < split_2_ts, expected_steps)) + expected_steps = expected_steps_1 + expected_steps_2 + expected_steps_3 + else: + expected_steps_1 = list(filter(lambda ts: ts >= split_1_ts, expected_steps)) + expected_steps_2 = list(filter(lambda ts: ts >= split_2_ts and ts < split_1_ts, expected_steps)) + expected_steps_3 = list(filter(lambda ts: ts < split_2_ts, expected_steps)) # now we monkey patch step `done_steps` # list into the step function for testing @@ -438,6 +446,7 @@ def new_step(self, *args, **kwargs): EulerDiscreteScheduler, DPMSolverMultistepScheduler, UniPCMultistepScheduler, + HeunDiscreteScheduler, ]: assert_run_mixture(steps, split_1, split_2, scheduler_cls) From de85e0bd5b98063244e371c57ab4d8fa3864ad72 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 24 Oct 2023 20:12:44 +0000 Subject: [PATCH 05/10] fix the other test too --- .../test_stable_diffusion_xl.py | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl.py b/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl.py index 65c7526e3aa2..323f7e349462 100644 --- a/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl.py +++ b/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl.py @@ -324,8 +324,13 @@ class scheduler_cls(scheduler_cls_orig): pipe_1.scheduler.set_timesteps(num_steps) expected_steps = pipe_1.scheduler.timesteps.tolist() - expected_steps_1 = list(filter(lambda ts: ts >= split, expected_tss)) - expected_steps_2 = list(filter(lambda ts: ts < split, expected_tss)) + if pipe_1.scheduler.order == 2: + expected_steps_1 = list(filter(lambda ts: ts >= split, expected_tss)) + expected_steps_2 = expected_steps_1[-1:] + list(filter(lambda ts: ts < split, expected_tss)) + expected_steps = expected_steps_1 + expected_steps_2 + else: + expected_steps_1 = list(filter(lambda ts: ts >= split, expected_tss)) + expected_steps_2 = list(filter(lambda ts: ts < split, expected_tss)) # now we monkey patch step `done_steps` # list into the step function for testing @@ -607,13 +612,18 @@ class scheduler_cls(scheduler_cls_orig): split_1_ts = num_train_timesteps - int(round(num_train_timesteps * split_1)) split_2_ts = num_train_timesteps - int(round(num_train_timesteps * split_2)) - expected_steps_1 = expected_steps[:split_1_ts] - expected_steps_2 = expected_steps[split_1_ts:split_2_ts] - expected_steps_3 = expected_steps[split_2_ts:] - expected_steps_1 = list(filter(lambda ts: ts >= split_1_ts, expected_steps)) - expected_steps_2 = list(filter(lambda ts: ts >= split_2_ts and ts < split_1_ts, expected_steps)) - expected_steps_3 = list(filter(lambda ts: ts < split_2_ts, expected_steps)) + if pipe_1.scheduler.order == 2: + expected_steps_1 = list(filter(lambda ts: ts >= split_1_ts, expected_steps)) + expected_steps_2 = expected_steps_1[-1:] + list( + filter(lambda ts: ts >= split_2_ts and ts < split_1_ts, expected_steps) + ) + expected_steps_3 = expected_steps_2[-1:] + list(filter(lambda ts: ts < split_2_ts, expected_steps)) + expected_steps = expected_steps_1 + expected_steps_2 + expected_steps_3 + else: + expected_steps_1 = list(filter(lambda ts: ts >= split_1_ts, expected_steps)) + expected_steps_2 = list(filter(lambda ts: ts >= split_2_ts and ts < split_1_ts, expected_steps)) + expected_steps_3 = list(filter(lambda ts: ts < split_2_ts, expected_steps)) # now we monkey patch step `done_steps` # list into the step function for testing From ceb0f45f6c46757f2f229b9ca9b679c7b6193819 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 25 Oct 2023 11:51:14 +0200 Subject: [PATCH 06/10] Apply suggestions from code review --- .../pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py | 2 +- .../stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py | 2 +- .../stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py index ccd5dca11f48..9e4b831bab70 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py @@ -868,7 +868,7 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N ) ) - num_inference_steps = len(list(filter(lambda ts: ts < discrete_timestep_cutoff, timesteps))) + num_inference_steps = (timesteps < discrete_timestep_cutoff).sum() if self.scheduler.order == 2: num_inference_steps = num_inference_steps + 1 timesteps = timesteps[-num_inference_steps:] diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py index 43b072bd2137..51afb7475567 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py @@ -516,7 +516,7 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N ) ) - num_inference_steps = len(list(filter(lambda ts: ts < discrete_timestep_cutoff, timesteps))) + num_inference_steps = (timesteps < discrete_timestep_cutoff).sum() if self.scheduler.order == 2: num_inference_steps = num_inference_steps + 1 timesteps = timesteps[-num_inference_steps:] diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py index f795c275ff2b..690ff939045e 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py @@ -800,7 +800,7 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N ) ) - num_inference_steps = len(list(filter(lambda ts: ts < discrete_timestep_cutoff, timesteps))) + num_inference_steps = (timesteps < discrete_timestep_cutoff).sum() if self.scheduler.order == 2: num_inference_steps = num_inference_steps + 1 timesteps = timesteps[-num_inference_steps:] From 62a9b6ed341d97cef8fa590cc5ed5bb077d127c1 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 25 Oct 2023 11:52:44 +0200 Subject: [PATCH 07/10] Apply suggestions from code review --- .../pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py | 2 ++ .../stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py | 2 ++ .../stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py index 9e4b831bab70..707a9fa89e23 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py @@ -871,6 +871,8 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N num_inference_steps = (timesteps < discrete_timestep_cutoff).sum() if self.scheduler.order == 2: num_inference_steps = num_inference_steps + 1 + + # because t_n+1 >= t_n, we slice the timesteps starting from the end timesteps = timesteps[-num_inference_steps:] return timesteps, num_inference_steps diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py index 51afb7475567..a762e32807eb 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py @@ -519,6 +519,8 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N num_inference_steps = (timesteps < discrete_timestep_cutoff).sum() if self.scheduler.order == 2: num_inference_steps = num_inference_steps + 1 + + # because t_n+1 >= t_n, we slice the timesteps starting from the end timesteps = timesteps[-num_inference_steps:] return timesteps, num_inference_steps diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py index 690ff939045e..1eae9e208ad8 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py @@ -803,6 +803,8 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N num_inference_steps = (timesteps < discrete_timestep_cutoff).sum() if self.scheduler.order == 2: num_inference_steps = num_inference_steps + 1 + + # because t_n+1 >= t_n, we slice the timesteps starting from the end timesteps = timesteps[-num_inference_steps:] return timesteps, num_inference_steps From a424a3fce4c6a96cce92aabbc0ab7fd8f6e8c41b Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 25 Oct 2023 11:53:19 +0200 Subject: [PATCH 08/10] Apply suggestions from code review --- .../pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py | 2 +- .../stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py | 2 +- .../stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py index 707a9fa89e23..0c53c5465131 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py @@ -872,7 +872,7 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N if self.scheduler.order == 2: num_inference_steps = num_inference_steps + 1 - # because t_n+1 >= t_n, we slice the timesteps starting from the end + # because t_n+1 >= t_n, we slice the timesteps starting from the end timesteps = timesteps[-num_inference_steps:] return timesteps, num_inference_steps diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py index a762e32807eb..e18d4508a9e1 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py @@ -520,7 +520,7 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N if self.scheduler.order == 2: num_inference_steps = num_inference_steps + 1 - # because t_n+1 >= t_n, we slice the timesteps starting from the end + # because t_n+1 >= t_n, we slice the timesteps starting from the end timesteps = timesteps[-num_inference_steps:] return timesteps, num_inference_steps diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py index 1eae9e208ad8..bccaacfb4846 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py @@ -804,7 +804,7 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N if self.scheduler.order == 2: num_inference_steps = num_inference_steps + 1 - # because t_n+1 >= t_n, we slice the timesteps starting from the end + # because t_n+1 >= t_n, we slice the timesteps starting from the end timesteps = timesteps[-num_inference_steps:] return timesteps, num_inference_steps From ea590b853a3134b701a926481660c22eac1a1b7b Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 25 Oct 2023 09:58:12 +0000 Subject: [PATCH 09/10] make style --- .../pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py | 2 +- .../stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py | 2 +- .../stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py index 0c53c5465131..39573d86594e 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py @@ -871,7 +871,7 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N num_inference_steps = (timesteps < discrete_timestep_cutoff).sum() if self.scheduler.order == 2: num_inference_steps = num_inference_steps + 1 - + # because t_n+1 >= t_n, we slice the timesteps starting from the end timesteps = timesteps[-num_inference_steps:] return timesteps, num_inference_steps diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py index e18d4508a9e1..c2704b3b9b00 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py @@ -519,7 +519,7 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N num_inference_steps = (timesteps < discrete_timestep_cutoff).sum() if self.scheduler.order == 2: num_inference_steps = num_inference_steps + 1 - + # because t_n+1 >= t_n, we slice the timesteps starting from the end timesteps = timesteps[-num_inference_steps:] return timesteps, num_inference_steps diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py index bccaacfb4846..3e477e242604 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py @@ -803,7 +803,7 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N num_inference_steps = (timesteps < discrete_timestep_cutoff).sum() if self.scheduler.order == 2: num_inference_steps = num_inference_steps + 1 - + # because t_n+1 >= t_n, we slice the timesteps starting from the end timesteps = timesteps[-num_inference_steps:] return timesteps, num_inference_steps From 2b959dfb6eb68622253e9751d096933c252e85f3 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 25 Oct 2023 10:11:18 +0000 Subject: [PATCH 10/10] add more comments --- .../controlnet/pipeline_controlnet_inpaint_sd_xl.py | 8 +++++++- .../pipeline_stable_diffusion_xl_img2img.py | 8 +++++++- .../pipeline_stable_diffusion_xl_inpaint.py | 8 +++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py index 39573d86594e..490e19fd3c97 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py @@ -868,8 +868,14 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N ) ) - num_inference_steps = (timesteps < discrete_timestep_cutoff).sum() + num_inference_steps = (timesteps < discrete_timestep_cutoff).sum().item() if self.scheduler.order == 2: + # if the scheduler is a 2nd order scheduler we ALWAYS have to do +1 + # because `num_inference_steps` will always be even given that every timestep + # (except the highest one) is duplicated. If `num_inference_steps` is even it would + # mean that we cut the timesteps in the middle of the denoising step + # (between 1st and 2nd devirative) which leads to incorrect results. By adding 1 + # we ensure that the denoising process always ends after the 2nd derivate step of the scheduler num_inference_steps = num_inference_steps + 1 # because t_n+1 >= t_n, we slice the timesteps starting from the end diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py index c2704b3b9b00..a9ae25e800c7 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py @@ -516,8 +516,14 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N ) ) - num_inference_steps = (timesteps < discrete_timestep_cutoff).sum() + num_inference_steps = (timesteps < discrete_timestep_cutoff).sum().item() if self.scheduler.order == 2: + # if the scheduler is a 2nd order scheduler we ALWAYS have to do +1 + # because `num_inference_steps` will always be even given that every timestep + # (except the highest one) is duplicated. If `num_inference_steps` is even it would + # mean that we cut the timesteps in the middle of the denoising step + # (between 1st and 2nd devirative) which leads to incorrect results. By adding 1 + # we ensure that the denoising process always ends after the 2nd derivate step of the scheduler num_inference_steps = num_inference_steps + 1 # because t_n+1 >= t_n, we slice the timesteps starting from the end diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py index 3e477e242604..bf32ff7bfd50 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py @@ -800,8 +800,14 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N ) ) - num_inference_steps = (timesteps < discrete_timestep_cutoff).sum() + num_inference_steps = (timesteps < discrete_timestep_cutoff).sum().item() if self.scheduler.order == 2: + # if the scheduler is a 2nd order scheduler we ALWAYS have to do +1 + # because `num_inference_steps` will always be even given that every timestep + # (except the highest one) is duplicated. If `num_inference_steps` is even it would + # mean that we cut the timesteps in the middle of the denoising step + # (between 1st and 2nd devirative) which leads to incorrect results. By adding 1 + # we ensure that the denoising process always ends after the 2nd derivate step of the scheduler num_inference_steps = num_inference_steps + 1 # because t_n+1 >= t_n, we slice the timesteps starting from the end