From 2652654d0bc112f9ffb68118fe310d27ea617492 Mon Sep 17 00:00:00 2001 From: George He Date: Thu, 3 Aug 2023 09:24:18 -0400 Subject: [PATCH 1/2] Handle single controlnet in multicontrolnet --- src/diffusers/pipelines/controlnet/pipeline_controlnet.py | 5 ++++- .../pipelines/controlnet/pipeline_controlnet_img2img.py | 5 ++++- .../pipelines/controlnet/pipeline_controlnet_inpaint.py | 5 ++++- .../pipelines/controlnet/pipeline_controlnet_sd_xl.py | 5 ++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet.py index 995c4ec0e488..f7c07deb6123 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet.py @@ -941,7 +941,10 @@ def __call__( if isinstance(controlnet_keep[i], list): cond_scale = [c * s for c, s in zip(controlnet_conditioning_scale, controlnet_keep[i])] else: - cond_scale = controlnet_conditioning_scale * controlnet_keep[i] + controlnet_cond_scale = controlnet_conditioning_scale + if isinstance(controlnet_cond_scale, list): + controlnet_cond_scale = controlnet_cond_scale[0] + cond_scale = controlnet_cond_scale * controlnet_keep[i] down_block_res_samples, mid_block_res_sample = self.controlnet( control_model_input, diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py index de1bd2f97f10..d92eb0b7a213 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py @@ -1034,7 +1034,10 @@ def __call__( if isinstance(controlnet_keep[i], list): cond_scale = [c * s for c, s in zip(controlnet_conditioning_scale, controlnet_keep[i])] else: - cond_scale = controlnet_conditioning_scale * controlnet_keep[i] + controlnet_cond_scale = controlnet_conditioning_scale + if isinstance(controlnet_cond_scale, list): + controlnet_cond_scale = controlnet_cond_scale[0] + cond_scale = controlnet_cond_scale * controlnet_keep[i] down_block_res_samples, mid_block_res_sample = self.controlnet( control_model_input, diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py index e37db367378a..02fadb04b299 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py @@ -1269,7 +1269,10 @@ def __call__( if isinstance(controlnet_keep[i], list): cond_scale = [c * s for c, s in zip(controlnet_conditioning_scale, controlnet_keep[i])] else: - cond_scale = controlnet_conditioning_scale * controlnet_keep[i] + controlnet_cond_scale = controlnet_conditioning_scale + if isinstance(controlnet_cond_scale, list): + controlnet_cond_scale = controlnet_cond_scale[0] + cond_scale = controlnet_cond_scale * controlnet_keep[i] down_block_res_samples, mid_block_res_sample = self.controlnet( control_model_input, diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py index 2fa4d3b04787..368154b60ef8 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py @@ -967,7 +967,10 @@ def __call__( if isinstance(controlnet_keep[i], list): cond_scale = [c * s for c, s in zip(controlnet_conditioning_scale, controlnet_keep[i])] else: - cond_scale = controlnet_conditioning_scale * controlnet_keep[i] + controlnet_cond_scale = controlnet_conditioning_scale + if isinstance(controlnet_cond_scale, list): + controlnet_cond_scale = controlnet_cond_scale[0] + cond_scale = controlnet_cond_scale * controlnet_keep[i] added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} down_block_res_samples, mid_block_res_sample = self.controlnet( From a6d4352d340c4df1197ba42979592a16bce8ebb0 Mon Sep 17 00:00:00 2001 From: George He Date: Thu, 3 Aug 2023 09:34:15 -0400 Subject: [PATCH 2/2] Fix formatting --- src/diffusers/pipelines/controlnet/pipeline_controlnet.py | 4 ++-- .../pipelines/controlnet/pipeline_controlnet_img2img.py | 4 ++-- .../pipelines/controlnet/pipeline_controlnet_inpaint.py | 4 ++-- .../pipelines/controlnet/pipeline_controlnet_sd_xl.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet.py index f7c07deb6123..03b0680d8372 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet.py @@ -941,9 +941,9 @@ def __call__( if isinstance(controlnet_keep[i], list): cond_scale = [c * s for c, s in zip(controlnet_conditioning_scale, controlnet_keep[i])] else: - controlnet_cond_scale = controlnet_conditioning_scale + controlnet_cond_scale = controlnet_conditioning_scale if isinstance(controlnet_cond_scale, list): - controlnet_cond_scale = controlnet_cond_scale[0] + controlnet_cond_scale = controlnet_cond_scale[0] cond_scale = controlnet_cond_scale * controlnet_keep[i] down_block_res_samples, mid_block_res_sample = self.controlnet( diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py index d92eb0b7a213..655d9db01c24 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py @@ -1034,9 +1034,9 @@ def __call__( if isinstance(controlnet_keep[i], list): cond_scale = [c * s for c, s in zip(controlnet_conditioning_scale, controlnet_keep[i])] else: - controlnet_cond_scale = controlnet_conditioning_scale + controlnet_cond_scale = controlnet_conditioning_scale if isinstance(controlnet_cond_scale, list): - controlnet_cond_scale = controlnet_cond_scale[0] + controlnet_cond_scale = controlnet_cond_scale[0] cond_scale = controlnet_cond_scale * controlnet_keep[i] down_block_res_samples, mid_block_res_sample = self.controlnet( diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py index 02fadb04b299..83bec1e190a9 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py @@ -1269,9 +1269,9 @@ def __call__( if isinstance(controlnet_keep[i], list): cond_scale = [c * s for c, s in zip(controlnet_conditioning_scale, controlnet_keep[i])] else: - controlnet_cond_scale = controlnet_conditioning_scale + controlnet_cond_scale = controlnet_conditioning_scale if isinstance(controlnet_cond_scale, list): - controlnet_cond_scale = controlnet_cond_scale[0] + controlnet_cond_scale = controlnet_cond_scale[0] cond_scale = controlnet_cond_scale * controlnet_keep[i] down_block_res_samples, mid_block_res_sample = self.controlnet( diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py index 368154b60ef8..bb9568221505 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py @@ -967,9 +967,9 @@ def __call__( if isinstance(controlnet_keep[i], list): cond_scale = [c * s for c, s in zip(controlnet_conditioning_scale, controlnet_keep[i])] else: - controlnet_cond_scale = controlnet_conditioning_scale + controlnet_cond_scale = controlnet_conditioning_scale if isinstance(controlnet_cond_scale, list): - controlnet_cond_scale = controlnet_cond_scale[0] + controlnet_cond_scale = controlnet_cond_scale[0] cond_scale = controlnet_cond_scale * controlnet_keep[i] added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}