From 0934d9dbbcfdb7ffb0d335db4daeb3069ea591d1 Mon Sep 17 00:00:00 2001 From: andrehuang0 Date: Tue, 7 Mar 2023 13:56:41 +0100 Subject: [PATCH 1/3] fix the in-place modification in unet condition when using controlnet, which will cause backprop errors when training --- src/diffusers/models/unet_2d_condition.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/diffusers/models/unet_2d_condition.py b/src/diffusers/models/unet_2d_condition.py index a5f997e9b400..c6a1e1f2f171 100644 --- a/src/diffusers/models/unet_2d_condition.py +++ b/src/diffusers/models/unet_2d_condition.py @@ -597,6 +597,7 @@ def forward( for down_block_res_sample, down_block_additional_residual in zip( down_block_res_samples, down_block_additional_residuals ): + down_block_res_sample = down_block_res_sample.clone() # make sure we don't modify the original in place down_block_res_sample += down_block_additional_residual new_down_block_res_samples += (down_block_res_sample,) From 35b7add1dbfcbf2433f40af6a952f72fe15a51fa Mon Sep 17 00:00:00 2001 From: William Berman Date: Mon, 13 Mar 2023 23:34:27 -0700 Subject: [PATCH 2/3] add clone to mid block --- src/diffusers/models/unet_2d_condition.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/diffusers/models/unet_2d_condition.py b/src/diffusers/models/unet_2d_condition.py index 393c8e4ecd1b..24e827a328f7 100644 --- a/src/diffusers/models/unet_2d_condition.py +++ b/src/diffusers/models/unet_2d_condition.py @@ -598,8 +598,7 @@ def forward( for down_block_res_sample, down_block_additional_residual in zip( down_block_res_samples, down_block_additional_residuals ): - down_block_res_sample = down_block_res_sample.clone() # make sure we don't modify the original in place - down_block_res_sample += down_block_additional_residual + down_block_res_sample = down_block_res_sample + down_block_additional_residual new_down_block_res_samples += (down_block_res_sample,) down_block_res_samples = new_down_block_res_samples @@ -615,7 +614,7 @@ def forward( ) if mid_block_additional_residual is not None: - sample += mid_block_additional_residual + sample = sample + mid_block_additional_residual # 5. up for i, upsample_block in enumerate(self.up_blocks): From eac1b232413a7d0ae61f93ded89448addd077e46 Mon Sep 17 00:00:00 2001 From: William Berman Date: Mon, 13 Mar 2023 23:36:54 -0700 Subject: [PATCH 3/3] fix-copies --- .../pipelines/versatile_diffusion/modeling_text_unet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/diffusers/pipelines/versatile_diffusion/modeling_text_unet.py b/src/diffusers/pipelines/versatile_diffusion/modeling_text_unet.py index e9e7b18eb02a..76bfdc4313ca 100644 --- a/src/diffusers/pipelines/versatile_diffusion/modeling_text_unet.py +++ b/src/diffusers/pipelines/versatile_diffusion/modeling_text_unet.py @@ -688,7 +688,7 @@ def forward( for down_block_res_sample, down_block_additional_residual in zip( down_block_res_samples, down_block_additional_residuals ): - down_block_res_sample += down_block_additional_residual + down_block_res_sample = down_block_res_sample + down_block_additional_residual new_down_block_res_samples += (down_block_res_sample,) down_block_res_samples = new_down_block_res_samples @@ -704,7 +704,7 @@ def forward( ) if mid_block_additional_residual is not None: - sample += mid_block_additional_residual + sample = sample + mid_block_additional_residual # 5. up for i, upsample_block in enumerate(self.up_blocks):