From a9aad0e968b75e5a90e9b137656e90148d0a78b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tolga=20Cang=C3=B6z?= Date: Thu, 30 May 2024 16:20:42 +0300 Subject: [PATCH 1/3] Refactor code to remove unnecessary calls to `to(torch_device)` --- .../pipeline_stable_diffusion_diffedit.py | 14 ++++++-------- .../stable_diffusion/test_stable_diffusion.py | 1 - .../test_stable_diffusion_img2img.py | 1 - .../test_stable_diffusion_inpaint.py | 2 -- .../test_stable_diffusion_instruction_pix2pix.py | 1 - .../stable_diffusion_2/test_stable_diffusion.py | 1 - .../test_stable_diffusion_depth.py | 1 - .../test_stable_diffusion_inpaint.py | 1 - .../test_stable_diffusion_upscale.py | 1 - .../test_stable_diffusion_v_pred.py | 1 - .../test_stable_diffusion_adapter.py | 2 -- .../test_stable_diffusion_image_variation.py | 1 - .../test_stable_diffusion_panorama.py | 1 - .../pipelines/stable_unclip/test_stable_unclip.py | 2 -- .../stable_unclip/test_stable_unclip_img2img.py | 3 --- tests/pipelines/unclip/test_unclip.py | 1 - 16 files changed, 6 insertions(+), 28 deletions(-) diff --git a/src/diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py b/src/diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py index 9cec7cc4e15b..9f1ad9ecb6fd 100644 --- a/src/diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py +++ b/src/diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py @@ -85,10 +85,9 @@ class DiffEditInversionPipelineOutput(BaseOutput): >>> init_image = download_image(img_url).resize((768, 768)) - >>> pipe = StableDiffusionDiffEditPipeline.from_pretrained( + >>> pipeline = StableDiffusionDiffEditPipeline.from_pretrained( ... "stabilityai/stable-diffusion-2-1", torch_dtype=torch.float16 ... ) - >>> pipe = pipe.to("cuda") >>> pipeline.scheduler = DDIMScheduler.from_config(pipeline.scheduler.config) >>> pipeline.inverse_scheduler = DDIMInverseScheduler.from_config(pipeline.scheduler.config) @@ -97,9 +96,9 @@ class DiffEditInversionPipelineOutput(BaseOutput): >>> mask_prompt = "A bowl of fruits" >>> prompt = "A bowl of pears" - >>> mask_image = pipe.generate_mask(image=init_image, source_prompt=prompt, target_prompt=mask_prompt) - >>> image_latents = pipe.invert(image=init_image, prompt=mask_prompt).latents - >>> image = pipe(prompt=prompt, mask_image=mask_image, image_latents=image_latents).images[0] + >>> mask_image = pipeline.generate_mask(image=init_image, source_prompt=prompt, target_prompt=mask_prompt) + >>> image_latents = pipeline.invert(image=init_image, prompt=mask_prompt).latents + >>> image = pipeline(prompt=prompt, mask_image=mask_image, image_latents=image_latents).images[0] ``` """ @@ -122,10 +121,9 @@ class DiffEditInversionPipelineOutput(BaseOutput): >>> init_image = download_image(img_url).resize((768, 768)) - >>> pipe = StableDiffusionDiffEditPipeline.from_pretrained( + >>> pipeline = StableDiffusionDiffEditPipeline.from_pretrained( ... "stabilityai/stable-diffusion-2-1", torch_dtype=torch.float16 ... ) - >>> pipe = pipe.to("cuda") >>> pipeline.scheduler = DDIMScheduler.from_config(pipeline.scheduler.config) >>> pipeline.inverse_scheduler = DDIMInverseScheduler.from_config(pipeline.scheduler.config) @@ -133,7 +131,7 @@ class DiffEditInversionPipelineOutput(BaseOutput): >>> prompt = "A bowl of fruits" - >>> inverted_latents = pipe.invert(image=init_image, prompt=prompt).latents + >>> inverted_latents = pipeline.invert(image=init_image, prompt=prompt).latents ``` """ diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_stable_diffusion.py index 5edfedefc553..85af45768065 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion.py @@ -1135,7 +1135,6 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): torch.cuda.reset_peak_memory_stats() pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", torch_dtype=torch.float16) - pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing(1) pipe.enable_sequential_cpu_offload() diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py index d691565248bc..2a26e1f478a6 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py @@ -513,7 +513,6 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): pipe = StableDiffusionImg2ImgPipeline.from_pretrained( "CompVis/stable-diffusion-v1-4", safety_checker=None, torch_dtype=torch.float16 ) - pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing(1) pipe.enable_sequential_cpu_offload() diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py index 0567c4c23650..737f8e90ac6b 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py @@ -705,7 +705,6 @@ def test_stable_diffusion_inpaint_with_sequential_cpu_offloading(self): pipe = StableDiffusionInpaintPipeline.from_pretrained( "runwayml/stable-diffusion-inpainting", safety_checker=None, torch_dtype=torch.float16 ) - pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing(1) pipe.enable_sequential_cpu_offload() @@ -911,7 +910,6 @@ def test_stable_diffusion_inpaint_with_sequential_cpu_offloading(self): "runwayml/stable-diffusion-inpainting", safety_checker=None, torch_dtype=torch.float16 ) pipe.vae = vae - pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing(1) pipe.enable_sequential_cpu_offload() diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_instruction_pix2pix.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_instruction_pix2pix.py index 5da8669215b2..b9b061c060c0 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_instruction_pix2pix.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_instruction_pix2pix.py @@ -394,7 +394,6 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): pipe = StableDiffusionInstructPix2PixPipeline.from_pretrained( "timbrooks/instruct-pix2pix", safety_checker=None, torch_dtype=torch.float16 ) - pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing(1) pipe.enable_sequential_cpu_offload() diff --git a/tests/pipelines/stable_diffusion_2/test_stable_diffusion.py b/tests/pipelines/stable_diffusion_2/test_stable_diffusion.py index 494036482a77..c39bfb1984b5 100644 --- a/tests/pipelines/stable_diffusion_2/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion_2/test_stable_diffusion.py @@ -462,7 +462,6 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): pipe = StableDiffusionPipeline.from_pretrained( "stabilityai/stable-diffusion-2-base", torch_dtype=torch.float16 ) - pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing(1) pipe.enable_sequential_cpu_offload() diff --git a/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py b/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py index c260d565a829..8fc76db311a6 100644 --- a/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py +++ b/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py @@ -506,7 +506,6 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): pipe = StableDiffusionDepth2ImgPipeline.from_pretrained( "stabilityai/stable-diffusion-2-depth", safety_checker=None, torch_dtype=torch.float16 ) - pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing(1) pipe.enable_sequential_cpu_offload() diff --git a/tests/pipelines/stable_diffusion_2/test_stable_diffusion_inpaint.py b/tests/pipelines/stable_diffusion_2/test_stable_diffusion_inpaint.py index e1dc363878c4..b99a1816456e 100644 --- a/tests/pipelines/stable_diffusion_2/test_stable_diffusion_inpaint.py +++ b/tests/pipelines/stable_diffusion_2/test_stable_diffusion_inpaint.py @@ -261,7 +261,6 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): scheduler=pndm, torch_dtype=torch.float16, ) - pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing(1) pipe.enable_sequential_cpu_offload() diff --git a/tests/pipelines/stable_diffusion_2/test_stable_diffusion_upscale.py b/tests/pipelines/stable_diffusion_2/test_stable_diffusion_upscale.py index f63aaeefd0b9..c21da7af6d2c 100644 --- a/tests/pipelines/stable_diffusion_2/test_stable_diffusion_upscale.py +++ b/tests/pipelines/stable_diffusion_2/test_stable_diffusion_upscale.py @@ -472,7 +472,6 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): model_id, torch_dtype=torch.float16, ) - pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing(1) pipe.enable_sequential_cpu_offload() diff --git a/tests/pipelines/stable_diffusion_2/test_stable_diffusion_v_pred.py b/tests/pipelines/stable_diffusion_2/test_stable_diffusion_v_pred.py index 923fba1272ce..703c3b7a39d8 100644 --- a/tests/pipelines/stable_diffusion_2/test_stable_diffusion_v_pred.py +++ b/tests/pipelines/stable_diffusion_2/test_stable_diffusion_v_pred.py @@ -537,7 +537,6 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading_v_pred(self): prompt = "Andromeda galaxy in a bottle" pipeline = StableDiffusionPipeline.from_pretrained(pipeline_id, torch_dtype=torch.float16) - pipeline = pipeline.to(torch_device) pipeline.enable_attention_slicing(1) pipeline.enable_sequential_cpu_offload() diff --git a/tests/pipelines/stable_diffusion_adapter/test_stable_diffusion_adapter.py b/tests/pipelines/stable_diffusion_adapter/test_stable_diffusion_adapter.py index 36bccaac9d93..9b5adcba1684 100644 --- a/tests/pipelines/stable_diffusion_adapter/test_stable_diffusion_adapter.py +++ b/tests/pipelines/stable_diffusion_adapter/test_stable_diffusion_adapter.py @@ -809,7 +809,6 @@ def test_stable_diffusion_adapter_zoedepth_sd_v15(self): adapter = T2IAdapter.from_pretrained(adapter_model, torch_dtype=torch.float16) pipe = StableDiffusionAdapterPipeline.from_pretrained(sd_model, adapter=adapter, safety_checker=None) - pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_model_cpu_offload() generator = torch.Generator(device="cpu").manual_seed(0) @@ -942,7 +941,6 @@ def test_stable_diffusion_adapter_pipeline_with_sequential_cpu_offloading(self): pipe = StableDiffusionAdapterPipeline.from_pretrained( "CompVis/stable-diffusion-v1-4", adapter=adapter, safety_checker=None ) - pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing(1) pipe.enable_sequential_cpu_offload() diff --git a/tests/pipelines/stable_diffusion_image_variation/test_stable_diffusion_image_variation.py b/tests/pipelines/stable_diffusion_image_variation/test_stable_diffusion_image_variation.py index 6ddb562aacf2..f707c15cdbb4 100644 --- a/tests/pipelines/stable_diffusion_image_variation/test_stable_diffusion_image_variation.py +++ b/tests/pipelines/stable_diffusion_image_variation/test_stable_diffusion_image_variation.py @@ -263,7 +263,6 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): pipe = StableDiffusionImageVariationPipeline.from_pretrained( "lambdalabs/sd-image-variations-diffusers", safety_checker=None, torch_dtype=torch.float16 ) - pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing(1) pipe.enable_sequential_cpu_offload() diff --git a/tests/pipelines/stable_diffusion_panorama/test_stable_diffusion_panorama.py b/tests/pipelines/stable_diffusion_panorama/test_stable_diffusion_panorama.py index 4e36dab5ac6c..6dc6c31ae9a7 100644 --- a/tests/pipelines/stable_diffusion_panorama/test_stable_diffusion_panorama.py +++ b/tests/pipelines/stable_diffusion_panorama/test_stable_diffusion_panorama.py @@ -415,7 +415,6 @@ def test_stable_diffusion_panorama_pipeline_with_sequential_cpu_offloading(self) model_ckpt = "stabilityai/stable-diffusion-2-base" scheduler = DDIMScheduler.from_pretrained(model_ckpt, subfolder="scheduler") pipe = StableDiffusionPanoramaPipeline.from_pretrained(model_ckpt, scheduler=scheduler, safety_checker=None) - pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing(1) pipe.enable_sequential_cpu_offload() diff --git a/tests/pipelines/stable_unclip/test_stable_unclip.py b/tests/pipelines/stable_unclip/test_stable_unclip.py index eaf4573c123a..bb54d212a786 100644 --- a/tests/pipelines/stable_unclip/test_stable_unclip.py +++ b/tests/pipelines/stable_unclip/test_stable_unclip.py @@ -206,7 +206,6 @@ def test_stable_unclip(self): ) pipe = StableUnCLIPPipeline.from_pretrained("fusing/stable-unclip-2-1-l", torch_dtype=torch.float16) - pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) # stable unclip will oom when integration tests are run on a V100, # so turn on memory savings @@ -228,7 +227,6 @@ def test_stable_unclip_pipeline_with_sequential_cpu_offloading(self): torch.cuda.reset_peak_memory_stats() pipe = StableUnCLIPPipeline.from_pretrained("fusing/stable-unclip-2-1-l", torch_dtype=torch.float16) - pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() pipe.enable_sequential_cpu_offload() diff --git a/tests/pipelines/stable_unclip/test_stable_unclip_img2img.py b/tests/pipelines/stable_unclip/test_stable_unclip_img2img.py index cd2787b2f5e3..7ccdc515c68b 100644 --- a/tests/pipelines/stable_unclip/test_stable_unclip_img2img.py +++ b/tests/pipelines/stable_unclip/test_stable_unclip_img2img.py @@ -233,7 +233,6 @@ def test_stable_unclip_l_img2img(self): pipe = StableUnCLIPImg2ImgPipeline.from_pretrained( "fusing/stable-unclip-2-1-l-img2img", torch_dtype=torch.float16 ) - pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) # stable unclip will oom when integration tests are run on a V100, # so turn on memory savings @@ -261,7 +260,6 @@ def test_stable_unclip_h_img2img(self): pipe = StableUnCLIPImg2ImgPipeline.from_pretrained( "fusing/stable-unclip-2-1-h-img2img", torch_dtype=torch.float16 ) - pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) # stable unclip will oom when integration tests are run on a V100, # so turn on memory savings @@ -289,7 +287,6 @@ def test_stable_unclip_img2img_pipeline_with_sequential_cpu_offloading(self): pipe = StableUnCLIPImg2ImgPipeline.from_pretrained( "fusing/stable-unclip-2-1-h-img2img", torch_dtype=torch.float16 ) - pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() pipe.enable_sequential_cpu_offload() diff --git a/tests/pipelines/unclip/test_unclip.py b/tests/pipelines/unclip/test_unclip.py index d3d026a448ca..07590c9db458 100644 --- a/tests/pipelines/unclip/test_unclip.py +++ b/tests/pipelines/unclip/test_unclip.py @@ -500,7 +500,6 @@ def test_unclip_pipeline_with_sequential_cpu_offloading(self): torch.cuda.reset_peak_memory_stats() pipe = UnCLIPPipeline.from_pretrained("kakaobrain/karlo-v1-alpha", torch_dtype=torch.float16) - pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() pipe.enable_sequential_cpu_offload() From 812a4c134d85eaefc2a78bcd09600f03e37b8354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tolga=20Cang=C3=B6z?= Date: Thu, 30 May 2024 19:02:40 +0300 Subject: [PATCH 2/3] Refactor code to remove unnecessary calls to `to("cuda")` --- docs/source/en/api/pipelines/animatediff.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/api/pipelines/animatediff.md b/docs/source/en/api/pipelines/animatediff.md index 425764541590..b21650aa2a4f 100644 --- a/docs/source/en/api/pipelines/animatediff.md +++ b/docs/source/en/api/pipelines/animatediff.md @@ -165,7 +165,7 @@ from PIL import Image adapter = MotionAdapter.from_pretrained("guoyww/animatediff-motion-adapter-v1-5-2", torch_dtype=torch.float16) # load SD 1.5 based finetuned model model_id = "SG161222/Realistic_Vision_V5.1_noVAE" -pipe = AnimateDiffVideoToVideoPipeline.from_pretrained(model_id, motion_adapter=adapter, torch_dtype=torch.float16).to("cuda") +pipe = AnimateDiffVideoToVideoPipeline.from_pretrained(model_id, motion_adapter=adapter, torch_dtype=torch.float16) scheduler = DDIMScheduler.from_pretrained( model_id, subfolder="scheduler", From 94425558e358a37f29cfea7f2132e8400bc20002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tolga=20Cang=C3=B6z?= <46008593+tolgacangoz@users.noreply.github.com> Date: Wed, 5 Jun 2024 20:39:32 +0300 Subject: [PATCH 3/3] Update pipeline_stable_diffusion_diffedit.py --- .../pipeline_stable_diffusion_diffedit.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py b/src/diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py index 9f1ad9ecb6fd..9cec7cc4e15b 100644 --- a/src/diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py +++ b/src/diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py @@ -85,9 +85,10 @@ class DiffEditInversionPipelineOutput(BaseOutput): >>> init_image = download_image(img_url).resize((768, 768)) - >>> pipeline = StableDiffusionDiffEditPipeline.from_pretrained( + >>> pipe = StableDiffusionDiffEditPipeline.from_pretrained( ... "stabilityai/stable-diffusion-2-1", torch_dtype=torch.float16 ... ) + >>> pipe = pipe.to("cuda") >>> pipeline.scheduler = DDIMScheduler.from_config(pipeline.scheduler.config) >>> pipeline.inverse_scheduler = DDIMInverseScheduler.from_config(pipeline.scheduler.config) @@ -96,9 +97,9 @@ class DiffEditInversionPipelineOutput(BaseOutput): >>> mask_prompt = "A bowl of fruits" >>> prompt = "A bowl of pears" - >>> mask_image = pipeline.generate_mask(image=init_image, source_prompt=prompt, target_prompt=mask_prompt) - >>> image_latents = pipeline.invert(image=init_image, prompt=mask_prompt).latents - >>> image = pipeline(prompt=prompt, mask_image=mask_image, image_latents=image_latents).images[0] + >>> mask_image = pipe.generate_mask(image=init_image, source_prompt=prompt, target_prompt=mask_prompt) + >>> image_latents = pipe.invert(image=init_image, prompt=mask_prompt).latents + >>> image = pipe(prompt=prompt, mask_image=mask_image, image_latents=image_latents).images[0] ``` """ @@ -121,9 +122,10 @@ class DiffEditInversionPipelineOutput(BaseOutput): >>> init_image = download_image(img_url).resize((768, 768)) - >>> pipeline = StableDiffusionDiffEditPipeline.from_pretrained( + >>> pipe = StableDiffusionDiffEditPipeline.from_pretrained( ... "stabilityai/stable-diffusion-2-1", torch_dtype=torch.float16 ... ) + >>> pipe = pipe.to("cuda") >>> pipeline.scheduler = DDIMScheduler.from_config(pipeline.scheduler.config) >>> pipeline.inverse_scheduler = DDIMInverseScheduler.from_config(pipeline.scheduler.config) @@ -131,7 +133,7 @@ class DiffEditInversionPipelineOutput(BaseOutput): >>> prompt = "A bowl of fruits" - >>> inverted_latents = pipeline.invert(image=init_image, prompt=prompt).latents + >>> inverted_latents = pipe.invert(image=init_image, prompt=prompt).latents ``` """