From 11547f0418ff0787169bdbfeb1bb9ce6e6abfe51 Mon Sep 17 00:00:00 2001 From: "G.O.D" <32255912+gameofdimension@users.noreply.github.com> Date: Mon, 29 Jul 2024 20:53:28 +0800 Subject: [PATCH 1/3] Update train_controlnet.py reduce float value error for bfloat16 --- examples/controlnet/train_controlnet.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/controlnet/train_controlnet.py b/examples/controlnet/train_controlnet.py index b4efe0bd574a..3acbaf6a0d9b 100644 --- a/examples/controlnet/train_controlnet.py +++ b/examples/controlnet/train_controlnet.py @@ -1048,7 +1048,8 @@ def load_model_hook(models, input_dir): # Add noise to the latents according to the noise magnitude at each timestep # (this is the forward diffusion process) - noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps) + noisy_latents = noise_scheduler.add_noise( + latents.float(), noise.float(), timesteps).to(dtype=weight_dtype) # Get the text embedding for conditioning encoder_hidden_states = text_encoder(batch["input_ids"], return_dict=False)[0] From eff5bcdb941f6cbdedd8d13ca5a59b9d762b0941 Mon Sep 17 00:00:00 2001 From: "G.O.D" <32255912+gameofdimension@users.noreply.github.com> Date: Mon, 29 Jul 2024 20:54:23 +0800 Subject: [PATCH 2/3] Update train_controlnet_sdxl.py --- examples/controlnet/train_controlnet_sdxl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/controlnet/train_controlnet_sdxl.py b/examples/controlnet/train_controlnet_sdxl.py index dee79bf9f190..e495e87c79cc 100644 --- a/examples/controlnet/train_controlnet_sdxl.py +++ b/examples/controlnet/train_controlnet_sdxl.py @@ -1210,7 +1210,8 @@ def compute_embeddings(batch, proportion_empty_prompts, text_encoders, tokenizer # Add noise to the latents according to the noise magnitude at each timestep # (this is the forward diffusion process) - noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps) + noisy_latents = noise_scheduler.add_noise( + latents.float(), noise.float(), timesteps).to(dtype=weight_dtype) # ControlNet conditioning. controlnet_image = batch["conditioning_pixel_values"].to(dtype=weight_dtype) From d761630ae664e759da82401aba32f3a83556f50b Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 22 Oct 2024 01:11:20 +0200 Subject: [PATCH 3/3] style --- examples/controlnet/train_controlnet.py | 5 +++-- examples/controlnet/train_controlnet_sdxl.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/controlnet/train_controlnet.py b/examples/controlnet/train_controlnet.py index 53b42190a419..eaeb697c64c0 100644 --- a/examples/controlnet/train_controlnet.py +++ b/examples/controlnet/train_controlnet.py @@ -1048,8 +1048,9 @@ def load_model_hook(models, input_dir): # Add noise to the latents according to the noise magnitude at each timestep # (this is the forward diffusion process) - noisy_latents = noise_scheduler.add_noise( - latents.float(), noise.float(), timesteps).to(dtype=weight_dtype) + noisy_latents = noise_scheduler.add_noise(latents.float(), noise.float(), timesteps).to( + dtype=weight_dtype + ) # Get the text embedding for conditioning encoder_hidden_states = text_encoder(batch["input_ids"], return_dict=False)[0] diff --git a/examples/controlnet/train_controlnet_sdxl.py b/examples/controlnet/train_controlnet_sdxl.py index c0c54cb6ba3d..ae627bb3a04c 100644 --- a/examples/controlnet/train_controlnet_sdxl.py +++ b/examples/controlnet/train_controlnet_sdxl.py @@ -1210,8 +1210,9 @@ def compute_embeddings(batch, proportion_empty_prompts, text_encoders, tokenizer # Add noise to the latents according to the noise magnitude at each timestep # (this is the forward diffusion process) - noisy_latents = noise_scheduler.add_noise( - latents.float(), noise.float(), timesteps).to(dtype=weight_dtype) + noisy_latents = noise_scheduler.add_noise(latents.float(), noise.float(), timesteps).to( + dtype=weight_dtype + ) # ControlNet conditioning. controlnet_image = batch["conditioning_pixel_values"].to(dtype=weight_dtype)