From eb8f2dbcb3672cc116ef077cfa81ce1e5da39a4f Mon Sep 17 00:00:00 2001 From: Shahmatov Arseniy <62886550+cene555@users.noreply.github.com> Date: Wed, 12 Jul 2023 03:45:40 +0300 Subject: [PATCH 01/38] Add files via upload --- examples/tune_decoder.py | 616 ++++++++++++++++++++++++++++++++++ examples/tune_decoder_lora.py | 582 ++++++++++++++++++++++++++++++++ examples/tune_prior.py | 610 +++++++++++++++++++++++++++++++++ examples/tune_prior_lora.py | 573 +++++++++++++++++++++++++++++++ 4 files changed, 2381 insertions(+) create mode 100644 examples/tune_decoder.py create mode 100644 examples/tune_decoder_lora.py create mode 100644 examples/tune_prior.py create mode 100644 examples/tune_prior_lora.py diff --git a/examples/tune_decoder.py b/examples/tune_decoder.py new file mode 100644 index 000000000000..6f5794224f85 --- /dev/null +++ b/examples/tune_decoder.py @@ -0,0 +1,616 @@ +import argparse +import os +import numpy as np +from PIL import Image, ImageOps +from tqdm import tqdm +import pandas as pd +import math +from packaging import version + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import accelerate +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from torchvision import transforms +import transformers +from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection +from transformers.utils import ContextManagers + +import diffusers +from diffusers import DDPMScheduler, UNet2DConditionModel, VQModel +from diffusers.optimization import get_scheduler +from diffusers.training_utils import EMAModel +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.models.attention_processor import LoRAAttnProcessor + +if is_wandb_available(): + import wandb + +logger = get_logger(__name__, log_level="INFO") + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of finetuning Kandinsky 2.2.") + parser.add_argument( + "--image_resolution", + type=int, + default=512, + required=False, + help="Image resolution", + ) + parser.add_argument( + "--pretrained_kandinsky_path", + type=str, + default='kandinsky-community/kandinsky-2-2-decoder', + required=False, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--pretrained_vae_path", + type=str, + default='kandinsky-community/kandinsky-2-2-decoder', + required=False, + help="Path to pretrained vae.", + ) + parser.add_argument( + "--pretrained_image_encoder", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to pretrained image encoder.", + ) + parser.add_argument( + "--scheduler_path", + type=str, + default='kandinsky-community/kandinsky-2-2-decoder', + required=False, + help="Path to scheduler.", + ) + parser.add_argument( + "--image_processor_path", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to image_processor.", + ) + parser.add_argument( + "--train_image_folder", + type=str, + default=None, + required=False, + help="Path to train image folder.", + ) + parser.add_argument( + "--train_images_paths_csv", + type=str, + default=None, + required=False, + help="Path to csv with train images paths(define train_image_folder or train_images_paths_csv) with column paths.", + ) + parser.add_argument( + "--val_image_folder", + type=str, + default=None, + required=False, + help="Path to val image folder.", + ) + parser.add_argument( + "--val_images_paths_csv", + type=str, + default=None, + required=False, + help="Path to csv with val images paths with column paths.", + ) + parser.add_argument( + "--train_batch_size", + type=int, + default=1, + required=False, + help="train batch size", + ) + parser.add_argument( + "--val_batch_size", + type=int, + default=1, + required=False, + help="val batch size", + ) + parser.add_argument( + "--lr", + type=float, + default=1e-4, + required=False, + help="learning rate", + ) + parser.add_argument( + "--weight_decay", + type=float, + default=0.0, + required=False, + help="weight decay", + ) + parser.add_argument( + "--num_epochs", + type=int, + default=0.0, + required=False, + help="num of epochs", + ) + parser.add_argument( + "--output_dir", + type=str, + default="kandi_2_2-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--snr_gamma", + type=float, + default=None, + help="SNR weighting gamma to be used if rebalancing the loss. Recommended value is 5.0. " + "More details here: https://arxiv.org/abs/2303.09556.", + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=( + "Max number of checkpoints to store. Passed as `total_limit` to the `Accelerator` `ProjectConfiguration`." + " See Accelerator::save_state https://huggingface.co/docs/accelerate/package_reference/accelerator#accelerate.Accelerator.save_state" + " for more docs" + ), + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # Sanity checks + if args.train_image_folder is None and args.train_images_paths_csv is None: + raise ValueError("Need either a train_image_folder or a train_images_paths_csv.") + + return args + +def center_crop(image): + width, height = image.size + new_size = min(width, height) + left = (width - new_size) / 2 + top = (height - new_size) / 2 + right = (width + new_size) / 2 + bottom = (height + new_size) / 2 + return image.crop((left, top, right, bottom)) + +class ImageDataset(torch.utils.data.Dataset): + def __init__(self, image_folder=None, images_paths_csv=None, image_processor=None, img_size=512): + assert image_folder is None or images_paths_csv is None + self.image_processor = image_processor + self.img_size = img_size + if images_paths_csv is not None: + self.paths = pd.read_csv(images_paths_csv)['paths'].values + else: + self.paths = [os.path.join(image_folder, path) for path in os.listdir(image_folder) if '.jpg' in path.lower() or '.png' in path.lower()] + + def __len__(self): + return len(self.paths) + + def __getitem__(self, i): + img = Image.open(self.paths[i]) + clip_image = self.image_processor(img) + img = center_crop(img) + img = img.resize((self.img_size, self.img_size), resample=Image.BICUBIC, + reducing_gap=1) + img = np.array(img.convert("RGB")) + img = img.astype(np.float32) / 127.5 - 1 + return np.transpose(img, [2, 0, 1]), clip_image.pixel_values[0] + +def main(): + args = parse_args() + logging_dir = os.path.join(args.output_dir, args.logging_dir) + accelerator_project_config = ProjectConfiguration( + total_limit=args.checkpoints_total_limit, project_dir=args.output_dir, logging_dir=logging_dir + ) + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + if args.seed is not None: + set_seed(args.seed) + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + noise_scheduler = DDPMScheduler.from_pretrained(args.scheduler_path, subfolder="scheduler") + image_processor = CLIPImageProcessor.from_pretrained(args.image_processor_path, subfolder='image_processor') + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + vae = VQModel.from_pretrained(args.pretrained_vae_path, subfolder='movq', torch_dtype=weight_dtype).eval() + image_encoder = CLIPVisionModelWithProjection.from_pretrained(args.pretrained_image_encoder, subfolder='image_encoder', torch_dtype=weight_dtype).eval() + print('args.pretrained_kandinsky_path =', args.pretrained_kandinsky_path) + unet = UNet2DConditionModel.from_pretrained( + args.pretrained_kandinsky_path, subfolder="unet" + ) + + # Freeze vae and image_encoder + vae.requires_grad_(False) + image_encoder.requires_grad_(False) + + # Create EMA for the unet. + if args.use_ema: + ema_unet = UNet2DConditionModel.from_pretrained( + args.pretrained_kandinsky_path, subfolder="unet" + ) + ema_unet = EMAModel(ema_unet.parameters(), model_cls=UNet2DConditionModel, model_config=ema_unet.config) + ema_unet.to(accelerator.device) + def compute_snr(timesteps): + """ + Computes SNR as per https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L847-L849 + """ + alphas_cumprod = noise_scheduler.alphas_cumprod + sqrt_alphas_cumprod = alphas_cumprod**0.5 + sqrt_one_minus_alphas_cumprod = (1.0 - alphas_cumprod) ** 0.5 + + # Expand the tensors. + # Adapted from https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L1026 + sqrt_alphas_cumprod = sqrt_alphas_cumprod.to(device=timesteps.device)[timesteps].float() + while len(sqrt_alphas_cumprod.shape) < len(timesteps.shape): + sqrt_alphas_cumprod = sqrt_alphas_cumprod[..., None] + alpha = sqrt_alphas_cumprod.expand(timesteps.shape) + + sqrt_one_minus_alphas_cumprod = sqrt_one_minus_alphas_cumprod.to(device=timesteps.device)[timesteps].float() + while len(sqrt_one_minus_alphas_cumprod.shape) < len(timesteps.shape): + sqrt_one_minus_alphas_cumprod = sqrt_one_minus_alphas_cumprod[..., None] + sigma = sqrt_one_minus_alphas_cumprod.expand(timesteps.shape) + + # Compute SNR. + snr = (alpha / sigma) ** 2 + return snr + + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if args.use_ema: + ema_unet.save_pretrained(os.path.join(output_dir, "unet_ema")) + + for i, model in enumerate(models): + model.save_pretrained(os.path.join(output_dir, "unet")) + + # make sure to pop weight so that corresponding model is not saved again + weights.pop() + + def load_model_hook(models, input_dir): + if args.use_ema: + load_model = EMAModel.from_pretrained(os.path.join(input_dir, "unet_ema"), UNet2DConditionModel) + ema_unet.load_state_dict(load_model.state_dict()) + ema_unet.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = UNet2DConditionModel.from_pretrained(input_dir, subfolder="unet") + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + unet.enable_gradient_checkpointing() + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + else: + optimizer_cls = torch.optim.AdamW + optimizer = optimizer_cls( + unet.parameters(), + lr=args.lr, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.weight_decay, + eps=args.adam_epsilon, + ) + + train_dataset = ImageDataset(image_folder=args.train_image_folder, images_paths_csv=args.train_images_paths_csv, image_processor=image_processor, img_size=args.image_resolution) + train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=args.train_batch_size, num_workers=args.dataloader_num_workers) + if args.val_image_folder is not None or args.val_images_paths_csv is not None: + do_val = True + val_dataset = ImageDataset(image_folder=args.val_image_folder, images_paths_csv=args.val_images_paths_csv, image_processor=image_processor, img_size=args.image_resolution) + val_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=args.val_batch_size, num_workers=args.dataloader_num_workers) + else: + do_val = False + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * args.gradient_accumulation_steps, + num_training_steps=args.max_train_steps * args.gradient_accumulation_steps, + ) + unet, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + unet, optimizer, train_dataloader, lr_scheduler + ) + + image_encoder.to(accelerator.device, dtype=weight_dtype) + vae.to(accelerator.device, dtype=weight_dtype) + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + accelerator.init_trackers('test', tracker_config)#args.tracker_project_name + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + else: + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + global_step = int(path.split("-")[1]) + + resume_global_step = global_step * args.gradient_accumulation_steps + first_epoch = global_step // num_update_steps_per_epoch + resume_step = resume_global_step % (num_update_steps_per_epoch * args.gradient_accumulation_steps) + + progress_bar = tqdm(range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process) + progress_bar.set_description("training goes brrr") + for epoch in range(first_epoch, args.num_train_epochs): + unet.train() + train_loss = 0.0 + for step, batch in enumerate(train_dataloader): + # Skip steps until we reach the resumed step + if args.resume_from_checkpoint and epoch == first_epoch and step < resume_step: + if step % args.gradient_accumulation_steps == 0: + progress_bar.update(1) + continue + + with accelerator.accumulate(unet): + # Convert images to latent space + images, clip_images = batch + images, clip_images = images.to(weight_dtype), clip_images.to(weight_dtype) + latents = vae.encode(images).latents + image_embeds = image_encoder(clip_images).image_embeds + # Sample noise that we'll add to the latents + noise = torch.randn_like(latents) + bsz = latents.shape[0] + # Sample a random timestep for each image + timesteps = torch.randint(0, noise_scheduler.config.num_train_timesteps, (bsz,), device=latents.device) + timesteps = timesteps.long() + + noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps) + + target = noise + + # Predict the noise residual and compute loss + added_cond_kwargs = {"image_embeds": image_embeds} + + model_pred = unet(noisy_latents, timesteps, None, added_cond_kwargs=added_cond_kwargs).sample[:, :4] + + if args.snr_gamma is None: + loss = F.mse_loss(model_pred.float(), target.float(), reduction="mean") + else: + # Compute loss-weights as per Section 3.4 of https://arxiv.org/abs/2303.09556. + # Since we predict the noise instead of x_0, the original formulation is slightly changed. + # This is discussed in Section 4.2 of the same paper. + snr = compute_snr(timesteps) + mse_loss_weights = ( + torch.stack([snr, args.snr_gamma * torch.ones_like(timesteps)], dim=1).min(dim=1)[0] / snr + ) + # We first calculate the original loss. Then we mean over the non-batch dimensions and + # rebalance the sample-wise losses with their respective loss weights. + # Finally, we take the mean of the rebalanced loss. + loss = F.mse_loss(model_pred.float(), target.float(), reduction="none") + loss = loss.mean(dim=list(range(1, len(loss.shape)))) * mse_loss_weights + loss = loss.mean() + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(unet.parameters(), args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + if args.use_ema: + ema_unet.step(unet.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if accelerator.is_main_process: + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + accelerator.end_training() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/examples/tune_decoder_lora.py b/examples/tune_decoder_lora.py new file mode 100644 index 000000000000..3af257d4a53d --- /dev/null +++ b/examples/tune_decoder_lora.py @@ -0,0 +1,582 @@ +import argparse +import os +import numpy as np +from PIL import Image, ImageOps +from tqdm import tqdm +import pandas as pd +import math +from packaging import version + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import accelerate +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from torchvision import transforms +import transformers +from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection +from transformers.utils import ContextManagers + +import diffusers +from diffusers import DDPMScheduler, UNet2DConditionModel, VQModel +from diffusers.optimization import get_scheduler +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.models.attention_processor import LoRAAttnProcessor, LoRAAttnAddedKVProcessor +from diffusers.loaders import AttnProcsLayers + +if is_wandb_available(): + import wandb + +logger = get_logger(__name__, log_level="INFO") + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of finetuning Kandinsky 2.2.") + parser.add_argument( + "--image_resolution", + type=int, + default=512, + required=False, + help="Image resolution", + ) + parser.add_argument( + "--pretrained_kandinsky_path", + type=str, + default='kandinsky-community/kandinsky-2-2-decoder', + required=False, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--pretrained_vae_path", + type=str, + default='kandinsky-community/kandinsky-2-2-decoder', + required=False, + help="Path to pretrained vae.", + ) + parser.add_argument( + "--pretrained_image_encoder", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to pretrained image encoder.", + ) + parser.add_argument( + "--scheduler_path", + type=str, + default='kandinsky-community/kandinsky-2-2-decoder', + required=False, + help="Path to scheduler.", + ) + parser.add_argument( + "--image_processor_path", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to image_processor.", + ) + parser.add_argument( + "--train_image_folder", + type=str, + default=None, + required=False, + help="Path to train image folder.", + ) + parser.add_argument( + "--train_images_paths_csv", + type=str, + default=None, + required=False, + help="Path to csv with train images paths(define train_image_folder or train_images_paths_csv) with column paths.", + ) + parser.add_argument( + "--val_image_folder", + type=str, + default=None, + required=False, + help="Path to val image folder.", + ) + parser.add_argument( + "--val_images_paths_csv", + type=str, + default=None, + required=False, + help="Path to csv with val images paths with column paths.", + ) + parser.add_argument( + "--train_batch_size", + type=int, + default=1, + required=False, + help="train batch size", + ) + parser.add_argument( + "--val_batch_size", + type=int, + default=1, + required=False, + help="val batch size", + ) + parser.add_argument( + "--lr", + type=float, + default=1e-4, + required=False, + help="learning rate", + ) + parser.add_argument( + "--weight_decay", + type=float, + default=0.0, + required=False, + help="weight decay", + ) + parser.add_argument( + "--num_epochs", + type=int, + default=0.0, + required=False, + help="num of epochs", + ) + parser.add_argument( + "--output_dir", + type=str, + default="kandi_2_2-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--snr_gamma", + type=float, + default=None, + help="SNR weighting gamma to be used if rebalancing the loss. Recommended value is 5.0. " + "More details here: https://arxiv.org/abs/2303.09556.", + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=( + "Max number of checkpoints to store. Passed as `total_limit` to the `Accelerator` `ProjectConfiguration`." + " See Accelerator::save_state https://huggingface.co/docs/accelerate/package_reference/accelerator#accelerate.Accelerator.save_state" + " for more docs" + ), + ) + parser.add_argument("--rank", type=int, default=4, help="A rank of LORA") + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # Sanity checks + if args.train_image_folder is None and args.train_images_paths_csv is None: + raise ValueError("Need either a train_image_folder or a train_images_paths_csv.") + + return args + +def center_crop(image): + width, height = image.size + new_size = min(width, height) + left = (width - new_size) / 2 + top = (height - new_size) / 2 + right = (width + new_size) / 2 + bottom = (height + new_size) / 2 + return image.crop((left, top, right, bottom)) + +class ImageDataset(torch.utils.data.Dataset): + def __init__(self, image_folder=None, images_paths_csv=None, image_processor=None, img_size=512): + assert image_folder is None or images_paths_csv is None + self.image_processor = image_processor + self.img_size = img_size + if images_paths_csv is not None: + self.paths = pd.read_csv(images_paths_csv)['paths'].values + else: + self.paths = [os.path.join(image_folder, path) for path in os.listdir(image_folder) if '.jpg' in path.lower() or '.png' in path.lower()] + + def __len__(self): + return len(self.paths) + + def __getitem__(self, i): + img = Image.open(self.paths[i]) + clip_image = self.image_processor(img) + img = center_crop(img) + img = img.resize((self.img_size, self.img_size), resample=Image.BICUBIC, + reducing_gap=1) + img = np.array(img.convert("RGB")) + img = img.astype(np.float32) / 127.5 - 1 + return np.transpose(img, [2, 0, 1]), clip_image.pixel_values[0] + +def main(): + args = parse_args() + logging_dir = os.path.join(args.output_dir, args.logging_dir) + accelerator_project_config = ProjectConfiguration( + total_limit=args.checkpoints_total_limit, project_dir=args.output_dir, logging_dir=logging_dir + ) + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + if args.seed is not None: + set_seed(args.seed) + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + noise_scheduler = DDPMScheduler.from_pretrained(args.scheduler_path, subfolder="scheduler") + image_processor = CLIPImageProcessor.from_pretrained(args.image_processor_path, subfolder='image_processor') + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + vae = VQModel.from_pretrained(args.pretrained_vae_path, subfolder='movq', torch_dtype=weight_dtype).eval() + image_encoder = CLIPVisionModelWithProjection.from_pretrained(args.pretrained_image_encoder, subfolder='image_encoder', torch_dtype=weight_dtype).eval() + + unet = UNet2DConditionModel.from_pretrained( + args.pretrained_kandinsky_path, subfolder="unet", torch_dtype=weight_dtype + ) + + # Freeze vae and image_encoder + vae.requires_grad_(False) + image_encoder.requires_grad_(False) + unet.requires_grad_(False) + image_encoder.to(accelerator.device) + vae.to(accelerator.device) + unet.to(accelerator.device) + lora_attn_procs = {} + for name in unet.attn_processors.keys(): + cross_attention_dim = None if name.endswith("attn1.processor") else unet.config.cross_attention_dim + if name.startswith("mid_block"): + hidden_size = unet.config.block_out_channels[-1] + elif name.startswith("up_blocks"): + block_id = int(name[len("up_blocks.")]) + hidden_size = list(reversed(unet.config.block_out_channels))[block_id] + elif name.startswith("down_blocks"): + block_id = int(name[len("down_blocks.")]) + hidden_size = unet.config.block_out_channels[block_id] + lora_attn_procs[name] = LoRAAttnAddedKVProcessor( + hidden_size=hidden_size, + cross_attention_dim=cross_attention_dim, + rank=args.rank, + ) + + unet.set_attn_processor(lora_attn_procs) + def compute_snr(timesteps): + """ + Computes SNR as per https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L847-L849 + """ + alphas_cumprod = noise_scheduler.alphas_cumprod + sqrt_alphas_cumprod = alphas_cumprod**0.5 + sqrt_one_minus_alphas_cumprod = (1.0 - alphas_cumprod) ** 0.5 + + # Expand the tensors. + # Adapted from https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L1026 + sqrt_alphas_cumprod = sqrt_alphas_cumprod.to(device=timesteps.device)[timesteps].float() + while len(sqrt_alphas_cumprod.shape) < len(timesteps.shape): + sqrt_alphas_cumprod = sqrt_alphas_cumprod[..., None] + alpha = sqrt_alphas_cumprod.expand(timesteps.shape) + + sqrt_one_minus_alphas_cumprod = sqrt_one_minus_alphas_cumprod.to(device=timesteps.device)[timesteps].float() + while len(sqrt_one_minus_alphas_cumprod.shape) < len(timesteps.shape): + sqrt_one_minus_alphas_cumprod = sqrt_one_minus_alphas_cumprod[..., None] + sigma = sqrt_one_minus_alphas_cumprod.expand(timesteps.shape) + + # Compute SNR. + snr = (alpha / sigma) ** 2 + return snr + lora_layers = AttnProcsLayers(unet.attn_processors) + + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + else: + optimizer_cls = torch.optim.AdamW + optimizer = optimizer_cls( + lora_layers.parameters(), + lr=args.lr, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.weight_decay, + eps=args.adam_epsilon, + ) + + train_dataset = ImageDataset(image_folder=args.train_image_folder, images_paths_csv=args.train_images_paths_csv, image_processor=image_processor, img_size=args.image_resolution) + train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=args.train_batch_size, num_workers=args.dataloader_num_workers) + if args.val_image_folder is not None or args.val_images_paths_csv is not None: + do_val = True + val_dataset = ImageDataset(image_folder=args.val_image_folder, images_paths_csv=args.val_images_paths_csv, image_processor=image_processor, img_size=args.image_resolution) + val_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=args.val_batch_size, num_workers=args.dataloader_num_workers) + else: + do_val = False + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * args.gradient_accumulation_steps, + num_training_steps=args.max_train_steps * args.gradient_accumulation_steps, + ) + lora_layers, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + lora_layers, optimizer, train_dataloader, lr_scheduler + ) + + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + accelerator.init_trackers('test', tracker_config)#args.tracker_project_name + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + else: + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + global_step = int(path.split("-")[1]) + + resume_global_step = global_step * args.gradient_accumulation_steps + first_epoch = global_step // num_update_steps_per_epoch + resume_step = resume_global_step % (num_update_steps_per_epoch * args.gradient_accumulation_steps) + + progress_bar = tqdm(range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process) + progress_bar.set_description("training goes brrr") + for epoch in range(first_epoch, args.num_train_epochs): + unet.train() + train_loss = 0.0 + for step, batch in enumerate(train_dataloader): + # Skip steps until we reach the resumed step + if args.resume_from_checkpoint and epoch == first_epoch and step < resume_step: + if step % args.gradient_accumulation_steps == 0: + progress_bar.update(1) + continue + + with accelerator.accumulate(unet): + # Convert images to latent space + images, clip_images = batch + images, clip_images = images.to(weight_dtype), clip_images.to(weight_dtype) + latents = vae.encode(images).latents + image_embeds = image_encoder(clip_images).image_embeds + # Sample noise that we'll add to the latents + noise = torch.randn_like(latents) + bsz = latents.shape[0] + # Sample a random timestep for each image + timesteps = torch.randint(0, noise_scheduler.config.num_train_timesteps, (bsz,), device=latents.device) + timesteps = timesteps.long() + + noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps) + + target = noise + + # Predict the noise residual and compute loss + added_cond_kwargs = {"image_embeds": image_embeds} + + model_pred = unet(noisy_latents, timesteps, None, added_cond_kwargs=added_cond_kwargs).sample[:, :4] + + if args.snr_gamma is None: + loss = F.mse_loss(model_pred.float(), target.float(), reduction="mean") + else: + # Compute loss-weights as per Section 3.4 of https://arxiv.org/abs/2303.09556. + # Since we predict the noise instead of x_0, the original formulation is slightly changed. + # This is discussed in Section 4.2 of the same paper. + snr = compute_snr(timesteps) + mse_loss_weights = ( + torch.stack([snr, args.snr_gamma * torch.ones_like(timesteps)], dim=1).min(dim=1)[0] / snr + ) + # We first calculate the original loss. Then we mean over the non-batch dimensions and + # rebalance the sample-wise losses with their respective loss weights. + # Finally, we take the mean of the rebalanced loss. + loss = F.mse_loss(model_pred.float(), target.float(), reduction="none") + loss = loss.mean(dim=list(range(1, len(loss.shape)))) * mse_loss_weights + loss = loss.mean() + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + params_to_clip = lora_layers.parameters() + accelerator.clip_grad_norm_(params_to_clip, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if accelerator.is_main_process: + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + accelerator.end_training() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/examples/tune_prior.py b/examples/tune_prior.py new file mode 100644 index 000000000000..18b48116b0bf --- /dev/null +++ b/examples/tune_prior.py @@ -0,0 +1,610 @@ +import argparse +import os +import numpy as np +from PIL import Image, ImageOps +from tqdm import tqdm +import pandas as pd +import math +from packaging import version + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import accelerate +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from torchvision import transforms +import transformers +from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection +from transformers.utils import ContextManagers + +import diffusers +from diffusers import PriorTransformer, UnCLIPScheduler, DDPMScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import EMAModel +from diffusers.utils import check_min_version, deprecate, is_wandb_available + +if is_wandb_available(): + import wandb + +logger = get_logger(__name__, log_level="INFO") + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of finetuning Kandinsky 2.2.") + parser.add_argument( + "--pretrained_prior_path", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to pretrained prior model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--pretrained_image_encoder", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to pretrained image encoder.", + ) + parser.add_argument( + "--scheduler_path", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to scheduler.", + ) + parser.add_argument( + "--image_processor_path", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to image_processor.", + ) + parser.add_argument( + "--text_encoder_path", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to text_encoder.", + ) + parser.add_argument( + "--tokenizer_path", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to tokenizer.", + ) + parser.add_argument( + "--train_images_paths_csv", + type=str, + default=None, + required=False, + help="Path to csv with train images paths with column paths and caption.", + ) + parser.add_argument( + "--val_images_paths_csv", + type=str, + default=None, + required=False, + help="Path to csv with val images paths with column paths and caption.", + ) + parser.add_argument( + "--train_batch_size", + type=int, + default=1, + required=False, + help="train batch size", + ) + parser.add_argument( + "--val_batch_size", + type=int, + default=1, + required=False, + help="val batch size", + ) + parser.add_argument( + "--lr", + type=float, + default=1e-4, + required=False, + help="learning rate", + ) + parser.add_argument( + "--weight_decay", + type=float, + default=0.0, + required=False, + help="weight decay", + ) + parser.add_argument( + "--num_epochs", + type=int, + default=0.0, + required=False, + help="num of epochs", + ) + parser.add_argument( + "--output_dir", + type=str, + default="kandi_2_2-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--snr_gamma", + type=float, + default=None, + help="SNR weighting gamma to be used if rebalancing the loss. Recommended value is 5.0. " + "More details here: https://arxiv.org/abs/2303.09556.", + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=( + "Max number of checkpoints to store. Passed as `total_limit` to the `Accelerator` `ProjectConfiguration`." + " See Accelerator::save_state https://huggingface.co/docs/accelerate/package_reference/accelerator#accelerate.Accelerator.save_state" + " for more docs" + ), + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # Sanity checks + if args.train_images_paths_csv is None: + raise ValueError("Need either a train_image_folder or a train_images_paths_csv.") + + return args + + +class ImageDataset(torch.utils.data.Dataset): + def __init__(self, images_paths_csv=None, image_processor=None, tokenizer=None): + self.image_processor = image_processor + self.tokenizer = tokenizer + df = pd.read_csv(images_paths_csv) + self.paths = df['paths'].values + self.captions = df['caption'].values + def __len__(self): + return len(self.paths) + + def __getitem__(self, i): + img = Image.open(self.paths[i]) + clip_image = self.image_processor(img) + text_inputs = self.tokenizer( + self.captions[i], + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids[0] + text_mask = text_inputs.attention_mask.bool()[0] + + return text_input_ids, text_mask, clip_image.pixel_values[0] + +def main(): + args = parse_args() + logging_dir = os.path.join(args.output_dir, args.logging_dir) + accelerator_project_config = ProjectConfiguration( + total_limit=args.checkpoints_total_limit, project_dir=args.output_dir, logging_dir=logging_dir + ) + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + if args.seed is not None: + set_seed(args.seed) + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + noise_scheduler = DDPMScheduler(beta_schedule='squaredcos_cap_v2', prediction_type='sample') + image_processor = CLIPImageProcessor.from_pretrained(args.image_processor_path, subfolder='image_processor') + tokenizer = CLIPTokenizer.from_pretrained(args.tokenizer_path, subfolder='tokenizer') + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + image_encoder = CLIPVisionModelWithProjection.from_pretrained(args.pretrained_image_encoder, subfolder='image_encoder', torch_dtype=weight_dtype).eval() + text_encoder = CLIPTextModelWithProjection.from_pretrained(args.text_encoder_path, subfolder='text_encoder', torch_dtype=weight_dtype).eval() + print('args.pretrained_prior_path =', args.pretrained_prior_path) + prior = PriorTransformer.from_pretrained( + args.pretrained_prior_path, subfolder="prior" + ) + + # Freeze text_encoder and image_encoder + text_encoder.requires_grad_(False) + image_encoder.requires_grad_(False) + + # Create EMA for the prior. + if args.use_ema: + ema_prior = PriorTransformer.from_pretrained( + args.pretrained_prior_path, subfolder="prior" + ) + ema_prior = EMAModel(ema_prior.parameters(), model_cls=PriorTransformer, model_config=ema_prior.config) + ema_prior.to(accelerator.device) + def compute_snr(timesteps): + """ + Computes SNR as per https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L847-L849 + """ + alphas_cumprod = noise_scheduler.alphas_cumprod + sqrt_alphas_cumprod = alphas_cumprod**0.5 + sqrt_one_minus_alphas_cumprod = (1.0 - alphas_cumprod) ** 0.5 + + # Expand the tensors. + # Adapted from https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L1026 + sqrt_alphas_cumprod = sqrt_alphas_cumprod.to(device=timesteps.device)[timesteps].float() + while len(sqrt_alphas_cumprod.shape) < len(timesteps.shape): + sqrt_alphas_cumprod = sqrt_alphas_cumprod[..., None] + alpha = sqrt_alphas_cumprod.expand(timesteps.shape) + + sqrt_one_minus_alphas_cumprod = sqrt_one_minus_alphas_cumprod.to(device=timesteps.device)[timesteps].float() + while len(sqrt_one_minus_alphas_cumprod.shape) < len(timesteps.shape): + sqrt_one_minus_alphas_cumprod = sqrt_one_minus_alphas_cumprod[..., None] + sigma = sqrt_one_minus_alphas_cumprod.expand(timesteps.shape) + + # Compute SNR. + snr = (alpha / sigma) ** 2 + return snr + + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if args.use_ema: + ema_prior.save_pretrained(os.path.join(output_dir, "prior_ema")) + + for i, model in enumerate(models): + model.save_pretrained(os.path.join(output_dir, "prior")) + + # make sure to pop weight so that corresponding model is not saved again + weights.pop() + + def load_model_hook(models, input_dir): + if args.use_ema: + load_model = EMAModel.from_pretrained(os.path.join(input_dir, "prior_ema"), PriorTransformer) + ema_prior.load_state_dict(load_model.state_dict()) + ema_prior.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = PriorTransformer.from_pretrained(input_dir, subfolder="prior") + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + else: + optimizer_cls = torch.optim.AdamW + optimizer = optimizer_cls( + prior.parameters(), + lr=args.lr, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.weight_decay, + eps=args.adam_epsilon, + ) + + train_dataset = ImageDataset(images_paths_csv=args.train_images_paths_csv, image_processor=image_processor, tokenizer=tokenizer) + train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=args.train_batch_size, num_workers=args.dataloader_num_workers) + if args.val_images_paths_csv is not None: + do_val = True + val_dataset = ImageDataset(images_paths_csv=args.val_images_paths_csv, image_processor=image_processor, tokenizer=tokenizer) + val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=args.val_batch_size, num_workers=args.dataloader_num_workers) + else: + do_val = False + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * args.gradient_accumulation_steps, + num_training_steps=args.max_train_steps * args.gradient_accumulation_steps, + ) + clip_mean = prior.clip_mean + clip_std = prior.clip_std + prior.clip_mean = None + prior.clip_std = None + prior, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + prior, optimizer, train_dataloader, lr_scheduler + ) + + image_encoder.to(accelerator.device, dtype=weight_dtype) + text_encoder.to(accelerator.device, dtype=weight_dtype) + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + accelerator.init_trackers('test', tracker_config)#args.tracker_project_name + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + else: + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + global_step = int(path.split("-")[1]) + + resume_global_step = global_step * args.gradient_accumulation_steps + first_epoch = global_step // num_update_steps_per_epoch + resume_step = resume_global_step % (num_update_steps_per_epoch * args.gradient_accumulation_steps) + + progress_bar = tqdm(range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process) + progress_bar.set_description("training goes brrr") + clip_mean = clip_mean.to(weight_dtype).to(accelerator.device) + clip_std = clip_std.to(weight_dtype).to(accelerator.device) + for epoch in range(first_epoch, args.num_train_epochs): + prior.train() + train_loss = 0.0 + for step, batch in enumerate(train_dataloader): + # Skip steps until we reach the resumed step + if args.resume_from_checkpoint and epoch == first_epoch and step < resume_step: + if step % args.gradient_accumulation_steps == 0: + progress_bar.update(1) + continue + + with accelerator.accumulate(prior): + # Convert images to latent space + text_input_ids, text_mask, clip_images = batch + text_input_ids, text_mask, clip_images = text_input_ids, text_mask, clip_images.to(weight_dtype) + with torch.no_grad(): + text_encoder_output = text_encoder(text_input_ids) + prompt_embeds = text_encoder_output.text_embeds + text_encoder_hidden_states = text_encoder_output.last_hidden_state + + + image_embeds = image_encoder(clip_images).image_embeds + # Sample noise that we'll add to the image_embeds + noise = torch.randn_like(image_embeds) + bsz = image_embeds.shape[0] + # Sample a random timestep for each image + timesteps = torch.randint(0, noise_scheduler.config.num_train_timesteps, (bsz,), device=image_embeds.device) + timesteps = timesteps.long() + image_embeds = (image_embeds - clip_mean) / clip_std + noisy_latents = noise_scheduler.add_noise(image_embeds, noise, timesteps) + + target = image_embeds + + # Predict the noise residual and compute loss + model_pred = prior( + noisy_latents, + timestep=timesteps, + proj_embedding=prompt_embeds, + encoder_hidden_states=text_encoder_hidden_states, + attention_mask=text_mask, + ).predicted_image_embedding + + if args.snr_gamma is None: + loss = F.mse_loss(model_pred.float(), target.float(), reduction="mean") + else: + # Compute loss-weights as per Section 3.4 of https://arxiv.org/abs/2303.09556. + # Since we predict the noise instead of x_0, the original formulation is slightly changed. + # This is discussed in Section 4.2 of the same paper. + snr = compute_snr(timesteps) + mse_loss_weights = ( + torch.stack([snr, args.snr_gamma * torch.ones_like(timesteps)], dim=1).min(dim=1)[0] / snr + ) + # We first calculate the original loss. Then we mean over the non-batch dimensions and + # rebalance the sample-wise losses with their respective loss weights. + # Finally, we take the mean of the rebalanced loss. + loss = F.mse_loss(model_pred.float(), target.float(), reduction="none") + loss = loss.mean(dim=list(range(1, len(loss.shape)))) * mse_loss_weights + loss = loss.mean() + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(prior.parameters(), args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + if args.use_ema: + ema_prior.step(prior.parameters()) + progress_bar.update(1) + global_step += 1 + #print(train_loss) + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if accelerator.is_main_process: + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + accelerator.end_training() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/examples/tune_prior_lora.py b/examples/tune_prior_lora.py new file mode 100644 index 000000000000..b7e3414aaf23 --- /dev/null +++ b/examples/tune_prior_lora.py @@ -0,0 +1,573 @@ +import argparse +import os +import numpy as np +from PIL import Image, ImageOps +from tqdm import tqdm +import pandas as pd +import math +from packaging import version + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import accelerate +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from torchvision import transforms +import transformers +from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection +from transformers.utils import ContextManagers + +import diffusers +from diffusers import PriorTransformer, UnCLIPScheduler, DDPMScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import EMAModel +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.models.attention_processor import LoRAAttnProcessor +from diffusers.loaders import AttnProcsLayers +if is_wandb_available(): + import wandb + +logger = get_logger(__name__, log_level="INFO") + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of finetuning Kandinsky 2.2.") + parser.add_argument( + "--pretrained_prior_path", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to pretrained prior model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--pretrained_image_encoder", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to pretrained image encoder.", + ) + parser.add_argument( + "--scheduler_path", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to scheduler.", + ) + parser.add_argument( + "--image_processor_path", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to image_processor.", + ) + parser.add_argument( + "--text_encoder_path", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to text_encoder.", + ) + parser.add_argument( + "--tokenizer_path", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to tokenizer.", + ) + parser.add_argument( + "--train_images_paths_csv", + type=str, + default=None, + required=False, + help="Path to csv with train images paths with column paths and caption.", + ) + parser.add_argument( + "--val_images_paths_csv", + type=str, + default=None, + required=False, + help="Path to csv with val images paths with column paths and caption.", + ) + parser.add_argument( + "--train_batch_size", + type=int, + default=1, + required=False, + help="train batch size", + ) + parser.add_argument( + "--val_batch_size", + type=int, + default=1, + required=False, + help="val batch size", + ) + parser.add_argument( + "--lr", + type=float, + default=1e-4, + required=False, + help="learning rate", + ) + parser.add_argument( + "--weight_decay", + type=float, + default=0.0, + required=False, + help="weight decay", + ) + parser.add_argument( + "--num_epochs", + type=int, + default=0.0, + required=False, + help="num of epochs", + ) + parser.add_argument( + "--output_dir", + type=str, + default="kandi_2_2-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--snr_gamma", + type=float, + default=None, + help="SNR weighting gamma to be used if rebalancing the loss. Recommended value is 5.0. " + "More details here: https://arxiv.org/abs/2303.09556.", + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=( + "Max number of checkpoints to store. Passed as `total_limit` to the `Accelerator` `ProjectConfiguration`." + " See Accelerator::save_state https://huggingface.co/docs/accelerate/package_reference/accelerator#accelerate.Accelerator.save_state" + " for more docs" + ), + ) + parser.add_argument("--rank", type=int, default=4, help="A rank of LORA") + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # Sanity checks + if args.train_images_paths_csv is None: + raise ValueError("Need either a train_image_folder or a train_images_paths_csv.") + + return args + + +class ImageDataset(torch.utils.data.Dataset): + def __init__(self, images_paths_csv=None, image_processor=None, tokenizer=None): + self.image_processor = image_processor + self.tokenizer = tokenizer + df = pd.read_csv(images_paths_csv) + self.paths = df['paths'].values + self.captions = df['caption'].values + def __len__(self): + return len(self.paths) + + def __getitem__(self, i): + img = Image.open(self.paths[i]) + clip_image = self.image_processor(img, return_tensors="pt") + text_inputs = self.tokenizer( + self.captions[i], + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids[0] + text_mask = text_inputs.attention_mask.bool()[0] + + return text_input_ids, text_mask, clip_image.pixel_values[0] + +def main(): + args = parse_args() + logging_dir = os.path.join(args.output_dir, args.logging_dir) + accelerator_project_config = ProjectConfiguration( + total_limit=args.checkpoints_total_limit, project_dir=args.output_dir, logging_dir=logging_dir + ) + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + if args.seed is not None: + set_seed(args.seed) + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + noise_scheduler = DDPMScheduler(beta_schedule='squaredcos_cap_v2', prediction_type='sample') + image_processor = CLIPImageProcessor.from_pretrained(args.image_processor_path, subfolder='image_processor') + tokenizer = CLIPTokenizer.from_pretrained(args.tokenizer_path, subfolder='tokenizer') + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + image_encoder = CLIPVisionModelWithProjection.from_pretrained(args.pretrained_image_encoder, subfolder='image_encoder', torch_dtype=weight_dtype).eval() + text_encoder = CLIPTextModelWithProjection.from_pretrained(args.text_encoder_path, subfolder='text_encoder', torch_dtype=weight_dtype).eval() + print('args.pretrained_prior_path =', args.pretrained_prior_path) + prior = PriorTransformer.from_pretrained( + args.pretrained_prior_path, subfolder="prior", torch_dtype=weight_dtype + ) + + # Freeze text_encoder and image_encoder + text_encoder.requires_grad_(False) + image_encoder.requires_grad_(False) + prior.requires_grad_(False) + image_encoder.to(accelerator.device, dtype=weight_dtype) + text_encoder.to(accelerator.device, dtype=weight_dtype) + prior.to(accelerator.device, dtype=weight_dtype) + lora_attn_procs = {} + for name in prior.attn_processors.keys(): + lora_attn_procs[name] = LoRAAttnProcessor(hidden_size=2048, rank=args.rank) + + prior.set_attn_processor(lora_attn_procs) + def compute_snr(timesteps): + """ + Computes SNR as per https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L847-L849 + """ + alphas_cumprod = noise_scheduler.alphas_cumprod + sqrt_alphas_cumprod = alphas_cumprod**0.5 + sqrt_one_minus_alphas_cumprod = (1.0 - alphas_cumprod) ** 0.5 + + # Expand the tensors. + # Adapted from https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L1026 + sqrt_alphas_cumprod = sqrt_alphas_cumprod.to(device=timesteps.device)[timesteps].float() + while len(sqrt_alphas_cumprod.shape) < len(timesteps.shape): + sqrt_alphas_cumprod = sqrt_alphas_cumprod[..., None] + alpha = sqrt_alphas_cumprod.expand(timesteps.shape) + + sqrt_one_minus_alphas_cumprod = sqrt_one_minus_alphas_cumprod.to(device=timesteps.device)[timesteps].float() + while len(sqrt_one_minus_alphas_cumprod.shape) < len(timesteps.shape): + sqrt_one_minus_alphas_cumprod = sqrt_one_minus_alphas_cumprod[..., None] + sigma = sqrt_one_minus_alphas_cumprod.expand(timesteps.shape) + + # Compute SNR. + snr = (alpha / sigma) ** 2 + return snr + lora_layers = AttnProcsLayers(prior.attn_processors) + + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + else: + optimizer_cls = torch.optim.AdamW + optimizer = optimizer_cls( + lora_layers.parameters(), + lr=args.lr, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.weight_decay, + eps=args.adam_epsilon, + ) + + train_dataset = ImageDataset(images_paths_csv=args.train_images_paths_csv, image_processor=image_processor, tokenizer=tokenizer) + train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=args.train_batch_size, num_workers=args.dataloader_num_workers) + if args.val_images_paths_csv is not None: + do_val = True + val_dataset = ImageDataset(images_paths_csv=args.val_images_paths_csv, image_processor=image_processor, tokenizer=tokenizer) + val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=args.val_batch_size, num_workers=args.dataloader_num_workers) + else: + do_val = False + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * args.gradient_accumulation_steps, + num_training_steps=args.max_train_steps * args.gradient_accumulation_steps, + ) + clip_mean = prior.clip_mean + clip_std = prior.clip_std + lora_layers, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + lora_layers, optimizer, train_dataloader, lr_scheduler + ) + + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + accelerator.init_trackers('test', tracker_config)#args.tracker_project_name + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + else: + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + global_step = int(path.split("-")[1]) + + resume_global_step = global_step * args.gradient_accumulation_steps + first_epoch = global_step // num_update_steps_per_epoch + resume_step = resume_global_step % (num_update_steps_per_epoch * args.gradient_accumulation_steps) + + progress_bar = tqdm(range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process) + progress_bar.set_description("training goes brrr") + clip_mean = clip_mean.to(weight_dtype).to(accelerator.device) + clip_std = clip_std.to(weight_dtype).to(accelerator.device) + for epoch in range(first_epoch, args.num_train_epochs): + prior.train() + train_loss = 0.0 + for step, batch in enumerate(train_dataloader): + # Skip steps until we reach the resumed step + if args.resume_from_checkpoint and epoch == first_epoch and step < resume_step: + if step % args.gradient_accumulation_steps == 0: + progress_bar.update(1) + continue + + with accelerator.accumulate(prior): + # Convert images to latent space + text_input_ids, text_mask, clip_images = batch + text_input_ids, text_mask, clip_images = text_input_ids, text_mask, clip_images.to(weight_dtype) + with torch.no_grad(): + text_encoder_output = text_encoder(text_input_ids) + prompt_embeds = text_encoder_output.text_embeds + text_encoder_hidden_states = text_encoder_output.last_hidden_state + + + image_embeds = image_encoder(clip_images).image_embeds + # Sample noise that we'll add to the image_embeds + noise = torch.randn_like(image_embeds) + bsz = image_embeds.shape[0] + # Sample a random timestep for each image + timesteps = torch.randint(0, noise_scheduler.config.num_train_timesteps, (bsz,), device=image_embeds.device) + timesteps = timesteps.long() + image_embeds = (image_embeds - clip_mean) / clip_std + noisy_latents = noise_scheduler.add_noise(image_embeds, noise, timesteps) + + target = image_embeds + + # Predict the noise residual and compute loss + model_pred = prior( + noisy_latents, + timestep=timesteps, + proj_embedding=prompt_embeds, + encoder_hidden_states=text_encoder_hidden_states, + attention_mask=text_mask, + ).predicted_image_embedding + + if args.snr_gamma is None: + loss = F.mse_loss(model_pred.float(), target.float(), reduction="mean") + else: + # Compute loss-weights as per Section 3.4 of https://arxiv.org/abs/2303.09556. + # Since we predict the noise instead of x_0, the original formulation is slightly changed. + # This is discussed in Section 4.2 of the same paper. + snr = compute_snr(timesteps) + mse_loss_weights = ( + torch.stack([snr, args.snr_gamma * torch.ones_like(timesteps)], dim=1).min(dim=1)[0] / snr + ) + # We first calculate the original loss. Then we mean over the non-batch dimensions and + # rebalance the sample-wise losses with their respective loss weights. + # Finally, we take the mean of the rebalanced loss. + loss = F.mse_loss(model_pred.float(), target.float(), reduction="none") + loss = loss.mean(dim=list(range(1, len(loss.shape)))) * mse_loss_weights + loss = loss.mean() + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(prior.parameters(), args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + #print(train_loss) + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if accelerator.is_main_process: + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + accelerator.end_training() + + +if __name__ == "__main__": + main() \ No newline at end of file From 13a409c87c46ce58bfa0eecd9c778a51984b04b0 Mon Sep 17 00:00:00 2001 From: Shahmatov Arseniy <62886550+cene555@users.noreply.github.com> Date: Wed, 12 Jul 2023 03:47:20 +0300 Subject: [PATCH 02/38] Rename examples/tune_decoder.py to examples/kandinsky2_2_train/tune_decoder.py --- examples/{ => kandinsky2_2_train}/tune_decoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename examples/{ => kandinsky2_2_train}/tune_decoder.py (99%) diff --git a/examples/tune_decoder.py b/examples/kandinsky2_2_train/tune_decoder.py similarity index 99% rename from examples/tune_decoder.py rename to examples/kandinsky2_2_train/tune_decoder.py index 6f5794224f85..8b321a74fd07 100644 --- a/examples/tune_decoder.py +++ b/examples/kandinsky2_2_train/tune_decoder.py @@ -613,4 +613,4 @@ def load_model_hook(models, input_dir): if __name__ == "__main__": - main() \ No newline at end of file + main() From 03502dc15d5c3a91bb68ba74526340108862342c Mon Sep 17 00:00:00 2001 From: Shahmatov Arseniy <62886550+cene555@users.noreply.github.com> Date: Wed, 12 Jul 2023 03:50:13 +0300 Subject: [PATCH 03/38] Rename examples/tune_prior.py to examples/kandinsky2_2_train/tune_prior.py --- examples/{ => kandinsky2_2_train}/tune_prior.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename examples/{ => kandinsky2_2_train}/tune_prior.py (99%) diff --git a/examples/tune_prior.py b/examples/kandinsky2_2_train/tune_prior.py similarity index 99% rename from examples/tune_prior.py rename to examples/kandinsky2_2_train/tune_prior.py index 18b48116b0bf..52b5628e94c9 100644 --- a/examples/tune_prior.py +++ b/examples/kandinsky2_2_train/tune_prior.py @@ -607,4 +607,4 @@ def load_model_hook(models, input_dir): if __name__ == "__main__": - main() \ No newline at end of file + main() From eb434ecb7d2d321d33c230ecf9d8dddb2ca04b98 Mon Sep 17 00:00:00 2001 From: Shahmatov Arseniy <62886550+cene555@users.noreply.github.com> Date: Wed, 12 Jul 2023 03:50:31 +0300 Subject: [PATCH 04/38] Rename examples/tune_decoder_lora.py to examples/kandinsky2_2_train/tune_decoder_lora.py --- examples/{ => kandinsky2_2_train}/tune_decoder_lora.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename examples/{ => kandinsky2_2_train}/tune_decoder_lora.py (99%) diff --git a/examples/tune_decoder_lora.py b/examples/kandinsky2_2_train/tune_decoder_lora.py similarity index 99% rename from examples/tune_decoder_lora.py rename to examples/kandinsky2_2_train/tune_decoder_lora.py index 3af257d4a53d..0a432d4e6b91 100644 --- a/examples/tune_decoder_lora.py +++ b/examples/kandinsky2_2_train/tune_decoder_lora.py @@ -579,4 +579,4 @@ def compute_snr(timesteps): if __name__ == "__main__": - main() \ No newline at end of file + main() From e7924e6874690d63c87878c3fea191780bea16a4 Mon Sep 17 00:00:00 2001 From: Shahmatov Arseniy <62886550+cene555@users.noreply.github.com> Date: Wed, 12 Jul 2023 03:50:52 +0300 Subject: [PATCH 05/38] Rename examples/tune_prior_lora.py to examples/kandinsky2_2_train/tune_prior_lora.py --- examples/{ => kandinsky2_2_train}/tune_prior_lora.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename examples/{ => kandinsky2_2_train}/tune_prior_lora.py (99%) diff --git a/examples/tune_prior_lora.py b/examples/kandinsky2_2_train/tune_prior_lora.py similarity index 99% rename from examples/tune_prior_lora.py rename to examples/kandinsky2_2_train/tune_prior_lora.py index b7e3414aaf23..c996e6176946 100644 --- a/examples/tune_prior_lora.py +++ b/examples/kandinsky2_2_train/tune_prior_lora.py @@ -570,4 +570,4 @@ def compute_snr(timesteps): if __name__ == "__main__": - main() \ No newline at end of file + main() From f17466f31e52240da5dfa28ba81e0fc7a2230c54 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Fri, 14 Jul 2023 15:08:20 +0000 Subject: [PATCH 06/38] style --- examples/kandinsky2_2_train/tune_decoder.py | 119 ++++++++++------- .../kandinsky2_2_train/tune_decoder_lora.py | 113 +++++++++------- examples/kandinsky2_2_train/tune_prior.py | 118 +++++++++-------- .../kandinsky2_2_train/tune_prior_lora.py | 125 ++++++++++-------- 4 files changed, 269 insertions(+), 206 deletions(-) diff --git a/examples/kandinsky2_2_train/tune_decoder.py b/examples/kandinsky2_2_train/tune_decoder.py index 8b321a74fd07..5ef16e36bc09 100644 --- a/examples/kandinsky2_2_train/tune_decoder.py +++ b/examples/kandinsky2_2_train/tune_decoder.py @@ -1,22 +1,21 @@ import argparse +import math import os + +import accelerate import numpy as np -from PIL import Image, ImageOps -from tqdm import tqdm import pandas as pd -import math -from packaging import version - import torch import torch.nn.functional as F import torch.utils.checkpoint -import accelerate +import transformers from accelerate import Accelerator from accelerate.logging import get_logger from accelerate.state import AcceleratorState from accelerate.utils import ProjectConfiguration, set_seed -from torchvision import transforms -import transformers +from packaging import version +from PIL import Image +from tqdm import tqdm from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection from transformers.utils import ContextManagers @@ -24,14 +23,15 @@ from diffusers import DDPMScheduler, UNet2DConditionModel, VQModel from diffusers.optimization import get_scheduler from diffusers.training_utils import EMAModel -from diffusers.utils import check_min_version, deprecate, is_wandb_available -from diffusers.models.attention_processor import LoRAAttnProcessor +from diffusers.utils import is_wandb_available + if is_wandb_available(): - import wandb + pass + +logger = get_logger(__name__, log_level="INFO") + -logger = get_logger(__name__, log_level="INFO") - def parse_args(): parser = argparse.ArgumentParser(description="Simple example of finetuning Kandinsky 2.2.") parser.add_argument( @@ -44,35 +44,35 @@ def parse_args(): parser.add_argument( "--pretrained_kandinsky_path", type=str, - default='kandinsky-community/kandinsky-2-2-decoder', + default="kandinsky-community/kandinsky-2-2-decoder", required=False, help="Path to pretrained model or model identifier from huggingface.co/models.", ) parser.add_argument( "--pretrained_vae_path", type=str, - default='kandinsky-community/kandinsky-2-2-decoder', + default="kandinsky-community/kandinsky-2-2-decoder", required=False, help="Path to pretrained vae.", ) parser.add_argument( "--pretrained_image_encoder", type=str, - default='kandinsky-community/kandinsky-2-2-prior', + default="kandinsky-community/kandinsky-2-2-prior", required=False, help="Path to pretrained image encoder.", ) parser.add_argument( "--scheduler_path", type=str, - default='kandinsky-community/kandinsky-2-2-decoder', + default="kandinsky-community/kandinsky-2-2-decoder", required=False, help="Path to scheduler.", ) parser.add_argument( "--image_processor_path", type=str, - default='kandinsky-community/kandinsky-2-2-prior', + default="kandinsky-community/kandinsky-2-2-prior", required=False, help="Path to image_processor.", ) @@ -264,7 +264,7 @@ def parse_args(): parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") - + args = parser.parse_args() env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) if env_local_rank != -1 and env_local_rank != args.local_rank: @@ -276,6 +276,7 @@ def parse_args(): return args + def center_crop(image): width, height = image.size new_size = min(width, height) @@ -285,29 +286,34 @@ def center_crop(image): bottom = (height + new_size) / 2 return image.crop((left, top, right, bottom)) + class ImageDataset(torch.utils.data.Dataset): def __init__(self, image_folder=None, images_paths_csv=None, image_processor=None, img_size=512): assert image_folder is None or images_paths_csv is None self.image_processor = image_processor self.img_size = img_size if images_paths_csv is not None: - self.paths = pd.read_csv(images_paths_csv)['paths'].values + self.paths = pd.read_csv(images_paths_csv)["paths"].values else: - self.paths = [os.path.join(image_folder, path) for path in os.listdir(image_folder) if '.jpg' in path.lower() or '.png' in path.lower()] - + self.paths = [ + os.path.join(image_folder, path) + for path in os.listdir(image_folder) + if ".jpg" in path.lower() or ".png" in path.lower() + ] + def __len__(self): return len(self.paths) - + def __getitem__(self, i): img = Image.open(self.paths[i]) clip_image = self.image_processor(img) img = center_crop(img) - img = img.resize((self.img_size, self.img_size), resample=Image.BICUBIC, - reducing_gap=1) + img = img.resize((self.img_size, self.img_size), resample=Image.BICUBIC, reducing_gap=1) img = np.array(img.convert("RGB")) img = img.astype(np.float32) / 127.5 - 1 return np.transpose(img, [2, 0, 1]), clip_image.pixel_values[0] + def main(): args = parse_args() logging_dir = os.path.join(args.output_dir, args.logging_dir) @@ -327,15 +333,16 @@ def main(): else: transformers.utils.logging.set_verbosity_error() diffusers.utils.logging.set_verbosity_error() - + if args.seed is not None: set_seed(args.seed) if accelerator.is_main_process: if args.output_dir is not None: os.makedirs(args.output_dir, exist_ok=True) - + noise_scheduler = DDPMScheduler.from_pretrained(args.scheduler_path, subfolder="scheduler") - image_processor = CLIPImageProcessor.from_pretrained(args.image_processor_path, subfolder='image_processor') + image_processor = CLIPImageProcessor.from_pretrained(args.image_processor_path, subfolder="image_processor") + def deepspeed_zero_init_disabled_context_manager(): """ returns either a context list that includes one that will disable zero.Init or an empty context list @@ -345,18 +352,19 @@ def deepspeed_zero_init_disabled_context_manager(): return [] return [deepspeed_plugin.zero3_init_context_manager(enable=False)] - weight_dtype = torch.float32 + + weight_dtype = torch.float32 if accelerator.mixed_precision == "fp16": weight_dtype = torch.float16 elif accelerator.mixed_precision == "bf16": - weight_dtype = torch.bfloat16 + weight_dtype = torch.bfloat16 with ContextManagers(deepspeed_zero_init_disabled_context_manager()): - vae = VQModel.from_pretrained(args.pretrained_vae_path, subfolder='movq', torch_dtype=weight_dtype).eval() - image_encoder = CLIPVisionModelWithProjection.from_pretrained(args.pretrained_image_encoder, subfolder='image_encoder', torch_dtype=weight_dtype).eval() - print('args.pretrained_kandinsky_path =', args.pretrained_kandinsky_path) - unet = UNet2DConditionModel.from_pretrained( - args.pretrained_kandinsky_path, subfolder="unet" - ) + vae = VQModel.from_pretrained(args.pretrained_vae_path, subfolder="movq", torch_dtype=weight_dtype).eval() + image_encoder = CLIPVisionModelWithProjection.from_pretrained( + args.pretrained_image_encoder, subfolder="image_encoder", torch_dtype=weight_dtype + ).eval() + print("args.pretrained_kandinsky_path =", args.pretrained_kandinsky_path) + unet = UNet2DConditionModel.from_pretrained(args.pretrained_kandinsky_path, subfolder="unet") # Freeze vae and image_encoder vae.requires_grad_(False) @@ -364,11 +372,10 @@ def deepspeed_zero_init_disabled_context_manager(): # Create EMA for the unet. if args.use_ema: - ema_unet = UNet2DConditionModel.from_pretrained( - args.pretrained_kandinsky_path, subfolder="unet" - ) + ema_unet = UNet2DConditionModel.from_pretrained(args.pretrained_kandinsky_path, subfolder="unet") ema_unet = EMAModel(ema_unet.parameters(), model_cls=UNet2DConditionModel, model_config=ema_unet.config) ema_unet.to(accelerator.device) + def compute_snr(timesteps): """ Computes SNR as per https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L847-L849 @@ -393,7 +400,6 @@ def compute_snr(timesteps): snr = (alpha / sigma) ** 2 return snr - # `accelerate` 0.16.0 will have better support for customized saving if version.parse(accelerate.__version__) >= version.parse("0.16.0"): # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format @@ -432,7 +438,7 @@ def load_model_hook(models, input_dir): unet.enable_gradient_checkpointing() if args.allow_tf32: torch.backends.cuda.matmul.allow_tf32 = True - + if args.use_8bit_adam: try: import bitsandbytes as bnb @@ -451,15 +457,28 @@ def load_model_hook(models, input_dir): weight_decay=args.weight_decay, eps=args.adam_epsilon, ) - - train_dataset = ImageDataset(image_folder=args.train_image_folder, images_paths_csv=args.train_images_paths_csv, image_processor=image_processor, img_size=args.image_resolution) - train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=args.train_batch_size, num_workers=args.dataloader_num_workers) + + train_dataset = ImageDataset( + image_folder=args.train_image_folder, + images_paths_csv=args.train_images_paths_csv, + image_processor=image_processor, + img_size=args.image_resolution, + ) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, batch_size=args.train_batch_size, num_workers=args.dataloader_num_workers + ) if args.val_image_folder is not None or args.val_images_paths_csv is not None: - do_val = True - val_dataset = ImageDataset(image_folder=args.val_image_folder, images_paths_csv=args.val_images_paths_csv, image_processor=image_processor, img_size=args.image_resolution) - val_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=args.val_batch_size, num_workers=args.dataloader_num_workers) + ImageDataset( + image_folder=args.val_image_folder, + images_paths_csv=args.val_images_paths_csv, + image_processor=image_processor, + img_size=args.image_resolution, + ) + torch.utils.data.DataLoader( + train_dataset, batch_size=args.val_batch_size, num_workers=args.dataloader_num_workers + ) else: - do_val = False + pass overrode_max_train_steps = False num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) if args.max_train_steps is None: @@ -489,7 +508,7 @@ def load_model_hook(models, input_dir): # The trackers initializes automatically on the main process. if accelerator.is_main_process: tracker_config = dict(vars(args)) - accelerator.init_trackers('test', tracker_config)#args.tracker_project_name + accelerator.init_trackers("test", tracker_config) # args.tracker_project_name # Train! total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps @@ -525,7 +544,7 @@ def load_model_hook(models, input_dir): resume_global_step = global_step * args.gradient_accumulation_steps first_epoch = global_step // num_update_steps_per_epoch resume_step = resume_global_step % (num_update_steps_per_epoch * args.gradient_accumulation_steps) - + progress_bar = tqdm(range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process) progress_bar.set_description("training goes brrr") for epoch in range(first_epoch, args.num_train_epochs): diff --git a/examples/kandinsky2_2_train/tune_decoder_lora.py b/examples/kandinsky2_2_train/tune_decoder_lora.py index 0a432d4e6b91..bb29398fa230 100644 --- a/examples/kandinsky2_2_train/tune_decoder_lora.py +++ b/examples/kandinsky2_2_train/tune_decoder_lora.py @@ -1,37 +1,34 @@ import argparse +import math import os + import numpy as np -from PIL import Image, ImageOps -from tqdm import tqdm import pandas as pd -import math -from packaging import version - import torch import torch.nn.functional as F import torch.utils.checkpoint -import accelerate +import transformers from accelerate import Accelerator from accelerate.logging import get_logger -from accelerate.state import AcceleratorState from accelerate.utils import ProjectConfiguration, set_seed -from torchvision import transforms -import transformers +from PIL import Image +from tqdm import tqdm from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection -from transformers.utils import ContextManagers import diffusers from diffusers import DDPMScheduler, UNet2DConditionModel, VQModel -from diffusers.optimization import get_scheduler -from diffusers.utils import check_min_version, deprecate, is_wandb_available -from diffusers.models.attention_processor import LoRAAttnProcessor, LoRAAttnAddedKVProcessor from diffusers.loaders import AttnProcsLayers +from diffusers.models.attention_processor import LoRAAttnAddedKVProcessor +from diffusers.optimization import get_scheduler +from diffusers.utils import is_wandb_available + if is_wandb_available(): - import wandb + pass + +logger = get_logger(__name__, log_level="INFO") + -logger = get_logger(__name__, log_level="INFO") - def parse_args(): parser = argparse.ArgumentParser(description="Simple example of finetuning Kandinsky 2.2.") parser.add_argument( @@ -44,35 +41,35 @@ def parse_args(): parser.add_argument( "--pretrained_kandinsky_path", type=str, - default='kandinsky-community/kandinsky-2-2-decoder', + default="kandinsky-community/kandinsky-2-2-decoder", required=False, help="Path to pretrained model or model identifier from huggingface.co/models.", ) parser.add_argument( "--pretrained_vae_path", type=str, - default='kandinsky-community/kandinsky-2-2-decoder', + default="kandinsky-community/kandinsky-2-2-decoder", required=False, help="Path to pretrained vae.", ) parser.add_argument( "--pretrained_image_encoder", type=str, - default='kandinsky-community/kandinsky-2-2-prior', + default="kandinsky-community/kandinsky-2-2-prior", required=False, help="Path to pretrained image encoder.", ) parser.add_argument( "--scheduler_path", type=str, - default='kandinsky-community/kandinsky-2-2-decoder', + default="kandinsky-community/kandinsky-2-2-decoder", required=False, help="Path to scheduler.", ) parser.add_argument( "--image_processor_path", type=str, - default='kandinsky-community/kandinsky-2-2-prior', + default="kandinsky-community/kandinsky-2-2-prior", required=False, help="Path to image_processor.", ) @@ -264,7 +261,7 @@ def parse_args(): parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") - + args = parser.parse_args() env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) if env_local_rank != -1 and env_local_rank != args.local_rank: @@ -276,6 +273,7 @@ def parse_args(): return args + def center_crop(image): width, height = image.size new_size = min(width, height) @@ -285,29 +283,34 @@ def center_crop(image): bottom = (height + new_size) / 2 return image.crop((left, top, right, bottom)) + class ImageDataset(torch.utils.data.Dataset): def __init__(self, image_folder=None, images_paths_csv=None, image_processor=None, img_size=512): assert image_folder is None or images_paths_csv is None self.image_processor = image_processor self.img_size = img_size if images_paths_csv is not None: - self.paths = pd.read_csv(images_paths_csv)['paths'].values + self.paths = pd.read_csv(images_paths_csv)["paths"].values else: - self.paths = [os.path.join(image_folder, path) for path in os.listdir(image_folder) if '.jpg' in path.lower() or '.png' in path.lower()] - + self.paths = [ + os.path.join(image_folder, path) + for path in os.listdir(image_folder) + if ".jpg" in path.lower() or ".png" in path.lower() + ] + def __len__(self): return len(self.paths) - + def __getitem__(self, i): img = Image.open(self.paths[i]) clip_image = self.image_processor(img) img = center_crop(img) - img = img.resize((self.img_size, self.img_size), resample=Image.BICUBIC, - reducing_gap=1) + img = img.resize((self.img_size, self.img_size), resample=Image.BICUBIC, reducing_gap=1) img = np.array(img.convert("RGB")) img = img.astype(np.float32) / 127.5 - 1 return np.transpose(img, [2, 0, 1]), clip_image.pixel_values[0] + def main(): args = parse_args() logging_dir = os.path.join(args.output_dir, args.logging_dir) @@ -327,23 +330,25 @@ def main(): else: transformers.utils.logging.set_verbosity_error() diffusers.utils.logging.set_verbosity_error() - + if args.seed is not None: set_seed(args.seed) if accelerator.is_main_process: if args.output_dir is not None: os.makedirs(args.output_dir, exist_ok=True) - + noise_scheduler = DDPMScheduler.from_pretrained(args.scheduler_path, subfolder="scheduler") - image_processor = CLIPImageProcessor.from_pretrained(args.image_processor_path, subfolder='image_processor') + image_processor = CLIPImageProcessor.from_pretrained(args.image_processor_path, subfolder="image_processor") weight_dtype = torch.float32 if accelerator.mixed_precision == "fp16": weight_dtype = torch.float16 elif accelerator.mixed_precision == "bf16": - weight_dtype = torch.bfloat16 - vae = VQModel.from_pretrained(args.pretrained_vae_path, subfolder='movq', torch_dtype=weight_dtype).eval() - image_encoder = CLIPVisionModelWithProjection.from_pretrained(args.pretrained_image_encoder, subfolder='image_encoder', torch_dtype=weight_dtype).eval() - + weight_dtype = torch.bfloat16 + vae = VQModel.from_pretrained(args.pretrained_vae_path, subfolder="movq", torch_dtype=weight_dtype).eval() + image_encoder = CLIPVisionModelWithProjection.from_pretrained( + args.pretrained_image_encoder, subfolder="image_encoder", torch_dtype=weight_dtype + ).eval() + unet = UNet2DConditionModel.from_pretrained( args.pretrained_kandinsky_path, subfolder="unet", torch_dtype=weight_dtype ) @@ -373,6 +378,7 @@ def main(): ) unet.set_attn_processor(lora_attn_procs) + def compute_snr(timesteps): """ Computes SNR as per https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L847-L849 @@ -396,11 +402,12 @@ def compute_snr(timesteps): # Compute SNR. snr = (alpha / sigma) ** 2 return snr + lora_layers = AttnProcsLayers(unet.attn_processors) - + if args.allow_tf32: torch.backends.cuda.matmul.allow_tf32 = True - + if args.use_8bit_adam: try: import bitsandbytes as bnb @@ -419,15 +426,28 @@ def compute_snr(timesteps): weight_decay=args.weight_decay, eps=args.adam_epsilon, ) - - train_dataset = ImageDataset(image_folder=args.train_image_folder, images_paths_csv=args.train_images_paths_csv, image_processor=image_processor, img_size=args.image_resolution) - train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=args.train_batch_size, num_workers=args.dataloader_num_workers) + + train_dataset = ImageDataset( + image_folder=args.train_image_folder, + images_paths_csv=args.train_images_paths_csv, + image_processor=image_processor, + img_size=args.image_resolution, + ) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, batch_size=args.train_batch_size, num_workers=args.dataloader_num_workers + ) if args.val_image_folder is not None or args.val_images_paths_csv is not None: - do_val = True - val_dataset = ImageDataset(image_folder=args.val_image_folder, images_paths_csv=args.val_images_paths_csv, image_processor=image_processor, img_size=args.image_resolution) - val_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=args.val_batch_size, num_workers=args.dataloader_num_workers) + ImageDataset( + image_folder=args.val_image_folder, + images_paths_csv=args.val_images_paths_csv, + image_processor=image_processor, + img_size=args.image_resolution, + ) + torch.utils.data.DataLoader( + train_dataset, batch_size=args.val_batch_size, num_workers=args.dataloader_num_workers + ) else: - do_val = False + pass overrode_max_train_steps = False num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) if args.max_train_steps is None: @@ -444,7 +464,6 @@ def compute_snr(timesteps): lora_layers, optimizer, train_dataloader, lr_scheduler ) - # We need to recalculate our total training steps as the size of the training dataloader may have changed. num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) if overrode_max_train_steps: @@ -456,7 +475,7 @@ def compute_snr(timesteps): # The trackers initializes automatically on the main process. if accelerator.is_main_process: tracker_config = dict(vars(args)) - accelerator.init_trackers('test', tracker_config)#args.tracker_project_name + accelerator.init_trackers("test", tracker_config) # args.tracker_project_name # Train! total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps @@ -492,7 +511,7 @@ def compute_snr(timesteps): resume_global_step = global_step * args.gradient_accumulation_steps first_epoch = global_step // num_update_steps_per_epoch resume_step = resume_global_step % (num_update_steps_per_epoch * args.gradient_accumulation_steps) - + progress_bar = tqdm(range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process) progress_bar.set_description("training goes brrr") for epoch in range(first_epoch, args.num_train_epochs): diff --git a/examples/kandinsky2_2_train/tune_prior.py b/examples/kandinsky2_2_train/tune_prior.py index 52b5628e94c9..8a75f73f5eb8 100644 --- a/examples/kandinsky2_2_train/tune_prior.py +++ b/examples/kandinsky2_2_train/tune_prior.py @@ -1,77 +1,77 @@ import argparse -import os -import numpy as np -from PIL import Image, ImageOps -from tqdm import tqdm -import pandas as pd import math -from packaging import version +import os +import accelerate +import pandas as pd import torch import torch.nn.functional as F import torch.utils.checkpoint -import accelerate +import transformers from accelerate import Accelerator from accelerate.logging import get_logger from accelerate.state import AcceleratorState from accelerate.utils import ProjectConfiguration, set_seed -from torchvision import transforms -import transformers +from packaging import version +from PIL import Image +from tqdm import tqdm from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection from transformers.utils import ContextManagers import diffusers -from diffusers import PriorTransformer, UnCLIPScheduler, DDPMScheduler +from diffusers import DDPMScheduler, PriorTransformer from diffusers.optimization import get_scheduler from diffusers.training_utils import EMAModel -from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils import is_wandb_available + if is_wandb_available(): - import wandb + pass + +logger = get_logger(__name__, log_level="INFO") -logger = get_logger(__name__, log_level="INFO") def parse_args(): parser = argparse.ArgumentParser(description="Simple example of finetuning Kandinsky 2.2.") parser.add_argument( "--pretrained_prior_path", type=str, - default='kandinsky-community/kandinsky-2-2-prior', + default="kandinsky-community/kandinsky-2-2-prior", required=False, help="Path to pretrained prior model or model identifier from huggingface.co/models.", ) parser.add_argument( "--pretrained_image_encoder", type=str, - default='kandinsky-community/kandinsky-2-2-prior', + default="kandinsky-community/kandinsky-2-2-prior", required=False, help="Path to pretrained image encoder.", ) parser.add_argument( "--scheduler_path", type=str, - default='kandinsky-community/kandinsky-2-2-prior', + default="kandinsky-community/kandinsky-2-2-prior", required=False, help="Path to scheduler.", ) parser.add_argument( "--image_processor_path", type=str, - default='kandinsky-community/kandinsky-2-2-prior', + default="kandinsky-community/kandinsky-2-2-prior", required=False, help="Path to image_processor.", ) parser.add_argument( "--text_encoder_path", type=str, - default='kandinsky-community/kandinsky-2-2-prior', + default="kandinsky-community/kandinsky-2-2-prior", required=False, help="Path to text_encoder.", ) parser.add_argument( "--tokenizer_path", type=str, - default='kandinsky-community/kandinsky-2-2-prior', + default="kandinsky-community/kandinsky-2-2-prior", required=False, help="Path to tokenizer.", ) @@ -249,7 +249,7 @@ def parse_args(): parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") - + args = parser.parse_args() env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) if env_local_rank != -1 and env_local_rank != args.local_rank: @@ -267,11 +267,12 @@ def __init__(self, images_paths_csv=None, image_processor=None, tokenizer=None): self.image_processor = image_processor self.tokenizer = tokenizer df = pd.read_csv(images_paths_csv) - self.paths = df['paths'].values - self.captions = df['caption'].values + self.paths = df["paths"].values + self.captions = df["caption"].values + def __len__(self): return len(self.paths) - + def __getitem__(self, i): img = Image.open(self.paths[i]) clip_image = self.image_processor(img) @@ -287,6 +288,7 @@ def __getitem__(self, i): return text_input_ids, text_mask, clip_image.pixel_values[0] + def main(): args = parse_args() logging_dir = os.path.join(args.output_dir, args.logging_dir) @@ -306,16 +308,17 @@ def main(): else: transformers.utils.logging.set_verbosity_error() diffusers.utils.logging.set_verbosity_error() - + if args.seed is not None: set_seed(args.seed) if accelerator.is_main_process: if args.output_dir is not None: os.makedirs(args.output_dir, exist_ok=True) - - noise_scheduler = DDPMScheduler(beta_schedule='squaredcos_cap_v2', prediction_type='sample') - image_processor = CLIPImageProcessor.from_pretrained(args.image_processor_path, subfolder='image_processor') - tokenizer = CLIPTokenizer.from_pretrained(args.tokenizer_path, subfolder='tokenizer') + + noise_scheduler = DDPMScheduler(beta_schedule="squaredcos_cap_v2", prediction_type="sample") + image_processor = CLIPImageProcessor.from_pretrained(args.image_processor_path, subfolder="image_processor") + tokenizer = CLIPTokenizer.from_pretrained(args.tokenizer_path, subfolder="tokenizer") + def deepspeed_zero_init_disabled_context_manager(): """ returns either a context list that includes one that will disable zero.Init or an empty context list @@ -325,18 +328,21 @@ def deepspeed_zero_init_disabled_context_manager(): return [] return [deepspeed_plugin.zero3_init_context_manager(enable=False)] - weight_dtype = torch.float32 + + weight_dtype = torch.float32 if accelerator.mixed_precision == "fp16": weight_dtype = torch.float16 elif accelerator.mixed_precision == "bf16": - weight_dtype = torch.bfloat16 + weight_dtype = torch.bfloat16 with ContextManagers(deepspeed_zero_init_disabled_context_manager()): - image_encoder = CLIPVisionModelWithProjection.from_pretrained(args.pretrained_image_encoder, subfolder='image_encoder', torch_dtype=weight_dtype).eval() - text_encoder = CLIPTextModelWithProjection.from_pretrained(args.text_encoder_path, subfolder='text_encoder', torch_dtype=weight_dtype).eval() - print('args.pretrained_prior_path =', args.pretrained_prior_path) - prior = PriorTransformer.from_pretrained( - args.pretrained_prior_path, subfolder="prior" - ) + image_encoder = CLIPVisionModelWithProjection.from_pretrained( + args.pretrained_image_encoder, subfolder="image_encoder", torch_dtype=weight_dtype + ).eval() + text_encoder = CLIPTextModelWithProjection.from_pretrained( + args.text_encoder_path, subfolder="text_encoder", torch_dtype=weight_dtype + ).eval() + print("args.pretrained_prior_path =", args.pretrained_prior_path) + prior = PriorTransformer.from_pretrained(args.pretrained_prior_path, subfolder="prior") # Freeze text_encoder and image_encoder text_encoder.requires_grad_(False) @@ -344,11 +350,10 @@ def deepspeed_zero_init_disabled_context_manager(): # Create EMA for the prior. if args.use_ema: - ema_prior = PriorTransformer.from_pretrained( - args.pretrained_prior_path, subfolder="prior" - ) + ema_prior = PriorTransformer.from_pretrained(args.pretrained_prior_path, subfolder="prior") ema_prior = EMAModel(ema_prior.parameters(), model_cls=PriorTransformer, model_config=ema_prior.config) ema_prior.to(accelerator.device) + def compute_snr(timesteps): """ Computes SNR as per https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L847-L849 @@ -373,7 +378,6 @@ def compute_snr(timesteps): snr = (alpha / sigma) ** 2 return snr - # `accelerate` 0.16.0 will have better support for customized saving if version.parse(accelerate.__version__) >= version.parse("0.16.0"): # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format @@ -410,7 +414,7 @@ def load_model_hook(models, input_dir): if args.allow_tf32: torch.backends.cuda.matmul.allow_tf32 = True - + if args.use_8bit_adam: try: import bitsandbytes as bnb @@ -429,15 +433,22 @@ def load_model_hook(models, input_dir): weight_decay=args.weight_decay, eps=args.adam_epsilon, ) - - train_dataset = ImageDataset(images_paths_csv=args.train_images_paths_csv, image_processor=image_processor, tokenizer=tokenizer) - train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=args.train_batch_size, num_workers=args.dataloader_num_workers) + + train_dataset = ImageDataset( + images_paths_csv=args.train_images_paths_csv, image_processor=image_processor, tokenizer=tokenizer + ) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, batch_size=args.train_batch_size, num_workers=args.dataloader_num_workers + ) if args.val_images_paths_csv is not None: - do_val = True - val_dataset = ImageDataset(images_paths_csv=args.val_images_paths_csv, image_processor=image_processor, tokenizer=tokenizer) - val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=args.val_batch_size, num_workers=args.dataloader_num_workers) + val_dataset = ImageDataset( + images_paths_csv=args.val_images_paths_csv, image_processor=image_processor, tokenizer=tokenizer + ) + torch.utils.data.DataLoader( + val_dataset, batch_size=args.val_batch_size, num_workers=args.dataloader_num_workers + ) else: - do_val = False + pass overrode_max_train_steps = False num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) if args.max_train_steps is None: @@ -471,7 +482,7 @@ def load_model_hook(models, input_dir): # The trackers initializes automatically on the main process. if accelerator.is_main_process: tracker_config = dict(vars(args)) - accelerator.init_trackers('test', tracker_config)#args.tracker_project_name + accelerator.init_trackers("test", tracker_config) # args.tracker_project_name # Train! total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps @@ -507,7 +518,7 @@ def load_model_hook(models, input_dir): resume_global_step = global_step * args.gradient_accumulation_steps first_epoch = global_step // num_update_steps_per_epoch resume_step = resume_global_step % (num_update_steps_per_epoch * args.gradient_accumulation_steps) - + progress_bar = tqdm(range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process) progress_bar.set_description("training goes brrr") clip_mean = clip_mean.to(weight_dtype).to(accelerator.device) @@ -531,13 +542,14 @@ def load_model_hook(models, input_dir): prompt_embeds = text_encoder_output.text_embeds text_encoder_hidden_states = text_encoder_output.last_hidden_state - image_embeds = image_encoder(clip_images).image_embeds # Sample noise that we'll add to the image_embeds noise = torch.randn_like(image_embeds) bsz = image_embeds.shape[0] # Sample a random timestep for each image - timesteps = torch.randint(0, noise_scheduler.config.num_train_timesteps, (bsz,), device=image_embeds.device) + timesteps = torch.randint( + 0, noise_scheduler.config.num_train_timesteps, (bsz,), device=image_embeds.device + ) timesteps = timesteps.long() image_embeds = (image_embeds - clip_mean) / clip_std noisy_latents = noise_scheduler.add_noise(image_embeds, noise, timesteps) @@ -588,7 +600,7 @@ def load_model_hook(models, input_dir): ema_prior.step(prior.parameters()) progress_bar.update(1) global_step += 1 - #print(train_loss) + # print(train_loss) accelerator.log({"train_loss": train_loss}, step=global_step) train_loss = 0.0 diff --git a/examples/kandinsky2_2_train/tune_prior_lora.py b/examples/kandinsky2_2_train/tune_prior_lora.py index c996e6176946..c2f5253acaa8 100644 --- a/examples/kandinsky2_2_train/tune_prior_lora.py +++ b/examples/kandinsky2_2_train/tune_prior_lora.py @@ -1,78 +1,76 @@ import argparse -import os -import numpy as np -from PIL import Image, ImageOps -from tqdm import tqdm -import pandas as pd import math -from packaging import version +import os +import accelerate +import pandas as pd import torch import torch.nn.functional as F import torch.utils.checkpoint -import accelerate +import transformers from accelerate import Accelerator from accelerate.logging import get_logger from accelerate.state import AcceleratorState from accelerate.utils import ProjectConfiguration, set_seed -from torchvision import transforms -import transformers +from PIL import Image +from tqdm import tqdm from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection -from transformers.utils import ContextManagers import diffusers -from diffusers import PriorTransformer, UnCLIPScheduler, DDPMScheduler -from diffusers.optimization import get_scheduler -from diffusers.training_utils import EMAModel -from diffusers.utils import check_min_version, deprecate, is_wandb_available -from diffusers.models.attention_processor import LoRAAttnProcessor +from diffusers import DDPMScheduler, PriorTransformer from diffusers.loaders import AttnProcsLayers +from diffusers.models.attention_processor import LoRAAttnProcessor +from diffusers.optimization import get_scheduler +from diffusers.utils import is_wandb_available + + if is_wandb_available(): - import wandb + pass + +logger = get_logger(__name__, log_level="INFO") -logger = get_logger(__name__, log_level="INFO") def parse_args(): parser = argparse.ArgumentParser(description="Simple example of finetuning Kandinsky 2.2.") parser.add_argument( "--pretrained_prior_path", type=str, - default='kandinsky-community/kandinsky-2-2-prior', + default="kandinsky-community/kandinsky-2-2-prior", required=False, help="Path to pretrained prior model or model identifier from huggingface.co/models.", ) parser.add_argument( "--pretrained_image_encoder", type=str, - default='kandinsky-community/kandinsky-2-2-prior', + default="kandinsky-community/kandinsky-2-2-prior", required=False, help="Path to pretrained image encoder.", ) parser.add_argument( "--scheduler_path", type=str, - default='kandinsky-community/kandinsky-2-2-prior', + default="kandinsky-community/kandinsky-2-2-prior", required=False, help="Path to scheduler.", ) parser.add_argument( "--image_processor_path", type=str, - default='kandinsky-community/kandinsky-2-2-prior', + default="kandinsky-community/kandinsky-2-2-prior", required=False, help="Path to image_processor.", ) parser.add_argument( "--text_encoder_path", type=str, - default='kandinsky-community/kandinsky-2-2-prior', + default="kandinsky-community/kandinsky-2-2-prior", required=False, help="Path to text_encoder.", ) parser.add_argument( "--tokenizer_path", type=str, - default='kandinsky-community/kandinsky-2-2-prior', + default="kandinsky-community/kandinsky-2-2-prior", required=False, help="Path to tokenizer.", ) @@ -251,7 +249,7 @@ def parse_args(): parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") - + args = parser.parse_args() env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) if env_local_rank != -1 and env_local_rank != args.local_rank: @@ -269,11 +267,12 @@ def __init__(self, images_paths_csv=None, image_processor=None, tokenizer=None): self.image_processor = image_processor self.tokenizer = tokenizer df = pd.read_csv(images_paths_csv) - self.paths = df['paths'].values - self.captions = df['caption'].values + self.paths = df["paths"].values + self.captions = df["caption"].values + def __len__(self): return len(self.paths) - + def __getitem__(self, i): img = Image.open(self.paths[i]) clip_image = self.image_processor(img, return_tensors="pt") @@ -289,6 +288,7 @@ def __getitem__(self, i): return text_input_ids, text_mask, clip_image.pixel_values[0] + def main(): args = parse_args() logging_dir = os.path.join(args.output_dir, args.logging_dir) @@ -308,16 +308,17 @@ def main(): else: transformers.utils.logging.set_verbosity_error() diffusers.utils.logging.set_verbosity_error() - + if args.seed is not None: set_seed(args.seed) if accelerator.is_main_process: if args.output_dir is not None: os.makedirs(args.output_dir, exist_ok=True) - - noise_scheduler = DDPMScheduler(beta_schedule='squaredcos_cap_v2', prediction_type='sample') - image_processor = CLIPImageProcessor.from_pretrained(args.image_processor_path, subfolder='image_processor') - tokenizer = CLIPTokenizer.from_pretrained(args.tokenizer_path, subfolder='tokenizer') + + noise_scheduler = DDPMScheduler(beta_schedule="squaredcos_cap_v2", prediction_type="sample") + image_processor = CLIPImageProcessor.from_pretrained(args.image_processor_path, subfolder="image_processor") + tokenizer = CLIPTokenizer.from_pretrained(args.tokenizer_path, subfolder="tokenizer") + def deepspeed_zero_init_disabled_context_manager(): """ returns either a context list that includes one that will disable zero.Init or an empty context list @@ -327,17 +328,20 @@ def deepspeed_zero_init_disabled_context_manager(): return [] return [deepspeed_plugin.zero3_init_context_manager(enable=False)] - weight_dtype = torch.float32 + + weight_dtype = torch.float32 if accelerator.mixed_precision == "fp16": weight_dtype = torch.float16 elif accelerator.mixed_precision == "bf16": - weight_dtype = torch.bfloat16 - image_encoder = CLIPVisionModelWithProjection.from_pretrained(args.pretrained_image_encoder, subfolder='image_encoder', torch_dtype=weight_dtype).eval() - text_encoder = CLIPTextModelWithProjection.from_pretrained(args.text_encoder_path, subfolder='text_encoder', torch_dtype=weight_dtype).eval() - print('args.pretrained_prior_path =', args.pretrained_prior_path) - prior = PriorTransformer.from_pretrained( - args.pretrained_prior_path, subfolder="prior", torch_dtype=weight_dtype - ) + weight_dtype = torch.bfloat16 + image_encoder = CLIPVisionModelWithProjection.from_pretrained( + args.pretrained_image_encoder, subfolder="image_encoder", torch_dtype=weight_dtype + ).eval() + text_encoder = CLIPTextModelWithProjection.from_pretrained( + args.text_encoder_path, subfolder="text_encoder", torch_dtype=weight_dtype + ).eval() + print("args.pretrained_prior_path =", args.pretrained_prior_path) + prior = PriorTransformer.from_pretrained(args.pretrained_prior_path, subfolder="prior", torch_dtype=weight_dtype) # Freeze text_encoder and image_encoder text_encoder.requires_grad_(False) @@ -351,6 +355,7 @@ def deepspeed_zero_init_disabled_context_manager(): lora_attn_procs[name] = LoRAAttnProcessor(hidden_size=2048, rank=args.rank) prior.set_attn_processor(lora_attn_procs) + def compute_snr(timesteps): """ Computes SNR as per https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L847-L849 @@ -374,11 +379,12 @@ def compute_snr(timesteps): # Compute SNR. snr = (alpha / sigma) ** 2 return snr + lora_layers = AttnProcsLayers(prior.attn_processors) - + if args.allow_tf32: torch.backends.cuda.matmul.allow_tf32 = True - + if args.use_8bit_adam: try: import bitsandbytes as bnb @@ -397,15 +403,22 @@ def compute_snr(timesteps): weight_decay=args.weight_decay, eps=args.adam_epsilon, ) - - train_dataset = ImageDataset(images_paths_csv=args.train_images_paths_csv, image_processor=image_processor, tokenizer=tokenizer) - train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=args.train_batch_size, num_workers=args.dataloader_num_workers) + + train_dataset = ImageDataset( + images_paths_csv=args.train_images_paths_csv, image_processor=image_processor, tokenizer=tokenizer + ) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, batch_size=args.train_batch_size, num_workers=args.dataloader_num_workers + ) if args.val_images_paths_csv is not None: - do_val = True - val_dataset = ImageDataset(images_paths_csv=args.val_images_paths_csv, image_processor=image_processor, tokenizer=tokenizer) - val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=args.val_batch_size, num_workers=args.dataloader_num_workers) + val_dataset = ImageDataset( + images_paths_csv=args.val_images_paths_csv, image_processor=image_processor, tokenizer=tokenizer + ) + torch.utils.data.DataLoader( + val_dataset, batch_size=args.val_batch_size, num_workers=args.dataloader_num_workers + ) else: - do_val = False + pass overrode_max_train_steps = False num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) if args.max_train_steps is None: @@ -424,7 +437,6 @@ def compute_snr(timesteps): lora_layers, optimizer, train_dataloader, lr_scheduler ) - # We need to recalculate our total training steps as the size of the training dataloader may have changed. num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) if overrode_max_train_steps: @@ -436,7 +448,7 @@ def compute_snr(timesteps): # The trackers initializes automatically on the main process. if accelerator.is_main_process: tracker_config = dict(vars(args)) - accelerator.init_trackers('test', tracker_config)#args.tracker_project_name + accelerator.init_trackers("test", tracker_config) # args.tracker_project_name # Train! total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps @@ -472,7 +484,7 @@ def compute_snr(timesteps): resume_global_step = global_step * args.gradient_accumulation_steps first_epoch = global_step // num_update_steps_per_epoch resume_step = resume_global_step % (num_update_steps_per_epoch * args.gradient_accumulation_steps) - + progress_bar = tqdm(range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process) progress_bar.set_description("training goes brrr") clip_mean = clip_mean.to(weight_dtype).to(accelerator.device) @@ -496,17 +508,18 @@ def compute_snr(timesteps): prompt_embeds = text_encoder_output.text_embeds text_encoder_hidden_states = text_encoder_output.last_hidden_state - image_embeds = image_encoder(clip_images).image_embeds # Sample noise that we'll add to the image_embeds noise = torch.randn_like(image_embeds) bsz = image_embeds.shape[0] # Sample a random timestep for each image - timesteps = torch.randint(0, noise_scheduler.config.num_train_timesteps, (bsz,), device=image_embeds.device) + timesteps = torch.randint( + 0, noise_scheduler.config.num_train_timesteps, (bsz,), device=image_embeds.device + ) timesteps = timesteps.long() image_embeds = (image_embeds - clip_mean) / clip_std noisy_latents = noise_scheduler.add_noise(image_embeds, noise, timesteps) - + target = image_embeds # Predict the noise residual and compute loss @@ -551,7 +564,7 @@ def compute_snr(timesteps): if accelerator.sync_gradients: progress_bar.update(1) global_step += 1 - #print(train_loss) + # print(train_loss) accelerator.log({"train_loss": train_loss}, step=global_step) train_loss = 0.0 From 9c368c101c06aa2b77690a594c0119d2670d6703 Mon Sep 17 00:00:00 2001 From: Shahmatov Arseniy <62886550+cene555@users.noreply.github.com> Date: Fri, 21 Jul 2023 17:20:25 +0300 Subject: [PATCH 07/38] Add files via upload --- .../tune_prior_dreambooth.py | 676 ++++++++++++++++++ 1 file changed, 676 insertions(+) create mode 100644 examples/kandinsky2_2_train/tune_prior_dreambooth.py diff --git a/examples/kandinsky2_2_train/tune_prior_dreambooth.py b/examples/kandinsky2_2_train/tune_prior_dreambooth.py new file mode 100644 index 000000000000..d9b1d811a9c2 --- /dev/null +++ b/examples/kandinsky2_2_train/tune_prior_dreambooth.py @@ -0,0 +1,676 @@ +import argparse +import os +import numpy as np +from PIL import Image, ImageOps +from tqdm import tqdm +import pandas as pd +import math +from packaging import version + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import accelerate +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from torchvision import transforms +import transformers +from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection +from transformers.utils import ContextManagers + +import diffusers +from diffusers import PriorTransformer, UnCLIPScheduler, DDPMScheduler, KandinskyV22PriorPipeline +from diffusers.optimization import get_scheduler +from diffusers.training_utils import EMAModel +from diffusers.utils import check_min_version, deprecate, is_wandb_available + +if is_wandb_available(): + import wandb + +logger = get_logger(__name__, log_level="INFO") + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of finetuning Kandinsky 2.2.") + parser.add_argument( + "--pretrained_prior_path", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to pretrained prior model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--pretrained_image_encoder", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to pretrained image encoder.", + ) + parser.add_argument( + "--scheduler_path", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to scheduler.", + ) + parser.add_argument( + "--image_processor_path", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to image_processor.", + ) + parser.add_argument( + "--text_encoder_path", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to text_encoder.", + ) + parser.add_argument( + "--instance_data_dir", + type=str, + default=None, + required=True, + help="A folder containing the training data of instance images.", + ) + parser.add_argument( + "--instance_prompt", + type=str, + default=None, + required=True, + help="The prompt with identifier specifying the instance", + ) + parser.add_argument( + "--class_prompt", + type=str, + default=None, + help="The prompt to specify images in the same class as provided instance images.", + ) + parser.add_argument( + "--with_prior_preservation", + default=False, + action="store_true", + help="Flag to add prior preservation loss.", + ) + parser.add_argument("--prior_loss_weight", type=float, default=1.0, help="The weight of prior preservation loss.") + parser.add_argument( + "--num_class_images", + type=int, + default=100, + help=( + "Minimal class images for prior preservation loss. If there are not enough images already present in" + " class_data_dir, additional images will be sampled with class_prompt." + ), + ) + parser.add_argument( + "--tokenizer_path", + type=str, + default='kandinsky-community/kandinsky-2-2-prior', + required=False, + help="Path to tokenizer.", + ) + parser.add_argument( + "--train_batch_size", + type=int, + default=1, + required=False, + help="train batch size", + ) + parser.add_argument( + "--lr", + type=float, + default=1e-4, + required=False, + help="learning rate", + ) + parser.add_argument( + "--weight_decay", + type=float, + default=0.0, + required=False, + help="weight decay", + ) + parser.add_argument( + "--num_epochs", + type=int, + default=0.0, + required=False, + help="num of epochs", + ) + parser.add_argument( + "--output_dir", + type=str, + default="kandi_2_2-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--snr_gamma", + type=float, + default=None, + help="SNR weighting gamma to be used if rebalancing the loss. Recommended value is 5.0. " + "More details here: https://arxiv.org/abs/2303.09556.", + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=( + "Max number of checkpoints to store. Passed as `total_limit` to the `Accelerator` `ProjectConfiguration`." + " See Accelerator::save_state https://huggingface.co/docs/accelerate/package_reference/accelerator#accelerate.Accelerator.save_state" + " for more docs" + ), + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # Sanity checks + + return args + + +class ImageDataset(torch.utils.data.Dataset): + def __init__(self, + instance_data_dir=None, + instance_prompt=None, + class_dataset=None, + class_prompt=None, + image_processor=None, + tokenizer=None): + self.image_processor = image_processor + self.tokenizer = tokenizer + self.instance_paths = [os.path.join(instance_data_dir, i) for i in os.listdir(instance_data_dir) if '.jpg' in i or '.png' in i or '.jpeg' in i] + self.instance_prompt = instance_prompt + self.class_dataset = class_dataset + self.class_prompt = class_prompt + def __len__(self): + return len(self.instance_paths) + + def __getitem__(self, i): + example = {} + img = Image.open(self.instance_paths[i]) + example["instance_images"] = torch.from_numpy(self.image_processor(img).pixel_values[0]) + text_inputs = self.tokenizer( + self.instance_prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + example["instance_prompt_ids"] = text_inputs.input_ids[0] + example["instance_attention_mask"] = text_inputs.attention_mask.bool()[0] + if self.class_dataset is not None: + example["class_images"] = self.class_dataset[i % self.class_dataset.shape[0]] + text_inputs = self.tokenizer( + self.class_prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + example["class_prompt_ids"] = text_inputs.input_ids[0] + example["class_attention_mask"] = text_inputs.attention_mask.bool()[0] + return example + +def collate_fn(examples, with_prior_preservation=False): + input_ids = [example["instance_prompt_ids"] for example in examples] + pixel_values = [example["instance_images"] for example in examples] + + attention_mask = [example["instance_attention_mask"] for example in examples] + + # Concat class and instance examples for prior preservation. + # We do this to avoid doing two forward passes. + if with_prior_preservation: + input_ids += [example["class_prompt_ids"] for example in examples] + class_pixel_values = [example["class_images"] for example in examples] + attention_mask += [example["class_attention_mask"] for example in examples] + class_pixel_values = torch.stack(class_pixel_values) + pixel_values = torch.stack(pixel_values) + pixel_values = pixel_values.to(memory_format=torch.contiguous_format).float() + + input_ids = torch.stack(input_ids) + attention_mask = torch.stack(attention_mask) + batch = { + "input_ids": input_ids, + "pixel_values": pixel_values, + "attention_mask": attention_mask + } + if with_prior_preservation: + batch["class_pixel_values"] = class_pixel_values + return batch + +def main(): + args = parse_args() + logging_dir = os.path.join(args.output_dir, args.logging_dir) + accelerator_project_config = ProjectConfiguration( + total_limit=args.checkpoints_total_limit, project_dir=args.output_dir, logging_dir=logging_dir + ) + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + if args.with_prior_preservation: + pipe_prior = KandinskyV22PriorPipeline.from_pretrained(args.pretrained_prior_path, torch_dtype=torch.float16).to(accelerator.device) + class_dataset = [] + for i in tqdm(range(args.num_class_images)): + image_emb = pipe_prior(args.class_prompt).image_embeds + class_dataset.append(image_emb.to('cpu')) + class_dataset = torch.cat(class_dataset, dim=0) + del pipe_prior + if torch.cuda.is_available(): + torch.cuda.empty_cache() + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + if args.seed is not None: + set_seed(args.seed) + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + noise_scheduler = DDPMScheduler(beta_schedule='squaredcos_cap_v2', prediction_type='sample') + image_processor = CLIPImageProcessor.from_pretrained(args.image_processor_path, subfolder='image_processor') + tokenizer = CLIPTokenizer.from_pretrained(args.tokenizer_path, subfolder='tokenizer') + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + image_encoder = CLIPVisionModelWithProjection.from_pretrained(args.pretrained_image_encoder, subfolder='image_encoder', torch_dtype=weight_dtype).eval() + text_encoder = CLIPTextModelWithProjection.from_pretrained(args.text_encoder_path, subfolder='text_encoder', torch_dtype=weight_dtype).eval() + print('args.pretrained_prior_path =', args.pretrained_prior_path) + prior = PriorTransformer.from_pretrained( + args.pretrained_prior_path, subfolder="prior" + ) + + # Freeze text_encoder and image_encoder + text_encoder.requires_grad_(False) + image_encoder.requires_grad_(False) + + # Create EMA for the prior. + if args.use_ema: + ema_prior = PriorTransformer.from_pretrained( + args.pretrained_prior_path, subfolder="prior" + ) + ema_prior = EMAModel(ema_prior.parameters(), model_cls=PriorTransformer, model_config=ema_prior.config) + ema_prior.to(accelerator.device) + def compute_snr(timesteps): + """ + Computes SNR as per https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L847-L849 + """ + alphas_cumprod = noise_scheduler.alphas_cumprod + sqrt_alphas_cumprod = alphas_cumprod**0.5 + sqrt_one_minus_alphas_cumprod = (1.0 - alphas_cumprod) ** 0.5 + + # Expand the tensors. + # Adapted from https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L1026 + sqrt_alphas_cumprod = sqrt_alphas_cumprod.to(device=timesteps.device)[timesteps].float() + while len(sqrt_alphas_cumprod.shape) < len(timesteps.shape): + sqrt_alphas_cumprod = sqrt_alphas_cumprod[..., None] + alpha = sqrt_alphas_cumprod.expand(timesteps.shape) + + sqrt_one_minus_alphas_cumprod = sqrt_one_minus_alphas_cumprod.to(device=timesteps.device)[timesteps].float() + while len(sqrt_one_minus_alphas_cumprod.shape) < len(timesteps.shape): + sqrt_one_minus_alphas_cumprod = sqrt_one_minus_alphas_cumprod[..., None] + sigma = sqrt_one_minus_alphas_cumprod.expand(timesteps.shape) + + # Compute SNR. + snr = (alpha / sigma) ** 2 + return snr + + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if args.use_ema: + ema_prior.save_pretrained(os.path.join(output_dir, "prior_ema")) + + for i, model in enumerate(models): + model.save_pretrained(os.path.join(output_dir, "prior")) + + # make sure to pop weight so that corresponding model is not saved again + weights.pop() + + def load_model_hook(models, input_dir): + if args.use_ema: + load_model = EMAModel.from_pretrained(os.path.join(input_dir, "prior_ema"), PriorTransformer) + ema_prior.load_state_dict(load_model.state_dict()) + ema_prior.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = PriorTransformer.from_pretrained(input_dir, subfolder="prior") + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + else: + optimizer_cls = torch.optim.AdamW + optimizer = optimizer_cls( + prior.parameters(), + lr=args.lr, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.weight_decay, + eps=args.adam_epsilon, + ) + + train_dataset = ImageDataset(instance_data_dir=args.instance_data_dir, + instance_prompt=args.instance_prompt, + class_dataset=class_dataset, + class_prompt=args.class_prompt, + image_processor=image_processor, + tokenizer=tokenizer) + train_dataloader = torch.utils.data.DataLoader(train_dataset, + collate_fn=lambda examples: collate_fn(examples, args.with_prior_preservation), + batch_size=args.train_batch_size, + num_workers=args.dataloader_num_workers) + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * args.gradient_accumulation_steps, + num_training_steps=args.max_train_steps * args.gradient_accumulation_steps, + ) + clip_mean = prior.clip_mean + clip_std = prior.clip_std + prior.clip_mean = None + prior.clip_std = None + prior, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + prior, optimizer, train_dataloader, lr_scheduler + ) + + image_encoder.to(accelerator.device, dtype=weight_dtype) + text_encoder.to(accelerator.device, dtype=weight_dtype) + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + accelerator.init_trackers('test', tracker_config)#args.tracker_project_name + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + else: + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + global_step = int(path.split("-")[1]) + + resume_global_step = global_step * args.gradient_accumulation_steps + first_epoch = global_step // num_update_steps_per_epoch + resume_step = resume_global_step % (num_update_steps_per_epoch * args.gradient_accumulation_steps) + + progress_bar = tqdm(range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process) + progress_bar.set_description("training goes brrr") + clip_mean = clip_mean.to(weight_dtype).to(accelerator.device) + clip_std = clip_std.to(weight_dtype).to(accelerator.device) + for epoch in range(first_epoch, args.num_train_epochs): + prior.train() + train_loss = 0.0 + for step, batch in enumerate(train_dataloader): + # Skip steps until we reach the resumed step + if args.resume_from_checkpoint and epoch == first_epoch and step < resume_step: + if step % args.gradient_accumulation_steps == 0: + progress_bar.update(1) + continue + + with accelerator.accumulate(prior): + # Convert images to latent space + text_input_ids, text_mask, clip_images = batch['input_ids'].to(accelerator.device), batch['attention_mask'].to(accelerator.device), batch['pixel_values'].to(accelerator.device).to(weight_dtype) + if args.with_prior_preservation: + class_pixel_values = batch['class_pixel_values'].to(accelerator.device) + with torch.no_grad(): + text_encoder_output = text_encoder(text_input_ids) + prompt_embeds = text_encoder_output.text_embeds + text_encoder_hidden_states = text_encoder_output.last_hidden_state + + + image_embeds = image_encoder(clip_images).image_embeds + if args.with_prior_preservation: + image_embeds = torch.cat((image_embeds, class_pixel_values), dim=0) + # Sample noise that we'll add to the image_embeds + noise = torch.randn_like(image_embeds) + bsz = image_embeds.shape[0] + # Sample a random timestep for each image + timesteps = torch.randint(0, noise_scheduler.config.num_train_timesteps, (bsz,), device=image_embeds.device) + timesteps = timesteps.long() + image_embeds = (image_embeds - clip_mean) / clip_std + noisy_latents = noise_scheduler.add_noise(image_embeds, noise, timesteps) + + target = image_embeds + + # Predict the noise residual and compute loss + model_pred = prior( + noisy_latents, + timestep=timesteps, + proj_embedding=prompt_embeds, + encoder_hidden_states=text_encoder_hidden_states, + attention_mask=text_mask, + ).predicted_image_embedding + + if args.with_prior_preservation: + model_pred, model_pred_prior = torch.chunk(model_pred, 2, dim=0) + target, target_prior = torch.chunk(target, 2, dim=0) + loss = F.mse_loss(model_pred.float(), target.float(), reduction="mean") + prior_loss = F.mse_loss(model_pred_prior.float(), target_prior.float(), reduction="mean") + loss = loss + args.prior_loss_weight * prior_loss + else: + loss = F.mse_loss(model_pred.float(), target.float(), reduction="mean") + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(prior.parameters(), args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + if args.use_ema: + ema_prior.step(prior.parameters()) + progress_bar.update(1) + global_step += 1 + #print(train_loss) + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if accelerator.is_main_process: + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + accelerator.end_training() + + +if __name__ == "__main__": + main() From 272f789dd36e859637126e9c2184e612cd5e0acf Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Mon, 4 Sep 2023 04:27:33 +0000 Subject: [PATCH 08/38] update tune_decoder --- examples/kandinsky2_2_train/requirements.txt | 7 + examples/kandinsky2_2_train/tune_decoder.py | 644 ++++++++++++++----- 2 files changed, 477 insertions(+), 174 deletions(-) create mode 100644 examples/kandinsky2_2_train/requirements.txt diff --git a/examples/kandinsky2_2_train/requirements.txt b/examples/kandinsky2_2_train/requirements.txt new file mode 100644 index 000000000000..31b9026efdc2 --- /dev/null +++ b/examples/kandinsky2_2_train/requirements.txt @@ -0,0 +1,7 @@ +accelerate>=0.16.0 +torchvision +transformers>=4.25.1 +datasets +ftfy +tensorboard +Jinja2 diff --git a/examples/kandinsky2_2_train/tune_decoder.py b/examples/kandinsky2_2_train/tune_decoder.py index 5ef16e36bc09..0b14413caec8 100644 --- a/examples/kandinsky2_2_train/tune_decoder.py +++ b/examples/kandinsky2_2_train/tune_decoder.py @@ -1,10 +1,26 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + import argparse +import logging import math import os import accelerate +import datasets import numpy as np -import pandas as pd import torch import torch.nn.functional as F import torch.utils.checkpoint @@ -13,6 +29,7 @@ from accelerate.logging import get_logger from accelerate.state import AcceleratorState from accelerate.utils import ProjectConfiguration, set_seed +from datasets import load_dataset from packaging import version from PIL import Image from tqdm import tqdm @@ -20,124 +37,220 @@ from transformers.utils import ContextManagers import diffusers -from diffusers import DDPMScheduler, UNet2DConditionModel, VQModel +from diffusers import DDPMScheduler, UNet2DConditionModel, VQModel, AutoPipelineForText2Image from diffusers.optimization import get_scheduler from diffusers.training_utils import EMAModel -from diffusers.utils import is_wandb_available +from diffusers.utils import check_min_version, deprecate, is_wandb_available if is_wandb_available(): - pass + import wandb + + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.19.0.dev0") logger = get_logger(__name__, log_level="INFO") + +def make_image_grid(imgs, rows, cols): + assert len(imgs) == rows * cols + + w, h = imgs[0].size + grid = Image.new("RGB", size=(cols * w, rows * h)) + + for i, img in enumerate(imgs): + grid.paste(img, box=(i % cols * w, i // cols * h)) + return grid + + +def save_model_card( + args, + repo_id: str, + images=None, + repo_folder=None, +): + img_str = "" + if len(images) > 0: + image_grid = make_image_grid(images, 1, len(args.validation_prompts)) + image_grid.save(os.path.join(repo_folder, "val_imgs_grid.png")) + img_str += "![val_imgs_grid](./val_imgs_grid.png)\n" + + yaml = f""" +--- +license: creativeml-openrail-m +base_model: {args.pretrained_decoder_model_name_or_path} +datasets: +- {args.dataset_name} +tags: +- kandinsky +- text-to-image +- diffusers +inference: true +--- + """ + model_card = f""" +# Text-to-image finetuning - {repo_id} + +This pipeline was finetuned from **{args.pretrained_decoder_model_name_or_path}** on the **{args.dataset_name}** dataset. Below are some example images generated with the finetuned pipeline using the following prompts: {args.validation_prompts}: \n +{img_str} + +## Pipeline usage + +You can use the pipeline like so: + +```python +from diffusers import DiffusionPipeline +import torch + +pipeline = DiffusionPipeline.from_pretrained("{repo_id}", torch_dtype=torch.float16) +prompt = "{args.validation_prompts[0]}" +image = pipeline(prompt).images[0] +image.save("my_image.png") +``` + +## Training info + +These are the key hyperparameters used during training: + +* Epochs: {args.num_train_epochs} +* Learning rate: {args.learning_rate} +* Batch size: {args.train_batch_size} +* Gradient accumulation steps: {args.gradient_accumulation_steps} +* Image resolution: {args.resolution} +* Mixed-precision: {args.mixed_precision} + +""" + wandb_info = "" + if is_wandb_available(): + wandb_run_url = None + if wandb.run is not None: + wandb_run_url = wandb.run.url + + if wandb_run_url is not None: + wandb_info = f""" +More information on all the CLI arguments and the environment are available on your [`wandb` run page]({wandb_run_url}). +""" + + model_card += wandb_info + + with open(os.path.join(repo_folder, "README.md"), "w") as f: + f.write(yaml + model_card) + + +def log_validation(vae, image_encoder, image_processor, unet, args, accelerator, weight_dtype, epoch): + logger.info("Running validation... ") + + pipeline = AutoPipelineForText2Image.from_pretrained( + args.pretrained_decoder_model_name_or_path, + vae=accelerator.unwrap_model(vae), + image_encoder=accelerator.unwrap_model(image_encoder), + image_processor=image_processor, + unet=accelerator.unwrap_model(unet), + safety_checker=None, + revision=args.revision, + torch_dtype=weight_dtype, + ) + pipeline = pipeline.to(accelerator.device) + pipeline.set_progress_bar_config(disable=True) + + if args.enable_xformers_memory_efficient_attention: + pipeline.enable_xformers_memory_efficient_attention() + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.autocast("cuda"): + image = pipeline(args.validation_prompts[i], num_inference_steps=20, generator=generator).images[0] + + images.append(image) + + for tracker in accelerator.trackers: + if tracker.name == "tensorboard": + np_images = np.stack([np.asarray(img) for img in images]) + tracker.writer.add_images("validation", np_images, epoch, dataformats="NHWC") + elif tracker.name == "wandb": + tracker.log( + { + "validation": [ + wandb.Image(image, caption=f"{i}: {args.validation_prompts[i]}") + for i, image in enumerate(images) + ] + } + ) + else: + logger.warn(f"image logging not implemented for {tracker.name}") + + del pipeline + torch.cuda.empty_cache() + + return images + def parse_args(): parser = argparse.ArgumentParser(description="Simple example of finetuning Kandinsky 2.2.") parser.add_argument( - "--image_resolution", - type=int, - default=512, - required=False, - help="Image resolution", - ) - parser.add_argument( - "--pretrained_kandinsky_path", + "--pretrained_decoder_model_name_or_path", type=str, default="kandinsky-community/kandinsky-2-2-decoder", required=False, help="Path to pretrained model or model identifier from huggingface.co/models.", ) parser.add_argument( - "--pretrained_vae_path", - type=str, - default="kandinsky-community/kandinsky-2-2-decoder", - required=False, - help="Path to pretrained vae.", - ) - parser.add_argument( - "--pretrained_image_encoder", + "--pretrained_prior_model_name_or_path", type=str, default="kandinsky-community/kandinsky-2-2-prior", required=False, - help="Path to pretrained image encoder.", - ) - parser.add_argument( - "--scheduler_path", - type=str, - default="kandinsky-community/kandinsky-2-2-decoder", - required=False, - help="Path to scheduler.", - ) - parser.add_argument( - "--image_processor_path", - type=str, - default="kandinsky-community/kandinsky-2-2-prior", - required=False, - help="Path to image_processor.", - ) - parser.add_argument( - "--train_image_folder", - type=str, - default=None, - required=False, - help="Path to train image folder.", + help="Path to pretrained model or model identifier from huggingface.co/models.", ) parser.add_argument( - "--train_images_paths_csv", + "--dataset_name", type=str, default=None, - required=False, - help="Path to csv with train images paths(define train_image_folder or train_images_paths_csv) with column paths.", + help=( + "The name of the Dataset (from the HuggingFace hub) to train on (could be your own, possibly private," + " dataset). It can also be a path pointing to a local copy of a dataset in your filesystem," + " or to a folder containing files that 🤗 Datasets can understand." + ), ) parser.add_argument( - "--val_image_folder", + "--dataset_config_name", type=str, default=None, - required=False, - help="Path to val image folder.", + help="The config of the Dataset, leave as None if there's only one config.", ) parser.add_argument( - "--val_images_paths_csv", + "--train_data_dir", type=str, default=None, - required=False, - help="Path to csv with val images paths with column paths.", + help=( + "A folder containing the training data. Folder contents must follow the structure described in" + " https://huggingface.co/docs/datasets/image_dataset#imagefolder. In particular, a `metadata.jsonl` file" + " must exist to provide the captions for the images. Ignored if `dataset_name` is specified." + ), ) parser.add_argument( - "--train_batch_size", - type=int, - default=1, - required=False, - help="train batch size", + "--image_column", type=str, default="image", help="The column of the dataset containing an image." ) parser.add_argument( - "--val_batch_size", + "--max_train_samples", type=int, - default=1, - required=False, - help="val batch size", - ) - parser.add_argument( - "--lr", - type=float, - default=1e-4, - required=False, - help="learning rate", - ) - parser.add_argument( - "--weight_decay", - type=float, - default=0.0, - required=False, - help="weight decay", + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), ) parser.add_argument( - "--num_epochs", - type=int, - default=0.0, - required=False, - help="num of epochs", + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), ) parser.add_argument( "--output_dir", @@ -146,19 +259,30 @@ def parse_args(): help="The output directory where the model predictions and checkpoints will be written.", ) parser.add_argument( - "--lr_scheduler", + "--cache_dir", type=str, - default="constant", + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--resolution", + type=int, + default=512, help=( - 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' - ' "constant", "constant_with_warmup"]' + "The resolution for input images, all the images in the train/validation dataset will be resized to this" + " resolution" ), ) + parser.add_argument( + "--train_batch_size", type=int, default=1, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) parser.add_argument( "--max_train_steps", type=int, default=None, - help="Total number of training steps to perform. If provided, overrides num_epochs.", + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", ) parser.add_argument( "--gradient_accumulation_steps", @@ -171,6 +295,22 @@ def parse_args(): action="store_true", help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + required=False, + help="learning rate", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) parser.add_argument( "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." ) @@ -184,16 +324,6 @@ def parse_args(): parser.add_argument( "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." ) - parser.add_argument( - "--resume_from_checkpoint", - type=str, - default=None, - help=( - "Whether training should be resumed from a previous checkpoint. Use a path saved by" - ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' - ), - ) - parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") parser.add_argument( "--allow_tf32", action="store_true", @@ -202,6 +332,7 @@ def parse_args(): " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" ), ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") parser.add_argument( "--dataloader_num_workers", type=int, @@ -210,6 +341,19 @@ def parse_args(): "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." ), ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay",type=float,default=0.0, required=False, help="weight decay_to_use",) + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) parser.add_argument( "--logging_dir", type=str, @@ -253,17 +397,35 @@ def parse_args(): "--checkpoints_total_limit", type=int, default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, help=( - "Max number of checkpoints to store. Passed as `total_limit` to the `Accelerator` `ProjectConfiguration`." - " See Accelerator::save_state https://huggingface.co/docs/accelerate/package_reference/accelerator#accelerate.Accelerator.save_state" - " for more docs" + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument( + "--enable_xformers_memory_efficient_attention", action="store_true", help="Whether or not to use xformers." + ) + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" ), ) - parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") - parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") - parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") - parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") - parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") args = parser.parse_args() env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) @@ -271,49 +433,12 @@ def parse_args(): args.local_rank = env_local_rank # Sanity checks - if args.train_image_folder is None and args.train_images_paths_csv is None: - raise ValueError("Need either a train_image_folder or a train_images_paths_csv.") + if args.dataset_name is None and args.train_data_dir is None: + raise ValueError("Need either a dataset name or a training folder.") return args -def center_crop(image): - width, height = image.size - new_size = min(width, height) - left = (width - new_size) / 2 - top = (height - new_size) / 2 - right = (width + new_size) / 2 - bottom = (height + new_size) / 2 - return image.crop((left, top, right, bottom)) - - -class ImageDataset(torch.utils.data.Dataset): - def __init__(self, image_folder=None, images_paths_csv=None, image_processor=None, img_size=512): - assert image_folder is None or images_paths_csv is None - self.image_processor = image_processor - self.img_size = img_size - if images_paths_csv is not None: - self.paths = pd.read_csv(images_paths_csv)["paths"].values - else: - self.paths = [ - os.path.join(image_folder, path) - for path in os.listdir(image_folder) - if ".jpg" in path.lower() or ".png" in path.lower() - ] - - def __len__(self): - return len(self.paths) - - def __getitem__(self, i): - img = Image.open(self.paths[i]) - clip_image = self.image_processor(img) - img = center_crop(img) - img = img.resize((self.img_size, self.img_size), resample=Image.BICUBIC, reducing_gap=1) - img = np.array(img.convert("RGB")) - img = img.astype(np.float32) / 127.5 - 1 - return np.transpose(img, [2, 0, 1]), clip_image.pixel_values[0] - - def main(): args = parse_args() logging_dir = os.path.join(args.output_dir, args.logging_dir) @@ -326,22 +451,38 @@ def main(): log_with=args.report_to, project_config=accelerator_project_config, ) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) logger.info(accelerator.state, main_process_only=False) if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() transformers.utils.logging.set_verbosity_warning() diffusers.utils.logging.set_verbosity_info() else: + datasets.utils.logging.set_verbosity_error() transformers.utils.logging.set_verbosity_error() diffusers.utils.logging.set_verbosity_error() + # If passed along, set the training seed now. if args.seed is not None: set_seed(args.seed) + + # Handle the repository creation if accelerator.is_main_process: if args.output_dir is not None: os.makedirs(args.output_dir, exist_ok=True) - noise_scheduler = DDPMScheduler.from_pretrained(args.scheduler_path, subfolder="scheduler") - image_processor = CLIPImageProcessor.from_pretrained(args.image_processor_path, subfolder="image_processor") + if args.push_to_hub: + repo_id = create_repo( + repo_id=args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token + ).repo_id + noise_scheduler = DDPMScheduler.from_pretrained(args.pretrained_decoder_model_name_or_path, subfolder="scheduler") + image_processor = CLIPImageProcessor.from_pretrained(args.pretrained_prior_model_name_or_path, subfolder="image_processor") def deepspeed_zero_init_disabled_context_manager(): """ @@ -359,12 +500,14 @@ def deepspeed_zero_init_disabled_context_manager(): elif accelerator.mixed_precision == "bf16": weight_dtype = torch.bfloat16 with ContextManagers(deepspeed_zero_init_disabled_context_manager()): - vae = VQModel.from_pretrained(args.pretrained_vae_path, subfolder="movq", torch_dtype=weight_dtype).eval() + vae = VQModel.from_pretrained( + args.pretrained_decoder_model_name_or_path, subfolder="movq", torch_dtype=weight_dtype).eval() image_encoder = CLIPVisionModelWithProjection.from_pretrained( - args.pretrained_image_encoder, subfolder="image_encoder", torch_dtype=weight_dtype + args.pretrained_prior_model_name_or_path, subfolder="image_encoder", torch_dtype=weight_dtype ).eval() - print("args.pretrained_kandinsky_path =", args.pretrained_kandinsky_path) - unet = UNet2DConditionModel.from_pretrained(args.pretrained_kandinsky_path, subfolder="unet") + unet = UNet2DConditionModel.from_pretrained( + args.pretrained_decoder_model_name_or_path, subfolder="unet" + ) # Freeze vae and image_encoder vae.requires_grad_(False) @@ -372,7 +515,8 @@ def deepspeed_zero_init_disabled_context_manager(): # Create EMA for the unet. if args.use_ema: - ema_unet = UNet2DConditionModel.from_pretrained(args.pretrained_kandinsky_path, subfolder="unet") + ema_unet = UNet2DConditionModel.from_pretrained( + args.pretrained_decoder_model_name_or_path, subfolder="unet") ema_unet = EMAModel(ema_unet.parameters(), model_cls=UNet2DConditionModel, model_config=ema_unet.config) ema_unet.to(accelerator.device) @@ -436,6 +580,9 @@ def load_model_hook(models, input_dir): if args.gradient_checkpointing: unet.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices if args.allow_tf32: torch.backends.cuda.matmul.allow_tf32 = True @@ -450,35 +597,95 @@ def load_model_hook(models, input_dir): optimizer_cls = bnb.optim.AdamW8bit else: optimizer_cls = torch.optim.AdamW + optimizer = optimizer_cls( unet.parameters(), - lr=args.lr, + lr=args.learning_rate, betas=(args.adam_beta1, args.adam_beta2), - weight_decay=args.weight_decay, + weight_decay=args.adam_weight_decay, eps=args.adam_epsilon, ) - train_dataset = ImageDataset( - image_folder=args.train_image_folder, - images_paths_csv=args.train_images_paths_csv, - image_processor=image_processor, - img_size=args.image_resolution, - ) - train_dataloader = torch.utils.data.DataLoader( - train_dataset, batch_size=args.train_batch_size, num_workers=args.dataloader_num_workers - ) - if args.val_image_folder is not None or args.val_images_paths_csv is not None: - ImageDataset( - image_folder=args.val_image_folder, - images_paths_csv=args.val_images_paths_csv, - image_processor=image_processor, - img_size=args.image_resolution, - ) - torch.utils.data.DataLoader( - train_dataset, batch_size=args.val_batch_size, num_workers=args.dataloader_num_workers + # Get the datasets: you can either provide your own training and evaluation files (see below) + # or specify a Dataset from the hub (the dataset will be downloaded automatically from the datasets Hub). + + # In distributed training, the load_dataset function guarantees that only one local process can concurrently + # download the dataset. + if args.dataset_name is not None: + # Downloading and loading a dataset from the hub. + dataset = load_dataset( + args.dataset_name, + args.dataset_config_name, + cache_dir=args.cache_dir, ) else: - pass + data_files = {} + if args.train_data_dir is not None: + data_files["train"] = os.path.join(args.train_data_dir, "**") + dataset = load_dataset( + "imagefolder", + data_files=data_files, + cache_dir=args.cache_dir, + ) + # See more about loading custom images at + # https://huggingface.co/docs/datasets/v2.4.0/en/image_load#imagefolder + + # Preprocessing the datasets. + # We need to tokenize inputs and targets. + column_names = dataset["train"].column_names + + image_column = args.image_column + if image_column not in column_names: + raise ValueError( + f"--image_column' value '{args.image_column}' needs to be one of: {', '.join(column_names)}" + ) + + def center_crop(image): + width, height = image.size + new_size = min(width, height) + left = (width - new_size) / 2 + top = (height - new_size) / 2 + right = (width + new_size) / 2 + bottom = (height + new_size) / 2 + return image.crop((left, top, right, bottom)) + + def train_transforms(img): + img = center_crop(img) + img = img.resize( + (args.resolution, args.resolution), + resample=Image.BICUBIC, + reducing_gap=1 + ) + img = np.array(img).astype(np.float32) / 127.5 - 1 + img = torch.from_numpy(np.transpose(img, [2, 0, 1])) + return img + + def preprocess_train(examples): + images = [image.convert("RGB") for image in examples[image_column]] + examples["pixel_values"] = [train_transforms(image) for image in images] + examples["clip_pixel_values"] = image_processor(images,return_tensors="pt").pixel_values + return examples + with accelerator.main_process_first(): + if args.max_train_samples is not None: + dataset["train"] = dataset["train"].shuffle(seed=args.seed).select(range(args.max_train_samples)) + # Set the training transforms + train_dataset = dataset["train"].with_transform(preprocess_train) + + def collate_fn(examples): + pixel_values = torch.stack([example["pixel_values"] for example in examples]) + pixel_values = pixel_values.to(memory_format=torch.contiguous_format).float() + clip_pixel_values = torch.stack([example["clip_pixel_values"] for example in examples]) + clip_pixel_values = clip_pixel_values.to(memory_format=torch.contiguous_format).float() + return {"pixel_values": pixel_values, "clip_pixel_values": clip_pixel_values} + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + shuffle=True, + collate_fn=collate_fn, + batch_size=args.train_batch_size, + num_workers=args.dataloader_num_workers, + ) + + # Scheduler and math around the number of training steps. overrode_max_train_steps = False num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) if args.max_train_steps is None: @@ -494,7 +701,7 @@ def load_model_hook(models, input_dir): unet, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( unet, optimizer, train_dataloader, lr_scheduler ) - + # Move image_encode and vae to gpu and cast to weight_dtype image_encoder.to(accelerator.device, dtype=weight_dtype) vae.to(accelerator.device, dtype=weight_dtype) # We need to recalculate our total training steps as the size of the training dataloader may have changed. @@ -508,7 +715,9 @@ def load_model_hook(models, input_dir): # The trackers initializes automatically on the main process. if accelerator.is_main_process: tracker_config = dict(vars(args)) - accelerator.init_trackers("test", tracker_config) # args.tracker_project_name + tracker_config.pop("validation_prompts") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + # Train! total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps @@ -546,7 +755,7 @@ def load_model_hook(models, input_dir): resume_step = resume_global_step % (num_update_steps_per_epoch * args.gradient_accumulation_steps) progress_bar = tqdm(range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process) - progress_bar.set_description("training goes brrr") + progress_bar.set_description("Steps") for epoch in range(first_epoch, args.num_train_epochs): unet.train() train_loss = 0.0 @@ -559,8 +768,8 @@ def load_model_hook(models, input_dir): with accelerator.accumulate(unet): # Convert images to latent space - images, clip_images = batch - images, clip_images = images.to(weight_dtype), clip_images.to(weight_dtype) + images = batch["pixel_values"].to(weight_dtype) + clip_images = batch["clip_pixel_values"].to(weight_dtype) latents = vae.encode(images).latents image_embeds = image_encoder(clip_images).image_embeds # Sample noise that we'll add to the latents @@ -619,6 +828,26 @@ def load_model_hook(models, input_dir): if global_step % args.checkpointing_steps == 0: if accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") accelerator.save_state(save_path) logger.info(f"Saved state to {save_path}") @@ -628,6 +857,73 @@ def load_model_hook(models, input_dir): if global_step >= args.max_train_steps: break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_unet.store(unet.parameters()) + ema_unet.copy_to(unet.parameters()) + log_validation( + vae, + image_encoder, + image_processor, + unet, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original UNet parameters. + ema_unet.restore(unet.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + unet = accelerator.unwrap_model(unet) + if args.use_ema: + ema_unet.copy_to(unet.parameters()) + + pipeline = AutoPipelineForText2Image.from_pretrained( + args.pretrained_decoder_model_name_or_path, + image_encoder=image_encoder, + vae=vae, + unet=unet, + revision=args.revision, + ) + pipeline.save_pretrained(args.output_dir) + + # Run a final round of inference. + images = [] + if args.validation_prompts is not None: + logger.info("Running inference for collecting generated images...") + pipeline = pipeline.to(accelerator.device) + pipeline.torch_dtype = weight_dtype + pipeline.set_progress_bar_config(disable=True) + + if args.enable_xformers_memory_efficient_attention: + pipeline.enable_xformers_memory_efficient_attention() + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.autocast("cuda"): + image = pipeline(args.validation_prompts[i], num_inference_steps=20, generator=generator).images[0] + images.append(image) + + if args.push_to_hub: + save_model_card(args, repo_id, images, repo_folder=args.output_dir) + upload_folder( + repo_id=repo_id, + folder_path=args.output_dir, + commit_message="End of training", + ignore_patterns=["step_*", "epoch_*"], + ) + accelerator.end_training() From 6e8faa434533e271238a1cf00a90733f28d41ebb Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Mon, 4 Sep 2023 21:34:29 +0000 Subject: [PATCH 09/38] rename --- .../requirements.txt | 0 .../train_decoder.py} | 45 ++++++++++--------- .../train_decoder_lora.py} | 0 .../train_prior.py} | 0 .../train_prior_dreambooth.py} | 0 .../train_prior_lora.py} | 0 6 files changed, 23 insertions(+), 22 deletions(-) rename examples/{kandinsky2_2_train => kandinsky2_2}/requirements.txt (100%) rename examples/{kandinsky2_2_train/tune_decoder.py => kandinsky2_2/train_decoder.py} (96%) rename examples/{kandinsky2_2_train/tune_decoder_lora.py => kandinsky2_2/train_decoder_lora.py} (100%) rename examples/{kandinsky2_2_train/tune_prior.py => kandinsky2_2/train_prior.py} (100%) rename examples/{kandinsky2_2_train/tune_prior_dreambooth.py => kandinsky2_2/train_prior_dreambooth.py} (100%) rename examples/{kandinsky2_2_train/tune_prior_lora.py => kandinsky2_2/train_prior_lora.py} (100%) diff --git a/examples/kandinsky2_2_train/requirements.txt b/examples/kandinsky2_2/requirements.txt similarity index 100% rename from examples/kandinsky2_2_train/requirements.txt rename to examples/kandinsky2_2/requirements.txt diff --git a/examples/kandinsky2_2_train/tune_decoder.py b/examples/kandinsky2_2/train_decoder.py similarity index 96% rename from examples/kandinsky2_2_train/tune_decoder.py rename to examples/kandinsky2_2/train_decoder.py index 0b14413caec8..8e05aefa1bc9 100644 --- a/examples/kandinsky2_2_train/tune_decoder.py +++ b/examples/kandinsky2_2/train_decoder.py @@ -30,6 +30,7 @@ from accelerate.state import AcceleratorState from accelerate.utils import ProjectConfiguration, set_seed from datasets import load_dataset +from huggingface_hub import create_repo, upload_folder from packaging import version from PIL import Image from tqdm import tqdm @@ -40,7 +41,8 @@ from diffusers import DDPMScheduler, UNet2DConditionModel, VQModel, AutoPipelineForText2Image from diffusers.optimization import get_scheduler from diffusers.training_utils import EMAModel -from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils import check_min_version, deprecate, is_wandb_available, make_image_grid +from diffusers.utils.import_utils import is_xformers_available if is_wandb_available(): @@ -48,23 +50,12 @@ # Will error if the minimal version of diffusers is not installed. Remove at your own risks. -check_min_version("0.19.0.dev0") +check_min_version("0.21.0.dev0") logger = get_logger(__name__, log_level="INFO") -def make_image_grid(imgs, rows, cols): - assert len(imgs) == rows * cols - - w, h = imgs[0].size - grid = Image.new("RGB", size=(cols * w, rows * h)) - - for i, img in enumerate(imgs): - grid.paste(img, box=(i % cols * w, i // cols * h)) - return grid - - def save_model_card( args, repo_id: str, @@ -83,6 +74,8 @@ def save_model_card( base_model: {args.pretrained_decoder_model_name_or_path} datasets: - {args.dataset_name} +prior: +- {args.pretrained_prior_model_name_or_path} tags: - kandinsky - text-to-image @@ -91,7 +84,7 @@ def save_model_card( --- """ model_card = f""" -# Text-to-image finetuning - {repo_id} +# Finetuning - {repo_id} This pipeline was finetuned from **{args.pretrained_decoder_model_name_or_path}** on the **{args.dataset_name}** dataset. Below are some example images generated with the finetuned pipeline using the following prompts: {args.validation_prompts}: \n {img_str} @@ -104,7 +97,7 @@ def save_model_card( from diffusers import DiffusionPipeline import torch -pipeline = DiffusionPipeline.from_pretrained("{repo_id}", torch_dtype=torch.float16) +pipeline = AutoPipelineForText2Image.from_pretrained("{repo_id}", torch_dtype=torch.float16) prompt = "{args.validation_prompts[0]}" image = pipeline(prompt).images[0] image.save("my_image.png") @@ -145,11 +138,9 @@ def log_validation(vae, image_encoder, image_processor, unet, args, accelerator, pipeline = AutoPipelineForText2Image.from_pretrained( args.pretrained_decoder_model_name_or_path, vae=accelerator.unwrap_model(vae), - image_encoder=accelerator.unwrap_model(image_encoder), - image_processor=image_processor, + prior_image_encoder=accelerator.unwrap_model(image_encoder), + prior_image_processor=image_processor, unet=accelerator.unwrap_model(unet), - safety_checker=None, - revision=args.revision, torch_dtype=weight_dtype, ) pipeline = pipeline.to(accelerator.device) @@ -191,6 +182,7 @@ def log_validation(vae, image_encoder, image_processor, unet, args, accelerator, return images + def parse_args(): parser = argparse.ArgumentParser(description="Simple example of finetuning Kandinsky 2.2.") parser.add_argument( @@ -299,7 +291,6 @@ def parse_args(): "--learning_rate", type=float, default=1e-4, - required=False, help="learning rate", ) parser.add_argument( @@ -519,6 +510,18 @@ def deepspeed_zero_init_disabled_context_manager(): args.pretrained_decoder_model_name_or_path, subfolder="unet") ema_unet = EMAModel(ema_unet.parameters(), model_cls=UNet2DConditionModel, model_config=ema_unet.config) ema_unet.to(accelerator.device) + if args.enable_xformers_memory_efficient_attention: + if is_xformers_available(): + import xformers + + xformers_version = version.parse(xformers.__version__) + if xformers_version == version.parse("0.0.16"): + logger.warn( + "xFormers 0.0.16 cannot be used for training in some GPUs. If you observe problems during training, please update xFormers to at least 0.0.17. See https://huggingface.co/docs/diffusers/main/en/optimization/xformers for more details." + ) + unet.enable_xformers_memory_efficient_attention() + else: + raise ValueError("xformers is not available. Make sure it is installed correctly") def compute_snr(timesteps): """ @@ -887,10 +890,8 @@ def collate_fn(examples): pipeline = AutoPipelineForText2Image.from_pretrained( args.pretrained_decoder_model_name_or_path, - image_encoder=image_encoder, vae=vae, unet=unet, - revision=args.revision, ) pipeline.save_pretrained(args.output_dir) diff --git a/examples/kandinsky2_2_train/tune_decoder_lora.py b/examples/kandinsky2_2/train_decoder_lora.py similarity index 100% rename from examples/kandinsky2_2_train/tune_decoder_lora.py rename to examples/kandinsky2_2/train_decoder_lora.py diff --git a/examples/kandinsky2_2_train/tune_prior.py b/examples/kandinsky2_2/train_prior.py similarity index 100% rename from examples/kandinsky2_2_train/tune_prior.py rename to examples/kandinsky2_2/train_prior.py diff --git a/examples/kandinsky2_2_train/tune_prior_dreambooth.py b/examples/kandinsky2_2/train_prior_dreambooth.py similarity index 100% rename from examples/kandinsky2_2_train/tune_prior_dreambooth.py rename to examples/kandinsky2_2/train_prior_dreambooth.py diff --git a/examples/kandinsky2_2_train/tune_prior_lora.py b/examples/kandinsky2_2/train_prior_lora.py similarity index 100% rename from examples/kandinsky2_2_train/tune_prior_lora.py rename to examples/kandinsky2_2/train_prior_lora.py From d39efc7e234a21492ca2059b271aec1f5c7e9550 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Mon, 4 Sep 2023 21:36:10 +0000 Subject: [PATCH 10/38] style --- examples/kandinsky2_2/train_decoder.py | 45 ++++++++++++++------------ 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/examples/kandinsky2_2/train_decoder.py b/examples/kandinsky2_2/train_decoder.py index 8e05aefa1bc9..cb60fadc9dda 100644 --- a/examples/kandinsky2_2/train_decoder.py +++ b/examples/kandinsky2_2/train_decoder.py @@ -17,6 +17,8 @@ import logging import math import os +import shutil +from pathlib import Path import accelerate import datasets @@ -38,10 +40,10 @@ from transformers.utils import ContextManagers import diffusers -from diffusers import DDPMScheduler, UNet2DConditionModel, VQModel, AutoPipelineForText2Image +from diffusers import AutoPipelineForText2Image, DDPMScheduler, UNet2DConditionModel, VQModel from diffusers.optimization import get_scheduler from diffusers.training_utils import EMAModel -from diffusers.utils import check_min_version, deprecate, is_wandb_available, make_image_grid +from diffusers.utils import check_min_version, is_wandb_available, make_image_grid from diffusers.utils.import_utils import is_xformers_available @@ -55,7 +57,6 @@ logger = get_logger(__name__, log_level="INFO") - def save_model_card( args, repo_id: str, @@ -74,7 +75,7 @@ def save_model_card( base_model: {args.pretrained_decoder_model_name_or_path} datasets: - {args.dataset_name} -prior: +prior: - {args.pretrained_prior_model_name_or_path} tags: - kandinsky @@ -334,7 +335,13 @@ def parse_args(): ) parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") - parser.add_argument("--adam_weight_decay",type=float,default=0.0, required=False, help="weight decay_to_use",) + parser.add_argument( + "--adam_weight_decay", + type=float, + default=0.0, + required=False, + help="weight decay_to_use", + ) parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") @@ -473,7 +480,9 @@ def main(): repo_id=args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token ).repo_id noise_scheduler = DDPMScheduler.from_pretrained(args.pretrained_decoder_model_name_or_path, subfolder="scheduler") - image_processor = CLIPImageProcessor.from_pretrained(args.pretrained_prior_model_name_or_path, subfolder="image_processor") + image_processor = CLIPImageProcessor.from_pretrained( + args.pretrained_prior_model_name_or_path, subfolder="image_processor" + ) def deepspeed_zero_init_disabled_context_manager(): """ @@ -492,13 +501,12 @@ def deepspeed_zero_init_disabled_context_manager(): weight_dtype = torch.bfloat16 with ContextManagers(deepspeed_zero_init_disabled_context_manager()): vae = VQModel.from_pretrained( - args.pretrained_decoder_model_name_or_path, subfolder="movq", torch_dtype=weight_dtype).eval() + args.pretrained_decoder_model_name_or_path, subfolder="movq", torch_dtype=weight_dtype + ).eval() image_encoder = CLIPVisionModelWithProjection.from_pretrained( args.pretrained_prior_model_name_or_path, subfolder="image_encoder", torch_dtype=weight_dtype ).eval() - unet = UNet2DConditionModel.from_pretrained( - args.pretrained_decoder_model_name_or_path, subfolder="unet" - ) + unet = UNet2DConditionModel.from_pretrained(args.pretrained_decoder_model_name_or_path, subfolder="unet") # Freeze vae and image_encoder vae.requires_grad_(False) @@ -506,8 +514,7 @@ def deepspeed_zero_init_disabled_context_manager(): # Create EMA for the unet. if args.use_ema: - ema_unet = UNet2DConditionModel.from_pretrained( - args.pretrained_decoder_model_name_or_path, subfolder="unet") + ema_unet = UNet2DConditionModel.from_pretrained(args.pretrained_decoder_model_name_or_path, subfolder="unet") ema_unet = EMAModel(ema_unet.parameters(), model_cls=UNet2DConditionModel, model_config=ema_unet.config) ema_unet.to(accelerator.device) if args.enable_xformers_memory_efficient_attention: @@ -639,9 +646,7 @@ def load_model_hook(models, input_dir): image_column = args.image_column if image_column not in column_names: - raise ValueError( - f"--image_column' value '{args.image_column}' needs to be one of: {', '.join(column_names)}" - ) + raise ValueError(f"--image_column' value '{args.image_column}' needs to be one of: {', '.join(column_names)}") def center_crop(image): width, height = image.size @@ -654,11 +659,7 @@ def center_crop(image): def train_transforms(img): img = center_crop(img) - img = img.resize( - (args.resolution, args.resolution), - resample=Image.BICUBIC, - reducing_gap=1 - ) + img = img.resize((args.resolution, args.resolution), resample=Image.BICUBIC, reducing_gap=1) img = np.array(img).astype(np.float32) / 127.5 - 1 img = torch.from_numpy(np.transpose(img, [2, 0, 1])) return img @@ -666,8 +667,9 @@ def train_transforms(img): def preprocess_train(examples): images = [image.convert("RGB") for image in examples[image_column]] examples["pixel_values"] = [train_transforms(image) for image in images] - examples["clip_pixel_values"] = image_processor(images,return_tensors="pt").pixel_values + examples["clip_pixel_values"] = image_processor(images, return_tensors="pt").pixel_values return examples + with accelerator.main_process_first(): if args.max_train_samples is not None: dataset["train"] = dataset["train"].shuffle(seed=args.seed).select(range(args.max_train_samples)) @@ -680,6 +682,7 @@ def collate_fn(examples): clip_pixel_values = torch.stack([example["clip_pixel_values"] for example in examples]) clip_pixel_values = clip_pixel_values.to(memory_format=torch.contiguous_format).float() return {"pixel_values": pixel_values, "clip_pixel_values": clip_pixel_values} + train_dataloader = torch.utils.data.DataLoader( train_dataset, shuffle=True, From 03c9da0249e13baa9031ce538bd0b48cb3079025 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Mon, 4 Sep 2023 23:11:16 +0000 Subject: [PATCH 11/38] save only decoder pipeline --- examples/kandinsky2_2/train_decoder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/kandinsky2_2/train_decoder.py b/examples/kandinsky2_2/train_decoder.py index cb60fadc9dda..364ed7e03189 100644 --- a/examples/kandinsky2_2/train_decoder.py +++ b/examples/kandinsky2_2/train_decoder.py @@ -896,7 +896,7 @@ def collate_fn(examples): vae=vae, unet=unet, ) - pipeline.save_pretrained(args.output_dir) + pipeline.decoder_pipe.save_pretrained(args.output_dir) # Run a final round of inference. images = [] @@ -905,6 +905,7 @@ def collate_fn(examples): pipeline = pipeline.to(accelerator.device) pipeline.torch_dtype = weight_dtype pipeline.set_progress_bar_config(disable=True) + pipeline.enable_model_cpu_offload() if args.enable_xformers_memory_efficient_attention: pipeline.enable_xformers_memory_efficient_attention() From 810425c29a4f03f1c2fd1c77fc9b483c58bb542f Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 5 Sep 2023 08:24:42 +0000 Subject: [PATCH 12/38] update text-to-image-lora --- examples/kandinsky2_2/train_decoder_lora.py | 599 ++++++++++++++------ 1 file changed, 418 insertions(+), 181 deletions(-) diff --git a/examples/kandinsky2_2/train_decoder_lora.py b/examples/kandinsky2_2/train_decoder_lora.py index bb29398fa230..7a7e24ae56f4 100644 --- a/examples/kandinsky2_2/train_decoder_lora.py +++ b/examples/kandinsky2_2/train_decoder_lora.py @@ -1,9 +1,28 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Fine-tuning script for Kandinsky with support for LoRA.""" + import argparse +import logging import math import os +import shutil +from pathlib import Path +import datasets import numpy as np -import pandas as pd import torch import torch.nn.functional as F import torch.utils.checkpoint @@ -11,151 +30,158 @@ from accelerate import Accelerator from accelerate.logging import get_logger from accelerate.utils import ProjectConfiguration, set_seed +from datasets import load_dataset +from huggingface_hub import create_repo, upload_folder +from packaging import version from PIL import Image from tqdm import tqdm from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection import diffusers -from diffusers import DDPMScheduler, UNet2DConditionModel, VQModel +from diffusers import AutoPipelineForText2Image, DDPMScheduler, UNet2DConditionModel, VQModel from diffusers.loaders import AttnProcsLayers from diffusers.models.attention_processor import LoRAAttnAddedKVProcessor from diffusers.optimization import get_scheduler -from diffusers.utils import is_wandb_available +from diffusers.utils import check_min_version, is_wandb_available +from diffusers.utils.import_utils import is_xformers_available -if is_wandb_available(): - pass +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.21.0.dev0") logger = get_logger(__name__, log_level="INFO") +def save_model_card(repo_id: str, images=None, base_model=str, dataset_name=str, repo_folder=None): + img_str = "" + for i, image in enumerate(images): + image.save(os.path.join(repo_folder, f"image_{i}.png")) + img_str += f"![img_{i}](./image_{i}.png)\n" + + yaml = f""" +--- +license: creativeml-openrail-m +base_model: {base_model} +tags: +- kandinsky +- text-to-image +- diffusers +- lora +inference: true +--- + """ + model_card = f""" +# LoRA text2image fine-tuning - {repo_id} +These are LoRA adaption weights for {base_model}. The weights were fine-tuned on the {dataset_name} dataset. You can find some example images in the following. \n +{img_str} +""" + with open(os.path.join(repo_folder, "README.md"), "w") as f: + f.write(yaml + model_card) + + def parse_args(): - parser = argparse.ArgumentParser(description="Simple example of finetuning Kandinsky 2.2.") + parser = argparse.ArgumentParser(description="Simple example of finetuning Kandinsky 2.2 with LoRA.") parser.add_argument( - "--image_resolution", - type=int, - default=512, - required=False, - help="Image resolution", - ) - parser.add_argument( - "--pretrained_kandinsky_path", + "--pretrained_decoder_model_name_or_path", type=str, default="kandinsky-community/kandinsky-2-2-decoder", required=False, help="Path to pretrained model or model identifier from huggingface.co/models.", ) parser.add_argument( - "--pretrained_vae_path", - type=str, - default="kandinsky-community/kandinsky-2-2-decoder", - required=False, - help="Path to pretrained vae.", - ) - parser.add_argument( - "--pretrained_image_encoder", + "--pretrained_prior_model_name_or_path", type=str, default="kandinsky-community/kandinsky-2-2-prior", required=False, - help="Path to pretrained image encoder.", - ) - parser.add_argument( - "--scheduler_path", - type=str, - default="kandinsky-community/kandinsky-2-2-decoder", - required=False, - help="Path to scheduler.", + help="Path to pretrained model or model identifier from huggingface.co/models.", ) parser.add_argument( - "--image_processor_path", + "--dataset_name", type=str, - default="kandinsky-community/kandinsky-2-2-prior", - required=False, - help="Path to image_processor.", + default=None, + help=( + "The name of the Dataset (from the HuggingFace hub) to train on (could be your own, possibly private," + " dataset). It can also be a path pointing to a local copy of a dataset in your filesystem," + " or to a folder containing files that 🤗 Datasets can understand." + ), ) parser.add_argument( - "--train_image_folder", + "--dataset_config_name", type=str, default=None, - required=False, - help="Path to train image folder.", + help="The config of the Dataset, leave as None if there's only one config.", ) parser.add_argument( - "--train_images_paths_csv", + "--train_data_dir", type=str, default=None, - required=False, - help="Path to csv with train images paths(define train_image_folder or train_images_paths_csv) with column paths.", + help=( + "A folder containing the training data. Folder contents must follow the structure described in" + " https://huggingface.co/docs/datasets/image_dataset#imagefolder. In particular, a `metadata.jsonl` file" + " must exist to provide the captions for the images. Ignored if `dataset_name` is specified." + ), ) parser.add_argument( - "--val_image_folder", - type=str, - default=None, - required=False, - help="Path to val image folder.", + "--image_column", type=str, default="image", help="The column of the dataset containing an image." ) parser.add_argument( - "--val_images_paths_csv", - type=str, - default=None, - required=False, - help="Path to csv with val images paths with column paths.", + "--validation_prompt", type=str, default=None, help="A prompt that is sampled during training for inference." ) parser.add_argument( - "--train_batch_size", + "--num_validation_images", type=int, - default=1, - required=False, - help="train batch size", + default=4, + help="Number of images that should be generated during validation with `validation_prompt`.", ) parser.add_argument( - "--val_batch_size", + "--validation_epochs", type=int, default=1, - required=False, - help="val batch size", - ) - parser.add_argument( - "--lr", - type=float, - default=1e-4, - required=False, - help="learning rate", - ) - parser.add_argument( - "--weight_decay", - type=float, - default=0.0, - required=False, - help="weight decay", + help=( + "Run fine-tuning validation every X epochs. The validation process consists of running the prompt" + " `args.validation_prompt` multiple times: `args.num_validation_images`." + ), ) parser.add_argument( - "--num_epochs", + "--max_train_samples", type=int, - default=0.0, - required=False, - help="num of epochs", + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), ) parser.add_argument( "--output_dir", type=str, - default="kandi_2_2-model-finetuned", + default="kandi_2_2-model-finetuned-lora", help="The output directory where the model predictions and checkpoints will be written.", ) parser.add_argument( - "--lr_scheduler", + "--cache_dir", type=str, - default="constant", + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--resolution", + type=int, + default=512, help=( - 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' - ' "constant", "constant_with_warmup"]' + "The resolution for input images, all the images in the train/validation dataset will be resized to this" + " resolution" ), ) + parser.add_argument( + "--train_batch_size", type=int, default=1, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) parser.add_argument( "--max_train_steps", type=int, default=None, - help="Total number of training steps to perform. If provided, overrides num_epochs.", + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", ) parser.add_argument( "--gradient_accumulation_steps", @@ -168,6 +194,21 @@ def parse_args(): action="store_true", help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) parser.add_argument( "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." ) @@ -181,15 +222,6 @@ def parse_args(): parser.add_argument( "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." ) - parser.add_argument( - "--resume_from_checkpoint", - type=str, - default=None, - help=( - "Whether training should be resumed from a previous checkpoint. Use a path saved by" - ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' - ), - ) parser.add_argument( "--allow_tf32", action="store_true", @@ -206,6 +238,19 @@ def parse_args(): "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." ), ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=0.0, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) parser.add_argument( "--logging_dir", type=str, @@ -249,18 +294,26 @@ def parse_args(): "--checkpoints_total_limit", type=int, default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, help=( - "Max number of checkpoints to store. Passed as `total_limit` to the `Accelerator` `ProjectConfiguration`." - " See Accelerator::save_state https://huggingface.co/docs/accelerate/package_reference/accelerator#accelerate.Accelerator.save_state" - " for more docs" + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' ), ) - parser.add_argument("--rank", type=int, default=4, help="A rank of LORA") - parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") - parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") - parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") - parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") - parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument( + "--enable_xformers_memory_efficient_attention", action="store_true", help="Whether or not to use xformers." + ) + parser.add_argument( + "--rank", + type=int, + default=4, + help=("The dimension of the LoRA update matrices."), + ) args = parser.parse_args() env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) @@ -268,52 +321,15 @@ def parse_args(): args.local_rank = env_local_rank # Sanity checks - if args.train_image_folder is None and args.train_images_paths_csv is None: - raise ValueError("Need either a train_image_folder or a train_images_paths_csv.") + if args.dataset_name is None and args.train_data_dir is None: + raise ValueError("Need either a dataset name or a training folder.") return args -def center_crop(image): - width, height = image.size - new_size = min(width, height) - left = (width - new_size) / 2 - top = (height - new_size) / 2 - right = (width + new_size) / 2 - bottom = (height + new_size) / 2 - return image.crop((left, top, right, bottom)) - - -class ImageDataset(torch.utils.data.Dataset): - def __init__(self, image_folder=None, images_paths_csv=None, image_processor=None, img_size=512): - assert image_folder is None or images_paths_csv is None - self.image_processor = image_processor - self.img_size = img_size - if images_paths_csv is not None: - self.paths = pd.read_csv(images_paths_csv)["paths"].values - else: - self.paths = [ - os.path.join(image_folder, path) - for path in os.listdir(image_folder) - if ".jpg" in path.lower() or ".png" in path.lower() - ] - - def __len__(self): - return len(self.paths) - - def __getitem__(self, i): - img = Image.open(self.paths[i]) - clip_image = self.image_processor(img) - img = center_crop(img) - img = img.resize((self.img_size, self.img_size), resample=Image.BICUBIC, reducing_gap=1) - img = np.array(img.convert("RGB")) - img = img.astype(np.float32) / 127.5 - 1 - return np.transpose(img, [2, 0, 1]), clip_image.pixel_values[0] - - def main(): args = parse_args() - logging_dir = os.path.join(args.output_dir, args.logging_dir) + logging_dir = Path(args.output_dir, args.logging_dir) accelerator_project_config = ProjectConfiguration( total_limit=args.checkpoints_total_limit, project_dir=args.output_dir, logging_dir=logging_dir ) @@ -323,43 +339,71 @@ def main(): log_with=args.report_to, project_config=accelerator_project_config, ) + if args.report_to == "wandb": + if not is_wandb_available(): + raise ImportError("Make sure to install wandb if you want to use it for logging during training.") + import wandb + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) logger.info(accelerator.state, main_process_only=False) if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() transformers.utils.logging.set_verbosity_warning() diffusers.utils.logging.set_verbosity_info() else: + datasets.utils.logging.set_verbosity_error() transformers.utils.logging.set_verbosity_error() diffusers.utils.logging.set_verbosity_error() + # If passed along, set the training seed now. if args.seed is not None: set_seed(args.seed) + + # Handle the repository creation if accelerator.is_main_process: if args.output_dir is not None: os.makedirs(args.output_dir, exist_ok=True) - noise_scheduler = DDPMScheduler.from_pretrained(args.scheduler_path, subfolder="scheduler") - image_processor = CLIPImageProcessor.from_pretrained(args.image_processor_path, subfolder="image_processor") + if args.push_to_hub: + repo_id = create_repo( + repo_id=args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token + ).repo_id + # Load scheduler, tokenizer and models. + noise_scheduler = DDPMScheduler.from_pretrained(args.pretrained_decoder_model_name_or_path, subfolder="scheduler") + image_processor = CLIPImageProcessor.from_pretrained( + args.pretrained_prior_model_name_or_path, subfolder="image_processor" + ) + image_encoder = CLIPVisionModelWithProjection.from_pretrained( + args.pretrained_prior_model_name_or_path, subfolder="image_encoder" + ) + + vae = VQModel.from_pretrained(args.pretrained_decoder_model_name_or_path, subfolder="movq") + + unet = UNet2DConditionModel.from_pretrained(args.pretrained_decoder_model_name_or_path, subfolder="unet") + # freeze parameters of models to save more memory + unet.requires_grad_(False) + vae.requires_grad_(False) + + image_encoder.requires_grad_(False) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora unet) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. weight_dtype = torch.float32 if accelerator.mixed_precision == "fp16": weight_dtype = torch.float16 elif accelerator.mixed_precision == "bf16": weight_dtype = torch.bfloat16 - vae = VQModel.from_pretrained(args.pretrained_vae_path, subfolder="movq", torch_dtype=weight_dtype).eval() - image_encoder = CLIPVisionModelWithProjection.from_pretrained( - args.pretrained_image_encoder, subfolder="image_encoder", torch_dtype=weight_dtype - ).eval() - unet = UNet2DConditionModel.from_pretrained( - args.pretrained_kandinsky_path, subfolder="unet", torch_dtype=weight_dtype - ) + # Move unet, vae and text_encoder to device and cast to weight_dtype + unet.to(accelerator.device, dtype=weight_dtype) + vae.to(accelerator.device, dtype=weight_dtype) + image_encoder.to(accelerator.device, dtype=weight_dtype) - # Freeze vae and image_encoder - vae.requires_grad_(False) - image_encoder.requires_grad_(False) - unet.requires_grad_(False) - image_encoder.to(accelerator.device) - vae.to(accelerator.device) - unet.to(accelerator.device) lora_attn_procs = {} for name in unet.attn_processors.keys(): cross_attention_dim = None if name.endswith("attn1.processor") else unet.config.cross_attention_dim @@ -371,6 +415,7 @@ def main(): elif name.startswith("down_blocks"): block_id = int(name[len("down_blocks.")]) hidden_size = unet.config.block_out_channels[block_id] + lora_attn_procs[name] = LoRAAttnAddedKVProcessor( hidden_size=hidden_size, cross_attention_dim=cross_attention_dim, @@ -379,6 +424,19 @@ def main(): unet.set_attn_processor(lora_attn_procs) + if args.enable_xformers_memory_efficient_attention: + if is_xformers_available(): + import xformers + + xformers_version = version.parse(xformers.__version__) + if xformers_version == version.parse("0.0.16"): + logger.warn( + "xFormers 0.0.16 cannot be used for training in some GPUs. If you observe problems during training, please update xFormers to at least 0.0.17. See https://huggingface.co/docs/diffusers/main/en/optimization/xformers for more details." + ) + unet.enable_xformers_memory_efficient_attention() + else: + raise ValueError("xformers is not available. Make sure it is installed correctly") + def compute_snr(timesteps): """ Computes SNR as per https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L847-L849 @@ -419,35 +477,91 @@ def compute_snr(timesteps): optimizer_cls = bnb.optim.AdamW8bit else: optimizer_cls = torch.optim.AdamW + optimizer = optimizer_cls( lora_layers.parameters(), - lr=args.lr, + lr=args.learning_rate, betas=(args.adam_beta1, args.adam_beta2), - weight_decay=args.weight_decay, + weight_decay=args.adam_weight_decay, eps=args.adam_epsilon, ) - train_dataset = ImageDataset( - image_folder=args.train_image_folder, - images_paths_csv=args.train_images_paths_csv, - image_processor=image_processor, - img_size=args.image_resolution, - ) - train_dataloader = torch.utils.data.DataLoader( - train_dataset, batch_size=args.train_batch_size, num_workers=args.dataloader_num_workers - ) - if args.val_image_folder is not None or args.val_images_paths_csv is not None: - ImageDataset( - image_folder=args.val_image_folder, - images_paths_csv=args.val_images_paths_csv, - image_processor=image_processor, - img_size=args.image_resolution, - ) - torch.utils.data.DataLoader( - train_dataset, batch_size=args.val_batch_size, num_workers=args.dataloader_num_workers + # Get the datasets: you can either provide your own training and evaluation files (see below) + # or specify a Dataset from the hub (the dataset will be downloaded automatically from the datasets Hub). + + # In distributed training, the load_dataset function guarantees that only one local process can concurrently + # download the dataset. + if args.dataset_name is not None: + # Downloading and loading a dataset from the hub. + dataset = load_dataset( + args.dataset_name, + args.dataset_config_name, + cache_dir=args.cache_dir, ) else: - pass + data_files = {} + if args.train_data_dir is not None: + data_files["train"] = os.path.join(args.train_data_dir, "**") + dataset = load_dataset( + "imagefolder", + data_files=data_files, + cache_dir=args.cache_dir, + ) + # See more about loading custom images at + # https://huggingface.co/docs/datasets/v2.4.0/en/image_load#imagefolder + + # Preprocessing the datasets. + # We need to tokenize inputs and targets. + column_names = dataset["train"].column_names + + image_column = args.image_column + if image_column not in column_names: + raise ValueError(f"--image_column' value '{args.image_column}' needs to be one of: {', '.join(column_names)}") + + def center_crop(image): + width, height = image.size + new_size = min(width, height) + left = (width - new_size) / 2 + top = (height - new_size) / 2 + right = (width + new_size) / 2 + bottom = (height + new_size) / 2 + return image.crop((left, top, right, bottom)) + + def train_transforms(img): + img = center_crop(img) + img = img.resize((args.resolution, args.resolution), resample=Image.BICUBIC, reducing_gap=1) + img = np.array(img).astype(np.float32) / 127.5 - 1 + img = torch.from_numpy(np.transpose(img, [2, 0, 1])) + return img + + def preprocess_train(examples): + images = [image.convert("RGB") for image in examples[image_column]] + examples["pixel_values"] = [train_transforms(image) for image in images] + examples["clip_pixel_values"] = image_processor(images, return_tensors="pt").pixel_values + return examples + + with accelerator.main_process_first(): + if args.max_train_samples is not None: + dataset["train"] = dataset["train"].shuffle(seed=args.seed).select(range(args.max_train_samples)) + # Set the training transforms + train_dataset = dataset["train"].with_transform(preprocess_train) + + def collate_fn(examples): + pixel_values = torch.stack([example["pixel_values"] for example in examples]) + pixel_values = pixel_values.to(memory_format=torch.contiguous_format).float() + clip_pixel_values = torch.stack([example["clip_pixel_values"] for example in examples]) + clip_pixel_values = clip_pixel_values.to(memory_format=torch.contiguous_format).float() + return {"pixel_values": pixel_values, "clip_pixel_values": clip_pixel_values} + + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + shuffle=True, + collate_fn=collate_fn, + batch_size=args.train_batch_size, + num_workers=args.dataloader_num_workers, + ) + + # Scheduler and math around the number of training steps. overrode_max_train_steps = False num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) if args.max_train_steps is None: @@ -460,6 +574,7 @@ def compute_snr(timesteps): num_warmup_steps=args.lr_warmup_steps * args.gradient_accumulation_steps, num_training_steps=args.max_train_steps * args.gradient_accumulation_steps, ) + # Prepare everything with our `accelerator`. lora_layers, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( lora_layers, optimizer, train_dataloader, lr_scheduler ) @@ -474,8 +589,8 @@ def compute_snr(timesteps): # We need to initialize the trackers we use, and also store our configuration. # The trackers initializes automatically on the main process. if accelerator.is_main_process: - tracker_config = dict(vars(args)) - accelerator.init_trackers("test", tracker_config) # args.tracker_project_name + accelerator.init_trackers("text2image-fine-tune", config=vars(args)) + # Train! total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps @@ -488,6 +603,8 @@ def compute_snr(timesteps): logger.info(f" Total optimization steps = {args.max_train_steps}") global_step = 0 first_epoch = 0 + + # Potentially load in the weights and states from a previous save if args.resume_from_checkpoint: if args.resume_from_checkpoint != "latest": path = os.path.basename(args.resume_from_checkpoint) @@ -512,8 +629,10 @@ def compute_snr(timesteps): first_epoch = global_step // num_update_steps_per_epoch resume_step = resume_global_step % (num_update_steps_per_epoch * args.gradient_accumulation_steps) + # Only show the progress bar once on each machine. progress_bar = tqdm(range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process) - progress_bar.set_description("training goes brrr") + progress_bar.set_description("Steps") + for epoch in range(first_epoch, args.num_train_epochs): unet.train() train_loss = 0.0 @@ -526,8 +645,8 @@ def compute_snr(timesteps): with accelerator.accumulate(unet): # Convert images to latent space - images, clip_images = batch - images, clip_images = images.to(weight_dtype), clip_images.to(weight_dtype) + images = batch["pixel_values"].to(weight_dtype) + clip_images = batch["clip_pixel_values"].to(weight_dtype) latents = vae.encode(images).latents image_embeds = image_encoder(clip_images).image_embeds # Sample noise that we'll add to the latents @@ -585,6 +704,26 @@ def compute_snr(timesteps): if global_step % args.checkpointing_steps == 0: if accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") accelerator.save_state(save_path) logger.info(f"Saved state to {save_path}") @@ -594,6 +733,104 @@ def compute_snr(timesteps): if global_step >= args.max_train_steps: break + + if accelerator.is_main_process: + if args.validation_prompt is not None and epoch % args.validation_epochs == 0: + logger.info( + f"Running validation... \n Generating {args.num_validation_images} images with prompt:" + f" {args.validation_prompt}." + ) + # create pipeline + pipeline = AutoPipelineForText2Image.from_pretrained( + args.pretrained_decoder_model_name_or_path, + unet=accelerator.unwrap_model(unet), + torch_dtype=weight_dtype, + ) + pipeline = pipeline.to(accelerator.device) + pipeline.set_progress_bar_config(disable=True) + + # run inference + generator = torch.Generator(device=accelerator.device) + if args.seed is not None: + generator = generator.manual_seed(args.seed) + images = [] + for _ in range(args.num_validation_images): + images.append( + pipeline(args.validation_prompt, num_inference_steps=30, generator=generator).images[0] + ) + + for tracker in accelerator.trackers: + if tracker.name == "tensorboard": + np_images = np.stack([np.asarray(img) for img in images]) + tracker.writer.add_images("validation", np_images, epoch, dataformats="NHWC") + if tracker.name == "wandb": + tracker.log( + { + "validation": [ + wandb.Image(image, caption=f"{i}: {args.validation_prompt}") + for i, image in enumerate(images) + ] + } + ) + + del pipeline + torch.cuda.empty_cache() + + # Save the lora layers + accelerator.wait_for_everyone() + if accelerator.is_main_process: + unet = unet.to(torch.float32) + unet.save_attn_procs(args.output_dir) + + if args.push_to_hub: + save_model_card( + repo_id, + images=images, + base_model=args.pretrained_decoder_model_name_or_path, + dataset_name=args.dataset_name, + repo_folder=args.output_dir, + ) + upload_folder( + repo_id=repo_id, + folder_path=args.output_dir, + commit_message="End of training", + ignore_patterns=["step_*", "epoch_*"], + ) + + # Final inference + # Load previous pipeline + pipeline = AutoPipelineForText2Image.from_pretrained( + args.pretrained_decoder_model_name_or_path, torch_dtype=weight_dtype + ) + pipeline = pipeline.to(accelerator.device) + + # load attention processors + pipeline.unet.load_attn_procs(args.output_dir) + + # run inference + generator = torch.Generator(device=accelerator.device) + if args.seed is not None: + generator = generator.manual_seed(args.seed) + images = [] + for _ in range(args.num_validation_images): + images.append(pipeline(args.validation_prompt, num_inference_steps=30, generator=generator).images[0]) + + if accelerator.is_main_process: + for tracker in accelerator.trackers: + if len(images) != 0: + if tracker.name == "tensorboard": + np_images = np.stack([np.asarray(img) for img in images]) + tracker.writer.add_images("test", np_images, epoch, dataformats="NHWC") + if tracker.name == "wandb": + tracker.log( + { + "test": [ + wandb.Image(image, caption=f"{i}: {args.validation_prompt}") + for i, image in enumerate(images) + ] + } + ) + accelerator.end_training() From 4e3a210caa41d6cc5fff122ae9f4ee50e5847bc4 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 5 Sep 2023 08:56:20 +0000 Subject: [PATCH 13/38] remove xformer --- examples/kandinsky2_2/train_decoder_lora.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/examples/kandinsky2_2/train_decoder_lora.py b/examples/kandinsky2_2/train_decoder_lora.py index 7a7e24ae56f4..9d96a936d0ca 100644 --- a/examples/kandinsky2_2/train_decoder_lora.py +++ b/examples/kandinsky2_2/train_decoder_lora.py @@ -32,7 +32,6 @@ from accelerate.utils import ProjectConfiguration, set_seed from datasets import load_dataset from huggingface_hub import create_repo, upload_folder -from packaging import version from PIL import Image from tqdm import tqdm from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection @@ -43,7 +42,6 @@ from diffusers.models.attention_processor import LoRAAttnAddedKVProcessor from diffusers.optimization import get_scheduler from diffusers.utils import check_min_version, is_wandb_available -from diffusers.utils.import_utils import is_xformers_available # Will error if the minimal version of diffusers is not installed. Remove at your own risks. @@ -305,9 +303,6 @@ def parse_args(): ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' ), ) - parser.add_argument( - "--enable_xformers_memory_efficient_attention", action="store_true", help="Whether or not to use xformers." - ) parser.add_argument( "--rank", type=int, @@ -424,19 +419,6 @@ def main(): unet.set_attn_processor(lora_attn_procs) - if args.enable_xformers_memory_efficient_attention: - if is_xformers_available(): - import xformers - - xformers_version = version.parse(xformers.__version__) - if xformers_version == version.parse("0.0.16"): - logger.warn( - "xFormers 0.0.16 cannot be used for training in some GPUs. If you observe problems during training, please update xFormers to at least 0.0.17. See https://huggingface.co/docs/diffusers/main/en/optimization/xformers for more details." - ) - unet.enable_xformers_memory_efficient_attention() - else: - raise ValueError("xformers is not available. Make sure it is installed correctly") - def compute_snr(timesteps): """ Computes SNR as per https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L847-L849 From 9ec48d2644f372a1e97f78a9f1333eb13b64d1fc Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Wed, 6 Sep 2023 06:59:32 +0000 Subject: [PATCH 14/38] update train_prior --- examples/kandinsky2_2/train_prior.py | 648 +++++++++++++++++++++------ 1 file changed, 499 insertions(+), 149 deletions(-) diff --git a/examples/kandinsky2_2/train_prior.py b/examples/kandinsky2_2/train_prior.py index 8a75f73f5eb8..b42602982e40 100644 --- a/examples/kandinsky2_2/train_prior.py +++ b/examples/kandinsky2_2/train_prior.py @@ -1,9 +1,29 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + import argparse +import logging import math import os +import random +import shutil +from pathlib import Path import accelerate -import pandas as pd +import datasets +import numpy as np import torch import torch.nn.functional as F import torch.utils.checkpoint @@ -12,117 +32,227 @@ from accelerate.logging import get_logger from accelerate.state import AcceleratorState from accelerate.utils import ProjectConfiguration, set_seed +from datasets import load_dataset +from huggingface_hub import create_repo, upload_folder from packaging import version -from PIL import Image from tqdm import tqdm from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection from transformers.utils import ContextManagers import diffusers -from diffusers import DDPMScheduler, PriorTransformer +from diffusers import AutoPipelineForText2Image, DDPMScheduler, PriorTransformer from diffusers.optimization import get_scheduler from diffusers.training_utils import EMAModel -from diffusers.utils import is_wandb_available +from diffusers.utils import check_min_version, is_wandb_available, make_image_grid if is_wandb_available(): - pass + import wandb + + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.21.0.dev0") logger = get_logger(__name__, log_level="INFO") +DATASET_NAME_MAPPING = { + "lambdalabs/pokemon-blip-captions": ("image", "text"), +} + + +def save_model_card( + args, + repo_id: str, + images=None, + repo_folder=None, +): + img_str = "" + if len(images) > 0: + image_grid = make_image_grid(images, 1, len(args.validation_prompts)) + image_grid.save(os.path.join(repo_folder, "val_imgs_grid.png")) + img_str += "![val_imgs_grid](./val_imgs_grid.png)\n" + + yaml = f""" +--- +license: creativeml-openrail-m +base_model: {args.pretrained_prior_model_name_or_path} +datasets: +- {args.dataset_name} +tags: +- kandinsky +- text-to-image +- diffusers +inference: true +--- + """ + model_card = f""" +# Finetuning - {repo_id} + +This pipeline was finetuned from **{args.pretrained_prior_model_name_or_path}** on the **{args.dataset_name}** dataset. Below are some example images generated with the finetuned pipeline using the following prompts: {args.validation_prompts}: \n +{img_str} + +## Pipeline usage + +You can use the pipeline like so: + +```python +from diffusers import DiffusionPipeline +import torch + +pipe_prior = DiffusionPipeline.from_pretrained("{repo_id}", torch_dtype=torch.float16) +pipe_t2i = DiffusionPipeline.from_pretrained("{args.pretrained_decoder_model_name_or_path}", torch_dtype=torch.float16) +prompt = "{args.validation_prompts[0]}" +image_embeds, negative_image_embeds = pipe_prior(prompt, guidance_scale=1.0).to_tuple() +image = pipe_t2i(image_embeds=image_embeds, negative_image_embeds=negative_image_embeds).images[0] +image.save("my_image.png") +``` + +## Training info + +These are the key hyperparameters used during training: + +* Epochs: {args.num_train_epochs} +* Learning rate: {args.learning_rate} +* Batch size: {args.train_batch_size} +* Gradient accumulation steps: {args.gradient_accumulation_steps} +* Image resolution: {args.resolution} +* Mixed-precision: {args.mixed_precision} + +""" + wandb_info = "" + if is_wandb_available(): + wandb_run_url = None + if wandb.run is not None: + wandb_run_url = wandb.run.url + + if wandb_run_url is not None: + wandb_info = f""" +More information on all the CLI arguments and the environment are available on your [`wandb` run page]({wandb_run_url}). +""" + + model_card += wandb_info + + with open(os.path.join(repo_folder, "README.md"), "w") as f: + f.write(yaml + model_card) + + +def log_validation( + vae, image_encoder, image_processor, text_encoder, tokenizer, prior, args, accelerator, weight_dtype, epoch +): + logger.info("Running validation... ") + + pipeline = AutoPipelineForText2Image.from_pretrained( + args.pretrained_decoder_model_name_or_path, + prior_image_encoder=accelerator.unwrap_model(image_encoder), + prior_image_processor=image_processor, + prior_text_encoder=accelerator.unwrap_model(text_encoder), + prior_tokenizer=tokenizer, + prior_prior=accelerator.unwrap_model(prior), + torch_dtype=weight_dtype, + ) + pipeline = pipeline.to(accelerator.device) + pipeline.set_progress_bar_config(disable=True) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.autocast("cuda"): + image = pipeline(args.validation_prompts[i], num_inference_steps=20, generator=generator).images[0] + + images.append(image) + + for tracker in accelerator.trackers: + if tracker.name == "tensorboard": + np_images = np.stack([np.asarray(img) for img in images]) + tracker.writer.add_images("validation", np_images, epoch, dataformats="NHWC") + elif tracker.name == "wandb": + tracker.log( + { + "validation": [ + wandb.Image(image, caption=f"{i}: {args.validation_prompts[i]}") + for i, image in enumerate(images) + ] + } + ) + else: + logger.warn(f"image logging not implemented for {tracker.name}") + + del pipeline + torch.cuda.empty_cache() + + return images + def parse_args(): parser = argparse.ArgumentParser(description="Simple example of finetuning Kandinsky 2.2.") parser.add_argument( - "--pretrained_prior_path", + "--pretrained_decoder_model_name_or_path", type=str, - default="kandinsky-community/kandinsky-2-2-prior", - required=False, - help="Path to pretrained prior model or model identifier from huggingface.co/models.", - ) - parser.add_argument( - "--pretrained_image_encoder", - type=str, - default="kandinsky-community/kandinsky-2-2-prior", - required=False, - help="Path to pretrained image encoder.", - ) - parser.add_argument( - "--scheduler_path", - type=str, - default="kandinsky-community/kandinsky-2-2-prior", + default="kandinsky-community/kandinsky-2-2-decoder", required=False, - help="Path to scheduler.", + help="Path to pretrained model or model identifier from huggingface.co/models.", ) parser.add_argument( - "--image_processor_path", + "--pretrained_prior_model_name_or_path", type=str, default="kandinsky-community/kandinsky-2-2-prior", required=False, - help="Path to image_processor.", + help="Path to pretrained model or model identifier from huggingface.co/models.", ) parser.add_argument( - "--text_encoder_path", + "--dataset_name", type=str, - default="kandinsky-community/kandinsky-2-2-prior", - required=False, - help="Path to text_encoder.", - ) - parser.add_argument( - "--tokenizer_path", - type=str, - default="kandinsky-community/kandinsky-2-2-prior", - required=False, - help="Path to tokenizer.", + default=None, + help=( + "The name of the Dataset (from the HuggingFace hub) to train on (could be your own, possibly private," + " dataset). It can also be a path pointing to a local copy of a dataset in your filesystem," + " or to a folder containing files that 🤗 Datasets can understand." + ), ) parser.add_argument( - "--train_images_paths_csv", + "--dataset_config_name", type=str, default=None, - required=False, - help="Path to csv with train images paths with column paths and caption.", + help="The config of the Dataset, leave as None if there's only one config.", ) parser.add_argument( - "--val_images_paths_csv", + "--train_data_dir", type=str, default=None, - required=False, - help="Path to csv with val images paths with column paths and caption.", + help=( + "A folder containing the training data. Folder contents must follow the structure described in" + " https://huggingface.co/docs/datasets/image_dataset#imagefolder. In particular, a `metadata.jsonl` file" + " must exist to provide the captions for the images. Ignored if `dataset_name` is specified." + ), ) parser.add_argument( - "--train_batch_size", - type=int, - default=1, - required=False, - help="train batch size", + "--image_column", type=str, default="image", help="The column of the dataset containing an image." ) parser.add_argument( - "--val_batch_size", - type=int, - default=1, - required=False, - help="val batch size", - ) - parser.add_argument( - "--lr", - type=float, - default=1e-4, - required=False, - help="learning rate", + "--caption_column", + type=str, + default="text", + help="The column of the dataset containing a caption or a list of captions.", ) parser.add_argument( - "--weight_decay", - type=float, - default=0.0, - required=False, - help="weight decay", + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), ) parser.add_argument( - "--num_epochs", - type=int, - default=0.0, - required=False, - help="num of epochs", + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), ) parser.add_argument( "--output_dir", @@ -131,19 +261,30 @@ def parse_args(): help="The output directory where the model predictions and checkpoints will be written.", ) parser.add_argument( - "--lr_scheduler", + "--cache_dir", type=str, - default="constant", + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--resolution", + type=int, + default=512, help=( - 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' - ' "constant", "constant_with_warmup"]' + "The resolution for input images, all the images in the train/validation dataset will be resized to this" + " resolution" ), ) + parser.add_argument( + "--train_batch_size", type=int, default=1, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) parser.add_argument( "--max_train_steps", type=int, default=None, - help="Total number of training steps to perform. If provided, overrides num_epochs.", + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", ) parser.add_argument( "--gradient_accumulation_steps", @@ -152,9 +293,19 @@ def parse_args(): help="Number of updates steps to accumulate before performing a backward/update pass.", ) parser.add_argument( - "--gradient_checkpointing", - action="store_true", - help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + "--learning_rate", + type=float, + default=1e-4, + help="learning rate", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), ) parser.add_argument( "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." @@ -169,16 +320,6 @@ def parse_args(): parser.add_argument( "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." ) - parser.add_argument( - "--resume_from_checkpoint", - type=str, - default=None, - help=( - "Whether training should be resumed from a previous checkpoint. Use a path saved by" - ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' - ), - ) - parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") parser.add_argument( "--allow_tf32", action="store_true", @@ -187,6 +328,7 @@ def parse_args(): " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" ), ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") parser.add_argument( "--dataloader_num_workers", type=int, @@ -195,6 +337,25 @@ def parse_args(): "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." ), ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument( + "--adam_weight_decay", + type=float, + default=0.0, + required=False, + help="weight decay_to_use", + ) + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) parser.add_argument( "--logging_dir", type=str, @@ -238,17 +399,32 @@ def parse_args(): "--checkpoints_total_limit", type=int, default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, help=( - "Max number of checkpoints to store. Passed as `total_limit` to the `Accelerator` `ProjectConfiguration`." - " See Accelerator::save_state https://huggingface.co/docs/accelerate/package_reference/accelerator#accelerate.Accelerator.save_state" - " for more docs" + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" ), ) - parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") - parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") - parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") - parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") - parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") args = parser.parse_args() env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) @@ -256,39 +432,12 @@ def parse_args(): args.local_rank = env_local_rank # Sanity checks - if args.train_images_paths_csv is None: - raise ValueError("Need either a train_image_folder or a train_images_paths_csv.") + if args.dataset_name is None and args.train_data_dir is None: + raise ValueError("Need either a dataset name or a training folder.") return args -class ImageDataset(torch.utils.data.Dataset): - def __init__(self, images_paths_csv=None, image_processor=None, tokenizer=None): - self.image_processor = image_processor - self.tokenizer = tokenizer - df = pd.read_csv(images_paths_csv) - self.paths = df["paths"].values - self.captions = df["caption"].values - - def __len__(self): - return len(self.paths) - - def __getitem__(self, i): - img = Image.open(self.paths[i]) - clip_image = self.image_processor(img) - text_inputs = self.tokenizer( - self.captions[i], - padding="max_length", - max_length=self.tokenizer.model_max_length, - truncation=True, - return_tensors="pt", - ) - text_input_ids = text_inputs.input_ids[0] - text_mask = text_inputs.attention_mask.bool()[0] - - return text_input_ids, text_mask, clip_image.pixel_values[0] - - def main(): args = parse_args() logging_dir = os.path.join(args.output_dir, args.logging_dir) @@ -301,23 +450,43 @@ def main(): log_with=args.report_to, project_config=accelerator_project_config, ) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) logger.info(accelerator.state, main_process_only=False) if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() transformers.utils.logging.set_verbosity_warning() diffusers.utils.logging.set_verbosity_info() else: + datasets.utils.logging.set_verbosity_error() transformers.utils.logging.set_verbosity_error() diffusers.utils.logging.set_verbosity_error() + # If passed along, set the training seed now. if args.seed is not None: set_seed(args.seed) + + # Handle the repository creation if accelerator.is_main_process: if args.output_dir is not None: os.makedirs(args.output_dir, exist_ok=True) + if args.push_to_hub: + repo_id = create_repo( + repo_id=args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token + ).repo_id + + # Load scheduler, image_processor, tokenizer and models. noise_scheduler = DDPMScheduler(beta_schedule="squaredcos_cap_v2", prediction_type="sample") - image_processor = CLIPImageProcessor.from_pretrained(args.image_processor_path, subfolder="image_processor") - tokenizer = CLIPTokenizer.from_pretrained(args.tokenizer_path, subfolder="tokenizer") + image_processor = CLIPImageProcessor.from_pretrained( + args.pretrained_prior_model_name_or_path, subfolder="image_processor" + ) + tokenizer = CLIPTokenizer.from_pretrained(args.pretrained_prior_model_name_or_path, subfolder="tokenizer") def deepspeed_zero_init_disabled_context_manager(): """ @@ -336,13 +505,13 @@ def deepspeed_zero_init_disabled_context_manager(): weight_dtype = torch.bfloat16 with ContextManagers(deepspeed_zero_init_disabled_context_manager()): image_encoder = CLIPVisionModelWithProjection.from_pretrained( - args.pretrained_image_encoder, subfolder="image_encoder", torch_dtype=weight_dtype + args.pretrained_prior_model_name_or_path, subfolder="image_encoder", torch_dtype=weight_dtype ).eval() text_encoder = CLIPTextModelWithProjection.from_pretrained( - args.text_encoder_path, subfolder="text_encoder", torch_dtype=weight_dtype + args.pretrained_prior_model_name_or_path, subfolder="text_encoder", torch_dtype=weight_dtype ).eval() - print("args.pretrained_prior_path =", args.pretrained_prior_path) - prior = PriorTransformer.from_pretrained(args.pretrained_prior_path, subfolder="prior") + + prior = PriorTransformer.from_pretrained(args.pretrained_prior_model_name_or_path, subfolder="prior") # Freeze text_encoder and image_encoder text_encoder.requires_grad_(False) @@ -350,7 +519,7 @@ def deepspeed_zero_init_disabled_context_manager(): # Create EMA for the prior. if args.use_ema: - ema_prior = PriorTransformer.from_pretrained(args.pretrained_prior_path, subfolder="prior") + ema_prior = PriorTransformer.from_pretrained(args.pretrained_prior_model_name_or_path, subfolder="prior") ema_prior = EMAModel(ema_prior.parameters(), model_cls=PriorTransformer, model_config=ema_prior.config) ema_prior.to(accelerator.device) @@ -428,27 +597,109 @@ def load_model_hook(models, input_dir): optimizer_cls = torch.optim.AdamW optimizer = optimizer_cls( prior.parameters(), - lr=args.lr, + lr=args.learning_rate, betas=(args.adam_beta1, args.adam_beta2), - weight_decay=args.weight_decay, + weight_decay=args.adam_weight_decay, eps=args.adam_epsilon, ) - train_dataset = ImageDataset( - images_paths_csv=args.train_images_paths_csv, image_processor=image_processor, tokenizer=tokenizer - ) - train_dataloader = torch.utils.data.DataLoader( - train_dataset, batch_size=args.train_batch_size, num_workers=args.dataloader_num_workers - ) - if args.val_images_paths_csv is not None: - val_dataset = ImageDataset( - images_paths_csv=args.val_images_paths_csv, image_processor=image_processor, tokenizer=tokenizer + # Get the datasets: you can either provide your own training and evaluation files (see below) + # or specify a Dataset from the hub (the dataset will be downloaded automatically from the datasets Hub). + + # In distributed training, the load_dataset function guarantees that only one local process can concurrently + # download the dataset. + if args.dataset_name is not None: + # Downloading and loading a dataset from the hub. + dataset = load_dataset( + args.dataset_name, + args.dataset_config_name, + cache_dir=args.cache_dir, ) - torch.utils.data.DataLoader( - val_dataset, batch_size=args.val_batch_size, num_workers=args.dataloader_num_workers + else: + data_files = {} + if args.train_data_dir is not None: + data_files["train"] = os.path.join(args.train_data_dir, "**") + dataset = load_dataset( + "imagefolder", + data_files=data_files, + cache_dir=args.cache_dir, ) + # See more about loading custom images at + # https://huggingface.co/docs/datasets/v2.4.0/en/image_load#imagefolder + + # Preprocessing the datasets. + # We need to tokenize inputs and targets. + column_names = dataset["train"].column_names + + # 6. Get the column names for input/target. + dataset_columns = DATASET_NAME_MAPPING.get(args.dataset_name, None) + if args.image_column is None: + image_column = dataset_columns[0] if dataset_columns is not None else column_names[0] + else: + image_column = args.image_column + if image_column not in column_names: + raise ValueError( + f"--image_column' value '{args.image_column}' needs to be one of: {', '.join(column_names)}" + ) + if args.caption_column is None: + caption_column = dataset_columns[1] if dataset_columns is not None else column_names[1] else: - pass + caption_column = args.caption_column + if caption_column not in column_names: + raise ValueError( + f"--caption_column' value '{args.caption_column}' needs to be one of: {', '.join(column_names)}" + ) + + # Preprocessing the datasets. + # We need to tokenize input captions and transform the images. + def tokenize_captions(examples, is_train=True): + captions = [] + for caption in examples[caption_column]: + if isinstance(caption, str): + captions.append(caption) + elif isinstance(caption, (list, np.ndarray)): + # take a random caption if there are multiple + captions.append(random.choice(caption) if is_train else caption[0]) + else: + raise ValueError( + f"Caption column `{caption_column}` should contain either strings or lists of strings." + ) + inputs = tokenizer( + captions, max_length=tokenizer.model_max_length, padding="max_length", truncation=True, return_tensors="pt" + ) + text_input_ids = inputs.input_ids + text_mask = inputs.attention_mask.bool() + return text_input_ids, text_mask + + def preprocess_train(examples): + images = [image.convert("RGB") for image in examples[image_column]] + examples["clip_pixel_values"] = image_processor(images, return_tensors="pt").pixel_values + examples["text_input_ids"], examples["text_mask"] = tokenize_captions(examples) + return examples + + with accelerator.main_process_first(): + if args.max_train_samples is not None: + dataset["train"] = dataset["train"].shuffle(seed=args.seed).select(range(args.max_train_samples)) + # Set the training transforms + train_dataset = dataset["train"].with_transform(preprocess_train) + + def collate_fn(examples): + clip_pixel_values = torch.stack([example["clip_pixel_values"] for example in examples]) + clip_pixel_values = clip_pixel_values.to(memory_format=torch.contiguous_format).float() + text_input_ids = torch.stack([example["text_input_ids"] for example in examples]) + text_mask = torch.stack([example["text_mask"] for example in examples]) + return {"clip_pixel_values": clip_pixel_values, "text_input_ids": text_input_ids, "text_mask": text_mask} + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + shuffle=True, + collate_fn=collate_fn, + batch_size=args.train_batch_size, + num_workers=args.dataloader_num_workers, + ) + + # Scheduler and math around the number of training steps. overrode_max_train_steps = False num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) if args.max_train_steps is None: @@ -461,10 +712,12 @@ def load_model_hook(models, input_dir): num_warmup_steps=args.lr_warmup_steps * args.gradient_accumulation_steps, num_training_steps=args.max_train_steps * args.gradient_accumulation_steps, ) + clip_mean = prior.clip_mean clip_std = prior.clip_std prior.clip_mean = None prior.clip_std = None + prior, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( prior, optimizer, train_dataloader, lr_scheduler ) @@ -482,7 +735,9 @@ def load_model_hook(models, input_dir): # The trackers initializes automatically on the main process. if accelerator.is_main_process: tracker_config = dict(vars(args)) - accelerator.init_trackers("test", tracker_config) # args.tracker_project_name + tracker_config.pop("validation_prompts") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + # Train! total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps @@ -495,6 +750,8 @@ def load_model_hook(models, input_dir): logger.info(f" Total optimization steps = {args.max_train_steps}") global_step = 0 first_epoch = 0 + + # Potentially load in the weights and states from a previous save if args.resume_from_checkpoint: if args.resume_from_checkpoint != "latest": path = os.path.basename(args.resume_from_checkpoint) @@ -519,10 +776,13 @@ def load_model_hook(models, input_dir): first_epoch = global_step // num_update_steps_per_epoch resume_step = resume_global_step % (num_update_steps_per_epoch * args.gradient_accumulation_steps) + # Only show the progress bar once on each machine. progress_bar = tqdm(range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process) - progress_bar.set_description("training goes brrr") + progress_bar.set_description("Steps") + clip_mean = clip_mean.to(weight_dtype).to(accelerator.device) clip_std = clip_std.to(weight_dtype).to(accelerator.device) + for epoch in range(first_epoch, args.num_train_epochs): prior.train() train_loss = 0.0 @@ -535,8 +795,11 @@ def load_model_hook(models, input_dir): with accelerator.accumulate(prior): # Convert images to latent space - text_input_ids, text_mask, clip_images = batch - text_input_ids, text_mask, clip_images = text_input_ids, text_mask, clip_images.to(weight_dtype) + text_input_ids, text_mask, clip_images = ( + batch["text_input_ids"], + batch["text_mask"], + batch["clip_pixel_values"].to(weight_dtype), + ) with torch.no_grad(): text_encoder_output = text_encoder(text_input_ids) prompt_embeds = text_encoder_output.text_embeds @@ -600,12 +863,31 @@ def load_model_hook(models, input_dir): ema_prior.step(prior.parameters()) progress_bar.update(1) global_step += 1 - # print(train_loss) accelerator.log({"train_loss": train_loss}, step=global_step) train_loss = 0.0 if global_step % args.checkpointing_steps == 0: if accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") accelerator.save_state(save_path) logger.info(f"Saved state to {save_path}") @@ -615,6 +897,74 @@ def load_model_hook(models, input_dir): if global_step >= args.max_train_steps: break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_prior.store(prior.parameters()) + ema_prior.copy_to(prior.parameters()) + log_validation( + image_encoder, + image_processor, + text_encoder, + tokenizer, + prior, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original UNet parameters. + ema_prior.restore(prior.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + prior = accelerator.unwrap_model(prior) + if args.use_ema: + ema_prior.copy_to(prior.parameters()) + + pipeline = AutoPipelineForText2Image.from_pretrained( + args.pretrained_decoder_model_name_or_path, + prior_image_encoder=image_encoder, + prior_text_encoder=text_encoder, + prior_prior=prior, + revision=args.revision, + ) + pipeline.prior_pipe.save_pretrained(args.output_dir) + + # Run a final round of inference. + images = [] + if args.validation_prompts is not None: + logger.info("Running inference for collecting generated images...") + pipeline = pipeline.to(accelerator.device) + pipeline.torch_dtype = weight_dtype + pipeline.set_progress_bar_config(disable=True) + + if args.enable_xformers_memory_efficient_attention: + pipeline.enable_xformers_memory_efficient_attention() + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.autocast("cuda"): + image = pipeline(args.validation_prompts[i], num_inference_steps=20, generator=generator).images[0] + images.append(image) + + if args.push_to_hub: + save_model_card(args, repo_id, images, repo_folder=args.output_dir) + upload_folder( + repo_id=repo_id, + folder_path=args.output_dir, + commit_message="End of training", + ignore_patterns=["step_*", "epoch_*"], + ) + accelerator.end_training() From 3f8ea1c8e0d523ac2a3c56dec85f553f5c3f7630 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Wed, 6 Sep 2023 09:07:47 +0000 Subject: [PATCH 15/38] fix clip_mean --- examples/kandinsky2_2/train_prior.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/examples/kandinsky2_2/train_prior.py b/examples/kandinsky2_2/train_prior.py index b42602982e40..3e5d1b12bc50 100644 --- a/examples/kandinsky2_2/train_prior.py +++ b/examples/kandinsky2_2/train_prior.py @@ -137,7 +137,7 @@ def save_model_card( def log_validation( - vae, image_encoder, image_processor, text_encoder, tokenizer, prior, args, accelerator, weight_dtype, epoch + image_encoder, image_processor, text_encoder, tokenizer, prior, args, accelerator, weight_dtype, epoch ): logger.info("Running validation... ") @@ -713,10 +713,8 @@ def collate_fn(examples): num_training_steps=args.max_train_steps * args.gradient_accumulation_steps, ) - clip_mean = prior.clip_mean - clip_std = prior.clip_std - prior.clip_mean = None - prior.clip_std = None + clip_mean = prior.clip_mean.clone() + clip_std = prior.clip_std.clone() prior, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( prior, optimizer, train_dataloader, lr_scheduler From a7d136b1cdb4295ae5523f6c1e2027f8b5e55394 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Wed, 6 Sep 2023 09:19:50 +0000 Subject: [PATCH 16/38] fix --- examples/kandinsky2_2/train_prior.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/kandinsky2_2/train_prior.py b/examples/kandinsky2_2/train_prior.py index 3e5d1b12bc50..efd29d53bf50 100644 --- a/examples/kandinsky2_2/train_prior.py +++ b/examples/kandinsky2_2/train_prior.py @@ -929,7 +929,6 @@ def collate_fn(examples): prior_image_encoder=image_encoder, prior_text_encoder=text_encoder, prior_prior=prior, - revision=args.revision, ) pipeline.prior_pipe.save_pretrained(args.output_dir) From 429498e0388c98b4af0c4e45a591ec6244ccdfb5 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Wed, 6 Sep 2023 09:31:40 +0000 Subject: [PATCH 17/38] fix more --- examples/kandinsky2_2/train_prior.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/examples/kandinsky2_2/train_prior.py b/examples/kandinsky2_2/train_prior.py index efd29d53bf50..d451e1bfe40d 100644 --- a/examples/kandinsky2_2/train_prior.py +++ b/examples/kandinsky2_2/train_prior.py @@ -940,9 +940,6 @@ def collate_fn(examples): pipeline.torch_dtype = weight_dtype pipeline.set_progress_bar_config(disable=True) - if args.enable_xformers_memory_efficient_attention: - pipeline.enable_xformers_memory_efficient_attention() - if args.seed is None: generator = None else: From 78812ef32358b2e99e4706f32644e12f09e447d0 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Thu, 7 Sep 2023 03:47:15 +0000 Subject: [PATCH 18/38] update prior lora --- examples/kandinsky2_2/train_prior_lora.py | 594 ++++++++++++++++------ 1 file changed, 429 insertions(+), 165 deletions(-) diff --git a/examples/kandinsky2_2/train_prior_lora.py b/examples/kandinsky2_2/train_prior_lora.py index c2f5253acaa8..68545df054d5 100644 --- a/examples/kandinsky2_2/train_prior_lora.py +++ b/examples/kandinsky2_2/train_prior_lora.py @@ -1,148 +1,191 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Fine-tuning script for Stable Diffusion for text2image with support for LoRA.""" + import argparse +import logging import math import os +import random +import shutil +from pathlib import Path -import accelerate -import pandas as pd +import datasets +import numpy as np import torch import torch.nn.functional as F import torch.utils.checkpoint import transformers from accelerate import Accelerator from accelerate.logging import get_logger -from accelerate.state import AcceleratorState from accelerate.utils import ProjectConfiguration, set_seed -from PIL import Image +from datasets import load_dataset +from huggingface_hub import create_repo, upload_folder from tqdm import tqdm from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection import diffusers -from diffusers import DDPMScheduler, PriorTransformer +from diffusers import AutoPipelineForText2Image, DDPMScheduler, PriorTransformer from diffusers.loaders import AttnProcsLayers from diffusers.models.attention_processor import LoRAAttnProcessor from diffusers.optimization import get_scheduler -from diffusers.utils import is_wandb_available +from diffusers.utils import check_min_version, is_wandb_available -if is_wandb_available(): - pass +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.21.0.dev0") logger = get_logger(__name__, log_level="INFO") +def save_model_card(repo_id: str, images=None, base_model=str, dataset_name=str, repo_folder=None): + img_str = "" + for i, image in enumerate(images): + image.save(os.path.join(repo_folder, f"image_{i}.png")) + img_str += f"![img_{i}](./image_{i}.png)\n" + + yaml = f""" +--- +license: creativeml-openrail-m +base_model: {base_model} +tags: +- kandinsky +- text-to-image +- diffusers +- lora +inference: true +--- + """ + model_card = f""" +# LoRA text2image fine-tuning - {repo_id} +These are LoRA adaption weights for {base_model}. The weights were fine-tuned on the {dataset_name} dataset. You can find some example images in the following. \n +{img_str} +""" + with open(os.path.join(repo_folder, "README.md"), "w") as f: + f.write(yaml + model_card) + + def parse_args(): parser = argparse.ArgumentParser(description="Simple example of finetuning Kandinsky 2.2.") parser.add_argument( - "--pretrained_prior_path", + "--pretrained_decoder_model_name_or_path", type=str, - default="kandinsky-community/kandinsky-2-2-prior", + default="kandinsky-community/kandinsky-2-2-decoder", required=False, - help="Path to pretrained prior model or model identifier from huggingface.co/models.", + help="Path to pretrained model or model identifier from huggingface.co/models.", ) parser.add_argument( - "--pretrained_image_encoder", + "--pretrained_prior_model_name_or_path", type=str, default="kandinsky-community/kandinsky-2-2-prior", required=False, - help="Path to pretrained image encoder.", + help="Path to pretrained model or model identifier from huggingface.co/models.", ) parser.add_argument( - "--scheduler_path", + "--dataset_name", type=str, - default="kandinsky-community/kandinsky-2-2-prior", - required=False, - help="Path to scheduler.", + default=None, + help=( + "The name of the Dataset (from the HuggingFace hub) to train on (could be your own, possibly private," + " dataset). It can also be a path pointing to a local copy of a dataset in your filesystem," + " or to a folder containing files that 🤗 Datasets can understand." + ), ) parser.add_argument( - "--image_processor_path", + "--dataset_config_name", type=str, - default="kandinsky-community/kandinsky-2-2-prior", - required=False, - help="Path to image_processor.", + default=None, + help="The config of the Dataset, leave as None if there's only one config.", ) parser.add_argument( - "--text_encoder_path", + "--train_data_dir", type=str, - default="kandinsky-community/kandinsky-2-2-prior", - required=False, - help="Path to text_encoder.", + default=None, + help=( + "A folder containing the training data. Folder contents must follow the structure described in" + " https://huggingface.co/docs/datasets/image_dataset#imagefolder. In particular, a `metadata.jsonl` file" + " must exist to provide the captions for the images. Ignored if `dataset_name` is specified." + ), ) parser.add_argument( - "--tokenizer_path", - type=str, - default="kandinsky-community/kandinsky-2-2-prior", - required=False, - help="Path to tokenizer.", + "--image_column", type=str, default="image", help="The column of the dataset containing an image." ) parser.add_argument( - "--train_images_paths_csv", + "--caption_column", type=str, - default=None, - required=False, - help="Path to csv with train images paths with column paths and caption.", + default="text", + help="The column of the dataset containing a caption or a list of captions.", ) parser.add_argument( - "--val_images_paths_csv", - type=str, - default=None, - required=False, - help="Path to csv with val images paths with column paths and caption.", + "--validation_prompt", type=str, default=None, help="A prompt that is sampled during training for inference." ) parser.add_argument( - "--train_batch_size", + "--num_validation_images", type=int, - default=1, - required=False, - help="train batch size", + default=4, + help="Number of images that should be generated during validation with `validation_prompt`.", ) parser.add_argument( - "--val_batch_size", + "--validation_epochs", type=int, default=1, - required=False, - help="val batch size", - ) - parser.add_argument( - "--lr", - type=float, - default=1e-4, - required=False, - help="learning rate", - ) - parser.add_argument( - "--weight_decay", - type=float, - default=0.0, - required=False, - help="weight decay", + help=( + "Run fine-tuning validation every X epochs. The validation process consists of running the prompt" + " `args.validation_prompt` multiple times: `args.num_validation_images`." + ), ) parser.add_argument( - "--num_epochs", + "--max_train_samples", type=int, - default=0.0, - required=False, - help="num of epochs", + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), ) parser.add_argument( "--output_dir", type=str, - default="kandi_2_2-model-finetuned", + default="kandi_2_2-model-finetuned-lora", help="The output directory where the model predictions and checkpoints will be written.", ) parser.add_argument( - "--lr_scheduler", + "--cache_dir", type=str, - default="constant", + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--resolution", + type=int, + default=512, help=( - 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' - ' "constant", "constant_with_warmup"]' + "The resolution for input images, all the images in the train/validation dataset will be resized to this" + " resolution" ), ) + parser.add_argument( + "--train_batch_size", type=int, default=1, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) parser.add_argument( "--max_train_steps", type=int, default=None, - help="Total number of training steps to perform. If provided, overrides num_epochs.", + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", ) parser.add_argument( "--gradient_accumulation_steps", @@ -151,9 +194,19 @@ def parse_args(): help="Number of updates steps to accumulate before performing a backward/update pass.", ) parser.add_argument( - "--gradient_checkpointing", - action="store_true", - help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + "--learning_rate", + type=float, + default=1e-4, + help="learning rate", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), ) parser.add_argument( "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." @@ -168,16 +221,6 @@ def parse_args(): parser.add_argument( "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." ) - parser.add_argument( - "--resume_from_checkpoint", - type=str, - default=None, - help=( - "Whether training should be resumed from a previous checkpoint. Use a path saved by" - ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' - ), - ) - parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") parser.add_argument( "--allow_tf32", action="store_true", @@ -194,6 +237,25 @@ def parse_args(): "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." ), ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument( + "--adam_weight_decay", + type=float, + default=0.0, + required=False, + help="weight decay_to_use", + ) + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) parser.add_argument( "--logging_dir", type=str, @@ -237,18 +299,23 @@ def parse_args(): "--checkpoints_total_limit", type=int, default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, help=( - "Max number of checkpoints to store. Passed as `total_limit` to the `Accelerator` `ProjectConfiguration`." - " See Accelerator::save_state https://huggingface.co/docs/accelerate/package_reference/accelerator#accelerate.Accelerator.save_state" - " for more docs" + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' ), ) - parser.add_argument("--rank", type=int, default=4, help="A rank of LORA") - parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") - parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") - parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") - parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") - parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument( + "--rank", + type=int, + default=4, + help=("The dimension of the LoRA update matrices."), + ) args = parser.parse_args() env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) @@ -256,42 +323,21 @@ def parse_args(): args.local_rank = env_local_rank # Sanity checks - if args.train_images_paths_csv is None: - raise ValueError("Need either a train_image_folder or a train_images_paths_csv.") + if args.dataset_name is None and args.train_data_dir is None: + raise ValueError("Need either a dataset name or a training folder.") return args -class ImageDataset(torch.utils.data.Dataset): - def __init__(self, images_paths_csv=None, image_processor=None, tokenizer=None): - self.image_processor = image_processor - self.tokenizer = tokenizer - df = pd.read_csv(images_paths_csv) - self.paths = df["paths"].values - self.captions = df["caption"].values - - def __len__(self): - return len(self.paths) - - def __getitem__(self, i): - img = Image.open(self.paths[i]) - clip_image = self.image_processor(img, return_tensors="pt") - text_inputs = self.tokenizer( - self.captions[i], - padding="max_length", - max_length=self.tokenizer.model_max_length, - truncation=True, - return_tensors="pt", - ) - text_input_ids = text_inputs.input_ids[0] - text_mask = text_inputs.attention_mask.bool()[0] - - return text_input_ids, text_mask, clip_image.pixel_values[0] +DATASET_NAME_MAPPING = { + "lambdalabs/pokemon-blip-captions": ("image", "text"), +} def main(): args = parse_args() - logging_dir = os.path.join(args.output_dir, args.logging_dir) + logging_dir = Path(args.output_dir, args.logging_dir) + accelerator_project_config = ProjectConfiguration( total_limit=args.checkpoints_total_limit, project_dir=args.output_dir, logging_dir=logging_dir ) @@ -301,55 +347,67 @@ def main(): log_with=args.report_to, project_config=accelerator_project_config, ) + if args.report_to == "wandb": + if not is_wandb_available(): + raise ImportError("Make sure to install wandb if you want to use it for logging during training.") + import wandb + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) logger.info(accelerator.state, main_process_only=False) if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() transformers.utils.logging.set_verbosity_warning() diffusers.utils.logging.set_verbosity_info() else: + datasets.utils.logging.set_verbosity_error() transformers.utils.logging.set_verbosity_error() diffusers.utils.logging.set_verbosity_error() + # If passed along, set the training seed now. if args.seed is not None: set_seed(args.seed) + + # Handle the repository creation if accelerator.is_main_process: if args.output_dir is not None: os.makedirs(args.output_dir, exist_ok=True) + if args.push_to_hub: + repo_id = create_repo( + repo_id=args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token + ).repo_id + # Load scheduler, image_processor, tokenizer and models. noise_scheduler = DDPMScheduler(beta_schedule="squaredcos_cap_v2", prediction_type="sample") - image_processor = CLIPImageProcessor.from_pretrained(args.image_processor_path, subfolder="image_processor") - tokenizer = CLIPTokenizer.from_pretrained(args.tokenizer_path, subfolder="tokenizer") - - def deepspeed_zero_init_disabled_context_manager(): - """ - returns either a context list that includes one that will disable zero.Init or an empty context list - """ - deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None - if deepspeed_plugin is None: - return [] - - return [deepspeed_plugin.zero3_init_context_manager(enable=False)] - + image_processor = CLIPImageProcessor.from_pretrained( + args.pretrained_prior_model_name_or_path, subfolder="image_processor" + ) + tokenizer = CLIPTokenizer.from_pretrained(args.pretrained_prior_model_name_or_path, subfolder="tokenizer") + image_encoder = CLIPVisionModelWithProjection.from_pretrained( + args.pretrained_prior_model_name_or_path, subfolder="image_encoder" + ) + text_encoder = CLIPTextModelWithProjection.from_pretrained( + args.pretrained_prior_model_name_or_path, subfolder="text_encoder" + ) + prior = PriorTransformer.from_pretrained(args.pretrained_prior_model_name_or_path, subfolder="prior") + # freeze parameters of models to save more memory + image_encoder.requires_grad_(False) + prior.requires_grad_(False) + text_encoder.requires_grad_(False) weight_dtype = torch.float32 if accelerator.mixed_precision == "fp16": weight_dtype = torch.float16 elif accelerator.mixed_precision == "bf16": weight_dtype = torch.bfloat16 - image_encoder = CLIPVisionModelWithProjection.from_pretrained( - args.pretrained_image_encoder, subfolder="image_encoder", torch_dtype=weight_dtype - ).eval() - text_encoder = CLIPTextModelWithProjection.from_pretrained( - args.text_encoder_path, subfolder="text_encoder", torch_dtype=weight_dtype - ).eval() - print("args.pretrained_prior_path =", args.pretrained_prior_path) - prior = PriorTransformer.from_pretrained(args.pretrained_prior_path, subfolder="prior", torch_dtype=weight_dtype) - # Freeze text_encoder and image_encoder - text_encoder.requires_grad_(False) - image_encoder.requires_grad_(False) - prior.requires_grad_(False) + # Move image_encoder, text_encoder and prior to device and cast to weight_dtype + prior.to(accelerator.device, dtype=weight_dtype) image_encoder.to(accelerator.device, dtype=weight_dtype) text_encoder.to(accelerator.device, dtype=weight_dtype) - prior.to(accelerator.device, dtype=weight_dtype) lora_attn_procs = {} for name in prior.attn_processors.keys(): lora_attn_procs[name] = LoRAAttnProcessor(hidden_size=2048, rank=args.rank) @@ -396,29 +454,112 @@ def compute_snr(timesteps): optimizer_cls = bnb.optim.AdamW8bit else: optimizer_cls = torch.optim.AdamW + optimizer = optimizer_cls( lora_layers.parameters(), - lr=args.lr, + lr=args.learning_rate, betas=(args.adam_beta1, args.adam_beta2), - weight_decay=args.weight_decay, + weight_decay=args.adam_weight_decay, eps=args.adam_epsilon, ) - train_dataset = ImageDataset( - images_paths_csv=args.train_images_paths_csv, image_processor=image_processor, tokenizer=tokenizer - ) - train_dataloader = torch.utils.data.DataLoader( - train_dataset, batch_size=args.train_batch_size, num_workers=args.dataloader_num_workers - ) - if args.val_images_paths_csv is not None: - val_dataset = ImageDataset( - images_paths_csv=args.val_images_paths_csv, image_processor=image_processor, tokenizer=tokenizer + # Get the datasets: you can either provide your own training and evaluation files (see below) + # or specify a Dataset from the hub (the dataset will be downloaded automatically from the datasets Hub). + + # In distributed training, the load_dataset function guarantees that only one local process can concurrently + # download the dataset. + if args.dataset_name is not None: + # Downloading and loading a dataset from the hub. + dataset = load_dataset( + args.dataset_name, + args.dataset_config_name, + cache_dir=args.cache_dir, ) - torch.utils.data.DataLoader( - val_dataset, batch_size=args.val_batch_size, num_workers=args.dataloader_num_workers + else: + data_files = {} + if args.train_data_dir is not None: + data_files["train"] = os.path.join(args.train_data_dir, "**") + dataset = load_dataset( + "imagefolder", + data_files=data_files, + cache_dir=args.cache_dir, ) + # See more about loading custom images at + # https://huggingface.co/docs/datasets/v2.4.0/en/image_load#imagefolder + + # Preprocessing the datasets. + # We need to tokenize inputs and targets. + column_names = dataset["train"].column_names + + # 6. Get the column names for input/target. + dataset_columns = DATASET_NAME_MAPPING.get(args.dataset_name, None) + if args.image_column is None: + image_column = dataset_columns[0] if dataset_columns is not None else column_names[0] + else: + image_column = args.image_column + if image_column not in column_names: + raise ValueError( + f"--image_column' value '{args.image_column}' needs to be one of: {', '.join(column_names)}" + ) + if args.caption_column is None: + caption_column = dataset_columns[1] if dataset_columns is not None else column_names[1] else: - pass + caption_column = args.caption_column + if caption_column not in column_names: + raise ValueError( + f"--caption_column' value '{args.caption_column}' needs to be one of: {', '.join(column_names)}" + ) + + # Preprocessing the datasets. + # We need to tokenize input captions and transform the images. + def tokenize_captions(examples, is_train=True): + captions = [] + for caption in examples[caption_column]: + if isinstance(caption, str): + captions.append(caption) + elif isinstance(caption, (list, np.ndarray)): + # take a random caption if there are multiple + captions.append(random.choice(caption) if is_train else caption[0]) + else: + raise ValueError( + f"Caption column `{caption_column}` should contain either strings or lists of strings." + ) + inputs = tokenizer( + captions, max_length=tokenizer.model_max_length, padding="max_length", truncation=True, return_tensors="pt" + ) + text_input_ids = inputs.input_ids + text_mask = inputs.attention_mask.bool() + return text_input_ids, text_mask + + def preprocess_train(examples): + images = [image.convert("RGB") for image in examples[image_column]] + examples["clip_pixel_values"] = image_processor(images, return_tensors="pt").pixel_values + examples["text_input_ids"], examples["text_mask"] = tokenize_captions(examples) + return examples + + with accelerator.main_process_first(): + if args.max_train_samples is not None: + dataset["train"] = dataset["train"].shuffle(seed=args.seed).select(range(args.max_train_samples)) + # Set the training transforms + train_dataset = dataset["train"].with_transform(preprocess_train) + + def collate_fn(examples): + clip_pixel_values = torch.stack([example["clip_pixel_values"] for example in examples]) + clip_pixel_values = clip_pixel_values.to(memory_format=torch.contiguous_format).float() + text_input_ids = torch.stack([example["text_input_ids"] for example in examples]) + text_mask = torch.stack([example["text_mask"] for example in examples]) + return {"clip_pixel_values": clip_pixel_values, "text_input_ids": text_input_ids, "text_mask": text_mask} + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + shuffle=True, + collate_fn=collate_fn, + batch_size=args.train_batch_size, + num_workers=args.dataloader_num_workers, + ) + + # Scheduler and math around the number of training steps. overrode_max_train_steps = False num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) if args.max_train_steps is None: @@ -447,8 +588,8 @@ def compute_snr(timesteps): # We need to initialize the trackers we use, and also store our configuration. # The trackers initializes automatically on the main process. if accelerator.is_main_process: - tracker_config = dict(vars(args)) - accelerator.init_trackers("test", tracker_config) # args.tracker_project_name + accelerator.init_trackers("text2image-fine-tune", config=vars(args)) + # Train! total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps @@ -461,6 +602,8 @@ def compute_snr(timesteps): logger.info(f" Total optimization steps = {args.max_train_steps}") global_step = 0 first_epoch = 0 + + # Potentially load in the weights and states from a previous save if args.resume_from_checkpoint: if args.resume_from_checkpoint != "latest": path = os.path.basename(args.resume_from_checkpoint) @@ -485,6 +628,7 @@ def compute_snr(timesteps): first_epoch = global_step // num_update_steps_per_epoch resume_step = resume_global_step % (num_update_steps_per_epoch * args.gradient_accumulation_steps) + # Only show the progress bar once on each machine. progress_bar = tqdm(range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process) progress_bar.set_description("training goes brrr") clip_mean = clip_mean.to(weight_dtype).to(accelerator.device) @@ -501,8 +645,11 @@ def compute_snr(timesteps): with accelerator.accumulate(prior): # Convert images to latent space - text_input_ids, text_mask, clip_images = batch - text_input_ids, text_mask, clip_images = text_input_ids, text_mask, clip_images.to(weight_dtype) + text_input_ids, text_mask, clip_images = ( + batch["text_input_ids"], + batch["text_mask"], + batch["clip_pixel_values"].to(weight_dtype), + ) with torch.no_grad(): text_encoder_output = text_encoder(text_input_ids) prompt_embeds = text_encoder_output.text_embeds @@ -564,12 +711,31 @@ def compute_snr(timesteps): if accelerator.sync_gradients: progress_bar.update(1) global_step += 1 - # print(train_loss) accelerator.log({"train_loss": train_loss}, step=global_step) train_loss = 0.0 if global_step % args.checkpointing_steps == 0: if accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") accelerator.save_state(save_path) logger.info(f"Saved state to {save_path}") @@ -579,6 +745,104 @@ def compute_snr(timesteps): if global_step >= args.max_train_steps: break + + if accelerator.is_main_process: + if args.validation_prompt is not None and epoch % args.validation_epochs == 0: + logger.info( + f"Running validation... \n Generating {args.num_validation_images} images with prompt:" + f" {args.validation_prompt}." + ) + # create pipeline + pipeline = AutoPipelineForText2Image.from_pretrained( + args.pretrained_decoder_model_name_or_path, + prior_prior=accelerator.unwrap_model(prior), + torch_dtype=weight_dtype, + ) + pipeline = pipeline.to(accelerator.device) + pipeline.set_progress_bar_config(disable=True) + + # run inference + generator = torch.Generator(device=accelerator.device) + if args.seed is not None: + generator = generator.manual_seed(args.seed) + images = [] + for _ in range(args.num_validation_images): + images.append( + pipeline(args.validation_prompt, num_inference_steps=30, generator=generator).images[0] + ) + + for tracker in accelerator.trackers: + if tracker.name == "tensorboard": + np_images = np.stack([np.asarray(img) for img in images]) + tracker.writer.add_images("validation", np_images, epoch, dataformats="NHWC") + if tracker.name == "wandb": + tracker.log( + { + "validation": [ + wandb.Image(image, caption=f"{i}: {args.validation_prompt}") + for i, image in enumerate(images) + ] + } + ) + + del pipeline + torch.cuda.empty_cache() + + # Save the lora layers + accelerator.wait_for_everyone() + if accelerator.is_main_process: + prior = prior.to(torch.float32) + prior.save_attn_procs(args.output_dir) + + if args.push_to_hub: + save_model_card( + repo_id, + images=images, + base_model=args.pretrained_model_name_or_path, + dataset_name=args.dataset_name, + repo_folder=args.output_dir, + ) + upload_folder( + repo_id=repo_id, + folder_path=args.output_dir, + commit_message="End of training", + ignore_patterns=["step_*", "epoch_*"], + ) + + # Final inference + # Load previous pipeline + pipeline = AutoPipelineForText2Image.from_pretrained( + args.pretrained_decoder_model_name_or_path, torch_dtype=weight_dtype + ) + pipeline = pipeline.to(accelerator.device) + + # load attention processors + pipeline.prior_prior.load_attn_procs(args.output_dir) + + # run inference + generator = torch.Generator(device=accelerator.device) + if args.seed is not None: + generator = generator.manual_seed(args.seed) + images = [] + for _ in range(args.num_validation_images): + images.append(pipeline(args.validation_prompt, num_inference_steps=30, generator=generator).images[0]) + + if accelerator.is_main_process: + for tracker in accelerator.trackers: + if len(images) != 0: + if tracker.name == "tensorboard": + np_images = np.stack([np.asarray(img) for img in images]) + tracker.writer.add_images("test", np_images, epoch, dataformats="NHWC") + if tracker.name == "wandb": + tracker.log( + { + "test": [ + wandb.Image(image, caption=f"{i}: {args.validation_prompt}") + for i, image in enumerate(images) + ] + } + ) + accelerator.end_training() From 982b2a6f453070c7d8a5c220928462b97fad480a Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Thu, 7 Sep 2023 07:03:34 +0000 Subject: [PATCH 19/38] fix --- .../kandinsky2_2/train_prior_dreambooth.py | 676 ------------------ examples/kandinsky2_2/train_prior_lora.py | 6 +- 2 files changed, 3 insertions(+), 679 deletions(-) delete mode 100644 examples/kandinsky2_2/train_prior_dreambooth.py diff --git a/examples/kandinsky2_2/train_prior_dreambooth.py b/examples/kandinsky2_2/train_prior_dreambooth.py deleted file mode 100644 index d9b1d811a9c2..000000000000 --- a/examples/kandinsky2_2/train_prior_dreambooth.py +++ /dev/null @@ -1,676 +0,0 @@ -import argparse -import os -import numpy as np -from PIL import Image, ImageOps -from tqdm import tqdm -import pandas as pd -import math -from packaging import version - -import torch -import torch.nn.functional as F -import torch.utils.checkpoint -import accelerate -from accelerate import Accelerator -from accelerate.logging import get_logger -from accelerate.state import AcceleratorState -from accelerate.utils import ProjectConfiguration, set_seed -from torchvision import transforms -import transformers -from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection -from transformers.utils import ContextManagers - -import diffusers -from diffusers import PriorTransformer, UnCLIPScheduler, DDPMScheduler, KandinskyV22PriorPipeline -from diffusers.optimization import get_scheduler -from diffusers.training_utils import EMAModel -from diffusers.utils import check_min_version, deprecate, is_wandb_available - -if is_wandb_available(): - import wandb - -logger = get_logger(__name__, log_level="INFO") - -def parse_args(): - parser = argparse.ArgumentParser(description="Simple example of finetuning Kandinsky 2.2.") - parser.add_argument( - "--pretrained_prior_path", - type=str, - default='kandinsky-community/kandinsky-2-2-prior', - required=False, - help="Path to pretrained prior model or model identifier from huggingface.co/models.", - ) - parser.add_argument( - "--pretrained_image_encoder", - type=str, - default='kandinsky-community/kandinsky-2-2-prior', - required=False, - help="Path to pretrained image encoder.", - ) - parser.add_argument( - "--scheduler_path", - type=str, - default='kandinsky-community/kandinsky-2-2-prior', - required=False, - help="Path to scheduler.", - ) - parser.add_argument( - "--image_processor_path", - type=str, - default='kandinsky-community/kandinsky-2-2-prior', - required=False, - help="Path to image_processor.", - ) - parser.add_argument( - "--text_encoder_path", - type=str, - default='kandinsky-community/kandinsky-2-2-prior', - required=False, - help="Path to text_encoder.", - ) - parser.add_argument( - "--instance_data_dir", - type=str, - default=None, - required=True, - help="A folder containing the training data of instance images.", - ) - parser.add_argument( - "--instance_prompt", - type=str, - default=None, - required=True, - help="The prompt with identifier specifying the instance", - ) - parser.add_argument( - "--class_prompt", - type=str, - default=None, - help="The prompt to specify images in the same class as provided instance images.", - ) - parser.add_argument( - "--with_prior_preservation", - default=False, - action="store_true", - help="Flag to add prior preservation loss.", - ) - parser.add_argument("--prior_loss_weight", type=float, default=1.0, help="The weight of prior preservation loss.") - parser.add_argument( - "--num_class_images", - type=int, - default=100, - help=( - "Minimal class images for prior preservation loss. If there are not enough images already present in" - " class_data_dir, additional images will be sampled with class_prompt." - ), - ) - parser.add_argument( - "--tokenizer_path", - type=str, - default='kandinsky-community/kandinsky-2-2-prior', - required=False, - help="Path to tokenizer.", - ) - parser.add_argument( - "--train_batch_size", - type=int, - default=1, - required=False, - help="train batch size", - ) - parser.add_argument( - "--lr", - type=float, - default=1e-4, - required=False, - help="learning rate", - ) - parser.add_argument( - "--weight_decay", - type=float, - default=0.0, - required=False, - help="weight decay", - ) - parser.add_argument( - "--num_epochs", - type=int, - default=0.0, - required=False, - help="num of epochs", - ) - parser.add_argument( - "--output_dir", - type=str, - default="kandi_2_2-model-finetuned", - help="The output directory where the model predictions and checkpoints will be written.", - ) - parser.add_argument( - "--lr_scheduler", - type=str, - default="constant", - help=( - 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' - ' "constant", "constant_with_warmup"]' - ), - ) - parser.add_argument( - "--max_train_steps", - type=int, - default=None, - help="Total number of training steps to perform. If provided, overrides num_epochs.", - ) - parser.add_argument( - "--gradient_accumulation_steps", - type=int, - default=1, - help="Number of updates steps to accumulate before performing a backward/update pass.", - ) - parser.add_argument( - "--gradient_checkpointing", - action="store_true", - help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", - ) - parser.add_argument( - "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." - ) - parser.add_argument( - "--snr_gamma", - type=float, - default=None, - help="SNR weighting gamma to be used if rebalancing the loss. Recommended value is 5.0. " - "More details here: https://arxiv.org/abs/2303.09556.", - ) - parser.add_argument( - "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." - ) - parser.add_argument( - "--resume_from_checkpoint", - type=str, - default=None, - help=( - "Whether training should be resumed from a previous checkpoint. Use a path saved by" - ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' - ), - ) - parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") - parser.add_argument( - "--allow_tf32", - action="store_true", - help=( - "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" - " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" - ), - ) - parser.add_argument( - "--dataloader_num_workers", - type=int, - default=0, - help=( - "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." - ), - ) - parser.add_argument( - "--logging_dir", - type=str, - default="logs", - help=( - "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" - " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." - ), - ) - parser.add_argument( - "--mixed_precision", - type=str, - default=None, - choices=["no", "fp16", "bf16"], - help=( - "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" - " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" - " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." - ), - ) - parser.add_argument( - "--report_to", - type=str, - default="tensorboard", - help=( - 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' - ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' - ), - ) - parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") - parser.add_argument( - "--checkpointing_steps", - type=int, - default=500, - help=( - "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" - " training using `--resume_from_checkpoint`." - ), - ) - parser.add_argument( - "--checkpoints_total_limit", - type=int, - default=None, - help=( - "Max number of checkpoints to store. Passed as `total_limit` to the `Accelerator` `ProjectConfiguration`." - " See Accelerator::save_state https://huggingface.co/docs/accelerate/package_reference/accelerator#accelerate.Accelerator.save_state" - " for more docs" - ), - ) - parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") - parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") - parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") - parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") - parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") - - args = parser.parse_args() - env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) - if env_local_rank != -1 and env_local_rank != args.local_rank: - args.local_rank = env_local_rank - - # Sanity checks - - return args - - -class ImageDataset(torch.utils.data.Dataset): - def __init__(self, - instance_data_dir=None, - instance_prompt=None, - class_dataset=None, - class_prompt=None, - image_processor=None, - tokenizer=None): - self.image_processor = image_processor - self.tokenizer = tokenizer - self.instance_paths = [os.path.join(instance_data_dir, i) for i in os.listdir(instance_data_dir) if '.jpg' in i or '.png' in i or '.jpeg' in i] - self.instance_prompt = instance_prompt - self.class_dataset = class_dataset - self.class_prompt = class_prompt - def __len__(self): - return len(self.instance_paths) - - def __getitem__(self, i): - example = {} - img = Image.open(self.instance_paths[i]) - example["instance_images"] = torch.from_numpy(self.image_processor(img).pixel_values[0]) - text_inputs = self.tokenizer( - self.instance_prompt, - padding="max_length", - max_length=self.tokenizer.model_max_length, - truncation=True, - return_tensors="pt", - ) - example["instance_prompt_ids"] = text_inputs.input_ids[0] - example["instance_attention_mask"] = text_inputs.attention_mask.bool()[0] - if self.class_dataset is not None: - example["class_images"] = self.class_dataset[i % self.class_dataset.shape[0]] - text_inputs = self.tokenizer( - self.class_prompt, - padding="max_length", - max_length=self.tokenizer.model_max_length, - truncation=True, - return_tensors="pt", - ) - example["class_prompt_ids"] = text_inputs.input_ids[0] - example["class_attention_mask"] = text_inputs.attention_mask.bool()[0] - return example - -def collate_fn(examples, with_prior_preservation=False): - input_ids = [example["instance_prompt_ids"] for example in examples] - pixel_values = [example["instance_images"] for example in examples] - - attention_mask = [example["instance_attention_mask"] for example in examples] - - # Concat class and instance examples for prior preservation. - # We do this to avoid doing two forward passes. - if with_prior_preservation: - input_ids += [example["class_prompt_ids"] for example in examples] - class_pixel_values = [example["class_images"] for example in examples] - attention_mask += [example["class_attention_mask"] for example in examples] - class_pixel_values = torch.stack(class_pixel_values) - pixel_values = torch.stack(pixel_values) - pixel_values = pixel_values.to(memory_format=torch.contiguous_format).float() - - input_ids = torch.stack(input_ids) - attention_mask = torch.stack(attention_mask) - batch = { - "input_ids": input_ids, - "pixel_values": pixel_values, - "attention_mask": attention_mask - } - if with_prior_preservation: - batch["class_pixel_values"] = class_pixel_values - return batch - -def main(): - args = parse_args() - logging_dir = os.path.join(args.output_dir, args.logging_dir) - accelerator_project_config = ProjectConfiguration( - total_limit=args.checkpoints_total_limit, project_dir=args.output_dir, logging_dir=logging_dir - ) - accelerator = Accelerator( - gradient_accumulation_steps=args.gradient_accumulation_steps, - mixed_precision=args.mixed_precision, - log_with=args.report_to, - project_config=accelerator_project_config, - ) - - if args.with_prior_preservation: - pipe_prior = KandinskyV22PriorPipeline.from_pretrained(args.pretrained_prior_path, torch_dtype=torch.float16).to(accelerator.device) - class_dataset = [] - for i in tqdm(range(args.num_class_images)): - image_emb = pipe_prior(args.class_prompt).image_embeds - class_dataset.append(image_emb.to('cpu')) - class_dataset = torch.cat(class_dataset, dim=0) - del pipe_prior - if torch.cuda.is_available(): - torch.cuda.empty_cache() - logger.info(accelerator.state, main_process_only=False) - if accelerator.is_local_main_process: - transformers.utils.logging.set_verbosity_warning() - diffusers.utils.logging.set_verbosity_info() - else: - transformers.utils.logging.set_verbosity_error() - diffusers.utils.logging.set_verbosity_error() - - if args.seed is not None: - set_seed(args.seed) - if accelerator.is_main_process: - if args.output_dir is not None: - os.makedirs(args.output_dir, exist_ok=True) - - noise_scheduler = DDPMScheduler(beta_schedule='squaredcos_cap_v2', prediction_type='sample') - image_processor = CLIPImageProcessor.from_pretrained(args.image_processor_path, subfolder='image_processor') - tokenizer = CLIPTokenizer.from_pretrained(args.tokenizer_path, subfolder='tokenizer') - def deepspeed_zero_init_disabled_context_manager(): - """ - returns either a context list that includes one that will disable zero.Init or an empty context list - """ - deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None - if deepspeed_plugin is None: - return [] - - return [deepspeed_plugin.zero3_init_context_manager(enable=False)] - weight_dtype = torch.float32 - if accelerator.mixed_precision == "fp16": - weight_dtype = torch.float16 - elif accelerator.mixed_precision == "bf16": - weight_dtype = torch.bfloat16 - with ContextManagers(deepspeed_zero_init_disabled_context_manager()): - image_encoder = CLIPVisionModelWithProjection.from_pretrained(args.pretrained_image_encoder, subfolder='image_encoder', torch_dtype=weight_dtype).eval() - text_encoder = CLIPTextModelWithProjection.from_pretrained(args.text_encoder_path, subfolder='text_encoder', torch_dtype=weight_dtype).eval() - print('args.pretrained_prior_path =', args.pretrained_prior_path) - prior = PriorTransformer.from_pretrained( - args.pretrained_prior_path, subfolder="prior" - ) - - # Freeze text_encoder and image_encoder - text_encoder.requires_grad_(False) - image_encoder.requires_grad_(False) - - # Create EMA for the prior. - if args.use_ema: - ema_prior = PriorTransformer.from_pretrained( - args.pretrained_prior_path, subfolder="prior" - ) - ema_prior = EMAModel(ema_prior.parameters(), model_cls=PriorTransformer, model_config=ema_prior.config) - ema_prior.to(accelerator.device) - def compute_snr(timesteps): - """ - Computes SNR as per https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L847-L849 - """ - alphas_cumprod = noise_scheduler.alphas_cumprod - sqrt_alphas_cumprod = alphas_cumprod**0.5 - sqrt_one_minus_alphas_cumprod = (1.0 - alphas_cumprod) ** 0.5 - - # Expand the tensors. - # Adapted from https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L1026 - sqrt_alphas_cumprod = sqrt_alphas_cumprod.to(device=timesteps.device)[timesteps].float() - while len(sqrt_alphas_cumprod.shape) < len(timesteps.shape): - sqrt_alphas_cumprod = sqrt_alphas_cumprod[..., None] - alpha = sqrt_alphas_cumprod.expand(timesteps.shape) - - sqrt_one_minus_alphas_cumprod = sqrt_one_minus_alphas_cumprod.to(device=timesteps.device)[timesteps].float() - while len(sqrt_one_minus_alphas_cumprod.shape) < len(timesteps.shape): - sqrt_one_minus_alphas_cumprod = sqrt_one_minus_alphas_cumprod[..., None] - sigma = sqrt_one_minus_alphas_cumprod.expand(timesteps.shape) - - # Compute SNR. - snr = (alpha / sigma) ** 2 - return snr - - - # `accelerate` 0.16.0 will have better support for customized saving - if version.parse(accelerate.__version__) >= version.parse("0.16.0"): - # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format - def save_model_hook(models, weights, output_dir): - if args.use_ema: - ema_prior.save_pretrained(os.path.join(output_dir, "prior_ema")) - - for i, model in enumerate(models): - model.save_pretrained(os.path.join(output_dir, "prior")) - - # make sure to pop weight so that corresponding model is not saved again - weights.pop() - - def load_model_hook(models, input_dir): - if args.use_ema: - load_model = EMAModel.from_pretrained(os.path.join(input_dir, "prior_ema"), PriorTransformer) - ema_prior.load_state_dict(load_model.state_dict()) - ema_prior.to(accelerator.device) - del load_model - - for i in range(len(models)): - # pop models so that they are not loaded again - model = models.pop() - - # load diffusers style into model - load_model = PriorTransformer.from_pretrained(input_dir, subfolder="prior") - model.register_to_config(**load_model.config) - - model.load_state_dict(load_model.state_dict()) - del load_model - - accelerator.register_save_state_pre_hook(save_model_hook) - accelerator.register_load_state_pre_hook(load_model_hook) - - if args.allow_tf32: - torch.backends.cuda.matmul.allow_tf32 = True - - if args.use_8bit_adam: - try: - import bitsandbytes as bnb - except ImportError: - raise ImportError( - "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" - ) - - optimizer_cls = bnb.optim.AdamW8bit - else: - optimizer_cls = torch.optim.AdamW - optimizer = optimizer_cls( - prior.parameters(), - lr=args.lr, - betas=(args.adam_beta1, args.adam_beta2), - weight_decay=args.weight_decay, - eps=args.adam_epsilon, - ) - - train_dataset = ImageDataset(instance_data_dir=args.instance_data_dir, - instance_prompt=args.instance_prompt, - class_dataset=class_dataset, - class_prompt=args.class_prompt, - image_processor=image_processor, - tokenizer=tokenizer) - train_dataloader = torch.utils.data.DataLoader(train_dataset, - collate_fn=lambda examples: collate_fn(examples, args.with_prior_preservation), - batch_size=args.train_batch_size, - num_workers=args.dataloader_num_workers) - overrode_max_train_steps = False - num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) - if args.max_train_steps is None: - args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch - overrode_max_train_steps = True - - lr_scheduler = get_scheduler( - args.lr_scheduler, - optimizer=optimizer, - num_warmup_steps=args.lr_warmup_steps * args.gradient_accumulation_steps, - num_training_steps=args.max_train_steps * args.gradient_accumulation_steps, - ) - clip_mean = prior.clip_mean - clip_std = prior.clip_std - prior.clip_mean = None - prior.clip_std = None - prior, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( - prior, optimizer, train_dataloader, lr_scheduler - ) - - image_encoder.to(accelerator.device, dtype=weight_dtype) - text_encoder.to(accelerator.device, dtype=weight_dtype) - # We need to recalculate our total training steps as the size of the training dataloader may have changed. - num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) - if overrode_max_train_steps: - args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch - # Afterwards we recalculate our number of training epochs - args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) - - # We need to initialize the trackers we use, and also store our configuration. - # The trackers initializes automatically on the main process. - if accelerator.is_main_process: - tracker_config = dict(vars(args)) - accelerator.init_trackers('test', tracker_config)#args.tracker_project_name - # Train! - total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps - - logger.info("***** Running training *****") - logger.info(f" Num examples = {len(train_dataset)}") - logger.info(f" Num Epochs = {args.num_train_epochs}") - logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") - logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") - logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") - logger.info(f" Total optimization steps = {args.max_train_steps}") - global_step = 0 - first_epoch = 0 - if args.resume_from_checkpoint: - if args.resume_from_checkpoint != "latest": - path = os.path.basename(args.resume_from_checkpoint) - else: - # Get the most recent checkpoint - dirs = os.listdir(args.output_dir) - dirs = [d for d in dirs if d.startswith("checkpoint")] - dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) - path = dirs[-1] if len(dirs) > 0 else None - - if path is None: - accelerator.print( - f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." - ) - args.resume_from_checkpoint = None - else: - accelerator.print(f"Resuming from checkpoint {path}") - accelerator.load_state(os.path.join(args.output_dir, path)) - global_step = int(path.split("-")[1]) - - resume_global_step = global_step * args.gradient_accumulation_steps - first_epoch = global_step // num_update_steps_per_epoch - resume_step = resume_global_step % (num_update_steps_per_epoch * args.gradient_accumulation_steps) - - progress_bar = tqdm(range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process) - progress_bar.set_description("training goes brrr") - clip_mean = clip_mean.to(weight_dtype).to(accelerator.device) - clip_std = clip_std.to(weight_dtype).to(accelerator.device) - for epoch in range(first_epoch, args.num_train_epochs): - prior.train() - train_loss = 0.0 - for step, batch in enumerate(train_dataloader): - # Skip steps until we reach the resumed step - if args.resume_from_checkpoint and epoch == first_epoch and step < resume_step: - if step % args.gradient_accumulation_steps == 0: - progress_bar.update(1) - continue - - with accelerator.accumulate(prior): - # Convert images to latent space - text_input_ids, text_mask, clip_images = batch['input_ids'].to(accelerator.device), batch['attention_mask'].to(accelerator.device), batch['pixel_values'].to(accelerator.device).to(weight_dtype) - if args.with_prior_preservation: - class_pixel_values = batch['class_pixel_values'].to(accelerator.device) - with torch.no_grad(): - text_encoder_output = text_encoder(text_input_ids) - prompt_embeds = text_encoder_output.text_embeds - text_encoder_hidden_states = text_encoder_output.last_hidden_state - - - image_embeds = image_encoder(clip_images).image_embeds - if args.with_prior_preservation: - image_embeds = torch.cat((image_embeds, class_pixel_values), dim=0) - # Sample noise that we'll add to the image_embeds - noise = torch.randn_like(image_embeds) - bsz = image_embeds.shape[0] - # Sample a random timestep for each image - timesteps = torch.randint(0, noise_scheduler.config.num_train_timesteps, (bsz,), device=image_embeds.device) - timesteps = timesteps.long() - image_embeds = (image_embeds - clip_mean) / clip_std - noisy_latents = noise_scheduler.add_noise(image_embeds, noise, timesteps) - - target = image_embeds - - # Predict the noise residual and compute loss - model_pred = prior( - noisy_latents, - timestep=timesteps, - proj_embedding=prompt_embeds, - encoder_hidden_states=text_encoder_hidden_states, - attention_mask=text_mask, - ).predicted_image_embedding - - if args.with_prior_preservation: - model_pred, model_pred_prior = torch.chunk(model_pred, 2, dim=0) - target, target_prior = torch.chunk(target, 2, dim=0) - loss = F.mse_loss(model_pred.float(), target.float(), reduction="mean") - prior_loss = F.mse_loss(model_pred_prior.float(), target_prior.float(), reduction="mean") - loss = loss + args.prior_loss_weight * prior_loss - else: - loss = F.mse_loss(model_pred.float(), target.float(), reduction="mean") - - # Gather the losses across all processes for logging (if we use distributed training). - avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() - train_loss += avg_loss.item() / args.gradient_accumulation_steps - - # Backpropagate - accelerator.backward(loss) - if accelerator.sync_gradients: - accelerator.clip_grad_norm_(prior.parameters(), args.max_grad_norm) - optimizer.step() - lr_scheduler.step() - optimizer.zero_grad() - - # Checks if the accelerator has performed an optimization step behind the scenes - if accelerator.sync_gradients: - if args.use_ema: - ema_prior.step(prior.parameters()) - progress_bar.update(1) - global_step += 1 - #print(train_loss) - accelerator.log({"train_loss": train_loss}, step=global_step) - train_loss = 0.0 - - if global_step % args.checkpointing_steps == 0: - if accelerator.is_main_process: - save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") - accelerator.save_state(save_path) - logger.info(f"Saved state to {save_path}") - - logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} - progress_bar.set_postfix(**logs) - - if global_step >= args.max_train_steps: - break - accelerator.end_training() - - -if __name__ == "__main__": - main() diff --git a/examples/kandinsky2_2/train_prior_lora.py b/examples/kandinsky2_2/train_prior_lora.py index 68545df054d5..449e834354c8 100644 --- a/examples/kandinsky2_2/train_prior_lora.py +++ b/examples/kandinsky2_2/train_prior_lora.py @@ -572,8 +572,8 @@ def collate_fn(examples): num_warmup_steps=args.lr_warmup_steps * args.gradient_accumulation_steps, num_training_steps=args.max_train_steps * args.gradient_accumulation_steps, ) - clip_mean = prior.clip_mean - clip_std = prior.clip_std + clip_mean = prior.clip_mean.clone() + clip_std = prior.clip_std.clone() lora_layers, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( lora_layers, optimizer, train_dataloader, lr_scheduler ) @@ -630,7 +630,7 @@ def collate_fn(examples): # Only show the progress bar once on each machine. progress_bar = tqdm(range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process) - progress_bar.set_description("training goes brrr") + progress_bar.set_description("Steps") clip_mean = clip_mean.to(weight_dtype).to(accelerator.device) clip_std = clip_std.to(weight_dtype).to(accelerator.device) for epoch in range(first_epoch, args.num_train_epochs): From 5ce839811c41d44284920a18b3341719eb126e9c Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Thu, 7 Sep 2023 08:48:25 +0000 Subject: [PATCH 20/38] test lora loader --- src/diffusers/models/prior_transformer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/diffusers/models/prior_transformer.py b/src/diffusers/models/prior_transformer.py index fd744e9d195e..763d6bbce2cd 100644 --- a/src/diffusers/models/prior_transformer.py +++ b/src/diffusers/models/prior_transformer.py @@ -5,6 +5,7 @@ import torch.nn.functional as F from torch import nn +from ..loaders import UNet2DConditionLoadersMixin from ..configuration_utils import ConfigMixin, register_to_config from ..utils import BaseOutput from .attention import BasicTransformerBlock @@ -32,7 +33,7 @@ class PriorTransformerOutput(BaseOutput): predicted_image_embedding: torch.FloatTensor -class PriorTransformer(ModelMixin, ConfigMixin): +class PriorTransformer(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin): """ A Prior Transformer model. From 9be844041a5a4fc86fcfd6de287a60c1d4e15ff3 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Thu, 7 Sep 2023 16:32:00 +0000 Subject: [PATCH 21/38] style + fix --- examples/kandinsky2_2/train_prior_lora.py | 2 +- src/diffusers/models/prior_transformer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/kandinsky2_2/train_prior_lora.py b/examples/kandinsky2_2/train_prior_lora.py index 449e834354c8..e4aec111b8f7 100644 --- a/examples/kandinsky2_2/train_prior_lora.py +++ b/examples/kandinsky2_2/train_prior_lora.py @@ -798,7 +798,7 @@ def collate_fn(examples): save_model_card( repo_id, images=images, - base_model=args.pretrained_model_name_or_path, + base_model=args.pretrained_prior_model_name_or_path, dataset_name=args.dataset_name, repo_folder=args.output_dir, ) diff --git a/src/diffusers/models/prior_transformer.py b/src/diffusers/models/prior_transformer.py index 763d6bbce2cd..8ada0a7c08a5 100644 --- a/src/diffusers/models/prior_transformer.py +++ b/src/diffusers/models/prior_transformer.py @@ -5,8 +5,8 @@ import torch.nn.functional as F from torch import nn -from ..loaders import UNet2DConditionLoadersMixin from ..configuration_utils import ConfigMixin, register_to_config +from ..loaders import UNet2DConditionLoadersMixin from ..utils import BaseOutput from .attention import BasicTransformerBlock from .attention_processor import ( From dd42e9de5f79c81c33ecaf7eac6d3ce22c9ffe10 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Thu, 7 Sep 2023 22:05:34 +0000 Subject: [PATCH 22/38] rename files and add readme --- examples/kandinsky2_2/text_to_image/README.md | 317 ++++++++++++++++++ .../{ => text_to_image}/requirements.txt | 0 .../train_text_to_image_decoder.py} | 0 .../train_text_to_image_lora_decoder.py} | 0 .../train_text_to_image_lora_prior.py} | 0 .../train_text_to_image_prior.py} | 0 6 files changed, 317 insertions(+) create mode 100644 examples/kandinsky2_2/text_to_image/README.md rename examples/kandinsky2_2/{ => text_to_image}/requirements.txt (100%) rename examples/kandinsky2_2/{train_decoder.py => text_to_image/train_text_to_image_decoder.py} (100%) rename examples/kandinsky2_2/{train_decoder_lora.py => text_to_image/train_text_to_image_lora_decoder.py} (100%) rename examples/kandinsky2_2/{train_prior_lora.py => text_to_image/train_text_to_image_lora_prior.py} (100%) rename examples/kandinsky2_2/{train_prior.py => text_to_image/train_text_to_image_prior.py} (100%) diff --git a/examples/kandinsky2_2/text_to_image/README.md b/examples/kandinsky2_2/text_to_image/README.md new file mode 100644 index 000000000000..6e7dacc7722d --- /dev/null +++ b/examples/kandinsky2_2/text_to_image/README.md @@ -0,0 +1,317 @@ +# Kandinsky2.2 text-to-image fine-tuning + +Kandinsky 2.2 includes a prior pipeline that generates image embeddings from text prompts, and a decoder pipeline to that generate the output image based on the image embeddings. We provide `train_text_to_image_prior.py` and `train_text_to_image_decoder.py` scripts to show you how to fine-tune the kandinsky prior and decoder models seperately based on your own dataset. To achieve the best resutls, you should fine-tune both your prior and decoder models. + +___Note___: + +___This script is experimental. The script fine-tunes the whole model and often times the model overfits and runs into issues like catastrophic forgetting. It's recommended to try different hyperparamters to get the best result on your dataset.___ + + +## Running locally with PyTorch +### Installing the dependencies + +Before running the scripts, make sure to install the library's training dependencies: + +**Important** + +To make sure you can successfully run the latest versions of the example scripts, we highly recommend **installing from source** and keeping the install up to date as we update the example scripts frequently and install some example-specific requirements. To do this, execute the following steps in a new virtual environment: +```bash +git clone https://github.com/huggingface/diffusers +cd diffusers +pip install . +``` + +Then cd in the example folder and run +```bash +pip install -r requirements.txt +``` + +And initialize an [🤗Accelerate](https://github.com/huggingface/accelerate/) environment with: + +```bash +accelerate config +``` +For this example we want to directly store the trained LoRA embeddings on the Hub, so we need to be logged in and add the --push_to_hub flag. + +___ + +### Pokemon example + +For all our examples, we will directly store the trained weights on the Hub, so we need to be logged in and add the `--push_to_hub` flag. In order to do that, You have to be a registered user in 🤗 Hugging Face Hub, and you'll also need to use an access token for the code to work. For more information on access tokens, please refer to [this section of the documentation](https://huggingface.co/docs/hub/security-tokens). + +Run the following command to authenticate your token + +```bash +huggingface-cli login +``` + +We also use [Weights and Biases](https://docs.wandb.ai/quickstart) logging by default, because it is really useful to monitor the training progress by regularly generating sample images during training. To install wandb, run + +```bash +pip install wandb +``` + +To diable wandb logging, remove the `--report_to=="wandb"` and `--validation_prompts="A robot pokemon, 4k photo"` flags from below examples + +#### Fine-tune decoder +
+ + +```bash +export DATASET_NAME="lambdalabs/pokemon-blip-captions" + +accelerate launch --mixed_precision="fp16" train_text_to_image_decoder.py \ + --dataset_name=$DATASET_NAME \ + --resolution=768 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=4 \ + --gradient_checkpointing \ + --max_train_steps=15000 \ + --learning_rate=1e-05 \ + --max_grad_norm=1 \ + --checkpoints_total_limit=3 \ + --lr_scheduler="constant" --lr_warmup_steps=0 \ + --validation_prompts="A robot pokemon, 4k photo" \ + --report_to="wandb" \ + --push_to_hub \ + --output_dir="kandi2-decoder-pokemon-model" +``` + + + +To run on your own training files prepare the dataset according to the format required by `datasets`, you can find the instructions for how to do that in this [document](https://huggingface.co/docs/datasets/v2.4.0/en/image_load#imagefolder-with-metadata). +If you wish to use custom loading logic, you should modify the script, we have left pointers for that in the training script. + +```bash +export TRAIN_DIR="path_to_your_dataset" + +accelerate launch --mixed_precision="fp16" train_text_to_image_decoder.py \ + --train_data_dir=$TRAIN_DIR \ + --resolution=768 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=4 \ + --gradient_checkpointing \ + --max_train_steps=15000 \ + --learning_rate=1e-05 \ + --max_grad_norm=1 \ + --checkpoints_total_limit=3 \ + --lr_scheduler="constant" --lr_warmup_steps=0 \ + --validation_prompts="A robot pokemon, 4k photo" \ + --report_to="wandb" \ + --push_to_hub \ + --output_dir="kandi22-decoder-pokemon-model" +``` + + +Once the training is finished the model will be saved in the `output_dir` specified in the command. In this example it's `kandi22-decoder-pokemon-model`. To load the fine-tuned model for inference just pass that path to `AutoPipelineForText2Image` + +```python +from diffusers import AutoPipelineForText2Image +import torch + +pipe = AutoPipelineForText2Image.from_pretrained(output_dir, torch_dtype=torch.float16) +pipe.enable_model_cpu_offload() + +prompt='A robot pokemon, 4k photo' +images = pipe(prompt=prompt).images +images[0].save("robot-pokemon.png") +``` + +Checkpoints only save the unet, so to run inference from a checkpoint, just load the unet +```python +from diffusers import AutoPipelineForText2Image, UNet2DConditionModel + +model_path = "path_to_saved_model" + +unet = UNet2DConditionModel.from_pretrained(model_path + "/checkpoint-/unet") + +pipe = AutoPipelineForText2Image.from_pretrained("kandinsky-community/kandinsky-2-2-decoder", unet=unet, torch_dtype=torch.float16) +pipe.enable_model_cpu_offload() + +image = pipe(prompt="A robot pokemon, 4k photo").images[0] +image.save("robot-pokemon.png") +``` + +#### Fine-tune prior + +You can fine-tune kandinsky prior model with `train_text_to_image_prior.py` script. Note that we currently do not support `--gradient_checkpointing` for prior model fine-tune. + +
+ + +```bash +export DATASET_NAME="lambdalabs/pokemon-blip-captions" + +accelerate launch --mixed_precision="fp16" train_text_to_image_prior.py \ + --dataset_name=$DATASET_NAME \ + --resolution=768 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=4 \ + --max_train_steps=15000 \ + --learning_rate=1e-05 \ + --max_grad_norm=1 \ + --checkpoints_total_limit=3 \ + --lr_scheduler="constant" --lr_warmup_steps=0 \ + --validation_prompts="A robot pokemon, 4k photo" \ + --report_to="wandb" \ + --push_to_hub \ + --output_dir="kandi2-prior-pokemon-model" +``` + + + +To perform inference with the fune-tuned prior model, you will need to first create a prior pipeline by passing the `output_dir` to `DiffusionPpeline`, and then you can create a `KandinskyV22CombinedPipeline` with a pretrained or fine-tuned decoder checkpoint, along with all the modules of the prior pipeline you just created. + +```python +from diffusers import AutoPipelineForText2Image, DiffusionPipeline +import torch + +pipe_prior = DiffusionPipeline.from_pretrained(output_dir, torch_dtype=torch.float16) +prior_components = {"prior_" + k: v for k,v in pipe_prior.components.items()} +pipe = AutoPipelineForText2Image.from_pretrained("kandinsky-community/kandinsky-2-2-decoder", **prior_components, torch_dtype=torch.float16) + +pipe.enable_model_cpu_offload() +prompt='A robot pokemon, 4k photo' +images = pipe(prompt=prompt, negative_prompt=negative_prompt).images +images[0] +``` + + +#### Training with multiple GPUs + +`accelerate` allows for seamless multi-GPU training. Follow the instructions [here](https://huggingface.co/docs/accelerate/basic_tutorials/launch) +for running distributed training with `accelerate`. Here is an example command: + +```bash +export DATASET_NAME="lambdalabs/pokemon-blip-captions" + +accelerate launch --mixed_precision="fp16" --multi_gpu train_text_to_image_decoder.py \ + --dataset_name=$DATASET_NAME \ + --resolution=768 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=4 \ + --gradient_checkpointing \ + --max_train_steps=15000 \ + --learning_rate=1e-05 \ + --max_grad_norm=1 \ + --checkpoints_total_limit=3 \ + --lr_scheduler="constant" --lr_warmup_steps=0 \ + --validation_prompts="A robot pokemon, 4k photo" \ + --report_to="wandb" \ + --push_to_hub \ + --output_dir="kandi2-decoder-pokemon-model" +``` + + +#### Training with Min-SNR weighting + +We support training with the Min-SNR weighting strategy proposed in [Efficient Diffusion Training via Min-SNR Weighting Strategy](https://arxiv.org/abs/2303.09556) which helps to achieve faster convergence +by rebalancing the loss. In order to use it, one needs to set the `--snr_gamma` argument. The recommended +value when using it is 5.0. + + +## Training with LoRA + +Low-Rank Adaption of Large Language Models was first introduced by Microsoft in [LoRA: Low-Rank Adaptation of Large Language Models](https://arxiv.org/abs/2106.09685) by *Edward J. Hu, Yelong Shen, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, Weizhu Chen*. + +In a nutshell, LoRA allows adapting pretrained models by adding pairs of rank-decomposition matrices to existing weights and **only** training those newly added weights. This has a couple of advantages: + +- Previous pretrained weights are kept frozen so that model is not prone to [catastrophic forgetting](https://www.pnas.org/doi/10.1073/pnas.1611835114). +- Rank-decomposition matrices have significantly fewer parameters than original model, which means that trained LoRA weights are easily portable. +- LoRA attention layers allow to control to which extent the model is adapted toward new training images via a `scale` parameter. + +[cloneofsimo](https://github.com/cloneofsimo) was the first to try out LoRA training for Stable Diffusion in the popular [lora](https://github.com/cloneofsimo/lora) GitHub repository. + +With LoRA, it's possible to fine-tune Kandinsky2.2 on a custom image-caption pair dataset +on consumer GPUs like Tesla T4, Tesla V100. + +### Training + +First, you need to set up your development environment as is explained in the [installation section](#installing-the-dependencies). Make sure to set the `MODEL_NAME` and `DATASET_NAME` environment variables. Here, we will use [Kandinsky2.2](https://huggingface.co/kandinsky-community/kandinsky-2-2-decoder) and the [Pokemons dataset](https://huggingface.co/datasets/lambdalabs/pokemon-blip-captions). + + +#### Train decoder + +```bash +export DATASET_NAME="lambdalabs/pokemon-blip-captions" + +accelerate launch --mixed_precision="fp16" train_text_to_image_decoder_lora.py \ + --dataset_name=$DATASET_NAME --caption_column="text" \ + --resolution=768 \ + --train_batch_size=1 \ + --num_train_epochs=100 --checkpointing_steps=5000 \ + --learning_rate=1e-04 --lr_scheduler="constant" --lr_warmup_steps=0 \ + --seed=42 \ + --rank=4 \ + --gradient_checkpointing \ + --output_dir="kandi22-decoder-pokemon-lora" \ + --validation_prompt="cute dragon creature" --report_to="wandb" \ + --push_to_hub \ +``` + +#### Train prior + +```bash +export DATASET_NAME="lambdalabs/pokemon-blip-captions" + +accelerate launch --mixed_precision="fp16" train_text_to_image_prior_lora.py \ + --dataset_name=$DATASET_NAME --caption_column="text" \ + --resolution=768 \ + --train_batch_size=1 \ + --num_train_epochs=100 --checkpointing_steps=5000 \ + --learning_rate=1e-04 --lr_scheduler="constant" --lr_warmup_steps=0 \ + --seed=42 \ + --rank=4 \ + --output_dir="kandi22-prior-pokemon-lora" \ + --validation_prompt="cute dragon creature" --report_to="wandb" \ + --push_to_hub \ +``` + +**___Note: When using LoRA we can use a much higher learning rate compared to non-LoRA fine-tuning. Here we use *1e-4* instead of the usual *1e-5*. Also, by using LoRA, it's possible to run above scripts in consumer GPUs like T4 or V100.___** + + +### Inference + +#### Inference using fine-tuned LoRA checkpoint for decoder + +Once you have trained a kandinsky decoder model using above command, the inference can be done simply using the `AutoPipelineForText2Image` after loading the trained LoRA weights. You need to pass the `output_dir` for loading the LoRA weights which, in this case, is `kandi22-decoder-pokemon-lora`. + + +```python +from diffusers import AutoPipelineForText2Image +import torch + +pipe = AutoPipelineForText2Image.from_pretrained("kandinsky-community/kandinsky-2-2-decoder", torch_dtype=torch.float16) +pipe.unet.load_attn_procs(output_dir) +pipe.enable_model_cpu_offload() + +prompt='A robot pokemon, 4k photo' +image = pipe(prompt=prompt).images[0] +image.save("robot_pokemon.png") +``` + +#### Inference using fine-tuned LoRA checkpoint for Prior + +```python +from diffusers import AutoPipelineForText2Image +import torch + +pipe = AutoPipelineForText2Image.from_pretrained("kandinsky-community/kandinsky-2-2-decoder", torch_dtype=torch.float16) +pipe.prior_prior.load_attn_procs(output_dir) +pipe.enable_model_cpu_offload() + +prompt='A robot pokemon, 4k photo' +image = pipe(prompt=prompt).images[0] +image.save("robot_pokemon.png") +image +``` + +### Training with xFormers: + +You can enable memory efficient attention by [installing xFormers](https://huggingface.co/docs/diffusers/main/en/optimization/xformers) and passing the `--enable_xformers_memory_efficient_attention` argument to the script. + +xFormers training is not available for Prior model fine-tune. + +**Note**: + +According to [this issue](https://github.com/huggingface/diffusers/issues/2234#issuecomment-1416931212), xFormers `v0.0.16` cannot be used for training in some GPUs. If you observe that problem, please install a development version as indicated in that comment. \ No newline at end of file diff --git a/examples/kandinsky2_2/requirements.txt b/examples/kandinsky2_2/text_to_image/requirements.txt similarity index 100% rename from examples/kandinsky2_2/requirements.txt rename to examples/kandinsky2_2/text_to_image/requirements.txt diff --git a/examples/kandinsky2_2/train_decoder.py b/examples/kandinsky2_2/text_to_image/train_text_to_image_decoder.py similarity index 100% rename from examples/kandinsky2_2/train_decoder.py rename to examples/kandinsky2_2/text_to_image/train_text_to_image_decoder.py diff --git a/examples/kandinsky2_2/train_decoder_lora.py b/examples/kandinsky2_2/text_to_image/train_text_to_image_lora_decoder.py similarity index 100% rename from examples/kandinsky2_2/train_decoder_lora.py rename to examples/kandinsky2_2/text_to_image/train_text_to_image_lora_decoder.py diff --git a/examples/kandinsky2_2/train_prior_lora.py b/examples/kandinsky2_2/text_to_image/train_text_to_image_lora_prior.py similarity index 100% rename from examples/kandinsky2_2/train_prior_lora.py rename to examples/kandinsky2_2/text_to_image/train_text_to_image_lora_prior.py diff --git a/examples/kandinsky2_2/train_prior.py b/examples/kandinsky2_2/text_to_image/train_text_to_image_prior.py similarity index 100% rename from examples/kandinsky2_2/train_prior.py rename to examples/kandinsky2_2/text_to_image/train_text_to_image_prior.py From 3545212571487d10729e52f50ddb62a68876dc76 Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Sun, 10 Sep 2023 18:48:09 -1000 Subject: [PATCH 23/38] Update examples/kandinsky2_2/text_to_image/README.md Co-authored-by: Sayak Paul --- examples/kandinsky2_2/text_to_image/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kandinsky2_2/text_to_image/README.md b/examples/kandinsky2_2/text_to_image/README.md index 6e7dacc7722d..aceea8e1be4d 100644 --- a/examples/kandinsky2_2/text_to_image/README.md +++ b/examples/kandinsky2_2/text_to_image/README.md @@ -1,6 +1,6 @@ # Kandinsky2.2 text-to-image fine-tuning -Kandinsky 2.2 includes a prior pipeline that generates image embeddings from text prompts, and a decoder pipeline to that generate the output image based on the image embeddings. We provide `train_text_to_image_prior.py` and `train_text_to_image_decoder.py` scripts to show you how to fine-tune the kandinsky prior and decoder models seperately based on your own dataset. To achieve the best resutls, you should fine-tune both your prior and decoder models. +Kandinsky 2.2 includes a prior pipeline that generates image embeddings from text prompts, and a decoder pipeline that generates the output image based on the image embeddings. We provide `train_text_to_image_prior.py` and `train_text_to_image_decoder.py` scripts to show you how to fine-tune the Kandinsky prior and decoder models separately based on your own dataset. To achieve the best results, you should fine-tune **_both_** your prior and decoder models. ___Note___: From a3581d5884c4e33ba2c3ac757667e30a6af4950f Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Sun, 10 Sep 2023 18:51:57 -1000 Subject: [PATCH 24/38] Update examples/kandinsky2_2/text_to_image/README.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- examples/kandinsky2_2/text_to_image/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kandinsky2_2/text_to_image/README.md b/examples/kandinsky2_2/text_to_image/README.md index aceea8e1be4d..a8186f07c903 100644 --- a/examples/kandinsky2_2/text_to_image/README.md +++ b/examples/kandinsky2_2/text_to_image/README.md @@ -4,7 +4,7 @@ Kandinsky 2.2 includes a prior pipeline that generates image embeddings from tex ___Note___: -___This script is experimental. The script fine-tunes the whole model and often times the model overfits and runs into issues like catastrophic forgetting. It's recommended to try different hyperparamters to get the best result on your dataset.___ +___This script is experimental. The script fine-tunes the whole model and often times the model overfits and runs into issues like catastrophic forgetting. It's recommended to try different hyperparameters to get the best result on your dataset.___ ## Running locally with PyTorch From 121f3cbd4e4bad715eb3826b4904356fcf7b8138 Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Sun, 10 Sep 2023 18:52:15 -1000 Subject: [PATCH 25/38] Update examples/kandinsky2_2/text_to_image/README.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- examples/kandinsky2_2/text_to_image/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/kandinsky2_2/text_to_image/README.md b/examples/kandinsky2_2/text_to_image/README.md index a8186f07c903..51c129108d36 100644 --- a/examples/kandinsky2_2/text_to_image/README.md +++ b/examples/kandinsky2_2/text_to_image/README.md @@ -8,7 +8,6 @@ ___This script is experimental. The script fine-tunes the whole model and often ## Running locally with PyTorch -### Installing the dependencies Before running the scripts, make sure to install the library's training dependencies: From 8ee5597fb4b97b062708776e527448a1e1ece1d2 Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Sun, 10 Sep 2023 18:52:26 -1000 Subject: [PATCH 26/38] Update examples/kandinsky2_2/text_to_image/README.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- examples/kandinsky2_2/text_to_image/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kandinsky2_2/text_to_image/README.md b/examples/kandinsky2_2/text_to_image/README.md index 51c129108d36..1715b319b299 100644 --- a/examples/kandinsky2_2/text_to_image/README.md +++ b/examples/kandinsky2_2/text_to_image/README.md @@ -36,7 +36,7 @@ ___ ### Pokemon example -For all our examples, we will directly store the trained weights on the Hub, so we need to be logged in and add the `--push_to_hub` flag. In order to do that, You have to be a registered user in 🤗 Hugging Face Hub, and you'll also need to use an access token for the code to work. For more information on access tokens, please refer to [this section of the documentation](https://huggingface.co/docs/hub/security-tokens). +For all our examples, we will directly store the trained weights on the Hub, so we need to be logged in and add the `--push_to_hub` flag. In order to do that, you have to be a registered user on the 🤗 Hugging Face Hub, and you'll also need to use an access token for the code to work. For more information on access tokens, please refer to the [User Access Tokens](https://huggingface.co/docs/hub/security-tokens) guide. Run the following command to authenticate your token From c3707a7c55366372b7a8409b06ff634384bfd2c0 Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Sun, 10 Sep 2023 18:52:34 -1000 Subject: [PATCH 27/38] Update examples/kandinsky2_2/text_to_image/README.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- examples/kandinsky2_2/text_to_image/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kandinsky2_2/text_to_image/README.md b/examples/kandinsky2_2/text_to_image/README.md index 1715b319b299..15d602914595 100644 --- a/examples/kandinsky2_2/text_to_image/README.md +++ b/examples/kandinsky2_2/text_to_image/README.md @@ -50,7 +50,7 @@ We also use [Weights and Biases](https://docs.wandb.ai/quickstart) logging by de pip install wandb ``` -To diable wandb logging, remove the `--report_to=="wandb"` and `--validation_prompts="A robot pokemon, 4k photo"` flags from below examples +To disable wandb logging, remove the `--report_to=="wandb"` and `--validation_prompts="A robot pokemon, 4k photo"` flags from below examples #### Fine-tune decoder
From 3676185bc0accd5b44d1a0cef2e9073b16044dd3 Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Sun, 10 Sep 2023 18:52:43 -1000 Subject: [PATCH 28/38] Update examples/kandinsky2_2/text_to_image/README.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- examples/kandinsky2_2/text_to_image/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/kandinsky2_2/text_to_image/README.md b/examples/kandinsky2_2/text_to_image/README.md index 15d602914595..6563cdaac561 100644 --- a/examples/kandinsky2_2/text_to_image/README.md +++ b/examples/kandinsky2_2/text_to_image/README.md @@ -78,8 +78,8 @@ accelerate launch --mixed_precision="fp16" train_text_to_image_decoder.py \ -To run on your own training files prepare the dataset according to the format required by `datasets`, you can find the instructions for how to do that in this [document](https://huggingface.co/docs/datasets/v2.4.0/en/image_load#imagefolder-with-metadata). -If you wish to use custom loading logic, you should modify the script, we have left pointers for that in the training script. +To train on your own training files, prepare the dataset according to the format required by `datasets`. You can find the instructions for how to do that in the [ImageFolder with metadata](https://huggingface.co/docs/datasets/en/image_load#imagefolder-with-metadata) guide. +If you wish to use custom loading logic, you should modify the script and we have left pointers for that in the training script. ```bash export TRAIN_DIR="path_to_your_dataset" From 814600363f0e90a542a8dce9dc5657dd2cc1a246 Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Sun, 10 Sep 2023 18:52:52 -1000 Subject: [PATCH 29/38] Update examples/kandinsky2_2/text_to_image/README.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- examples/kandinsky2_2/text_to_image/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kandinsky2_2/text_to_image/README.md b/examples/kandinsky2_2/text_to_image/README.md index 6563cdaac561..e2616d8515fd 100644 --- a/examples/kandinsky2_2/text_to_image/README.md +++ b/examples/kandinsky2_2/text_to_image/README.md @@ -226,7 +226,7 @@ on consumer GPUs like Tesla T4, Tesla V100. ### Training -First, you need to set up your development environment as is explained in the [installation section](#installing-the-dependencies). Make sure to set the `MODEL_NAME` and `DATASET_NAME` environment variables. Here, we will use [Kandinsky2.2](https://huggingface.co/kandinsky-community/kandinsky-2-2-decoder) and the [Pokemons dataset](https://huggingface.co/datasets/lambdalabs/pokemon-blip-captions). +First, you need to set up your development environment as explained in the [installation](#installing-the-dependencies). Make sure to set the `MODEL_NAME` and `DATASET_NAME` environment variables. Here, we will use [Kandinsky 2.2](https://huggingface.co/kandinsky-community/kandinsky-2-2-decoder) and the [Pokemons dataset](https://huggingface.co/datasets/lambdalabs/pokemon-blip-captions). #### Train decoder From af96d292ae051841e8e2334170ca6acb4e815ac4 Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Sun, 10 Sep 2023 18:52:59 -1000 Subject: [PATCH 30/38] Update examples/kandinsky2_2/text_to_image/README.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- examples/kandinsky2_2/text_to_image/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kandinsky2_2/text_to_image/README.md b/examples/kandinsky2_2/text_to_image/README.md index e2616d8515fd..4a5e37ff4922 100644 --- a/examples/kandinsky2_2/text_to_image/README.md +++ b/examples/kandinsky2_2/text_to_image/README.md @@ -273,7 +273,7 @@ accelerate launch --mixed_precision="fp16" train_text_to_image_prior_lora.py \ #### Inference using fine-tuned LoRA checkpoint for decoder -Once you have trained a kandinsky decoder model using above command, the inference can be done simply using the `AutoPipelineForText2Image` after loading the trained LoRA weights. You need to pass the `output_dir` for loading the LoRA weights which, in this case, is `kandi22-decoder-pokemon-lora`. +Once you have trained a Kandinsky decoder model using the above command, inference can be done with the `AutoPipelineForText2Image` after loading the trained LoRA weights. You need to pass the `output_dir` for loading the LoRA weights, which in this case is `kandi22-decoder-pokemon-lora`. ```python From cdc83a4977dc04b2a517e6045fa06ccab4e51395 Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Sun, 10 Sep 2023 18:53:06 -1000 Subject: [PATCH 31/38] Update examples/kandinsky2_2/text_to_image/README.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- examples/kandinsky2_2/text_to_image/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kandinsky2_2/text_to_image/README.md b/examples/kandinsky2_2/text_to_image/README.md index 4a5e37ff4922..e07f1f85f5cf 100644 --- a/examples/kandinsky2_2/text_to_image/README.md +++ b/examples/kandinsky2_2/text_to_image/README.md @@ -289,7 +289,7 @@ image = pipe(prompt=prompt).images[0] image.save("robot_pokemon.png") ``` -#### Inference using fine-tuned LoRA checkpoint for Prior +#### Inference using fine-tuned LoRA checkpoint for prior ```python from diffusers import AutoPipelineForText2Image From d5b06b95b490ef52ea68fb707c217e7d7fed7864 Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Sun, 10 Sep 2023 18:53:44 -1000 Subject: [PATCH 32/38] Update examples/kandinsky2_2/text_to_image/README.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- examples/kandinsky2_2/text_to_image/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kandinsky2_2/text_to_image/README.md b/examples/kandinsky2_2/text_to_image/README.md index e07f1f85f5cf..70482b1572ec 100644 --- a/examples/kandinsky2_2/text_to_image/README.md +++ b/examples/kandinsky2_2/text_to_image/README.md @@ -309,7 +309,7 @@ image You can enable memory efficient attention by [installing xFormers](https://huggingface.co/docs/diffusers/main/en/optimization/xformers) and passing the `--enable_xformers_memory_efficient_attention` argument to the script. -xFormers training is not available for Prior model fine-tune. +xFormers training is not available for fine-tuning the prior model. **Note**: From eaf67973ff90a5c2ca930827feb0a51d3c687416 Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Sun, 10 Sep 2023 18:55:26 -1000 Subject: [PATCH 33/38] Update examples/kandinsky2_2/text_to_image/README.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- examples/kandinsky2_2/text_to_image/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kandinsky2_2/text_to_image/README.md b/examples/kandinsky2_2/text_to_image/README.md index 70482b1572ec..8e6421498aa8 100644 --- a/examples/kandinsky2_2/text_to_image/README.md +++ b/examples/kandinsky2_2/text_to_image/README.md @@ -133,7 +133,7 @@ image.save("robot-pokemon.png") #### Fine-tune prior -You can fine-tune kandinsky prior model with `train_text_to_image_prior.py` script. Note that we currently do not support `--gradient_checkpointing` for prior model fine-tune. +You can fine-tune the Kandinsky prior model with `train_text_to_image_prior.py` script. Note that we currently do not support `--gradient_checkpointing` for prior model fine-tuning.
From 8e8cb1c1d33b141fae462076babf5929224f197c Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Sun, 10 Sep 2023 18:58:14 -1000 Subject: [PATCH 34/38] Update examples/kandinsky2_2/text_to_image/README.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- examples/kandinsky2_2/text_to_image/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kandinsky2_2/text_to_image/README.md b/examples/kandinsky2_2/text_to_image/README.md index 8e6421498aa8..72fa5f6d704d 100644 --- a/examples/kandinsky2_2/text_to_image/README.md +++ b/examples/kandinsky2_2/text_to_image/README.md @@ -221,7 +221,7 @@ In a nutshell, LoRA allows adapting pretrained models by adding pairs of rank-de [cloneofsimo](https://github.com/cloneofsimo) was the first to try out LoRA training for Stable Diffusion in the popular [lora](https://github.com/cloneofsimo/lora) GitHub repository. -With LoRA, it's possible to fine-tune Kandinsky2.2 on a custom image-caption pair dataset +With LoRA, it's possible to fine-tune Kandinsky 2.2 on a custom image-caption pair dataset on consumer GPUs like Tesla T4, Tesla V100. ### Training From db250480274ee30402d523f31f905e686cf43826 Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Sun, 10 Sep 2023 18:58:22 -1000 Subject: [PATCH 35/38] Update examples/kandinsky2_2/text_to_image/README.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- examples/kandinsky2_2/text_to_image/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kandinsky2_2/text_to_image/README.md b/examples/kandinsky2_2/text_to_image/README.md index 72fa5f6d704d..cf23bc51bdf6 100644 --- a/examples/kandinsky2_2/text_to_image/README.md +++ b/examples/kandinsky2_2/text_to_image/README.md @@ -159,7 +159,7 @@ accelerate launch --mixed_precision="fp16" train_text_to_image_prior.py \ -To perform inference with the fune-tuned prior model, you will need to first create a prior pipeline by passing the `output_dir` to `DiffusionPpeline`, and then you can create a `KandinskyV22CombinedPipeline` with a pretrained or fine-tuned decoder checkpoint, along with all the modules of the prior pipeline you just created. +To perform inference with the fine-tuned prior model, you will need to first create a prior pipeline by passing the `output_dir` to `DiffusionPipeline`. Then create a `KandinskyV22CombinedPipeline` from a pretrained or fine-tuned decoder checkpoint along with all the modules of the prior pipeline you just created. ```python from diffusers import AutoPipelineForText2Image, DiffusionPipeline From 91c836786a9578538dfc08e0b00297fb2ab03092 Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Sun, 10 Sep 2023 18:59:14 -1000 Subject: [PATCH 36/38] Update examples/kandinsky2_2/text_to_image/README.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- examples/kandinsky2_2/text_to_image/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/kandinsky2_2/text_to_image/README.md b/examples/kandinsky2_2/text_to_image/README.md index cf23bc51bdf6..4305c4e06124 100644 --- a/examples/kandinsky2_2/text_to_image/README.md +++ b/examples/kandinsky2_2/text_to_image/README.md @@ -204,9 +204,9 @@ accelerate launch --mixed_precision="fp16" --multi_gpu train_text_to_image_deco #### Training with Min-SNR weighting -We support training with the Min-SNR weighting strategy proposed in [Efficient Diffusion Training via Min-SNR Weighting Strategy](https://arxiv.org/abs/2303.09556) which helps to achieve faster convergence -by rebalancing the loss. In order to use it, one needs to set the `--snr_gamma` argument. The recommended -value when using it is 5.0. +We support training with the Min-SNR weighting strategy proposed in [Efficient Diffusion Training via Min-SNR Weighting Strategy](https://arxiv.org/abs/2303.09556) which helps achieve faster convergence +by rebalancing the loss. Enable the `--snr_gamma` argument and set it to the recommended +value of 5.0. ## Training with LoRA From 7066be980017dd9cd763010ba3ba7db1e2ec2d41 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Mon, 11 Sep 2023 05:38:32 +0000 Subject: [PATCH 37/38] add --- examples/kandinsky2_2/text_to_image/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/kandinsky2_2/text_to_image/README.md b/examples/kandinsky2_2/text_to_image/README.md index 4305c4e06124..6e5a1835593f 100644 --- a/examples/kandinsky2_2/text_to_image/README.md +++ b/examples/kandinsky2_2/text_to_image/README.md @@ -175,6 +175,7 @@ images = pipe(prompt=prompt, negative_prompt=negative_prompt).images images[0] ``` +If you want to use a fine-tuned decoder checkpoint along with your fine-tuned prior checkpoint, you can simply replace the "kandinsky-community/kandinsky-2-2-decoder" in above code with your custom model repo name. Note that in order to be able to create a `KandinskyV22CombinedPipeline`, your model repository need to have a prior tag. If you have created your model repo using our training script, the prior tag is automatically included. #### Training with multiple GPUs From cf8ea15775ae3f345a75a15643ba1cbe1fb456e5 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Wed, 13 Sep 2023 21:46:03 +0000 Subject: [PATCH 38/38] make style --- docs/source/en/training/text2image.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/source/en/training/text2image.md b/docs/source/en/training/text2image.md index 9585141c1468..6aa39572ab34 100644 --- a/docs/source/en/training/text2image.md +++ b/docs/source/en/training/text2image.md @@ -281,3 +281,8 @@ image.save("yoda-pokemon.png") * We support fine-tuning the UNet shipped in [Stable Diffusion XL](https://huggingface.co/papers/2307.01952) via the `train_text_to_image_sdxl.py` script. Please refer to the docs [here](https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/README_sdxl.md). * We also support fine-tuning of the UNet and Text Encoder shipped in [Stable Diffusion XL](https://huggingface.co/papers/2307.01952) with LoRA via the `train_text_to_image_lora_sdxl.py` script. Please refer to the docs [here](https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/README_sdxl.md). + + +## Kandinsky 2.2 + +* We support fine-tuning both the decoder and prior in Kandinsky2.2 with the `train_text_to_image_prior.py` and `train_text_to_image_decoder.py` scripts. LoRA support is also included. Please refer to the docs [here](https://github.com/huggingface/diffusers/blob/main/examples/kandinsky2_2/text_to_image/README_sdxl.md). \ No newline at end of file