Diffusion_model_DTD.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from datetime import datetime
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader, ConcatDataset
from torchvision import transforms
from torchvision.io import read_image
from torchvision.utils import save_image
from tqdm import tqdm
import math
import glob

# GLOBALS
device = torch.device("cuda")
diffusion_steps = 1000
max_epoch = 1000
n_hold_final = 10


class SelfAttention(nn.Module):
    def __init__(self, h_size):
        super(SelfAttention, self).__init__()
        self.h_size = h_size
        self.mha = nn.MultiheadAttention(h_size, 4, batch_first=True)
        self.ln = nn.LayerNorm([h_size])
        self.ff_self = nn.Sequential(
            nn.LayerNorm([h_size]),
            nn.Linear(h_size, h_size),
            nn.GELU(),
            nn.Linear(h_size, h_size),
        )

    def forward(self, x):
        x_ln = self.ln(x)
        attention_value, _ = self.mha(x_ln, x_ln, x_ln)
        attention_value = attention_value + x
        attention_value = self.ff_self(attention_value) + attention_value
        return attention_value


class SAWrapper(nn.Module):
    def __init__(self, h_size, num_s):
        super(SAWrapper, self).__init__()
        self.sa = nn.Sequential(*[SelfAttention(h_size) for _ in range(1)])
        self.num_s = num_s
        self.h_size = h_size

    def forward(self, x):
        x = x.view(-1, self.h_size, self.num_s * self.num_s).swapaxes(1, 2)
        x = self.sa(x)
        x = x.swapaxes(2, 1).view(-1, self.h_size, self.num_s, self.num_s)
        return x


# U-Net code adapted from: https://github.com/milesial/Pytorch-UNet
class DoubleConv(nn.Module):
    def __init__(self, in_channels, out_channels, mid_channels=None, residual=False):
        super().__init__()
        self.residual = residual
        if not mid_channels:
            mid_channels = out_channels
        self.double_conv = nn.Sequential(
            nn.Conv2d(in_channels, mid_channels, kernel_size=3, padding=1, bias=False),
            nn.GroupNorm(1, mid_channels),
            nn.GELU(),
            nn.Conv2d(mid_channels, out_channels, kernel_size=3, padding=1, bias=False),
            nn.GroupNorm(1, out_channels),
        )

    def forward(self, x):
        if self.residual:
            return F.gelu(x + self.double_conv(x))
        else:
            return self.double_conv(x)


class Down(nn.Module):
    def __init__(self, in_channels, out_channels):
        super().__init__()
        self.maxpool_conv = nn.Sequential(
            nn.MaxPool2d(2),
            DoubleConv(in_channels, in_channels, residual=True),
            DoubleConv(in_channels, out_channels),
        )

    def forward(self, x):
        return self.maxpool_conv(x)


class Up(nn.Module):
    def __init__(self, in_channels, out_channels, bilinear=True):
        super().__init__()

        # if bilinear, use the normal convolutions to reduce the number of channels
        if bilinear:
            self.up = nn.Upsample(scale_factor=2, mode="bilinear", align_corners=True)
            self.conv = DoubleConv(in_channels, in_channels, residual=True)
            self.conv2 = DoubleConv(in_channels, out_channels, in_channels // 2)
        else:
            self.up = nn.ConvTranspose2d(
                in_channels, in_channels // 2, kernel_size=2, stride=2
            )
            self.conv = DoubleConv(in_channels, out_channels)

    def forward(self, x1, x2):
        x1 = self.up(x1)
        # input is CHW
        diffY = x2.size()[2] - x1.size()[2]
        diffX = x2.size()[3] - x1.size()[3]

        x1 = F.pad(x1, [diffX // 2, diffX - diffX // 2, diffY // 2, diffY - diffY // 2])
        x = torch.cat([x2, x1], dim=1)
        x = self.conv(x)
        x = self.conv2(x)
        return x


class OutConv(nn.Module):
    def __init__(self, in_channels, out_channels):
        super(OutConv, self).__init__()
        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=1)

    def forward(self, x):
        return self.conv(x)


class DiffusionModel(nn.Module):
    def __init__(self, t_range):
        super().__init__()
        self.beta_small = 1e-4
        self.beta_large = 0.02
        self.t_range = t_range
        self.device = torch.device("cuda")

        bilinear = True
        self.inc = DoubleConv(3, 64)
        self.down1 = Down(64, 128)
        self.down2 = Down(128, 256)
        factor = 2 if bilinear else 1
        self.down3 = Down(256, 512 // factor)
        self.up1 = Up(512, 256 // factor, bilinear)
        self.up2 = Up(256, 128 // factor, bilinear)
        self.up3 = Up(128, 64, bilinear)
        self.outc = OutConv(64, 3)
        self.sa1 = SAWrapper(256, 32)
        self.sa2 = SAWrapper(256, 16)
        self.sa3 = SAWrapper(128, 32)

    def pos_encoding(self, t, channels, embed_size):
        inv_freq = 1.0 / (10000 ** (torch.arange(0, channels, 2, device=self.device).float() / channels))
        pos_enc_a = torch.sin(t.repeat(1, channels // 2) * inv_freq)
        pos_enc_b = torch.cos(t.repeat(1, channels // 2) * inv_freq)
        pos_enc = torch.cat([pos_enc_a, pos_enc_b], dim=-1)
        return pos_enc.view(-1, channels, 1, 1).repeat(1, 1, embed_size, embed_size)

    def forward(self, x, t):
        """
        Model is U-Net with added positional encodings and self-attention layers.
        """
        x1 = self.inc(x)
        x2 = self.down1(x1) + self.pos_encoding(t, 128, 64)
        x3 = self.down2(x2) + self.pos_encoding(t, 256, 32)
        x3 = self.sa1(x3)
        x4 = self.down3(x3) + self.pos_encoding(t, 256, 16)
        x4 = self.sa2(x4)
        x = self.up1(x4, x3) + self.pos_encoding(t, 128, 32)
        x = self.sa3(x)
        x = self.up2(x, x2) + self.pos_encoding(t, 64, 64)
        x = self.up3(x, x1) + self.pos_encoding(t, 64, 128)
        output = self.outc(x)
        return output

    def beta(self, t):
        return self.beta_small + (t / self.t_range) * (
            self.beta_large - self.beta_small
        )

    def alpha(self, t):
        return 1 - self.beta(t)

    def alpha_bar(self, t):
        return math.prod([self.alpha(j) for j in range(t)])

    def get_loss(self, batch):
        """
        Corresponds to Algorithm 1 from (Ho et al., 2020).
        """
        ts = torch.randint(0, self.t_range, [batch.shape[0]], device=self.device)
        noise_imgs = []
        epsilons = torch.randn(batch.shape, device=self.device)
        for i in range(len(ts)):
            a_hat = self.alpha_bar(ts[i])
            noise_imgs.append((math.sqrt(a_hat) * batch[i]) + (math.sqrt(1 - a_hat) * epsilons[i]))
        noise_imgs = torch.stack(noise_imgs, dim=0)
        e_hat = self.forward(noise_imgs, ts.unsqueeze(-1).type(torch.float))
        loss = F.mse_loss(e_hat.reshape(-1, 32*32), epsilons.reshape(-1, 32*32))
        return loss

    def denoise_sample(self, x, t):
        """
        Corresponds to the inner loop of Algorithm 2 from (Ho et al., 2020).
        """
        with torch.no_grad():
            if t > 1:
                z = torch.randn(x.shape)
            else:
                z = torch.zeros(1)
            e_hat = self.forward(x, t.view(1, 1).repeat(x.shape[0], 1)).to(self.device)
            pre_scale = (1 / math.sqrt(self.alpha(t)))
            e_scale = ((1 - self.alpha(t)) / math.sqrt(1 - self.alpha_bar(t))).to(self.device)
            post_sigma = (math.sqrt(self.beta(t)) * z).to(self.device)
            x = pre_scale * (x - e_scale * e_hat) + post_sigma
            return x

    def training_step(self, batch, batch_idx):
        loss = self.get_loss(batch, batch_idx)
        self.log("train/loss", loss)
        return loss

    def validation_step(self, batch, batch_idx):
        loss = self.get_loss(batch, batch_idx)
        self.log("val/loss", loss)
        return

    def configure_optimizers(self):
        optimizer = torch.optim.Adam(self.parameters(), lr=2e-4)
        return optimizer


class DtdDataset(Dataset):
    def __init__(self, img_dir, transform=None, target_transform=None):
        self.img_labels, self.img_files = [], []
        self.img_dir = img_dir
        self.transform = transform
        self.target_transform = target_transform
        with open(self.img_dir+"labels/labels_joint_anno.txt", "r") as f:
            for line in f:
                self.img_labels.append(line.split(" ")[1])
        for file in sorted(glob.glob(self.img_dir+"images/*/**")):
            self.img_files.append(file[11:])

    def __len__(self):
        return len(glob.glob(self.img_dir + "images/*/**"))

    def __getitem__(self, idx):
        image = read_image(self.img_dir + "images/" + self.img_files[idx])
        label = self.img_labels[idx]
        if self.transform:
            image = self.transform(image)
        if self.target_transform:
            label = self.target_transform(label)
        return image, label


def main():
    transformations = transforms.Compose([
        transforms.RandomCrop(128),
        transforms.RandomHorizontalFlip(0.3),
        transforms.ConvertImageDtype(torch.float32),
        transforms.Normalize((0.533, 0.475, 0.426), (0.271, 0.261, 0.269))
    ])
    trainset1 = DtdDataset("dtd/", transform=transformations)
    trainset2 = DtdDataset("dtd/", transform=transformations)
    trainset3 = DtdDataset("dtd/", transform=transformations)
    trainset_norm = ConcatDataset([trainset1, trainset2, trainset3])
    data_train = DataLoader(trainset_norm, batch_size=64, shuffle=True)
    model = DiffusionModel(diffusion_steps).to(device)
    optim = model.configure_optimizers()

    # Training
    losses = []
    for e in tqdm(range(max_epoch)):
        train_loss = 0
        for batch in data_train:
            x = batch[0].to(torch.device("cuda"))
            optim.zero_grad()
            loss = model.get_loss(x)
            train_loss += loss.item()
            loss.backward()
            optim.step()
        print(f"{datetime.now().strftime('%H:%M:%S')} : Epoch {e} - Average loss {train_loss:.3f}")
        losses.append(train_loss)
    np.save(f"loss_dtd_{max_epoch}e.npy", losses)
    torch.save({'model_state_dict': model.state_dict(), 'optim_state_dict': optim.state_dict()},
               f"models/diffusion_dtd_{max_epoch}e.pt")

    # Sampling
    model.eval()
    gen_samples = []
    x = torch.randn((9, 3, 128, 128))
    sample_steps = torch.arange(model.t_range - 1, 0, -1)
    for t in tqdm(sample_steps):
        x = model.denoise_sample(x.to(device), t.to(device))
        if t % 50 == 0:
            gen_samples.append(x)
    for _ in range(n_hold_final):
        gen_samples.append(x)
    gen_samples = torch.stack(gen_samples, dim=0).moveaxis(2, 4).squeeze(-1)
    clamped_samples = (gen_samples.clamp(-1, 1) + 1) / 2
    for i in range(29):
        for j in range(9):
            save_image(clamped_samples[i, j].permute(2, 0, 1).cpu(), f"images/sample{j}_time{i}.png")


if __name__ == "__main__":
    main()