06_sgd-with-scheduler.py

import time

import lightning as L
from lightning import Fabric
import torch
import torch.nn.functional as F
import torchmetrics
from torchvision import transforms
from torchvision.models import vit_l_16
from torchvision.models import ViT_L_16_Weights
from watermark import watermark

from local_utilities import get_dataloaders_cifar10


def train(num_epochs, model, optimizer, scheduler, train_loader, val_loader, fabric, accumulation_steps):

    for epoch in range(num_epochs):
        train_acc = torchmetrics.Accuracy(task="multiclass", num_classes=10).to(fabric.device)

        model.train()
        for batch_idx, (features, targets) in enumerate(train_loader):
            model.train()

            ### FORWARD AND BACK PROP   
            logits = model(features)
            loss = F.cross_entropy(logits, targets) / accumulation_steps # NEW

            fabric.backward(loss)


            ### UPDATE MODEL PARAMETERS
            if batch_idx % accumulation_steps == 0:  # NEW
                optimizer.step()
                scheduler.step()
                optimizer.zero_grad()

            ### LOGGING
            if not batch_idx % 300:
                print(f"Epoch: {epoch+1:04d}/{num_epochs:04d} | Batch {batch_idx:04d}/{len(train_loader):04d} | Loss: {loss:.4f}")

            model.eval()
            with torch.no_grad():
                predicted_labels = torch.argmax(logits, 1)
                train_acc.update(predicted_labels, targets)

        ### MORE LOGGING
        model.eval()
        with torch.no_grad():
            val_acc = torchmetrics.Accuracy(task="multiclass", num_classes=10).to(fabric.device)

            for (features, targets) in val_loader:
                outputs = model(features)
                predicted_labels = torch.argmax(outputs, 1)
                val_acc.update(predicted_labels, targets)

            print(f"Epoch: {epoch+1:04d}/{num_epochs:04d} | Train acc.: {train_acc.compute()*100:.2f}% | Val acc.: {val_acc.compute()*100:.2f}%")
            train_acc.reset(), val_acc.reset()


if __name__ == "__main__":

    print(watermark(packages="torch,lightning", python=True))
    print("Torch CUDA available?", torch.cuda.is_available())

    fabric = Fabric(accelerator="cuda", devices=1, precision="bf16-true")
    fabric.launch()

    torch.set_float32_matmul_precision('high')

    L.seed_everything(123)

    ##########################
    ### 1 Loading the Dataset
    ##########################
    train_transforms = transforms.Compose([transforms.Resize((224, 224)),
                                           #transforms.RandomCrop((224, 224)),
                                           transforms.ToTensor()])
    
    test_transforms = transforms.Compose([transforms.Resize((224, 224)),
                                          #transforms.CenterCrop((224, 224)),
                                          transforms.ToTensor()])
    
    NUM_EPOCHS = 1
    BATCHSIZE = 16
    ACCUMULATION_STEPS = 4
    MICROBATCHSIZE = int(BATCHSIZE / ACCUMULATION_STEPS)

    train_loader, val_loader, test_loader = get_dataloaders_cifar10(
        batch_size=MICROBATCHSIZE,
        num_workers=3,
        train_transforms=train_transforms,
        test_transforms=test_transforms,
        validation_fraction=0.1)
    
    train_loader, val_loader, test_loader = fabric.setup_dataloaders(
        train_loader, val_loader, test_loader)


    #########################################
    ### 2 Initializing the Model
    #########################################

    model = vit_l_16(weights=ViT_L_16_Weights.IMAGENET1K_V1)

    # replace output layer
    model.heads.head = torch.nn.Linear(in_features=1024, out_features=10)

    optimizer = torch.optim.SGD(model.parameters(), lr=0.01)

    num_steps = NUM_EPOCHS * len(train_loader)
    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=num_steps)
    model, optimizer = fabric.setup(model, optimizer)

    #########################################
    ### 3 Finetuning
    #########################################

    start = time.time()
    train(
        num_epochs=NUM_EPOCHS,
        model=model,
        optimizer=optimizer,
        scheduler=scheduler,
        train_loader=train_loader,
        val_loader=val_loader,
        fabric=fabric,
        accumulation_steps=ACCUMULATION_STEPS
    )

    end = time.time()
    elapsed = end-start
    print(f"Time elapsed {elapsed/60:.2f} min")
    print(f"Memory used: {torch.cuda.max_memory_reserved() / 1e9:.02f} GB")

    #########################################
    ### 4 Evaluation
    #########################################
    
    with torch.no_grad():
        model.eval()
        test_acc = torchmetrics.Accuracy(task="multiclass", num_classes=10).to(fabric.device)

        for (features, targets) in test_loader:
            outputs = model(features)
            predicted_labels = torch.argmax(outputs, 1)
            test_acc.update(predicted_labels, targets)

    print(f"Test accuracy {test_acc.compute()*100:.2f}%")