examples/04_Training_with_Pytorch_Learning_Rate_Scheduling.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "04_Training_with_Pytorch_Learning_Rate_Scheduling.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n3tYy2PfpYz_"
      },
      "source": [
        "# **Using LibAUC optimizers with Pytorch Learning Rate Scheduling**\n",
        "\n",
        "**Author**: Zhuoning Yuan\n",
        "\n",
        "**Introduction**\n",
        "\n",
        "In this tutorial, you will learn how to quickly train models using LibAUC with [Pytorch Learning Rate Scheduler](https:/https://www.kaggle.com/code/isbhargav/guide-to-pytorch-learning-rate-scheduling/notebook/). After completion of this tutorial, you should be able to use LibAUC to train your own models on your own datasets.\n",
        "\n",
        "**Useful Resources**:\n",
        "* Website: https://libauc.org\n",
        "* Github: https://github.com/Optimization-AI/LibAUC\n",
        "\n",
        "**Reference**:  \n",
        "\n",
        "If you find this tutorial helpful in your work,  please acknowledge our library and cite the following paper:\n",
        "\n",
        "<pre>\n",
        "@inproceedings{yuan2021large,\n",
        "  title={Large-scale robust deep auc maximization: A new surrogate loss and empirical studies on medical image classification},\n",
        "  author={Yuan, Zhuoning and Yan, Yan and Sonka, Milan and Yang, Tianbao},\n",
        "  booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},\n",
        "  pages={3040--3049},\n",
        "  year={2021}\n",
        "  }\n",
        "</pre>\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pSR3EPy_n1Cc"
      },
      "source": [
        "# **Installing LibAUC**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dBQk04JIm6Kb"
      },
      "source": [
        "!pip install libauc==1.2.0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FSmuI7S2n0uu"
      },
      "source": [
        "# **Importing AUC Training Pipeline**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0m60hg13nsc4",
        "outputId": "f98cabc7-0d77-4683-9391-50b1bfa167ce"
      },
      "source": [
        "from libauc.losses import AUCMLoss\n",
        "from libauc.optimizers import PESG\n",
        "from libauc.models import resnet20 as ResNet20\n",
        "from libauc.datasets import CIFAR10\n",
        "from libauc.utils import ImbalancedDataGenerator\n",
        "from libauc.metrics import auc_roc_score\n",
        "\n",
        "import torch \n",
        "from PIL import Image\n",
        "import numpy as np\n",
        "import torchvision.transforms as transforms\n",
        "from torch.utils.data import Dataset\n",
        "\n",
        "class ImageDataset(Dataset):\n",
        "    def __init__(self, images, targets, image_size=32, crop_size=30, mode='train'):\n",
        "       self.images = images.astype(np.uint8)\n",
        "       self.targets = targets\n",
        "       self.mode = mode\n",
        "       self.transform_train = transforms.Compose([                                                \n",
        "                              transforms.ToTensor(),\n",
        "                              transforms.RandomCrop((crop_size, crop_size), padding=None),\n",
        "                              transforms.RandomHorizontalFlip(),\n",
        "                              transforms.Resize((image_size, image_size)),\n",
        "                              ])\n",
        "       self.transform_test = transforms.Compose([\n",
        "                             transforms.ToTensor(),\n",
        "                             transforms.Resize((image_size, image_size)),\n",
        "                              ])\n",
        "    def __len__(self):\n",
        "        return len(self.images)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        image = self.images[idx]\n",
        "        target = self.targets[idx]\n",
        "        image = Image.fromarray(image.astype('uint8'))\n",
        "        if self.mode == 'train':\n",
        "            image = self.transform_train(image)\n",
        "        else:\n",
        "            image = self.transform_test(image)\n",
        "        return image, target\n",
        "\n",
        "# paramaters\n",
        "SEED = 123\n",
        "BATCH_SIZE = 128\n",
        "imratio = 0.1\n",
        "lr = 0.1\n",
        "epoch_decay = 2e-3 # 1/gamma\n",
        "weight_decay = 1e-4\n",
        "margin = 1.0\n",
        "\n",
        "\n",
        "# dataloader \n",
        "(train_data, train_label) = CIFAR10(root='./data', train=True) \n",
        "(test_data, test_label) = CIFAR10(root='./data', train=False) \n",
        "\n",
        "generator = ImbalancedDataGenerator(verbose=True, random_seed=0)\n",
        "(train_images, train_labels) = generator.transform(train_data, train_label, imratio=imratio)\n",
        "(test_images, test_labels) = generator.transform(test_data, test_label, imratio=0.5)\n",
        "\n",
        "trainloader = torch.utils.data.DataLoader(ImageDataset(train_images, train_labels), batch_size=BATCH_SIZE, shuffle=True, num_workers=1, pin_memory=True, drop_last=True)\n",
        "testloader = torch.utils.data.DataLoader(ImageDataset(test_images, test_labels, mode='test'), batch_size=BATCH_SIZE, shuffle=False, num_workers=1,  pin_memory=True)\n",
        "\n",
        "# model \n",
        "model = ResNet20(pretrained=False, num_classes=1)\n",
        "model = model.cuda()\n",
        "\n",
        "# loss & optimizer\n",
        "loss_fn = AUCMLoss()\n",
        "optimizer = PESG(model, \n",
        "                 loss_fn=loss_fn,\n",
        "                 lr=lr, \n",
        "                 margin=margin,\n",
        "                 epoch_decay=epoch_decay, \n",
        "                 weight_decay=weight_decay)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n",
            "#SAMPLES: [27777], POS:NEG: [2777 : 25000], POS RATIO: 0.1000\n",
            "#SAMPLES: [10000], POS:NEG: [5000 : 5000], POS RATIO: 0.5000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qouS-m5soBbL"
      },
      "source": [
        "# **Pytorch Learning Rate Scheduling**\n",
        "We will cover three scheduling functions in this section: \n",
        "*   CosineAnnealingLR\n",
        "*   ReduceLROnPlateau\n",
        "*   MultiStepLR\n",
        "\n",
        "For more details, please refer to orginal PyTorch [doc](https://pytorch.org/docs/stable/optim.html).\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0nTXyToIyov6"
      },
      "source": [
        "def reset_model():\n",
        "    # loss & optimizer\n",
        "    loss_fn = AUCMLoss()\n",
        "    optimizer = PESG(model, \n",
        "                    loss_fn=loss_fn,\n",
        "                    lr=lr, \n",
        "                    epoch_decay=epoch_decay, \n",
        "                    margin=margin, \n",
        "                    weight_decay=weight_decay)\n",
        "    return loss_fn, optimizer"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D7M3g29upT1r"
      },
      "source": [
        "### CosineAnnealingLR"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zyFWKbrpoBFz"
      },
      "source": [
        "total_epochs = 10\n",
        "loss_fn, optimizer = reset_model()\n",
        "scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, len(trainloader)*total_epochs)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JP8isMj4txc3",
        "outputId": "91142e90-96ce-4819-c221-fe1dd9118d36"
      },
      "source": [
        "model.train()    \n",
        "for epoch in range(total_epochs):\n",
        "     for i, (data, targets) in enumerate(trainloader):\n",
        "         data, targets  = data.cuda(), targets.cuda()\n",
        "         y_pred = model(data)\n",
        "         y_pred = torch.sigmoid(y_pred)\n",
        "         loss = loss_fn(y_pred, targets)\n",
        "         optimizer.zero_grad()\n",
        "         loss.backward()\n",
        "         optimizer.step()\n",
        "         scheduler.step()\n",
        "     print(\"epoch: {}, loss: {:4f}, lr:{:4f}\".format(epoch, loss.item(), optimizer.lr))          "
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch: 0, loss: 0.096573, lr:0.097575\n",
            "epoch: 1, loss: 0.093474, lr:0.090493\n",
            "epoch: 2, loss: 0.102245, lr:0.079448\n",
            "epoch: 3, loss: 0.054700, lr:0.065520\n",
            "epoch: 4, loss: 0.070871, lr:0.050072\n",
            "epoch: 5, loss: 0.075728, lr:0.034618\n",
            "epoch: 6, loss: 0.061491, lr:0.020669\n",
            "epoch: 7, loss: 0.072430, lr:0.009592\n",
            "epoch: 8, loss: 0.024156, lr:0.002470\n",
            "epoch: 9, loss: 0.089273, lr:0.000000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OZ6_-eT2v44n"
      },
      "source": [
        "### ReduceLROnPlateau"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vNDn1TDMv4PP"
      },
      "source": [
        "total_epochs = 20\n",
        "loss_fn, optimizer = reset_model()\n",
        "scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, \n",
        "                                                       patience=3,  \n",
        "                                                       verbose=True, \n",
        "                                                       factor=0.5, \n",
        "                                                       threshold=0.001,\n",
        "                                                       min_lr=0.00001)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RojZiIXJwagO",
        "outputId": "8d4c2792-6693-4070-a205-0b2ab603c150"
      },
      "source": [
        "model.train()    \n",
        "for epoch in range(total_epochs):\n",
        "     for i, (data, targets) in enumerate(trainloader):\n",
        "         data, targets  = data.cuda(), targets.cuda()\n",
        "         y_pred = model(data)\n",
        "         y_pred = torch.sigmoid(y_pred)\n",
        "         loss = loss_fn(y_pred, targets)\n",
        "         optimizer.zero_grad()\n",
        "         loss.backward()\n",
        "         optimizer.step()\n",
        "     scheduler.step(loss)\n",
        "     print(\"epoch: {}, loss: {:4f}, lr:{:4f}\".format(epoch, loss.item(), optimizer.lr))          "
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch: 0, loss: 0.051323, lr:0.100000\n",
            "epoch: 1, loss: 0.073475, lr:0.100000\n",
            "epoch: 2, loss: 0.072317, lr:0.100000\n",
            "epoch: 3, loss: 0.063765, lr:0.100000\n",
            "Epoch 00005: reducing learning rate of group 0 to 5.0000e-02.\n",
            "epoch: 4, loss: 0.075719, lr:0.100000\n",
            "epoch: 5, loss: 0.028987, lr:0.050000\n",
            "epoch: 6, loss: 0.042710, lr:0.050000\n",
            "epoch: 7, loss: 0.047592, lr:0.050000\n",
            "epoch: 8, loss: 0.049839, lr:0.050000\n",
            "epoch: 9, loss: 0.022842, lr:0.050000\n",
            "epoch: 10, loss: 0.035902, lr:0.050000\n",
            "epoch: 11, loss: 0.067043, lr:0.050000\n",
            "epoch: 12, loss: 0.040740, lr:0.050000\n",
            "Epoch 00014: reducing learning rate of group 0 to 2.5000e-02.\n",
            "epoch: 13, loss: 0.039672, lr:0.050000\n",
            "epoch: 14, loss: 0.023661, lr:0.025000\n",
            "epoch: 15, loss: 0.031596, lr:0.025000\n",
            "epoch: 16, loss: 0.054208, lr:0.025000\n",
            "Epoch 00018: reducing learning rate of group 0 to 1.2500e-02.\n",
            "epoch: 17, loss: 0.044520, lr:0.025000\n",
            "epoch: 18, loss: 0.035632, lr:0.012500\n",
            "epoch: 19, loss: 0.016720, lr:0.012500\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ol6mnV8dyHZa"
      },
      "source": [
        "### MultiStepLR"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4zSK_ENbx8oR"
      },
      "source": [
        "total_epochs = 20\n",
        "loss_fn, optimizer = reset_model()\n",
        "scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[10,15], gamma=0.1)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VihFwQ1UyIxm",
        "outputId": "3d312a55-2ab2-425c-e28f-ace8fcb86e64"
      },
      "source": [
        "# reset model\n",
        "model.train()    \n",
        "for epoch in range(total_epochs):\n",
        "     for i, (data, targets) in enumerate(trainloader):\n",
        "         data, targets  = data.cuda(), targets.cuda()\n",
        "         y_pred = model(data)\n",
        "         y_pred = torch.sigmoid(y_pred)\n",
        "         loss = loss_fn(y_pred, targets)\n",
        "         optimizer.zero_grad()\n",
        "         loss.backward()\n",
        "         optimizer.step()\n",
        "     scheduler.step()\n",
        "     print(\"epoch: {}, loss: {:4f}, lr:{:4f}\".format(epoch, loss.item(), optimizer.lr))          "
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch: 0, loss: 0.036762, lr:0.100000\n",
            "epoch: 1, loss: 0.036951, lr:0.100000\n",
            "epoch: 2, loss: 0.050001, lr:0.100000\n",
            "epoch: 3, loss: 0.031577, lr:0.100000\n",
            "epoch: 4, loss: 0.049376, lr:0.100000\n",
            "epoch: 5, loss: 0.030413, lr:0.100000\n",
            "epoch: 6, loss: 0.066324, lr:0.100000\n",
            "epoch: 7, loss: 0.017938, lr:0.100000\n",
            "epoch: 8, loss: 0.040055, lr:0.100000\n",
            "epoch: 9, loss: 0.055795, lr:0.100000\n",
            "epoch: 10, loss: 0.036341, lr:0.010000\n",
            "epoch: 11, loss: 0.029210, lr:0.010000\n",
            "epoch: 12, loss: 0.015228, lr:0.010000\n",
            "epoch: 13, loss: 0.037940, lr:0.010000\n",
            "epoch: 14, loss: 0.025263, lr:0.010000\n",
            "epoch: 15, loss: 0.024623, lr:0.001000\n",
            "epoch: 16, loss: 0.041440, lr:0.001000\n",
            "epoch: 17, loss: 0.019552, lr:0.001000\n",
            "epoch: 18, loss: 0.014872, lr:0.001000\n",
            "epoch: 19, loss: 0.024439, lr:0.001000\n"
          ]
        }
      ]
    }
  ]
}