pytorch/15_transfer_learning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "model-zoo.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0Hk4wNbNytnB"
      },
      "source": [
        "# Transfer Learning  on CIFAR-10\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LFpHDxd6HHLy"
      },
      "source": [
        "### Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lEpof2LG7NA3"
      },
      "source": [
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torchvision import datasets\n",
        "from torchvision import transforms\n",
        "from torch.utils.data import DataLoader\n",
        "from torchvision import models\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    torch.backends.cudnn.deterministic = True"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PXz_dHMTHHL0"
      },
      "source": [
        "### Settings and Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9zWGjESt2QYa",
        "outputId": "6021b1c1-07d8-46f1-a3a0-d1f8137fa6d2"
      },
      "source": [
        "# Device\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "\n",
        "# Hyperparameters\n",
        "random_seed = 1\n",
        "learning_rate = 0.0001\n",
        "num_epochs = 5\n",
        "batch_size = 128\n",
        "torch.manual_seed(random_seed)\n",
        "\n",
        "\n",
        "# Architecture\n",
        "num_features = 784\n",
        "num_classes = 10\n",
        "\n",
        "\n",
        "# Data\n",
        "custom_transform = transforms.Compose([\n",
        "    transforms.Resize((224, 224)),\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
        "                          std=[0.229, 0.224, 0.225])])\n",
        "\n",
        "\n",
        "train_dataset = datasets.CIFAR10(root='data', \n",
        "                                 train=True, \n",
        "                                 transform=custom_transform,\n",
        "                                 download=True)\n",
        "\n",
        "test_dataset = datasets.CIFAR10(root='data', \n",
        "                                train=False, \n",
        "                                transform=custom_transform)\n",
        "\n",
        "\n",
        "train_loader = DataLoader(dataset=train_dataset, \n",
        "                          batch_size=batch_size,\n",
        "                          shuffle=True)\n",
        "\n",
        "test_loader = DataLoader(dataset=test_dataset, \n",
        "                         batch_size=batch_size,\n",
        "                         shuffle=False)\n",
        "\n",
        "\n",
        "# Checking the dataset\n",
        "for images, labels in train_loader:  \n",
        "    print('Image batch dimensions:', images.shape)\n",
        "    print('Image label dimensions:', labels.shape)\n",
        "    break"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Files already downloaded and verified\n",
            "Image batch dimensions: torch.Size([128, 3, 224, 224])\n",
            "Image label dimensions: torch.Size([128])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F1a6OD4-HHL3"
      },
      "source": [
        "### Loading the Pre-Trained Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8PPnruRVXvOP",
        "outputId": "f07acd48-2180-4c8b-bcec-bce3d15988ac"
      },
      "source": [
        "model = models.vgg16(pretrained=True)\n",
        "model"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "VGG(\n",
              "  (features): Sequential(\n",
              "    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (1): ReLU(inplace=True)\n",
              "    (2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (3): ReLU(inplace=True)\n",
              "    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (5): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (6): ReLU(inplace=True)\n",
              "    (7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (8): ReLU(inplace=True)\n",
              "    (9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (10): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (11): ReLU(inplace=True)\n",
              "    (12): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (13): ReLU(inplace=True)\n",
              "    (14): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (15): ReLU(inplace=True)\n",
              "    (16): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (17): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (18): ReLU(inplace=True)\n",
              "    (19): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (20): ReLU(inplace=True)\n",
              "    (21): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (22): ReLU(inplace=True)\n",
              "    (23): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (24): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (25): ReLU(inplace=True)\n",
              "    (26): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (27): ReLU(inplace=True)\n",
              "    (28): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (29): ReLU(inplace=True)\n",
              "    (30): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "  )\n",
              "  (avgpool): AdaptiveAvgPool2d(output_size=(7, 7))\n",
              "  (classifier): Sequential(\n",
              "    (0): Linear(in_features=25088, out_features=4096, bias=True)\n",
              "    (1): ReLU(inplace=True)\n",
              "    (2): Dropout(p=0.5, inplace=False)\n",
              "    (3): Linear(in_features=4096, out_features=4096, bias=True)\n",
              "    (4): ReLU(inplace=True)\n",
              "    (5): Dropout(p=0.5, inplace=False)\n",
              "    (6): Linear(in_features=4096, out_features=1000, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gBxXwJfQgd1S"
      },
      "source": [
        "### Freezing the Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cd3BYUkLgnM8"
      },
      "source": [
        "for param in model.parameters():\n",
        "    param.requires_grad = False"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HukRzjmiguzd"
      },
      "source": [
        "model.classifier[3].requires_grad = True"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PgapubKlgyks"
      },
      "source": [
        "model.classifier[6] = nn.Sequential(\n",
        "                      nn.Linear(4096, 512), \n",
        "                      nn.ReLU(), \n",
        "                      nn.Dropout(0.5),\n",
        "                      nn.Linear(512, num_classes))"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PhpvTQ-OhJB5"
      },
      "source": [
        "model = model.to(device)\n",
        "optimizer = torch.optim.Adam(model.parameters())"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rzFIKC5mSCNs"
      },
      "source": [
        "### Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Pnl1_8H7SHK9",
        "outputId": "e93a5883-3517-4214-ce6c-dc06208d6ae0"
      },
      "source": [
        "def compute_accuracy(model, data_loader):\n",
        "    correct_pred, num_examples = 0, 0\n",
        "    for i, (features, targets) in enumerate(data_loader):\n",
        "            \n",
        "        features = features.to(device)\n",
        "        targets = targets.to(device)\n",
        "\n",
        "        logits = model(features)\n",
        "        _, predicted_labels = torch.max(logits, 1)\n",
        "        num_examples += targets.size(0)\n",
        "        correct_pred += (predicted_labels == targets).sum()\n",
        "    return correct_pred.float()/num_examples * 100\n",
        "    \n",
        "for epoch in range(num_epochs):\n",
        "    \n",
        "    model.train()\n",
        "    for batch_idx, (features, targets) in enumerate(train_loader):\n",
        "        \n",
        "        features = features.to(device)\n",
        "        targets = targets.to(device)\n",
        "            \n",
        "        ### FORWARD AND BACK PROP\n",
        "        logits = model(features)\n",
        "        cost = F.cross_entropy(logits, targets)\n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        cost.backward()\n",
        "        \n",
        "        ### UPDATE MODEL PARAMETERS\n",
        "        optimizer.step()\n",
        "        \n",
        "        ### LOGGING\n",
        "        if not batch_idx % 50:\n",
        "            print ('Epoch: %03d/%03d | Batch %04d/%04d | Cost: %.4f' \n",
        "                   %(epoch+1, num_epochs, batch_idx, \n",
        "                     len(train_loader), cost))\n",
        "\n",
        "    model.eval()\n",
        "    with torch.set_grad_enabled(False):\n",
        "        print('Epoch: %03d/%03d | Train: %.3f%% ' %(\n",
        "              epoch+1, num_epochs, \n",
        "              compute_accuracy(model, train_loader)))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 001/005 | Batch 0000/0391 | Cost: 2.3117\n",
            "Epoch: 001/005 | Batch 0050/0391 | Cost: 0.6383\n",
            "Epoch: 001/005 | Batch 0100/0391 | Cost: 0.8338\n",
            "Epoch: 001/005 | Batch 0150/0391 | Cost: 0.6465\n",
            "Epoch: 001/005 | Batch 0200/0391 | Cost: 0.5130\n",
            "Epoch: 001/005 | Batch 0250/0391 | Cost: 0.6768\n",
            "Epoch: 001/005 | Batch 0300/0391 | Cost: 0.5832\n",
            "Epoch: 001/005 | Batch 0350/0391 | Cost: 0.6239\n",
            "Epoch: 001/005 | Train: 83.400% \n",
            "Epoch: 002/005 | Batch 0000/0391 | Cost: 0.6694\n",
            "Epoch: 002/005 | Batch 0050/0391 | Cost: 0.5546\n",
            "Epoch: 002/005 | Batch 0100/0391 | Cost: 0.4742\n",
            "Epoch: 002/005 | Batch 0150/0391 | Cost: 0.5282\n",
            "Epoch: 002/005 | Batch 0200/0391 | Cost: 0.6725\n",
            "Epoch: 002/005 | Batch 0250/0391 | Cost: 0.6962\n",
            "Epoch: 002/005 | Batch 0300/0391 | Cost: 0.4866\n",
            "Epoch: 002/005 | Batch 0350/0391 | Cost: 0.6809\n",
            "Epoch: 002/005 | Train: 85.468% \n",
            "Epoch: 003/005 | Batch 0000/0391 | Cost: 0.4796\n",
            "Epoch: 003/005 | Batch 0050/0391 | Cost: 0.5336\n",
            "Epoch: 003/005 | Batch 0100/0391 | Cost: 0.4859\n",
            "Epoch: 003/005 | Batch 0150/0391 | Cost: 0.5673\n",
            "Epoch: 003/005 | Batch 0200/0391 | Cost: 0.6507\n",
            "Epoch: 003/005 | Batch 0250/0391 | Cost: 0.6041\n",
            "Epoch: 003/005 | Batch 0300/0391 | Cost: 0.7358\n",
            "Epoch: 003/005 | Batch 0350/0391 | Cost: 0.6902\n",
            "Epoch: 003/005 | Train: 86.382% \n",
            "Epoch: 004/005 | Batch 0000/0391 | Cost: 0.4464\n",
            "Epoch: 004/005 | Batch 0050/0391 | Cost: 0.5532\n",
            "Epoch: 004/005 | Batch 0100/0391 | Cost: 0.5370\n",
            "Epoch: 004/005 | Batch 0150/0391 | Cost: 0.6030\n",
            "Epoch: 004/005 | Batch 0200/0391 | Cost: 0.6853\n",
            "Epoch: 004/005 | Batch 0250/0391 | Cost: 0.5403\n",
            "Epoch: 004/005 | Batch 0300/0391 | Cost: 0.5543\n",
            "Epoch: 004/005 | Batch 0350/0391 | Cost: 0.5761\n",
            "Epoch: 004/005 | Train: 86.576% \n",
            "Epoch: 005/005 | Batch 0000/0391 | Cost: 0.4798\n",
            "Epoch: 005/005 | Batch 0050/0391 | Cost: 0.5277\n",
            "Epoch: 005/005 | Batch 0100/0391 | Cost: 0.4972\n",
            "Epoch: 005/005 | Batch 0150/0391 | Cost: 0.5645\n",
            "Epoch: 005/005 | Batch 0200/0391 | Cost: 0.6429\n",
            "Epoch: 005/005 | Batch 0250/0391 | Cost: 0.6062\n",
            "Epoch: 005/005 | Batch 0300/0391 | Cost: 0.5547\n",
            "Epoch: 005/005 | Batch 0350/0391 | Cost: 0.4263\n",
            "Epoch: 005/005 | Train: 87.136% \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7sQBgaPtHHL7"
      },
      "source": [
        "### Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B0MYkRbBQs5_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a392f43c-9aae-4c4a-cf4c-1db17d0bb227"
      },
      "source": [
        "with torch.set_grad_enabled(False):\n",
        "  print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader)))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test accuracy: 83.94%\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}