notebooks/cis6930_week2_minibatch_sampling_followup.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "cis6930-week2-minibatch-sampling-followup.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oRgpqL0KVO43"
      },
      "source": [
        "# CIS6930 Week 2: Mini-batch sampling follow-up (for in-class demo)\n",
        "\n",
        "In this notebook, you will learn how PyTorch components/functions handle \"batched\" samples.\n",
        "\n",
        "---\n",
        "\n",
        "Preparation: Go to `Runtime > Change runtime type` and choose `GPU` for the hardware accelerator.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b1dpc5PwIghS"
      },
      "source": [
        "## A magic command to check your assigned GPU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uayE3ioc3k_h",
        "outputId": "2ac00230-9694-4b50-c237-e15e1a3adf90"
      },
      "source": [
        "gpu_info = !nvidia-smi -L\n",
        "gpu_info = \"\\n\".join(gpu_info)\n",
        "if gpu_info.find(\"failed\") >= 0:\n",
        "  print(\"Not connected to a GPU\")\n",
        "else:\n",
        "  print(gpu_info)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "GPU 0: Tesla P100-PCIE-16GB (UUID: GPU-c9328b96-4cf0-0d22-d40b-9a021ea2fd42)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_Jpe1bw6ImPn"
      },
      "source": [
        "## Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a7h-6LpjVDHW"
      },
      "source": [
        "import copy\n",
        "import random\n",
        "from time import time\n",
        "from typing import Any, Dict\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "from sklearn.datasets import load_digits\n",
        "from sklearn.decomposition import PCA\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.model_selection import train_test_split\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch import optim\n",
        "from torch.utils.data import Dataset, TensorDataset, DataLoader"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2MesACC414_v"
      },
      "source": [
        "#pred = torch.FloatTensor([0.4, 0.6])\n",
        "#true = torch.LongTensor([1])\n",
        "\n",
        "#loss_func = nn.CrossEntropy()\n",
        "#loss_func(true, pred)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oxddim5C2dhs"
      },
      "source": [
        "## Code from the examples of the last class"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ClCzZBOK2pNr"
      },
      "source": [
        "### Logistic Regression"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NKKv1noF2RW0"
      },
      "source": [
        "class LogisticRegression(nn.Module):\n",
        "    def __init__(self,\n",
        "                 num_input,\n",
        "                 num_output):\n",
        "        super(LogisticRegression, self).__init__()\n",
        "        self.linear = nn.Linear(num_input, num_output)\n",
        "\n",
        "    def forward(self, X):\n",
        "        out = self.linear(X)\n",
        "        return out"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eymHeJOl2r-4"
      },
      "source": [
        "### Training script"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6QbX0WOb2a9-",
        "outputId": "c1553417-d255-402b-885d-b014dca84479"
      },
      "source": [
        "## Configurations ======\n",
        "n_epochs = 10\n",
        "batch_size = 16\n",
        "\n",
        "lr = 0.01\n",
        "momentum = 0.\n",
        "\n",
        "num_input = 64\n",
        "num_output = 10\n",
        "\n",
        "# Random Seeds\n",
        "torch.manual_seed(0)\n",
        "random.seed(0)\n",
        "np.random.seed(0)\n",
        "## ======================\n",
        "\n",
        "\n",
        "# GPU configuration\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "# Load the handwritten digit dataset\n",
        "# https://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html\n",
        "data = load_digits()\n",
        "X, y = data.data, data.target\n",
        "\n",
        "# Splint into 60% train, 20% valid, 20% test\n",
        "X_train, X_test, y_train, y_test = train_test_split(\n",
        "    X, y, test_size=0.2, random_state=1)\n",
        "\n",
        "X_train, X_valid, y_train, y_valid = train_test_split(\n",
        "    X_train, y_train, test_size=0.25, random_state=1)  # 0.25 x 0.8 = 0.2\n",
        "\n",
        "# NumPy array -> Torch tensor -> Dataset -> DataLoader\n",
        "# for the train, validation, test datasets\n",
        "dataset_train = TensorDataset(torch.Tensor(X_train),\n",
        "                              torch.LongTensor(y_train))\n",
        "dl_train = DataLoader(dataset_train,\n",
        "                      batch_size=batch_size,\n",
        "                      shuffle=True)\n",
        "\n",
        "dataset_valid = TensorDataset(torch.Tensor(X_valid),\n",
        "                              torch.LongTensor(y_valid))\n",
        "dl_valid = DataLoader(dataset_valid)\n",
        "\n",
        "dataset_test = TensorDataset(torch.Tensor(X_test),\n",
        "                              torch.LongTensor(y_test))\n",
        "dl_test = DataLoader(dataset_test)\n",
        "\n",
        "# Model, Optimzier, Loss function\n",
        "model = LogisticRegression(num_input=num_input,\n",
        "                           num_output=num_output).to(device)\n",
        "optimizer = optim.SGD(model.parameters(),\n",
        "                      lr=lr, momentum=momentum)\n",
        "\n",
        "loss_fn = nn.CrossEntropyLoss()\n",
        "\n",
        "# For each epoch\n",
        "eval_list = []\n",
        "for n in range(n_epochs):\n",
        "    print(\"Epoch {}\".format(n))\n",
        "    # Training\n",
        "    train_loss = 0.\n",
        "    train_pred_list = []\n",
        "    train_true_list = []\n",
        "    model.train()  # Switch to the training mode\n",
        "\n",
        "    # For each batch\n",
        "    for batch in dl_train:\n",
        "        optimizer.zero_grad()              # Initialize gradient information\n",
        "        X, y = batch\n",
        "        out = model(X.to(device))          # Call `forward()` function of the model\n",
        "        loss = loss_fn(out, y.to(device))  # Calculate loss \n",
        "        loss.backward()                    # Backpropagate the loss value\n",
        "        optimizer.step()                   # Update the parameters\n",
        "        \n",
        "        # import sys; sys.exit()\n",
        "        import pdb; pdb.set_trace()\n",
        "\n",
        "        train_loss += loss.data.item() * batch_size\n",
        "        train_pred_list += out.argmax(1).detach().cpu().tolist()\n",
        "        train_true_list += y.detach().cpu().tolist()\n",
        "\n",
        "    train_loss /= len(dl_train)\n",
        "    train_acc = accuracy_score(train_true_list, train_pred_list)\n",
        "    print(\"    Training loss: {:.4f}\\t  Training acc: {:.4f}\".format(train_loss, train_acc))\n",
        "\n",
        "    # Validation\n",
        "    valid_loss = 0.\n",
        "    valid_pred_list = []\n",
        "    valid_true_list = []\n",
        "\n",
        "    model.eval()  # Switch to the evaluation mode\n",
        "    for batch in dl_valid:\n",
        "        X, y = batch\n",
        "        out = model(X.to(device))\n",
        "        loss = loss_fn(out, y.to(device))\n",
        "        valid_loss += loss.data.item() * batch_size\n",
        "        valid_pred_list.append(out.argmax(1).detach().cpu())\n",
        "        valid_true_list.append(y.detach().cpu())\n",
        "\n",
        "    valid_loss /= len(dl_valid)\n",
        "    valid_acc = accuracy_score(valid_true_list, valid_pred_list)\n",
        "    print(\"  Validation loss: {:.4f}\\tValidation acc: {:.4f}\".format(valid_loss, valid_acc))\n",
        "    # Store train/validation loss, accuracy values\n",
        "    eval_list.append([n, train_loss, train_acc, valid_loss, valid_acc])\n",
        "\n",
        "eval_df = pd.DataFrame(eval_list, columns=[\"epoch\", \"train_loss\", \"train_acc\",\n",
        "                                           \"valid_loss\", \"valid_acc\"])\n",
        "\n",
        "# Test\n",
        "model.eval()\n",
        "pred_list = []\n",
        "true_list = []\n",
        "for batch in dl_test:\n",
        "    X, y = batch\n",
        "    out = model(X.to(device))\n",
        "    pred = out.argmax().item()\n",
        "    pred_list.append(pred)\n",
        "    true_list.append(y.item())\n",
        "y_pred = np.array(pred_list)\n",
        "y_true = np.array(true_list)\n",
        "\n",
        "test_accuracy = accuracy_score(y_true, y_pred)\n",
        "print(\"\\nTest accuracy: {:.4f}\".format(test_accuracy))\n",
        "\n",
        "eval_df[[\"train_loss\", \"valid_loss\"]].plot()\n",
        "eval_df[[\"train_acc\", \"valid_acc\"]].plot()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "PYDEV DEBUGGER WARNING:\n",
            "sys.settrace() should not be used when the debugger is being used.\n",
            "This may cause the debugger to stop working correctly.\n",
            "If this is needed, please check: \n",
            "http://pydev.blogspot.com/2007/06/why-cant-pydev-debugger-work-with.html\n",
            "to see how to restore the debug tracing back correctly.\n",
            "Call Location:\n",
            "  File \"/usr/lib/python3.7/bdb.py\", line 332, in set_trace\n",
            "    sys.settrace(self.trace_dispatch)\n",
            "\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 0\n",
            "> <ipython-input-20-f1cad334684c>(79)<module>()\n",
            "-> train_loss += loss.data.item() * batch_size\n",
            "*** AttributeError: 'list' object has no attribute 'shape'\n",
            "torch.Size([16, 64])\n",
            "*** NameError: name 'pred' is not defined\n",
            "torch.Size([16, 10])\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/nn/modules/loss.py:528: UserWarning: Using a target size (torch.Size([16])) that is different to the input size (torch.Size([16, 10])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n",
            "  return F.mse_loss(input, target, reduction=self.reduction)\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "*** RuntimeError: The size of tensor a (10) must match the size of tensor b (16) at non-singleton dimension 1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wEbHS-tt7lBE"
      },
      "source": [
        "## Checking variables"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EWEbWyMg6rTk"
      },
      "source": [
        "## Input data\n",
        "# X, y = batch\n",
        "X\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4e0KzOtA6-hV"
      },
      "source": [
        "X.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3inyg-Sw7uli"
      },
      "source": [
        "model(X.to(device))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lZlcZLF774U6"
      },
      "source": [
        "X[0:1]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HW8YIDKW781D",
        "outputId": "8e1b40a3-8e9c-4fae-87e7-83749e8d74ed"
      },
      "source": [
        "X[0:1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 0.,  0.,  0., 14., 14.,  1.,  0.,  0.,  0.,  0.,  6., 16., 12.,  0.,\n",
              "          0.,  0.,  0.,  0., 12., 16.,  2.,  0.,  0.,  0.,  0.,  0., 16., 16.,\n",
              "         16.,  9.,  0.,  0.,  0.,  1., 16., 15.,  8., 14.,  9.,  0.,  0.,  0.,\n",
              "         14., 12.,  0., 12., 13.,  0.,  0.,  0.,  6., 14.,  7., 16., 10.,  0.,\n",
              "          0.,  0.,  1., 13., 16., 13.,  1.,  0.]])"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZBzVekNy70-L",
        "outputId": "f8b02e0f-25a9-43c8-f985-2988a3f24a90"
      },
      "source": [
        "model(X[0:1].to(device))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ -4.5490,   4.0253,  -1.5301,  -5.7145,   3.7009, -12.5724,   6.5383,\n",
              "           3.3752,   2.5099,   1.3851]], device='cuda:0',\n",
              "       grad_fn=<AddmmBackward>)"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "364AheAS7Dt9",
        "outputId": "0c86d1df-19c8-49a6-a554-de847e594ec5"
      },
      "source": [
        "## True labels\n",
        "y"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([6, 8, 4, 0, 1, 6, 1, 1, 3, 1, 5, 4, 2, 4, 9, 7])"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fdrKv2D77EpY"
      },
      "source": [
        "y.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rBMUlkMa7F7I"
      },
      "source": [
        "out.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-fCBP4qq7JAT",
        "outputId": "c17a0f2a-7d7c-48cc-8360-8ade696a6587"
      },
      "source": [
        "## Loss func\n",
        "loss_fn(out, y.to(device))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(8.0736, device='cuda:0', grad_fn=<NllLossBackward>)"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aU0EdAu57UZw",
        "outputId": "b6803a16-9e82-4b3b-d523-aa802e9f2484"
      },
      "source": [
        "loss_fn(out[0:1], y[0:1].to(device))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(7.3873, device='cuda:0', grad_fn=<NllLossBackward>)"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MX9hqVU47XSF",
        "outputId": "4ea86941-23a9-439c-ce1d-4caa74a23712"
      },
      "source": [
        "for i in range(len(y)):\n",
        "  print(loss_fn(out[i:i+1], y[i:i+1].to(device)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(7.3873, device='cuda:0', grad_fn=<NllLossBackward>)\n",
            "tensor(8.7995, device='cuda:0', grad_fn=<NllLossBackward>)\n",
            "tensor(12.9916, device='cuda:0', grad_fn=<NllLossBackward>)\n",
            "tensor(10.6342, device='cuda:0', grad_fn=<NllLossBackward>)\n",
            "tensor(7.0666, device='cuda:0', grad_fn=<NllLossBackward>)\n",
            "tensor(7.0021, device='cuda:0', grad_fn=<NllLossBackward>)\n",
            "tensor(11.7180, device='cuda:0', grad_fn=<NllLossBackward>)\n",
            "tensor(6.3022, device='cuda:0', grad_fn=<NllLossBackward>)\n",
            "tensor(14.9491, device='cuda:0', grad_fn=<NllLossBackward>)\n",
            "tensor(9.1619, device='cuda:0', grad_fn=<NllLossBackward>)\n",
            "tensor(0.5710, device='cuda:0', grad_fn=<NllLossBackward>)\n",
            "tensor(7.8638, device='cuda:0', grad_fn=<NllLossBackward>)\n",
            "tensor(1.6965, device='cuda:0', grad_fn=<NllLossBackward>)\n",
            "tensor(10.4794, device='cuda:0', grad_fn=<NllLossBackward>)\n",
            "tensor(5.5548, device='cuda:0', grad_fn=<NllLossBackward>)\n",
            "tensor(6.9994, device='cuda:0', grad_fn=<NllLossBackward>)\n"
          ]
        }
      ]
    }
  ]
}