demo_notebooks/MNIST_Example_Detail.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Torch Version: 1.4.0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import torch\n",
    "import numpy as np\n",
    "print(\"Torch Version:\",torch.__version__)\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "from torchvision import datasets,transforms\n",
    "import torch.optim as optim\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from classification_report import Report, Config, HyperParameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "h_p = HyperParameters(training_config = Config(batch_size=64 , lr=0.01 , no_epochs=10),\n",
    "                      inference_config = Config(batch_size=64),\n",
    "                      model_config = Config(no_of_channels=10 ,classes = [\"zero\",\"one\",\"two\",\"three\",\"four\",\"five\",\"six\",\"seven\",\"eight\",\"nine\"])\n",
    "                     )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training batch size 64\n",
      "Number of classes 10\n"
     ]
    }
   ],
   "source": [
    "print(f\"Training batch size {h_p.training_config.batch_size}\")\n",
    "print(f\"Number of classes {h_p.model_config.classes.__len__()}\") "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Datasets and Dataloaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "transform=transforms.Compose([transforms.ToTensor(),\n",
    "                              transforms.Normalize((0.1307,), (0.3081,))]\n",
    "                            )\n",
    "\n",
    "trainset=datasets.MNIST('~/.pytorch/MNIST_data/',\n",
    "                        train=True,\n",
    "                        transform=transform,\n",
    "                        download=True)\n",
    "\n",
    "validset=datasets.MNIST('~/.pytorch/MNIST_data/',\n",
    "                        train=False,\n",
    "                        transform=transform,\n",
    "                        download=True)\n",
    "\n",
    "train_loader=torch.utils.data.DataLoader(trainset,\n",
    "                                         batch_size=h_p.training_config.batch_size,\n",
    "                                         shuffle=True,\n",
    "                                         num_workers=0)\n",
    "\n",
    "valid_loader=torch.utils.data.DataLoader(validset,\n",
    "                                         batch_size=h_p.inference_config.batch_size,\n",
    "                                         shuffle=True,\n",
    "                                         num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Defination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net(\n",
      "  (conv1): Conv2d(1, 10, kernel_size=(3, 3), stride=(1, 1))\n",
      "  (fc2): Linear(in_features=640, out_features=10, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, \n",
    "                               h_p.model_config.no_of_channels,\n",
    "                               3)  #[in_channel,out_channel,filter_size,stride=1]\n",
    "        \n",
    "        self.fc2 = nn.Linear(8*8*h_p.model_config.no_of_channels,\n",
    "                             h_p.model_config.classes.__len__())\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.max_pool2d(x,3)\n",
    "        x = x.view(-1, 8*8*10)\n",
    "        x = self.fc2(x)\n",
    "        return x # return raw logits.\n",
    "      \n",
    "# create a complete CNN\n",
    "model = Net()\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total_params 6510\n",
      "Trainable_params 6510\n"
     ]
    }
   ],
   "source": [
    "pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
    "print(\"Total_params\",pytorch_total_params)\n",
    "pytorch_total_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "print(\"Trainable_params\",pytorch_total_params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Report Intialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The first input batch shape torch.Size([64, 1, 28, 28])\n"
     ]
    }
   ],
   "source": [
    "data , label = next(iter(train_loader))\n",
    "print(f\"The first input batch shape {data.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<src.report.Report at 0x7f0a4cbc6208>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report = Report(classes=h_p.model_config.classes) # intialize the class\n",
    "report.plot_model(model, data) # plot model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Criterion and Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=h_p.training_config.lr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training Method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_an_epoch():\n",
    "    model.train()\n",
    "    train_loss = 0.0\n",
    "    for data, target in train_loader:\n",
    "        # clear the gradients of all optimized variables\n",
    "        optimizer.zero_grad()\n",
    "        # forward pass: compute predicted outputs by passing inputs to the model\n",
    "        output = model(data)\n",
    "        # calculate the batch loss\n",
    "        loss = criterion(output, target)\n",
    "        # backward pass: compute gradient of the loss with respect to model parameters\n",
    "        loss.backward()\n",
    "        # perform a single optimization step (parameter update)\n",
    "        optimizer.step()\n",
    "        # update training loss\n",
    "        train_loss += loss.item()*data.size(0)\n",
    "        \n",
    "        # converting raw _logits to softmax output\n",
    "        output = F.softmax(output,dim=-1)\n",
    "        \n",
    "        # write training batch information into report\n",
    "        report.write_a_batch(loss=loss,\n",
    "                             batch_size=data.size(0),\n",
    "                             actual=target,\n",
    "                             prediction=output,\n",
    "                             train=True)\n",
    "        \n",
    "        #plot histogram of model weight, bias and gradients 2 times in an epoch\n",
    "        report.plot_model_data_grad(at_which_iter = len(train_loader)/2)\n",
    "        \n",
    "    return train_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Validation Method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def valid_an_epoch():\n",
    "        \n",
    "    valid_loss = 0.0\n",
    "    model.eval()\n",
    "    for data, target in valid_loader:\n",
    "        # forward pass: compute predicted outputs by passing inputs to the model\n",
    "        output = model(data)\n",
    "        # calculate the batch loss\n",
    "        loss = criterion(output, target)\n",
    "        # update average validation loss \n",
    "        valid_loss += loss.item()*data.size(0)\n",
    "        \n",
    "        \n",
    "        # converting raw _logits to softmax output\n",
    "        output = F.softmax(output,dim=-1)\n",
    "        \n",
    "        # write validation batch information into report\n",
    "        report.write_a_batch(loss=loss,\n",
    "                             batch_size=data.size(0),\n",
    "                             actual=target,\n",
    "                             prediction=output,\n",
    "                             train=False)\n",
    "    return valid_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training and Plotting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 \tTraining Loss: 0.439560 \tValidation Loss: 0.256226\n",
      "Epoch: 2 \tTraining Loss: 0.238404 \tValidation Loss: 0.185166\n",
      "Epoch: 3 \tTraining Loss: 0.185101 \tValidation Loss: 0.149065\n",
      "Epoch: 4 \tTraining Loss: 0.155258 \tValidation Loss: 0.126869\n",
      "Epoch: 5 \tTraining Loss: 0.137192 \tValidation Loss: 0.131225\n",
      "Epoch: 6 \tTraining Loss: 0.124772 \tValidation Loss: 0.105704\n",
      "Epoch: 7 \tTraining Loss: 0.114972 \tValidation Loss: 0.101094\n",
      "Epoch: 8 \tTraining Loss: 0.107882 \tValidation Loss: 0.097820\n",
      "Epoch: 9 \tTraining Loss: 0.102315 \tValidation Loss: 0.092370\n",
      "Epoch: 10 \tTraining Loss: 0.097554 \tValidation Loss: 0.090351\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(1, h_p.training_config.no_epochs+1):\n",
    "\n",
    "    train_loss = train_an_epoch()\n",
    "    valid_loss = valid_an_epoch()\n",
    "    \n",
    "    train_loss = train_loss/len(train_loader.sampler)\n",
    "    valid_loss = valid_loss/len(valid_loader.sampler)\n",
    "        \n",
    "    # print training/validation statistics \n",
    "    print(f'Epoch: {epoch} \\tTraining Loss: {train_loss:.6f} \\tValidation Loss: {valid_loss:.6f}')\n",
    "    \n",
    "    \n",
    "    \n",
    "    report.plot_an_epoch(detail=True)\\\n",
    "    .plot_loss()\\\n",
    "    .plot_precision_recall()\\\n",
    "    .plot_mcc()\\\n",
    "    .plot_confusion_matrix(at_which_epoch=h_p.training_config.no_epochs/2)\\\n",
    "    .plot_missclassification_count(at_which_epoch=h_p.training_config.no_epochs/2)\\\n",
    "    .plot_pred_prob(at_which_epoch=h_p.training_config.no_epochs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot Hyperparams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<src.report.Report at 0x7f0a4cbc6208>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report.plot_hparams(h_p) # record hyper params once training is completed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#    Last Epoch Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'train': 0.09755439987679322, 'valid': 0.09035115723609924}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report.loss_count # final loss for 10th epoch "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'train': {'actual': array([2, 2, 3, ..., 3, 5, 8]),\n",
       "  'pred': array([[2.48052995e-04, 2.56126282e-06, 9.99349177e-01, ...,\n",
       "          1.75650166e-06, 7.01895033e-07, 1.77736339e-07],\n",
       "         [9.09831726e-07, 4.25156759e-06, 9.97760773e-01, ...,\n",
       "          2.16383650e-03, 3.05638241e-05, 1.48316531e-06],\n",
       "         [1.25071892e-05, 6.65944419e-04, 1.18769694e-03, ...,\n",
       "          8.64696403e-07, 2.03786956e-04, 1.00202044e-04],\n",
       "         ...,\n",
       "         [6.38893098e-08, 7.92739058e-07, 1.20110897e-04, ...,\n",
       "          3.35105483e-07, 2.92154786e-04, 2.76299164e-04],\n",
       "         [5.92773741e-09, 1.71622860e-09, 3.84681016e-06, ...,\n",
       "          3.61187311e-08, 1.76191481e-03, 1.02757906e-04],\n",
       "         [4.52033646e-06, 1.93666483e-06, 5.22399414e-03, ...,\n",
       "          2.39940956e-08, 3.00623268e-01, 7.03178375e-05]], dtype=float32)},\n",
       " 'valid': {'actual': array([5, 7, 2, ..., 9, 7, 2]),\n",
       "  'pred': array([[1.3033445e-05, 1.3001286e-04, 3.8345825e-04, ..., 1.7812144e-07,\n",
       "          7.1492861e-04, 8.7654627e-07],\n",
       "         [8.6563177e-02, 8.2975626e-09, 6.2081340e-04, ..., 6.3928354e-01,\n",
       "          5.9633305e-05, 2.7313110e-01],\n",
       "         [9.4460900e-04, 2.3505608e-04, 1.7317668e-01, ..., 1.5286578e-01,\n",
       "          4.2600259e-01, 1.3843805e-01],\n",
       "         ...,\n",
       "         [8.3830128e-06, 2.4636527e-11, 3.9478678e-06, ..., 7.5490918e-04,\n",
       "          1.7611294e-04, 9.9803096e-01],\n",
       "         [8.9650812e-06, 6.4869447e-08, 3.4324595e-04, ..., 9.7839659e-01,\n",
       "          1.9081427e-04, 2.1030616e-02],\n",
       "         [4.2341799e-07, 7.6391705e-05, 9.9970990e-01, ..., 8.1078997e-06,\n",
       "          1.7953636e-04, 1.6452940e-07]], dtype=float32)}}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report.act_pred_dict # actual and prediction for 10th epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report.counter # number of epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'train': 9380, 'valid': 1570})"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report.iter_count # train_loader and valid_loader size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'train': 60000, 'valid': 10000})"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report.data_count # data point count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'train': 0.9679713295792429, 'valid': 0.9693575850914065})"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report.mcc # Mathews correlation coefficient for 10th epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "report.close() # close the writer object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}