exercise_850.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Image Classification using a Multi-Layer Neural Network\n",
    "\n",
    "Next, we measure the performance of a multi-layer neural network on the MNIST dataset to set a baseline.\n",
    "\n",
    "The layers of the same are wrapped in an **nn.Sequential** object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train - Epoch 1, Batch: 0, Loss: 2.297173\n",
      "Train - Epoch 1, Batch: 10, Loss: 1.088993\n",
      "Train - Epoch 1, Batch: 20, Loss: 0.574045\n",
      "Train - Epoch 1, Batch: 30, Loss: 0.421659\n",
      "Train - Epoch 1, Batch: 40, Loss: 0.409437\n",
      "Train - Epoch 1, Batch: 50, Loss: 0.323081\n",
      "Train - Epoch 1, Batch: 60, Loss: 0.363600\n",
      "Train - Epoch 1, Batch: 70, Loss: 0.311784\n",
      "Train - Epoch 1, Batch: 80, Loss: 0.319516\n",
      "Train - Epoch 1, Batch: 90, Loss: 0.283992\n",
      "Train - Epoch 1, Batch: 100, Loss: 0.279556\n",
      "Train - Epoch 1, Batch: 110, Loss: 0.315841\n",
      "Train - Epoch 1, Batch: 120, Loss: 0.324343\n",
      "Train - Epoch 1, Batch: 130, Loss: 0.290451\n",
      "Train - Epoch 1, Batch: 140, Loss: 0.356294\n",
      "Train - Epoch 1, Batch: 150, Loss: 0.243043\n",
      "Train - Epoch 1, Batch: 160, Loss: 0.274326\n",
      "Train - Epoch 1, Batch: 170, Loss: 0.281660\n",
      "Train - Epoch 1, Batch: 180, Loss: 0.270453\n",
      "Train - Epoch 1, Batch: 190, Loss: 0.353078\n",
      "Train - Epoch 1, Batch: 200, Loss: 0.255094\n",
      "Train - Epoch 1, Batch: 210, Loss: 0.197983\n",
      "Train - Epoch 1, Batch: 220, Loss: 0.177464\n",
      "Train - Epoch 1, Batch: 230, Loss: 0.201570\n",
      "Test Avg. Loss: 0.000210, Accuracy: 0.937800\n",
      "Train - Epoch 2, Batch: 0, Loss: 0.212923\n",
      "Train - Epoch 2, Batch: 10, Loss: 0.221799\n",
      "Train - Epoch 2, Batch: 20, Loss: 0.144665\n",
      "Train - Epoch 2, Batch: 30, Loss: 0.221862\n",
      "Train - Epoch 2, Batch: 40, Loss: 0.167832\n",
      "Train - Epoch 2, Batch: 50, Loss: 0.134292\n",
      "Train - Epoch 2, Batch: 60, Loss: 0.125244\n",
      "Train - Epoch 2, Batch: 70, Loss: 0.244757\n",
      "Train - Epoch 2, Batch: 80, Loss: 0.151447\n",
      "Train - Epoch 2, Batch: 90, Loss: 0.144052\n",
      "Train - Epoch 2, Batch: 100, Loss: 0.193691\n",
      "Train - Epoch 2, Batch: 110, Loss: 0.160129\n",
      "Train - Epoch 2, Batch: 120, Loss: 0.244493\n",
      "Train - Epoch 2, Batch: 130, Loss: 0.139838\n",
      "Train - Epoch 2, Batch: 140, Loss: 0.223725\n",
      "Train - Epoch 2, Batch: 150, Loss: 0.193246\n",
      "Train - Epoch 2, Batch: 160, Loss: 0.116174\n",
      "Train - Epoch 2, Batch: 170, Loss: 0.144729\n",
      "Train - Epoch 2, Batch: 180, Loss: 0.124929\n",
      "Train - Epoch 2, Batch: 190, Loss: 0.164392\n",
      "Train - Epoch 2, Batch: 200, Loss: 0.171080\n",
      "Train - Epoch 2, Batch: 210, Loss: 0.141962\n",
      "Train - Epoch 2, Batch: 220, Loss: 0.186430\n",
      "Train - Epoch 2, Batch: 230, Loss: 0.130967\n",
      "Test Avg. Loss: 0.000142, Accuracy: 0.957700\n",
      "Train - Epoch 3, Batch: 0, Loss: 0.128827\n",
      "Train - Epoch 3, Batch: 10, Loss: 0.160667\n",
      "Train - Epoch 3, Batch: 20, Loss: 0.151637\n",
      "Train - Epoch 3, Batch: 30, Loss: 0.144104\n",
      "Train - Epoch 3, Batch: 40, Loss: 0.135468\n",
      "Train - Epoch 3, Batch: 50, Loss: 0.150068\n",
      "Train - Epoch 3, Batch: 60, Loss: 0.130085\n",
      "Train - Epoch 3, Batch: 70, Loss: 0.124879\n",
      "Train - Epoch 3, Batch: 80, Loss: 0.158337\n",
      "Train - Epoch 3, Batch: 90, Loss: 0.091751\n",
      "Train - Epoch 3, Batch: 100, Loss: 0.134458\n",
      "Train - Epoch 3, Batch: 110, Loss: 0.139275\n",
      "Train - Epoch 3, Batch: 120, Loss: 0.086146\n",
      "Train - Epoch 3, Batch: 130, Loss: 0.101201\n",
      "Train - Epoch 3, Batch: 140, Loss: 0.142036\n",
      "Train - Epoch 3, Batch: 150, Loss: 0.113172\n",
      "Train - Epoch 3, Batch: 160, Loss: 0.095839\n",
      "Train - Epoch 3, Batch: 170, Loss: 0.119627\n",
      "Train - Epoch 3, Batch: 180, Loss: 0.084755\n",
      "Train - Epoch 3, Batch: 190, Loss: 0.085531\n",
      "Train - Epoch 3, Batch: 200, Loss: 0.123627\n",
      "Train - Epoch 3, Batch: 210, Loss: 0.127291\n",
      "Train - Epoch 3, Batch: 220, Loss: 0.064456\n",
      "Train - Epoch 3, Batch: 230, Loss: 0.079468\n",
      "Test Avg. Loss: 0.000111, Accuracy: 0.968100\n",
      "Train - Epoch 4, Batch: 0, Loss: 0.111215\n",
      "Train - Epoch 4, Batch: 10, Loss: 0.069494\n",
      "Train - Epoch 4, Batch: 20, Loss: 0.116470\n",
      "Train - Epoch 4, Batch: 30, Loss: 0.064025\n",
      "Train - Epoch 4, Batch: 40, Loss: 0.142354\n",
      "Train - Epoch 4, Batch: 50, Loss: 0.075807\n",
      "Train - Epoch 4, Batch: 60, Loss: 0.082231\n",
      "Train - Epoch 4, Batch: 70, Loss: 0.096365\n",
      "Train - Epoch 4, Batch: 80, Loss: 0.089357\n",
      "Train - Epoch 4, Batch: 90, Loss: 0.065143\n",
      "Train - Epoch 4, Batch: 100, Loss: 0.082779\n",
      "Train - Epoch 4, Batch: 110, Loss: 0.169316\n",
      "Train - Epoch 4, Batch: 120, Loss: 0.127078\n",
      "Train - Epoch 4, Batch: 130, Loss: 0.082476\n",
      "Train - Epoch 4, Batch: 140, Loss: 0.109661\n",
      "Train - Epoch 4, Batch: 150, Loss: 0.065477\n",
      "Train - Epoch 4, Batch: 160, Loss: 0.125068\n",
      "Train - Epoch 4, Batch: 170, Loss: 0.086409\n",
      "Train - Epoch 4, Batch: 180, Loss: 0.066471\n",
      "Train - Epoch 4, Batch: 190, Loss: 0.084903\n",
      "Train - Epoch 4, Batch: 200, Loss: 0.080134\n",
      "Train - Epoch 4, Batch: 210, Loss: 0.092685\n",
      "Train - Epoch 4, Batch: 220, Loss: 0.095929\n",
      "Train - Epoch 4, Batch: 230, Loss: 0.056947\n",
      "Test Avg. Loss: 0.000095, Accuracy: 0.972700\n"
     ]
    }
   ],
   "source": [
    "# Code adapted from https://github.com/activatedgeek/LeNet-5/\n",
    "\n",
    "from collections import OrderedDict\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "from torchvision.datasets.mnist import MNIST\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "class MultilayerClassifier(nn.Module):\n",
    "    \"\"\"\n",
    "    Input - 1x32x32\n",
    "    Output - 10 (Output)\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        super(MultilayerClassifier, self).__init__()\n",
    "\n",
    "        self.fc = nn.Sequential(OrderedDict([\n",
    "            ('linear_combination', nn.Linear(32 * 32, 32 * 4)),\n",
    "            ('relu1', nn.ReLU()),\n",
    "            ('fc2', nn.Linear(32 * 4, 10)),\n",
    "        ]))\n",
    "\n",
    "    def forward(self, img):\n",
    "        output = img.view(-1, 32 * 32)\n",
    "        output = self.fc(output)\n",
    "        return output\n",
    "    \n",
    "\n",
    "data_train = MNIST('data/mnist',\n",
    "                   download=True,\n",
    "                   transform=transforms.Compose([\n",
    "                       transforms.Resize((32, 32)),\n",
    "                       transforms.ToTensor()]))\n",
    "data_test = MNIST('data/mnist',\n",
    "                  train=False,\n",
    "                  download=True,\n",
    "                  transform=transforms.Compose([\n",
    "                      transforms.Resize((32, 32)),\n",
    "                      transforms.ToTensor()]))\n",
    "data_train_loader = DataLoader(data_train, batch_size=256, shuffle=True, num_workers=8)\n",
    "data_test_loader = DataLoader(data_test, batch_size=1024, num_workers=8)\n",
    "\n",
    "net = MultilayerClassifier()\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(net.parameters(), lr=2e-3)\n",
    "\n",
    "def train(epoch):\n",
    "    net.train()\n",
    "    loss_list, batch_list = [], []\n",
    "    for i,(images, labels) in enumerate(data_train_loader):\n",
    "        images, labels = Variable(images), Variable(labels)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        output = net(images)\n",
    "\n",
    "        loss = criterion(output, labels)\n",
    "\n",
    "        loss_list.append(loss.data.item())\n",
    "        batch_list.append(i+1)\n",
    "\n",
    "        if i % 10 == 0:\n",
    "            print('Train - Epoch %d, Batch: %d, Loss: %f' % (epoch, i, loss.data.item()))\n",
    "\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "\n",
    "def test():\n",
    "    net.eval()\n",
    "    total_correct = 0\n",
    "    avg_loss = 0.0\n",
    "    for i, (images, labels) in enumerate(data_test_loader):\n",
    "        images, labels = Variable(images), Variable(labels)\n",
    "        output = net(images)\n",
    "        avg_loss += criterion(output, labels).sum()\n",
    "        pred = output.data.max(1)[1]\n",
    "        total_correct += pred.eq(labels.data.view_as(pred)).sum()\n",
    "\n",
    "    avg_loss /= len(data_test)\n",
    "    print('Test Avg. Loss: %f, Accuracy: %f' % (avg_loss.data.item(), float(total_correct) / len(data_test)))\n",
    "\n",
    "\n",
    "def train_and_test(epoch):\n",
    "    train(epoch)\n",
    "    test()\n",
    "\n",
    "\n",
    "def main():\n",
    "    for e in range(1, 5):  # Change 5 to 16 for better performance\n",
    "        train_and_test(e)\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    main()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "The accuracy we get with just two layers is around 97%, which is a huge improvement over single-layer models on the MNIST dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}