03_CNN_Basics/0_MNIST/3_CNN_clean.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convolutional Neural Network\n",
    "- MNIST data\n",
    "- 3 convolutional layers\n",
    "- 2 fully connected layers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Settings\n",
    "### 1) Import required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.init as init\n",
    "import torchvision.datasets as dset\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.autograd import Variable\n",
    "from visdom import Visdom\n",
    "viz = Visdom()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2) Set hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 16\n",
    "learning_rate = 0.0002\n",
    "num_epoch = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Data\n",
    "\n",
    "### 1) Download Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mnist_train = dset.MNIST(\"./\", train=True, transform=transforms.ToTensor(), target_transform=None, download=True)\n",
    "mnist_test = dset.MNIST(\"./\", train=False, transform=transforms.ToTensor(), target_transform=None, download=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2) Check Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 28, 28]) 60000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(torch.Size([1, 28, 28]), 10000)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(mnist_train.__getitem__(0)[0].size(), mnist_train.__len__())\n",
    "mnist_test.__getitem__(0)[0].size(), mnist_test.__len__()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3) Set DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(mnist_train,batch_size=batch_size, shuffle=True,num_workers=2,drop_last=True)\n",
    "test_loader = torch.utils.data.DataLoader(mnist_test,batch_size=batch_size, shuffle=False,num_workers=2,drop_last=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Model & Optimizer\n",
    "\n",
    "### 1) CNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class CNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(CNN,self).__init__()\n",
    "        self.layer = nn.Sequential(\n",
    "            nn.Conv2d(1,16,5),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(16,32,5),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(2,2),\n",
    "            nn.Conv2d(32,64,5),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(2,2)\n",
    "        )\n",
    "        self.fc_layer = nn.Sequential(\n",
    "            nn.Linear(64*3*3,100),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(100,10)\n",
    "        )       \n",
    "        \n",
    "    def forward(self,x):\n",
    "        out = self.layer(x)\n",
    "        out = out.view(batch_size,-1)\n",
    "        out = self.fc_layer(out)\n",
    "\n",
    "        return out\n",
    "\n",
    "model = CNN().cuda()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2) Loss func & Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss_func = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Train "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 2.2991\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.3071\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.2968\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.2922\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.2815\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.3111\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.2976\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.2919\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.3026\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.2916\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.2892\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.2883\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.2887\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.2822\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.2942\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.2602\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.2668\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.2629\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.2458\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.2079\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.1923\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.1865\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.0999\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.0429\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.0384\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Process Process-13:\n",
      "Process Process-14:\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/gunho/anaconda3/envs/pytorch/lib/python3.6/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/gunho/anaconda3/envs/pytorch/lib/python3.6/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/gunho/anaconda3/envs/pytorch/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/gunho/anaconda3/envs/pytorch/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 34, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/home/gunho/anaconda3/envs/pytorch/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/gunho/anaconda3/envs/pytorch/lib/python3.6/multiprocessing/queues.py\", line 341, in get\n",
      "    with self._rlock:\n",
      "KeyboardInterrupt\n",
      "  File \"/home/gunho/anaconda3/envs/pytorch/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 34, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/home/gunho/anaconda3/envs/pytorch/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/home/gunho/anaconda3/envs/pytorch/lib/python3.6/multiprocessing/queues.py\", line 342, in get\n",
      "    res = self._reader.recv_bytes()\n",
      "  File \"/home/gunho/anaconda3/envs/pytorch/lib/python3.6/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/home/gunho/anaconda3/envs/pytorch/lib/python3.6/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/home/gunho/anaconda3/envs/pytorch/lib/python3.6/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-5a8b747f40e9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.6/site-packages/torch/autograd/variable.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph, retain_variables)\u001b[0m\n\u001b[1;32m    154\u001b[0m                 \u001b[0mVariable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    155\u001b[0m         \"\"\"\n\u001b[0;32m--> 156\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_variables\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    157\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    158\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.6/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(variables, grad_variables, retain_graph, create_graph, retain_variables)\u001b[0m\n\u001b[1;32m     96\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[0;32m---> 98\u001b[0;31m         variables, grad_variables, retain_graph)\n\u001b[0m\u001b[1;32m     99\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for i in range(num_epoch):\n",
    "    for j,[image,label] in enumerate(train_loader):\n",
    "        x = Variable(image).cuda()\n",
    "        y_= Variable(label).cuda()\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        output = model.forward(x)\n",
    "        loss = loss_func(output,y_)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if j % 1000 == 0:\n",
    "            print(loss)          "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#param_list = list(model.parameters())\n",
    "#print(param_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of Test Data: Variable containing:\n",
      " 67.8300\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "correct = 0\n",
    "total = 0\n",
    "\n",
    "for image,label in test_loader:\n",
    "    x = Variable(image,volatile=True).cuda()\n",
    "    y_= Variable(label).cuda()\n",
    "\n",
    "    output = model.forward(x)\n",
    "    _,output_index = torch.max(output,1)\n",
    "        \n",
    "    total += label.size(0)\n",
    "    correct += (output_index == y_).sum().float()\n",
    "    \n",
    "print(\"Accuracy of Test Data: {}\".format(100*correct/total))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}