06_Prob&Solutions/0_Weight_Regularization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Weight Regularization\n",
    "\n",
    "torch.optim.SGD(params, lr=1, momentum=0, dampening=0, weight_decay=0, nesterov=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Settings\n",
    "### 1) Import required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.init as init\n",
    "import torchvision.datasets as dset\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.autograd import Variable\n",
    "from visdom import Visdom\n",
    "viz = Visdom()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2) Set hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 16\n",
    "learning_rate = 0.0002\n",
    "num_epoch = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Data\n",
    "\n",
    "### 1) Download Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mnist_train = dset.MNIST(\"./\", train=True, transform=transforms.ToTensor(), target_transform=None, download=True)\n",
    "mnist_test = dset.MNIST(\"./\", train=False, transform=transforms.ToTensor(), target_transform=None, download=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2) Check Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 28, 28]) 60000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(torch.Size([1, 28, 28]), 10000)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(mnist_train.__getitem__(0)[0].size(), mnist_train.__len__())\n",
    "mnist_test.__getitem__(0)[0].size(), mnist_test.__len__()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3) Set DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(mnist_train,batch_size=batch_size, shuffle=True,num_workers=2,drop_last=True)\n",
    "test_loader = torch.utils.data.DataLoader(mnist_test,batch_size=batch_size, shuffle=False,num_workers=2,drop_last=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Model & Optimizer\n",
    "\n",
    "### 1) CNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class CNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(CNN,self).__init__()\n",
    "        self.layer = nn.Sequential(\n",
    "            nn.Conv2d(1,16,3,padding=1),  # 28\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(16,32,3,padding=1), # 28\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(2,2), # 14\n",
    "            nn.Conv2d(32,64,3,padding=1), #14\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(2,2) # 7\n",
    "        )\n",
    "        self.fc_layer = nn.Sequential(\n",
    "            nn.Linear(64*7*7,100),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(100,10)\n",
    "        )       \n",
    "        \n",
    "    def forward(self,x):\n",
    "        out = self.layer(x)\n",
    "        #print(out.size())\n",
    "        out = out.view(batch_size,-1)\n",
    "        out = self.fc_layer(out)\n",
    "\n",
    "        return out\n",
    "\n",
    "model = CNN().cuda()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2) Loss func & Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss_func = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Train "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 2.3163\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.3087\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.2993\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 2.3043\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(num_epoch):\n",
    "    for j,[image,label] in enumerate(train_loader):\n",
    "        x = Variable(image).cuda()\n",
    "        y_= Variable(label).cuda()\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        output = model.forward(x)\n",
    "        loss = loss_func(output,y_)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if j % 1000 == 0:\n",
    "            print(loss)          "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#param_list = list(model.parameters())\n",
    "#print(param_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of Test Data: Variable containing:\n",
      " 10.0900\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "correct = 0\n",
    "total = 0\n",
    "\n",
    "for image,label in test_loader:\n",
    "    x = Variable(image,volatile=True).cuda()\n",
    "    y_= Variable(label).cuda()\n",
    "\n",
    "    output = model.forward(x)\n",
    "    _,output_index = torch.max(output,1)\n",
    "        \n",
    "    total += label.size(0)\n",
    "    correct += (output_index == y_).sum().float()\n",
    "    \n",
    "print(\"Accuracy of Test Data: {}\".format(100*correct/total))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}