A2_PyTorch/PyTorch_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": [
     "pdf-ignore"
    ]
   },
   "outputs": [],
   "source": [
    "# Preparation\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data import sampler\n",
    "\n",
    "import torchvision.datasets as dset\n",
    "import torchvision.transforms as T\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.4.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": [
     "pdf-ignore"
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "NUM_TRAIN = 49000\n",
    "\n",
    "# The torchvision.transforms package provides tools for preprocessing data\n",
    "# and for performing data augmentation; here we set up a transform to\n",
    "# preprocess the data by subtracting the mean RGB value and dividing by the\n",
    "# standard deviation of each RGB value; we've hardcoded the mean and std.\n",
    "transform = T.Compose([\n",
    "                T.ToTensor(),\n",
    "                T.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))\n",
    "            ])\n",
    "\n",
    "# We set up a Dataset object for each split (train / val / test); Datasets load\n",
    "# training examples one at a time, so we wrap each Dataset in a DataLoader which\n",
    "# iterates through the Dataset and forms minibatches. We divide the CIFAR-10\n",
    "# training set into train and val sets by passing a Sampler object to the\n",
    "# DataLoader telling how it should sample from the underlying Dataset.\n",
    "cifar10_train = dset.CIFAR10('./datasets', train=True, download=True,\n",
    "                             transform=transform)\n",
    "loader_train = DataLoader(cifar10_train, batch_size=64, \n",
    "                          sampler=sampler.SubsetRandomSampler(range(NUM_TRAIN)))\n",
    "\n",
    "cifar10_val = dset.CIFAR10('./datasets', train=True, download=True,\n",
    "                           transform=transform)\n",
    "loader_val = DataLoader(cifar10_val, batch_size=64, \n",
    "                        sampler=sampler.SubsetRandomSampler(range(NUM_TRAIN, 50000)))\n",
    "\n",
    "cifar10_test = dset.CIFAR10('./datasets', train=False, download=True, \n",
    "                            transform=transform)\n",
    "loader_test = DataLoader(cifar10_test, batch_size=64)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": [
     "pdf-ignore"
    ]
   },
   "source": [
    "You have an option to **use GPU by setting the flag to True below**. It is not necessary to use GPU for this assignment. Note that if your computer does not have CUDA enabled, `torch.cuda.is_available()` will return False and this notebook will fallback to CPU mode.\n",
    "\n",
    "The global variables `dtype` and `device` will control the data types throughout this assignment. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": [
     "pdf-ignore-input"
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using device: cpu\n"
     ]
    }
   ],
   "source": [
    "USE_GPU = True\n",
    "\n",
    "dtype = torch.float32 # we will be using float throughout this tutorial\n",
    "\n",
    "if USE_GPU and torch.cuda.is_available():\n",
    "    device = torch.device('cuda')\n",
    "else:\n",
    "    device = torch.device('cpu')\n",
    "\n",
    "# Constant to control how frequently we print train loss\n",
    "print_every = 100\n",
    "\n",
    "print('using device:', device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": [
     "pdf-ignore-input"
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before flattening:  tensor([[[[ 0,  1],\n",
      "          [ 2,  3],\n",
      "          [ 4,  5]]],\n",
      "\n",
      "\n",
      "        [[[ 6,  7],\n",
      "          [ 8,  9],\n",
      "          [10, 11]]]])\n",
      "After flattening:  tensor([[ 0,  1,  2,  3,  4,  5],\n",
      "        [ 6,  7,  8,  9, 10, 11]])\n"
     ]
    }
   ],
   "source": [
    "def flatten(x):\n",
    "    N = x.shape[0] # read in N, C, H, W\n",
    "    return x.view(N, -1)  # \"flatten\" the C * H * W values into a single vector per image\n",
    "\n",
    "def test_flatten():\n",
    "    x = torch.arange(12).view(2, 1, 3, 2)\n",
    "    print('Before flattening: ', x)\n",
    "    print('After flattening: ', flatten(x))\n",
    "\n",
    "test_flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "tags": [
     "pdf-ignore-input"
    ]
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-1.9643, -1.2111,  0.3739, -0.2833,  0.2331],\n",
       "        [-0.6898, -0.9484, -1.3388, -0.4535,  0.5783],\n",
       "        [-0.4867, -0.5598, -0.5088, -0.5024,  1.3244]], requires_grad=True)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def random_weight(shape):\n",
    "    \"\"\"\n",
    "    Create random Tensors for weights; setting requires_grad=True means that we\n",
    "    want to compute gradients for these Tensors during the backward pass.\n",
    "    We use Kaiming normalization: sqrt(2 / fan_in)\n",
    "    \"\"\"\n",
    "    if len(shape) == 2:  # FC weight\n",
    "        fan_in = shape[0]\n",
    "    else:\n",
    "        fan_in = np.prod(shape[1:]) # conv weight [out_channel, in_channel, kH, kW]\n",
    "    # randn is standard normal distribution generator. \n",
    "    w = torch.randn(shape, device=device, dtype=dtype) * np.sqrt(2. / fan_in)\n",
    "    w.requires_grad = True\n",
    "    return w\n",
    "\n",
    "def zero_weight(shape):\n",
    "    return torch.zeros(shape, device=device, dtype=dtype, requires_grad=True)\n",
    "\n",
    "# create a weight of shape [3 x 5]\n",
    "# you should see the type `torch.cuda.FloatTensor` if you use GPU. \n",
    "# Otherwise it should be `torch.FloatTensor`\n",
    "random_weight((3, 5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Module API: Check Accuracy\n",
    "Given the validation or test set, we can check the classification accuracy of a neural network. \n",
    "\n",
    "This version is slightly different from the one in part II. You don't manually pass in the parameters anymore."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_accuracy_part34(loader, model):\n",
    "    if loader.dataset.train:\n",
    "        print('Checking accuracy on validation set')\n",
    "    else:\n",
    "        print('Checking accuracy on test set')   \n",
    "    num_correct = 0\n",
    "    num_samples = 0\n",
    "    model.eval()  # set model to evaluation mode\n",
    "    with torch.no_grad():\n",
    "        for x, y in loader:\n",
    "            x = x.to(device=device, dtype=dtype)  # move to device, e.g. GPU\n",
    "            y = y.to(device=device, dtype=torch.long)\n",
    "            scores = model(x)\n",
    "            _, preds = scores.max(1)\n",
    "            num_correct += (preds == y).sum()\n",
    "            num_samples += preds.size(0)\n",
    "        acc = float(num_correct) / num_samples\n",
    "        print('Got %d / %d correct (%.2f)' % (num_correct, num_samples, 100 * acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Module API: Training Loop\n",
    "We also use a slightly different training loop. Rather than updating the values of the weights ourselves, we use an Optimizer object from the `torch.optim` package, which abstract the notion of an optimization algorithm and provides implementations of most of the algorithms commonly used to optimize neural networks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_part34(model, optimizer, epochs=1):\n",
    "    \"\"\"\n",
    "    Train a model on CIFAR-10 using the PyTorch Module API.\n",
    "    \n",
    "    Inputs:\n",
    "    - model: A PyTorch Module giving the model to train.\n",
    "    - optimizer: An Optimizer object we will use to train the model\n",
    "    - epochs: (Optional) A Python integer giving the number of epochs to train for\n",
    "    \n",
    "    Returns: Nothing, but prints model accuracies during training.\n",
    "    \"\"\"\n",
    "    model = model.to(device=device)  # move the model parameters to CPU/GPU\n",
    "    for e in range(epochs):\n",
    "        for t, (x, y) in enumerate(loader_train):\n",
    "            model.train()  # put model to training mode\n",
    "            x = x.to(device=device, dtype=dtype)  # move to device, e.g. GPU\n",
    "            y = y.to(device=device, dtype=torch.long)\n",
    "\n",
    "            scores = model(x)\n",
    "            loss = F.cross_entropy(scores, y)\n",
    "\n",
    "            # Zero out all of the gradients for the variables which the optimizer\n",
    "            # will update.\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            # This is the backwards pass: compute the gradient of the loss with\n",
    "            # respect to each  parameter of the model.\n",
    "            loss.backward()\n",
    "\n",
    "            # Actually update the parameters of the model using the gradients\n",
    "            # computed by the backwards pass.\n",
    "            optimizer.step()\n",
    "\n",
    "            if t % print_every == 0:\n",
    "                print('Iteration %d, loss = %.4f' % (t, loss.item()))\n",
    "                check_accuracy_part34(loader_val, model)\n",
    "                print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Module API: Train a Three-Layer ConvNet\n",
    "You should now use the Module API to train a three-layer ConvNet on CIFAR. This should look very similar to training the two-layer network! You don't need to tune any hyperparameters, but you should achieve above above 45% after training for one epoch.\n",
    "\n",
    "You should train the model using stochastic gradient descent without momentum."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part V. CIFAR-10 open-ended challenge\n",
    "\n",
    "In this section, you can experiment with whatever ConvNet architecture you'd like on CIFAR-10. \n",
    "\n",
    "Now it's your job to experiment with architectures, hyperparameters, loss functions, and optimizers to train a model that achieves **at least 70%** accuracy on the CIFAR-10 **validation** set within 10 epochs. You can use the check_accuracy and train functions from above. You can use either `nn.Module` or `nn.Sequential` API. \n",
    "\n",
    "Describe what you did at the end of this notebook.\n",
    "\n",
    "Here are the official API documentation for each component. One note: what we call in the class \"spatial batch norm\" is called \"BatchNorm2D\" in PyTorch.\n",
    "\n",
    "* Layers in torch.nn package: http://pytorch.org/docs/stable/nn.html\n",
    "* Activations: http://pytorch.org/docs/stable/nn.html#non-linear-activations\n",
    "* Loss functions: http://pytorch.org/docs/stable/nn.html#loss-functions\n",
    "* Optimizers: http://pytorch.org/docs/stable/optim.html\n",
    "\n",
    "\n",
    "### Things you might try:\n",
    "- **Filter size**: Above we used 5x5; would smaller filters be more efficient?\n",
    "- **Number of filters**: Above we used 32 filters. Do more or fewer do better?\n",
    "- **Pooling vs Strided Convolution**: Do you use max pooling or just stride convolutions?\n",
    "- **Batch normalization**: Try adding spatial batch normalization after convolution layers and vanilla batch normalization after affine layers. Do your networks train faster?\n",
    "- **Network architecture**: The network above has two layers of trainable parameters. Can you do better with a deep network? Good architectures to try include:\n",
    "    - [conv-relu-pool]xN -> [affine]xM -> [softmax or SVM]\n",
    "    - [conv-relu-conv-relu-pool]xN -> [affine]xM -> [softmax or SVM]\n",
    "    - [batchnorm-relu-conv]xN -> [affine]xM -> [softmax or SVM]\n",
    "- **Global Average Pooling**: Instead of flattening and then having multiple affine layers, perform convolutions until your image gets small (7x7 or so) and then perform an average pooling operation to get to a 1x1 image picture (1, 1 , Filter#), which is then reshaped into a (Filter#) vector. This is used in [Google's Inception Network](https://arxiv.org/abs/1512.00567) (See Table 1 for their architecture).\n",
    "- **Regularization**: Add l2 weight regularization, or perhaps use Dropout.\n",
    "\n",
    "### Tips for training\n",
    "For each network architecture that you try, you should tune the learning rate and other hyperparameters. When doing this there are a couple important things to keep in mind:\n",
    "\n",
    "- If the parameters are working well, you should see improvement within a few hundred iterations\n",
    "- Remember the coarse-to-fine approach for hyperparameter tuning: start by testing a large range of hyperparameters for just a few training iterations to find the combinations of parameters that are working at all.\n",
    "- Once you have found some sets of parameters that seem to work, search more finely around these parameters. You may need to train for more epochs.\n",
    "- You should use the validation set for hyperparameter search, and save your test set for evaluating your architecture on the best parameters as selected by the validation set.\n",
    "\n",
    "### Going above and beyond\n",
    "If you are feeling adventurous there are many other features you can implement to try and improve your performance. You are **not required** to implement any of these, but don't miss the fun if you have time!\n",
    "\n",
    "- Alternative optimizers: you can try Adam, Adagrad, RMSprop, etc.\n",
    "- Alternative activation functions such as leaky ReLU, parametric ReLU, ELU, or MaxOut.\n",
    "- Model ensembles\n",
    "- Data augmentation\n",
    "- New Architectures\n",
    "  - [ResNets](https://arxiv.org/abs/1512.03385) where the input from the previous layer is added to the output.\n",
    "  - [DenseNets](https://arxiv.org/abs/1608.06993) where inputs into previous layers are concatenated together.\n",
    "  - [This blog has an in-depth overview](https://chatbotslife.com/resnets-highwaynets-and-densenets-oh-my-9bb15918ee32)\n",
    "\n",
    "### Have fun and happy training! "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn.functional as F  # useful stateless functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 0, loss = 2.3004\n",
      "Checking accuracy on validation set\n",
      "Got 105 / 1000 correct (10.50)\n",
      "\n",
      "Iteration 100, loss = 1.8291\n",
      "Checking accuracy on validation set\n",
      "Got 427 / 1000 correct (42.70)\n",
      "\n",
      "Iteration 200, loss = 1.4274\n",
      "Checking accuracy on validation set\n",
      "Got 463 / 1000 correct (46.30)\n",
      "\n",
      "Iteration 300, loss = 1.3071\n",
      "Checking accuracy on validation set\n",
      "Got 535 / 1000 correct (53.50)\n",
      "\n",
      "Iteration 400, loss = 1.2173\n",
      "Checking accuracy on validation set\n",
      "Got 581 / 1000 correct (58.10)\n",
      "\n",
      "Iteration 500, loss = 1.0874\n",
      "Checking accuracy on validation set\n",
      "Got 594 / 1000 correct (59.40)\n",
      "\n",
      "Iteration 600, loss = 1.1839\n",
      "Checking accuracy on validation set\n",
      "Got 607 / 1000 correct (60.70)\n",
      "\n",
      "Iteration 700, loss = 1.2205\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Iteration 0, loss = 1.4973\n",
      "Checking accuracy on validation set\n",
      "Got 604 / 1000 correct (60.40)\n",
      "\n",
      "Iteration 100, loss = 1.2352\n",
      "Checking accuracy on validation set\n",
      "Got 637 / 1000 correct (63.70)\n",
      "\n",
      "Iteration 200, loss = 0.9215\n",
      "Checking accuracy on validation set\n",
      "Got 665 / 1000 correct (66.50)\n",
      "\n",
      "Iteration 300, loss = 0.9295\n",
      "Checking accuracy on validation set\n",
      "Got 679 / 1000 correct (67.90)\n",
      "\n",
      "Iteration 400, loss = 0.9107\n",
      "Checking accuracy on validation set\n",
      "Got 643 / 1000 correct (64.30)\n",
      "\n",
      "Iteration 500, loss = 0.9072\n",
      "Checking accuracy on validation set\n",
      "Got 649 / 1000 correct (64.90)\n",
      "\n",
      "Iteration 600, loss = 1.1438\n",
      "Checking accuracy on validation set\n",
      "Got 676 / 1000 correct (67.60)\n",
      "\n",
      "Iteration 700, loss = 1.0477\n",
      "Checking accuracy on validation set\n",
      "Got 686 / 1000 correct (68.60)\n",
      "\n",
      "Iteration 0, loss = 1.1503\n",
      "Checking accuracy on validation set\n",
      "Got 658 / 1000 correct (65.80)\n",
      "\n",
      "Iteration 100, loss = 1.0792\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Iteration 200, loss = 0.9380\n",
      "Checking accuracy on validation set\n",
      "Got 687 / 1000 correct (68.70)\n",
      "\n",
      "Iteration 300, loss = 1.1713\n",
      "Checking accuracy on validation set\n",
      "Got 703 / 1000 correct (70.30)\n",
      "\n",
      "Iteration 400, loss = 0.6416\n",
      "Checking accuracy on validation set\n",
      "Got 719 / 1000 correct (71.90)\n",
      "\n",
      "Iteration 500, loss = 0.8619\n",
      "Checking accuracy on validation set\n",
      "Got 704 / 1000 correct (70.40)\n",
      "\n",
      "Iteration 600, loss = 0.8271\n",
      "Checking accuracy on validation set\n",
      "Got 703 / 1000 correct (70.30)\n",
      "\n",
      "Iteration 700, loss = 0.8209\n",
      "Checking accuracy on validation set\n",
      "Got 721 / 1000 correct (72.10)\n",
      "\n",
      "Iteration 0, loss = 0.7774\n",
      "Checking accuracy on validation set\n",
      "Got 715 / 1000 correct (71.50)\n",
      "\n",
      "Iteration 100, loss = 0.7453\n",
      "Checking accuracy on validation set\n",
      "Got 705 / 1000 correct (70.50)\n",
      "\n",
      "Iteration 200, loss = 0.8113\n",
      "Checking accuracy on validation set\n",
      "Got 676 / 1000 correct (67.60)\n",
      "\n",
      "Iteration 300, loss = 0.6603\n",
      "Checking accuracy on validation set\n",
      "Got 741 / 1000 correct (74.10)\n",
      "\n",
      "Iteration 400, loss = 0.8265\n",
      "Checking accuracy on validation set\n",
      "Got 716 / 1000 correct (71.60)\n",
      "\n",
      "Iteration 500, loss = 0.7668\n",
      "Checking accuracy on validation set\n",
      "Got 732 / 1000 correct (73.20)\n",
      "\n",
      "Iteration 600, loss = 0.6525\n",
      "Checking accuracy on validation set\n",
      "Got 751 / 1000 correct (75.10)\n",
      "\n",
      "Iteration 700, loss = 0.6965\n",
      "Checking accuracy on validation set\n",
      "Got 750 / 1000 correct (75.00)\n",
      "\n",
      "Iteration 0, loss = 0.8928\n",
      "Checking accuracy on validation set\n",
      "Got 743 / 1000 correct (74.30)\n",
      "\n",
      "Iteration 100, loss = 0.6969\n",
      "Checking accuracy on validation set\n",
      "Got 747 / 1000 correct (74.70)\n",
      "\n",
      "Iteration 200, loss = 0.5869\n",
      "Checking accuracy on validation set\n",
      "Got 740 / 1000 correct (74.00)\n",
      "\n",
      "Iteration 300, loss = 0.7444\n",
      "Checking accuracy on validation set\n",
      "Got 736 / 1000 correct (73.60)\n",
      "\n",
      "Iteration 400, loss = 0.7381\n",
      "Checking accuracy on validation set\n",
      "Got 749 / 1000 correct (74.90)\n",
      "\n",
      "Iteration 500, loss = 0.6301\n",
      "Checking accuracy on validation set\n",
      "Got 748 / 1000 correct (74.80)\n",
      "\n",
      "Iteration 600, loss = 0.5014\n",
      "Checking accuracy on validation set\n",
      "Got 713 / 1000 correct (71.30)\n",
      "\n",
      "Iteration 700, loss = 0.7290\n",
      "Checking accuracy on validation set\n",
      "Got 740 / 1000 correct (74.00)\n",
      "\n",
      "Iteration 0, loss = 0.5534\n",
      "Checking accuracy on validation set\n",
      "Got 751 / 1000 correct (75.10)\n",
      "\n",
      "Iteration 100, loss = 0.5632\n",
      "Checking accuracy on validation set\n",
      "Got 756 / 1000 correct (75.60)\n",
      "\n",
      "Iteration 200, loss = 0.7633\n",
      "Checking accuracy on validation set\n",
      "Got 773 / 1000 correct (77.30)\n",
      "\n",
      "Iteration 300, loss = 0.7615\n",
      "Checking accuracy on validation set\n",
      "Got 764 / 1000 correct (76.40)\n",
      "\n",
      "Iteration 400, loss = 0.6765\n",
      "Checking accuracy on validation set\n",
      "Got 757 / 1000 correct (75.70)\n",
      "\n",
      "Iteration 500, loss = 0.4891\n",
      "Checking accuracy on validation set\n",
      "Got 758 / 1000 correct (75.80)\n",
      "\n",
      "Iteration 600, loss = 0.8060\n",
      "Checking accuracy on validation set\n",
      "Got 757 / 1000 correct (75.70)\n",
      "\n",
      "Iteration 700, loss = 0.7148\n",
      "Checking accuracy on validation set\n",
      "Got 776 / 1000 correct (77.60)\n",
      "\n",
      "Iteration 0, loss = 0.6695\n",
      "Checking accuracy on validation set\n",
      "Got 748 / 1000 correct (74.80)\n",
      "\n",
      "Iteration 100, loss = 0.4388\n",
      "Checking accuracy on validation set\n",
      "Got 741 / 1000 correct (74.10)\n",
      "\n",
      "Iteration 200, loss = 0.4872\n",
      "Checking accuracy on validation set\n",
      "Got 784 / 1000 correct (78.40)\n",
      "\n",
      "Iteration 300, loss = 0.6099\n",
      "Checking accuracy on validation set\n",
      "Got 782 / 1000 correct (78.20)\n",
      "\n",
      "Iteration 400, loss = 0.6472\n",
      "Checking accuracy on validation set\n",
      "Got 762 / 1000 correct (76.20)\n",
      "\n",
      "Iteration 500, loss = 0.6479\n",
      "Checking accuracy on validation set\n",
      "Got 782 / 1000 correct (78.20)\n",
      "\n",
      "Iteration 600, loss = 0.4108\n",
      "Checking accuracy on validation set\n",
      "Got 784 / 1000 correct (78.40)\n",
      "\n",
      "Iteration 700, loss = 0.7012\n",
      "Checking accuracy on validation set\n",
      "Got 780 / 1000 correct (78.00)\n",
      "\n",
      "Iteration 0, loss = 0.4364\n",
      "Checking accuracy on validation set\n",
      "Got 760 / 1000 correct (76.00)\n",
      "\n",
      "Iteration 100, loss = 0.6106\n",
      "Checking accuracy on validation set\n",
      "Got 756 / 1000 correct (75.60)\n",
      "\n",
      "Iteration 200, loss = 0.5930\n",
      "Checking accuracy on validation set\n",
      "Got 760 / 1000 correct (76.00)\n",
      "\n",
      "Iteration 300, loss = 0.7947\n",
      "Checking accuracy on validation set\n",
      "Got 784 / 1000 correct (78.40)\n",
      "\n",
      "Iteration 400, loss = 0.3848\n",
      "Checking accuracy on validation set\n",
      "Got 790 / 1000 correct (79.00)\n",
      "\n",
      "Iteration 500, loss = 0.5923\n",
      "Checking accuracy on validation set\n",
      "Got 787 / 1000 correct (78.70)\n",
      "\n",
      "Iteration 600, loss = 0.4217\n",
      "Checking accuracy on validation set\n",
      "Got 788 / 1000 correct (78.80)\n",
      "\n",
      "Iteration 700, loss = 0.3918\n",
      "Checking accuracy on validation set\n",
      "Got 776 / 1000 correct (77.60)\n",
      "\n",
      "Iteration 0, loss = 0.4110\n",
      "Checking accuracy on validation set\n",
      "Got 794 / 1000 correct (79.40)\n",
      "\n",
      "Iteration 100, loss = 0.4149\n",
      "Checking accuracy on validation set\n",
      "Got 777 / 1000 correct (77.70)\n",
      "\n",
      "Iteration 200, loss = 0.4324\n",
      "Checking accuracy on validation set\n",
      "Got 776 / 1000 correct (77.60)\n",
      "\n",
      "Iteration 300, loss = 0.5714\n",
      "Checking accuracy on validation set\n",
      "Got 775 / 1000 correct (77.50)\n",
      "\n",
      "Iteration 400, loss = 0.5102\n",
      "Checking accuracy on validation set\n",
      "Got 789 / 1000 correct (78.90)\n",
      "\n",
      "Iteration 500, loss = 0.4575\n",
      "Checking accuracy on validation set\n",
      "Got 787 / 1000 correct (78.70)\n",
      "\n",
      "Iteration 600, loss = 0.3914\n",
      "Checking accuracy on validation set\n",
      "Got 785 / 1000 correct (78.50)\n",
      "\n",
      "Iteration 700, loss = 0.5307\n",
      "Checking accuracy on validation set\n",
      "Got 770 / 1000 correct (77.00)\n",
      "\n",
      "Iteration 0, loss = 0.5233\n",
      "Checking accuracy on validation set\n",
      "Got 783 / 1000 correct (78.30)\n",
      "\n",
      "Iteration 100, loss = 0.5133\n",
      "Checking accuracy on validation set\n",
      "Got 790 / 1000 correct (79.00)\n",
      "\n",
      "Iteration 200, loss = 0.6844\n",
      "Checking accuracy on validation set\n",
      "Got 796 / 1000 correct (79.60)\n",
      "\n",
      "Iteration 300, loss = 0.3085\n",
      "Checking accuracy on validation set\n",
      "Got 769 / 1000 correct (76.90)\n",
      "\n",
      "Iteration 400, loss = 0.5566\n",
      "Checking accuracy on validation set\n",
      "Got 784 / 1000 correct (78.40)\n",
      "\n",
      "Iteration 500, loss = 0.4439\n",
      "Checking accuracy on validation set\n",
      "Got 789 / 1000 correct (78.90)\n",
      "\n",
      "Iteration 600, loss = 0.5490\n",
      "Checking accuracy on validation set\n",
      "Got 793 / 1000 correct (79.30)\n",
      "\n",
      "Iteration 700, loss = 0.4864\n",
      "Checking accuracy on validation set\n",
      "Got 783 / 1000 correct (78.30)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "################################################################################\n",
    "# TODO:                                                                        #         \n",
    "# Experiment with any architectures, optimizers, and hyperparameters.          #\n",
    "# Achieve AT LEAST 70% accuracy on the *validation set* within 10 epochs.      #\n",
    "#                                                                              #\n",
    "# Note that you can use the check_accuracy function to evaluate on either      #\n",
    "# the test set or the validation set, by passing either loader_test or         #\n",
    "# loader_val as the second argument to check_accuracy. You should not touch    #\n",
    "# the test set until you have finished your architecture and  hyperparameter   #\n",
    "# tuning, and only run the test set once at the end to report a final value.   #\n",
    "################################################################################\n",
    "model = None\n",
    "optimizer = None\n",
    "\n",
    "# *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
    "\n",
    "learning_rate = 1e-3\n",
    "\n",
    "class Net(nn.Module):\n",
    "\n",
    "    def __init__(self, num_classes=10):\n",
    "        super(Net, self).__init__()\n",
    "        self.features = nn.Sequential(\n",
    "            nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1), #(64,32,32,32)\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=1, padding=1), #(64,32,32,32)\n",
    "            \n",
    "            nn.Conv2d(32, 64, kernel_size=5, stride=1, padding=0), #(64,64,28,28)\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=1, padding=1), #(64,64,28,28)\n",
    "            \n",
    "            nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),  #(64,128,28,28)\n",
    "            nn.ReLU(inplace=True),\n",
    "            \n",
    "            nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1), #(64,256,28,28)\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2), #(64,256,14,14)\n",
    "        )\n",
    "        self.avgpool = nn.AdaptiveAvgPool2d((7, 7)) #(64,256,7,7)\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Dropout(),\n",
    "            nn.Linear(256 * 7 * 7, num_classes),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        x = self.avgpool(x)\n",
    "        x = x.view(x.size(0), 256 * 7 * 7)\n",
    "        x = self.classifier(x)\n",
    "        return x\n",
    "\n",
    "model = Net()\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "# *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
    "################################################################################\n",
    "#                                 END OF YOUR CODE                             \n",
    "################################################################################\n",
    "\n",
    "# You should get at least 70% accuracy\n",
    "train_part34(model, optimizer, epochs=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test set -- run this only once\n",
    "\n",
    "Now that we've gotten a result we're happy with, we test our final model on the test set (which you should store in best_model). Think about how this compares to your validation set accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking accuracy on test set\n",
      "Got 7820 / 10000 correct (78.20)\n"
     ]
    }
   ],
   "source": [
    "best_model = model\n",
    "check_accuracy_part34(loader_test, best_model)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}