Self-Attention for Computer Vision.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Lw6n8sqlFIjZ",
    "tags": [
     "pdf-title"
    ]
   },
   "source": [
    "# Self-Attention for Vision\n",
    "\n",
    "We're going to implement self-attention blocks in a convolutional neural network for CIFAR-10 Classification."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CWmQwMDlFIjj"
   },
   "source": [
    "# Part I. Preparation\n",
    "\n",
    "First, we load the CIFAR-10 dataset. This might take a couple minutes the first time you do it, but the files should stay cached after that."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "LhcP-S1VFIjj",
    "tags": [
     "pdf-ignore"
    ]
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data import sampler\n",
    "\n",
    "import torchvision.datasets as dset\n",
    "import torchvision.transforms as T\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 134,
     "referenced_widgets": [
      "6646a364fe5b431eb578707bae8329b8",
      "92a914b698524f5ebd49603d108fbff9",
      "149c512369044de1951159ab8d9ec5ed",
      "15e4679b552e4d9c85a490b59222d1cf",
      "e7cb787fa0d743c08b5d4c395a7431f7",
      "e3796ddf7334425dbc86e68ff37d8a18",
      "d080590971d14b158aea92032dfebecc",
      "4dfce8d8475644e69a52e20410e4fd1b"
     ]
    },
    "id": "ZJJBVo0YFIjk",
    "outputId": "0ad37cdb-9bab-4f74-bdc5-0463ab7cbbad",
    "tags": [
     "pdf-ignore"
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "NUM_TRAIN = 49000\n",
    "\n",
    "# The torchvision.transforms package provides tools for preprocessing data\n",
    "# and for performing data augmentation; here we set up a transform to\n",
    "# preprocess the data by subtracting the mean RGB value and dividing by the\n",
    "# standard deviation of each RGB value; we've hardcoded the mean and std.\n",
    "transform = T.Compose([\n",
    "                T.ToTensor(),\n",
    "                T.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))\n",
    "            ])\n",
    "\n",
    "# We set up a Dataset object for each split (train / val / test); Datasets load\n",
    "# training examples one at a time, so we wrap each Dataset in a DataLoader which\n",
    "# iterates through the Dataset and forms minibatches. We divide the CIFAR-10\n",
    "# training set into train and val sets by passing a Sampler object to the\n",
    "# DataLoader telling how it should sample from the underlying Dataset.\n",
    "cifar10_train = dset.CIFAR10('./data/datasets', train=True, download=True,\n",
    "                             transform=transform)\n",
    "loader_train = DataLoader(cifar10_train, batch_size=64, \n",
    "                          sampler=sampler.SubsetRandomSampler(range(NUM_TRAIN)))\n",
    "\n",
    "cifar10_val = dset.CIFAR10('./data/datasets', train=True, download=True,\n",
    "                           transform=transform)\n",
    "loader_val = DataLoader(cifar10_val, batch_size=64, \n",
    "                        sampler=sampler.SubsetRandomSampler(range(NUM_TRAIN, 50000)))\n",
    "\n",
    "cifar10_test = dset.CIFAR10('./data/datasets', train=False, download=True, \n",
    "                            transform=transform)\n",
    "loader_test = DataLoader(cifar10_test, batch_size=64)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "An3cfS0tFIjm",
    "tags": [
     "pdf-ignore"
    ]
   },
   "source": [
    "You have an option to **use GPU by setting the flag to True below**. It is not necessary to use GPU for this assignment. Note that if your computer does not have CUDA enabled, `torch.cuda.is_available()` will return False and this notebook will fallback to CPU mode.\n",
    "\n",
    "The global variables `dtype` and `device` will control the data types throughout this assignment. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "cOu2LLoEFIjm",
    "outputId": "683549ab-f649-4709-fd83-d44ef527e4df",
    "tags": [
     "pdf-ignore-input"
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using device: cuda\n"
     ]
    }
   ],
   "source": [
    "USE_GPU = True\n",
    "\n",
    "dtype = torch.float32 # we will be using float throughout this tutorial\n",
    "\n",
    "if USE_GPU and torch.cuda.is_available():\n",
    "    device = torch.device('cuda')\n",
    "else:\n",
    "    device = torch.device('cpu')\n",
    "\n",
    "# Constant to control how frequently we print train loss\n",
    "print_every = 100\n",
    "\n",
    "print('using device:', device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "d0Cp2ir0FIjn",
    "tags": [
     "pdf-ignore"
    ]
   },
   "source": [
    "## Flatten Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "R4QkOBJdFIjo",
    "outputId": "286fe612-3115-42ad-a8aa-352adb9d74e2",
    "tags": [
     "pdf-ignore-input"
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before flattening:  tensor([[[[ 0,  1],\n",
      "          [ 2,  3],\n",
      "          [ 4,  5]]],\n",
      "\n",
      "\n",
      "        [[[ 6,  7],\n",
      "          [ 8,  9],\n",
      "          [10, 11]]]])\n",
      "After flattening:  tensor([[ 0,  1,  2,  3,  4,  5],\n",
      "        [ 6,  7,  8,  9, 10, 11]])\n"
     ]
    }
   ],
   "source": [
    "def flatten(x):\n",
    "    N = x.shape[0] # read in N, C, H, W\n",
    "    return x.view(N, -1)  # \"flatten\" the C * H * W values into a single vector per image\n",
    "\n",
    "def test_flatten():\n",
    "    x = torch.arange(12).view(2, 1, 3, 2)\n",
    "    print('Before flattening: ', x)\n",
    "    print('After flattening: ', flatten(x))\n",
    "\n",
    "test_flatten()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XGHNkBu-FIjr"
   },
   "source": [
    "### Check Accuracy Function\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "zeM908EmFIjv"
   },
   "outputs": [],
   "source": [
    "import torch.nn.functional as F  # useful stateless functions\n",
    "def check_accuracy(loader, model):\n",
    "    if loader.dataset.train:\n",
    "        print('Checking accuracy on validation set')\n",
    "    else:\n",
    "        print('Checking accuracy on test set')\n",
    "    num_correct = 0\n",
    "    num_samples = 0\n",
    "    model.eval()  # set model to evaluation mode\n",
    "    with torch.no_grad():\n",
    "        for x, y in loader:\n",
    "            x = x.to(device=device, dtype=dtype)  # move to device, e.g. GPU\n",
    "            y = y.to(device=device, dtype=torch.long)\n",
    "            scores = model(x)\n",
    "            _, preds = scores.max(1)\n",
    "            num_correct += (preds == y).sum()\n",
    "            num_samples += preds.size(0)\n",
    "        acc = float(num_correct) / num_samples\n",
    "        print('Got %d / %d correct (%.2f)' % (num_correct, num_samples, 100 * acc))\n",
    "        return 100 * acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-Ma0ntxxFIjv"
   },
   "source": [
    "### Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "0kP6SeaiFIjv"
   },
   "outputs": [],
   "source": [
    "def train(model, optimizer, epochs=1):\n",
    "    \"\"\"\n",
    "    Train a model on CIFAR-10 using the PyTorch Module API.\n",
    "    \n",
    "    Inputs:\n",
    "    - model: A PyTorch Module giving the model to train.\n",
    "    - optimizer: An Optimizer object we will use to train the model\n",
    "    - epochs: (Optional) A Python integer giving the number of epochs to train for\n",
    "    \n",
    "    Returns: Nothing, but prints model accuracies during training.\n",
    "    \"\"\"\n",
    "    model = model.to(device=device)  # move the model parameters to CPU/GPU\n",
    "    acc_max = 0\n",
    "    for e in range(epochs):\n",
    "        for t, (x, y) in enumerate(loader_train):\n",
    "            \n",
    "            model.train()  # put model to training mode\n",
    "            x = x.to(device=device, dtype=dtype)  # move to device, e.g. GPU\n",
    "            y = y.to(device=device, dtype=torch.long)\n",
    "\n",
    "            scores = model(x)\n",
    "            loss = F.cross_entropy(scores, y)\n",
    "\n",
    "            # Zero out all of the gradients for the variables which the optimizer\n",
    "            # will update.\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            # This is the backwards pass: compute the gradient of the loss with\n",
    "            # respect to each  parameter of the model.\n",
    "            loss.backward()\n",
    "\n",
    "            # Actually update the parameters of the model using the gradients\n",
    "            # computed by the backwards pass.\n",
    "            optimizer.step()\n",
    "\n",
    "            if t % print_every == 0:\n",
    "                print('Epoch %d, Iteration %d, loss = %.4f' % (e, t, loss.item()))\n",
    "                acc = check_accuracy(loader_val, model)\n",
    "                if acc >= acc_max:\n",
    "                    acc_max = acc\n",
    "                print()\n",
    "    print(\"Maximum accuracy attained: \", acc_max)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We need to wrap `flatten` function in a module in order to stack it\n",
    "# in nn.Sequential\n",
    "class Flatten(nn.Module):\n",
    "    def forward(self, x):\n",
    "        return flatten(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vanilla CNN; No Attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, Iteration 0, loss = 2.3140\n",
      "Checking accuracy on validation set\n",
      "Got 141 / 1000 correct (14.10)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.5275\n",
      "Checking accuracy on validation set\n",
      "Got 419 / 1000 correct (41.90)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.4770\n",
      "Checking accuracy on validation set\n",
      "Got 445 / 1000 correct (44.50)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.4464\n",
      "Checking accuracy on validation set\n",
      "Got 475 / 1000 correct (47.50)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.4642\n",
      "Checking accuracy on validation set\n",
      "Got 532 / 1000 correct (53.20)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.4925\n",
      "Checking accuracy on validation set\n",
      "Got 508 / 1000 correct (50.80)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.4339\n",
      "Checking accuracy on validation set\n",
      "Got 534 / 1000 correct (53.40)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.3148\n",
      "Checking accuracy on validation set\n",
      "Got 560 / 1000 correct (56.00)\n",
      "\n",
      "Maximum accuracy attained:  56.00000000000001\n"
     ]
    }
   ],
   "source": [
    "channel_1 = 64\n",
    "channel_2 = 32\n",
    "learning_rate = 1e-3\n",
    "num_classes = 10\n",
    "\n",
    "model = nn.Sequential(\n",
    "    nn.Conv2d(3, channel_1, 3, padding=1, stride=1),\n",
    "    nn.ReLU(),\n",
    "    nn.Conv2d(channel_1, channel_2, 3, padding=1),\n",
    "    nn.ReLU(),\n",
    "    Flatten(),\n",
    "    nn.Linear(channel_2*32*32, num_classes),\n",
    ")\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "\n",
    "train(model, optimizer, epochs=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cTw89bPoFIjy"
   },
   "source": [
    "## Test set "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kpj3j6aEFIjy",
    "outputId": "2220f2f2-b24d-4c1d-9d9a-ed212567fce9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking accuracy on test set\n",
      "Got 5534 / 10000 correct (55.34)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "55.34"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vanillaModel = model\n",
    "check_accuracy(loader_test, vanillaModel)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-gH8-iqGUOO1"
   },
   "source": [
    "## Part II Self-Attention\n",
    "\n",
    "In the next section, you will implement an Attention layer which you will then use within a convnet architecture defined above for cifar 10 classification task.\n",
    "\n",
    "A self-attention layer is formulated as following:\n",
    "\n",
    "Input: $X$ of shape $(H\\times W, C)$\n",
    "\n",
    "Query, key, value linear transforms are $W_Q$, $W_K$, $W_V$, of shape $(C, C)$. We implement these linear transforms as 1x1 convolutional layers of the same dimensions.\n",
    "\n",
    "$XW_Q$, $XW_K$, $XW_V$, represent the output volumes when input X is passed through the transforms.\n",
    "\n",
    "\n",
    "Self-Attention is given by the formula: $Attention(X) = X + Softmax(\\frac{XW_Q(XW_K)^\\top}{\\sqrt{C}})XW_V$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Here you implement the Attention module, and run it in the next section"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "uvcTf4eUY-FX"
   },
   "outputs": [],
   "source": [
    "# Initialize the attention module as a nn.Module subclass\n",
    "class Attention(nn.Module):\n",
    "    def __init__(self, in_channels):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.conv_query = nn.Conv2d(in_channels, in_channels, kernel_size=1)\n",
    "        self.conv_key = nn.Conv2d(in_channels, in_channels, kernel_size=1)\n",
    "        self.conv_value = nn.Conv2d(in_channels, in_channels, kernel_size=1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        N, C, H, W = x.shape\n",
    "        \n",
    "        q = self.conv_query(x).reshape(N, C, H*W)\n",
    "\n",
    "        k = self.conv_key(x).reshape(N, C, H*W)\n",
    "\n",
    "        v = self.conv_value(x).reshape(N, C, H*W)\n",
    "\n",
    "        attention = torch.bmm(torch.softmax(torch.bmm(q, k.transpose(1, 2)) / int(C**0.5), dim=2),v)\n",
    "\n",
    "        attention = attention.reshape(N, C, H, W)\n",
    "        return x + attention"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Single Attention Block: Early attention; After the first conv layer. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6A3xh7aVFWzM",
    "outputId": "f45f9049-70ce-414f-ab1d-446f619ad367",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, Iteration 0, loss = 2.2951\n",
      "Checking accuracy on validation set\n",
      "Got 135 / 1000 correct (13.50)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.5557\n",
      "Checking accuracy on validation set\n",
      "Got 451 / 1000 correct (45.10)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.4654\n",
      "Checking accuracy on validation set\n",
      "Got 499 / 1000 correct (49.90)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.1809\n",
      "Checking accuracy on validation set\n",
      "Got 506 / 1000 correct (50.60)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.3901\n",
      "Checking accuracy on validation set\n",
      "Got 566 / 1000 correct (56.60)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.2920\n",
      "Checking accuracy on validation set\n",
      "Got 551 / 1000 correct (55.10)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 0.9551\n",
      "Checking accuracy on validation set\n",
      "Got 580 / 1000 correct (58.00)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.2441\n",
      "Checking accuracy on validation set\n",
      "Got 585 / 1000 correct (58.50)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 0.9908\n",
      "Checking accuracy on validation set\n",
      "Got 608 / 1000 correct (60.80)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 0.8331\n",
      "Checking accuracy on validation set\n",
      "Got 606 / 1000 correct (60.60)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 0.9495\n",
      "Checking accuracy on validation set\n",
      "Got 588 / 1000 correct (58.80)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 1.1831\n",
      "Checking accuracy on validation set\n",
      "Got 604 / 1000 correct (60.40)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 1.0644\n",
      "Checking accuracy on validation set\n",
      "Got 617 / 1000 correct (61.70)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 0.8790\n",
      "Checking accuracy on validation set\n",
      "Got 606 / 1000 correct (60.60)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 0.9228\n",
      "Checking accuracy on validation set\n",
      "Got 632 / 1000 correct (63.20)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 1.1159\n",
      "Checking accuracy on validation set\n",
      "Got 635 / 1000 correct (63.50)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 0.8389\n",
      "Checking accuracy on validation set\n",
      "Got 631 / 1000 correct (63.10)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 0.7202\n",
      "Checking accuracy on validation set\n",
      "Got 625 / 1000 correct (62.50)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.7884\n",
      "Checking accuracy on validation set\n",
      "Got 628 / 1000 correct (62.80)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.7562\n",
      "Checking accuracy on validation set\n",
      "Got 631 / 1000 correct (63.10)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 0.8354\n",
      "Checking accuracy on validation set\n",
      "Got 645 / 1000 correct (64.50)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 0.8239\n",
      "Checking accuracy on validation set\n",
      "Got 655 / 1000 correct (65.50)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.5982\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.8234\n",
      "Checking accuracy on validation set\n",
      "Got 642 / 1000 correct (64.20)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.6068\n",
      "Checking accuracy on validation set\n",
      "Got 630 / 1000 correct (63.00)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.5275\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.5527\n",
      "Checking accuracy on validation set\n",
      "Got 630 / 1000 correct (63.00)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.8258\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.6166\n",
      "Checking accuracy on validation set\n",
      "Got 634 / 1000 correct (63.40)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.8229\n",
      "Checking accuracy on validation set\n",
      "Got 627 / 1000 correct (62.70)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.7252\n",
      "Checking accuracy on validation set\n",
      "Got 635 / 1000 correct (63.50)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.4869\n",
      "Checking accuracy on validation set\n",
      "Got 614 / 1000 correct (61.40)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.4342\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.3652\n",
      "Checking accuracy on validation set\n",
      "Got 639 / 1000 correct (63.90)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.2804\n",
      "Checking accuracy on validation set\n",
      "Got 635 / 1000 correct (63.50)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.4311\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.7790\n",
      "Checking accuracy on validation set\n",
      "Got 639 / 1000 correct (63.90)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.5164\n",
      "Checking accuracy on validation set\n",
      "Got 616 / 1000 correct (61.60)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.4852\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.5793\n",
      "Checking accuracy on validation set\n",
      "Got 632 / 1000 correct (63.20)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.2022\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.3501\n",
      "Checking accuracy on validation set\n",
      "Got 640 / 1000 correct (64.00)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.1397\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.3891\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.3471\n",
      "Checking accuracy on validation set\n",
      "Got 642 / 1000 correct (64.20)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.2549\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.4250\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.5219\n",
      "Checking accuracy on validation set\n",
      "Got 599 / 1000 correct (59.90)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.1344\n",
      "Checking accuracy on validation set\n",
      "Got 626 / 1000 correct (62.60)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.2268\n",
      "Checking accuracy on validation set\n",
      "Got 630 / 1000 correct (63.00)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.1639\n",
      "Checking accuracy on validation set\n",
      "Got 643 / 1000 correct (64.30)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.1234\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.2449\n",
      "Checking accuracy on validation set\n",
      "Got 627 / 1000 correct (62.70)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.1166\n",
      "Checking accuracy on validation set\n",
      "Got 615 / 1000 correct (61.50)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.4342\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.3435\n",
      "Checking accuracy on validation set\n",
      "Got 618 / 1000 correct (61.80)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.1773\n",
      "Checking accuracy on validation set\n",
      "Got 615 / 1000 correct (61.50)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.0814\n",
      "Checking accuracy on validation set\n",
      "Got 619 / 1000 correct (61.90)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.1724\n",
      "Checking accuracy on validation set\n",
      "Got 627 / 1000 correct (62.70)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.1198\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.1732\n",
      "Checking accuracy on validation set\n",
      "Got 630 / 1000 correct (63.00)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.1659\n",
      "Checking accuracy on validation set\n",
      "Got 596 / 1000 correct (59.60)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.2625\n",
      "Checking accuracy on validation set\n",
      "Got 603 / 1000 correct (60.30)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.1682\n",
      "Checking accuracy on validation set\n",
      "Got 607 / 1000 correct (60.70)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.1307\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.0766\n",
      "Checking accuracy on validation set\n",
      "Got 636 / 1000 correct (63.60)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.1573\n",
      "Checking accuracy on validation set\n",
      "Got 627 / 1000 correct (62.70)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.0812\n",
      "Checking accuracy on validation set\n",
      "Got 620 / 1000 correct (62.00)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.1821\n",
      "Checking accuracy on validation set\n",
      "Got 623 / 1000 correct (62.30)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.0749\n",
      "Checking accuracy on validation set\n",
      "Got 616 / 1000 correct (61.60)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.0675\n",
      "Checking accuracy on validation set\n",
      "Got 614 / 1000 correct (61.40)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.1127\n",
      "Checking accuracy on validation set\n",
      "Got 617 / 1000 correct (61.70)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.1185\n",
      "Checking accuracy on validation set\n",
      "Got 626 / 1000 correct (62.60)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.0855\n",
      "Checking accuracy on validation set\n",
      "Got 626 / 1000 correct (62.60)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.1271\n",
      "Checking accuracy on validation set\n",
      "Got 627 / 1000 correct (62.70)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.0330\n",
      "Checking accuracy on validation set\n",
      "Got 621 / 1000 correct (62.10)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.1301\n",
      "Checking accuracy on validation set\n",
      "Got 624 / 1000 correct (62.40)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.0849\n",
      "Checking accuracy on validation set\n",
      "Got 627 / 1000 correct (62.70)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.1218\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.1387\n",
      "Checking accuracy on validation set\n",
      "Got 620 / 1000 correct (62.00)\n",
      "\n",
      "Maximum accuracy attained:  65.5\n"
     ]
    }
   ],
   "source": [
    "channel_1 = 64\n",
    "channel_2 = 32\n",
    "learning_rate = 1e-3\n",
    "\n",
    "\n",
    "model = nn.Sequential(\n",
    "    nn.Conv2d(3, channel_1, 3, padding=1, stride=1),\n",
    "    nn.ReLU(),\n",
    "    Attention(channel_1),\n",
    "    nn.ReLU(),\n",
    "    nn.Conv2d(channel_1, channel_2, 3, padding=1),\n",
    "    nn.ReLU(),\n",
    "    Flatten(),\n",
    "    nn.Linear(channel_2*32*32, num_classes),\n",
    ")\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "train(model, optimizer, epochs=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cTw89bPoFIjy"
   },
   "source": [
    "## Test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kpj3j6aEFIjy",
    "outputId": "2220f2f2-b24d-4c1d-9d9a-ed212567fce9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking accuracy on test set\n",
      "Got 6033 / 10000 correct (60.33)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "60.33"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "earlyAttention = model\n",
    "check_accuracy(loader_test, earlyAttention)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Single Attention Block: Late attention; After the second conv layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, Iteration 0, loss = 2.3106\n",
      "Checking accuracy on validation set\n",
      "Got 107 / 1000 correct (10.70)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.4990\n",
      "Checking accuracy on validation set\n",
      "Got 451 / 1000 correct (45.10)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.5117\n",
      "Checking accuracy on validation set\n",
      "Got 488 / 1000 correct (48.80)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.1954\n",
      "Checking accuracy on validation set\n",
      "Got 527 / 1000 correct (52.70)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.3444\n",
      "Checking accuracy on validation set\n",
      "Got 527 / 1000 correct (52.70)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.3058\n",
      "Checking accuracy on validation set\n",
      "Got 539 / 1000 correct (53.90)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.3221\n",
      "Checking accuracy on validation set\n",
      "Got 554 / 1000 correct (55.40)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.0496\n",
      "Checking accuracy on validation set\n",
      "Got 564 / 1000 correct (56.40)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 0.9104\n",
      "Checking accuracy on validation set\n",
      "Got 598 / 1000 correct (59.80)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 1.2924\n",
      "Checking accuracy on validation set\n",
      "Got 587 / 1000 correct (58.70)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 1.0363\n",
      "Checking accuracy on validation set\n",
      "Got 602 / 1000 correct (60.20)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 1.3267\n",
      "Checking accuracy on validation set\n",
      "Got 584 / 1000 correct (58.40)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 0.9510\n",
      "Checking accuracy on validation set\n",
      "Got 616 / 1000 correct (61.60)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 1.1619\n",
      "Checking accuracy on validation set\n",
      "Got 631 / 1000 correct (63.10)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 1.2036\n",
      "Checking accuracy on validation set\n",
      "Got 606 / 1000 correct (60.60)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 0.8760\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 0.8443\n",
      "Checking accuracy on validation set\n",
      "Got 639 / 1000 correct (63.90)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 0.6435\n",
      "Checking accuracy on validation set\n",
      "Got 617 / 1000 correct (61.70)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.7006\n",
      "Checking accuracy on validation set\n",
      "Got 616 / 1000 correct (61.60)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.9314\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 1.0327\n",
      "Checking accuracy on validation set\n",
      "Got 649 / 1000 correct (64.90)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 0.9542\n",
      "Checking accuracy on validation set\n",
      "Got 642 / 1000 correct (64.20)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.7026\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.8256\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.6246\n",
      "Checking accuracy on validation set\n",
      "Got 639 / 1000 correct (63.90)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.7804\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.6787\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.7402\n",
      "Checking accuracy on validation set\n",
      "Got 661 / 1000 correct (66.10)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.9845\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.8530\n",
      "Checking accuracy on validation set\n",
      "Got 664 / 1000 correct (66.40)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.7348\n",
      "Checking accuracy on validation set\n",
      "Got 665 / 1000 correct (66.50)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.8610\n",
      "Checking accuracy on validation set\n",
      "Got 660 / 1000 correct (66.00)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.5695\n",
      "Checking accuracy on validation set\n",
      "Got 663 / 1000 correct (66.30)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.5710\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.3702\n",
      "Checking accuracy on validation set\n",
      "Got 645 / 1000 correct (64.50)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.6352\n",
      "Checking accuracy on validation set\n",
      "Got 660 / 1000 correct (66.00)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.7369\n",
      "Checking accuracy on validation set\n",
      "Got 664 / 1000 correct (66.40)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.7486\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.5449\n",
      "Checking accuracy on validation set\n",
      "Got 656 / 1000 correct (65.60)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.6389\n",
      "Checking accuracy on validation set\n",
      "Got 664 / 1000 correct (66.40)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.3910\n",
      "Checking accuracy on validation set\n",
      "Got 658 / 1000 correct (65.80)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.3961\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.3125\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.3729\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.4303\n",
      "Checking accuracy on validation set\n",
      "Got 664 / 1000 correct (66.40)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.5042\n",
      "Checking accuracy on validation set\n",
      "Got 662 / 1000 correct (66.20)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.5877\n",
      "Checking accuracy on validation set\n",
      "Got 662 / 1000 correct (66.20)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.5719\n",
      "Checking accuracy on validation set\n",
      "Got 642 / 1000 correct (64.20)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.3757\n",
      "Checking accuracy on validation set\n",
      "Got 675 / 1000 correct (67.50)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.3737\n",
      "Checking accuracy on validation set\n",
      "Got 643 / 1000 correct (64.30)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.1693\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.4865\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.3347\n",
      "Checking accuracy on validation set\n",
      "Got 653 / 1000 correct (65.30)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.5700\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.2978\n",
      "Checking accuracy on validation set\n",
      "Got 640 / 1000 correct (64.00)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.5148\n",
      "Checking accuracy on validation set\n",
      "Got 656 / 1000 correct (65.60)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.2038\n",
      "Checking accuracy on validation set\n",
      "Got 653 / 1000 correct (65.30)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.2369\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.1781\n",
      "Checking accuracy on validation set\n",
      "Got 658 / 1000 correct (65.80)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.3192\n",
      "Checking accuracy on validation set\n",
      "Got 636 / 1000 correct (63.60)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.1893\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.2481\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.2716\n",
      "Checking accuracy on validation set\n",
      "Got 639 / 1000 correct (63.90)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.5002\n",
      "Checking accuracy on validation set\n",
      "Got 640 / 1000 correct (64.00)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.0824\n",
      "Checking accuracy on validation set\n",
      "Got 617 / 1000 correct (61.70)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.2903\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.1422\n",
      "Checking accuracy on validation set\n",
      "Got 641 / 1000 correct (64.10)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.1945\n",
      "Checking accuracy on validation set\n",
      "Got 631 / 1000 correct (63.10)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.1283\n",
      "Checking accuracy on validation set\n",
      "Got 630 / 1000 correct (63.00)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.3194\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.2348\n",
      "Checking accuracy on validation set\n",
      "Got 616 / 1000 correct (61.60)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.3631\n",
      "Checking accuracy on validation set\n",
      "Got 632 / 1000 correct (63.20)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.1492\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.0755\n",
      "Checking accuracy on validation set\n",
      "Got 640 / 1000 correct (64.00)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.0950\n",
      "Checking accuracy on validation set\n",
      "Got 616 / 1000 correct (61.60)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.0692\n",
      "Checking accuracy on validation set\n",
      "Got 631 / 1000 correct (63.10)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.1106\n",
      "Checking accuracy on validation set\n",
      "Got 614 / 1000 correct (61.40)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.1314\n",
      "Checking accuracy on validation set\n",
      "Got 626 / 1000 correct (62.60)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.2031\n",
      "Checking accuracy on validation set\n",
      "Got 635 / 1000 correct (63.50)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.2569\n",
      "Checking accuracy on validation set\n",
      "Got 614 / 1000 correct (61.40)\n",
      "\n",
      "Maximum accuracy attained:  67.5\n"
     ]
    }
   ],
   "source": [
    "channel_1 = 64\n",
    "channel_2 = 32\n",
    "learning_rate = 1e-3\n",
    "\n",
    "# TODO: Use the above Attention module after the Second Convolutional layer.\n",
    "# Essentially the architecture should be [Conv->Relu->Conv->Relu->Attention->Relu->Linear]\n",
    "\n",
    "model = nn.Sequential(\n",
    "    nn.Conv2d(3, channel_1, 3, padding=1, stride=1),\n",
    "    nn.ReLU(),\n",
    "    nn.Conv2d(channel_1, channel_2, 3, padding=1),\n",
    "    nn.ReLU(),\n",
    "    Attention(channel_2),\n",
    "    nn.ReLU(),\n",
    "    Flatten(),\n",
    "    nn.Linear(channel_2*32*32, num_classes),\n",
    ")\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "train(model, optimizer, epochs=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cTw89bPoFIjy"
   },
   "source": [
    "## Test set "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kpj3j6aEFIjy",
    "outputId": "2220f2f2-b24d-4c1d-9d9a-ed212567fce9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking accuracy on test set\n",
      "Got 6054 / 10000 correct (60.54)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "60.540000000000006"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lateAttention = model\n",
    "check_accuracy(loader_test, lateAttention)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Double Attention Blocks: After conv layers 1 and 2 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, Iteration 0, loss = 2.3165\n",
      "Checking accuracy on validation set\n",
      "Got 166 / 1000 correct (16.60)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.4627\n",
      "Checking accuracy on validation set\n",
      "Got 437 / 1000 correct (43.70)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.7036\n",
      "Checking accuracy on validation set\n",
      "Got 459 / 1000 correct (45.90)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.5321\n",
      "Checking accuracy on validation set\n",
      "Got 492 / 1000 correct (49.20)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.4116\n",
      "Checking accuracy on validation set\n",
      "Got 526 / 1000 correct (52.60)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.1967\n",
      "Checking accuracy on validation set\n",
      "Got 541 / 1000 correct (54.10)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.3342\n",
      "Checking accuracy on validation set\n",
      "Got 562 / 1000 correct (56.20)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.3691\n",
      "Checking accuracy on validation set\n",
      "Got 596 / 1000 correct (59.60)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 0.8812\n",
      "Checking accuracy on validation set\n",
      "Got 571 / 1000 correct (57.10)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 1.1494\n",
      "Checking accuracy on validation set\n",
      "Got 604 / 1000 correct (60.40)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 0.9883\n",
      "Checking accuracy on validation set\n",
      "Got 607 / 1000 correct (60.70)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 0.9623\n",
      "Checking accuracy on validation set\n",
      "Got 621 / 1000 correct (62.10)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 0.9955\n",
      "Checking accuracy on validation set\n",
      "Got 611 / 1000 correct (61.10)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 1.0265\n",
      "Checking accuracy on validation set\n",
      "Got 618 / 1000 correct (61.80)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 0.8762\n",
      "Checking accuracy on validation set\n",
      "Got 630 / 1000 correct (63.00)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 1.3119\n",
      "Checking accuracy on validation set\n",
      "Got 609 / 1000 correct (60.90)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 0.7887\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 0.8161\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.9441\n",
      "Checking accuracy on validation set\n",
      "Got 630 / 1000 correct (63.00)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.7474\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 0.8491\n",
      "Checking accuracy on validation set\n",
      "Got 643 / 1000 correct (64.30)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 0.9150\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.6227\n",
      "Checking accuracy on validation set\n",
      "Got 667 / 1000 correct (66.70)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.9589\n",
      "Checking accuracy on validation set\n",
      "Got 681 / 1000 correct (68.10)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.6500\n",
      "Checking accuracy on validation set\n",
      "Got 676 / 1000 correct (67.60)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.4874\n",
      "Checking accuracy on validation set\n",
      "Got 672 / 1000 correct (67.20)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.6595\n",
      "Checking accuracy on validation set\n",
      "Got 667 / 1000 correct (66.70)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.7583\n",
      "Checking accuracy on validation set\n",
      "Got 671 / 1000 correct (67.10)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.7915\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.5663\n",
      "Checking accuracy on validation set\n",
      "Got 665 / 1000 correct (66.50)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.7374\n",
      "Checking accuracy on validation set\n",
      "Got 673 / 1000 correct (67.30)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.6443\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.3376\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.2837\n",
      "Checking accuracy on validation set\n",
      "Got 663 / 1000 correct (66.30)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.4647\n",
      "Checking accuracy on validation set\n",
      "Got 665 / 1000 correct (66.50)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.5949\n",
      "Checking accuracy on validation set\n",
      "Got 634 / 1000 correct (63.40)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.5973\n",
      "Checking accuracy on validation set\n",
      "Got 653 / 1000 correct (65.30)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.3557\n",
      "Checking accuracy on validation set\n",
      "Got 661 / 1000 correct (66.10)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.5399\n",
      "Checking accuracy on validation set\n",
      "Got 649 / 1000 correct (64.90)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.7581\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.4118\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.3607\n",
      "Checking accuracy on validation set\n",
      "Got 664 / 1000 correct (66.40)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.4042\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.3087\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.3103\n",
      "Checking accuracy on validation set\n",
      "Got 657 / 1000 correct (65.70)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.2747\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.5531\n",
      "Checking accuracy on validation set\n",
      "Got 667 / 1000 correct (66.70)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.5016\n",
      "Checking accuracy on validation set\n",
      "Got 657 / 1000 correct (65.70)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.2198\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.1901\n",
      "Checking accuracy on validation set\n",
      "Got 662 / 1000 correct (66.20)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.2304\n",
      "Checking accuracy on validation set\n",
      "Got 663 / 1000 correct (66.30)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.3667\n",
      "Checking accuracy on validation set\n",
      "Got 663 / 1000 correct (66.30)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.4046\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.2921\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.5293\n",
      "Checking accuracy on validation set\n",
      "Got 666 / 1000 correct (66.60)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.2302\n",
      "Checking accuracy on validation set\n",
      "Got 645 / 1000 correct (64.50)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.3245\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.3404\n",
      "Checking accuracy on validation set\n",
      "Got 663 / 1000 correct (66.30)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.1529\n",
      "Checking accuracy on validation set\n",
      "Got 656 / 1000 correct (65.60)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.4345\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.3414\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.2054\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.3198\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.3552\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.1933\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.1254\n",
      "Checking accuracy on validation set\n",
      "Got 660 / 1000 correct (66.00)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.3105\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.1376\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.1197\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.2408\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.1477\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.2970\n",
      "Checking accuracy on validation set\n",
      "Got 639 / 1000 correct (63.90)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.1101\n",
      "Checking accuracy on validation set\n",
      "Got 634 / 1000 correct (63.40)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.1321\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.0704\n",
      "Checking accuracy on validation set\n",
      "Got 645 / 1000 correct (64.50)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.1244\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.1088\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.1526\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.1803\n",
      "Checking accuracy on validation set\n",
      "Got 643 / 1000 correct (64.30)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.1201\n",
      "Checking accuracy on validation set\n",
      "Got 643 / 1000 correct (64.30)\n",
      "\n",
      "Maximum accuracy attained:  68.10000000000001\n"
     ]
    }
   ],
   "source": [
    "channel_1 = 64\n",
    "channel_2 = 32\n",
    "learning_rate = 1e-3\n",
    "\n",
    "model = nn.Sequential(\n",
    "    nn.Conv2d(3, channel_1, 3, padding=1, stride=1),\n",
    "    nn.ReLU(),\n",
    "    Attention(channel_1),\n",
    "    nn.ReLU(),\n",
    "    nn.Conv2d(channel_1, channel_2, 3, padding=1),\n",
    "    nn.ReLU(),\n",
    "    Attention(channel_2),\n",
    "    nn.ReLU(),\n",
    "    Flatten(),\n",
    "    nn.Linear(channel_2*32*32, num_classes),\n",
    ")\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "train(model, optimizer, epochs=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cTw89bPoFIjy"
   },
   "source": [
    "## Test set "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kpj3j6aEFIjy",
    "outputId": "2220f2f2-b24d-4c1d-9d9a-ed212567fce9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking accuracy on test set\n",
      "Got 6184 / 10000 correct (61.84)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "61.839999999999996"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doubleAttentionModel = model\n",
    "check_accuracy(loader_test, doubleAttentionModel)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resnet with Attention \n",
    "\n",
    "Now we will experiment with applying attention within the Resnet10 architecture "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vanilla Resnet, No Attention\n",
    "\n",
    "The architecture for Resnet is given below, we will train it and evaluate it on the test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "class ResNet(nn.Module):\n",
    "\n",
    "    def __init__(self, block, layers, img_channels=3, num_classes=100, batchnorm=False):\n",
    "        super(ResNet, self).__init__() #layers = [1, 1, 1, 1] \n",
    "        self.in_channels = 64\n",
    "        self.conv1 = nn.Conv2d(img_channels, 64, kernel_size=7, stride=2, padding=3)\n",
    "        self.bn1 = nn.BatchNorm2d(64)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n",
    "        self.batchnorm = batchnorm\n",
    "        self.layer1 = self.make_layer(block, layers[0], out_channels=64, stride=1, batchnorm=batchnorm)\n",
    "        self.layer2 = self.make_layer(block, layers[1], out_channels=128, stride=1, batchnorm=batchnorm)\n",
    "        self.layer3 = self.make_layer(block, layers[2], out_channels=256, stride=1, batchnorm=batchnorm)\n",
    "        self.layer4 = self.make_layer(block, layers[3], out_channels=512, stride=2, batchnorm=batchnorm)\n",
    "\n",
    "        self.averagepool = nn.AdaptiveAvgPool2d((1, 1))\n",
    "        self.fc = nn.Linear(512, num_classes)\n",
    "\n",
    "    \n",
    "    def forward(self, x):\n",
    "\n",
    "        x = self.conv1(x)\n",
    "        if self.batchnorm:\n",
    "            x = self.bn1(x)\n",
    "        x = self.relu(x)\n",
    "        x = self.maxpool(x)\n",
    "        x = self.layer1(x) \n",
    "        x = self.layer2(x)\n",
    "        x = self.layer3(x)\n",
    "        x = self.layer4(x)\n",
    "        x = self.averagepool(x)\n",
    "        x = x.reshape(x.shape[0], -1)\n",
    "        x = x.reshape(x.shape[0], -1)\n",
    "        x = self.fc(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "    def make_layer(self, block, num_blocks, out_channels, stride, batchnorm=False):\n",
    "        downsampler = None\n",
    "        layers = []\n",
    "        if stride != 1 or self.in_channels != out_channels:\n",
    "            downsampler = nn.Sequential(nn.Conv2d(self.in_channels, out_channels, kernel_size = 1, stride = stride), nn.BatchNorm2d(out_channels))\n",
    "\n",
    "        layers.append(block(self.in_channels, out_channels, downsampler, stride, batchnorm=batchnorm))\n",
    "\n",
    "        self.in_channels = out_channels\n",
    "\n",
    "        for i in range(num_blocks - 1):\n",
    "            layers.append(block(self.in_channels, out_channels))\n",
    "\n",
    "        \n",
    "        return nn.Sequential(*layers)\n",
    "        \n",
    "class block(nn.Module):\n",
    "\n",
    "    def __init__(self, in_channels, out_channels, downsampler = None, stride = 1, batchnorm=False):\n",
    "        \n",
    "        super(block, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size = 3, padding = 2)\n",
    "        self.bn1 = nn.BatchNorm2d(out_channels)\n",
    "        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size = 3, stride = stride)\n",
    "        self.bn2 = nn.BatchNorm2d(out_channels)\n",
    "        self.downsampler = downsampler\n",
    "        self.relu = nn.ReLU()\n",
    "        self.batchnorm = batchnorm\n",
    "\n",
    "    \n",
    "    def forward(self, x):\n",
    "\n",
    "        residual = x\n",
    "        x = self.conv1(x)\n",
    "        if self.batchnorm:\n",
    "            x = self.bn1(x)\n",
    "        x = self.relu(x)\n",
    "        x = self.conv2(x)\n",
    "        if self.batchnorm:\n",
    "            x = self.bn2(x)\n",
    "        x = self.relu(x)\n",
    "        \n",
    "        if self.downsampler:\n",
    "            residual = self.downsampler(residual)\n",
    "\n",
    "        return self.relu(residual + x)\n",
    "    \n",
    "def ResNet10(num_classes = 100, batchnorm= False):\n",
    "\n",
    "    return ResNet(block, [1, 1, 1, 1], num_classes=num_classes, batchnorm=batchnorm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test set "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.9/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  /pytorch/c10/core/TensorImpl.h:1156.)\n",
      "  return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, Iteration 0, loss = 4.5606\n",
      "Checking accuracy on validation set\n",
      "Got 105 / 1000 correct (10.50)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.7618\n",
      "Checking accuracy on validation set\n",
      "Got 390 / 1000 correct (39.00)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.3404\n",
      "Checking accuracy on validation set\n",
      "Got 403 / 1000 correct (40.30)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.3388\n",
      "Checking accuracy on validation set\n",
      "Got 511 / 1000 correct (51.10)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.2707\n",
      "Checking accuracy on validation set\n",
      "Got 489 / 1000 correct (48.90)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.3293\n",
      "Checking accuracy on validation set\n",
      "Got 527 / 1000 correct (52.70)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.1351\n",
      "Checking accuracy on validation set\n",
      "Got 525 / 1000 correct (52.50)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.1695\n",
      "Checking accuracy on validation set\n",
      "Got 541 / 1000 correct (54.10)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 1.3914\n",
      "Checking accuracy on validation set\n",
      "Got 578 / 1000 correct (57.80)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 1.1982\n",
      "Checking accuracy on validation set\n",
      "Got 565 / 1000 correct (56.50)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 1.1140\n",
      "Checking accuracy on validation set\n",
      "Got 603 / 1000 correct (60.30)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 0.9572\n",
      "Checking accuracy on validation set\n",
      "Got 640 / 1000 correct (64.00)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 0.8814\n",
      "Checking accuracy on validation set\n",
      "Got 576 / 1000 correct (57.60)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 0.9673\n",
      "Checking accuracy on validation set\n",
      "Got 657 / 1000 correct (65.70)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 0.8588\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 0.9227\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 1.0683\n",
      "Checking accuracy on validation set\n",
      "Got 658 / 1000 correct (65.80)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 0.7600\n",
      "Checking accuracy on validation set\n",
      "Got 641 / 1000 correct (64.10)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.8439\n",
      "Checking accuracy on validation set\n",
      "Got 653 / 1000 correct (65.30)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.6492\n",
      "Checking accuracy on validation set\n",
      "Got 690 / 1000 correct (69.00)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 0.7471\n",
      "Checking accuracy on validation set\n",
      "Got 663 / 1000 correct (66.30)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 0.6780\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.9913\n",
      "Checking accuracy on validation set\n",
      "Got 683 / 1000 correct (68.30)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.6712\n",
      "Checking accuracy on validation set\n",
      "Got 688 / 1000 correct (68.80)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.9294\n",
      "Checking accuracy on validation set\n",
      "Got 675 / 1000 correct (67.50)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.6541\n",
      "Checking accuracy on validation set\n",
      "Got 703 / 1000 correct (70.30)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.7450\n",
      "Checking accuracy on validation set\n",
      "Got 727 / 1000 correct (72.70)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.6069\n",
      "Checking accuracy on validation set\n",
      "Got 713 / 1000 correct (71.30)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.7548\n",
      "Checking accuracy on validation set\n",
      "Got 700 / 1000 correct (70.00)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.7459\n",
      "Checking accuracy on validation set\n",
      "Got 708 / 1000 correct (70.80)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.6731\n",
      "Checking accuracy on validation set\n",
      "Got 734 / 1000 correct (73.40)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.7185\n",
      "Checking accuracy on validation set\n",
      "Got 706 / 1000 correct (70.60)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.7727\n",
      "Checking accuracy on validation set\n",
      "Got 678 / 1000 correct (67.80)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.7117\n",
      "Checking accuracy on validation set\n",
      "Got 728 / 1000 correct (72.80)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.7685\n",
      "Checking accuracy on validation set\n",
      "Got 703 / 1000 correct (70.30)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.5160\n",
      "Checking accuracy on validation set\n",
      "Got 725 / 1000 correct (72.50)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.7816\n",
      "Checking accuracy on validation set\n",
      "Got 720 / 1000 correct (72.00)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.9551\n",
      "Checking accuracy on validation set\n",
      "Got 698 / 1000 correct (69.80)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.8399\n",
      "Checking accuracy on validation set\n",
      "Got 739 / 1000 correct (73.90)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.6225\n",
      "Checking accuracy on validation set\n",
      "Got 738 / 1000 correct (73.80)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.6046\n",
      "Checking accuracy on validation set\n",
      "Got 739 / 1000 correct (73.90)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.6642\n",
      "Checking accuracy on validation set\n",
      "Got 734 / 1000 correct (73.40)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.4583\n",
      "Checking accuracy on validation set\n",
      "Got 725 / 1000 correct (72.50)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.5907\n",
      "Checking accuracy on validation set\n",
      "Got 740 / 1000 correct (74.00)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.5625\n",
      "Checking accuracy on validation set\n",
      "Got 759 / 1000 correct (75.90)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.5458\n",
      "Checking accuracy on validation set\n",
      "Got 735 / 1000 correct (73.50)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.6488\n",
      "Checking accuracy on validation set\n",
      "Got 739 / 1000 correct (73.90)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.7098\n",
      "Checking accuracy on validation set\n",
      "Got 736 / 1000 correct (73.60)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.4698\n",
      "Checking accuracy on validation set\n",
      "Got 733 / 1000 correct (73.30)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.5445\n",
      "Checking accuracy on validation set\n",
      "Got 742 / 1000 correct (74.20)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.5497\n",
      "Checking accuracy on validation set\n",
      "Got 735 / 1000 correct (73.50)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.5489\n",
      "Checking accuracy on validation set\n",
      "Got 744 / 1000 correct (74.40)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.6409\n",
      "Checking accuracy on validation set\n",
      "Got 765 / 1000 correct (76.50)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.4601\n",
      "Checking accuracy on validation set\n",
      "Got 750 / 1000 correct (75.00)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.6843\n",
      "Checking accuracy on validation set\n",
      "Got 721 / 1000 correct (72.10)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.6252\n",
      "Checking accuracy on validation set\n",
      "Got 759 / 1000 correct (75.90)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.3443\n",
      "Checking accuracy on validation set\n",
      "Got 750 / 1000 correct (75.00)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.3638\n",
      "Checking accuracy on validation set\n",
      "Got 755 / 1000 correct (75.50)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.4753\n",
      "Checking accuracy on validation set\n",
      "Got 758 / 1000 correct (75.80)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.3946\n",
      "Checking accuracy on validation set\n",
      "Got 755 / 1000 correct (75.50)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.7004\n",
      "Checking accuracy on validation set\n",
      "Got 751 / 1000 correct (75.10)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.6234\n",
      "Checking accuracy on validation set\n",
      "Got 759 / 1000 correct (75.90)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.5132\n",
      "Checking accuracy on validation set\n",
      "Got 757 / 1000 correct (75.70)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.5515\n",
      "Checking accuracy on validation set\n",
      "Got 762 / 1000 correct (76.20)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.5907\n",
      "Checking accuracy on validation set\n",
      "Got 747 / 1000 correct (74.70)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.4528\n",
      "Checking accuracy on validation set\n",
      "Got 759 / 1000 correct (75.90)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.3539\n",
      "Checking accuracy on validation set\n",
      "Got 751 / 1000 correct (75.10)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.6163\n",
      "Checking accuracy on validation set\n",
      "Got 743 / 1000 correct (74.30)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.3798\n",
      "Checking accuracy on validation set\n",
      "Got 778 / 1000 correct (77.80)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.4376\n",
      "Checking accuracy on validation set\n",
      "Got 763 / 1000 correct (76.30)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.5794\n",
      "Checking accuracy on validation set\n",
      "Got 758 / 1000 correct (75.80)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.4897\n",
      "Checking accuracy on validation set\n",
      "Got 761 / 1000 correct (76.10)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.4165\n",
      "Checking accuracy on validation set\n",
      "Got 763 / 1000 correct (76.30)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.3187\n",
      "Checking accuracy on validation set\n",
      "Got 769 / 1000 correct (76.90)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.3098\n",
      "Checking accuracy on validation set\n",
      "Got 758 / 1000 correct (75.80)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.4035\n",
      "Checking accuracy on validation set\n",
      "Got 764 / 1000 correct (76.40)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.2689\n",
      "Checking accuracy on validation set\n",
      "Got 746 / 1000 correct (74.60)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.2688\n",
      "Checking accuracy on validation set\n",
      "Got 761 / 1000 correct (76.10)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.4768\n",
      "Checking accuracy on validation set\n",
      "Got 747 / 1000 correct (74.70)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.4813\n",
      "Checking accuracy on validation set\n",
      "Got 747 / 1000 correct (74.70)\n",
      "\n",
      "Maximum accuracy attained:  77.8\n"
     ]
    }
   ],
   "source": [
    "learning_rate = 1e-3\n",
    "\n",
    "model = ResNet10()\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "train(model, optimizer, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking accuracy on test set\n",
      "Got 7494 / 10000 correct (74.94)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "74.94"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vanillaResnet = model\n",
    "check_accuracy(loader_test, vanillaResnet)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resnet with Attention "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, Iteration 0, loss = 4.6522\n",
      "Checking accuracy on validation set\n",
      "Got 113 / 1000 correct (11.30)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.6097\n",
      "Checking accuracy on validation set\n",
      "Got 367 / 1000 correct (36.70)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.5243\n",
      "Checking accuracy on validation set\n",
      "Got 464 / 1000 correct (46.40)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.2668\n",
      "Checking accuracy on validation set\n",
      "Got 505 / 1000 correct (50.50)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.3435\n",
      "Checking accuracy on validation set\n",
      "Got 459 / 1000 correct (45.90)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.1794\n",
      "Checking accuracy on validation set\n",
      "Got 541 / 1000 correct (54.10)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.1000\n",
      "Checking accuracy on validation set\n",
      "Got 569 / 1000 correct (56.90)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.1838\n",
      "Checking accuracy on validation set\n",
      "Got 559 / 1000 correct (55.90)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 1.1745\n",
      "Checking accuracy on validation set\n",
      "Got 580 / 1000 correct (58.00)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 1.2302\n",
      "Checking accuracy on validation set\n",
      "Got 562 / 1000 correct (56.20)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 1.0572\n",
      "Checking accuracy on validation set\n",
      "Got 618 / 1000 correct (61.80)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 0.8613\n",
      "Checking accuracy on validation set\n",
      "Got 617 / 1000 correct (61.70)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 1.0495\n",
      "Checking accuracy on validation set\n",
      "Got 599 / 1000 correct (59.90)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 1.1262\n",
      "Checking accuracy on validation set\n",
      "Got 632 / 1000 correct (63.20)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 1.0978\n",
      "Checking accuracy on validation set\n",
      "Got 558 / 1000 correct (55.80)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 1.2042\n",
      "Checking accuracy on validation set\n",
      "Got 623 / 1000 correct (62.30)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 0.9687\n",
      "Checking accuracy on validation set\n",
      "Got 636 / 1000 correct (63.60)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 0.8709\n",
      "Checking accuracy on validation set\n",
      "Got 678 / 1000 correct (67.80)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.8298\n",
      "Checking accuracy on validation set\n",
      "Got 666 / 1000 correct (66.60)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.8347\n",
      "Checking accuracy on validation set\n",
      "Got 649 / 1000 correct (64.90)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 0.7688\n",
      "Checking accuracy on validation set\n",
      "Got 663 / 1000 correct (66.30)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 0.8397\n",
      "Checking accuracy on validation set\n",
      "Got 676 / 1000 correct (67.60)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 1.0610\n",
      "Checking accuracy on validation set\n",
      "Got 667 / 1000 correct (66.70)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.8408\n",
      "Checking accuracy on validation set\n",
      "Got 687 / 1000 correct (68.70)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.6558\n",
      "Checking accuracy on validation set\n",
      "Got 683 / 1000 correct (68.30)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.9094\n",
      "Checking accuracy on validation set\n",
      "Got 708 / 1000 correct (70.80)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.8993\n",
      "Checking accuracy on validation set\n",
      "Got 701 / 1000 correct (70.10)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 1.0125\n",
      "Checking accuracy on validation set\n",
      "Got 701 / 1000 correct (70.10)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.9695\n",
      "Checking accuracy on validation set\n",
      "Got 683 / 1000 correct (68.30)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.7872\n",
      "Checking accuracy on validation set\n",
      "Got 691 / 1000 correct (69.10)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.9011\n",
      "Checking accuracy on validation set\n",
      "Got 736 / 1000 correct (73.60)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.8339\n",
      "Checking accuracy on validation set\n",
      "Got 738 / 1000 correct (73.80)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.9282\n",
      "Checking accuracy on validation set\n",
      "Got 725 / 1000 correct (72.50)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.7451\n",
      "Checking accuracy on validation set\n",
      "Got 714 / 1000 correct (71.40)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.4722\n",
      "Checking accuracy on validation set\n",
      "Got 720 / 1000 correct (72.00)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.6084\n",
      "Checking accuracy on validation set\n",
      "Got 716 / 1000 correct (71.60)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.5177\n",
      "Checking accuracy on validation set\n",
      "Got 724 / 1000 correct (72.40)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.8156\n",
      "Checking accuracy on validation set\n",
      "Got 750 / 1000 correct (75.00)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.6929\n",
      "Checking accuracy on validation set\n",
      "Got 718 / 1000 correct (71.80)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.8129\n",
      "Checking accuracy on validation set\n",
      "Got 721 / 1000 correct (72.10)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.6361\n",
      "Checking accuracy on validation set\n",
      "Got 735 / 1000 correct (73.50)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.6535\n",
      "Checking accuracy on validation set\n",
      "Got 741 / 1000 correct (74.10)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.6891\n",
      "Checking accuracy on validation set\n",
      "Got 751 / 1000 correct (75.10)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.6069\n",
      "Checking accuracy on validation set\n",
      "Got 748 / 1000 correct (74.80)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.5061\n",
      "Checking accuracy on validation set\n",
      "Got 757 / 1000 correct (75.70)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.5045\n",
      "Checking accuracy on validation set\n",
      "Got 749 / 1000 correct (74.90)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.6014\n",
      "Checking accuracy on validation set\n",
      "Got 743 / 1000 correct (74.30)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.5241\n",
      "Checking accuracy on validation set\n",
      "Got 734 / 1000 correct (73.40)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.6219\n",
      "Checking accuracy on validation set\n",
      "Got 737 / 1000 correct (73.70)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.3847\n",
      "Checking accuracy on validation set\n",
      "Got 754 / 1000 correct (75.40)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.4181\n",
      "Checking accuracy on validation set\n",
      "Got 749 / 1000 correct (74.90)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.7566\n",
      "Checking accuracy on validation set\n",
      "Got 747 / 1000 correct (74.70)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.6267\n",
      "Checking accuracy on validation set\n",
      "Got 738 / 1000 correct (73.80)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.6739\n",
      "Checking accuracy on validation set\n",
      "Got 759 / 1000 correct (75.90)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.6253\n",
      "Checking accuracy on validation set\n",
      "Got 714 / 1000 correct (71.40)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.5801\n",
      "Checking accuracy on validation set\n",
      "Got 752 / 1000 correct (75.20)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.3989\n",
      "Checking accuracy on validation set\n",
      "Got 748 / 1000 correct (74.80)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.3928\n",
      "Checking accuracy on validation set\n",
      "Got 777 / 1000 correct (77.70)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.3798\n",
      "Checking accuracy on validation set\n",
      "Got 756 / 1000 correct (75.60)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.6683\n",
      "Checking accuracy on validation set\n",
      "Got 754 / 1000 correct (75.40)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.4259\n",
      "Checking accuracy on validation set\n",
      "Got 765 / 1000 correct (76.50)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.4231\n",
      "Checking accuracy on validation set\n",
      "Got 757 / 1000 correct (75.70)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.4280\n",
      "Checking accuracy on validation set\n",
      "Got 752 / 1000 correct (75.20)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.3344\n",
      "Checking accuracy on validation set\n",
      "Got 762 / 1000 correct (76.20)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.2015\n",
      "Checking accuracy on validation set\n",
      "Got 776 / 1000 correct (77.60)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.2137\n",
      "Checking accuracy on validation set\n",
      "Got 772 / 1000 correct (77.20)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.3168\n",
      "Checking accuracy on validation set\n",
      "Got 779 / 1000 correct (77.90)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.3979\n",
      "Checking accuracy on validation set\n",
      "Got 752 / 1000 correct (75.20)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.3253\n",
      "Checking accuracy on validation set\n",
      "Got 740 / 1000 correct (74.00)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.2051\n",
      "Checking accuracy on validation set\n",
      "Got 770 / 1000 correct (77.00)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.3719\n",
      "Checking accuracy on validation set\n",
      "Got 761 / 1000 correct (76.10)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.4083\n",
      "Checking accuracy on validation set\n",
      "Got 771 / 1000 correct (77.10)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.2891\n",
      "Checking accuracy on validation set\n",
      "Got 722 / 1000 correct (72.20)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.2693\n",
      "Checking accuracy on validation set\n",
      "Got 764 / 1000 correct (76.40)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.3063\n",
      "Checking accuracy on validation set\n",
      "Got 768 / 1000 correct (76.80)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.5492\n",
      "Checking accuracy on validation set\n",
      "Got 772 / 1000 correct (77.20)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.2843\n",
      "Checking accuracy on validation set\n",
      "Got 761 / 1000 correct (76.10)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.5319\n",
      "Checking accuracy on validation set\n",
      "Got 755 / 1000 correct (75.50)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.4500\n",
      "Checking accuracy on validation set\n",
      "Got 760 / 1000 correct (76.00)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.4244\n",
      "Checking accuracy on validation set\n",
      "Got 767 / 1000 correct (76.70)\n",
      "\n",
      "Maximum accuracy attained:  77.9\n"
     ]
    }
   ],
   "source": [
    "## Resnet with Attention\n",
    "\n",
    "learning_rate = 1e-3\n",
    "\n",
    "model = ResNet10()\n",
    "model.layer2.add_module('attention', Attention(128))\n",
    "model.layer2.add_module('relu', nn.ReLU())\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "train(model, optimizer, epochs=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test set "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking accuracy on test set\n",
      "Got 7636 / 10000 correct (76.36)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "76.36"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AttentionResnet = model\n",
    "check_accuracy(loader_test, AttentionResnet)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ranking the above models based on their performance on test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model_def(model_name):\n",
    "    \n",
    "    if model_name == 'vanilla_cnn':\n",
    "        \n",
    "        return nn.Sequential(\n",
    "                        nn.Conv2d(3, channel_1, 3, padding=1, stride=1),\n",
    "                        nn.ReLU(),\n",
    "                        nn.Conv2d(channel_1, channel_2, 3, padding=1),\n",
    "                        nn.ReLU(),\n",
    "                        Flatten(),\n",
    "                        nn.Linear(channel_2*32*32, num_classes),\n",
    "                    )\n",
    "    \n",
    "    elif model_name == 'early_attention': \n",
    "        \n",
    "        return nn.Sequential(\n",
    "                            nn.Conv2d(3, channel_1, 3, padding=1, stride=1),\n",
    "                            nn.ReLU(),\n",
    "                            Attention(channel_1),\n",
    "                            nn.ReLU(),\n",
    "                            nn.Conv2d(channel_1, channel_2, 3, padding=1),\n",
    "                            nn.ReLU(),\n",
    "                            Flatten(),\n",
    "                            nn.Linear(channel_2*32*32, num_classes),\n",
    "                        )\n",
    "    \n",
    "    elif model_name == 'late_attention': \n",
    "        \n",
    "        return nn.Sequential(\n",
    "                            nn.Conv2d(3, channel_1, 3, padding=1, stride=1),\n",
    "                            nn.ReLU(),\n",
    "                            nn.Conv2d(channel_1, channel_2, 3, padding=1),\n",
    "                            nn.ReLU(),\n",
    "                            Attention(channel_2),\n",
    "                            nn.ReLU(),\n",
    "                            Flatten(),\n",
    "                            nn.Linear(channel_2*32*32, num_classes),\n",
    "                        )\n",
    "    \n",
    "    elif model_name == 'double_attention': \n",
    "        \n",
    "        return nn.Sequential(\n",
    "                            nn.Conv2d(3, channel_1, 3, padding=1, stride=1),\n",
    "                            nn.ReLU(),\n",
    "                            Attention(channel_1),\n",
    "                            nn.ReLU(),\n",
    "                            nn.Conv2d(channel_1, channel_2, 3, padding=1),\n",
    "                            nn.ReLU(),\n",
    "                            Attention(channel_2),\n",
    "                            nn.ReLU(),\n",
    "                            Flatten(),\n",
    "                            nn.Linear(channel_2*32*32, num_classes),\n",
    "                        )\n",
    "    \n",
    "    elif model_name == 'vanilla_resnet': \n",
    "        \n",
    "        return ResNet10()\n",
    "    \n",
    "    elif model_name == 'resnet_w_attention': \n",
    "        \n",
    "        model = ResNet10()\n",
    "        model.layer2.add_module('attention', Attention(128))\n",
    "        model.layer2.add_module('relu', nn.ReLU())\n",
    "        \n",
    "        return model\n",
    "    \n",
    "    else:\n",
    "        print(\"Invalid Model Name\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, Iteration 0, loss = 2.3181\n",
      "Checking accuracy on validation set\n",
      "Got 148 / 1000 correct (14.80)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.7987\n",
      "Checking accuracy on validation set\n",
      "Got 407 / 1000 correct (40.70)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.6420\n",
      "Checking accuracy on validation set\n",
      "Got 460 / 1000 correct (46.00)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.5291\n",
      "Checking accuracy on validation set\n",
      "Got 472 / 1000 correct (47.20)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.3316\n",
      "Checking accuracy on validation set\n",
      "Got 524 / 1000 correct (52.40)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.0969\n",
      "Checking accuracy on validation set\n",
      "Got 544 / 1000 correct (54.40)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.3414\n",
      "Checking accuracy on validation set\n",
      "Got 550 / 1000 correct (55.00)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.2150\n",
      "Checking accuracy on validation set\n",
      "Got 559 / 1000 correct (55.90)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 1.1066\n",
      "Checking accuracy on validation set\n",
      "Got 562 / 1000 correct (56.20)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 1.1401\n",
      "Checking accuracy on validation set\n",
      "Got 592 / 1000 correct (59.20)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 1.1257\n",
      "Checking accuracy on validation set\n",
      "Got 600 / 1000 correct (60.00)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 0.9414\n",
      "Checking accuracy on validation set\n",
      "Got 598 / 1000 correct (59.80)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 1.1175\n",
      "Checking accuracy on validation set\n",
      "Got 575 / 1000 correct (57.50)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 1.0865\n",
      "Checking accuracy on validation set\n",
      "Got 583 / 1000 correct (58.30)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 0.8298\n",
      "Checking accuracy on validation set\n",
      "Got 611 / 1000 correct (61.10)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 1.0856\n",
      "Checking accuracy on validation set\n",
      "Got 620 / 1000 correct (62.00)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 1.0018\n",
      "Checking accuracy on validation set\n",
      "Got 613 / 1000 correct (61.30)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 0.7658\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.9901\n",
      "Checking accuracy on validation set\n",
      "Got 628 / 1000 correct (62.80)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.7965\n",
      "Checking accuracy on validation set\n",
      "Got 606 / 1000 correct (60.60)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 1.0180\n",
      "Checking accuracy on validation set\n",
      "Got 608 / 1000 correct (60.80)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 1.0808\n",
      "Checking accuracy on validation set\n",
      "Got 618 / 1000 correct (61.80)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 1.0963\n",
      "Checking accuracy on validation set\n",
      "Got 619 / 1000 correct (61.90)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.8945\n",
      "Checking accuracy on validation set\n",
      "Got 626 / 1000 correct (62.60)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.8741\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.6815\n",
      "Checking accuracy on validation set\n",
      "Got 624 / 1000 correct (62.40)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.9992\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.7398\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.6864\n",
      "Checking accuracy on validation set\n",
      "Got 623 / 1000 correct (62.30)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.9348\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.8476\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.8356\n",
      "Checking accuracy on validation set\n",
      "Got 616 / 1000 correct (61.60)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.7583\n",
      "Checking accuracy on validation set\n",
      "Got 628 / 1000 correct (62.80)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.7113\n",
      "Checking accuracy on validation set\n",
      "Got 641 / 1000 correct (64.10)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.5352\n",
      "Checking accuracy on validation set\n",
      "Got 621 / 1000 correct (62.10)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.8495\n",
      "Checking accuracy on validation set\n",
      "Got 608 / 1000 correct (60.80)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.7636\n",
      "Checking accuracy on validation set\n",
      "Got 626 / 1000 correct (62.60)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.5008\n",
      "Checking accuracy on validation set\n",
      "Got 626 / 1000 correct (62.60)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.7140\n",
      "Checking accuracy on validation set\n",
      "Got 620 / 1000 correct (62.00)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 1.0167\n",
      "Checking accuracy on validation set\n",
      "Got 627 / 1000 correct (62.70)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.7599\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.4490\n",
      "Checking accuracy on validation set\n",
      "Got 641 / 1000 correct (64.10)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.5147\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.6233\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.6212\n",
      "Checking accuracy on validation set\n",
      "Got 624 / 1000 correct (62.40)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.6141\n",
      "Checking accuracy on validation set\n",
      "Got 621 / 1000 correct (62.10)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.6914\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.4709\n",
      "Checking accuracy on validation set\n",
      "Got 619 / 1000 correct (61.90)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.3837\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.4415\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.2829\n",
      "Checking accuracy on validation set\n",
      "Got 624 / 1000 correct (62.40)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.3312\n",
      "Checking accuracy on validation set\n",
      "Got 625 / 1000 correct (62.50)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.6422\n",
      "Checking accuracy on validation set\n",
      "Got 617 / 1000 correct (61.70)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.5621\n",
      "Checking accuracy on validation set\n",
      "Got 623 / 1000 correct (62.30)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.5810\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.6090\n",
      "Checking accuracy on validation set\n",
      "Got 621 / 1000 correct (62.10)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.2434\n",
      "Checking accuracy on validation set\n",
      "Got 616 / 1000 correct (61.60)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.3549\n",
      "Checking accuracy on validation set\n",
      "Got 621 / 1000 correct (62.10)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.3464\n",
      "Checking accuracy on validation set\n",
      "Got 618 / 1000 correct (61.80)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.3333\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.3036\n",
      "Checking accuracy on validation set\n",
      "Got 624 / 1000 correct (62.40)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.2555\n",
      "Checking accuracy on validation set\n",
      "Got 609 / 1000 correct (60.90)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.6338\n",
      "Checking accuracy on validation set\n",
      "Got 626 / 1000 correct (62.60)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.5322\n",
      "Checking accuracy on validation set\n",
      "Got 609 / 1000 correct (60.90)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.1556\n",
      "Checking accuracy on validation set\n",
      "Got 615 / 1000 correct (61.50)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.2063\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.2707\n",
      "Checking accuracy on validation set\n",
      "Got 604 / 1000 correct (60.40)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.1515\n",
      "Checking accuracy on validation set\n",
      "Got 611 / 1000 correct (61.10)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.1134\n",
      "Checking accuracy on validation set\n",
      "Got 607 / 1000 correct (60.70)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.1516\n",
      "Checking accuracy on validation set\n",
      "Got 592 / 1000 correct (59.20)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.4483\n",
      "Checking accuracy on validation set\n",
      "Got 606 / 1000 correct (60.60)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.0981\n",
      "Checking accuracy on validation set\n",
      "Got 618 / 1000 correct (61.80)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.1509\n",
      "Checking accuracy on validation set\n",
      "Got 613 / 1000 correct (61.30)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.1240\n",
      "Checking accuracy on validation set\n",
      "Got 608 / 1000 correct (60.80)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.1717\n",
      "Checking accuracy on validation set\n",
      "Got 613 / 1000 correct (61.30)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.1546\n",
      "Checking accuracy on validation set\n",
      "Got 623 / 1000 correct (62.30)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.1714\n",
      "Checking accuracy on validation set\n",
      "Got 616 / 1000 correct (61.60)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.1722\n",
      "Checking accuracy on validation set\n",
      "Got 601 / 1000 correct (60.10)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.1422\n",
      "Checking accuracy on validation set\n",
      "Got 588 / 1000 correct (58.80)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.2611\n",
      "Checking accuracy on validation set\n",
      "Got 609 / 1000 correct (60.90)\n",
      "\n",
      "Maximum accuracy attained:  65.0\n",
      "Checking accuracy on test set\n",
      "Got 5879 / 10000 correct (58.79)\n",
      "Epoch 0, Iteration 0, loss = 2.3404\n",
      "Checking accuracy on validation set\n",
      "Got 79 / 1000 correct (7.90)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.4681\n",
      "Checking accuracy on validation set\n",
      "Got 420 / 1000 correct (42.00)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.6832\n",
      "Checking accuracy on validation set\n",
      "Got 481 / 1000 correct (48.10)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.4752\n",
      "Checking accuracy on validation set\n",
      "Got 523 / 1000 correct (52.30)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.3861\n",
      "Checking accuracy on validation set\n",
      "Got 518 / 1000 correct (51.80)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.1984\n",
      "Checking accuracy on validation set\n",
      "Got 528 / 1000 correct (52.80)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.2809\n",
      "Checking accuracy on validation set\n",
      "Got 564 / 1000 correct (56.40)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.5180\n",
      "Checking accuracy on validation set\n",
      "Got 557 / 1000 correct (55.70)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 1.0895\n",
      "Checking accuracy on validation set\n",
      "Got 579 / 1000 correct (57.90)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 1.3334\n",
      "Checking accuracy on validation set\n",
      "Got 552 / 1000 correct (55.20)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 1.1581\n",
      "Checking accuracy on validation set\n",
      "Got 571 / 1000 correct (57.10)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 0.9967\n",
      "Checking accuracy on validation set\n",
      "Got 554 / 1000 correct (55.40)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 1.0058\n",
      "Checking accuracy on validation set\n",
      "Got 547 / 1000 correct (54.70)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 1.1899\n",
      "Checking accuracy on validation set\n",
      "Got 559 / 1000 correct (55.90)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 0.8690\n",
      "Checking accuracy on validation set\n",
      "Got 592 / 1000 correct (59.20)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 1.0686\n",
      "Checking accuracy on validation set\n",
      "Got 595 / 1000 correct (59.50)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 1.0818\n",
      "Checking accuracy on validation set\n",
      "Got 582 / 1000 correct (58.20)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 1.0307\n",
      "Checking accuracy on validation set\n",
      "Got 589 / 1000 correct (58.90)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 1.2250\n",
      "Checking accuracy on validation set\n",
      "Got 591 / 1000 correct (59.10)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.9122\n",
      "Checking accuracy on validation set\n",
      "Got 570 / 1000 correct (57.00)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 1.1041\n",
      "Checking accuracy on validation set\n",
      "Got 598 / 1000 correct (59.80)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 0.9740\n",
      "Checking accuracy on validation set\n",
      "Got 600 / 1000 correct (60.00)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.8229\n",
      "Checking accuracy on validation set\n",
      "Got 602 / 1000 correct (60.20)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 1.1660\n",
      "Checking accuracy on validation set\n",
      "Got 606 / 1000 correct (60.60)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.8938\n",
      "Checking accuracy on validation set\n",
      "Got 584 / 1000 correct (58.40)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 1.2420\n",
      "Checking accuracy on validation set\n",
      "Got 609 / 1000 correct (60.90)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 1.1022\n",
      "Checking accuracy on validation set\n",
      "Got 601 / 1000 correct (60.10)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.7897\n",
      "Checking accuracy on validation set\n",
      "Got 609 / 1000 correct (60.90)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.9618\n",
      "Checking accuracy on validation set\n",
      "Got 625 / 1000 correct (62.50)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.8565\n",
      "Checking accuracy on validation set\n",
      "Got 618 / 1000 correct (61.80)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.9477\n",
      "Checking accuracy on validation set\n",
      "Got 608 / 1000 correct (60.80)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.6328\n",
      "Checking accuracy on validation set\n",
      "Got 600 / 1000 correct (60.00)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.6761\n",
      "Checking accuracy on validation set\n",
      "Got 618 / 1000 correct (61.80)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.7452\n",
      "Checking accuracy on validation set\n",
      "Got 609 / 1000 correct (60.90)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.6562\n",
      "Checking accuracy on validation set\n",
      "Got 609 / 1000 correct (60.90)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.7255\n",
      "Checking accuracy on validation set\n",
      "Got 606 / 1000 correct (60.60)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.7182\n",
      "Checking accuracy on validation set\n",
      "Got 599 / 1000 correct (59.90)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.9014\n",
      "Checking accuracy on validation set\n",
      "Got 608 / 1000 correct (60.80)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.9535\n",
      "Checking accuracy on validation set\n",
      "Got 600 / 1000 correct (60.00)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.7325\n",
      "Checking accuracy on validation set\n",
      "Got 609 / 1000 correct (60.90)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.5139\n",
      "Checking accuracy on validation set\n",
      "Got 598 / 1000 correct (59.80)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.4321\n",
      "Checking accuracy on validation set\n",
      "Got 586 / 1000 correct (58.60)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.5634\n",
      "Checking accuracy on validation set\n",
      "Got 605 / 1000 correct (60.50)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.7345\n",
      "Checking accuracy on validation set\n",
      "Got 603 / 1000 correct (60.30)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.5417\n",
      "Checking accuracy on validation set\n",
      "Got 595 / 1000 correct (59.50)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.5248\n",
      "Checking accuracy on validation set\n",
      "Got 592 / 1000 correct (59.20)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.4312\n",
      "Checking accuracy on validation set\n",
      "Got 598 / 1000 correct (59.80)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.6119\n",
      "Checking accuracy on validation set\n",
      "Got 607 / 1000 correct (60.70)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.3306\n",
      "Checking accuracy on validation set\n",
      "Got 604 / 1000 correct (60.40)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.4862\n",
      "Checking accuracy on validation set\n",
      "Got 605 / 1000 correct (60.50)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.4354\n",
      "Checking accuracy on validation set\n",
      "Got 592 / 1000 correct (59.20)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.4023\n",
      "Checking accuracy on validation set\n",
      "Got 587 / 1000 correct (58.70)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.7122\n",
      "Checking accuracy on validation set\n",
      "Got 593 / 1000 correct (59.30)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.4144\n",
      "Checking accuracy on validation set\n",
      "Got 598 / 1000 correct (59.80)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.6997\n",
      "Checking accuracy on validation set\n",
      "Got 597 / 1000 correct (59.70)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.6885\n",
      "Checking accuracy on validation set\n",
      "Got 599 / 1000 correct (59.90)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.4139\n",
      "Checking accuracy on validation set\n",
      "Got 586 / 1000 correct (58.60)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.3839\n",
      "Checking accuracy on validation set\n",
      "Got 601 / 1000 correct (60.10)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.4630\n",
      "Checking accuracy on validation set\n",
      "Got 597 / 1000 correct (59.70)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.4746\n",
      "Checking accuracy on validation set\n",
      "Got 590 / 1000 correct (59.00)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.3918\n",
      "Checking accuracy on validation set\n",
      "Got 601 / 1000 correct (60.10)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.5955\n",
      "Checking accuracy on validation set\n",
      "Got 593 / 1000 correct (59.30)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.7104\n",
      "Checking accuracy on validation set\n",
      "Got 585 / 1000 correct (58.50)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.6928\n",
      "Checking accuracy on validation set\n",
      "Got 572 / 1000 correct (57.20)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.3185\n",
      "Checking accuracy on validation set\n",
      "Got 590 / 1000 correct (59.00)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.3008\n",
      "Checking accuracy on validation set\n",
      "Got 595 / 1000 correct (59.50)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.4555\n",
      "Checking accuracy on validation set\n",
      "Got 594 / 1000 correct (59.40)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.3421\n",
      "Checking accuracy on validation set\n",
      "Got 587 / 1000 correct (58.70)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.5006\n",
      "Checking accuracy on validation set\n",
      "Got 590 / 1000 correct (59.00)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.1862\n",
      "Checking accuracy on validation set\n",
      "Got 597 / 1000 correct (59.70)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.4318\n",
      "Checking accuracy on validation set\n",
      "Got 582 / 1000 correct (58.20)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.4515\n",
      "Checking accuracy on validation set\n",
      "Got 575 / 1000 correct (57.50)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.3189\n",
      "Checking accuracy on validation set\n",
      "Got 588 / 1000 correct (58.80)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.2455\n",
      "Checking accuracy on validation set\n",
      "Got 583 / 1000 correct (58.30)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.1975\n",
      "Checking accuracy on validation set\n",
      "Got 581 / 1000 correct (58.10)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.3304\n",
      "Checking accuracy on validation set\n",
      "Got 573 / 1000 correct (57.30)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.3533\n",
      "Checking accuracy on validation set\n",
      "Got 582 / 1000 correct (58.20)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.4503\n",
      "Checking accuracy on validation set\n",
      "Got 581 / 1000 correct (58.10)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.7514\n",
      "Checking accuracy on validation set\n",
      "Got 589 / 1000 correct (58.90)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.3440\n",
      "Checking accuracy on validation set\n",
      "Got 605 / 1000 correct (60.50)\n",
      "\n",
      "Maximum accuracy attained:  62.5\n",
      "Checking accuracy on test set\n",
      "Got 5649 / 10000 correct (56.49)\n",
      "Epoch 0, Iteration 0, loss = 2.3211\n",
      "Checking accuracy on validation set\n",
      "Got 142 / 1000 correct (14.20)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.5029\n",
      "Checking accuracy on validation set\n",
      "Got 359 / 1000 correct (35.90)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.8716\n",
      "Checking accuracy on validation set\n",
      "Got 464 / 1000 correct (46.40)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.5599\n",
      "Checking accuracy on validation set\n",
      "Got 478 / 1000 correct (47.80)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.5632\n",
      "Checking accuracy on validation set\n",
      "Got 508 / 1000 correct (50.80)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.5019\n",
      "Checking accuracy on validation set\n",
      "Got 506 / 1000 correct (50.60)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.2307\n",
      "Checking accuracy on validation set\n",
      "Got 510 / 1000 correct (51.00)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.1431\n",
      "Checking accuracy on validation set\n",
      "Got 526 / 1000 correct (52.60)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 1.2351\n",
      "Checking accuracy on validation set\n",
      "Got 525 / 1000 correct (52.50)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 1.5400\n",
      "Checking accuracy on validation set\n",
      "Got 523 / 1000 correct (52.30)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 1.1946\n",
      "Checking accuracy on validation set\n",
      "Got 531 / 1000 correct (53.10)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 1.7057\n",
      "Checking accuracy on validation set\n",
      "Got 561 / 1000 correct (56.10)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 1.2747\n",
      "Checking accuracy on validation set\n",
      "Got 525 / 1000 correct (52.50)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 1.4418\n",
      "Checking accuracy on validation set\n",
      "Got 551 / 1000 correct (55.10)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 1.2747\n",
      "Checking accuracy on validation set\n",
      "Got 555 / 1000 correct (55.50)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 1.0828\n",
      "Checking accuracy on validation set\n",
      "Got 577 / 1000 correct (57.70)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 1.2794\n",
      "Checking accuracy on validation set\n",
      "Got 556 / 1000 correct (55.60)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 0.7789\n",
      "Checking accuracy on validation set\n",
      "Got 585 / 1000 correct (58.50)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 1.1922\n",
      "Checking accuracy on validation set\n",
      "Got 566 / 1000 correct (56.60)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.8758\n",
      "Checking accuracy on validation set\n",
      "Got 585 / 1000 correct (58.50)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 1.0956\n",
      "Checking accuracy on validation set\n",
      "Got 579 / 1000 correct (57.90)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 1.0317\n",
      "Checking accuracy on validation set\n",
      "Got 603 / 1000 correct (60.30)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.7587\n",
      "Checking accuracy on validation set\n",
      "Got 594 / 1000 correct (59.40)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.9760\n",
      "Checking accuracy on validation set\n",
      "Got 610 / 1000 correct (61.00)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 1.0328\n",
      "Checking accuracy on validation set\n",
      "Got 618 / 1000 correct (61.80)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.9048\n",
      "Checking accuracy on validation set\n",
      "Got 615 / 1000 correct (61.50)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.9260\n",
      "Checking accuracy on validation set\n",
      "Got 611 / 1000 correct (61.10)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 1.1825\n",
      "Checking accuracy on validation set\n",
      "Got 594 / 1000 correct (59.40)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.9598\n",
      "Checking accuracy on validation set\n",
      "Got 593 / 1000 correct (59.30)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.8950\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 1.0226\n",
      "Checking accuracy on validation set\n",
      "Got 611 / 1000 correct (61.10)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 1.0053\n",
      "Checking accuracy on validation set\n",
      "Got 619 / 1000 correct (61.90)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.8020\n",
      "Checking accuracy on validation set\n",
      "Got 604 / 1000 correct (60.40)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.5358\n",
      "Checking accuracy on validation set\n",
      "Got 607 / 1000 correct (60.70)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.9785\n",
      "Checking accuracy on validation set\n",
      "Got 602 / 1000 correct (60.20)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.7801\n",
      "Checking accuracy on validation set\n",
      "Got 616 / 1000 correct (61.60)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.6499\n",
      "Checking accuracy on validation set\n",
      "Got 615 / 1000 correct (61.50)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.7226\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.6210\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.9565\n",
      "Checking accuracy on validation set\n",
      "Got 611 / 1000 correct (61.10)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.3933\n",
      "Checking accuracy on validation set\n",
      "Got 640 / 1000 correct (64.00)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.6615\n",
      "Checking accuracy on validation set\n",
      "Got 614 / 1000 correct (61.40)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.4172\n",
      "Checking accuracy on validation set\n",
      "Got 634 / 1000 correct (63.40)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.5059\n",
      "Checking accuracy on validation set\n",
      "Got 603 / 1000 correct (60.30)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.7557\n",
      "Checking accuracy on validation set\n",
      "Got 618 / 1000 correct (61.80)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.6716\n",
      "Checking accuracy on validation set\n",
      "Got 613 / 1000 correct (61.30)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.6436\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.8404\n",
      "Checking accuracy on validation set\n",
      "Got 609 / 1000 correct (60.90)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.3941\n",
      "Checking accuracy on validation set\n",
      "Got 620 / 1000 correct (62.00)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.4547\n",
      "Checking accuracy on validation set\n",
      "Got 628 / 1000 correct (62.80)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.4715\n",
      "Checking accuracy on validation set\n",
      "Got 620 / 1000 correct (62.00)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.5445\n",
      "Checking accuracy on validation set\n",
      "Got 611 / 1000 correct (61.10)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.4345\n",
      "Checking accuracy on validation set\n",
      "Got 589 / 1000 correct (58.90)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.6202\n",
      "Checking accuracy on validation set\n",
      "Got 617 / 1000 correct (61.70)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.4227\n",
      "Checking accuracy on validation set\n",
      "Got 617 / 1000 correct (61.70)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.5277\n",
      "Checking accuracy on validation set\n",
      "Got 607 / 1000 correct (60.70)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.3001\n",
      "Checking accuracy on validation set\n",
      "Got 596 / 1000 correct (59.60)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.2720\n",
      "Checking accuracy on validation set\n",
      "Got 621 / 1000 correct (62.10)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.4943\n",
      "Checking accuracy on validation set\n",
      "Got 618 / 1000 correct (61.80)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.5196\n",
      "Checking accuracy on validation set\n",
      "Got 607 / 1000 correct (60.70)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.6775\n",
      "Checking accuracy on validation set\n",
      "Got 606 / 1000 correct (60.60)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.4333\n",
      "Checking accuracy on validation set\n",
      "Got 609 / 1000 correct (60.90)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.4610\n",
      "Checking accuracy on validation set\n",
      "Got 599 / 1000 correct (59.90)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.5043\n",
      "Checking accuracy on validation set\n",
      "Got 615 / 1000 correct (61.50)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.3329\n",
      "Checking accuracy on validation set\n",
      "Got 614 / 1000 correct (61.40)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.3392\n",
      "Checking accuracy on validation set\n",
      "Got 605 / 1000 correct (60.50)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.3440\n",
      "Checking accuracy on validation set\n",
      "Got 590 / 1000 correct (59.00)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.3466\n",
      "Checking accuracy on validation set\n",
      "Got 603 / 1000 correct (60.30)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.3980\n",
      "Checking accuracy on validation set\n",
      "Got 599 / 1000 correct (59.90)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.4325\n",
      "Checking accuracy on validation set\n",
      "Got 568 / 1000 correct (56.80)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.5074\n",
      "Checking accuracy on validation set\n",
      "Got 593 / 1000 correct (59.30)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.4655\n",
      "Checking accuracy on validation set\n",
      "Got 614 / 1000 correct (61.40)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.1922\n",
      "Checking accuracy on validation set\n",
      "Got 597 / 1000 correct (59.70)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.3912\n",
      "Checking accuracy on validation set\n",
      "Got 605 / 1000 correct (60.50)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.2682\n",
      "Checking accuracy on validation set\n",
      "Got 593 / 1000 correct (59.30)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.3395\n",
      "Checking accuracy on validation set\n",
      "Got 609 / 1000 correct (60.90)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.3604\n",
      "Checking accuracy on validation set\n",
      "Got 600 / 1000 correct (60.00)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.3321\n",
      "Checking accuracy on validation set\n",
      "Got 598 / 1000 correct (59.80)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.2609\n",
      "Checking accuracy on validation set\n",
      "Got 578 / 1000 correct (57.80)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.4078\n",
      "Checking accuracy on validation set\n",
      "Got 595 / 1000 correct (59.50)\n",
      "\n",
      "Maximum accuracy attained:  64.0\n",
      "Checking accuracy on test set\n",
      "Got 5880 / 10000 correct (58.80)\n",
      "Epoch 0, Iteration 0, loss = 2.2905\n",
      "Checking accuracy on validation set\n",
      "Got 180 / 1000 correct (18.00)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.6649\n",
      "Checking accuracy on validation set\n",
      "Got 436 / 1000 correct (43.60)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.3127\n",
      "Checking accuracy on validation set\n",
      "Got 476 / 1000 correct (47.60)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.4770\n",
      "Checking accuracy on validation set\n",
      "Got 514 / 1000 correct (51.40)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.3378\n",
      "Checking accuracy on validation set\n",
      "Got 548 / 1000 correct (54.80)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.2682\n",
      "Checking accuracy on validation set\n",
      "Got 559 / 1000 correct (55.90)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 0.9900\n",
      "Checking accuracy on validation set\n",
      "Got 550 / 1000 correct (55.00)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.0306\n",
      "Checking accuracy on validation set\n",
      "Got 593 / 1000 correct (59.30)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 0.9568\n",
      "Checking accuracy on validation set\n",
      "Got 600 / 1000 correct (60.00)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 0.9977\n",
      "Checking accuracy on validation set\n",
      "Got 626 / 1000 correct (62.60)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 0.6910\n",
      "Checking accuracy on validation set\n",
      "Got 607 / 1000 correct (60.70)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 1.1226\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 1.1462\n",
      "Checking accuracy on validation set\n",
      "Got 620 / 1000 correct (62.00)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 1.0019\n",
      "Checking accuracy on validation set\n",
      "Got 635 / 1000 correct (63.50)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 1.0484\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 1.0346\n",
      "Checking accuracy on validation set\n",
      "Got 655 / 1000 correct (65.50)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 0.7398\n",
      "Checking accuracy on validation set\n",
      "Got 649 / 1000 correct (64.90)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 0.6576\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.8647\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 1.2697\n",
      "Checking accuracy on validation set\n",
      "Got 649 / 1000 correct (64.90)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 0.8834\n",
      "Checking accuracy on validation set\n",
      "Got 635 / 1000 correct (63.50)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 0.7843\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.7192\n",
      "Checking accuracy on validation set\n",
      "Got 625 / 1000 correct (62.50)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.7293\n",
      "Checking accuracy on validation set\n",
      "Got 649 / 1000 correct (64.90)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.7850\n",
      "Checking accuracy on validation set\n",
      "Got 653 / 1000 correct (65.30)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.5900\n",
      "Checking accuracy on validation set\n",
      "Got 637 / 1000 correct (63.70)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.6260\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.5484\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.6391\n",
      "Checking accuracy on validation set\n",
      "Got 640 / 1000 correct (64.00)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.7136\n",
      "Checking accuracy on validation set\n",
      "Got 632 / 1000 correct (63.20)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.7136\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.9637\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.5866\n",
      "Checking accuracy on validation set\n",
      "Got 664 / 1000 correct (66.40)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.4246\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.6057\n",
      "Checking accuracy on validation set\n",
      "Got 635 / 1000 correct (63.50)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.3705\n",
      "Checking accuracy on validation set\n",
      "Got 625 / 1000 correct (62.50)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.5090\n",
      "Checking accuracy on validation set\n",
      "Got 653 / 1000 correct (65.30)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.7181\n",
      "Checking accuracy on validation set\n",
      "Got 657 / 1000 correct (65.70)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.4038\n",
      "Checking accuracy on validation set\n",
      "Got 653 / 1000 correct (65.30)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.2916\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.3670\n",
      "Checking accuracy on validation set\n",
      "Got 655 / 1000 correct (65.50)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.2934\n",
      "Checking accuracy on validation set\n",
      "Got 645 / 1000 correct (64.50)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.4659\n",
      "Checking accuracy on validation set\n",
      "Got 635 / 1000 correct (63.50)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.1750\n",
      "Checking accuracy on validation set\n",
      "Got 628 / 1000 correct (62.80)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.5659\n",
      "Checking accuracy on validation set\n",
      "Got 642 / 1000 correct (64.20)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.5024\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.4641\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.4400\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.2039\n",
      "Checking accuracy on validation set\n",
      "Got 630 / 1000 correct (63.00)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.1222\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.1121\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.1695\n",
      "Checking accuracy on validation set\n",
      "Got 640 / 1000 correct (64.00)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.2679\n",
      "Checking accuracy on validation set\n",
      "Got 635 / 1000 correct (63.50)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.2305\n",
      "Checking accuracy on validation set\n",
      "Got 628 / 1000 correct (62.80)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.1877\n",
      "Checking accuracy on validation set\n",
      "Got 627 / 1000 correct (62.70)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.2554\n",
      "Checking accuracy on validation set\n",
      "Got 634 / 1000 correct (63.40)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.1447\n",
      "Checking accuracy on validation set\n",
      "Got 632 / 1000 correct (63.20)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.0680\n",
      "Checking accuracy on validation set\n",
      "Got 620 / 1000 correct (62.00)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.0989\n",
      "Checking accuracy on validation set\n",
      "Got 628 / 1000 correct (62.80)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.1236\n",
      "Checking accuracy on validation set\n",
      "Got 631 / 1000 correct (63.10)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.1386\n",
      "Checking accuracy on validation set\n",
      "Got 636 / 1000 correct (63.60)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.1049\n",
      "Checking accuracy on validation set\n",
      "Got 628 / 1000 correct (62.80)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.0973\n",
      "Checking accuracy on validation set\n",
      "Got 631 / 1000 correct (63.10)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.1730\n",
      "Checking accuracy on validation set\n",
      "Got 616 / 1000 correct (61.60)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.0913\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.0709\n",
      "Checking accuracy on validation set\n",
      "Got 631 / 1000 correct (63.10)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.0623\n",
      "Checking accuracy on validation set\n",
      "Got 630 / 1000 correct (63.00)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.0567\n",
      "Checking accuracy on validation set\n",
      "Got 628 / 1000 correct (62.80)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.1017\n",
      "Checking accuracy on validation set\n",
      "Got 620 / 1000 correct (62.00)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.0797\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.0915\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.0609\n",
      "Checking accuracy on validation set\n",
      "Got 627 / 1000 correct (62.70)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.2210\n",
      "Checking accuracy on validation set\n",
      "Got 623 / 1000 correct (62.30)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.0529\n",
      "Checking accuracy on validation set\n",
      "Got 631 / 1000 correct (63.10)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.0493\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.1159\n",
      "Checking accuracy on validation set\n",
      "Got 628 / 1000 correct (62.80)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.0932\n",
      "Checking accuracy on validation set\n",
      "Got 603 / 1000 correct (60.30)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.1420\n",
      "Checking accuracy on validation set\n",
      "Got 621 / 1000 correct (62.10)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.1552\n",
      "Checking accuracy on validation set\n",
      "Got 606 / 1000 correct (60.60)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.1104\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Maximum accuracy attained:  66.4\n",
      "Checking accuracy on test set\n",
      "Got 6109 / 10000 correct (61.09)\n",
      "Epoch 0, Iteration 0, loss = 2.3044\n",
      "Checking accuracy on validation set\n",
      "Got 123 / 1000 correct (12.30)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.5609\n",
      "Checking accuracy on validation set\n",
      "Got 461 / 1000 correct (46.10)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.4211\n",
      "Checking accuracy on validation set\n",
      "Got 472 / 1000 correct (47.20)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.4801\n",
      "Checking accuracy on validation set\n",
      "Got 525 / 1000 correct (52.50)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.1089\n",
      "Checking accuracy on validation set\n",
      "Got 566 / 1000 correct (56.60)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.1597\n",
      "Checking accuracy on validation set\n",
      "Got 582 / 1000 correct (58.20)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.3717\n",
      "Checking accuracy on validation set\n",
      "Got 582 / 1000 correct (58.20)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.1822\n",
      "Checking accuracy on validation set\n",
      "Got 626 / 1000 correct (62.60)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 0.9278\n",
      "Checking accuracy on validation set\n",
      "Got 598 / 1000 correct (59.80)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 1.1526\n",
      "Checking accuracy on validation set\n",
      "Got 619 / 1000 correct (61.90)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 1.1113\n",
      "Checking accuracy on validation set\n",
      "Got 626 / 1000 correct (62.60)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 0.8836\n",
      "Checking accuracy on validation set\n",
      "Got 626 / 1000 correct (62.60)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 0.9425\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 0.9950\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 0.8604\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 0.9877\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 0.6832\n",
      "Checking accuracy on validation set\n",
      "Got 671 / 1000 correct (67.10)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 1.0084\n",
      "Checking accuracy on validation set\n",
      "Got 620 / 1000 correct (62.00)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.8814\n",
      "Checking accuracy on validation set\n",
      "Got 653 / 1000 correct (65.30)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.4020\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 0.9101\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 0.8931\n",
      "Checking accuracy on validation set\n",
      "Got 665 / 1000 correct (66.50)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.6509\n",
      "Checking accuracy on validation set\n",
      "Got 671 / 1000 correct (67.10)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.5128\n",
      "Checking accuracy on validation set\n",
      "Got 655 / 1000 correct (65.50)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.4451\n",
      "Checking accuracy on validation set\n",
      "Got 668 / 1000 correct (66.80)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.4819\n",
      "Checking accuracy on validation set\n",
      "Got 656 / 1000 correct (65.60)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.8755\n",
      "Checking accuracy on validation set\n",
      "Got 645 / 1000 correct (64.50)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.7883\n",
      "Checking accuracy on validation set\n",
      "Got 661 / 1000 correct (66.10)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.6401\n",
      "Checking accuracy on validation set\n",
      "Got 658 / 1000 correct (65.80)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.6180\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.7930\n",
      "Checking accuracy on validation set\n",
      "Got 658 / 1000 correct (65.80)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.7175\n",
      "Checking accuracy on validation set\n",
      "Got 660 / 1000 correct (66.00)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.4610\n",
      "Checking accuracy on validation set\n",
      "Got 656 / 1000 correct (65.60)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.4342\n",
      "Checking accuracy on validation set\n",
      "Got 660 / 1000 correct (66.00)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.5496\n",
      "Checking accuracy on validation set\n",
      "Got 645 / 1000 correct (64.50)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.6230\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.4294\n",
      "Checking accuracy on validation set\n",
      "Got 662 / 1000 correct (66.20)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.4498\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.6992\n",
      "Checking accuracy on validation set\n",
      "Got 655 / 1000 correct (65.50)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.5045\n",
      "Checking accuracy on validation set\n",
      "Got 657 / 1000 correct (65.70)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.2470\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.3367\n",
      "Checking accuracy on validation set\n",
      "Got 653 / 1000 correct (65.30)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.5695\n",
      "Checking accuracy on validation set\n",
      "Got 649 / 1000 correct (64.90)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.2924\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.4681\n",
      "Checking accuracy on validation set\n",
      "Got 649 / 1000 correct (64.90)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.3254\n",
      "Checking accuracy on validation set\n",
      "Got 637 / 1000 correct (63.70)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.4854\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.3130\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.2294\n",
      "Checking accuracy on validation set\n",
      "Got 642 / 1000 correct (64.20)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.1898\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.3046\n",
      "Checking accuracy on validation set\n",
      "Got 645 / 1000 correct (64.50)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.3525\n",
      "Checking accuracy on validation set\n",
      "Got 637 / 1000 correct (63.70)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.2195\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.2506\n",
      "Checking accuracy on validation set\n",
      "Got 631 / 1000 correct (63.10)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.4065\n",
      "Checking accuracy on validation set\n",
      "Got 610 / 1000 correct (61.00)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.2091\n",
      "Checking accuracy on validation set\n",
      "Got 645 / 1000 correct (64.50)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.1161\n",
      "Checking accuracy on validation set\n",
      "Got 632 / 1000 correct (63.20)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.0792\n",
      "Checking accuracy on validation set\n",
      "Got 640 / 1000 correct (64.00)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.1088\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.1371\n",
      "Checking accuracy on validation set\n",
      "Got 628 / 1000 correct (62.80)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.1915\n",
      "Checking accuracy on validation set\n",
      "Got 615 / 1000 correct (61.50)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.1976\n",
      "Checking accuracy on validation set\n",
      "Got 627 / 1000 correct (62.70)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.3116\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.2100\n",
      "Checking accuracy on validation set\n",
      "Got 607 / 1000 correct (60.70)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.1453\n",
      "Checking accuracy on validation set\n",
      "Got 620 / 1000 correct (62.00)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.1781\n",
      "Checking accuracy on validation set\n",
      "Got 626 / 1000 correct (62.60)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.0931\n",
      "Checking accuracy on validation set\n",
      "Got 636 / 1000 correct (63.60)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.2657\n",
      "Checking accuracy on validation set\n",
      "Got 628 / 1000 correct (62.80)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.0993\n",
      "Checking accuracy on validation set\n",
      "Got 637 / 1000 correct (63.70)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.2040\n",
      "Checking accuracy on validation set\n",
      "Got 618 / 1000 correct (61.80)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.1947\n",
      "Checking accuracy on validation set\n",
      "Got 630 / 1000 correct (63.00)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.1309\n",
      "Checking accuracy on validation set\n",
      "Got 621 / 1000 correct (62.10)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.0958\n",
      "Checking accuracy on validation set\n",
      "Got 620 / 1000 correct (62.00)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.0716\n",
      "Checking accuracy on validation set\n",
      "Got 628 / 1000 correct (62.80)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.0906\n",
      "Checking accuracy on validation set\n",
      "Got 619 / 1000 correct (61.90)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.0854\n",
      "Checking accuracy on validation set\n",
      "Got 620 / 1000 correct (62.00)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.1626\n",
      "Checking accuracy on validation set\n",
      "Got 614 / 1000 correct (61.40)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.1236\n",
      "Checking accuracy on validation set\n",
      "Got 611 / 1000 correct (61.10)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.2179\n",
      "Checking accuracy on validation set\n",
      "Got 624 / 1000 correct (62.40)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.1445\n",
      "Checking accuracy on validation set\n",
      "Got 615 / 1000 correct (61.50)\n",
      "\n",
      "Maximum accuracy attained:  67.10000000000001\n",
      "Checking accuracy on test set\n",
      "Got 6002 / 10000 correct (60.02)\n",
      "Epoch 0, Iteration 0, loss = 2.3321\n",
      "Checking accuracy on validation set\n",
      "Got 124 / 1000 correct (12.40)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.6988\n",
      "Checking accuracy on validation set\n",
      "Got 429 / 1000 correct (42.90)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.5850\n",
      "Checking accuracy on validation set\n",
      "Got 497 / 1000 correct (49.70)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.4037\n",
      "Checking accuracy on validation set\n",
      "Got 518 / 1000 correct (51.80)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.3269\n",
      "Checking accuracy on validation set\n",
      "Got 559 / 1000 correct (55.90)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.1789\n",
      "Checking accuracy on validation set\n",
      "Got 576 / 1000 correct (57.60)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.0447\n",
      "Checking accuracy on validation set\n",
      "Got 600 / 1000 correct (60.00)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 0.9842\n",
      "Checking accuracy on validation set\n",
      "Got 590 / 1000 correct (59.00)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 1.1243\n",
      "Checking accuracy on validation set\n",
      "Got 616 / 1000 correct (61.60)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 1.2564\n",
      "Checking accuracy on validation set\n",
      "Got 625 / 1000 correct (62.50)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 1.1859\n",
      "Checking accuracy on validation set\n",
      "Got 612 / 1000 correct (61.20)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 0.8652\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 0.7943\n",
      "Checking accuracy on validation set\n",
      "Got 641 / 1000 correct (64.10)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 1.0374\n",
      "Checking accuracy on validation set\n",
      "Got 642 / 1000 correct (64.20)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 0.9928\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 1.1376\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 0.8360\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 0.7325\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.5878\n",
      "Checking accuracy on validation set\n",
      "Got 666 / 1000 correct (66.60)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.8867\n",
      "Checking accuracy on validation set\n",
      "Got 653 / 1000 correct (65.30)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 0.9835\n",
      "Checking accuracy on validation set\n",
      "Got 639 / 1000 correct (63.90)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 0.8387\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 1.0645\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.9450\n",
      "Checking accuracy on validation set\n",
      "Got 661 / 1000 correct (66.10)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.5384\n",
      "Checking accuracy on validation set\n",
      "Got 688 / 1000 correct (68.80)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.6919\n",
      "Checking accuracy on validation set\n",
      "Got 677 / 1000 correct (67.70)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.7051\n",
      "Checking accuracy on validation set\n",
      "Got 661 / 1000 correct (66.10)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.7407\n",
      "Checking accuracy on validation set\n",
      "Got 669 / 1000 correct (66.90)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.6623\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.4962\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.6611\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.7813\n",
      "Checking accuracy on validation set\n",
      "Got 660 / 1000 correct (66.00)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.4893\n",
      "Checking accuracy on validation set\n",
      "Got 657 / 1000 correct (65.70)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.3615\n",
      "Checking accuracy on validation set\n",
      "Got 669 / 1000 correct (66.90)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.4638\n",
      "Checking accuracy on validation set\n",
      "Got 671 / 1000 correct (67.10)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.4384\n",
      "Checking accuracy on validation set\n",
      "Got 662 / 1000 correct (66.20)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.3489\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.5854\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.6665\n",
      "Checking accuracy on validation set\n",
      "Got 630 / 1000 correct (63.00)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.6711\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.2686\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.2044\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.2673\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.2749\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.2929\n",
      "Checking accuracy on validation set\n",
      "Got 649 / 1000 correct (64.90)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.2720\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.5116\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.2907\n",
      "Checking accuracy on validation set\n",
      "Got 636 / 1000 correct (63.60)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.1998\n",
      "Checking accuracy on validation set\n",
      "Got 630 / 1000 correct (63.00)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.0969\n",
      "Checking accuracy on validation set\n",
      "Got 632 / 1000 correct (63.20)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.1801\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.2075\n",
      "Checking accuracy on validation set\n",
      "Got 640 / 1000 correct (64.00)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.2907\n",
      "Checking accuracy on validation set\n",
      "Got 649 / 1000 correct (64.90)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.1982\n",
      "Checking accuracy on validation set\n",
      "Got 639 / 1000 correct (63.90)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.4467\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.2235\n",
      "Checking accuracy on validation set\n",
      "Got 641 / 1000 correct (64.10)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.0852\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.1594\n",
      "Checking accuracy on validation set\n",
      "Got 628 / 1000 correct (62.80)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.0855\n",
      "Checking accuracy on validation set\n",
      "Got 641 / 1000 correct (64.10)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.1566\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.1530\n",
      "Checking accuracy on validation set\n",
      "Got 624 / 1000 correct (62.40)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.1989\n",
      "Checking accuracy on validation set\n",
      "Got 624 / 1000 correct (62.40)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.1102\n",
      "Checking accuracy on validation set\n",
      "Got 615 / 1000 correct (61.50)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.2482\n",
      "Checking accuracy on validation set\n",
      "Got 620 / 1000 correct (62.00)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.0770\n",
      "Checking accuracy on validation set\n",
      "Got 632 / 1000 correct (63.20)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.0777\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.0967\n",
      "Checking accuracy on validation set\n",
      "Got 639 / 1000 correct (63.90)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.1075\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.2044\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.1116\n",
      "Checking accuracy on validation set\n",
      "Got 636 / 1000 correct (63.60)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.1174\n",
      "Checking accuracy on validation set\n",
      "Got 611 / 1000 correct (61.10)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.0498\n",
      "Checking accuracy on validation set\n",
      "Got 621 / 1000 correct (62.10)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.0358\n",
      "Checking accuracy on validation set\n",
      "Got 618 / 1000 correct (61.80)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.1131\n",
      "Checking accuracy on validation set\n",
      "Got 608 / 1000 correct (60.80)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.0783\n",
      "Checking accuracy on validation set\n",
      "Got 628 / 1000 correct (62.80)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.0473\n",
      "Checking accuracy on validation set\n",
      "Got 619 / 1000 correct (61.90)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.0620\n",
      "Checking accuracy on validation set\n",
      "Got 617 / 1000 correct (61.70)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.0502\n",
      "Checking accuracy on validation set\n",
      "Got 623 / 1000 correct (62.30)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.2230\n",
      "Checking accuracy on validation set\n",
      "Got 611 / 1000 correct (61.10)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.1048\n",
      "Checking accuracy on validation set\n",
      "Got 637 / 1000 correct (63.70)\n",
      "\n",
      "Maximum accuracy attained:  68.8\n",
      "Checking accuracy on test set\n",
      "Got 6055 / 10000 correct (60.55)\n",
      "Epoch 0, Iteration 0, loss = 2.3005\n",
      "Checking accuracy on validation set\n",
      "Got 93 / 1000 correct (9.30)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.5408\n",
      "Checking accuracy on validation set\n",
      "Got 453 / 1000 correct (45.30)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.3756\n",
      "Checking accuracy on validation set\n",
      "Got 474 / 1000 correct (47.40)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.2893\n",
      "Checking accuracy on validation set\n",
      "Got 507 / 1000 correct (50.70)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.3975\n",
      "Checking accuracy on validation set\n",
      "Got 523 / 1000 correct (52.30)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.3529\n",
      "Checking accuracy on validation set\n",
      "Got 552 / 1000 correct (55.20)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.1688\n",
      "Checking accuracy on validation set\n",
      "Got 530 / 1000 correct (53.00)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.0667\n",
      "Checking accuracy on validation set\n",
      "Got 545 / 1000 correct (54.50)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 1.1601\n",
      "Checking accuracy on validation set\n",
      "Got 579 / 1000 correct (57.90)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 1.0752\n",
      "Checking accuracy on validation set\n",
      "Got 574 / 1000 correct (57.40)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 1.1672\n",
      "Checking accuracy on validation set\n",
      "Got 566 / 1000 correct (56.60)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 1.0999\n",
      "Checking accuracy on validation set\n",
      "Got 606 / 1000 correct (60.60)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 0.9015\n",
      "Checking accuracy on validation set\n",
      "Got 580 / 1000 correct (58.00)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 1.0244\n",
      "Checking accuracy on validation set\n",
      "Got 621 / 1000 correct (62.10)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 0.9224\n",
      "Checking accuracy on validation set\n",
      "Got 623 / 1000 correct (62.30)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 1.0318\n",
      "Checking accuracy on validation set\n",
      "Got 627 / 1000 correct (62.70)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 0.9082\n",
      "Checking accuracy on validation set\n",
      "Got 634 / 1000 correct (63.40)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 0.9081\n",
      "Checking accuracy on validation set\n",
      "Got 619 / 1000 correct (61.90)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.9281\n",
      "Checking accuracy on validation set\n",
      "Got 617 / 1000 correct (61.70)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.8758\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 0.9179\n",
      "Checking accuracy on validation set\n",
      "Got 631 / 1000 correct (63.10)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 0.8218\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.9628\n",
      "Checking accuracy on validation set\n",
      "Got 639 / 1000 correct (63.90)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.8767\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.8645\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.5844\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.5855\n",
      "Checking accuracy on validation set\n",
      "Got 660 / 1000 correct (66.00)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.5850\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.7837\n",
      "Checking accuracy on validation set\n",
      "Got 660 / 1000 correct (66.00)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.7325\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.5390\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 1.0134\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.4748\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.3305\n",
      "Checking accuracy on validation set\n",
      "Got 642 / 1000 correct (64.20)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.4542\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.5926\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.4529\n",
      "Checking accuracy on validation set\n",
      "Got 657 / 1000 correct (65.70)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.6728\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.6146\n",
      "Checking accuracy on validation set\n",
      "Got 658 / 1000 correct (65.80)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.5363\n",
      "Checking accuracy on validation set\n",
      "Got 655 / 1000 correct (65.50)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.4155\n",
      "Checking accuracy on validation set\n",
      "Got 637 / 1000 correct (63.70)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.1988\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.3036\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.3219\n",
      "Checking accuracy on validation set\n",
      "Got 649 / 1000 correct (64.90)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.5402\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.3787\n",
      "Checking accuracy on validation set\n",
      "Got 655 / 1000 correct (65.50)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.4482\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.6088\n",
      "Checking accuracy on validation set\n",
      "Got 640 / 1000 correct (64.00)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.3524\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.3000\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.3208\n",
      "Checking accuracy on validation set\n",
      "Got 664 / 1000 correct (66.40)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.2924\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.3809\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.4058\n",
      "Checking accuracy on validation set\n",
      "Got 639 / 1000 correct (63.90)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.4852\n",
      "Checking accuracy on validation set\n",
      "Got 645 / 1000 correct (64.50)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.3709\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.1015\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.2315\n",
      "Checking accuracy on validation set\n",
      "Got 645 / 1000 correct (64.50)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.2587\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.4117\n",
      "Checking accuracy on validation set\n",
      "Got 641 / 1000 correct (64.10)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.3004\n",
      "Checking accuracy on validation set\n",
      "Got 643 / 1000 correct (64.30)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.1562\n",
      "Checking accuracy on validation set\n",
      "Got 628 / 1000 correct (62.80)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.4212\n",
      "Checking accuracy on validation set\n",
      "Got 625 / 1000 correct (62.50)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.4349\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.1427\n",
      "Checking accuracy on validation set\n",
      "Got 632 / 1000 correct (63.20)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.2193\n",
      "Checking accuracy on validation set\n",
      "Got 634 / 1000 correct (63.40)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.1202\n",
      "Checking accuracy on validation set\n",
      "Got 630 / 1000 correct (63.00)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.2255\n",
      "Checking accuracy on validation set\n",
      "Got 643 / 1000 correct (64.30)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.2048\n",
      "Checking accuracy on validation set\n",
      "Got 627 / 1000 correct (62.70)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.3092\n",
      "Checking accuracy on validation set\n",
      "Got 632 / 1000 correct (63.20)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.4449\n",
      "Checking accuracy on validation set\n",
      "Got 618 / 1000 correct (61.80)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.1878\n",
      "Checking accuracy on validation set\n",
      "Got 624 / 1000 correct (62.40)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.1454\n",
      "Checking accuracy on validation set\n",
      "Got 634 / 1000 correct (63.40)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.1073\n",
      "Checking accuracy on validation set\n",
      "Got 627 / 1000 correct (62.70)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.0803\n",
      "Checking accuracy on validation set\n",
      "Got 623 / 1000 correct (62.30)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.1518\n",
      "Checking accuracy on validation set\n",
      "Got 626 / 1000 correct (62.60)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.0999\n",
      "Checking accuracy on validation set\n",
      "Got 617 / 1000 correct (61.70)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.1100\n",
      "Checking accuracy on validation set\n",
      "Got 621 / 1000 correct (62.10)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.2822\n",
      "Checking accuracy on validation set\n",
      "Got 614 / 1000 correct (61.40)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.2300\n",
      "Checking accuracy on validation set\n",
      "Got 606 / 1000 correct (60.60)\n",
      "\n",
      "Maximum accuracy attained:  66.4\n",
      "Checking accuracy on test set\n",
      "Got 6048 / 10000 correct (60.48)\n",
      "Epoch 0, Iteration 0, loss = 2.3073\n",
      "Checking accuracy on validation set\n",
      "Got 145 / 1000 correct (14.50)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.5477\n",
      "Checking accuracy on validation set\n",
      "Got 462 / 1000 correct (46.20)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.2817\n",
      "Checking accuracy on validation set\n",
      "Got 506 / 1000 correct (50.60)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.1233\n",
      "Checking accuracy on validation set\n",
      "Got 534 / 1000 correct (53.40)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.1926\n",
      "Checking accuracy on validation set\n",
      "Got 546 / 1000 correct (54.60)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.2348\n",
      "Checking accuracy on validation set\n",
      "Got 572 / 1000 correct (57.20)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 0.9229\n",
      "Checking accuracy on validation set\n",
      "Got 578 / 1000 correct (57.80)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.1202\n",
      "Checking accuracy on validation set\n",
      "Got 593 / 1000 correct (59.30)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 0.9473\n",
      "Checking accuracy on validation set\n",
      "Got 608 / 1000 correct (60.80)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 0.9298\n",
      "Checking accuracy on validation set\n",
      "Got 607 / 1000 correct (60.70)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 1.1292\n",
      "Checking accuracy on validation set\n",
      "Got 624 / 1000 correct (62.40)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 0.8185\n",
      "Checking accuracy on validation set\n",
      "Got 573 / 1000 correct (57.30)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 1.0878\n",
      "Checking accuracy on validation set\n",
      "Got 608 / 1000 correct (60.80)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 0.8998\n",
      "Checking accuracy on validation set\n",
      "Got 628 / 1000 correct (62.80)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 0.7492\n",
      "Checking accuracy on validation set\n",
      "Got 634 / 1000 correct (63.40)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 0.9023\n",
      "Checking accuracy on validation set\n",
      "Got 641 / 1000 correct (64.10)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 0.8586\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 0.8842\n",
      "Checking accuracy on validation set\n",
      "Got 630 / 1000 correct (63.00)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.6151\n",
      "Checking accuracy on validation set\n",
      "Got 649 / 1000 correct (64.90)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.7561\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 0.5743\n",
      "Checking accuracy on validation set\n",
      "Got 661 / 1000 correct (66.10)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 0.7836\n",
      "Checking accuracy on validation set\n",
      "Got 642 / 1000 correct (64.20)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.9745\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.7992\n",
      "Checking accuracy on validation set\n",
      "Got 655 / 1000 correct (65.50)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.7265\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.7725\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.6948\n",
      "Checking accuracy on validation set\n",
      "Got 645 / 1000 correct (64.50)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.6789\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.5084\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.6389\n",
      "Checking accuracy on validation set\n",
      "Got 653 / 1000 correct (65.30)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.7374\n",
      "Checking accuracy on validation set\n",
      "Got 657 / 1000 correct (65.70)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.6753\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.6238\n",
      "Checking accuracy on validation set\n",
      "Got 662 / 1000 correct (66.20)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.5307\n",
      "Checking accuracy on validation set\n",
      "Got 673 / 1000 correct (67.30)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.2395\n",
      "Checking accuracy on validation set\n",
      "Got 641 / 1000 correct (64.10)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.4832\n",
      "Checking accuracy on validation set\n",
      "Got 641 / 1000 correct (64.10)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.4531\n",
      "Checking accuracy on validation set\n",
      "Got 632 / 1000 correct (63.20)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.7184\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.5680\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.6346\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.3688\n",
      "Checking accuracy on validation set\n",
      "Got 642 / 1000 correct (64.20)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.2889\n",
      "Checking accuracy on validation set\n",
      "Got 660 / 1000 correct (66.00)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.2229\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.3905\n",
      "Checking accuracy on validation set\n",
      "Got 640 / 1000 correct (64.00)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.5353\n",
      "Checking accuracy on validation set\n",
      "Got 658 / 1000 correct (65.80)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.5014\n",
      "Checking accuracy on validation set\n",
      "Got 632 / 1000 correct (63.20)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.3596\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.3654\n",
      "Checking accuracy on validation set\n",
      "Got 653 / 1000 correct (65.30)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.2449\n",
      "Checking accuracy on validation set\n",
      "Got 640 / 1000 correct (64.00)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.3373\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.3324\n",
      "Checking accuracy on validation set\n",
      "Got 645 / 1000 correct (64.50)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.4201\n",
      "Checking accuracy on validation set\n",
      "Got 634 / 1000 correct (63.40)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.2637\n",
      "Checking accuracy on validation set\n",
      "Got 655 / 1000 correct (65.50)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.3290\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.3130\n",
      "Checking accuracy on validation set\n",
      "Got 641 / 1000 correct (64.10)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.3663\n",
      "Checking accuracy on validation set\n",
      "Got 636 / 1000 correct (63.60)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.2022\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.2016\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.2342\n",
      "Checking accuracy on validation set\n",
      "Got 658 / 1000 correct (65.80)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.2272\n",
      "Checking accuracy on validation set\n",
      "Got 649 / 1000 correct (64.90)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.2908\n",
      "Checking accuracy on validation set\n",
      "Got 642 / 1000 correct (64.20)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.3111\n",
      "Checking accuracy on validation set\n",
      "Got 630 / 1000 correct (63.00)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.3838\n",
      "Checking accuracy on validation set\n",
      "Got 639 / 1000 correct (63.90)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.1349\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.1294\n",
      "Checking accuracy on validation set\n",
      "Got 635 / 1000 correct (63.50)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.1691\n",
      "Checking accuracy on validation set\n",
      "Got 635 / 1000 correct (63.50)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.1401\n",
      "Checking accuracy on validation set\n",
      "Got 627 / 1000 correct (62.70)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.0928\n",
      "Checking accuracy on validation set\n",
      "Got 656 / 1000 correct (65.60)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.1554\n",
      "Checking accuracy on validation set\n",
      "Got 636 / 1000 correct (63.60)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.1023\n",
      "Checking accuracy on validation set\n",
      "Got 632 / 1000 correct (63.20)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.1430\n",
      "Checking accuracy on validation set\n",
      "Got 635 / 1000 correct (63.50)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.3458\n",
      "Checking accuracy on validation set\n",
      "Got 625 / 1000 correct (62.50)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.0621\n",
      "Checking accuracy on validation set\n",
      "Got 643 / 1000 correct (64.30)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.1194\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.0783\n",
      "Checking accuracy on validation set\n",
      "Got 636 / 1000 correct (63.60)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.2021\n",
      "Checking accuracy on validation set\n",
      "Got 639 / 1000 correct (63.90)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.0985\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.0890\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.1978\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.1580\n",
      "Checking accuracy on validation set\n",
      "Got 606 / 1000 correct (60.60)\n",
      "\n",
      "Maximum accuracy attained:  67.30000000000001\n",
      "Checking accuracy on test set\n",
      "Got 6219 / 10000 correct (62.19)\n",
      "Epoch 0, Iteration 0, loss = 2.3155\n",
      "Checking accuracy on validation set\n",
      "Got 115 / 1000 correct (11.50)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.6172\n",
      "Checking accuracy on validation set\n",
      "Got 435 / 1000 correct (43.50)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.5119\n",
      "Checking accuracy on validation set\n",
      "Got 485 / 1000 correct (48.50)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.4978\n",
      "Checking accuracy on validation set\n",
      "Got 527 / 1000 correct (52.70)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.2963\n",
      "Checking accuracy on validation set\n",
      "Got 530 / 1000 correct (53.00)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.3409\n",
      "Checking accuracy on validation set\n",
      "Got 541 / 1000 correct (54.10)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.1184\n",
      "Checking accuracy on validation set\n",
      "Got 551 / 1000 correct (55.10)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 0.9630\n",
      "Checking accuracy on validation set\n",
      "Got 574 / 1000 correct (57.40)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 1.1442\n",
      "Checking accuracy on validation set\n",
      "Got 566 / 1000 correct (56.60)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 1.1164\n",
      "Checking accuracy on validation set\n",
      "Got 565 / 1000 correct (56.50)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 1.1446\n",
      "Checking accuracy on validation set\n",
      "Got 575 / 1000 correct (57.50)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 1.0859\n",
      "Checking accuracy on validation set\n",
      "Got 592 / 1000 correct (59.20)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 1.2786\n",
      "Checking accuracy on validation set\n",
      "Got 598 / 1000 correct (59.80)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 1.1885\n",
      "Checking accuracy on validation set\n",
      "Got 611 / 1000 correct (61.10)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 1.0722\n",
      "Checking accuracy on validation set\n",
      "Got 598 / 1000 correct (59.80)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 0.9113\n",
      "Checking accuracy on validation set\n",
      "Got 599 / 1000 correct (59.90)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 0.7754\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 0.8758\n",
      "Checking accuracy on validation set\n",
      "Got 617 / 1000 correct (61.70)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.9557\n",
      "Checking accuracy on validation set\n",
      "Got 621 / 1000 correct (62.10)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.8620\n",
      "Checking accuracy on validation set\n",
      "Got 613 / 1000 correct (61.30)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 0.7270\n",
      "Checking accuracy on validation set\n",
      "Got 602 / 1000 correct (60.20)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 1.0268\n",
      "Checking accuracy on validation set\n",
      "Got 619 / 1000 correct (61.90)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.7595\n",
      "Checking accuracy on validation set\n",
      "Got 630 / 1000 correct (63.00)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 1.0273\n",
      "Checking accuracy on validation set\n",
      "Got 640 / 1000 correct (64.00)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.7809\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.8978\n",
      "Checking accuracy on validation set\n",
      "Got 637 / 1000 correct (63.70)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.6495\n",
      "Checking accuracy on validation set\n",
      "Got 614 / 1000 correct (61.40)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.8206\n",
      "Checking accuracy on validation set\n",
      "Got 637 / 1000 correct (63.70)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.9022\n",
      "Checking accuracy on validation set\n",
      "Got 645 / 1000 correct (64.50)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.6786\n",
      "Checking accuracy on validation set\n",
      "Got 667 / 1000 correct (66.70)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.8577\n",
      "Checking accuracy on validation set\n",
      "Got 645 / 1000 correct (64.50)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.7264\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.5673\n",
      "Checking accuracy on validation set\n",
      "Got 615 / 1000 correct (61.50)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.6629\n",
      "Checking accuracy on validation set\n",
      "Got 639 / 1000 correct (63.90)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.7063\n",
      "Checking accuracy on validation set\n",
      "Got 627 / 1000 correct (62.70)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 1.0096\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.5951\n",
      "Checking accuracy on validation set\n",
      "Got 625 / 1000 correct (62.50)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.7434\n",
      "Checking accuracy on validation set\n",
      "Got 637 / 1000 correct (63.70)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.7403\n",
      "Checking accuracy on validation set\n",
      "Got 631 / 1000 correct (63.10)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.7542\n",
      "Checking accuracy on validation set\n",
      "Got 631 / 1000 correct (63.10)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.4523\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.5568\n",
      "Checking accuracy on validation set\n",
      "Got 637 / 1000 correct (63.70)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.2851\n",
      "Checking accuracy on validation set\n",
      "Got 598 / 1000 correct (59.80)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.5838\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.3074\n",
      "Checking accuracy on validation set\n",
      "Got 627 / 1000 correct (62.70)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.6526\n",
      "Checking accuracy on validation set\n",
      "Got 610 / 1000 correct (61.00)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.3865\n",
      "Checking accuracy on validation set\n",
      "Got 624 / 1000 correct (62.40)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.6543\n",
      "Checking accuracy on validation set\n",
      "Got 622 / 1000 correct (62.20)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.4841\n",
      "Checking accuracy on validation set\n",
      "Got 626 / 1000 correct (62.60)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.3349\n",
      "Checking accuracy on validation set\n",
      "Got 625 / 1000 correct (62.50)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.5040\n",
      "Checking accuracy on validation set\n",
      "Got 624 / 1000 correct (62.40)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.4854\n",
      "Checking accuracy on validation set\n",
      "Got 627 / 1000 correct (62.70)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.4841\n",
      "Checking accuracy on validation set\n",
      "Got 608 / 1000 correct (60.80)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.5026\n",
      "Checking accuracy on validation set\n",
      "Got 620 / 1000 correct (62.00)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.4116\n",
      "Checking accuracy on validation set\n",
      "Got 634 / 1000 correct (63.40)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.7470\n",
      "Checking accuracy on validation set\n",
      "Got 604 / 1000 correct (60.40)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.4218\n",
      "Checking accuracy on validation set\n",
      "Got 615 / 1000 correct (61.50)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.1793\n",
      "Checking accuracy on validation set\n",
      "Got 609 / 1000 correct (60.90)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.2845\n",
      "Checking accuracy on validation set\n",
      "Got 608 / 1000 correct (60.80)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.3878\n",
      "Checking accuracy on validation set\n",
      "Got 596 / 1000 correct (59.60)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.5007\n",
      "Checking accuracy on validation set\n",
      "Got 605 / 1000 correct (60.50)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.2460\n",
      "Checking accuracy on validation set\n",
      "Got 608 / 1000 correct (60.80)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.4411\n",
      "Checking accuracy on validation set\n",
      "Got 590 / 1000 correct (59.00)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.4103\n",
      "Checking accuracy on validation set\n",
      "Got 590 / 1000 correct (59.00)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.1995\n",
      "Checking accuracy on validation set\n",
      "Got 608 / 1000 correct (60.80)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.2610\n",
      "Checking accuracy on validation set\n",
      "Got 605 / 1000 correct (60.50)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.3096\n",
      "Checking accuracy on validation set\n",
      "Got 600 / 1000 correct (60.00)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.2306\n",
      "Checking accuracy on validation set\n",
      "Got 595 / 1000 correct (59.50)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.3040\n",
      "Checking accuracy on validation set\n",
      "Got 604 / 1000 correct (60.40)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.4676\n",
      "Checking accuracy on validation set\n",
      "Got 592 / 1000 correct (59.20)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.3456\n",
      "Checking accuracy on validation set\n",
      "Got 607 / 1000 correct (60.70)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.1982\n",
      "Checking accuracy on validation set\n",
      "Got 593 / 1000 correct (59.30)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.2246\n",
      "Checking accuracy on validation set\n",
      "Got 581 / 1000 correct (58.10)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.1586\n",
      "Checking accuracy on validation set\n",
      "Got 609 / 1000 correct (60.90)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.1672\n",
      "Checking accuracy on validation set\n",
      "Got 583 / 1000 correct (58.30)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.1670\n",
      "Checking accuracy on validation set\n",
      "Got 597 / 1000 correct (59.70)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.1874\n",
      "Checking accuracy on validation set\n",
      "Got 596 / 1000 correct (59.60)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.1389\n",
      "Checking accuracy on validation set\n",
      "Got 586 / 1000 correct (58.60)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.3966\n",
      "Checking accuracy on validation set\n",
      "Got 600 / 1000 correct (60.00)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.4130\n",
      "Checking accuracy on validation set\n",
      "Got 604 / 1000 correct (60.40)\n",
      "\n",
      "Maximum accuracy attained:  66.7\n",
      "Checking accuracy on test set\n",
      "Got 5960 / 10000 correct (59.60)\n",
      "Epoch 0, Iteration 0, loss = 2.2827\n",
      "Checking accuracy on validation set\n",
      "Got 137 / 1000 correct (13.70)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.5338\n",
      "Checking accuracy on validation set\n",
      "Got 460 / 1000 correct (46.00)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.5040\n",
      "Checking accuracy on validation set\n",
      "Got 520 / 1000 correct (52.00)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.4561\n",
      "Checking accuracy on validation set\n",
      "Got 533 / 1000 correct (53.30)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.1274\n",
      "Checking accuracy on validation set\n",
      "Got 563 / 1000 correct (56.30)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.3925\n",
      "Checking accuracy on validation set\n",
      "Got 564 / 1000 correct (56.40)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 0.9592\n",
      "Checking accuracy on validation set\n",
      "Got 583 / 1000 correct (58.30)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.0355\n",
      "Checking accuracy on validation set\n",
      "Got 600 / 1000 correct (60.00)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 1.1680\n",
      "Checking accuracy on validation set\n",
      "Got 626 / 1000 correct (62.60)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 0.8906\n",
      "Checking accuracy on validation set\n",
      "Got 616 / 1000 correct (61.60)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 1.0551\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 0.9203\n",
      "Checking accuracy on validation set\n",
      "Got 618 / 1000 correct (61.80)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 0.9160\n",
      "Checking accuracy on validation set\n",
      "Got 634 / 1000 correct (63.40)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 0.9999\n",
      "Checking accuracy on validation set\n",
      "Got 612 / 1000 correct (61.20)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 1.1548\n",
      "Checking accuracy on validation set\n",
      "Got 623 / 1000 correct (62.30)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 0.9808\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 0.7389\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 0.8253\n",
      "Checking accuracy on validation set\n",
      "Got 672 / 1000 correct (67.20)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.9141\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.7426\n",
      "Checking accuracy on validation set\n",
      "Got 655 / 1000 correct (65.50)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 0.7508\n",
      "Checking accuracy on validation set\n",
      "Got 642 / 1000 correct (64.20)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 0.8171\n",
      "Checking accuracy on validation set\n",
      "Got 663 / 1000 correct (66.30)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.7553\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.9052\n",
      "Checking accuracy on validation set\n",
      "Got 632 / 1000 correct (63.20)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.6441\n",
      "Checking accuracy on validation set\n",
      "Got 669 / 1000 correct (66.90)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.7331\n",
      "Checking accuracy on validation set\n",
      "Got 672 / 1000 correct (67.20)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.5783\n",
      "Checking accuracy on validation set\n",
      "Got 636 / 1000 correct (63.60)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.6178\n",
      "Checking accuracy on validation set\n",
      "Got 667 / 1000 correct (66.70)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.3411\n",
      "Checking accuracy on validation set\n",
      "Got 668 / 1000 correct (66.80)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.7514\n",
      "Checking accuracy on validation set\n",
      "Got 674 / 1000 correct (67.40)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.7350\n",
      "Checking accuracy on validation set\n",
      "Got 666 / 1000 correct (66.60)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.6858\n",
      "Checking accuracy on validation set\n",
      "Got 665 / 1000 correct (66.50)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.5389\n",
      "Checking accuracy on validation set\n",
      "Got 649 / 1000 correct (64.90)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.4521\n",
      "Checking accuracy on validation set\n",
      "Got 670 / 1000 correct (67.00)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.6378\n",
      "Checking accuracy on validation set\n",
      "Got 655 / 1000 correct (65.50)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.6712\n",
      "Checking accuracy on validation set\n",
      "Got 664 / 1000 correct (66.40)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.5311\n",
      "Checking accuracy on validation set\n",
      "Got 657 / 1000 correct (65.70)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.5034\n",
      "Checking accuracy on validation set\n",
      "Got 634 / 1000 correct (63.40)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.5713\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.7361\n",
      "Checking accuracy on validation set\n",
      "Got 666 / 1000 correct (66.60)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.2853\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.2737\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.4559\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.3102\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.4010\n",
      "Checking accuracy on validation set\n",
      "Got 656 / 1000 correct (65.60)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.4495\n",
      "Checking accuracy on validation set\n",
      "Got 669 / 1000 correct (66.90)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.2977\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.4529\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.2028\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.3059\n",
      "Checking accuracy on validation set\n",
      "Got 669 / 1000 correct (66.90)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.2271\n",
      "Checking accuracy on validation set\n",
      "Got 645 / 1000 correct (64.50)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.1647\n",
      "Checking accuracy on validation set\n",
      "Got 653 / 1000 correct (65.30)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.2728\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.1894\n",
      "Checking accuracy on validation set\n",
      "Got 653 / 1000 correct (65.30)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.1586\n",
      "Checking accuracy on validation set\n",
      "Got 665 / 1000 correct (66.50)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.3179\n",
      "Checking accuracy on validation set\n",
      "Got 656 / 1000 correct (65.60)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.2967\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.2033\n",
      "Checking accuracy on validation set\n",
      "Got 666 / 1000 correct (66.60)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.2921\n",
      "Checking accuracy on validation set\n",
      "Got 643 / 1000 correct (64.30)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.3008\n",
      "Checking accuracy on validation set\n",
      "Got 626 / 1000 correct (62.60)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.2163\n",
      "Checking accuracy on validation set\n",
      "Got 621 / 1000 correct (62.10)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.3555\n",
      "Checking accuracy on validation set\n",
      "Got 655 / 1000 correct (65.50)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.2743\n",
      "Checking accuracy on validation set\n",
      "Got 643 / 1000 correct (64.30)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.3006\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.1628\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.1338\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.1373\n",
      "Checking accuracy on validation set\n",
      "Got 670 / 1000 correct (67.00)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.1379\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.1730\n",
      "Checking accuracy on validation set\n",
      "Got 636 / 1000 correct (63.60)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.1833\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.1867\n",
      "Checking accuracy on validation set\n",
      "Got 632 / 1000 correct (63.20)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.2116\n",
      "Checking accuracy on validation set\n",
      "Got 649 / 1000 correct (64.90)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.1052\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.0726\n",
      "Checking accuracy on validation set\n",
      "Got 656 / 1000 correct (65.60)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.1473\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.0691\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.1283\n",
      "Checking accuracy on validation set\n",
      "Got 655 / 1000 correct (65.50)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.2653\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.1009\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.1184\n",
      "Checking accuracy on validation set\n",
      "Got 642 / 1000 correct (64.20)\n",
      "\n",
      "Maximum accuracy attained:  67.4\n",
      "Checking accuracy on test set\n",
      "Got 6327 / 10000 correct (63.27)\n",
      "Epoch 0, Iteration 0, loss = 2.3114\n",
      "Checking accuracy on validation set\n",
      "Got 108 / 1000 correct (10.80)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.5938\n",
      "Checking accuracy on validation set\n",
      "Got 460 / 1000 correct (46.00)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.2313\n",
      "Checking accuracy on validation set\n",
      "Got 536 / 1000 correct (53.60)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.3337\n",
      "Checking accuracy on validation set\n",
      "Got 554 / 1000 correct (55.40)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.1263\n",
      "Checking accuracy on validation set\n",
      "Got 569 / 1000 correct (56.90)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.1433\n",
      "Checking accuracy on validation set\n",
      "Got 590 / 1000 correct (59.00)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.1769\n",
      "Checking accuracy on validation set\n",
      "Got 587 / 1000 correct (58.70)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 0.9728\n",
      "Checking accuracy on validation set\n",
      "Got 624 / 1000 correct (62.40)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 1.0080\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 0.8030\n",
      "Checking accuracy on validation set\n",
      "Got 619 / 1000 correct (61.90)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 0.9972\n",
      "Checking accuracy on validation set\n",
      "Got 634 / 1000 correct (63.40)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 0.8281\n",
      "Checking accuracy on validation set\n",
      "Got 629 / 1000 correct (62.90)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 0.9823\n",
      "Checking accuracy on validation set\n",
      "Got 640 / 1000 correct (64.00)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 0.7057\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 0.9488\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 0.9808\n",
      "Checking accuracy on validation set\n",
      "Got 657 / 1000 correct (65.70)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 0.7206\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 0.7927\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.6214\n",
      "Checking accuracy on validation set\n",
      "Got 653 / 1000 correct (65.30)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.7614\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 0.6237\n",
      "Checking accuracy on validation set\n",
      "Got 656 / 1000 correct (65.60)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 0.8438\n",
      "Checking accuracy on validation set\n",
      "Got 656 / 1000 correct (65.60)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.9883\n",
      "Checking accuracy on validation set\n",
      "Got 673 / 1000 correct (67.30)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.8661\n",
      "Checking accuracy on validation set\n",
      "Got 670 / 1000 correct (67.00)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.5151\n",
      "Checking accuracy on validation set\n",
      "Got 675 / 1000 correct (67.50)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.7141\n",
      "Checking accuracy on validation set\n",
      "Got 661 / 1000 correct (66.10)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.5383\n",
      "Checking accuracy on validation set\n",
      "Got 657 / 1000 correct (65.70)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.6535\n",
      "Checking accuracy on validation set\n",
      "Got 642 / 1000 correct (64.20)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.4560\n",
      "Checking accuracy on validation set\n",
      "Got 683 / 1000 correct (68.30)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.9523\n",
      "Checking accuracy on validation set\n",
      "Got 660 / 1000 correct (66.00)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.7464\n",
      "Checking accuracy on validation set\n",
      "Got 679 / 1000 correct (67.90)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.8881\n",
      "Checking accuracy on validation set\n",
      "Got 670 / 1000 correct (67.00)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.3699\n",
      "Checking accuracy on validation set\n",
      "Got 688 / 1000 correct (68.80)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.3509\n",
      "Checking accuracy on validation set\n",
      "Got 675 / 1000 correct (67.50)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.2780\n",
      "Checking accuracy on validation set\n",
      "Got 684 / 1000 correct (68.40)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.5201\n",
      "Checking accuracy on validation set\n",
      "Got 675 / 1000 correct (67.50)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.3963\n",
      "Checking accuracy on validation set\n",
      "Got 689 / 1000 correct (68.90)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.3362\n",
      "Checking accuracy on validation set\n",
      "Got 688 / 1000 correct (68.80)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.5404\n",
      "Checking accuracy on validation set\n",
      "Got 678 / 1000 correct (67.80)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.4156\n",
      "Checking accuracy on validation set\n",
      "Got 672 / 1000 correct (67.20)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.2921\n",
      "Checking accuracy on validation set\n",
      "Got 673 / 1000 correct (67.30)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.4581\n",
      "Checking accuracy on validation set\n",
      "Got 675 / 1000 correct (67.50)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.2744\n",
      "Checking accuracy on validation set\n",
      "Got 670 / 1000 correct (67.00)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.4138\n",
      "Checking accuracy on validation set\n",
      "Got 682 / 1000 correct (68.20)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.2701\n",
      "Checking accuracy on validation set\n",
      "Got 679 / 1000 correct (67.90)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.6087\n",
      "Checking accuracy on validation set\n",
      "Got 660 / 1000 correct (66.00)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.5692\n",
      "Checking accuracy on validation set\n",
      "Got 673 / 1000 correct (67.30)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.5790\n",
      "Checking accuracy on validation set\n",
      "Got 677 / 1000 correct (67.70)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.2874\n",
      "Checking accuracy on validation set\n",
      "Got 684 / 1000 correct (68.40)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.1911\n",
      "Checking accuracy on validation set\n",
      "Got 688 / 1000 correct (68.80)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.2632\n",
      "Checking accuracy on validation set\n",
      "Got 691 / 1000 correct (69.10)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.2281\n",
      "Checking accuracy on validation set\n",
      "Got 692 / 1000 correct (69.20)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.3409\n",
      "Checking accuracy on validation set\n",
      "Got 658 / 1000 correct (65.80)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.4686\n",
      "Checking accuracy on validation set\n",
      "Got 678 / 1000 correct (67.80)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.3707\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.3913\n",
      "Checking accuracy on validation set\n",
      "Got 672 / 1000 correct (67.20)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.2338\n",
      "Checking accuracy on validation set\n",
      "Got 657 / 1000 correct (65.70)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.1985\n",
      "Checking accuracy on validation set\n",
      "Got 668 / 1000 correct (66.80)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.1978\n",
      "Checking accuracy on validation set\n",
      "Got 674 / 1000 correct (67.40)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.3762\n",
      "Checking accuracy on validation set\n",
      "Got 667 / 1000 correct (66.70)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.1433\n",
      "Checking accuracy on validation set\n",
      "Got 655 / 1000 correct (65.50)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.3509\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.3835\n",
      "Checking accuracy on validation set\n",
      "Got 657 / 1000 correct (65.70)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.1664\n",
      "Checking accuracy on validation set\n",
      "Got 662 / 1000 correct (66.20)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.1268\n",
      "Checking accuracy on validation set\n",
      "Got 666 / 1000 correct (66.60)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.0969\n",
      "Checking accuracy on validation set\n",
      "Got 674 / 1000 correct (67.40)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.1012\n",
      "Checking accuracy on validation set\n",
      "Got 682 / 1000 correct (68.20)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.1788\n",
      "Checking accuracy on validation set\n",
      "Got 670 / 1000 correct (67.00)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.0956\n",
      "Checking accuracy on validation set\n",
      "Got 660 / 1000 correct (66.00)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.2770\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.2659\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.2054\n",
      "Checking accuracy on validation set\n",
      "Got 666 / 1000 correct (66.60)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.0729\n",
      "Checking accuracy on validation set\n",
      "Got 657 / 1000 correct (65.70)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.0284\n",
      "Checking accuracy on validation set\n",
      "Got 666 / 1000 correct (66.60)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.0788\n",
      "Checking accuracy on validation set\n",
      "Got 671 / 1000 correct (67.10)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.0838\n",
      "Checking accuracy on validation set\n",
      "Got 667 / 1000 correct (66.70)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.0520\n",
      "Checking accuracy on validation set\n",
      "Got 680 / 1000 correct (68.00)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.1502\n",
      "Checking accuracy on validation set\n",
      "Got 657 / 1000 correct (65.70)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.1607\n",
      "Checking accuracy on validation set\n",
      "Got 672 / 1000 correct (67.20)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.1234\n",
      "Checking accuracy on validation set\n",
      "Got 653 / 1000 correct (65.30)\n",
      "\n",
      "Maximum accuracy attained:  69.19999999999999\n",
      "Checking accuracy on test set\n",
      "Got 6278 / 10000 correct (62.78)\n",
      "Epoch 0, Iteration 0, loss = 2.3056\n",
      "Checking accuracy on validation set\n",
      "Got 112 / 1000 correct (11.20)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.5170\n",
      "Checking accuracy on validation set\n",
      "Got 399 / 1000 correct (39.90)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.3428\n",
      "Checking accuracy on validation set\n",
      "Got 466 / 1000 correct (46.60)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.5138\n",
      "Checking accuracy on validation set\n",
      "Got 484 / 1000 correct (48.40)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.1746\n",
      "Checking accuracy on validation set\n",
      "Got 540 / 1000 correct (54.00)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.2741\n",
      "Checking accuracy on validation set\n",
      "Got 551 / 1000 correct (55.10)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.1445\n",
      "Checking accuracy on validation set\n",
      "Got 589 / 1000 correct (58.90)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.0795\n",
      "Checking accuracy on validation set\n",
      "Got 595 / 1000 correct (59.50)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 1.1149\n",
      "Checking accuracy on validation set\n",
      "Got 599 / 1000 correct (59.90)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 0.9165\n",
      "Checking accuracy on validation set\n",
      "Got 597 / 1000 correct (59.70)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 1.2045\n",
      "Checking accuracy on validation set\n",
      "Got 602 / 1000 correct (60.20)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 1.0274\n",
      "Checking accuracy on validation set\n",
      "Got 598 / 1000 correct (59.80)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 1.0785\n",
      "Checking accuracy on validation set\n",
      "Got 614 / 1000 correct (61.40)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 1.6141\n",
      "Checking accuracy on validation set\n",
      "Got 615 / 1000 correct (61.50)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 1.0786\n",
      "Checking accuracy on validation set\n",
      "Got 616 / 1000 correct (61.60)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 0.9233\n",
      "Checking accuracy on validation set\n",
      "Got 631 / 1000 correct (63.10)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 0.6942\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 0.8303\n",
      "Checking accuracy on validation set\n",
      "Got 639 / 1000 correct (63.90)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.9071\n",
      "Checking accuracy on validation set\n",
      "Got 628 / 1000 correct (62.80)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.9791\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 1.2172\n",
      "Checking accuracy on validation set\n",
      "Got 624 / 1000 correct (62.40)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 0.6643\n",
      "Checking accuracy on validation set\n",
      "Got 653 / 1000 correct (65.30)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.7048\n",
      "Checking accuracy on validation set\n",
      "Got 661 / 1000 correct (66.10)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.9750\n",
      "Checking accuracy on validation set\n",
      "Got 635 / 1000 correct (63.50)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.8398\n",
      "Checking accuracy on validation set\n",
      "Got 641 / 1000 correct (64.10)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.7183\n",
      "Checking accuracy on validation set\n",
      "Got 658 / 1000 correct (65.80)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.7369\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.6357\n",
      "Checking accuracy on validation set\n",
      "Got 636 / 1000 correct (63.60)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.7946\n",
      "Checking accuracy on validation set\n",
      "Got 656 / 1000 correct (65.60)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.7799\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.9278\n",
      "Checking accuracy on validation set\n",
      "Got 662 / 1000 correct (66.20)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.9672\n",
      "Checking accuracy on validation set\n",
      "Got 668 / 1000 correct (66.80)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.6000\n",
      "Checking accuracy on validation set\n",
      "Got 660 / 1000 correct (66.00)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.6811\n",
      "Checking accuracy on validation set\n",
      "Got 643 / 1000 correct (64.30)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.4992\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.6320\n",
      "Checking accuracy on validation set\n",
      "Got 673 / 1000 correct (67.30)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.3753\n",
      "Checking accuracy on validation set\n",
      "Got 672 / 1000 correct (67.20)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.5803\n",
      "Checking accuracy on validation set\n",
      "Got 661 / 1000 correct (66.10)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.6080\n",
      "Checking accuracy on validation set\n",
      "Got 649 / 1000 correct (64.90)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.7066\n",
      "Checking accuracy on validation set\n",
      "Got 658 / 1000 correct (65.80)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.5193\n",
      "Checking accuracy on validation set\n",
      "Got 670 / 1000 correct (67.00)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.4607\n",
      "Checking accuracy on validation set\n",
      "Got 660 / 1000 correct (66.00)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.3798\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.3235\n",
      "Checking accuracy on validation set\n",
      "Got 641 / 1000 correct (64.10)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.3568\n",
      "Checking accuracy on validation set\n",
      "Got 655 / 1000 correct (65.50)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.2490\n",
      "Checking accuracy on validation set\n",
      "Got 665 / 1000 correct (66.50)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.3002\n",
      "Checking accuracy on validation set\n",
      "Got 666 / 1000 correct (66.60)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.6339\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.5188\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.3251\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.3387\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.2080\n",
      "Checking accuracy on validation set\n",
      "Got 643 / 1000 correct (64.30)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.2244\n",
      "Checking accuracy on validation set\n",
      "Got 640 / 1000 correct (64.00)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.3326\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.3472\n",
      "Checking accuracy on validation set\n",
      "Got 656 / 1000 correct (65.60)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.5247\n",
      "Checking accuracy on validation set\n",
      "Got 661 / 1000 correct (66.10)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.1820\n",
      "Checking accuracy on validation set\n",
      "Got 658 / 1000 correct (65.80)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.1420\n",
      "Checking accuracy on validation set\n",
      "Got 637 / 1000 correct (63.70)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.1969\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.3281\n",
      "Checking accuracy on validation set\n",
      "Got 647 / 1000 correct (64.70)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.2414\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.1687\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.3779\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.2366\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.0838\n",
      "Checking accuracy on validation set\n",
      "Got 637 / 1000 correct (63.70)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.1532\n",
      "Checking accuracy on validation set\n",
      "Got 645 / 1000 correct (64.50)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.1674\n",
      "Checking accuracy on validation set\n",
      "Got 653 / 1000 correct (65.30)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.2385\n",
      "Checking accuracy on validation set\n",
      "Got 655 / 1000 correct (65.50)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.3468\n",
      "Checking accuracy on validation set\n",
      "Got 636 / 1000 correct (63.60)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.3443\n",
      "Checking accuracy on validation set\n",
      "Got 635 / 1000 correct (63.50)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.2199\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.1448\n",
      "Checking accuracy on validation set\n",
      "Got 641 / 1000 correct (64.10)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.2181\n",
      "Checking accuracy on validation set\n",
      "Got 648 / 1000 correct (64.80)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.0716\n",
      "Checking accuracy on validation set\n",
      "Got 643 / 1000 correct (64.30)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.1760\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.1858\n",
      "Checking accuracy on validation set\n",
      "Got 649 / 1000 correct (64.90)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.2152\n",
      "Checking accuracy on validation set\n",
      "Got 654 / 1000 correct (65.40)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.1345\n",
      "Checking accuracy on validation set\n",
      "Got 636 / 1000 correct (63.60)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.3360\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.1382\n",
      "Checking accuracy on validation set\n",
      "Got 635 / 1000 correct (63.50)\n",
      "\n",
      "Maximum accuracy attained:  67.30000000000001\n",
      "Checking accuracy on test set\n",
      "Got 6339 / 10000 correct (63.39)\n",
      "Epoch 0, Iteration 0, loss = 4.7319\n",
      "Checking accuracy on validation set\n",
      "Got 79 / 1000 correct (7.90)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.6998\n",
      "Checking accuracy on validation set\n",
      "Got 370 / 1000 correct (37.00)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.4343\n",
      "Checking accuracy on validation set\n",
      "Got 415 / 1000 correct (41.50)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.5710\n",
      "Checking accuracy on validation set\n",
      "Got 418 / 1000 correct (41.80)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.2908\n",
      "Checking accuracy on validation set\n",
      "Got 505 / 1000 correct (50.50)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.1744\n",
      "Checking accuracy on validation set\n",
      "Got 511 / 1000 correct (51.10)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.4747\n",
      "Checking accuracy on validation set\n",
      "Got 501 / 1000 correct (50.10)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.1154\n",
      "Checking accuracy on validation set\n",
      "Got 558 / 1000 correct (55.80)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 0.9529\n",
      "Checking accuracy on validation set\n",
      "Got 545 / 1000 correct (54.50)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 1.0953\n",
      "Checking accuracy on validation set\n",
      "Got 539 / 1000 correct (53.90)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 1.0076\n",
      "Checking accuracy on validation set\n",
      "Got 634 / 1000 correct (63.40)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 1.1192\n",
      "Checking accuracy on validation set\n",
      "Got 606 / 1000 correct (60.60)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 1.2416\n",
      "Checking accuracy on validation set\n",
      "Got 592 / 1000 correct (59.20)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 1.0990\n",
      "Checking accuracy on validation set\n",
      "Got 657 / 1000 correct (65.70)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 0.7879\n",
      "Checking accuracy on validation set\n",
      "Got 637 / 1000 correct (63.70)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 1.0178\n",
      "Checking accuracy on validation set\n",
      "Got 650 / 1000 correct (65.00)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 0.9125\n",
      "Checking accuracy on validation set\n",
      "Got 656 / 1000 correct (65.60)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 1.0078\n",
      "Checking accuracy on validation set\n",
      "Got 662 / 1000 correct (66.20)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.8928\n",
      "Checking accuracy on validation set\n",
      "Got 667 / 1000 correct (66.70)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.7899\n",
      "Checking accuracy on validation set\n",
      "Got 651 / 1000 correct (65.10)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 0.7139\n",
      "Checking accuracy on validation set\n",
      "Got 684 / 1000 correct (68.40)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 0.6812\n",
      "Checking accuracy on validation set\n",
      "Got 692 / 1000 correct (69.20)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.8416\n",
      "Checking accuracy on validation set\n",
      "Got 673 / 1000 correct (67.30)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.9879\n",
      "Checking accuracy on validation set\n",
      "Got 686 / 1000 correct (68.60)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.7484\n",
      "Checking accuracy on validation set\n",
      "Got 621 / 1000 correct (62.10)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.8396\n",
      "Checking accuracy on validation set\n",
      "Got 687 / 1000 correct (68.70)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.7023\n",
      "Checking accuracy on validation set\n",
      "Got 709 / 1000 correct (70.90)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.5582\n",
      "Checking accuracy on validation set\n",
      "Got 676 / 1000 correct (67.60)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.6857\n",
      "Checking accuracy on validation set\n",
      "Got 673 / 1000 correct (67.30)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.9102\n",
      "Checking accuracy on validation set\n",
      "Got 680 / 1000 correct (68.00)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.7075\n",
      "Checking accuracy on validation set\n",
      "Got 721 / 1000 correct (72.10)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.8891\n",
      "Checking accuracy on validation set\n",
      "Got 707 / 1000 correct (70.70)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.6316\n",
      "Checking accuracy on validation set\n",
      "Got 743 / 1000 correct (74.30)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.8667\n",
      "Checking accuracy on validation set\n",
      "Got 705 / 1000 correct (70.50)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.9930\n",
      "Checking accuracy on validation set\n",
      "Got 728 / 1000 correct (72.80)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.4913\n",
      "Checking accuracy on validation set\n",
      "Got 700 / 1000 correct (70.00)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.9718\n",
      "Checking accuracy on validation set\n",
      "Got 708 / 1000 correct (70.80)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.7642\n",
      "Checking accuracy on validation set\n",
      "Got 734 / 1000 correct (73.40)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.9638\n",
      "Checking accuracy on validation set\n",
      "Got 731 / 1000 correct (73.10)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.8005\n",
      "Checking accuracy on validation set\n",
      "Got 704 / 1000 correct (70.40)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.4632\n",
      "Checking accuracy on validation set\n",
      "Got 727 / 1000 correct (72.70)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.5805\n",
      "Checking accuracy on validation set\n",
      "Got 719 / 1000 correct (71.90)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.5277\n",
      "Checking accuracy on validation set\n",
      "Got 728 / 1000 correct (72.80)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.7981\n",
      "Checking accuracy on validation set\n",
      "Got 735 / 1000 correct (73.50)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.4251\n",
      "Checking accuracy on validation set\n",
      "Got 718 / 1000 correct (71.80)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.6424\n",
      "Checking accuracy on validation set\n",
      "Got 758 / 1000 correct (75.80)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.5824\n",
      "Checking accuracy on validation set\n",
      "Got 734 / 1000 correct (73.40)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.3972\n",
      "Checking accuracy on validation set\n",
      "Got 741 / 1000 correct (74.10)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.4605\n",
      "Checking accuracy on validation set\n",
      "Got 725 / 1000 correct (72.50)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.7844\n",
      "Checking accuracy on validation set\n",
      "Got 769 / 1000 correct (76.90)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.5174\n",
      "Checking accuracy on validation set\n",
      "Got 776 / 1000 correct (77.60)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.5083\n",
      "Checking accuracy on validation set\n",
      "Got 732 / 1000 correct (73.20)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.3220\n",
      "Checking accuracy on validation set\n",
      "Got 749 / 1000 correct (74.90)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.6685\n",
      "Checking accuracy on validation set\n",
      "Got 732 / 1000 correct (73.20)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.4705\n",
      "Checking accuracy on validation set\n",
      "Got 742 / 1000 correct (74.20)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.4896\n",
      "Checking accuracy on validation set\n",
      "Got 763 / 1000 correct (76.30)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.3361\n",
      "Checking accuracy on validation set\n",
      "Got 735 / 1000 correct (73.50)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.5368\n",
      "Checking accuracy on validation set\n",
      "Got 748 / 1000 correct (74.80)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.6885\n",
      "Checking accuracy on validation set\n",
      "Got 731 / 1000 correct (73.10)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.3799\n",
      "Checking accuracy on validation set\n",
      "Got 762 / 1000 correct (76.20)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.5543\n",
      "Checking accuracy on validation set\n",
      "Got 759 / 1000 correct (75.90)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.3632\n",
      "Checking accuracy on validation set\n",
      "Got 746 / 1000 correct (74.60)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.4577\n",
      "Checking accuracy on validation set\n",
      "Got 761 / 1000 correct (76.10)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.6681\n",
      "Checking accuracy on validation set\n",
      "Got 752 / 1000 correct (75.20)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.4790\n",
      "Checking accuracy on validation set\n",
      "Got 733 / 1000 correct (73.30)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.5314\n",
      "Checking accuracy on validation set\n",
      "Got 780 / 1000 correct (78.00)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.3012\n",
      "Checking accuracy on validation set\n",
      "Got 754 / 1000 correct (75.40)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.5838\n",
      "Checking accuracy on validation set\n",
      "Got 768 / 1000 correct (76.80)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.3275\n",
      "Checking accuracy on validation set\n",
      "Got 757 / 1000 correct (75.70)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.2727\n",
      "Checking accuracy on validation set\n",
      "Got 749 / 1000 correct (74.90)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.3067\n",
      "Checking accuracy on validation set\n",
      "Got 749 / 1000 correct (74.90)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.3387\n",
      "Checking accuracy on validation set\n",
      "Got 749 / 1000 correct (74.90)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.4465\n",
      "Checking accuracy on validation set\n",
      "Got 772 / 1000 correct (77.20)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.3585\n",
      "Checking accuracy on validation set\n",
      "Got 759 / 1000 correct (75.90)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.4998\n",
      "Checking accuracy on validation set\n",
      "Got 767 / 1000 correct (76.70)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.3668\n",
      "Checking accuracy on validation set\n",
      "Got 757 / 1000 correct (75.70)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.3020\n",
      "Checking accuracy on validation set\n",
      "Got 743 / 1000 correct (74.30)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.8140\n",
      "Checking accuracy on validation set\n",
      "Got 748 / 1000 correct (74.80)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.4208\n",
      "Checking accuracy on validation set\n",
      "Got 771 / 1000 correct (77.10)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.2733\n",
      "Checking accuracy on validation set\n",
      "Got 756 / 1000 correct (75.60)\n",
      "\n",
      "Maximum accuracy attained:  78.0\n",
      "Checking accuracy on test set\n",
      "Got 7647 / 10000 correct (76.47)\n",
      "Epoch 0, Iteration 0, loss = 4.7114\n",
      "Checking accuracy on validation set\n",
      "Got 105 / 1000 correct (10.50)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.3889\n",
      "Checking accuracy on validation set\n",
      "Got 407 / 1000 correct (40.70)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.4160\n",
      "Checking accuracy on validation set\n",
      "Got 451 / 1000 correct (45.10)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.3875\n",
      "Checking accuracy on validation set\n",
      "Got 539 / 1000 correct (53.90)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.4823\n",
      "Checking accuracy on validation set\n",
      "Got 496 / 1000 correct (49.60)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.2685\n",
      "Checking accuracy on validation set\n",
      "Got 598 / 1000 correct (59.80)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.0164\n",
      "Checking accuracy on validation set\n",
      "Got 529 / 1000 correct (52.90)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.2775\n",
      "Checking accuracy on validation set\n",
      "Got 555 / 1000 correct (55.50)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 1.0914\n",
      "Checking accuracy on validation set\n",
      "Got 527 / 1000 correct (52.70)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 1.0501\n",
      "Checking accuracy on validation set\n",
      "Got 537 / 1000 correct (53.70)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 1.0944\n",
      "Checking accuracy on validation set\n",
      "Got 618 / 1000 correct (61.80)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 1.0740\n",
      "Checking accuracy on validation set\n",
      "Got 598 / 1000 correct (59.80)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 1.1065\n",
      "Checking accuracy on validation set\n",
      "Got 592 / 1000 correct (59.20)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 0.7581\n",
      "Checking accuracy on validation set\n",
      "Got 603 / 1000 correct (60.30)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 1.0761\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 1.0116\n",
      "Checking accuracy on validation set\n",
      "Got 660 / 1000 correct (66.00)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 0.8228\n",
      "Checking accuracy on validation set\n",
      "Got 626 / 1000 correct (62.60)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 0.8324\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.8814\n",
      "Checking accuracy on validation set\n",
      "Got 666 / 1000 correct (66.60)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.7369\n",
      "Checking accuracy on validation set\n",
      "Got 669 / 1000 correct (66.90)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 0.8795\n",
      "Checking accuracy on validation set\n",
      "Got 684 / 1000 correct (68.40)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 0.7968\n",
      "Checking accuracy on validation set\n",
      "Got 689 / 1000 correct (68.90)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.9434\n",
      "Checking accuracy on validation set\n",
      "Got 669 / 1000 correct (66.90)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.8406\n",
      "Checking accuracy on validation set\n",
      "Got 675 / 1000 correct (67.50)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.9372\n",
      "Checking accuracy on validation set\n",
      "Got 672 / 1000 correct (67.20)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.7449\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.5999\n",
      "Checking accuracy on validation set\n",
      "Got 703 / 1000 correct (70.30)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.7631\n",
      "Checking accuracy on validation set\n",
      "Got 677 / 1000 correct (67.70)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.9883\n",
      "Checking accuracy on validation set\n",
      "Got 668 / 1000 correct (66.80)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.8739\n",
      "Checking accuracy on validation set\n",
      "Got 707 / 1000 correct (70.70)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.9097\n",
      "Checking accuracy on validation set\n",
      "Got 685 / 1000 correct (68.50)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.7972\n",
      "Checking accuracy on validation set\n",
      "Got 712 / 1000 correct (71.20)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.4522\n",
      "Checking accuracy on validation set\n",
      "Got 706 / 1000 correct (70.60)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.5494\n",
      "Checking accuracy on validation set\n",
      "Got 711 / 1000 correct (71.10)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.3964\n",
      "Checking accuracy on validation set\n",
      "Got 688 / 1000 correct (68.80)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.6281\n",
      "Checking accuracy on validation set\n",
      "Got 705 / 1000 correct (70.50)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.6992\n",
      "Checking accuracy on validation set\n",
      "Got 698 / 1000 correct (69.80)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.8721\n",
      "Checking accuracy on validation set\n",
      "Got 728 / 1000 correct (72.80)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.7193\n",
      "Checking accuracy on validation set\n",
      "Got 701 / 1000 correct (70.10)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.7378\n",
      "Checking accuracy on validation set\n",
      "Got 707 / 1000 correct (70.70)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.9139\n",
      "Checking accuracy on validation set\n",
      "Got 728 / 1000 correct (72.80)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.7626\n",
      "Checking accuracy on validation set\n",
      "Got 704 / 1000 correct (70.40)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.7847\n",
      "Checking accuracy on validation set\n",
      "Got 697 / 1000 correct (69.70)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.8589\n",
      "Checking accuracy on validation set\n",
      "Got 728 / 1000 correct (72.80)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.5693\n",
      "Checking accuracy on validation set\n",
      "Got 734 / 1000 correct (73.40)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.9560\n",
      "Checking accuracy on validation set\n",
      "Got 720 / 1000 correct (72.00)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.7642\n",
      "Checking accuracy on validation set\n",
      "Got 719 / 1000 correct (71.90)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.5419\n",
      "Checking accuracy on validation set\n",
      "Got 717 / 1000 correct (71.70)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.5812\n",
      "Checking accuracy on validation set\n",
      "Got 730 / 1000 correct (73.00)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.5422\n",
      "Checking accuracy on validation set\n",
      "Got 755 / 1000 correct (75.50)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.5253\n",
      "Checking accuracy on validation set\n",
      "Got 739 / 1000 correct (73.90)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.7597\n",
      "Checking accuracy on validation set\n",
      "Got 745 / 1000 correct (74.50)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.6030\n",
      "Checking accuracy on validation set\n",
      "Got 740 / 1000 correct (74.00)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.6542\n",
      "Checking accuracy on validation set\n",
      "Got 722 / 1000 correct (72.20)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.4389\n",
      "Checking accuracy on validation set\n",
      "Got 765 / 1000 correct (76.50)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.5373\n",
      "Checking accuracy on validation set\n",
      "Got 687 / 1000 correct (68.70)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.5244\n",
      "Checking accuracy on validation set\n",
      "Got 728 / 1000 correct (72.80)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.6921\n",
      "Checking accuracy on validation set\n",
      "Got 761 / 1000 correct (76.10)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.6223\n",
      "Checking accuracy on validation set\n",
      "Got 750 / 1000 correct (75.00)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.3864\n",
      "Checking accuracy on validation set\n",
      "Got 724 / 1000 correct (72.40)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.6215\n",
      "Checking accuracy on validation set\n",
      "Got 741 / 1000 correct (74.10)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.6589\n",
      "Checking accuracy on validation set\n",
      "Got 765 / 1000 correct (76.50)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.5450\n",
      "Checking accuracy on validation set\n",
      "Got 735 / 1000 correct (73.50)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.6288\n",
      "Checking accuracy on validation set\n",
      "Got 765 / 1000 correct (76.50)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.4966\n",
      "Checking accuracy on validation set\n",
      "Got 761 / 1000 correct (76.10)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.5567\n",
      "Checking accuracy on validation set\n",
      "Got 745 / 1000 correct (74.50)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.4464\n",
      "Checking accuracy on validation set\n",
      "Got 767 / 1000 correct (76.70)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.3705\n",
      "Checking accuracy on validation set\n",
      "Got 756 / 1000 correct (75.60)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.6033\n",
      "Checking accuracy on validation set\n",
      "Got 758 / 1000 correct (75.80)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.4620\n",
      "Checking accuracy on validation set\n",
      "Got 764 / 1000 correct (76.40)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.5267\n",
      "Checking accuracy on validation set\n",
      "Got 777 / 1000 correct (77.70)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.7067\n",
      "Checking accuracy on validation set\n",
      "Got 756 / 1000 correct (75.60)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.3469\n",
      "Checking accuracy on validation set\n",
      "Got 763 / 1000 correct (76.30)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.3069\n",
      "Checking accuracy on validation set\n",
      "Got 770 / 1000 correct (77.00)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.3651\n",
      "Checking accuracy on validation set\n",
      "Got 769 / 1000 correct (76.90)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.4157\n",
      "Checking accuracy on validation set\n",
      "Got 750 / 1000 correct (75.00)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.4992\n",
      "Checking accuracy on validation set\n",
      "Got 759 / 1000 correct (75.90)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.5068\n",
      "Checking accuracy on validation set\n",
      "Got 754 / 1000 correct (75.40)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.3220\n",
      "Checking accuracy on validation set\n",
      "Got 773 / 1000 correct (77.30)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.3446\n",
      "Checking accuracy on validation set\n",
      "Got 771 / 1000 correct (77.10)\n",
      "\n",
      "Maximum accuracy attained:  77.7\n",
      "Checking accuracy on test set\n",
      "Got 7499 / 10000 correct (74.99)\n",
      "Epoch 0, Iteration 0, loss = 4.5205\n",
      "Checking accuracy on validation set\n",
      "Got 112 / 1000 correct (11.20)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.5386\n",
      "Checking accuracy on validation set\n",
      "Got 350 / 1000 correct (35.00)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.4655\n",
      "Checking accuracy on validation set\n",
      "Got 426 / 1000 correct (42.60)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.5109\n",
      "Checking accuracy on validation set\n",
      "Got 458 / 1000 correct (45.80)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.4534\n",
      "Checking accuracy on validation set\n",
      "Got 483 / 1000 correct (48.30)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.0713\n",
      "Checking accuracy on validation set\n",
      "Got 525 / 1000 correct (52.50)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.1569\n",
      "Checking accuracy on validation set\n",
      "Got 593 / 1000 correct (59.30)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.0391\n",
      "Checking accuracy on validation set\n",
      "Got 564 / 1000 correct (56.40)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 1.2434\n",
      "Checking accuracy on validation set\n",
      "Got 599 / 1000 correct (59.90)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 1.1654\n",
      "Checking accuracy on validation set\n",
      "Got 582 / 1000 correct (58.20)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 1.2103\n",
      "Checking accuracy on validation set\n",
      "Got 570 / 1000 correct (57.00)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 0.9765\n",
      "Checking accuracy on validation set\n",
      "Got 635 / 1000 correct (63.50)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 0.8608\n",
      "Checking accuracy on validation set\n",
      "Got 645 / 1000 correct (64.50)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 1.0864\n",
      "Checking accuracy on validation set\n",
      "Got 640 / 1000 correct (64.00)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 0.6510\n",
      "Checking accuracy on validation set\n",
      "Got 634 / 1000 correct (63.40)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 1.0295\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 0.6910\n",
      "Checking accuracy on validation set\n",
      "Got 636 / 1000 correct (63.60)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 0.9674\n",
      "Checking accuracy on validation set\n",
      "Got 649 / 1000 correct (64.90)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.7412\n",
      "Checking accuracy on validation set\n",
      "Got 665 / 1000 correct (66.50)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.7768\n",
      "Checking accuracy on validation set\n",
      "Got 635 / 1000 correct (63.50)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 0.5843\n",
      "Checking accuracy on validation set\n",
      "Got 669 / 1000 correct (66.90)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 0.9494\n",
      "Checking accuracy on validation set\n",
      "Got 677 / 1000 correct (67.70)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.9236\n",
      "Checking accuracy on validation set\n",
      "Got 671 / 1000 correct (67.10)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.7368\n",
      "Checking accuracy on validation set\n",
      "Got 659 / 1000 correct (65.90)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.7942\n",
      "Checking accuracy on validation set\n",
      "Got 675 / 1000 correct (67.50)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.6157\n",
      "Checking accuracy on validation set\n",
      "Got 705 / 1000 correct (70.50)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.6763\n",
      "Checking accuracy on validation set\n",
      "Got 708 / 1000 correct (70.80)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.8731\n",
      "Checking accuracy on validation set\n",
      "Got 710 / 1000 correct (71.00)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.7683\n",
      "Checking accuracy on validation set\n",
      "Got 700 / 1000 correct (70.00)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.6466\n",
      "Checking accuracy on validation set\n",
      "Got 710 / 1000 correct (71.00)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.6199\n",
      "Checking accuracy on validation set\n",
      "Got 714 / 1000 correct (71.40)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.6452\n",
      "Checking accuracy on validation set\n",
      "Got 713 / 1000 correct (71.30)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.6664\n",
      "Checking accuracy on validation set\n",
      "Got 715 / 1000 correct (71.50)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.4496\n",
      "Checking accuracy on validation set\n",
      "Got 703 / 1000 correct (70.30)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.5941\n",
      "Checking accuracy on validation set\n",
      "Got 737 / 1000 correct (73.70)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.6469\n",
      "Checking accuracy on validation set\n",
      "Got 718 / 1000 correct (71.80)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.5758\n",
      "Checking accuracy on validation set\n",
      "Got 709 / 1000 correct (70.90)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.6204\n",
      "Checking accuracy on validation set\n",
      "Got 717 / 1000 correct (71.70)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.7003\n",
      "Checking accuracy on validation set\n",
      "Got 709 / 1000 correct (70.90)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.5063\n",
      "Checking accuracy on validation set\n",
      "Got 742 / 1000 correct (74.20)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.4776\n",
      "Checking accuracy on validation set\n",
      "Got 723 / 1000 correct (72.30)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.4344\n",
      "Checking accuracy on validation set\n",
      "Got 716 / 1000 correct (71.60)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.7864\n",
      "Checking accuracy on validation set\n",
      "Got 721 / 1000 correct (72.10)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.4870\n",
      "Checking accuracy on validation set\n",
      "Got 732 / 1000 correct (73.20)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.7984\n",
      "Checking accuracy on validation set\n",
      "Got 746 / 1000 correct (74.60)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.6023\n",
      "Checking accuracy on validation set\n",
      "Got 733 / 1000 correct (73.30)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.5090\n",
      "Checking accuracy on validation set\n",
      "Got 728 / 1000 correct (72.80)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.6523\n",
      "Checking accuracy on validation set\n",
      "Got 738 / 1000 correct (73.80)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.5368\n",
      "Checking accuracy on validation set\n",
      "Got 726 / 1000 correct (72.60)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.3941\n",
      "Checking accuracy on validation set\n",
      "Got 735 / 1000 correct (73.50)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.5249\n",
      "Checking accuracy on validation set\n",
      "Got 735 / 1000 correct (73.50)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.4588\n",
      "Checking accuracy on validation set\n",
      "Got 764 / 1000 correct (76.40)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.3611\n",
      "Checking accuracy on validation set\n",
      "Got 747 / 1000 correct (74.70)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.4535\n",
      "Checking accuracy on validation set\n",
      "Got 755 / 1000 correct (75.50)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.4207\n",
      "Checking accuracy on validation set\n",
      "Got 760 / 1000 correct (76.00)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.3956\n",
      "Checking accuracy on validation set\n",
      "Got 744 / 1000 correct (74.40)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.3061\n",
      "Checking accuracy on validation set\n",
      "Got 757 / 1000 correct (75.70)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.3263\n",
      "Checking accuracy on validation set\n",
      "Got 746 / 1000 correct (74.60)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.5521\n",
      "Checking accuracy on validation set\n",
      "Got 760 / 1000 correct (76.00)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.4668\n",
      "Checking accuracy on validation set\n",
      "Got 741 / 1000 correct (74.10)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.4656\n",
      "Checking accuracy on validation set\n",
      "Got 743 / 1000 correct (74.30)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.3661\n",
      "Checking accuracy on validation set\n",
      "Got 745 / 1000 correct (74.50)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.3310\n",
      "Checking accuracy on validation set\n",
      "Got 747 / 1000 correct (74.70)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.3685\n",
      "Checking accuracy on validation set\n",
      "Got 774 / 1000 correct (77.40)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.3051\n",
      "Checking accuracy on validation set\n",
      "Got 763 / 1000 correct (76.30)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.4565\n",
      "Checking accuracy on validation set\n",
      "Got 780 / 1000 correct (78.00)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.1743\n",
      "Checking accuracy on validation set\n",
      "Got 764 / 1000 correct (76.40)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.2962\n",
      "Checking accuracy on validation set\n",
      "Got 760 / 1000 correct (76.00)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.3833\n",
      "Checking accuracy on validation set\n",
      "Got 746 / 1000 correct (74.60)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.4364\n",
      "Checking accuracy on validation set\n",
      "Got 733 / 1000 correct (73.30)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.4041\n",
      "Checking accuracy on validation set\n",
      "Got 748 / 1000 correct (74.80)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.3578\n",
      "Checking accuracy on validation set\n",
      "Got 741 / 1000 correct (74.10)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.2959\n",
      "Checking accuracy on validation set\n",
      "Got 747 / 1000 correct (74.70)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.2279\n",
      "Checking accuracy on validation set\n",
      "Got 754 / 1000 correct (75.40)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.2461\n",
      "Checking accuracy on validation set\n",
      "Got 756 / 1000 correct (75.60)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.2824\n",
      "Checking accuracy on validation set\n",
      "Got 766 / 1000 correct (76.60)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.3427\n",
      "Checking accuracy on validation set\n",
      "Got 756 / 1000 correct (75.60)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.2421\n",
      "Checking accuracy on validation set\n",
      "Got 755 / 1000 correct (75.50)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.5224\n",
      "Checking accuracy on validation set\n",
      "Got 765 / 1000 correct (76.50)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.3173\n",
      "Checking accuracy on validation set\n",
      "Got 770 / 1000 correct (77.00)\n",
      "\n",
      "Maximum accuracy attained:  78.0\n",
      "Checking accuracy on test set\n",
      "Got 7576 / 10000 correct (75.76)\n",
      "Epoch 0, Iteration 0, loss = 4.7955\n",
      "Checking accuracy on validation set\n",
      "Got 117 / 1000 correct (11.70)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.5580\n",
      "Checking accuracy on validation set\n",
      "Got 412 / 1000 correct (41.20)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.3965\n",
      "Checking accuracy on validation set\n",
      "Got 443 / 1000 correct (44.30)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.2631\n",
      "Checking accuracy on validation set\n",
      "Got 420 / 1000 correct (42.00)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.3273\n",
      "Checking accuracy on validation set\n",
      "Got 450 / 1000 correct (45.00)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.2333\n",
      "Checking accuracy on validation set\n",
      "Got 481 / 1000 correct (48.10)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.1422\n",
      "Checking accuracy on validation set\n",
      "Got 570 / 1000 correct (57.00)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.0941\n",
      "Checking accuracy on validation set\n",
      "Got 576 / 1000 correct (57.60)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 1.4151\n",
      "Checking accuracy on validation set\n",
      "Got 485 / 1000 correct (48.50)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 1.3344\n",
      "Checking accuracy on validation set\n",
      "Got 601 / 1000 correct (60.10)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 1.2227\n",
      "Checking accuracy on validation set\n",
      "Got 581 / 1000 correct (58.10)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 1.0972\n",
      "Checking accuracy on validation set\n",
      "Got 579 / 1000 correct (57.90)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 0.9910\n",
      "Checking accuracy on validation set\n",
      "Got 620 / 1000 correct (62.00)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 0.9251\n",
      "Checking accuracy on validation set\n",
      "Got 625 / 1000 correct (62.50)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 0.9598\n",
      "Checking accuracy on validation set\n",
      "Got 580 / 1000 correct (58.00)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 0.9044\n",
      "Checking accuracy on validation set\n",
      "Got 631 / 1000 correct (63.10)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 0.7793\n",
      "Checking accuracy on validation set\n",
      "Got 669 / 1000 correct (66.90)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 1.0590\n",
      "Checking accuracy on validation set\n",
      "Got 671 / 1000 correct (67.10)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.8693\n",
      "Checking accuracy on validation set\n",
      "Got 646 / 1000 correct (64.60)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 1.0169\n",
      "Checking accuracy on validation set\n",
      "Got 656 / 1000 correct (65.60)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 0.6020\n",
      "Checking accuracy on validation set\n",
      "Got 673 / 1000 correct (67.30)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 1.1517\n",
      "Checking accuracy on validation set\n",
      "Got 653 / 1000 correct (65.30)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.8586\n",
      "Checking accuracy on validation set\n",
      "Got 677 / 1000 correct (67.70)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.9161\n",
      "Checking accuracy on validation set\n",
      "Got 657 / 1000 correct (65.70)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.6508\n",
      "Checking accuracy on validation set\n",
      "Got 700 / 1000 correct (70.00)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.8259\n",
      "Checking accuracy on validation set\n",
      "Got 687 / 1000 correct (68.70)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.5233\n",
      "Checking accuracy on validation set\n",
      "Got 705 / 1000 correct (70.50)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.8344\n",
      "Checking accuracy on validation set\n",
      "Got 698 / 1000 correct (69.80)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.7865\n",
      "Checking accuracy on validation set\n",
      "Got 693 / 1000 correct (69.30)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.8475\n",
      "Checking accuracy on validation set\n",
      "Got 669 / 1000 correct (66.90)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.5445\n",
      "Checking accuracy on validation set\n",
      "Got 716 / 1000 correct (71.60)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.8353\n",
      "Checking accuracy on validation set\n",
      "Got 704 / 1000 correct (70.40)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.6440\n",
      "Checking accuracy on validation set\n",
      "Got 694 / 1000 correct (69.40)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.5499\n",
      "Checking accuracy on validation set\n",
      "Got 719 / 1000 correct (71.90)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.4963\n",
      "Checking accuracy on validation set\n",
      "Got 711 / 1000 correct (71.10)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.6394\n",
      "Checking accuracy on validation set\n",
      "Got 736 / 1000 correct (73.60)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.5642\n",
      "Checking accuracy on validation set\n",
      "Got 715 / 1000 correct (71.50)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.7538\n",
      "Checking accuracy on validation set\n",
      "Got 703 / 1000 correct (70.30)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.5796\n",
      "Checking accuracy on validation set\n",
      "Got 722 / 1000 correct (72.20)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.7599\n",
      "Checking accuracy on validation set\n",
      "Got 717 / 1000 correct (71.70)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.7743\n",
      "Checking accuracy on validation set\n",
      "Got 735 / 1000 correct (73.50)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.6617\n",
      "Checking accuracy on validation set\n",
      "Got 734 / 1000 correct (73.40)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.5924\n",
      "Checking accuracy on validation set\n",
      "Got 726 / 1000 correct (72.60)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.6965\n",
      "Checking accuracy on validation set\n",
      "Got 739 / 1000 correct (73.90)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.6008\n",
      "Checking accuracy on validation set\n",
      "Got 726 / 1000 correct (72.60)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.9036\n",
      "Checking accuracy on validation set\n",
      "Got 755 / 1000 correct (75.50)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.7843\n",
      "Checking accuracy on validation set\n",
      "Got 745 / 1000 correct (74.50)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.6816\n",
      "Checking accuracy on validation set\n",
      "Got 743 / 1000 correct (74.30)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.5375\n",
      "Checking accuracy on validation set\n",
      "Got 751 / 1000 correct (75.10)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.5127\n",
      "Checking accuracy on validation set\n",
      "Got 732 / 1000 correct (73.20)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.5913\n",
      "Checking accuracy on validation set\n",
      "Got 725 / 1000 correct (72.50)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.7149\n",
      "Checking accuracy on validation set\n",
      "Got 755 / 1000 correct (75.50)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.5218\n",
      "Checking accuracy on validation set\n",
      "Got 750 / 1000 correct (75.00)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.6610\n",
      "Checking accuracy on validation set\n",
      "Got 751 / 1000 correct (75.10)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.7022\n",
      "Checking accuracy on validation set\n",
      "Got 729 / 1000 correct (72.90)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.7730\n",
      "Checking accuracy on validation set\n",
      "Got 737 / 1000 correct (73.70)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.4542\n",
      "Checking accuracy on validation set\n",
      "Got 759 / 1000 correct (75.90)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.4204\n",
      "Checking accuracy on validation set\n",
      "Got 728 / 1000 correct (72.80)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.5967\n",
      "Checking accuracy on validation set\n",
      "Got 755 / 1000 correct (75.50)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.4309\n",
      "Checking accuracy on validation set\n",
      "Got 752 / 1000 correct (75.20)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.5562\n",
      "Checking accuracy on validation set\n",
      "Got 756 / 1000 correct (75.60)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.4357\n",
      "Checking accuracy on validation set\n",
      "Got 749 / 1000 correct (74.90)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.5735\n",
      "Checking accuracy on validation set\n",
      "Got 741 / 1000 correct (74.10)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.3458\n",
      "Checking accuracy on validation set\n",
      "Got 758 / 1000 correct (75.80)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.4027\n",
      "Checking accuracy on validation set\n",
      "Got 767 / 1000 correct (76.70)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.4496\n",
      "Checking accuracy on validation set\n",
      "Got 754 / 1000 correct (75.40)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.2816\n",
      "Checking accuracy on validation set\n",
      "Got 764 / 1000 correct (76.40)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.5096\n",
      "Checking accuracy on validation set\n",
      "Got 769 / 1000 correct (76.90)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.3879\n",
      "Checking accuracy on validation set\n",
      "Got 761 / 1000 correct (76.10)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.4371\n",
      "Checking accuracy on validation set\n",
      "Got 761 / 1000 correct (76.10)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.3553\n",
      "Checking accuracy on validation set\n",
      "Got 744 / 1000 correct (74.40)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.3611\n",
      "Checking accuracy on validation set\n",
      "Got 743 / 1000 correct (74.30)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.1459\n",
      "Checking accuracy on validation set\n",
      "Got 752 / 1000 correct (75.20)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.2162\n",
      "Checking accuracy on validation set\n",
      "Got 764 / 1000 correct (76.40)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.3739\n",
      "Checking accuracy on validation set\n",
      "Got 766 / 1000 correct (76.60)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.2656\n",
      "Checking accuracy on validation set\n",
      "Got 752 / 1000 correct (75.20)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.4089\n",
      "Checking accuracy on validation set\n",
      "Got 744 / 1000 correct (74.40)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.1632\n",
      "Checking accuracy on validation set\n",
      "Got 760 / 1000 correct (76.00)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.3711\n",
      "Checking accuracy on validation set\n",
      "Got 751 / 1000 correct (75.10)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.3712\n",
      "Checking accuracy on validation set\n",
      "Got 741 / 1000 correct (74.10)\n",
      "\n",
      "Maximum accuracy attained:  76.9\n",
      "Checking accuracy on test set\n",
      "Got 7589 / 10000 correct (75.89)\n",
      "Epoch 0, Iteration 0, loss = 4.6754\n",
      "Checking accuracy on validation set\n",
      "Got 105 / 1000 correct (10.50)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.5127\n",
      "Checking accuracy on validation set\n",
      "Got 444 / 1000 correct (44.40)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.5267\n",
      "Checking accuracy on validation set\n",
      "Got 426 / 1000 correct (42.60)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.3271\n",
      "Checking accuracy on validation set\n",
      "Got 520 / 1000 correct (52.00)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.3624\n",
      "Checking accuracy on validation set\n",
      "Got 529 / 1000 correct (52.90)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.2451\n",
      "Checking accuracy on validation set\n",
      "Got 478 / 1000 correct (47.80)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.0894\n",
      "Checking accuracy on validation set\n",
      "Got 553 / 1000 correct (55.30)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.0207\n",
      "Checking accuracy on validation set\n",
      "Got 604 / 1000 correct (60.40)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 1.2835\n",
      "Checking accuracy on validation set\n",
      "Got 582 / 1000 correct (58.20)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 1.3675\n",
      "Checking accuracy on validation set\n",
      "Got 581 / 1000 correct (58.10)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 1.0983\n",
      "Checking accuracy on validation set\n",
      "Got 624 / 1000 correct (62.40)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 1.0017\n",
      "Checking accuracy on validation set\n",
      "Got 595 / 1000 correct (59.50)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 1.1335\n",
      "Checking accuracy on validation set\n",
      "Got 503 / 1000 correct (50.30)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 0.8881\n",
      "Checking accuracy on validation set\n",
      "Got 628 / 1000 correct (62.80)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 0.9677\n",
      "Checking accuracy on validation set\n",
      "Got 597 / 1000 correct (59.70)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 1.1097\n",
      "Checking accuracy on validation set\n",
      "Got 661 / 1000 correct (66.10)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 0.9456\n",
      "Checking accuracy on validation set\n",
      "Got 638 / 1000 correct (63.80)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 1.0557\n",
      "Checking accuracy on validation set\n",
      "Got 668 / 1000 correct (66.80)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.9034\n",
      "Checking accuracy on validation set\n",
      "Got 633 / 1000 correct (63.30)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 0.9949\n",
      "Checking accuracy on validation set\n",
      "Got 667 / 1000 correct (66.70)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 1.0227\n",
      "Checking accuracy on validation set\n",
      "Got 691 / 1000 correct (69.10)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 0.6193\n",
      "Checking accuracy on validation set\n",
      "Got 707 / 1000 correct (70.70)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.8523\n",
      "Checking accuracy on validation set\n",
      "Got 664 / 1000 correct (66.40)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.6885\n",
      "Checking accuracy on validation set\n",
      "Got 700 / 1000 correct (70.00)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.6646\n",
      "Checking accuracy on validation set\n",
      "Got 687 / 1000 correct (68.70)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.6054\n",
      "Checking accuracy on validation set\n",
      "Got 700 / 1000 correct (70.00)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.6451\n",
      "Checking accuracy on validation set\n",
      "Got 668 / 1000 correct (66.80)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.7653\n",
      "Checking accuracy on validation set\n",
      "Got 703 / 1000 correct (70.30)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.6801\n",
      "Checking accuracy on validation set\n",
      "Got 717 / 1000 correct (71.70)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.5635\n",
      "Checking accuracy on validation set\n",
      "Got 712 / 1000 correct (71.20)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.7005\n",
      "Checking accuracy on validation set\n",
      "Got 705 / 1000 correct (70.50)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.7271\n",
      "Checking accuracy on validation set\n",
      "Got 732 / 1000 correct (73.20)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.6484\n",
      "Checking accuracy on validation set\n",
      "Got 705 / 1000 correct (70.50)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.6824\n",
      "Checking accuracy on validation set\n",
      "Got 731 / 1000 correct (73.10)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.6196\n",
      "Checking accuracy on validation set\n",
      "Got 713 / 1000 correct (71.30)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.5540\n",
      "Checking accuracy on validation set\n",
      "Got 716 / 1000 correct (71.60)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.7565\n",
      "Checking accuracy on validation set\n",
      "Got 701 / 1000 correct (70.10)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.6802\n",
      "Checking accuracy on validation set\n",
      "Got 726 / 1000 correct (72.60)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.6444\n",
      "Checking accuracy on validation set\n",
      "Got 722 / 1000 correct (72.20)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.7207\n",
      "Checking accuracy on validation set\n",
      "Got 718 / 1000 correct (71.80)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.6258\n",
      "Checking accuracy on validation set\n",
      "Got 754 / 1000 correct (75.40)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.4647\n",
      "Checking accuracy on validation set\n",
      "Got 756 / 1000 correct (75.60)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.6376\n",
      "Checking accuracy on validation set\n",
      "Got 742 / 1000 correct (74.20)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.5547\n",
      "Checking accuracy on validation set\n",
      "Got 779 / 1000 correct (77.90)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.6350\n",
      "Checking accuracy on validation set\n",
      "Got 756 / 1000 correct (75.60)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.5890\n",
      "Checking accuracy on validation set\n",
      "Got 734 / 1000 correct (73.40)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.6231\n",
      "Checking accuracy on validation set\n",
      "Got 745 / 1000 correct (74.50)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.5711\n",
      "Checking accuracy on validation set\n",
      "Got 743 / 1000 correct (74.30)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.4147\n",
      "Checking accuracy on validation set\n",
      "Got 743 / 1000 correct (74.30)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.6414\n",
      "Checking accuracy on validation set\n",
      "Got 739 / 1000 correct (73.90)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.6360\n",
      "Checking accuracy on validation set\n",
      "Got 752 / 1000 correct (75.20)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.4646\n",
      "Checking accuracy on validation set\n",
      "Got 763 / 1000 correct (76.30)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.5293\n",
      "Checking accuracy on validation set\n",
      "Got 756 / 1000 correct (75.60)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.5468\n",
      "Checking accuracy on validation set\n",
      "Got 751 / 1000 correct (75.10)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.4039\n",
      "Checking accuracy on validation set\n",
      "Got 757 / 1000 correct (75.70)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.5365\n",
      "Checking accuracy on validation set\n",
      "Got 763 / 1000 correct (76.30)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.4983\n",
      "Checking accuracy on validation set\n",
      "Got 740 / 1000 correct (74.00)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.1945\n",
      "Checking accuracy on validation set\n",
      "Got 765 / 1000 correct (76.50)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.3210\n",
      "Checking accuracy on validation set\n",
      "Got 754 / 1000 correct (75.40)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.4506\n",
      "Checking accuracy on validation set\n",
      "Got 756 / 1000 correct (75.60)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.6202\n",
      "Checking accuracy on validation set\n",
      "Got 764 / 1000 correct (76.40)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.5138\n",
      "Checking accuracy on validation set\n",
      "Got 769 / 1000 correct (76.90)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.4851\n",
      "Checking accuracy on validation set\n",
      "Got 734 / 1000 correct (73.40)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.5392\n",
      "Checking accuracy on validation set\n",
      "Got 765 / 1000 correct (76.50)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.3241\n",
      "Checking accuracy on validation set\n",
      "Got 738 / 1000 correct (73.80)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.3789\n",
      "Checking accuracy on validation set\n",
      "Got 772 / 1000 correct (77.20)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.3853\n",
      "Checking accuracy on validation set\n",
      "Got 778 / 1000 correct (77.80)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.3688\n",
      "Checking accuracy on validation set\n",
      "Got 775 / 1000 correct (77.50)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.1691\n",
      "Checking accuracy on validation set\n",
      "Got 760 / 1000 correct (76.00)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.3163\n",
      "Checking accuracy on validation set\n",
      "Got 749 / 1000 correct (74.90)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.3137\n",
      "Checking accuracy on validation set\n",
      "Got 790 / 1000 correct (79.00)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.3710\n",
      "Checking accuracy on validation set\n",
      "Got 762 / 1000 correct (76.20)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.3570\n",
      "Checking accuracy on validation set\n",
      "Got 757 / 1000 correct (75.70)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.1855\n",
      "Checking accuracy on validation set\n",
      "Got 772 / 1000 correct (77.20)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.1400\n",
      "Checking accuracy on validation set\n",
      "Got 780 / 1000 correct (78.00)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.3354\n",
      "Checking accuracy on validation set\n",
      "Got 766 / 1000 correct (76.60)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.3734\n",
      "Checking accuracy on validation set\n",
      "Got 771 / 1000 correct (77.10)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.4241\n",
      "Checking accuracy on validation set\n",
      "Got 751 / 1000 correct (75.10)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.3983\n",
      "Checking accuracy on validation set\n",
      "Got 760 / 1000 correct (76.00)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.2967\n",
      "Checking accuracy on validation set\n",
      "Got 782 / 1000 correct (78.20)\n",
      "\n",
      "Maximum accuracy attained:  79.0\n",
      "Checking accuracy on test set\n",
      "Got 7619 / 10000 correct (76.19)\n",
      "Epoch 0, Iteration 0, loss = 4.5860\n",
      "Checking accuracy on validation set\n",
      "Got 119 / 1000 correct (11.90)\n",
      "\n",
      "Epoch 0, Iteration 100, loss = 1.5159\n",
      "Checking accuracy on validation set\n",
      "Got 429 / 1000 correct (42.90)\n",
      "\n",
      "Epoch 0, Iteration 200, loss = 1.3430\n",
      "Checking accuracy on validation set\n",
      "Got 463 / 1000 correct (46.30)\n",
      "\n",
      "Epoch 0, Iteration 300, loss = 1.3530\n",
      "Checking accuracy on validation set\n",
      "Got 490 / 1000 correct (49.00)\n",
      "\n",
      "Epoch 0, Iteration 400, loss = 1.3376\n",
      "Checking accuracy on validation set\n",
      "Got 520 / 1000 correct (52.00)\n",
      "\n",
      "Epoch 0, Iteration 500, loss = 1.0456\n",
      "Checking accuracy on validation set\n",
      "Got 537 / 1000 correct (53.70)\n",
      "\n",
      "Epoch 0, Iteration 600, loss = 1.0062\n",
      "Checking accuracy on validation set\n",
      "Got 550 / 1000 correct (55.00)\n",
      "\n",
      "Epoch 0, Iteration 700, loss = 1.0628\n",
      "Checking accuracy on validation set\n",
      "Got 581 / 1000 correct (58.10)\n",
      "\n",
      "Epoch 1, Iteration 0, loss = 1.1840\n",
      "Checking accuracy on validation set\n",
      "Got 564 / 1000 correct (56.40)\n",
      "\n",
      "Epoch 1, Iteration 100, loss = 1.1636\n",
      "Checking accuracy on validation set\n",
      "Got 562 / 1000 correct (56.20)\n",
      "\n",
      "Epoch 1, Iteration 200, loss = 1.2020\n",
      "Checking accuracy on validation set\n",
      "Got 605 / 1000 correct (60.50)\n",
      "\n",
      "Epoch 1, Iteration 300, loss = 1.1170\n",
      "Checking accuracy on validation set\n",
      "Got 541 / 1000 correct (54.10)\n",
      "\n",
      "Epoch 1, Iteration 400, loss = 1.1692\n",
      "Checking accuracy on validation set\n",
      "Got 607 / 1000 correct (60.70)\n",
      "\n",
      "Epoch 1, Iteration 500, loss = 1.2081\n",
      "Checking accuracy on validation set\n",
      "Got 608 / 1000 correct (60.80)\n",
      "\n",
      "Epoch 1, Iteration 600, loss = 1.0567\n",
      "Checking accuracy on validation set\n",
      "Got 621 / 1000 correct (62.10)\n",
      "\n",
      "Epoch 1, Iteration 700, loss = 0.8210\n",
      "Checking accuracy on validation set\n",
      "Got 644 / 1000 correct (64.40)\n",
      "\n",
      "Epoch 2, Iteration 0, loss = 0.8579\n",
      "Checking accuracy on validation set\n",
      "Got 652 / 1000 correct (65.20)\n",
      "\n",
      "Epoch 2, Iteration 100, loss = 1.1148\n",
      "Checking accuracy on validation set\n",
      "Got 677 / 1000 correct (67.70)\n",
      "\n",
      "Epoch 2, Iteration 200, loss = 0.8651\n",
      "Checking accuracy on validation set\n",
      "Got 670 / 1000 correct (67.00)\n",
      "\n",
      "Epoch 2, Iteration 300, loss = 1.0729\n",
      "Checking accuracy on validation set\n",
      "Got 657 / 1000 correct (65.70)\n",
      "\n",
      "Epoch 2, Iteration 400, loss = 1.0063\n",
      "Checking accuracy on validation set\n",
      "Got 679 / 1000 correct (67.90)\n",
      "\n",
      "Epoch 2, Iteration 500, loss = 0.7355\n",
      "Checking accuracy on validation set\n",
      "Got 658 / 1000 correct (65.80)\n",
      "\n",
      "Epoch 2, Iteration 600, loss = 0.8600\n",
      "Checking accuracy on validation set\n",
      "Got 676 / 1000 correct (67.60)\n",
      "\n",
      "Epoch 2, Iteration 700, loss = 0.8756\n",
      "Checking accuracy on validation set\n",
      "Got 693 / 1000 correct (69.30)\n",
      "\n",
      "Epoch 3, Iteration 0, loss = 0.7903\n",
      "Checking accuracy on validation set\n",
      "Got 683 / 1000 correct (68.30)\n",
      "\n",
      "Epoch 3, Iteration 100, loss = 0.6688\n",
      "Checking accuracy on validation set\n",
      "Got 718 / 1000 correct (71.80)\n",
      "\n",
      "Epoch 3, Iteration 200, loss = 0.8287\n",
      "Checking accuracy on validation set\n",
      "Got 679 / 1000 correct (67.90)\n",
      "\n",
      "Epoch 3, Iteration 300, loss = 0.9284\n",
      "Checking accuracy on validation set\n",
      "Got 676 / 1000 correct (67.60)\n",
      "\n",
      "Epoch 3, Iteration 400, loss = 0.8458\n",
      "Checking accuracy on validation set\n",
      "Got 693 / 1000 correct (69.30)\n",
      "\n",
      "Epoch 3, Iteration 500, loss = 0.6362\n",
      "Checking accuracy on validation set\n",
      "Got 736 / 1000 correct (73.60)\n",
      "\n",
      "Epoch 3, Iteration 600, loss = 0.7443\n",
      "Checking accuracy on validation set\n",
      "Got 727 / 1000 correct (72.70)\n",
      "\n",
      "Epoch 3, Iteration 700, loss = 0.8930\n",
      "Checking accuracy on validation set\n",
      "Got 746 / 1000 correct (74.60)\n",
      "\n",
      "Epoch 4, Iteration 0, loss = 0.6287\n",
      "Checking accuracy on validation set\n",
      "Got 721 / 1000 correct (72.10)\n",
      "\n",
      "Epoch 4, Iteration 100, loss = 0.9871\n",
      "Checking accuracy on validation set\n",
      "Got 715 / 1000 correct (71.50)\n",
      "\n",
      "Epoch 4, Iteration 200, loss = 0.6514\n",
      "Checking accuracy on validation set\n",
      "Got 722 / 1000 correct (72.20)\n",
      "\n",
      "Epoch 4, Iteration 300, loss = 0.7023\n",
      "Checking accuracy on validation set\n",
      "Got 707 / 1000 correct (70.70)\n",
      "\n",
      "Epoch 4, Iteration 400, loss = 0.6180\n",
      "Checking accuracy on validation set\n",
      "Got 736 / 1000 correct (73.60)\n",
      "\n",
      "Epoch 4, Iteration 500, loss = 0.8274\n",
      "Checking accuracy on validation set\n",
      "Got 733 / 1000 correct (73.30)\n",
      "\n",
      "Epoch 4, Iteration 600, loss = 0.6705\n",
      "Checking accuracy on validation set\n",
      "Got 721 / 1000 correct (72.10)\n",
      "\n",
      "Epoch 4, Iteration 700, loss = 0.4899\n",
      "Checking accuracy on validation set\n",
      "Got 740 / 1000 correct (74.00)\n",
      "\n",
      "Epoch 5, Iteration 0, loss = 0.7063\n",
      "Checking accuracy on validation set\n",
      "Got 739 / 1000 correct (73.90)\n",
      "\n",
      "Epoch 5, Iteration 100, loss = 0.6523\n",
      "Checking accuracy on validation set\n",
      "Got 727 / 1000 correct (72.70)\n",
      "\n",
      "Epoch 5, Iteration 200, loss = 0.6487\n",
      "Checking accuracy on validation set\n",
      "Got 741 / 1000 correct (74.10)\n",
      "\n",
      "Epoch 5, Iteration 300, loss = 0.5792\n",
      "Checking accuracy on validation set\n",
      "Got 735 / 1000 correct (73.50)\n",
      "\n",
      "Epoch 5, Iteration 400, loss = 0.6376\n",
      "Checking accuracy on validation set\n",
      "Got 713 / 1000 correct (71.30)\n",
      "\n",
      "Epoch 5, Iteration 500, loss = 0.7478\n",
      "Checking accuracy on validation set\n",
      "Got 750 / 1000 correct (75.00)\n",
      "\n",
      "Epoch 5, Iteration 600, loss = 0.6060\n",
      "Checking accuracy on validation set\n",
      "Got 737 / 1000 correct (73.70)\n",
      "\n",
      "Epoch 5, Iteration 700, loss = 0.4816\n",
      "Checking accuracy on validation set\n",
      "Got 766 / 1000 correct (76.60)\n",
      "\n",
      "Epoch 6, Iteration 0, loss = 0.3898\n",
      "Checking accuracy on validation set\n",
      "Got 765 / 1000 correct (76.50)\n",
      "\n",
      "Epoch 6, Iteration 100, loss = 0.7553\n",
      "Checking accuracy on validation set\n",
      "Got 770 / 1000 correct (77.00)\n",
      "\n",
      "Epoch 6, Iteration 200, loss = 0.3805\n",
      "Checking accuracy on validation set\n",
      "Got 747 / 1000 correct (74.70)\n",
      "\n",
      "Epoch 6, Iteration 300, loss = 0.4605\n",
      "Checking accuracy on validation set\n",
      "Got 740 / 1000 correct (74.00)\n",
      "\n",
      "Epoch 6, Iteration 400, loss = 0.6952\n",
      "Checking accuracy on validation set\n",
      "Got 739 / 1000 correct (73.90)\n",
      "\n",
      "Epoch 6, Iteration 500, loss = 0.6683\n",
      "Checking accuracy on validation set\n",
      "Got 763 / 1000 correct (76.30)\n",
      "\n",
      "Epoch 6, Iteration 600, loss = 0.4838\n",
      "Checking accuracy on validation set\n",
      "Got 728 / 1000 correct (72.80)\n",
      "\n",
      "Epoch 6, Iteration 700, loss = 0.5598\n",
      "Checking accuracy on validation set\n",
      "Got 765 / 1000 correct (76.50)\n",
      "\n",
      "Epoch 7, Iteration 0, loss = 0.4871\n",
      "Checking accuracy on validation set\n",
      "Got 780 / 1000 correct (78.00)\n",
      "\n",
      "Epoch 7, Iteration 100, loss = 0.5938\n",
      "Checking accuracy on validation set\n",
      "Got 757 / 1000 correct (75.70)\n",
      "\n",
      "Epoch 7, Iteration 200, loss = 0.3090\n",
      "Checking accuracy on validation set\n",
      "Got 754 / 1000 correct (75.40)\n",
      "\n",
      "Epoch 7, Iteration 300, loss = 0.5879\n",
      "Checking accuracy on validation set\n",
      "Got 771 / 1000 correct (77.10)\n",
      "\n",
      "Epoch 7, Iteration 400, loss = 0.4428\n",
      "Checking accuracy on validation set\n",
      "Got 753 / 1000 correct (75.30)\n",
      "\n",
      "Epoch 7, Iteration 500, loss = 0.4305\n",
      "Checking accuracy on validation set\n",
      "Got 755 / 1000 correct (75.50)\n",
      "\n",
      "Epoch 7, Iteration 600, loss = 0.3456\n",
      "Checking accuracy on validation set\n",
      "Got 743 / 1000 correct (74.30)\n",
      "\n",
      "Epoch 7, Iteration 700, loss = 0.5998\n",
      "Checking accuracy on validation set\n",
      "Got 752 / 1000 correct (75.20)\n",
      "\n",
      "Epoch 8, Iteration 0, loss = 0.4348\n",
      "Checking accuracy on validation set\n",
      "Got 769 / 1000 correct (76.90)\n",
      "\n",
      "Epoch 8, Iteration 100, loss = 0.4288\n",
      "Checking accuracy on validation set\n",
      "Got 766 / 1000 correct (76.60)\n",
      "\n",
      "Epoch 8, Iteration 200, loss = 0.3982\n",
      "Checking accuracy on validation set\n",
      "Got 763 / 1000 correct (76.30)\n",
      "\n",
      "Epoch 8, Iteration 300, loss = 0.5536\n",
      "Checking accuracy on validation set\n",
      "Got 745 / 1000 correct (74.50)\n",
      "\n",
      "Epoch 8, Iteration 400, loss = 0.5946\n",
      "Checking accuracy on validation set\n",
      "Got 752 / 1000 correct (75.20)\n",
      "\n",
      "Epoch 8, Iteration 500, loss = 0.5651\n",
      "Checking accuracy on validation set\n",
      "Got 750 / 1000 correct (75.00)\n",
      "\n",
      "Epoch 8, Iteration 600, loss = 0.2966\n",
      "Checking accuracy on validation set\n",
      "Got 760 / 1000 correct (76.00)\n",
      "\n",
      "Epoch 8, Iteration 700, loss = 0.5334\n",
      "Checking accuracy on validation set\n",
      "Got 753 / 1000 correct (75.30)\n",
      "\n",
      "Epoch 9, Iteration 0, loss = 0.3681\n",
      "Checking accuracy on validation set\n",
      "Got 763 / 1000 correct (76.30)\n",
      "\n",
      "Epoch 9, Iteration 100, loss = 0.2114\n",
      "Checking accuracy on validation set\n",
      "Got 771 / 1000 correct (77.10)\n",
      "\n",
      "Epoch 9, Iteration 200, loss = 0.3314\n",
      "Checking accuracy on validation set\n",
      "Got 768 / 1000 correct (76.80)\n",
      "\n",
      "Epoch 9, Iteration 300, loss = 0.2503\n",
      "Checking accuracy on validation set\n",
      "Got 768 / 1000 correct (76.80)\n",
      "\n",
      "Epoch 9, Iteration 400, loss = 0.3529\n",
      "Checking accuracy on validation set\n",
      "Got 775 / 1000 correct (77.50)\n",
      "\n",
      "Epoch 9, Iteration 500, loss = 0.4722\n",
      "Checking accuracy on validation set\n",
      "Got 747 / 1000 correct (74.70)\n",
      "\n",
      "Epoch 9, Iteration 600, loss = 0.2535\n",
      "Checking accuracy on validation set\n",
      "Got 743 / 1000 correct (74.30)\n",
      "\n",
      "Epoch 9, Iteration 700, loss = 0.3358\n",
      "Checking accuracy on validation set\n",
      "Got 788 / 1000 correct (78.80)\n",
      "\n",
      "Maximum accuracy attained:  78.8\n",
      "Checking accuracy on test set\n",
      "Got 7601 / 10000 correct (76.01)\n"
     ]
    }
   ],
   "source": [
    "#Training all the models three times and reporting the average accuracy\n",
    "\n",
    "models = ['vanilla_cnn',\n",
    "          'early_attention',\n",
    "          'late_attention',\n",
    "          'double_attention',\n",
    "          'vanilla_resnet',\n",
    "          'resnet_w_attention']\n",
    "\n",
    "model_acc = {}\n",
    "\n",
    "learning_rate = 1e-3\n",
    "\n",
    "epochs = 10\n",
    "\n",
    "for i in models:\n",
    "    test_acc = []\n",
    "    for j in range(3):\n",
    "        model = get_model_def(i)\n",
    "\n",
    "        optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "        \n",
    "        train(model, optimizer, epochs=epochs)\n",
    "        \n",
    "        acc = check_accuracy(loader_test, model)\n",
    "        \n",
    "        test_acc.append(acc)\n",
    "        \n",
    "    model_acc[i] = test_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'vanilla_cnn': [58.79, 56.489999999999995, 58.8],\n",
       " 'early_attention': [61.09, 60.019999999999996, 60.550000000000004],\n",
       " 'late_attention': [60.480000000000004, 62.19, 59.599999999999994],\n",
       " 'double_attention': [63.27, 62.78, 63.39],\n",
       " 'vanilla_resnet': [76.47, 74.99, 75.76],\n",
       " 'resnet_w_attention': [75.89, 76.19, 76.01]}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Test accuracies of each model\n",
    "model_acc  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: vanilla_cnn Average Accuracy: 58.03%\n",
      "Model: early_attention Average Accuracy: 60.55%\n",
      "Model: late_attention Average Accuracy: 60.76%\n",
      "Model: double_attention Average Accuracy: 63.15%\n",
      "Model: vanilla_resnet Average Accuracy: 75.74%\n",
      "Model: resnet_w_attention Average Accuracy: 76.03%\n"
     ]
    }
   ],
   "source": [
    "for i in model_acc:\n",
    "    print(\"Model: \"+i+\" Average Accuracy: \"+str(round(sum(model_acc[i])/3,2))+\"%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ranking the Models:\n",
    "\n",
    "|Rank|Model|Average Accuracy|\n",
    "|----|-------- |-------|\n",
    "|1.|ResNet with Attention|76.03%|\n",
    "|2.|Vanilla ResNet|75.74%|\n",
    "|3.|Double Attention|63.15%|\n",
    "|4.|Late Attention|60.76%|\n",
    "|5.|Early Attention|60.55%|\n",
    "|6.|Vanilla CNN|58.03%|"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "assignment3_solutions.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "149c512369044de1951159ab8d9ec5ed": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e3796ddf7334425dbc86e68ff37d8a18",
      "max": 170498071,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_e7cb787fa0d743c08b5d4c395a7431f7",
      "value": 170498071
     }
    },
    "15e4679b552e4d9c85a490b59222d1cf": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4dfce8d8475644e69a52e20410e4fd1b",
      "placeholder": "​",
      "style": "IPY_MODEL_d080590971d14b158aea92032dfebecc",
      "value": " 170499072/? [00:18&lt;00:00, 9456903.71it/s]"
     }
    },
    "4dfce8d8475644e69a52e20410e4fd1b": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6646a364fe5b431eb578707bae8329b8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_149c512369044de1951159ab8d9ec5ed",
       "IPY_MODEL_15e4679b552e4d9c85a490b59222d1cf"
      ],
      "layout": "IPY_MODEL_92a914b698524f5ebd49603d108fbff9"
     }
    },
    "92a914b698524f5ebd49603d108fbff9": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d080590971d14b158aea92032dfebecc": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "e3796ddf7334425dbc86e68ff37d8a18": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e7cb787fa0d743c08b5d4c395a7431f7": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}