Jun20/Discussion1/distill_basic_teacher.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import required packages and limit GPU usage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import pickle\n",
    "import argparse\n",
    "import time\n",
    "import itertools\n",
    "from copy import deepcopy\n",
    "import os\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "import networks\n",
    "import utils\n",
    "    \n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_gpu = True    # set use_gpu to True if system has gpu\n",
    "gpu_id = 2        # id of gpu to be used\n",
    "cpu_device = torch.device('cpu')\n",
    "# fast_device is where computation (training, inference) happens\n",
    "fast_device = torch.device('cpu')\n",
    "if use_gpu:\n",
    "    os.environ['CUDA_VISIBLE_DEVICES'] = '0,1,2'    # set visible devices depending on system configuration\n",
    "    fast_device = torch.device('cuda:' + str(gpu_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reproducibilitySeed():\n",
    "    \"\"\"\n",
    "    Ensure reproducibility of results; Seeds to 0\n",
    "    \"\"\"\n",
    "    torch_init_seed = 0\n",
    "    torch.manual_seed(torch_init_seed)\n",
    "    numpy_init_seed = 0\n",
    "    np.random.seed(numpy_init_seed)\n",
    "    if use_gpu:\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "\n",
    "reproducibilitySeed()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "mnist_image_shape = (28, 28)\n",
    "random_pad_size = 2\n",
    "# Training images augmented by randomly shifting images by at max. 2 pixels in any of 4 directions\n",
    "transform_train = transforms.Compose(\n",
    "                [\n",
    "                    transforms.RandomCrop(mnist_image_shape, random_pad_size),\n",
    "                    transforms.ToTensor(),\n",
    "                    transforms.Normalize((0.5, 0.5), (0.5, 0.5))\n",
    "                ]\n",
    "            )\n",
    "\n",
    "transform_test = transforms.Compose(\n",
    "                [\n",
    "                    transforms.ToTensor(),\n",
    "                    transforms.Normalize((0.5, 0.5), (0.5, 0.5))\n",
    "                ]\n",
    "            )\n",
    "\n",
    "train_val_dataset = torchvision.datasets.MNIST(root='./MNIST_dataset/', train=True, \n",
    "                                            download=True, transform=transform_train)\n",
    "\n",
    "test_dataset = torchvision.datasets.MNIST(root='./MNIST_dataset/', train=False, \n",
    "                                            download=True, transform=transform_test)\n",
    "\n",
    "num_train = int(1.0 * len(train_val_dataset) * 95 / 100)\n",
    "num_val = len(train_val_dataset) - num_train\n",
    "train_dataset, val_dataset = torch.utils.data.random_split(train_val_dataset, [num_train, num_val])\n",
    "\n",
    "batch_size = 128\n",
    "train_val_loader = torch.utils.data.DataLoader(train_val_dataset, batch_size=128, shuffle=True, num_workers=2)\n",
    "train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=128, shuffle=True, num_workers=2)\n",
    "val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=128, shuffle=False, num_workers=2)\n",
    "test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=128, shuffle=False, num_workers=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train teacher network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoints_path = 'checkpoints_teacher/'\n",
    "if not os.path.exists(checkpoints_path):\n",
    "    os.mkdir(checkpoints_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 60\n",
    "print_every = 100    # Interval size for which to print statistics of training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training with hparamsdropout_hidden=0.0, dropout_input=0.0, lr=0.01, lr_decay=0.95, momentum=0.9, weight_decay=1e-05\n",
      "[1,   100/  469] train loss: 0.936 train accuracy: 0.766\n",
      "[1,   200/  469] train loss: 0.717 train accuracy: 0.805\n",
      "[1,   300/  469] train loss: 0.548 train accuracy: 0.852\n",
      "[1,   400/  469] train loss: 0.311 train accuracy: 0.922\n",
      "[2,   100/  469] train loss: 0.327 train accuracy: 0.891\n",
      "[2,   200/  469] train loss: 0.301 train accuracy: 0.906\n",
      "[2,   300/  469] train loss: 0.187 train accuracy: 0.961\n",
      "[2,   400/  469] train loss: 0.248 train accuracy: 0.914\n",
      "[3,   100/  469] train loss: 0.152 train accuracy: 0.977\n",
      "[3,   200/  469] train loss: 0.194 train accuracy: 0.953\n",
      "[3,   300/  469] train loss: 0.171 train accuracy: 0.961\n",
      "[3,   400/  469] train loss: 0.145 train accuracy: 0.961\n",
      "[4,   100/  469] train loss: 0.297 train accuracy: 0.914\n",
      "[4,   200/  469] train loss: 0.166 train accuracy: 0.969\n",
      "[4,   300/  469] train loss: 0.249 train accuracy: 0.930\n",
      "[4,   400/  469] train loss: 0.120 train accuracy: 0.961\n",
      "[5,   100/  469] train loss: 0.178 train accuracy: 0.945\n",
      "[5,   200/  469] train loss: 0.173 train accuracy: 0.945\n",
      "[5,   300/  469] train loss: 0.108 train accuracy: 0.969\n",
      "[5,   400/  469] train loss: 0.139 train accuracy: 0.938\n",
      "[6,   100/  469] train loss: 0.165 train accuracy: 0.961\n",
      "[6,   200/  469] train loss: 0.166 train accuracy: 0.945\n",
      "[6,   300/  469] train loss: 0.142 train accuracy: 0.961\n",
      "[6,   400/  469] train loss: 0.189 train accuracy: 0.961\n",
      "[7,   100/  469] train loss: 0.083 train accuracy: 0.961\n",
      "[7,   200/  469] train loss: 0.111 train accuracy: 0.969\n",
      "[7,   300/  469] train loss: 0.163 train accuracy: 0.953\n",
      "[7,   400/  469] train loss: 0.159 train accuracy: 0.961\n",
      "[8,   100/  469] train loss: 0.180 train accuracy: 0.953\n",
      "[8,   200/  469] train loss: 0.146 train accuracy: 0.945\n",
      "[8,   300/  469] train loss: 0.043 train accuracy: 0.992\n",
      "[8,   400/  469] train loss: 0.081 train accuracy: 0.961\n",
      "[9,   100/  469] train loss: 0.215 train accuracy: 0.953\n",
      "[9,   200/  469] train loss: 0.052 train accuracy: 0.977\n",
      "[9,   300/  469] train loss: 0.085 train accuracy: 0.992\n",
      "[9,   400/  469] train loss: 0.078 train accuracy: 0.969\n",
      "[10,   100/  469] train loss: 0.067 train accuracy: 0.969\n",
      "[10,   200/  469] train loss: 0.138 train accuracy: 0.969\n",
      "[10,   300/  469] train loss: 0.092 train accuracy: 0.961\n",
      "[10,   400/  469] train loss: 0.111 train accuracy: 0.969\n",
      "[11,   100/  469] train loss: 0.112 train accuracy: 0.984\n",
      "[11,   200/  469] train loss: 0.104 train accuracy: 0.984\n",
      "[11,   300/  469] train loss: 0.031 train accuracy: 0.992\n",
      "[11,   400/  469] train loss: 0.043 train accuracy: 0.984\n",
      "[12,   100/  469] train loss: 0.125 train accuracy: 0.945\n",
      "[12,   200/  469] train loss: 0.026 train accuracy: 1.000\n",
      "[12,   300/  469] train loss: 0.074 train accuracy: 0.977\n",
      "[12,   400/  469] train loss: 0.128 train accuracy: 0.969\n",
      "[13,   100/  469] train loss: 0.105 train accuracy: 0.961\n",
      "[13,   200/  469] train loss: 0.123 train accuracy: 0.969\n",
      "[13,   300/  469] train loss: 0.101 train accuracy: 0.969\n",
      "[13,   400/  469] train loss: 0.047 train accuracy: 0.984\n",
      "[14,   100/  469] train loss: 0.046 train accuracy: 0.992\n",
      "[14,   200/  469] train loss: 0.067 train accuracy: 0.977\n",
      "[14,   300/  469] train loss: 0.088 train accuracy: 0.977\n",
      "[14,   400/  469] train loss: 0.048 train accuracy: 0.984\n",
      "[15,   100/  469] train loss: 0.036 train accuracy: 0.992\n",
      "[15,   200/  469] train loss: 0.045 train accuracy: 0.984\n",
      "[15,   300/  469] train loss: 0.087 train accuracy: 0.969\n",
      "[15,   400/  469] train loss: 0.052 train accuracy: 0.984\n",
      "[16,   100/  469] train loss: 0.045 train accuracy: 0.984\n",
      "[16,   200/  469] train loss: 0.045 train accuracy: 0.977\n",
      "[16,   300/  469] train loss: 0.161 train accuracy: 0.969\n",
      "[16,   400/  469] train loss: 0.091 train accuracy: 0.961\n",
      "[17,   100/  469] train loss: 0.075 train accuracy: 0.992\n",
      "[17,   200/  469] train loss: 0.030 train accuracy: 0.984\n",
      "[17,   300/  469] train loss: 0.047 train accuracy: 0.984\n",
      "[17,   400/  469] train loss: 0.099 train accuracy: 0.984\n",
      "[18,   100/  469] train loss: 0.044 train accuracy: 0.984\n",
      "[18,   200/  469] train loss: 0.050 train accuracy: 0.992\n",
      "[18,   300/  469] train loss: 0.021 train accuracy: 0.992\n",
      "[18,   400/  469] train loss: 0.031 train accuracy: 0.992\n",
      "[19,   100/  469] train loss: 0.039 train accuracy: 0.984\n",
      "[19,   200/  469] train loss: 0.029 train accuracy: 0.992\n",
      "[19,   300/  469] train loss: 0.016 train accuracy: 1.000\n",
      "[19,   400/  469] train loss: 0.064 train accuracy: 0.969\n",
      "[20,   100/  469] train loss: 0.035 train accuracy: 1.000\n",
      "[20,   200/  469] train loss: 0.073 train accuracy: 0.984\n",
      "[20,   300/  469] train loss: 0.024 train accuracy: 0.992\n",
      "[20,   400/  469] train loss: 0.055 train accuracy: 0.977\n",
      "[21,   100/  469] train loss: 0.008 train accuracy: 1.000\n",
      "[21,   200/  469] train loss: 0.017 train accuracy: 1.000\n",
      "[21,   300/  469] train loss: 0.037 train accuracy: 0.984\n",
      "[21,   400/  469] train loss: 0.083 train accuracy: 0.969\n",
      "[22,   100/  469] train loss: 0.064 train accuracy: 0.977\n",
      "[22,   200/  469] train loss: 0.137 train accuracy: 0.961\n",
      "[22,   300/  469] train loss: 0.034 train accuracy: 0.992\n",
      "[22,   400/  469] train loss: 0.079 train accuracy: 0.977\n",
      "[23,   100/  469] train loss: 0.077 train accuracy: 0.961\n",
      "[23,   200/  469] train loss: 0.076 train accuracy: 0.984\n",
      "[23,   300/  469] train loss: 0.099 train accuracy: 0.969\n",
      "[23,   400/  469] train loss: 0.066 train accuracy: 0.969\n",
      "[24,   100/  469] train loss: 0.029 train accuracy: 0.992\n",
      "[24,   200/  469] train loss: 0.022 train accuracy: 0.992\n",
      "[24,   300/  469] train loss: 0.046 train accuracy: 0.984\n",
      "[24,   400/  469] train loss: 0.125 train accuracy: 0.969\n",
      "[25,   100/  469] train loss: 0.032 train accuracy: 0.992\n",
      "[25,   200/  469] train loss: 0.066 train accuracy: 0.969\n",
      "[25,   300/  469] train loss: 0.032 train accuracy: 0.984\n",
      "[25,   400/  469] train loss: 0.104 train accuracy: 0.984\n",
      "[26,   100/  469] train loss: 0.028 train accuracy: 0.992\n",
      "[26,   200/  469] train loss: 0.028 train accuracy: 0.992\n",
      "[26,   300/  469] train loss: 0.098 train accuracy: 0.961\n",
      "[26,   400/  469] train loss: 0.030 train accuracy: 0.984\n",
      "[27,   100/  469] train loss: 0.078 train accuracy: 0.961\n",
      "[27,   200/  469] train loss: 0.078 train accuracy: 0.977\n",
      "[27,   300/  469] train loss: 0.026 train accuracy: 0.992\n",
      "[27,   400/  469] train loss: 0.056 train accuracy: 0.977\n",
      "[28,   100/  469] train loss: 0.042 train accuracy: 0.977\n",
      "[28,   200/  469] train loss: 0.012 train accuracy: 1.000\n",
      "[28,   300/  469] train loss: 0.042 train accuracy: 0.984\n",
      "[28,   400/  469] train loss: 0.058 train accuracy: 0.977\n",
      "[29,   100/  469] train loss: 0.018 train accuracy: 0.992\n",
      "[29,   200/  469] train loss: 0.023 train accuracy: 0.992\n",
      "[29,   300/  469] train loss: 0.073 train accuracy: 0.969\n",
      "[29,   400/  469] train loss: 0.033 train accuracy: 0.984\n",
      "[30,   100/  469] train loss: 0.016 train accuracy: 1.000\n",
      "[30,   200/  469] train loss: 0.036 train accuracy: 0.984\n",
      "[30,   300/  469] train loss: 0.024 train accuracy: 0.992\n",
      "[30,   400/  469] train loss: 0.021 train accuracy: 0.992\n",
      "[31,   100/  469] train loss: 0.026 train accuracy: 0.992\n",
      "[31,   200/  469] train loss: 0.077 train accuracy: 0.977\n",
      "[31,   300/  469] train loss: 0.012 train accuracy: 1.000\n",
      "[31,   400/  469] train loss: 0.058 train accuracy: 0.992\n",
      "[32,   100/  469] train loss: 0.027 train accuracy: 1.000\n",
      "[32,   200/  469] train loss: 0.023 train accuracy: 0.992\n",
      "[32,   300/  469] train loss: 0.040 train accuracy: 0.984\n",
      "[32,   400/  469] train loss: 0.010 train accuracy: 1.000\n",
      "[33,   100/  469] train loss: 0.024 train accuracy: 0.992\n",
      "[33,   200/  469] train loss: 0.068 train accuracy: 0.992\n",
      "[33,   300/  469] train loss: 0.019 train accuracy: 0.992\n",
      "[33,   400/  469] train loss: 0.029 train accuracy: 0.984\n",
      "[34,   100/  469] train loss: 0.032 train accuracy: 0.992\n",
      "[34,   200/  469] train loss: 0.010 train accuracy: 1.000\n",
      "[34,   300/  469] train loss: 0.074 train accuracy: 0.984\n",
      "[34,   400/  469] train loss: 0.043 train accuracy: 0.992\n",
      "[35,   100/  469] train loss: 0.052 train accuracy: 0.984\n",
      "[35,   200/  469] train loss: 0.075 train accuracy: 0.969\n",
      "[35,   300/  469] train loss: 0.062 train accuracy: 0.969\n",
      "[35,   400/  469] train loss: 0.043 train accuracy: 0.984\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[36,   100/  469] train loss: 0.094 train accuracy: 0.984\n",
      "[36,   200/  469] train loss: 0.034 train accuracy: 0.984\n",
      "[36,   300/  469] train loss: 0.057 train accuracy: 0.992\n",
      "[36,   400/  469] train loss: 0.017 train accuracy: 0.992\n",
      "[37,   100/  469] train loss: 0.070 train accuracy: 0.961\n",
      "[37,   200/  469] train loss: 0.046 train accuracy: 0.984\n",
      "[37,   300/  469] train loss: 0.028 train accuracy: 0.984\n",
      "[37,   400/  469] train loss: 0.006 train accuracy: 1.000\n",
      "[38,   100/  469] train loss: 0.017 train accuracy: 1.000\n",
      "[38,   200/  469] train loss: 0.096 train accuracy: 0.977\n",
      "[38,   300/  469] train loss: 0.050 train accuracy: 0.992\n",
      "[38,   400/  469] train loss: 0.017 train accuracy: 0.992\n",
      "[39,   100/  469] train loss: 0.019 train accuracy: 0.992\n",
      "[39,   200/  469] train loss: 0.032 train accuracy: 0.984\n",
      "[39,   300/  469] train loss: 0.137 train accuracy: 0.961\n",
      "[39,   400/  469] train loss: 0.071 train accuracy: 0.953\n",
      "[40,   100/  469] train loss: 0.077 train accuracy: 0.977\n",
      "[40,   200/  469] train loss: 0.046 train accuracy: 0.992\n",
      "[40,   300/  469] train loss: 0.016 train accuracy: 1.000\n",
      "[40,   400/  469] train loss: 0.026 train accuracy: 0.992\n",
      "[41,   100/  469] train loss: 0.056 train accuracy: 0.992\n",
      "[41,   200/  469] train loss: 0.029 train accuracy: 0.992\n",
      "[41,   300/  469] train loss: 0.008 train accuracy: 1.000\n",
      "[41,   400/  469] train loss: 0.028 train accuracy: 0.992\n",
      "[42,   100/  469] train loss: 0.044 train accuracy: 0.984\n",
      "[42,   200/  469] train loss: 0.019 train accuracy: 0.992\n",
      "[42,   300/  469] train loss: 0.007 train accuracy: 1.000\n",
      "[42,   400/  469] train loss: 0.009 train accuracy: 1.000\n",
      "[43,   100/  469] train loss: 0.022 train accuracy: 0.992\n",
      "[43,   200/  469] train loss: 0.039 train accuracy: 0.984\n",
      "[43,   300/  469] train loss: 0.015 train accuracy: 1.000\n",
      "[43,   400/  469] train loss: 0.015 train accuracy: 1.000\n",
      "[44,   100/  469] train loss: 0.024 train accuracy: 0.992\n",
      "[44,   200/  469] train loss: 0.013 train accuracy: 1.000\n",
      "[44,   300/  469] train loss: 0.019 train accuracy: 0.992\n",
      "[44,   400/  469] train loss: 0.028 train accuracy: 0.992\n",
      "[45,   100/  469] train loss: 0.029 train accuracy: 0.984\n",
      "[45,   200/  469] train loss: 0.036 train accuracy: 0.992\n",
      "[45,   300/  469] train loss: 0.021 train accuracy: 0.992\n",
      "[45,   400/  469] train loss: 0.026 train accuracy: 0.984\n",
      "[46,   100/  469] train loss: 0.016 train accuracy: 1.000\n",
      "[46,   200/  469] train loss: 0.025 train accuracy: 0.992\n",
      "[46,   300/  469] train loss: 0.044 train accuracy: 0.984\n",
      "[46,   400/  469] train loss: 0.055 train accuracy: 0.977\n",
      "[47,   100/  469] train loss: 0.054 train accuracy: 0.984\n",
      "[47,   200/  469] train loss: 0.018 train accuracy: 0.992\n",
      "[47,   300/  469] train loss: 0.066 train accuracy: 0.977\n",
      "[47,   400/  469] train loss: 0.020 train accuracy: 0.992\n",
      "[48,   100/  469] train loss: 0.030 train accuracy: 0.992\n",
      "[48,   200/  469] train loss: 0.025 train accuracy: 0.992\n",
      "[48,   300/  469] train loss: 0.029 train accuracy: 0.992\n",
      "[48,   400/  469] train loss: 0.064 train accuracy: 0.984\n",
      "[49,   100/  469] train loss: 0.015 train accuracy: 1.000\n",
      "[49,   200/  469] train loss: 0.029 train accuracy: 0.984\n",
      "[49,   300/  469] train loss: 0.015 train accuracy: 1.000\n",
      "[49,   400/  469] train loss: 0.063 train accuracy: 0.984\n",
      "[50,   100/  469] train loss: 0.042 train accuracy: 0.984\n",
      "[50,   200/  469] train loss: 0.039 train accuracy: 0.992\n",
      "[50,   300/  469] train loss: 0.083 train accuracy: 0.961\n",
      "[50,   400/  469] train loss: 0.014 train accuracy: 0.992\n",
      "[51,   100/  469] train loss: 0.019 train accuracy: 1.000\n",
      "[51,   200/  469] train loss: 0.043 train accuracy: 0.984\n",
      "[51,   300/  469] train loss: 0.018 train accuracy: 0.992\n",
      "[51,   400/  469] train loss: 0.032 train accuracy: 0.992\n",
      "[52,   100/  469] train loss: 0.025 train accuracy: 0.992\n",
      "[52,   200/  469] train loss: 0.071 train accuracy: 0.969\n",
      "[52,   300/  469] train loss: 0.039 train accuracy: 0.992\n",
      "[52,   400/  469] train loss: 0.073 train accuracy: 0.984\n",
      "[53,   100/  469] train loss: 0.017 train accuracy: 1.000\n",
      "[53,   200/  469] train loss: 0.032 train accuracy: 0.984\n",
      "[53,   300/  469] train loss: 0.070 train accuracy: 0.961\n",
      "[53,   400/  469] train loss: 0.038 train accuracy: 0.992\n",
      "[54,   100/  469] train loss: 0.033 train accuracy: 0.984\n",
      "[54,   200/  469] train loss: 0.056 train accuracy: 0.984\n",
      "[54,   300/  469] train loss: 0.023 train accuracy: 1.000\n",
      "[54,   400/  469] train loss: 0.078 train accuracy: 0.977\n",
      "[55,   100/  469] train loss: 0.041 train accuracy: 0.984\n",
      "[55,   200/  469] train loss: 0.040 train accuracy: 0.984\n",
      "[55,   300/  469] train loss: 0.035 train accuracy: 0.977\n",
      "[55,   400/  469] train loss: 0.031 train accuracy: 0.992\n",
      "[56,   100/  469] train loss: 0.014 train accuracy: 1.000\n",
      "[56,   200/  469] train loss: 0.023 train accuracy: 1.000\n",
      "[56,   300/  469] train loss: 0.019 train accuracy: 1.000\n",
      "[56,   400/  469] train loss: 0.021 train accuracy: 1.000\n",
      "[57,   100/  469] train loss: 0.013 train accuracy: 1.000\n",
      "[57,   200/  469] train loss: 0.038 train accuracy: 0.984\n",
      "[57,   300/  469] train loss: 0.034 train accuracy: 0.992\n",
      "[57,   400/  469] train loss: 0.030 train accuracy: 0.992\n",
      "[58,   100/  469] train loss: 0.021 train accuracy: 0.992\n",
      "[58,   200/  469] train loss: 0.011 train accuracy: 1.000\n",
      "[58,   300/  469] train loss: 0.020 train accuracy: 0.992\n",
      "[58,   400/  469] train loss: 0.020 train accuracy: 1.000\n",
      "[59,   100/  469] train loss: 0.044 train accuracy: 0.984\n",
      "[59,   200/  469] train loss: 0.044 train accuracy: 0.984\n",
      "[59,   300/  469] train loss: 0.024 train accuracy: 0.992\n",
      "[59,   400/  469] train loss: 0.026 train accuracy: 0.992\n",
      "[60,   100/  469] train loss: 0.036 train accuracy: 0.969\n",
      "[60,   200/  469] train loss: 0.022 train accuracy: 0.984\n",
      "[60,   300/  469] train loss: 0.027 train accuracy: 0.984\n",
      "[60,   400/  469] train loss: 0.030 train accuracy: 0.984\n"
     ]
    }
   ],
   "source": [
    "# Hyperparamters can be tuned by setting required range below\n",
    "# learning_rates = list(np.logspace(-4, -2, 3))\n",
    "learning_rates = [1e-2]\n",
    "learning_rate_decays = [0.95]    # learning rate decays at every epoch\n",
    "# weight_decays = [0.0] + list(np.logspace(-5, -1, 5))\n",
    "weight_decays = [1e-5]           # regularization weight\n",
    "momentums = [0.9]\n",
    "# dropout_probabilities = [(0.2, 0.5), (0.0, 0.0)]\n",
    "dropout_probabilities = [(0.0, 0.0)]\n",
    "hparams_list = []\n",
    "for hparam_tuple in itertools.product(dropout_probabilities, weight_decays, learning_rate_decays, \n",
    "                                        momentums, learning_rates):\n",
    "    hparam = {}\n",
    "    hparam['dropout_input'] = hparam_tuple[0][0]\n",
    "    hparam['dropout_hidden'] = hparam_tuple[0][1]\n",
    "    hparam['weight_decay'] = hparam_tuple[1]\n",
    "    hparam['lr_decay'] = hparam_tuple[2]\n",
    "    hparam['momentum'] = hparam_tuple[3]\n",
    "    hparam['lr'] = hparam_tuple[4]\n",
    "    hparams_list.append(hparam)\n",
    "\n",
    "results = {}\n",
    "for hparam in hparams_list:\n",
    "    print('Training with hparams' + utils.hparamToString(hparam))\n",
    "    reproducibilitySeed()\n",
    "    teacher_net = networks.TeacherNetwork()\n",
    "    teacher_net = teacher_net.to(fast_device)\n",
    "    hparam_tuple = utils.hparamDictToTuple(hparam)\n",
    "    results[hparam_tuple] = utils.trainTeacherOnHparam(teacher_net, hparam, num_epochs, \n",
    "                                                        train_val_loader, None, \n",
    "                                                        print_every=print_every, \n",
    "                                                        fast_device=fast_device)\n",
    "    save_path = checkpoints_path + utils.hparamToString(hparam) + '_final.tar'\n",
    "    torch.save({'results' : results[hparam_tuple], \n",
    "                'model_state_dict' : teacher_net.state_dict(), \n",
    "                'epoch' : num_epochs}, save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test accuracy:  0.9892\n"
     ]
    }
   ],
   "source": [
    "# Calculate test accuracy\n",
    "_, test_accuracy = utils.getLossAccuracyOnDataset(teacher_net, test_loader, fast_device)\n",
    "print('test accuracy: ', test_accuracy)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}