Jun20/Discussion1/distill_basic_student.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import required packages and limit GPU usage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import pickle\n",
    "import argparse\n",
    "import time\n",
    "import itertools\n",
    "from copy import deepcopy\n",
    "import os\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "import networks\n",
    "import utils\n",
    "    \n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_gpu = True    # set use_gpu to True if system has gpu\n",
    "gpu_id = 2        # id of gpu to be used\n",
    "cpu_device = torch.device('cpu')\n",
    "# fast_device is where computation (training, inference) happens\n",
    "fast_device = torch.device('cpu')\n",
    "if use_gpu:\n",
    "    os.environ['CUDA_VISIBLE_DEVICES'] = '0,1,2'    # set visible devices depending on system configuration\n",
    "    fast_device = torch.device('cuda:' + str(gpu_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ensure reproducibility\n",
    "def reproducibilitySeed():\n",
    "    \"\"\"\n",
    "    Ensure reproducibility of results; Seeds to 0\n",
    "    \"\"\"\n",
    "    torch_init_seed = 0\n",
    "    torch.manual_seed(torch_init_seed)\n",
    "    numpy_init_seed = 0\n",
    "    np.random.seed(numpy_init_seed)\n",
    "    if use_gpu:\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "\n",
    "reproducibilitySeed()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "# Student trained without data augmentation\n",
    "transform = transforms.Compose(\n",
    "                [\n",
    "                    transforms.ToTensor(),\n",
    "                    transforms.Normalize((0.5, 0.5), (0.5, 0.5))\n",
    "                ]\n",
    "            )\n",
    "\n",
    "train_val_dataset = torchvision.datasets.MNIST(root='./MNIST_dataset/', train=True, \n",
    "                                            download=True, transform=transform)\n",
    "\n",
    "test_dataset = torchvision.datasets.MNIST(root='./MNIST_dataset/', train=False, \n",
    "                                            download=True, transform=transform)\n",
    "\n",
    "num_train = int(1.0 * len(train_val_dataset) * 95 / 100)\n",
    "num_val = len(train_val_dataset) - num_train\n",
    "train_dataset, val_dataset = torch.utils.data.random_split(train_val_dataset, [num_train, num_val])\n",
    "\n",
    "batch_size = 128\n",
    "train_val_loader = torch.utils.data.DataLoader(train_val_dataset, batch_size=128, shuffle=True, num_workers=2)\n",
    "train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=128, shuffle=True, num_workers=2)\n",
    "val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=128, shuffle=False, num_workers=2)\n",
    "test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=128, shuffle=False, num_workers=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoints_path_teacher = 'checkpoints_teacher/'\n",
    "checkpoints_path_student = 'checkpoints_student/'\n",
    "if not os.path.exists(checkpoints_path_student):\n",
    "    os.makedirs(checkpoints_path_student)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load teacher network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set the hparams used for training teacher to load the teacher network\n",
    "learning_rates = [1e-2]\n",
    "learning_rate_decays = [0.95]\n",
    "weight_decays = [1e-5]\n",
    "momentums = [0.9]\n",
    "# keeping dropout input = dropout hidden\n",
    "dropout_probabilities = [(0.0, 0.0)]\n",
    "hparams_list = []\n",
    "for hparam_tuple in itertools.product(dropout_probabilities, weight_decays, learning_rate_decays, \n",
    "                                        momentums, learning_rates):\n",
    "    hparam = {}\n",
    "    hparam['dropout_input'] = hparam_tuple[0][0]\n",
    "    hparam['dropout_hidden'] = hparam_tuple[0][1]\n",
    "    hparam['weight_decay'] = hparam_tuple[1]\n",
    "    hparam['lr_decay'] = hparam_tuple[2]\n",
    "    hparam['momentum'] = hparam_tuple[3]\n",
    "    hparam['lr'] = hparam_tuple[4]\n",
    "    hparams_list.append(hparam)\n",
    "    \n",
    "load_path = checkpoints_path_teacher + utils.hparamToString(hparams_list[0]) + '_final.tar'\n",
    "teacher_net = networks.TeacherNetwork()\n",
    "teacher_net.load_state_dict(torch.load(load_path, map_location=fast_device)['model_state_dict'])\n",
    "teacher_net = teacher_net.to(fast_device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "teacher test accuracy:  0.9892\n"
     ]
    }
   ],
   "source": [
    "# Calculate teacher test accuracy\n",
    "_, test_accuracy = utils.getLossAccuracyOnDataset(teacher_net, test_loader, fast_device)\n",
    "print('teacher test accuracy: ', test_accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train student network without distillation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 60\n",
    "print_every = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training with hparamsT=1, alpha=0.0, dropout_hidden=0.0, dropout_input=0.0, lr=0.01, lr_decay=0.95, momentum=0.9, weight_decay=1e-05\n",
      "[1,   100/  469] train loss: 0.426 train accuracy: 0.875\n",
      "[1,   200/  469] train loss: 0.326 train accuracy: 0.898\n",
      "[1,   300/  469] train loss: 0.229 train accuracy: 0.945\n",
      "[1,   400/  469] train loss: 0.232 train accuracy: 0.922\n",
      "[2,   100/  469] train loss: 0.245 train accuracy: 0.938\n",
      "[2,   200/  469] train loss: 0.216 train accuracy: 0.945\n",
      "[2,   300/  469] train loss: 0.134 train accuracy: 0.984\n",
      "[2,   400/  469] train loss: 0.201 train accuracy: 0.961\n",
      "[3,   100/  469] train loss: 0.108 train accuracy: 0.977\n",
      "[3,   200/  469] train loss: 0.119 train accuracy: 0.961\n",
      "[3,   300/  469] train loss: 0.151 train accuracy: 0.969\n",
      "[3,   400/  469] train loss: 0.189 train accuracy: 0.945\n",
      "[4,   100/  469] train loss: 0.191 train accuracy: 0.945\n",
      "[4,   200/  469] train loss: 0.259 train accuracy: 0.938\n",
      "[4,   300/  469] train loss: 0.141 train accuracy: 0.977\n",
      "[4,   400/  469] train loss: 0.095 train accuracy: 0.984\n",
      "[5,   100/  469] train loss: 0.235 train accuracy: 0.953\n",
      "[5,   200/  469] train loss: 0.070 train accuracy: 0.984\n",
      "[5,   300/  469] train loss: 0.072 train accuracy: 0.977\n",
      "[5,   400/  469] train loss: 0.166 train accuracy: 0.930\n",
      "[6,   100/  469] train loss: 0.175 train accuracy: 0.953\n",
      "[6,   200/  469] train loss: 0.113 train accuracy: 0.969\n",
      "[6,   300/  469] train loss: 0.111 train accuracy: 0.977\n",
      "[6,   400/  469] train loss: 0.245 train accuracy: 0.945\n",
      "[7,   100/  469] train loss: 0.127 train accuracy: 0.984\n",
      "[7,   200/  469] train loss: 0.058 train accuracy: 0.992\n",
      "[7,   300/  469] train loss: 0.045 train accuracy: 0.984\n",
      "[7,   400/  469] train loss: 0.084 train accuracy: 0.984\n",
      "[8,   100/  469] train loss: 0.116 train accuracy: 0.977\n",
      "[8,   200/  469] train loss: 0.057 train accuracy: 0.992\n",
      "[8,   300/  469] train loss: 0.060 train accuracy: 0.977\n",
      "[8,   400/  469] train loss: 0.037 train accuracy: 0.992\n",
      "[9,   100/  469] train loss: 0.028 train accuracy: 1.000\n",
      "[9,   200/  469] train loss: 0.174 train accuracy: 0.953\n",
      "[9,   300/  469] train loss: 0.029 train accuracy: 0.984\n",
      "[9,   400/  469] train loss: 0.071 train accuracy: 1.000\n",
      "[10,   100/  469] train loss: 0.045 train accuracy: 0.992\n",
      "[10,   200/  469] train loss: 0.033 train accuracy: 0.992\n",
      "[10,   300/  469] train loss: 0.041 train accuracy: 0.992\n",
      "[10,   400/  469] train loss: 0.037 train accuracy: 1.000\n",
      "[11,   100/  469] train loss: 0.130 train accuracy: 0.953\n",
      "[11,   200/  469] train loss: 0.049 train accuracy: 0.984\n",
      "[11,   300/  469] train loss: 0.023 train accuracy: 1.000\n",
      "[11,   400/  469] train loss: 0.055 train accuracy: 0.992\n",
      "[12,   100/  469] train loss: 0.107 train accuracy: 0.961\n",
      "[12,   200/  469] train loss: 0.040 train accuracy: 0.984\n",
      "[12,   300/  469] train loss: 0.029 train accuracy: 0.992\n",
      "[12,   400/  469] train loss: 0.024 train accuracy: 1.000\n",
      "[13,   100/  469] train loss: 0.081 train accuracy: 0.984\n",
      "[13,   200/  469] train loss: 0.076 train accuracy: 0.984\n",
      "[13,   300/  469] train loss: 0.138 train accuracy: 0.961\n",
      "[13,   400/  469] train loss: 0.036 train accuracy: 0.992\n",
      "[14,   100/  469] train loss: 0.051 train accuracy: 0.992\n",
      "[14,   200/  469] train loss: 0.045 train accuracy: 0.984\n",
      "[14,   300/  469] train loss: 0.043 train accuracy: 0.984\n",
      "[14,   400/  469] train loss: 0.058 train accuracy: 0.984\n",
      "[15,   100/  469] train loss: 0.034 train accuracy: 0.992\n",
      "[15,   200/  469] train loss: 0.020 train accuracy: 1.000\n",
      "[15,   300/  469] train loss: 0.020 train accuracy: 1.000\n",
      "[15,   400/  469] train loss: 0.036 train accuracy: 0.984\n",
      "[16,   100/  469] train loss: 0.030 train accuracy: 0.984\n",
      "[16,   200/  469] train loss: 0.016 train accuracy: 1.000\n",
      "[16,   300/  469] train loss: 0.022 train accuracy: 0.992\n",
      "[16,   400/  469] train loss: 0.048 train accuracy: 0.984\n",
      "[17,   100/  469] train loss: 0.031 train accuracy: 0.992\n",
      "[17,   200/  469] train loss: 0.034 train accuracy: 0.984\n",
      "[17,   300/  469] train loss: 0.104 train accuracy: 0.977\n",
      "[17,   400/  469] train loss: 0.031 train accuracy: 0.992\n",
      "[18,   100/  469] train loss: 0.053 train accuracy: 0.977\n",
      "[18,   200/  469] train loss: 0.049 train accuracy: 0.992\n",
      "[18,   300/  469] train loss: 0.041 train accuracy: 0.992\n",
      "[18,   400/  469] train loss: 0.054 train accuracy: 0.977\n",
      "[19,   100/  469] train loss: 0.019 train accuracy: 1.000\n",
      "[19,   200/  469] train loss: 0.029 train accuracy: 1.000\n",
      "[19,   300/  469] train loss: 0.019 train accuracy: 1.000\n",
      "[19,   400/  469] train loss: 0.029 train accuracy: 0.992\n",
      "[20,   100/  469] train loss: 0.023 train accuracy: 0.992\n",
      "[20,   200/  469] train loss: 0.018 train accuracy: 1.000\n",
      "[20,   300/  469] train loss: 0.039 train accuracy: 0.992\n",
      "[20,   400/  469] train loss: 0.042 train accuracy: 0.992\n",
      "[21,   100/  469] train loss: 0.040 train accuracy: 0.984\n",
      "[21,   200/  469] train loss: 0.019 train accuracy: 1.000\n",
      "[21,   300/  469] train loss: 0.028 train accuracy: 1.000\n",
      "[21,   400/  469] train loss: 0.050 train accuracy: 0.984\n",
      "[22,   100/  469] train loss: 0.018 train accuracy: 1.000\n",
      "[22,   200/  469] train loss: 0.029 train accuracy: 0.992\n",
      "[22,   300/  469] train loss: 0.057 train accuracy: 0.977\n",
      "[22,   400/  469] train loss: 0.017 train accuracy: 1.000\n",
      "[23,   100/  469] train loss: 0.017 train accuracy: 1.000\n",
      "[23,   200/  469] train loss: 0.032 train accuracy: 0.992\n",
      "[23,   300/  469] train loss: 0.046 train accuracy: 0.984\n",
      "[23,   400/  469] train loss: 0.015 train accuracy: 1.000\n",
      "[24,   100/  469] train loss: 0.008 train accuracy: 1.000\n",
      "[24,   200/  469] train loss: 0.019 train accuracy: 1.000\n",
      "[24,   300/  469] train loss: 0.045 train accuracy: 0.992\n",
      "[24,   400/  469] train loss: 0.025 train accuracy: 0.992\n",
      "[25,   100/  469] train loss: 0.031 train accuracy: 1.000\n",
      "[25,   200/  469] train loss: 0.029 train accuracy: 0.992\n",
      "[25,   300/  469] train loss: 0.016 train accuracy: 1.000\n",
      "[25,   400/  469] train loss: 0.025 train accuracy: 1.000\n",
      "[26,   100/  469] train loss: 0.043 train accuracy: 0.992\n",
      "[26,   200/  469] train loss: 0.042 train accuracy: 0.984\n",
      "[26,   300/  469] train loss: 0.028 train accuracy: 0.992\n",
      "[26,   400/  469] train loss: 0.011 train accuracy: 1.000\n",
      "[27,   100/  469] train loss: 0.008 train accuracy: 1.000\n",
      "[27,   200/  469] train loss: 0.073 train accuracy: 0.992\n",
      "[27,   300/  469] train loss: 0.008 train accuracy: 1.000\n",
      "[27,   400/  469] train loss: 0.031 train accuracy: 0.992\n",
      "[28,   100/  469] train loss: 0.013 train accuracy: 1.000\n",
      "[28,   200/  469] train loss: 0.022 train accuracy: 1.000\n",
      "[28,   300/  469] train loss: 0.013 train accuracy: 1.000\n",
      "[28,   400/  469] train loss: 0.020 train accuracy: 1.000\n",
      "[29,   100/  469] train loss: 0.028 train accuracy: 0.992\n",
      "[29,   200/  469] train loss: 0.019 train accuracy: 1.000\n",
      "[29,   300/  469] train loss: 0.032 train accuracy: 0.992\n",
      "[29,   400/  469] train loss: 0.024 train accuracy: 0.992\n",
      "[30,   100/  469] train loss: 0.027 train accuracy: 0.992\n",
      "[30,   200/  469] train loss: 0.019 train accuracy: 0.992\n",
      "[30,   300/  469] train loss: 0.053 train accuracy: 0.992\n",
      "[30,   400/  469] train loss: 0.016 train accuracy: 1.000\n",
      "[31,   100/  469] train loss: 0.034 train accuracy: 0.992\n",
      "[31,   200/  469] train loss: 0.015 train accuracy: 1.000\n",
      "[31,   300/  469] train loss: 0.049 train accuracy: 0.992\n",
      "[31,   400/  469] train loss: 0.023 train accuracy: 1.000\n",
      "[32,   100/  469] train loss: 0.018 train accuracy: 1.000\n",
      "[32,   200/  469] train loss: 0.031 train accuracy: 1.000\n",
      "[32,   300/  469] train loss: 0.021 train accuracy: 1.000\n",
      "[32,   400/  469] train loss: 0.009 train accuracy: 1.000\n",
      "[33,   100/  469] train loss: 0.013 train accuracy: 0.992\n",
      "[33,   200/  469] train loss: 0.019 train accuracy: 1.000\n",
      "[33,   300/  469] train loss: 0.031 train accuracy: 1.000\n",
      "[33,   400/  469] train loss: 0.007 train accuracy: 1.000\n",
      "[34,   100/  469] train loss: 0.016 train accuracy: 0.992\n",
      "[34,   200/  469] train loss: 0.015 train accuracy: 1.000\n",
      "[34,   300/  469] train loss: 0.011 train accuracy: 1.000\n",
      "[34,   400/  469] train loss: 0.024 train accuracy: 0.992\n",
      "[35,   100/  469] train loss: 0.024 train accuracy: 1.000\n",
      "[35,   200/  469] train loss: 0.052 train accuracy: 0.992\n",
      "[35,   300/  469] train loss: 0.040 train accuracy: 0.977\n",
      "[35,   400/  469] train loss: 0.043 train accuracy: 0.977\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[36,   100/  469] train loss: 0.019 train accuracy: 1.000\n",
      "[36,   200/  469] train loss: 0.020 train accuracy: 0.992\n",
      "[36,   300/  469] train loss: 0.027 train accuracy: 0.992\n",
      "[36,   400/  469] train loss: 0.023 train accuracy: 1.000\n",
      "[37,   100/  469] train loss: 0.047 train accuracy: 0.984\n",
      "[37,   200/  469] train loss: 0.035 train accuracy: 0.984\n",
      "[37,   300/  469] train loss: 0.010 train accuracy: 1.000\n",
      "[37,   400/  469] train loss: 0.056 train accuracy: 0.984\n",
      "[38,   100/  469] train loss: 0.044 train accuracy: 0.984\n",
      "[38,   200/  469] train loss: 0.023 train accuracy: 0.992\n",
      "[38,   300/  469] train loss: 0.037 train accuracy: 0.984\n",
      "[38,   400/  469] train loss: 0.029 train accuracy: 0.992\n",
      "[39,   100/  469] train loss: 0.008 train accuracy: 1.000\n",
      "[39,   200/  469] train loss: 0.017 train accuracy: 1.000\n",
      "[39,   300/  469] train loss: 0.044 train accuracy: 0.992\n",
      "[39,   400/  469] train loss: 0.014 train accuracy: 1.000\n",
      "[40,   100/  469] train loss: 0.052 train accuracy: 0.984\n",
      "[40,   200/  469] train loss: 0.016 train accuracy: 1.000\n",
      "[40,   300/  469] train loss: 0.011 train accuracy: 1.000\n",
      "[40,   400/  469] train loss: 0.016 train accuracy: 1.000\n",
      "[41,   100/  469] train loss: 0.015 train accuracy: 1.000\n",
      "[41,   200/  469] train loss: 0.028 train accuracy: 0.992\n",
      "[41,   300/  469] train loss: 0.008 train accuracy: 1.000\n",
      "[41,   400/  469] train loss: 0.010 train accuracy: 1.000\n",
      "[42,   100/  469] train loss: 0.019 train accuracy: 1.000\n",
      "[42,   200/  469] train loss: 0.010 train accuracy: 1.000\n",
      "[42,   300/  469] train loss: 0.019 train accuracy: 0.992\n",
      "[42,   400/  469] train loss: 0.006 train accuracy: 1.000\n",
      "[43,   100/  469] train loss: 0.016 train accuracy: 0.992\n",
      "[43,   200/  469] train loss: 0.029 train accuracy: 0.992\n",
      "[43,   300/  469] train loss: 0.019 train accuracy: 1.000\n",
      "[43,   400/  469] train loss: 0.015 train accuracy: 1.000\n",
      "[44,   100/  469] train loss: 0.015 train accuracy: 1.000\n",
      "[44,   200/  469] train loss: 0.021 train accuracy: 0.992\n",
      "[44,   300/  469] train loss: 0.053 train accuracy: 0.977\n",
      "[44,   400/  469] train loss: 0.010 train accuracy: 1.000\n",
      "[45,   100/  469] train loss: 0.012 train accuracy: 1.000\n",
      "[45,   200/  469] train loss: 0.009 train accuracy: 1.000\n",
      "[45,   300/  469] train loss: 0.015 train accuracy: 1.000\n",
      "[45,   400/  469] train loss: 0.014 train accuracy: 1.000\n",
      "[46,   100/  469] train loss: 0.024 train accuracy: 1.000\n",
      "[46,   200/  469] train loss: 0.011 train accuracy: 1.000\n",
      "[46,   300/  469] train loss: 0.031 train accuracy: 0.992\n",
      "[46,   400/  469] train loss: 0.015 train accuracy: 1.000\n",
      "[47,   100/  469] train loss: 0.015 train accuracy: 1.000\n",
      "[47,   200/  469] train loss: 0.016 train accuracy: 1.000\n",
      "[47,   300/  469] train loss: 0.016 train accuracy: 0.992\n",
      "[47,   400/  469] train loss: 0.007 train accuracy: 1.000\n",
      "[48,   100/  469] train loss: 0.022 train accuracy: 1.000\n",
      "[48,   200/  469] train loss: 0.009 train accuracy: 1.000\n",
      "[48,   300/  469] train loss: 0.004 train accuracy: 1.000\n",
      "[48,   400/  469] train loss: 0.012 train accuracy: 1.000\n",
      "[49,   100/  469] train loss: 0.020 train accuracy: 0.992\n",
      "[49,   200/  469] train loss: 0.023 train accuracy: 0.992\n",
      "[49,   300/  469] train loss: 0.029 train accuracy: 0.992\n",
      "[49,   400/  469] train loss: 0.021 train accuracy: 1.000\n",
      "[50,   100/  469] train loss: 0.013 train accuracy: 1.000\n",
      "[50,   200/  469] train loss: 0.016 train accuracy: 1.000\n",
      "[50,   300/  469] train loss: 0.022 train accuracy: 0.992\n",
      "[50,   400/  469] train loss: 0.023 train accuracy: 0.992\n",
      "[51,   100/  469] train loss: 0.018 train accuracy: 0.992\n",
      "[51,   200/  469] train loss: 0.018 train accuracy: 1.000\n",
      "[51,   300/  469] train loss: 0.014 train accuracy: 1.000\n",
      "[51,   400/  469] train loss: 0.017 train accuracy: 1.000\n",
      "[52,   100/  469] train loss: 0.014 train accuracy: 1.000\n",
      "[52,   200/  469] train loss: 0.016 train accuracy: 1.000\n",
      "[52,   300/  469] train loss: 0.016 train accuracy: 1.000\n",
      "[52,   400/  469] train loss: 0.020 train accuracy: 0.992\n",
      "[53,   100/  469] train loss: 0.021 train accuracy: 0.992\n",
      "[53,   200/  469] train loss: 0.011 train accuracy: 1.000\n",
      "[53,   300/  469] train loss: 0.013 train accuracy: 1.000\n",
      "[53,   400/  469] train loss: 0.026 train accuracy: 0.992\n",
      "[54,   100/  469] train loss: 0.011 train accuracy: 1.000\n",
      "[54,   200/  469] train loss: 0.024 train accuracy: 0.992\n",
      "[54,   300/  469] train loss: 0.012 train accuracy: 1.000\n",
      "[54,   400/  469] train loss: 0.011 train accuracy: 1.000\n",
      "[55,   100/  469] train loss: 0.015 train accuracy: 1.000\n",
      "[55,   200/  469] train loss: 0.026 train accuracy: 0.984\n",
      "[55,   300/  469] train loss: 0.017 train accuracy: 1.000\n",
      "[55,   400/  469] train loss: 0.010 train accuracy: 1.000\n",
      "[56,   100/  469] train loss: 0.015 train accuracy: 0.992\n",
      "[56,   200/  469] train loss: 0.018 train accuracy: 1.000\n",
      "[56,   300/  469] train loss: 0.028 train accuracy: 0.984\n",
      "[56,   400/  469] train loss: 0.024 train accuracy: 0.992\n",
      "[57,   100/  469] train loss: 0.003 train accuracy: 1.000\n",
      "[57,   200/  469] train loss: 0.013 train accuracy: 1.000\n",
      "[57,   300/  469] train loss: 0.016 train accuracy: 1.000\n",
      "[57,   400/  469] train loss: 0.019 train accuracy: 1.000\n",
      "[58,   100/  469] train loss: 0.018 train accuracy: 0.992\n",
      "[58,   200/  469] train loss: 0.010 train accuracy: 1.000\n",
      "[58,   300/  469] train loss: 0.013 train accuracy: 1.000\n",
      "[58,   400/  469] train loss: 0.017 train accuracy: 0.992\n",
      "[59,   100/  469] train loss: 0.019 train accuracy: 1.000\n",
      "[59,   200/  469] train loss: 0.009 train accuracy: 1.000\n",
      "[59,   300/  469] train loss: 0.022 train accuracy: 0.992\n",
      "[59,   400/  469] train loss: 0.011 train accuracy: 1.000\n",
      "[60,   100/  469] train loss: 0.039 train accuracy: 0.992\n",
      "[60,   200/  469] train loss: 0.018 train accuracy: 1.000\n",
      "[60,   300/  469] train loss: 0.008 train accuracy: 1.000\n",
      "[60,   400/  469] train loss: 0.009 train accuracy: 1.000\n"
     ]
    }
   ],
   "source": [
    "temperatures = [1]    # temperature for distillation loss\n",
    "# trade-off between soft-target (st) cross-entropy and true-target (tt) cross-entropy;\n",
    "# loss = alpha * st + (1 - alpha) * tt\n",
    "alphas = [0.0]\n",
    "learning_rates = [1e-2]\n",
    "learning_rate_decays = [0.95]\n",
    "weight_decays = [1e-5]\n",
    "momentums = [0.9]\n",
    "# No dropout used\n",
    "dropout_probabilities = [(0.0, 0.0)]\n",
    "hparams_list = []\n",
    "for hparam_tuple in itertools.product(alphas, temperatures, dropout_probabilities, weight_decays, learning_rate_decays, \n",
    "                                        momentums, learning_rates):\n",
    "    hparam = {}\n",
    "    hparam['alpha'] = hparam_tuple[0]\n",
    "    hparam['T'] = hparam_tuple[1]\n",
    "    hparam['dropout_input'] = hparam_tuple[2][0]\n",
    "    hparam['dropout_hidden'] = hparam_tuple[2][1]\n",
    "    hparam['weight_decay'] = hparam_tuple[3]\n",
    "    hparam['lr_decay'] = hparam_tuple[4]\n",
    "    hparam['momentum'] = hparam_tuple[5]\n",
    "    hparam['lr'] = hparam_tuple[6]\n",
    "    hparams_list.append(hparam)\n",
    "\n",
    "results_no_distill = {}\n",
    "for hparam in hparams_list:\n",
    "    print('Training with hparams' + utils.hparamToString(hparam))\n",
    "    reproducibilitySeed()\n",
    "    student_net = networks.StudentNetwork()\n",
    "    student_net = student_net.to(fast_device)\n",
    "    hparam_tuple = utils.hparamDictToTuple(hparam)\n",
    "    results_no_distill[hparam_tuple] = utils.trainStudentOnHparam(teacher_net, student_net, hparam, num_epochs, \n",
    "                                                                    train_val_loader, None, \n",
    "                                                                    print_every=print_every, \n",
    "                                                                    fast_device=fast_device)\n",
    "    save_path = checkpoints_path_student + utils.hparamToString(hparam) + '_final.tar'\n",
    "    torch.save({'results' : results_no_distill[hparam_tuple], \n",
    "                'model_state_dict' : student_net.state_dict(), \n",
    "                'epoch' : num_epochs}, save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "student test accuracy (w/o distillation):  0.9819\n"
     ]
    }
   ],
   "source": [
    "# Calculate student test accuracy\n",
    "_, test_accuracy = utils.getLossAccuracyOnDataset(student_net, test_loader, fast_device)\n",
    "print('student test accuracy (w/o distillation): ', test_accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyperparameter search utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plt.rcParams['figure.figsize'] = [10, 5]\n",
    "weight_decay_scatter = ([math.log10(h['weight_decay']) if h['weight_decay'] > 0 else -6 for h in hparams_list])\n",
    "dropout_scatter = [int(h['dropout_input'] == 0.2) for h in hparams_list]\n",
    "colors = []\n",
    "for i in range(len(hparams_list)):\n",
    "    cur_hparam_tuple = utils.hparamDictToTuple(hparams_list[i])\n",
    "    colors.append(results_no_distill[cur_hparam_tuple]['val_acc'][-1])\n",
    "    \n",
    "marker_size = 100\n",
    "fig, ax = plt.subplots()\n",
    "plt.scatter(weight_decay_scatter, dropout_scatter, marker_size, c=colors, edgecolors='black')\n",
    "plt.colorbar()\n",
    "for i in range(len(weight_decay_scatter)):\n",
    "    ax.annotate(str('%0.4f' % (colors[i], )), (weight_decay_scatter[i], dropout_scatter[i]))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train student network using distillation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 60\n",
    "print_every = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training with hparamsT=10, alpha=0.5, dropout_hidden=0.0, dropout_input=0.0, lr=0.01, lr_decay=0.95, momentum=0.9, weight_decay=1e-05\n",
      "[1,   100/  469] train loss: 1.911 train accuracy: 0.891\n",
      "[1,   200/  469] train loss: 1.322 train accuracy: 0.930\n",
      "[1,   300/  469] train loss: 1.071 train accuracy: 0.969\n",
      "[1,   400/  469] train loss: 0.722 train accuracy: 0.953\n",
      "[2,   100/  469] train loss: 0.664 train accuracy: 0.961\n",
      "[2,   200/  469] train loss: 0.582 train accuracy: 0.953\n",
      "[2,   300/  469] train loss: 0.429 train accuracy: 0.984\n",
      "[2,   400/  469] train loss: 0.480 train accuracy: 0.961\n",
      "[3,   100/  469] train loss: 0.351 train accuracy: 0.992\n",
      "[3,   200/  469] train loss: 0.297 train accuracy: 1.000\n",
      "[3,   300/  469] train loss: 0.310 train accuracy: 0.984\n",
      "[3,   400/  469] train loss: 0.308 train accuracy: 0.992\n",
      "[4,   100/  469] train loss: 0.246 train accuracy: 0.977\n",
      "[4,   200/  469] train loss: 0.321 train accuracy: 0.969\n",
      "[4,   300/  469] train loss: 0.250 train accuracy: 0.984\n",
      "[4,   400/  469] train loss: 0.210 train accuracy: 0.992\n",
      "[5,   100/  469] train loss: 0.241 train accuracy: 0.969\n",
      "[5,   200/  469] train loss: 0.197 train accuracy: 1.000\n",
      "[5,   300/  469] train loss: 0.183 train accuracy: 1.000\n",
      "[5,   400/  469] train loss: 0.173 train accuracy: 0.984\n",
      "[6,   100/  469] train loss: 0.201 train accuracy: 0.953\n",
      "[6,   200/  469] train loss: 0.181 train accuracy: 0.984\n",
      "[6,   300/  469] train loss: 0.162 train accuracy: 0.984\n",
      "[6,   400/  469] train loss: 0.227 train accuracy: 0.969\n",
      "[7,   100/  469] train loss: 0.186 train accuracy: 0.984\n",
      "[7,   200/  469] train loss: 0.137 train accuracy: 0.984\n",
      "[7,   300/  469] train loss: 0.145 train accuracy: 1.000\n",
      "[7,   400/  469] train loss: 0.155 train accuracy: 0.992\n",
      "[8,   100/  469] train loss: 0.167 train accuracy: 0.977\n",
      "[8,   200/  469] train loss: 0.126 train accuracy: 0.992\n",
      "[8,   300/  469] train loss: 0.125 train accuracy: 1.000\n",
      "[8,   400/  469] train loss: 0.130 train accuracy: 0.992\n",
      "[9,   100/  469] train loss: 0.139 train accuracy: 1.000\n",
      "[9,   200/  469] train loss: 0.155 train accuracy: 0.969\n",
      "[9,   300/  469] train loss: 0.112 train accuracy: 1.000\n",
      "[9,   400/  469] train loss: 0.128 train accuracy: 1.000\n",
      "[10,   100/  469] train loss: 0.117 train accuracy: 0.992\n",
      "[10,   200/  469] train loss: 0.102 train accuracy: 0.992\n",
      "[10,   300/  469] train loss: 0.113 train accuracy: 0.984\n",
      "[10,   400/  469] train loss: 0.110 train accuracy: 1.000\n",
      "[11,   100/  469] train loss: 0.152 train accuracy: 0.977\n",
      "[11,   200/  469] train loss: 0.108 train accuracy: 1.000\n",
      "[11,   300/  469] train loss: 0.110 train accuracy: 1.000\n",
      "[11,   400/  469] train loss: 0.137 train accuracy: 0.984\n",
      "[12,   100/  469] train loss: 0.130 train accuracy: 0.992\n",
      "[12,   200/  469] train loss: 0.122 train accuracy: 0.984\n",
      "[12,   300/  469] train loss: 0.106 train accuracy: 0.992\n",
      "[12,   400/  469] train loss: 0.095 train accuracy: 1.000\n",
      "[13,   100/  469] train loss: 0.142 train accuracy: 0.977\n",
      "[13,   200/  469] train loss: 0.122 train accuracy: 0.992\n",
      "[13,   300/  469] train loss: 0.139 train accuracy: 0.977\n",
      "[13,   400/  469] train loss: 0.114 train accuracy: 0.992\n",
      "[14,   100/  469] train loss: 0.114 train accuracy: 0.977\n",
      "[14,   200/  469] train loss: 0.103 train accuracy: 0.984\n",
      "[14,   300/  469] train loss: 0.116 train accuracy: 1.000\n",
      "[14,   400/  469] train loss: 0.102 train accuracy: 0.992\n",
      "[15,   100/  469] train loss: 0.109 train accuracy: 0.992\n",
      "[15,   200/  469] train loss: 0.100 train accuracy: 0.992\n",
      "[15,   300/  469] train loss: 0.098 train accuracy: 1.000\n",
      "[15,   400/  469] train loss: 0.090 train accuracy: 0.992\n",
      "[16,   100/  469] train loss: 0.095 train accuracy: 1.000\n",
      "[16,   200/  469] train loss: 0.097 train accuracy: 0.992\n",
      "[16,   300/  469] train loss: 0.081 train accuracy: 1.000\n",
      "[16,   400/  469] train loss: 0.091 train accuracy: 0.992\n",
      "[17,   100/  469] train loss: 0.084 train accuracy: 1.000\n",
      "[17,   200/  469] train loss: 0.102 train accuracy: 1.000\n",
      "[17,   300/  469] train loss: 0.111 train accuracy: 0.969\n",
      "[17,   400/  469] train loss: 0.087 train accuracy: 1.000\n",
      "[18,   100/  469] train loss: 0.105 train accuracy: 0.984\n",
      "[18,   200/  469] train loss: 0.114 train accuracy: 0.977\n",
      "[18,   300/  469] train loss: 0.098 train accuracy: 0.992\n",
      "[18,   400/  469] train loss: 0.105 train accuracy: 0.977\n",
      "[19,   100/  469] train loss: 0.085 train accuracy: 1.000\n",
      "[19,   200/  469] train loss: 0.095 train accuracy: 0.992\n",
      "[19,   300/  469] train loss: 0.099 train accuracy: 1.000\n",
      "[19,   400/  469] train loss: 0.080 train accuracy: 1.000\n",
      "[20,   100/  469] train loss: 0.078 train accuracy: 0.992\n",
      "[20,   200/  469] train loss: 0.080 train accuracy: 0.984\n",
      "[20,   300/  469] train loss: 0.096 train accuracy: 0.992\n",
      "[20,   400/  469] train loss: 0.091 train accuracy: 0.992\n",
      "[21,   100/  469] train loss: 0.081 train accuracy: 0.992\n",
      "[21,   200/  469] train loss: 0.076 train accuracy: 1.000\n",
      "[21,   300/  469] train loss: 0.093 train accuracy: 0.984\n",
      "[21,   400/  469] train loss: 0.075 train accuracy: 0.992\n",
      "[22,   100/  469] train loss: 0.087 train accuracy: 1.000\n",
      "[22,   200/  469] train loss: 0.075 train accuracy: 1.000\n",
      "[22,   300/  469] train loss: 0.088 train accuracy: 1.000\n",
      "[22,   400/  469] train loss: 0.073 train accuracy: 1.000\n",
      "[23,   100/  469] train loss: 0.088 train accuracy: 0.992\n",
      "[23,   200/  469] train loss: 0.089 train accuracy: 0.984\n",
      "[23,   300/  469] train loss: 0.100 train accuracy: 0.984\n",
      "[23,   400/  469] train loss: 0.093 train accuracy: 0.992\n",
      "[24,   100/  469] train loss: 0.065 train accuracy: 1.000\n",
      "[24,   200/  469] train loss: 0.074 train accuracy: 1.000\n",
      "[24,   300/  469] train loss: 0.116 train accuracy: 0.984\n",
      "[24,   400/  469] train loss: 0.082 train accuracy: 0.984\n",
      "[25,   100/  469] train loss: 0.092 train accuracy: 0.992\n",
      "[25,   200/  469] train loss: 0.076 train accuracy: 1.000\n",
      "[25,   300/  469] train loss: 0.079 train accuracy: 1.000\n",
      "[25,   400/  469] train loss: 0.072 train accuracy: 1.000\n",
      "[26,   100/  469] train loss: 0.102 train accuracy: 0.984\n",
      "[26,   200/  469] train loss: 0.099 train accuracy: 1.000\n",
      "[26,   300/  469] train loss: 0.078 train accuracy: 0.992\n",
      "[26,   400/  469] train loss: 0.073 train accuracy: 1.000\n",
      "[27,   100/  469] train loss: 0.066 train accuracy: 1.000\n",
      "[27,   200/  469] train loss: 0.101 train accuracy: 0.992\n",
      "[27,   300/  469] train loss: 0.070 train accuracy: 1.000\n",
      "[27,   400/  469] train loss: 0.083 train accuracy: 0.992\n",
      "[28,   100/  469] train loss: 0.062 train accuracy: 1.000\n",
      "[28,   200/  469] train loss: 0.069 train accuracy: 1.000\n",
      "[28,   300/  469] train loss: 0.071 train accuracy: 0.992\n",
      "[28,   400/  469] train loss: 0.078 train accuracy: 0.992\n",
      "[29,   100/  469] train loss: 0.084 train accuracy: 0.992\n",
      "[29,   200/  469] train loss: 0.079 train accuracy: 0.992\n",
      "[29,   300/  469] train loss: 0.093 train accuracy: 0.992\n",
      "[29,   400/  469] train loss: 0.083 train accuracy: 0.992\n",
      "[30,   100/  469] train loss: 0.080 train accuracy: 1.000\n",
      "[30,   200/  469] train loss: 0.073 train accuracy: 1.000\n",
      "[30,   300/  469] train loss: 0.089 train accuracy: 0.984\n",
      "[30,   400/  469] train loss: 0.070 train accuracy: 1.000\n",
      "[31,   100/  469] train loss: 0.078 train accuracy: 0.992\n",
      "[31,   200/  469] train loss: 0.078 train accuracy: 0.992\n",
      "[31,   300/  469] train loss: 0.116 train accuracy: 0.977\n",
      "[31,   400/  469] train loss: 0.066 train accuracy: 1.000\n",
      "[32,   100/  469] train loss: 0.076 train accuracy: 0.992\n",
      "[32,   200/  469] train loss: 0.081 train accuracy: 1.000\n",
      "[32,   300/  469] train loss: 0.074 train accuracy: 1.000\n",
      "[32,   400/  469] train loss: 0.062 train accuracy: 1.000\n",
      "[33,   100/  469] train loss: 0.067 train accuracy: 1.000\n",
      "[33,   200/  469] train loss: 0.067 train accuracy: 0.992\n",
      "[33,   300/  469] train loss: 0.081 train accuracy: 1.000\n",
      "[33,   400/  469] train loss: 0.064 train accuracy: 1.000\n",
      "[34,   100/  469] train loss: 0.100 train accuracy: 0.992\n",
      "[34,   200/  469] train loss: 0.065 train accuracy: 1.000\n",
      "[34,   300/  469] train loss: 0.080 train accuracy: 0.992\n",
      "[34,   400/  469] train loss: 0.097 train accuracy: 0.984\n",
      "[35,   100/  469] train loss: 0.069 train accuracy: 1.000\n",
      "[35,   200/  469] train loss: 0.078 train accuracy: 0.984\n",
      "[35,   300/  469] train loss: 0.076 train accuracy: 0.992\n",
      "[35,   400/  469] train loss: 0.093 train accuracy: 0.984\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[36,   100/  469] train loss: 0.065 train accuracy: 1.000\n",
      "[36,   200/  469] train loss: 0.066 train accuracy: 1.000\n",
      "[36,   300/  469] train loss: 0.079 train accuracy: 0.992\n",
      "[36,   400/  469] train loss: 0.068 train accuracy: 1.000\n",
      "[37,   100/  469] train loss: 0.088 train accuracy: 0.984\n",
      "[37,   200/  469] train loss: 0.078 train accuracy: 0.984\n",
      "[37,   300/  469] train loss: 0.062 train accuracy: 1.000\n",
      "[37,   400/  469] train loss: 0.089 train accuracy: 0.984\n",
      "[38,   100/  469] train loss: 0.099 train accuracy: 0.977\n",
      "[38,   200/  469] train loss: 0.079 train accuracy: 0.992\n",
      "[38,   300/  469] train loss: 0.077 train accuracy: 0.984\n",
      "[38,   400/  469] train loss: 0.078 train accuracy: 1.000\n",
      "[39,   100/  469] train loss: 0.063 train accuracy: 1.000\n",
      "[39,   200/  469] train loss: 0.082 train accuracy: 1.000\n",
      "[39,   300/  469] train loss: 0.090 train accuracy: 0.984\n",
      "[39,   400/  469] train loss: 0.061 train accuracy: 1.000\n",
      "[40,   100/  469] train loss: 0.086 train accuracy: 0.984\n",
      "[40,   200/  469] train loss: 0.071 train accuracy: 1.000\n",
      "[40,   300/  469] train loss: 0.063 train accuracy: 1.000\n",
      "[40,   400/  469] train loss: 0.072 train accuracy: 1.000\n",
      "[41,   100/  469] train loss: 0.069 train accuracy: 1.000\n",
      "[41,   200/  469] train loss: 0.078 train accuracy: 0.992\n",
      "[41,   300/  469] train loss: 0.074 train accuracy: 1.000\n",
      "[41,   400/  469] train loss: 0.064 train accuracy: 1.000\n",
      "[42,   100/  469] train loss: 0.070 train accuracy: 1.000\n",
      "[42,   200/  469] train loss: 0.069 train accuracy: 1.000\n",
      "[42,   300/  469] train loss: 0.068 train accuracy: 0.992\n",
      "[42,   400/  469] train loss: 0.060 train accuracy: 0.992\n",
      "[43,   100/  469] train loss: 0.073 train accuracy: 0.992\n",
      "[43,   200/  469] train loss: 0.092 train accuracy: 0.977\n",
      "[43,   300/  469] train loss: 0.075 train accuracy: 0.984\n",
      "[43,   400/  469] train loss: 0.079 train accuracy: 0.984\n",
      "[44,   100/  469] train loss: 0.067 train accuracy: 1.000\n",
      "[44,   200/  469] train loss: 0.067 train accuracy: 0.992\n",
      "[44,   300/  469] train loss: 0.089 train accuracy: 0.977\n",
      "[44,   400/  469] train loss: 0.058 train accuracy: 1.000\n",
      "[45,   100/  469] train loss: 0.074 train accuracy: 1.000\n",
      "[45,   200/  469] train loss: 0.066 train accuracy: 1.000\n",
      "[45,   300/  469] train loss: 0.076 train accuracy: 1.000\n",
      "[45,   400/  469] train loss: 0.069 train accuracy: 1.000\n",
      "[46,   100/  469] train loss: 0.071 train accuracy: 1.000\n",
      "[46,   200/  469] train loss: 0.065 train accuracy: 1.000\n",
      "[46,   300/  469] train loss: 0.085 train accuracy: 0.992\n",
      "[46,   400/  469] train loss: 0.067 train accuracy: 1.000\n",
      "[47,   100/  469] train loss: 0.078 train accuracy: 1.000\n",
      "[47,   200/  469] train loss: 0.061 train accuracy: 1.000\n",
      "[47,   300/  469] train loss: 0.076 train accuracy: 1.000\n",
      "[47,   400/  469] train loss: 0.066 train accuracy: 1.000\n",
      "[48,   100/  469] train loss: 0.077 train accuracy: 1.000\n",
      "[48,   200/  469] train loss: 0.067 train accuracy: 1.000\n",
      "[48,   300/  469] train loss: 0.059 train accuracy: 1.000\n",
      "[48,   400/  469] train loss: 0.061 train accuracy: 1.000\n",
      "[49,   100/  469] train loss: 0.065 train accuracy: 1.000\n",
      "[49,   200/  469] train loss: 0.072 train accuracy: 0.992\n",
      "[49,   300/  469] train loss: 0.076 train accuracy: 1.000\n",
      "[49,   400/  469] train loss: 0.081 train accuracy: 0.992\n",
      "[50,   100/  469] train loss: 0.062 train accuracy: 1.000\n",
      "[50,   200/  469] train loss: 0.060 train accuracy: 1.000\n",
      "[50,   300/  469] train loss: 0.067 train accuracy: 0.992\n",
      "[50,   400/  469] train loss: 0.068 train accuracy: 0.992\n",
      "[51,   100/  469] train loss: 0.060 train accuracy: 1.000\n",
      "[51,   200/  469] train loss: 0.077 train accuracy: 0.977\n",
      "[51,   300/  469] train loss: 0.060 train accuracy: 1.000\n",
      "[51,   400/  469] train loss: 0.068 train accuracy: 1.000\n",
      "[52,   100/  469] train loss: 0.072 train accuracy: 1.000\n",
      "[52,   200/  469] train loss: 0.065 train accuracy: 0.992\n",
      "[52,   300/  469] train loss: 0.061 train accuracy: 1.000\n",
      "[52,   400/  469] train loss: 0.067 train accuracy: 1.000\n",
      "[53,   100/  469] train loss: 0.074 train accuracy: 0.992\n",
      "[53,   200/  469] train loss: 0.065 train accuracy: 1.000\n",
      "[53,   300/  469] train loss: 0.070 train accuracy: 1.000\n",
      "[53,   400/  469] train loss: 0.088 train accuracy: 0.992\n",
      "[54,   100/  469] train loss: 0.067 train accuracy: 1.000\n",
      "[54,   200/  469] train loss: 0.081 train accuracy: 0.992\n",
      "[54,   300/  469] train loss: 0.064 train accuracy: 1.000\n",
      "[54,   400/  469] train loss: 0.063 train accuracy: 1.000\n",
      "[55,   100/  469] train loss: 0.060 train accuracy: 1.000\n",
      "[55,   200/  469] train loss: 0.076 train accuracy: 0.992\n",
      "[55,   300/  469] train loss: 0.065 train accuracy: 0.992\n",
      "[55,   400/  469] train loss: 0.055 train accuracy: 0.992\n",
      "[56,   100/  469] train loss: 0.067 train accuracy: 1.000\n",
      "[56,   200/  469] train loss: 0.077 train accuracy: 0.992\n",
      "[56,   300/  469] train loss: 0.082 train accuracy: 0.992\n",
      "[56,   400/  469] train loss: 0.066 train accuracy: 0.984\n",
      "[57,   100/  469] train loss: 0.061 train accuracy: 1.000\n",
      "[57,   200/  469] train loss: 0.060 train accuracy: 1.000\n",
      "[57,   300/  469] train loss: 0.073 train accuracy: 0.992\n",
      "[57,   400/  469] train loss: 0.072 train accuracy: 0.992\n",
      "[58,   100/  469] train loss: 0.062 train accuracy: 1.000\n",
      "[58,   200/  469] train loss: 0.064 train accuracy: 0.992\n",
      "[58,   300/  469] train loss: 0.059 train accuracy: 0.992\n",
      "[58,   400/  469] train loss: 0.067 train accuracy: 0.992\n",
      "[59,   100/  469] train loss: 0.079 train accuracy: 0.992\n",
      "[59,   200/  469] train loss: 0.065 train accuracy: 1.000\n",
      "[59,   300/  469] train loss: 0.078 train accuracy: 0.984\n",
      "[59,   400/  469] train loss: 0.075 train accuracy: 0.992\n",
      "[60,   100/  469] train loss: 0.070 train accuracy: 0.992\n",
      "[60,   200/  469] train loss: 0.061 train accuracy: 0.992\n",
      "[60,   300/  469] train loss: 0.065 train accuracy: 0.992\n",
      "[60,   400/  469] train loss: 0.081 train accuracy: 0.992\n"
     ]
    }
   ],
   "source": [
    "temperatures = [10]\n",
    "# trade-off between soft-target (st) cross-entropy and true-target (tt) cross-entropy;\n",
    "# loss = alpha * st + (1 - alpha) * tt\n",
    "alphas = [0.5]\n",
    "learning_rates = [1e-2]\n",
    "learning_rate_decays = [0.95]\n",
    "weight_decays = [1e-5]\n",
    "momentums = [0.9]\n",
    "dropout_probabilities = [(0.0, 0.0)]\n",
    "hparams_list = []\n",
    "for hparam_tuple in itertools.product(alphas, temperatures, dropout_probabilities, weight_decays, learning_rate_decays, \n",
    "                                        momentums, learning_rates):\n",
    "    hparam = {}\n",
    "    hparam['alpha'] = hparam_tuple[0]\n",
    "    hparam['T'] = hparam_tuple[1]\n",
    "    hparam['dropout_input'] = hparam_tuple[2][0]\n",
    "    hparam['dropout_hidden'] = hparam_tuple[2][1]\n",
    "    hparam['weight_decay'] = hparam_tuple[3]\n",
    "    hparam['lr_decay'] = hparam_tuple[4]\n",
    "    hparam['momentum'] = hparam_tuple[5]\n",
    "    hparam['lr'] = hparam_tuple[6]\n",
    "    hparams_list.append(hparam)\n",
    "\n",
    "results_distill = {}\n",
    "for hparam in hparams_list:\n",
    "    print('Training with hparams' + utils.hparamToString(hparam))\n",
    "    reproducibilitySeed()\n",
    "    student_net = networks.StudentNetwork()\n",
    "    student_net = student_net.to(fast_device)\n",
    "    hparam_tuple = utils.hparamDictToTuple(hparam)\n",
    "    results_distill[hparam_tuple] = utils.trainStudentOnHparam(teacher_net, student_net, hparam, num_epochs, \n",
    "                                                                train_val_loader, None, \n",
    "                                                                print_every=print_every, \n",
    "                                                                fast_device=fast_device)\n",
    "    save_path = checkpoints_path_student + utils.hparamToString(hparam) + '_final.tar'\n",
    "    torch.save({'results' : results_distill[hparam_tuple], \n",
    "                'model_state_dict' : student_net.state_dict(), \n",
    "                'epoch' : num_epochs}, save_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "student test accuracy (w distillation):  0.9866\n"
     ]
    }
   ],
   "source": [
    "# Calculate student test accuracy\n",
    "_, test_accuracy = utils.getLossAccuracyOnDataset(student_net, test_loader, fast_device)\n",
    "print('student test accuracy (w distillation): ', test_accuracy)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}