notebooks/mlp_mnist_with_validation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multi-Layer Perceptron MNIST with Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Load and Visualize data\n",
    "# 2. Define a neural network\n",
    "# 3. Train the model\n",
    "# 4. Evaluate the performance on test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load and Visualize data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision import datasets\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data.sampler import SubsetRandomSampler\n",
    "\n",
    "num_workers = 0\n",
    "batch_size = 20\n",
    "valid_size = 0.2\n",
    "\n",
    "transform = transforms.ToTensor()\n",
    "\n",
    "train_data = datasets.MNIST('data', train=True, download=True, transform=transform)\n",
    "\n",
    "test_data = datasets.MNIST('data', train=False, download=True, transform=transform)\n",
    "\n",
    "# obtrain training indices\n",
    "num_train = len(train_data)\n",
    "indices = list(range(num_train))\n",
    "np.random.shuffle(indices)\n",
    "split = int(np.floor(valid_size * num_train))\n",
    "train_idx, valid_idx = indices[split:], indices[:split]\n",
    "\n",
    "# define samplers for obtaining training and validation batches\n",
    "train_sampler = SubsetRandomSampler(train_idx)\n",
    "valid_sampler = SubsetRandomSampler(valid_idx)\n",
    "\n",
    "# prepare data loaders\n",
    "train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, sampler=train_sampler, num_workers=num_workers)\n",
    "valid_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, sampler=valid_sampler, num_workers=num_workers)\n",
    "test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size, num_workers=num_workers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visualize a Batch if Training Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABXUAAAD7CAYAAAAl6XdWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xm8VfP6wPHnq0Gj0kSFCg2UFCljutdQmSqVQnFlzFQR0aA0cLmIpIlGkUgDQubMUm4RQtxKc6d51rB+f5x8f9/nq73ts8/ZZ+11zuf9evX6PY9n7b2e36t1117721rPNkEQCAAAAAAAAAAgGg4JuwEAAAAAAAAAQOJY1AUAAAAAAACACGFRFwAAAAAAAAAihEVdAAAAAAAAAIgQFnUBAAAAAAAAIEJY1AUAAAAAAACACGFRFwAAAAAAAAAiJN8s6hpjqhpj3jTGbDTGrDbGDDXGFAy7L6Q3Y8wJxpgPjDGbjTGLjTGtwu4J6c0Yc7sxZq4xZrcxZlzY/SA6ON8gWcaY6saYXcaYiWH3gvTHuQbJMMZMNMasMsZsMcb8bIy5IeyeEB18TiGrjDHtjTE/GmO2G2N+NcacE3ZPiIb8dr7JN4u6IjJMRNaKSEURqSci54rIraF2hLR2YNF/hoi8ISJlROQmEZlojKkRamNIdytFZKCIjAm7EUQH5xtk0zMi8nXYTSD9ca5BNjwsIlWDIDhMRC4TkYHGmFND7gnRwecUEmaMuUBEHhGR60SkpIg0FpHfQm0KUZKvzjf5aVG3moi8HATBriAIVovI2yJSO+SekN5qiUglERkcBMG+IAg+EJHPRKRjuG0hnQVBMDUIgukisj7sXhApnG+QFGNMexHZJCLvh90LIoFzDZISBMH3QRDs/jM98Oe4EFtCRPA5hSQ8KCL9gyD4MgiC/UEQrAiCYEXYTSH95cfzTX5a1H1KRNobY4oZYyqLSHPJXNgFYjEx/lud3G4EQJ7H+QZZZow5TET6i8jdYfeCyOBcg6QZY4YZY3aIyCIRWSUib4bcEtIcn1PIKmNMARFpICLlD4wIWn5gdGbRsHtDesuv55v8tKg7WzLvzN0iIstFZK6ITA+1I6S7RZI5suMeY0whY8yFkjm2o1i4bQHIgzjfIBkDRGR0EAS/h90IIoNzDZIWBMGtkvko9DkiMlVEdsd/BcDnFLLsCBEpJCJtJPNcU09E6otI7zCbQiTky/NNvljUNcYcIiKzJPPio7iIlBORwyVzTgtwUEEQ7BGRliJysYislsx/8XlZMv9RAAByDOcbZJUxpp6InC8ig8PuBdHBuQbZdWBsx6cicpSIdA67H6QvPqeQpJ0H/u/TQRCsCoIgQ0SeEJGLQuwJaS4/n28Kht1ALikjIkeLyNADs6B2G2PGSuaPGd0bamdIa0EQfCuZd7CIiIgx5nMRGR9eRwDyKs43yKImIlJVRJYZY0RESohIAWPMiUEQnBJiX0hznGuQQwoKM3URXxPhcwpZFATBRmPMcsmc2w0kqonk0/NNvrhT98C/7vxPRDobYwoaY0qLyLUisiDczpDujDF1jTFFDsxi7i4iFUVkXMhtIY0dOMcUEZECkvlBUuTAr40DcXG+QRaNkswFlXoH/owQkZki0jTMppD+ONcgq4wxFYwx7Y0xJYwxBYwxTUXkShH5IOzekNb4nEKyxorIHQfOPYeLSFcReSPknpDe8u35Jl8s6h5wuYg0E5F1IrJYRPaKSLdQO0IUdJTMH4JYKyLnicgFzi//AgfTWzIfG7pPRDociJkBhURwvkHCgiDYEQTB6j//iMg2EdkVBMG6sHtD2uNcg6wKJHPUwnIR2Sgij4lI1yAIZoTaFdIan1PIhgEi8rWI/CwiP4rIf0VkUKgdIa3l5/ONCQLuagcAAAAAAACAqMhPd+oCAAAAAAAAQOSxqAsAAAAAAAAAEcKiLgAAAAAAAABECIu6AAAAAAAAABAhLOoCAAAAAAAAQIQUzMrGxpggVY0gyzKCICgfdhOJ4LhJH0EQmLB7SATHTFrhXINkcNwgGRw3SAbHDZLBcYNkcNwgy/gOjiQkfK7hTt3oWhp2AwDyBc41SAbHDZLBcYNkcNwgGRw3SAbHDYDckPC5hkVdAAAAAAAAAIgQFnUBAAAAAAAAIEJY1AUAAAAAAACACGFRFwAAAAAAAAAihEVdAAAAAAAAAIgQFnUBAAAAAAAAIEJY1AUAAAAAAACACGFRFwAAAAAAAAAihEVdAAAAAAAAAIiQgmE3AAAAAAAAkKyjjz5a5e+8846Na9WqpWr33nuvjf/zn/+ktjHkGGOMynv06GHjf/3rX6pWs2ZNGwdBoGq33XabjYcPH56DHQK5jzt1AQAAAAAAACBCWNQFAAAAAAAAgAhh/AIAAAAAAIis8ePHq9x9/D4jI0PVhg4dmis9IXsOOUTfgzho0CCVu+MX9u/fr2pffvmljWvUqKFqS5YsyaEOgfBxpy4AAAAAAAAARAiLugAAAAAAAAAQISzqAgAAAAAAAECEMFMXAJLUr18/lfft29fGDz74YNxtAQAA0sWRRx5p4/POOy/mdoUKFVL5mDFjVG6MsXEQBKq2YsUKGzdv3lzVFi5cmHizwEEUK1YsZu3RRx9V+c6dO1PdDpLkztEdOHCgqrkzdEVE9u3bZ+Nu3bqpGnOTkV9wpy4AAAAAAAAARAiLugAAAAAAAAAQIZEav1CyZEkbX3LJJap2xhln2Lh169aqVrFiRRu7jwQdjPuY0KxZs1Tt1FNPtfGUKVNU7ccff7Tx008/HXcfCE/RokVV3r17dxu3adNG1erWrWvjV199VdX69++v8u+++87G/qNmyJ/cUQwiIueee67K3fEMH330UW60hBC5j6vefPPNqubmderUUbW77rpL5YMHD05Bd0hX7dq1s/GLL76oakuXLrXxWWedpWqrVq1KbWNZULx4cZVv3749pE7wd/zH6hs2bGjjJ554QtVOO+00G69du1bVBgwYYONnnnkmJ1tEAgoUKGDjww47TNXcR9DdcQsiIuXKlbOx+3f/d/zr3njXwZUqVbLxpZdeqmqMX4gG9/gS0eeCs88+W9XGjRtn44yMjJT0U6ZMGRv7nzfuiIUffvghJftHzrvgggtsfN9996maPzbDrTNuAdl11FFH2fj+++9XtSuuuMLGxx13nKpt2bIltY39De7UBQAAAAAAAIAIYVEXAAAAAAAAACKERV0AAAAAAAAAiJBIzdR1Z9V27Ngx4de5s52yMu/0wgsvjFnzZyJu3LjRxq+99pqquXPvEK5bbrlF5e5cU597rFx++eWq5ufvvfeejUeNGqVq/vxl5E9NmjRR+ezZs23MTN28p1ixYiqfMWOGjc877zxV++yzz2zsz0L159MxUzdva968ucp79+5tY//6Jd4cwdzgzqh359+J6N82WLJkiar588aRu4oUKWJjfyacO/9SROSUU06J+T7u8VihQgVVe+CBB2zMTN3cd8MNN9h42LBhIXaCvMKdt+3+HomIyKBBg2K+btKkSSnr6U9jx461ce3atVXN/c2bmTNnprwXJMe/ZnaPsb1796pap06dVD558uTUNYY8z53zLiLyzjvv2Nj/LS732jbsGbo+7tQFAAAAAAAAgAhhURcAAAAAAAAAIiStxy+0a9dO5W3bto257cqVK2380ksvqdqrr75qY/cx15x0+OGH2/jWW29VtR49eqRkn0hMqVKlbNyrV6+Y23333Xcqnzdvno1POOEEVTvttNNUfv7559u4QYMGqnb00Ufb2H8M8Y8//ojZD9Jfv379VJ6Vx4rdbf33QTS5j8BPmzZN1dyRC/6jio888oiN3UfqRUTWrFkTc39XX321ynft2mVj93MP0eE+ti4icuKJJ9rYH7/gPp64ePHi1DYmIjVq1FD5PffcY+Prrrsu5usef/zxlPWEv9e+fXuVu39v9evXV7V4I8oGDBig8oyMDBsPGTJE1Q477DAbt2nTRtUYSZUzSpYsaeO3335b1erUqZPt93f/fkVEPvjgAxs/9dRTqnbxxRervGfPntneP9JL9erVbRxv3IL/vWb//v053kvjxo1V7o83c7nXRUhfLVq0ULl7zfzNN9+oGuMW8Hf8MaljxoxRefny5W3sr/+sW7fOxmeddZaqrV+/PqdazHHcqQsAAAAAAAAAEcKiLgAAAAAAAABECIu6AAAAAAAAABAhaTdTt2DB/2+pd+/eqlakSBEb79ixQ9XcuaW+U0891cadO3dOujd3Lpk/z8cYY+MqVarErMWbV4bUKFy4sI39eZWuESNGqHz48OExt/Xnh7lzEP15u+48wcqVK6uaOxMRQLQcddRRKn/uueds7M4DExG57777bOzP1t6+fbuNt23bpmr+rDj3fFKvXj1Vc2cgMlM3fVWqVEnl48ePt7E/4zQed4ZzqtSuXdvG//73v1WtefPmKd8/EuPPI+zTp4+NTz75ZFU75JD/v59j1apVqtatWzeVv//++zZ2r89F/no8uNzrrpo1a8bcDsnbu3evjTdv3qxqJUqUiPm61atX29j9zPLNmDFD5f5cS5d7nkDecMMNN6j87rvvTuh1/ncp/xyTE/zzlDtf2ufPqUf6cP/e/O/De/bssXGXLl1yrSdEl/udyJ/7/uuvv6p84MCBNl62bJmqnXvuuTbesmVLTraYUtypCwAAAAAAAAARwqIuAAAAAAAAAERI2o1fcBUrVixH3mfevHkHjbNq1KhRNp4wYYKqXX311TZu27atqnXs2NHG7uMESC/fffddwtvOnDlT5V988YWN3333XVWrU6eOjQ899NAkuwOQDtzHWl988UVVO/vss23sP5r86KOPxnzPY445xsb+I47+yKDffvvNxv45q2LFijH3gdzlj0ZwH4f3H2stXbp0Uvu45ZZbbOwfi+vWrUv4fapWrWpjt08RkdatW9vYf8SVcVLhGjNmjI396073+nnTpk2q9sgjj9jYHf0hIrJmzZqY+/PPaddcc03Mbd19xhtlheTt3LnTxu53EBGRYcOG2fj4449XNfdR5s8//zzh/R1++OE2rlGjhqr1798/4fdxR3q436sQPvezwB0ZJSJy7LHHxnxdv379bPzQQw/ldFsiItKwYUMbN2vWLOZ2n3zyicrfeuutlPSDrPNH+LjHjT96avLkyTb+7LPPUtoX8gb3muTnn39WNXcUq4hI9erVbXzBBReoWpRGLri4UxcAAAAAAAAAIoRFXQAAAAAAAACIEBZ1AQAAAAAAACBC0m6m7t69e23sz+8aMWKEjYsWLapqHTp0sPHEiRNT1N3/W7FiRcr3gdx16aWXqvzTTz+Nue1RRx2l8jvuuMPGGRkZqtaoUSMbz58/PzstAgiZOy/OnaErIvLee+/ZeODAgTHfo0mTJip//vnnbezO7BURufbaa1U+bdo0G3/77bd/3zBCsWvXLpW78wjd2ZR/x5152qlTp+w3JiKFChVS+dixY23cuHHjmK875BB9H8Avv/xiY382ptt3Vub7Qrvwwgtt7M+/PfLII238xx9/qNpLL71k44cffljVFi5cGHN//rWNO6vZP9+5tm7dqvJ27drZeMOGDTFfh5yxceNGlV955ZU5vo/nnnvOxi1btkz6fe69914br1+/Pls9IXuOO+44lb/++us2jjdD159bO3jwYBu73+NzUtOmTW0c7/dJBgwYoPL9+/enpB9k3UUXXaTybt262dhfV7n++utzpSdE12WXXaby22+/3cb+ms5jjz2mcvc88c0336Sgu9zHnboAAAAAAAAAECEs6gIAAAAAAABAhKTd+AXXhx9+qPJt27bZ2H9EtWfPnjaePn16zNdlR6tWrWzcvHnzmNv5jxMFQZAj+0dy3Edvdu/erWruIzzVq1eP+R41atRQ+auvvqry2rVr23jp0qWq9uuvvybeLCKlX79+YbeAFHMfcRYR6dixo43Xrl2ral27drVx3bp1Ve2qq66y8a233qpqM2bMsPFNN92kavEeT61atarK/UdwEZ5rrrlG5ZdffrmN410T+I/Yp+IRxLvvvlvl55xzjo3j9eaOWxDRowGWLFmSM83lc3Xq1FG5e61RrFgxVduyZYuN3UfaRURGjhyZ0P78x+gfeeQRlR9//PE2jndsjB49WuXuKBpEx80332zjvn37qlqFChVyZB/uuJe77rpL1b788ksb79y5M0f2h9j8z5datWrF3Nb9Tn7FFVeomj9+JScULlxY5Z07d465rXsttnz58hzvBckrU6aMjV944YWY2/mPx+/YscPG/piQeOM3fO51y549exJ+HdJTqVKlbOyOZRXR1z3+96PixYurPN6xGFXcqQsAAAAAAAAAEcKiLgAAAAAAAABECIu6AAAAAAAAABAhaT1Td/HixSqfNGmSjW+88UZVq1mzpo3deU0iegbirl27Et7/mWeeqfLHH3/cxlWqVFG1b775xsbt27dXtb179ya8T+Q8dyZlnz59VO3RRx+1sT8n2Z3t5c7MFfnrbBaXPweqQIECiTcLIK3489vdeU4ff/yxqj344IM2dmewi+jPgSlTpqjaLbfcYuNNmzYl3yzSxgknnJDwtu480jvuuCMV7Ui9evVsfNtttyX8utmzZ9vYn7/IHN2c4c4HfPbZZ1XNn6PrGjdunI1Xr16tai1atLDxlVdeGXN/7nYiWfsNCHdurj8bFenL/f7iXxO3a9fOxvGOvexw5837s5cfeuihmL0hZ7jzcP356q7t27erfODAgTaON+s/O9zvT7169VI1//cNXJMnT7bxokWLcr4xJM2dqet/d87IyLDxt99+q2rumos/T7lIkSIJ79+dqfvkk0+q2vDhwxN+H6SHk046ycb+b2a510+zZs1SNfe3t0RE1q1bl4LuwsWdugAAAAAAAAAQISzqAgAAAAAAAECEpPX4BV+XLl1s3LRpU1U75phjbHz55Zermvv4e4cOHVTNfQzN38e9994bc1v3kQERkZtvvtnGv/7668H/H0DonnrqKZW7Yzv8R0sbNmyY1D6WLVumch6nBqJr5cqVKv/+++9t3LhxY1XbuHGjjadNm6Zq7qOkPB4IV7ly5WzsPv6cFf64B3+cUKVKlWxctmzZmO/z22+/qdwdI7Jly5akekN8hxzy//dXZGVclzuq4/bbb09q3/4j1gUL6q8F/jgp19SpU5PaJ8LVrFkzG1933XUp2cecOXNsnJVr6fvuu8/Gy5cvV7WRI0dmvzGox5ALFSqU0HYiIh9++GHKevpT165dbRxv/Mb777+vcn9UA9JH27ZtY9bcax//79Tljv4QEZkwYULMbU8++WSVu8fGY489pmru9bw/Tg3pwb8GccevumNZReKPD/JHs+ZF3KkLAAAAAAAAABHCoi4AAAAAAAAARAiLugAAAAAAAAAQIZGaqbt7924bu3PeRPR8ldq1a6taixYtbDx37lxVc2eq/p2FCxfa+JprrlG1BQsWJPw+CM+ePXtUftttt9nYna0jIlK6dOmY79OkSROV+7M1AeQNO3bsULk7j7BixYqqtnTpUhuvW7cutY0hz3CvUdxYRMQYY+MgCJLeR6LvM2TIEJUzRzf1du7caWP/7//BBx+0cevWrVXtyCOPjPme7rnIn1U4c+ZMG2/btk3Vxo8fH3Mf7733nqr58+wQDfH+979//34b//HHH6rmzsn1zxM+97tWgwYNVK1y5co2fvzxx1XNnens/laJiMjEiRNt7M+CRuLcmaPxjgX/e80XX3xh4++++y5HernppptU3qlTp4Re97///U/l7vqAP4Nz3759B42RGqVKlVJ57969E3qd+7kkomfG+79t4Z+bXIsXL1b56tWrbfzJJ5+omjuDlZm66cn/nuWuv3300UeqdsYZZ9jY/XsXERk2bFjC+3TPdf7vDBxxxBEx3zPs62Xu1AUAAAAAAACACGFRFwAAAAAAAAAihEVdAAAAAAAAAIiQSM3Udc2fP1/ll112mY2//PJLVStfvryNszJD9+WXX1b59ddfb2N/ziKiyZ3L8+STTyb8On9bZuoC+cOKFSsOGgOuN998U+WnnnqqjevVq6dq8ea3H3LI///buzvvMqvc9/Hn0fXs2dPGQ4cOTXofyL4NGzao3J0r6MY55aefflK5P7/O9eKLL6o87PlxSM6sWbNsPHr0aFVzjwd/3m2yfv/995g1dz6hiD4XubNfRUTOP/98G8+YMSNHekNs/gxvPw/TDTfcoHJ3lqb/eerO3fz5559V7d1337Wxv3aA5LjXGiIiRYsWtfGaNWtUrUePHjb2Z9ouWbIkR/px5y/7n1mnnXaajf3ZqXv37s2R/SN71q5dq/JLL73Uxt27d1c197dM/GvwePyZze5vEkRp1jJ36gIAAAAAAABAhLCoCwAAAAAAAAAREtnxC8WLF1d51apVbVymTJmk3zcjI8PG1157rar5jywi/3AfHxHRt/8DyXAfCQOQt/j/+3Zz/xH38847z8bumAYRkQYNGth45MiRqtakSRMb/+tf/4rbjzu64dFHH1W1nHrMGtFQt25dG1evXl3VgiBQ+cKFC238yiuvpLYx5Ar30dKbbrop1/dfqFAhG5coUSLh17mP3DN+IXnuuMJu3bqpmvuZEiW1a9e2sf951r9//5iv2717d8p6wl/5I6QmTJiQ8n1WqlTJxv7a0XHHHWfjIkWKqNq2bdtS2xgSsnPnTpW7oxLc/92LiDz77LM2dkf55BfcqQsAAAAAAAAAEcKiLgAAAAAAAABECIu6AAAAAAAAABAhkZ2pO2jQIJXfcccdMbf9/fffbbxhwwZVO/nkk1VesmRJG/tzU1999dUs94m8oVq1anFz13fffZfqdpAHzJ49O+wWAIRg1apVKp84ceJB47/ToUOHhLedP3++jYcNG5bw6xB97nWtiMgjjzwSc1v/tyNuvPFGG+/YsSNnG0O+1KVLFxvfeeedCb/uzTffTEU7+c7rr79uY3/2u3tuaNOmjaqVK1cupX35/Pmr06ZNi7mt+73Lnxm/a9eunG0MkXL00UfbuECBAqr29ttv25gZutFw7rnn2njjxo2qNnTo0NxuJ61wpy4AAAAAAAAARAiLugAAAAAAAAAQIZEavzBq1CgbX3vttTG3mzlzpsq7du1q4zVr1qjaF198ofLatWvb2H1ESITxC/mZP6YjnilTpqSwE0SV/5hbv379QukDQDS1atVK5aeffnrCr23WrJmN161bl2M9If2deOKJKr/wwgtjbjt16lSVz5kzJyU9IfsOO+wwG3fu3FnV4o3YyA2FChWysf9dqm/fvgm9x+rVq1WeldE0SMzWrVtVfuutt9p4yJAhquYeb1lRoUIFG48cOVLVKlasqPKvvvrKxv5Yxblz5ya1f+Quf0zPjz/+aOPSpUunfP/+ddFzzz1nY38E53/+85+U94PsKVy4sMrbtWtnY/d8ISKycuXKXOkpXXGnLgAAAAAAAABECIu6AAAAAAAAABAhLOoCAAAAAAAAQISk9Uzdhg0bqvy6666z8SGH6PXozz//3Mb+3Ll9+/bF3MfgwYNV7s5eAf50xhlnxK1/++23Nv7ggw9S3Q4iaPbs2WG3ACBi3HmEEyZMULWiRYvGfN3111+vcubo5l89evRIeNv+/funsBPkpPbt29v48ssvV7VJkybZeNmyZSnvpUGDBiq/+uqrbXznnXcm/D5ur23atFE1f/4rUmvRokU58j7vvvuujf0Zuj53ji8zdKNp9+7dKnevRT777DNV++STT2z8wAMPqNqHH35o48aNG6uaPzfXPVfUrVtX1dyZrPfcc4+qzZ8//6//DyCt+H+fjRo1svHAgQNzu520xp26AAAAAAAAABAhLOoCAAAAAAAAQISk9fgFd9yCiB654N/e36FDBxvHG7fg27t3b5LdIa874ogjbHzllVfG3XbatGk25pgCAOSEf/zjHzYuVqxYzO1Gjhyp8vHjx6esJ6S/Fi1a2Lhly5Yxt3OvXUREfvrpp5T1hJzljvoaPny4qs2YMcPGt912m6r997//tfHOnTtVrVSpUjauXbt2zH136dJF5e55SkSkbNmyMV/r8kdDXHrppTZeuHBhQu+B9FarVq2YtRUrVqjcPTaRN3z11Vc2Pvfcc1Xt448/tvF7772nau46jztCQeSvIzhdkydPVvmgQYNs/P333yfQMdLJ2LFjVT5x4kQbv/HGG7ndTlrjTl0AAAAAAAAAiBAWdQEAAAAAAAAgQljUBQAAAAAAAIAISeuZuvHMmzdP5UuXLk3qfeLNSn3xxReTek/kDf/85z9t7M8HW758ucqfeeaZXOkJAP60aNEilVeuXNnGxYsXV7Xt27fnSk/IngoVKqi8d+/eNg6CQNU+/PBDG/fq1Su1jSGt+TMG27dvb2P/uHF16tQpZT0htdavXx+zVrduXRt/8sknquZ+t9m0aZOqHXXUUTa+7LLLstviQa1evdrGbdq0UTXm6OY9b775po1vvPFGVXOvWURE6tevb2P/+gbR5H7+fPbZZ6p24okn2viKK65QtdNPP93G7rWOiMhbb72lcvdY8X9XKd7nH9KTO4fb/UwS0b8XAI07dQEAAAAAAAAgQljUBQAAAAAAAIAISevxC1u3bo1ZO/PMM1U+fvx4G/ft2zfm66655hqVN23aVOXr1q2z8ahRoxLqE3lD+fLlVf7EE0/E3HbWrFkqz8jISElPABDLgw8+qPJXXnnFxh07dlS1ESNG5EpPyB7/GsV9PNF/jPDRRx+18caNG1PbGNKa/+iqn7tGjx5t482bN6esJ6SWOzqhTJkyqvbII4/Y2H/k/aqrrsrxXn7//XeVuyMfbr/9dlXbv3+/jeN9z0PecPPNNx80Rv7jX8O4YxP69++f2+0gTbnjoyZNmqRqv/32W263ExncqQsAAAAAAAAAEcKiLgAAAAAAAABECIu6AAAAAAAAABAhaT1Td8CAASq/8MILbXzSSSepWocOHQ4a/52VK1eqfNiwYTZ25z4h76tZs6bKjzjiCBuvWLFC1Xr06JErPSG9ffTRRyp353n7tX79+qW+IeQrb775ZsyaP2+3WrVqNvY/W7dt25azjSEl/Lm5q1atCqkTpJt//vOiWFstAAAgAElEQVSfMWtr165VuX9uQDS58yn92cjdu3e38ezZs1XNncVdqVKlhPc3Z84cGw8ZMkTV/DmHX331VcLvCwDAn6677job169fP8ROooU7dQEAAAAAAAAgQljUBQAAAAAAAIAISevxC1u3blX5JZdcYuPRo0er2vnnnx/zfXbv3m3jZ599VtXGjBmj8gULFmS5T0RXsWLFbBzvkcSBAweqfMOGDSnrCdHhj1gwxoTTCPKlXbt2qbxLly42fuqpp1StfPnyNmbcQjRNnTpV5QsXLgypE6SbxYsXq3zp0qU29kf/+OOkkPe45/hJkyapmp8DABCWypUrq9y9RmG9JXHcqQsAAAAAAAAAEcKiLgAAAAAAAABECIu6AAAAAAAAABAhJgiCxDc2JvGNkWrzgiBoEHYTieC4SR9BEERi6CvHTFrhXINkcNwgGRw3SAbHDZLBcYNkcNwgy/gOjiQkfK7hTl0AAAAAAAAAiBAWdQEAAAAAAAAgQljUBQAAAAAAAIAIYVEXAAAAAAAAACKERV0AAAAAAAAAiBAWdQEAAAAAAAAgQgpmcfsMEVmaikaQZVXCbiALOG7SA8cMksFxg2Rw3CAZHDdIBscNksFxg2Rw3CCrOGaQjISPGxMEQSobAQAAAAAAAADkIMYvAAAAAAAAAECEsKgLAAAAAAAAABHCoi4AAAAAAAAAREi+WdQ1xlQ1xrxpjNlojFltjBlqjMnqD8UhHzHG3G6MmWuM2W2MGRd2P4gOY8wJxpgPjDGbjTGLjTGtwu4J6c8Y85ExZpcxZtuBPz+F3ROiwRhT/cCxMzHsXhANnG+QFcaYQ40xo40xS40xW40x/zXGNA+7L6Q/Y0wZY8w0Y8z2A8fPVWH3hPTmfC79+WefMebpsPtC+jPGTDTGrDLGbDHG/GyMuSHsnnJDvlnUFZFhIrJWRCqKSD0ROVdEbg21I6S7lSIyUETGhN0IouPAPxbNEJE3RKSMiNwkIhONMTVCbQxRcXsQBCUO/KkZdjOIjGdE5Ouwm0DkcL5BogqKyO+S+f2plIj0EZGXjTFVQ+wJ0fCMiPwhIkeIyNUiMtwYUzvclpDOnM+lEpJ53OwUkVdCbgvR8LCIVA2C4DARuUxEBhpjTg25p5TLT4u61UTk5SAIdgVBsFpE3hYRPlAQUxAEU4MgmC4i68PuBZFSS0QqicjgIAj2BUHwgYh8JiIdw20LQF5kjGkvIptE5P2wewGQNwVBsD0Ign5BECwJgmB/EARviMj/RCTPf1lG8owxxUWktYj0CYJgWxAEn4rIa8I1MRLXRjJvzPsk7EaQ/oIg+D4Igt1/pgf+HBdiS7kiPy3qPiUi7Y0xxYwxlUWkuWQu7AJATjIx/lud3G4EkfSwMSbDGPOZMaZJ2M0gvRljDhOR/iJyd9i9IJI43yApxpgjRKSGiHwfdi9IazVEZF8QBD87/22BcGMVEnetiEwIgiAIuxFEgzFmmDFmh4gsEpFVIvJmyC2lXH5a1J0tmR8gW0RkuYjMFZHpoXYEIC9aJJn/onyPMaaQMeZCyXxcsVi4bSECeojIsSJSWURGicjrxpg8/6/LyJYBIjI6CILfw24EkcP5BkkxxhQSkRdEZHwQBIvC7gdprYSIbPb+22YRKRlCL4gYY8wxkvkdanzYvSA6giC4VTLPMeeIyFQR2R3/FdGXLxZ1jTGHiMgsyfxLLS4i5UTkcBF5JMy+AOQ9QRDsEZGWInKxiKyWzDvoXpbMf0wCYgqC4KsgCLYGQbA7CILxkjm246Kw+0J6MsbUE5HzRWRw2L0gejjfIBkHvlM9L5kzUm8PuR2kv20icpj33w4Tka0h9ILouUZEPg2C4H9hN4JoOTAC8VMROUpEOofdT6rli0VdyfyxoqNFZOiBi9f1IjJWuHgFkAJBEHwbBMG5QRCUDYKgqWTeDTUn7L4QOYEcfJwHICLSRESqisgyY8xqEekuIq2NMd+E2RQii/MN4jLGGBEZLZk/XNT6wD9iA/H8LCIFjTHVnf92sjC2A4m5RrhLF9lTUJipmzcEQZAhmcP8OxtjChpjSkvmfJYF4XaGdHbgWCkiIgVEpIAxpogxpmDYfSH9GWPqHjheihljuotIRREZF3JbSGPGmNLGmKZ/nmeMMVeLSGPJfMoEOJhRknmhWu/AnxEiMlNEmobZFNIf5xskabiInCAilwZBsDPsZpD+giDYLplPyvY3xhQ3xpwlIi0k825vICZjzJmSOR7olbB7QTQYYyoYY9obY0oYYwoYY5qKyJUi8kHYvaVavljUPeByEWkmIutEZLGI7BWRbqF2hHTXW0R2ish9ItLhQNw71I4QFR0lczD7WhE5T0QucH6JEziYQiIyUDI/ozJE5A4RaRkEwU+hdoW0FQTBjiAIVv/5RzIfc90VBMG6sHtD2uN8gywxxlQRkZsl8x+QVhtjth34c3XIrSH93SoiRSXzmniSiHQOgoA7dfF3rhWRqUEQMKoDiQokc9TCchHZKCKPiUjXIAhmhNpVLjD8kCAAAAAAAAAAREd+ulMXAAAAAAAAACKPRV0AAAAAAAAAiBAWdQEAAAAAAAAgQljUBQAAAAAAAIAIKZiVjY0x/Kpa+sgIgqB82E0kguMmfQRBYMLuIREcM2mFcw2SwXGDZHDcIBkcN0gGxw2SwXGDLOM7OJKQ8LmGO3Wja2nYDQDIFzjXIBkcN0gGxw2SwXGDZHDcIBkcNwByQ8LnGhZ1AQAAAAAAACBCWNQFAAAAAAAAgAhhURcAAAAAAAAAIoRFXQAAAAAAAACIEBZ1AQAAAAAAACBCWNQFAAAAAAAAgAhhURcAAAAAAAAAIoRFXQAAAAAAAACIEBZ1AQAAAAAAACBCWNQFAAAAAAAAgAhhURcAAAAAAAAAIoRFXQAAAAAAAACIkIJhNwDkJ/v371f5Y489ZuOePXuq2t69e3OlJwB53ymnnGLjokWLqtrSpUttvHz58lzrCUDeUL58eZXXqVPHxq1atVK1s88+28b169dXtQULFti4Xr16OdkiAABAnsSdugAAAAAAAAAQISzqAgAAAAAAAECEsKgLAAAAAAAAABHCTF0gxbp3727jIAhU7a677rLxf/7zH1Vbt25dahsDEDmHHnqojY866ihVa926tY07deqkalWrVrVx4cKFVc0914waNUrV+vTpk3SvANKbP7fWP6cUK1bMxv45xVW9enWVu+ebePzfGTjppJMSeh2A9HHFFVfY2P+eY4yxcZcuXVTtzDPPVLl7PjjkEH3fmfsbJF9//bWqff755zbmdwGAaHNn9B9zzDExtzvhhBNUfv/999s4IyND1R566CGVz5o1KzstpiXu1AUAAAAAAACACGFRFwAAAAAAAAAiJJTxC0WLFrXx2LFjVa1t27ZJvef7779v4++//17V3MfLmjRpomr+o1+uHTt2qPy5556zcbt27VRt2bJlNu7atauqffnllzH3gbzHf+zQHb/g++GHH2zsH2+ItuOPP17l/mNnLVu2tHGlSpUSfl/3Uba+ffuq2oABA7LSIiLgvPPOU7n7GdarV6+Yr3OPE5G/PhLpch91aty4cRY7RFhuvvlmG/vXHf5jaYk6/PDDVT5ixAgbX3TRRapWv359Gy9evDip/SH33XnnnTYeOHCgqhUvXjxH9rFnz56EtitYUH8N8c9bSC1/FE/p0qVtfOutt6ra9OnTbfzLL7+o2vbt21PQHdJVt27dVO6ORvC/V7tjFPza3+Uu9zPOH83gjn9g/AIQLf53mRtuuMHG/vgF97tMvO85tWrVUrWZM2eqfNKkSTbu2LFjFjtOT9ypCwAAAAAAAAARwqIuAAAAAAAAAEQIi7oAAAAAAAAAECGhzNStWbOmjZs3b65q8eb+xfPPf/7zoLHPn9cTb3/u7F8RkTvuuCPmthUqVLDxVVddpWrM1M37SpUqZeMxY8aoWrly5WK+bvDgwTZmJln01K5dW+XuzC9/7rY/q9A99+zatUvVXn/99Zj7POWUU2zsz7w7/fTTbezO2xRhzlhUFCtWTOWjRo1SebVq1RJ6H/+zbe/evTb2559+/fXXNnbnTCG9+LO3hw4damP/HJIV7hzdhQsXqtqRRx5pY3+OInN0o8G9PhHR897/bobuokWLbLxv3z5Vmzx5so2XLl2qahMnTkyot3Hjxqnc/QxDari/XdK5c2dVc2eq+/MKe/fubWP39yBERF599VWVP/jgg9nuE+nFnVvrztAV+euMW5d7HPnb+ceYW89KzT1v+MciUs+de+r+xpGI/rzZvHmzqk2ZMsXG/jVrq1atbFy2bNmEe/GvU5588smEX4vc437W+LP93bW6eHP2/Zp7veL+VojIX4+hq6++2sbz5s1TtageM9ypCwAAAAAAAAARwqIuAAAAAAAAAERIKOMX5s+fb+MlS5aomv84cywvvPCCyt1HD93HAERENm7caONt27apWrLjHl5++WWVr1mzxsYrVqxI6j0RXY8++qiN3UcKfDNmzFC5fxwh/VSpUkXlAwYMsHHLli1VzX903uU/EuYeC2+//baqrV+/Pub7lChRwsb9+/dXtTvvvNPG/mOs7mNOSF/+I/ZlypSJua07NkFE5Ntvv7Xx2rVrVc093ubMmZOdFpGL3DFQQ4YMUTX3cdSMjIyk93HPPffY2B23ICKyYMECGyf6SD3Sy4gRI1TuPkbtXh+LiNx2220qdz+33BEuOeWuu+5SeXbGiODgunfvHjOPNx4snhNPPFHlVatWVbn7Pc+/7kU0ud+X/VGGLr/mjiD0H2v2H5929+HXunTpYuMzzzxT1dzRZytXrlQ1d8wdUmPZsmU2njBhgqr16tXLxocddpiqderUKaH3z8pajXs9IxLdR+nzulq1atk43mhU/9p26tSpNn722WdVzR2/4H+2PfHEEyp3x3v43+Wjesxwpy4AAAAAAAAARAiLugAAAAAAAAAQISzqAgAAAAAAAECEhDJT13X99derfNasWTYuVapUzNf5MygnTZpk44IF9f9b7qyOeHOAsuOCCy6w8apVq1KyD6QPd36TiEiHDh1s7M+BcmfWPfDAA6q2ffv2FHSH7GrYsKGN/TmSxx57bMzX/fDDDzZ++umnVc2f/ZMsdy74F198oWqdO3e28bp163Jkf8gZ7vyoX3/9VdX27Nlj48WLF6vaU089pfLq1avb+JFHHlE1d6Yuosmfy+3OzXZngImILF++3MbNmzdPeB/+PEJ3VqH/meTOX92wYUPC+0Du8q873FmF/rw4V48ePVQ+efLknG3sb3BMpYb7+yT+54Rrx44dKn/jjTds/Omnn6qa+/3JP26OPvpolbvnLWbq5g3ub4D436Xd+e4+dwam/9sSWdG2bVsb++c7d//+70kgd7m/OyIiMn78eBtffPHFqtamTRsbN2nSRNXc2cg///yzqvl/x+5vD7AGEw3ud1T//OGeJ9xjJCvKly+v8nPOOUfl/jkkL+BOXQAAAAAAAACIEBZ1AQAAAAAAACBCQh+/MHfuXJX36tXLxg8++KCqlS1b1sZjx45VNfeRxSlTpqja5s2bk+qtUKFCKj/11FNt7D4GIiJyyy232PiTTz5RtWbNmiW1f6QPfxRIo0aNVH7ooYfaOAgCVZs2bZqNFy5cmILukF2VK1dWufvoR6VKlVTNfTx+0KBBqjZ9+nQbb9myJSdbPKj27durfNOmTTaePXt2yveP2E4++WSVu2M87rnnHlXzxwm5+vXrl6N9If241y833nijqnXr1s3G/jmladOmNl60aFHC+/NHehQpUuSg+xP56zgQpKcqVaqofNSoUTG3/eWXX2w8evTolPWE8CxZssTG8+fPV7W9e/fa+KabblK1BQsWJPT+FSpUUHmfPn1U7o5/QDT5nwWJjjL0a4MHD86RftzvVv73LHeffg3hWrZsmY2HDx+uam7uj7Zzr3cyMjJUzf2uJSJy2WWX2XjmzJnJN4tc466NjBgxQtUefvjhbL+/fy3triGK6PEPd911V7b3lw64UxcAAAAAAAAAIoRFXQAAAAAAAACIEBZ1AQAAAAAAACBCQp+p63PnavjzR905l/5sjJEjR9q4RYsWqubOJPzmm2/i7r958+Y2vvTSS1XNnz3lcmdU/fvf/467D0TP3XffrfKzzjor5rY//vijynv27JmSnpA9JUuWtPHTTz+tau4c3TVr1qia+7/vCRMmpKi7xDz//PMq79ChQ0idwOfPB3NnDHbt2lXV4s3URd7XqVMnGz/xxBOqtmfPHhv7nyVZmaNbs2ZNG9etW1fVduzYYeOXXnop4fdE+ujdu3fC2/qz4JH3bN++3cbnnXeeqrnnFHe7v1O6dGkbN2zYMO62X3/9dcLvi/R0+umnq/yQQ2LfB+bW2rVrl5J+jDEHjf39+zVEw2+//RazVqZMGZW7v3Hke/fdd3OsJ+SOzp0758j71KpVy8aNGzdWNf+84K4H/t3aYFRwpy4AAAAAAAAARAiLugAAAAAAAAAQIWk3fsH16aefqty9rXrMmDGq5o5KuOiii1TtH//4h40feuihmK8T0bf0FyhQIGZv7rgFEZFrr73Wxh999FHM1yGa/JEe7uP5PndMiIjIunXrUtITsqd9+/Y2vuyyy1QtCAIbX3fddao2a9as1DaWBdOnTw+7BcQQb0TLd999l4udIN3Uq1dP5Y899ljMbWfMmGHj4cOHJ73PK6+80sYFC+pLv2HDhtl47dq1Se8DuatVq1Y2dq9Bff5ohhdeeCFlPSH9bNq0KanXFS9eXOUffPCBjf0RLr7XXnstqX0ifbjXwSIi+/fvP2j8d69LRT/xehs8eHBK9o/w+ONeKleurPLZs2fb+IsvvsiVnhC+8uXLq/ytt96y8THHHKNqP/zwg8o7duyYusZCwp26AAAAAAAAABAhLOoCAAAAAAAAQISwqAsAAAAAAAAAEZLWM3V9GzdutPEVV1yhai+99JKN/fmnRYsWtfHAgQNVLd7sH3/+xptvvmnjt99+W9WYo5v3uLMM69Spo2r+cfPxxx/b+IknnkhtY8gRvXr1illz/z5TNUO3SJEiNr7kkktUbejQoTFfd//999t47NixOd8YUq5KlSpht4AQ1a9fX+WFChWy8W+//aZqnTp1Smofhx56qMofeOABG/ufX/37909qH8hdVatWVXnPnj1tfMghse/RaNy4scpPOOGEhPc5Z84cG7szVUX+eo2MvGX+/Pkqr1atWsxtv/76a5VzHRx9xpiYuX++cWv+65LVtm3bmLm/D7efL7/8Mkf2j/Th/gbKwezevdvG+/btS3U7CJH7WwL+bxi517b+OWL06NEqz8jISEF34eJOXQAAAAAAAACIEBZ1AQAAAAAAACBCIjV+wbVnzx6VX3vttTb2RyHUq1cv4fd96623bNyyZUtV45b+vKdgwf//n8A//vEPVTvrrLNs7D9qtGnTJpX369fPxlu2bMnBDpFTTj/9dJVXqFAh5rZz587N8f27jz+LiDRt2tTGjRo1Svh93LENSF/+qJ/rrrvOxm3atFG1n376ycb//ve/Vc0dLSQisnPnzpxqESE55ZRTYtb889K4ceNs7D9q5o4I2rVrl6o9/fTTMfcxb948lW/bti3mti7/c9Ad8fDHH3+oGtdLOePEE0+08ZAhQ1Qt3nHkuvDCC5Pe/9VXX21j/ziZOnWqjR977DFV+/7775PeJ3KPf+0xc+ZMG5ctW1bV9u/fb+OJEyeqmj/CxX0cOlllypRRecWKFW3sfz+77LLLbNygQQNVW7hwoY1PPvnkbPeVX/hjetzcPRZE9GdDvLGGWdG1a1eVu/v0P4sef/zxHNknoumrr74KuwXkEnfsVLxzlO++++5TufsdfNq0aao2atSo7LQYGu7UBQAAAAAAAIAIYVEXAAAAAAAAACKERV0AAAAAAAAAiJDIztT1XXzxxTauWbNm0u9TunRpG9evX1/VUjFnE+GqU6eOjYcOHapq7rHgz4+aNWuWyr/++usUdIec9MMPP6jcncF0zjnnqNpdd91lY38W76RJkxLa35VXXqlyd0azSPJzx5hVGA3+3G13ntNTTz2latWrV7fxmDFjVK1atWoqd2f1+nNMEQ3uLFwRkfPOO8/G/vXL5ZdfftBYRP/9v/POO6p2ySWXxNx/oUKFVN66deuY27rnrSOPPFLV3GPTnU8mIrJ+/fqY74nEuXOUa9SokfDr9u7da+ONGzeq2pw5cw4aH4w7v65EiRKqds0119jY/ww9/vjjE+4Vuetf//qXjf0Z7vGue5977jkb33777arm/85JotxZuCIixx57rI1vueUWVXM/J+NdP/m12rVrJ9VbfmeMiZn7M23dmv+6rJg8ebKNzzjjDFVz/179ffzdeQzR4/7mTfPmzVXNP9+8/vrrudITwvfss8/aeMSIEaq2aNEiG+/YsUPVatWqpXL3mtX/3QG31rFjR1Xz3zedcKcuAAAAAAAAAEQIi7oAAAAAAAAAECEs6gIAAAAAAABAhER2pu6hhx6q8l69etm4WLFiMV+3detWlZcsWVLlZ555po39GS0VK1bMcp9IL0WKFFH5a6+9ZuNKlSrFfN2yZctU7s7HFBHZuXNnDnSHVNqyZYvK+/bta+Nx48apWpUqVWzszwr0Z+Mmyp/p6x5TzZo1i/k6fzbvxx9/nNT+ES7382TXrl2q5s5KbdGihar17t1b5e7MVX/WEzN2o+G9995T+Yknnmjjs88+W9VGjhxp4xNOOEHV3OugSy+9NO4+3RmEJ598sqq9+OKLMV/3448/2njIkCGq9sorr9h4w4YNcfeP5PjXui53bnGPHj1UbcWKFTb25y1nhTvDvXv37qrmzpsvXLhw0vtAajVo0EDljz76qI3LlCkT83XuDF0RPUfXn2lZqlQplZ9yyik2Pvfcc1XNnaN70kknqVqi81j9mfWff/65jf3fP/nuu+8Sek9o/mxiN/fnLbszdpP9vQj/tf77uPv0Z/pmZ59IT+53r/Lly6va5s2bVc5vHuUfU6dOtbH/956VmbpXX321je+//35Va9mypY0nTJigam3atMlix7mHO3UBAAAAAAAAIEJY1AUAAAAAAACACIns+IVq1aqp3H18Md5jGP6jjc8++6zKTzvtNBv7jxO5j0FPmjQp8WaRNp5++mmVxxu54OrWrZvKlyxZklMtISTuGAP3UUER/Xhy48aNVa127doJvf9LL72k8unTp6v8+eeft7F/znr11VdtPHPmzIT2h/S2evVqG7t/935+7733qtoNN9yg8rZt29p44cKFqjZgwIBs94lwffrppyp3z0UFC+pLtkKFCtn4o48+UrV69eqp3D2PuOcXEf04mz/CY+/evQeNkTsuueQSG/sjDtxH0FN1TTJt2rSYtSlTpqRkn8i+Ro0a2XjGjBmqFm/kgsv/nuOObfCvnd1RHCIilStXtrE/UiHedzT3mF6wYIGqDR8+3MazZ89WtXXr1sV8TyTH/3tzc3/8gVtLdISGiMjRRx8dM/ffx92nf+7xP9MQfe66js8dC4X8JSMj46Dx33FHM4iI9OnTx8ZVq1ZVNXc0Q6tWrVTtpptusvGoUaMS3n9u4E5dAAAAAAAAAIgQFnUBAAAAAAAAIEJY1AUAAAAAAACACInsTN2szM954YUXbOzPIGzfvr3K33nnHRsfd9xxqta/f38bv//++6q2du3ahPtB7jr++ONt3K5du4Rf584T8+ehIm9x57iJ6Hlt/uy2ZN14440qb9OmjY3Xr1+vatdcc42Nd+/enSP7RzS4cwtFRN544w2Vu59hp556aq70hPDEm2lbrFgxG/szdH2tW7e2sT83F+lrw4YNNvZnka5ZsyZXe6levXqu7g/Jc2doly5dOqn38L/nxJuFmxXujF9/3q97vbV06dIc2R+S4/99u/n+/ftVzZ13m5XjxJ/F3LBhw5jv4+4zp45FpK9OnTrFrH322We52Anyolq1atm4ZcuWquaeX/xzjfsbFOmGO3UBAAAAAAAAIEJY1AUAAAAAAACACIns+IUyZcrErPmPFsZ7dN5/vOfhhx+28bPPPqtq1apVs7H/WID7yKz/WArC5d4qX7x48Zjb/fLLLyrv0qVLynpC3te0aVOVjxgxQuXuIx1vvfWWqjFyIe8pW7asjd3HfnznnHOOyu+7776Y2xpjst8YIqtFixYxa//9739VzsiF9OWO0fBHRA0ZMsTG//vf/1Stbt26qW1MRI499lgbN2vWLOX7Q85wvyMl+6i6+0i9iP5u89NPP6na77//rvJBgwbZ+OOPP05q/wiXf33h5v6x4db8kQruuMTJkyerWtu2bVXuHqv+/t19fvXVV3F7B5C/ValSReU33HCDynv16mVj/zPSPfe44xBFRDIyMnKqxRzHnboAAAAAAAAAECEs6gIAAAAAAABAhLCoCwAAAAAAAAAREtmZuvHMmjVL5dOmTUv4tePGjbNxz549Vc2dLTZw4EBVW7lypY0nTJiQ8P6Q87p3767y2rVr29ifm7Jp0yYb+7PsNmzYkILukF+483oO5tdff7Vxnz59Ut0O4ihXrpzK3fPEnj17VK1w4cI2rlGjhqq582+rV6+uaiVKlLBx5cqVY/biz5Hzz1lbtmyx8Zw5c2K+D/KeQoUKqdyd9eXP8h87dmyu9ITsc68ZW7VqFXM7f27pLbfcYuPnn39e1bZv357Qvo844giV+/t3r4PjnbdefPHFhPaH3OF+bsSbqbtu3TqVz54928bDhw+P+brPP/9c5f7nJKJv8ODBKm/Tpo2N/c8bd95t165dVe2YY46xcaNGjVTNPzbd9/Xn9j7++OMxewMA9/dKRo4cqWpnnXWWyuN9RrrX1llZQwwbd+oCAAAAAAAAQISwqAsAAAAAAAAAERLZ8Qs///yzysuXL5/j+3AfNRER+eabb2Ju6z9qi9xVtWpVG/vjF8zpuTAAAAYhSURBVFy7d+9W+ZNPPmnjBQsW5HhfyNtKliyp8nnz5tn4uOOOUzX/UZBBgwbZeMWKFSnoDomaO3euyo888kgb+2NY3FoqrF+/XuXDhg2Lma9evTqlvSC9VKhQQeWNGze28Y4dO1TtmWeeyZWekH3+32sszZo1i5kPGTJE1eI9cu/yx70UKFAgodeJiDz99NM2/rtxQ8hdiX5O+WMT3JFkyN++/PJLlX/11Vc29scouOcRf2xC27Ztbeyfl/zzj/ta/7p46tSpibSNPMg/Tl566aWQOkHY3PW+KlWqqNrMmTNt7F9X+SNj3Gvmhx56SNVeeOGFbPcZBu7UBQAAAAAAAIAIYVEXAAAAAAAAACKERV0AAAAAAAAAiJDIztRt3bq1ypcvX27jpk2bqtrDDz9s4wEDBqiaP4fOtWbNmuy0iBQqXLiwyt15buXKlVO1X375xcb333+/qk2bNi0F3SEvO/roo23crVs3VTv22GNt/Nhjj6lajx49UtsYkta3b1+Vu3Nr/Rlw+/bts7H7uSMismvXLhtPmTJF1RYuXGjjpUuXqpp7jtq7d6+qbd68OW7vyD8uvPDCmLXXXnstFztBTurZs6eN/dm07tzkIkWKxHyPrMzCjWflypUq//rrr208evRoVfv0009t7J4XEb5169aF3QLymMGDB9v4xRdfVDV3Fq4/uzLRml+/4oorVM2f8Yv8w78O37ZtW0idIDe0atXKxu71kYhe4ylWrJiqlS1b1sb+ueaHH35Q+QMPPGDjvLIWxJ26AAAAAAAAABAhLOoCAAAAAAAAQIREdvxCRkaGyt3HQu655x5Vc/Nbb71V1SZPnhxzH0WLFs1Oi0ihQw89VOVlypSJue1NN91k448//jhlPSF/uP3222185513qpr7WL0/fgHpa/z48SqfN2+ejf3HWKtUqWLjRYsWqdqWLVtS0B2Q6bTTTlO5e7y55yVEizvGoHnz5qpWtWpVG/vjN2rXrp3Q+y9evFjllStXPui+RUTmzp2r8tWrVye0DwB52yuvvHLQWEQ/6uyPVDDGJFTz64xbyF/87/XumEX/OClZsmSu9ITwNWjQQOXxzjVr16618QsvvKBqDz30kMr9dcS8gDt1AQAAAAAAACBCWNQFAAAAAAAAgAhhURcAAAAAAAAAIiSyM3V9gwYNsvFxxx2napdffrmNS5QooWqdOnVKan8rVqxQebzZvMh5W7duVXnr1q1D6gR5XaNGjVR+7bXX2njPnj2qNnDgQBv7s1gRHQsXLoxZW7NmTS52Avy/U089VeW7d++28caNG3O7HeSCJUuW2HjUqFHhNQIAMTz++OM27tq1q6q5cy/deZgif52b++STT6agO0SBPyPezYMgUDV/zqo/Cx7RNm3aNBu736tFRGrVqmXj6dOnq9onn3xi42XLlqWou/TFnboAAAAAAAAAECEs6gIAAAAAAABAhOSZ8Qvbtm2zcf/+/VVt+fLlNu7SpUvS+1i1apWNL774YlWL97gugOjq0KGDysuVK2djf8TC2LFjc6UnAPmP/zjZnDlzQuoEAIBM99xzz0FjIFHz589X+YIFC2x8/PHHq9rbb7+dKz0hfH369Am7hcjgTl0AAAAAAAAAiBAWdQEAAAAAAAAgQljUBQAAAAAAAIAIMUEQJL6xMYlvjFSbFwRBg7CbSATHTfoIgsCE3UMiOGbSCucaJIPjBsnguEEyOG6QDI4bJIPjJsWef/55G0+ZMkXVZsyYkdvt5Ai+gyMJCZ9ruFMXAAAAAAAAACKERV0AAAAAAAAAiJCCYTcAAAAAAACA/K1jx45htwBECnfqAgAAAAAAAECEsKgLAAAAAAAAABHCoi4AAAAAAAAAREhWZ+pmiMjSVDSCLKsSdgNZwHGTHjhmkAyOGySD4wbJ4LhBMjhukAyOGySD4wZZxTGDZCR83JggCFLZCAAAAAAAAAAgBzF+AQAAAAAAAAAihEVdAAAAAAAAAIgQFnUBAAAAAAAAIEJY1AUAAAAAAACACGFRFwAAAAAAAAAihEVdAAAAAAAAAIgQFnUBAAAAAAAAIEJY1AUAAAAAAACACGFRFwAAAAAAAAAi5P8AOrfGDLYljOoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f14c3700c50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "# obtain one batch of training images\n",
    "dataiter = iter(train_loader)\n",
    "images, labels = dataiter.next()\n",
    "images = images.numpy()\n",
    "\n",
    "# plot the images in batch, along with corresponding labels\n",
    "fig = plt.figure(figsize=(25, 4))\n",
    "for idx in np.arange(20):\n",
    "    ax = fig.add_subplot(2, 20/2, idx+1, xticks=[], yticks=[])\n",
    "    ax.imshow(np.squeeze(images[idx]), cmap='gray')\n",
    "    ax.set_title(str(labels[idx].item()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define the Network Architecture\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import nn, optim\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Network(\n",
       "  (fc1): Linear(in_features=784, out_features=512, bias=True)\n",
       "  (fc2): Linear(in_features=512, out_features=512, bias=True)\n",
       "  (fc3): Linear(in_features=512, out_features=10, bias=True)\n",
       "  (dropout): Dropout(p=0.2)\n",
       ")"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class Network(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "        hidden_1 = 512\n",
    "        hidden_2 = 512\n",
    "        self.fc1 = nn.Linear(28 * 28, hidden_1)\n",
    "        self.fc2 = nn.Linear(hidden_1, hidden_2)\n",
    "        self.fc3 = nn.Linear(hidden_2, 10)\n",
    "        self.dropout = nn.Dropout(0.2)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.view(-1, 28 * 28)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.dropout(x)\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.dropout(x)\n",
    "        x = self.fc3(x)\n",
    "        return x\n",
    "    \n",
    "model = Network()\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define loss\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "# define optimizer\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.01)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train the Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Clear the gradients\n",
    "# 2. Forwards pass\n",
    "# 3. Calculate loss\n",
    "# 4. Backward pass\n",
    "# 5. optimization step\n",
    "# 6. Update average training loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 \tTraining Loss: 0.284437 \tValidation Loss: 0.059595\n",
      "Validation loss decreased (inf --> 0.059595), saving model ...\n",
      "Epoch: 2 \tTraining Loss: 0.226962 \tValidation Loss: 0.048655\n",
      "Validation loss decreased (0.059595 --> 0.048655), saving model ...\n",
      "Epoch: 3 \tTraining Loss: 0.186444 \tValidation Loss: 0.041765\n",
      "Validation loss decreased (0.048655 --> 0.041765), saving model ...\n",
      "Epoch: 4 \tTraining Loss: 0.158884 \tValidation Loss: 0.037206\n",
      "Validation loss decreased (0.041765 --> 0.037206), saving model ...\n",
      "Epoch: 5 \tTraining Loss: 0.137609 \tValidation Loss: 0.032738\n",
      "Validation loss decreased (0.037206 --> 0.032738), saving model ...\n",
      "Epoch: 6 \tTraining Loss: 0.123785 \tValidation Loss: 0.030007\n",
      "Validation loss decreased (0.032738 --> 0.030007), saving model ...\n",
      "Epoch: 7 \tTraining Loss: 0.109513 \tValidation Loss: 0.028166\n",
      "Validation loss decreased (0.030007 --> 0.028166), saving model ...\n",
      "Epoch: 8 \tTraining Loss: 0.099745 \tValidation Loss: 0.025637\n",
      "Validation loss decreased (0.028166 --> 0.025637), saving model ...\n",
      "Epoch: 9 \tTraining Loss: 0.090415 \tValidation Loss: 0.024140\n",
      "Validation loss decreased (0.025637 --> 0.024140), saving model ...\n",
      "Epoch: 10 \tTraining Loss: 0.082282 \tValidation Loss: 0.022827\n",
      "Validation loss decreased (0.024140 --> 0.022827), saving model ...\n",
      "Epoch: 11 \tTraining Loss: 0.076340 \tValidation Loss: 0.021754\n",
      "Validation loss decreased (0.022827 --> 0.021754), saving model ...\n",
      "Epoch: 12 \tTraining Loss: 0.069974 \tValidation Loss: 0.020848\n",
      "Validation loss decreased (0.021754 --> 0.020848), saving model ...\n",
      "Epoch: 13 \tTraining Loss: 0.065834 \tValidation Loss: 0.019946\n",
      "Validation loss decreased (0.020848 --> 0.019946), saving model ...\n",
      "Epoch: 14 \tTraining Loss: 0.061062 \tValidation Loss: 0.019915\n",
      "Validation loss decreased (0.019946 --> 0.019915), saving model ...\n",
      "Epoch: 15 \tTraining Loss: 0.056406 \tValidation Loss: 0.018542\n",
      "Validation loss decreased (0.019915 --> 0.018542), saving model ...\n",
      "Epoch: 16 \tTraining Loss: 0.052844 \tValidation Loss: 0.018045\n",
      "Validation loss decreased (0.018542 --> 0.018045), saving model ...\n",
      "Epoch: 17 \tTraining Loss: 0.050615 \tValidation Loss: 0.017747\n",
      "Validation loss decreased (0.018045 --> 0.017747), saving model ...\n",
      "Epoch: 18 \tTraining Loss: 0.045913 \tValidation Loss: 0.017242\n",
      "Validation loss decreased (0.017747 --> 0.017242), saving model ...\n",
      "Epoch: 19 \tTraining Loss: 0.044441 \tValidation Loss: 0.016862\n",
      "Validation loss decreased (0.017242 --> 0.016862), saving model ...\n",
      "Epoch: 20 \tTraining Loss: 0.041873 \tValidation Loss: 0.016937\n"
     ]
    }
   ],
   "source": [
    "epochs = 20\n",
    "\n",
    "# initialize tracker for minimum validation loss\n",
    "valid_loss_min = np.Inf\n",
    "\n",
    "for e in range(epochs):\n",
    "    train_loss = 0.0\n",
    "    valid_loss = 0.0\n",
    "    \n",
    "    model.train()\n",
    "    for images, labels in train_loader:\n",
    "        optimizer.zero_grad()\n",
    "        output = model(images)\n",
    "        loss = criterion(output, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        train_loss += loss.item()*images.size(0)\n",
    "    \n",
    "    \n",
    "    model.eval()\n",
    "    for images, labels in valid_loader:\n",
    "        output = model(images)\n",
    "        loss = criterion(output, labels)\n",
    "        valid_loss += loss.item()*images.size(0)\n",
    "    \n",
    "    \n",
    "    train_loss = train_loss/len(train_loader.dataset)\n",
    "    valid_loss = valid_loss/len(valid_loader.dataset)\n",
    "    \n",
    "    print('Epoch: {} \\tTraining Loss: {:.6f} \\tValidation Loss: {:.6f}'.format(\n",
    "        e+1, \n",
    "        train_loss, \n",
    "        valid_loss))\n",
    "    \n",
    "    # save model if validation loss has decreased\n",
    "    if valid_loss <= valid_loss_min:\n",
    "        print('Validation loss decreased ({:.6f} --> {:.6f}), saving model ...'.format(\n",
    "            valid_loss_min,\n",
    "            valid_loss))\n",
    "        torch.save(model.state_dict(), 'model.pt')\n",
    "        valid_loss_min = valid_loss\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load('model.pt'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test the Trained Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss: 0.071090\n",
      "\n",
      "Test Accuracy of     0: 98% (970/980)\n",
      "Test Accuracy of     1: 99% (1126/1135)\n",
      "Test Accuracy of     2: 97% (1007/1032)\n",
      "Test Accuracy of     3: 98% (990/1010)\n",
      "Test Accuracy of     4: 97% (955/982)\n",
      "Test Accuracy of     5: 97% (870/892)\n",
      "Test Accuracy of     6: 98% (940/958)\n",
      "Test Accuracy of     7: 97% (999/1028)\n",
      "Test Accuracy of     8: 97% (946/974)\n",
      "Test Accuracy of     9: 96% (975/1009)\n",
      "\n",
      "Test Accuracy (Overall): 97% (9778/10000)\n"
     ]
    }
   ],
   "source": [
    "# initialize lists to monitor test loss and accuracy\n",
    "test_loss = 0.0\n",
    "class_correct = list(0. for i in range(10))\n",
    "class_total = list(0. for i in range(10))\n",
    "\n",
    "model.eval()\n",
    "\n",
    "for images, labels in test_loader:\n",
    "    # forward pass\n",
    "    output = model(images)\n",
    "    # calculate loss\n",
    "    loss = criterion(output, labels)\n",
    "    # update test loss\n",
    "    test_loss += loss.item()*images.size(0)\n",
    "    # convert output probabilities to predicted class\n",
    "    _, pred = torch.max(output, 1)\n",
    "    # compare predictions to true label\n",
    "    correct = np.squeeze(pred.eq(labels.data.view_as(pred)))\n",
    "    # calculate test accuracy for each object class\n",
    "    for i in range(len(labels)):\n",
    "        label = labels.data[i]\n",
    "        class_correct[label] += correct[i].item()\n",
    "        class_total[label] += 1\n",
    "\n",
    "# calculate and print avg test loss\n",
    "test_loss = test_loss/len(test_loader.dataset)\n",
    "print('Test Loss: {:.6f}\\n'.format(test_loss))\n",
    "\n",
    "for i in range(10):\n",
    "    if class_total[i] > 0:\n",
    "        print('Test Accuracy of %5s: %2d%% (%2d/%2d)' % (\n",
    "            str(i), 100 * class_correct[i] / class_total[i],\n",
    "            np.sum(class_correct[i]), np.sum(class_total[i])))\n",
    "    else:\n",
    "        print('Test Accuracy of %5s: N/A (no training examples)' % (classes[i]))\n",
    "\n",
    "print('\\nTest Accuracy (Overall): %2d%% (%2d/%2d)' % (\n",
    "    100. * np.sum(class_correct) / np.sum(class_total),\n",
    "    np.sum(class_correct), np.sum(class_total)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}