multilayer_perceptron.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Multi-Layer Perceptrons (MLPs)\n",
    "======"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# install pytorch (http://pytorch.org/) if run from Google Colaboratory\n",
    "import sys\n",
    "if 'google.colab' in sys.modules and 'torch' not in sys.modules:\n",
    "    from os.path import exists\n",
    "    from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag\n",
    "    platform = '{}{}-{}'.format(get_abbr_impl(), get_impl_ver(), get_abi_tag())\n",
    "    cuda_output = !ldconfig -p|grep cudart.so|sed -e 's/.*\\.\\([0-9]*\\)\\.\\([0-9]*\\)$/cu\\1\\2/'\n",
    "    accelerator = cuda_output[0] if exists('/dev/nvidia0') else 'cpu'\n",
    "\n",
    "    !pip3 install https://download.pytorch.org/whl/{accelerator}/torch-1.1.0-{platform}-linux_x86_64.whl\n",
    "    !pip3 install https://download.pytorch.org/whl/{accelerator}/torchvision-0.3.0-{platform}-linux_x86_64.whl\n",
    "\n",
    "%matplotlib inline\n",
    "import os\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Parameter Settings\n",
    "-------------------\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 20\n",
    "batch_size = 128\n",
    "learning_rate = 1e-3\n",
    "use_gpu = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MNIST Data Loading\n",
    "-------------------\n",
    "\n",
    "MNIST images show digits from 0-9 in 28x28 grayscale images. We normalize and center them around 0, which gives a slight performance boost during training.\n",
    "We create both a training set and a test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAADMCAYAAAB9RGhMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzsXXdYFNfbfZcqIFVQUREbsUUSiSXGEmIs2HuL2GOJJbaoSSBC1OjP2BJTrLGgxl6jYkMQu4gNpKkIiNJFBJSyM+f7A/Z+u+yCqDOzCc55nvPs7sydee/euXPmnfc2BQCSIUOGDBkVFwb6zoAMGTJkyBAXstDLkCFDRgWHLPQyZMiQUcEhC70MGTJkVHDIQi9DhgwZFRyy0MuQIUNGBYdoQq9QKDwUCkW0QqG4r1AovhXLjgwZMmTIKBsKMfrRKxQKQyKKIaLORJRIRCFENAxAhODGZMiQIUNGmRDLo29FRPcBxAIoIKJdRNRHJFsyZMiQIaMMGIl03ppE9EjtdyIRtVZPoFAoJhDRhOKfH4mUDxkyZMioyEgH4PCqRGIJvULHNo0YEYD1RLSeiEihUMjzMMiQIUPG6yO+PInECt0kEpGT2u9aRPREJFsyZMiQIaMMiCX0IUTkolAo6ioUChMiGkpER0SyJUOGDBkyyoAooRsASoVCMZWIThKRIRFtAnBXDFsyZMiQIaNsiNK98rUzIcfoZciQIeNNEAqgxasSySNjZciQIaOCQxZ6GTJkyKjgkIVehgwZMio4KpzQv/feezRq1CgyNDSkiIgIunPnDvXt21fSPHTp0oWePXtGzZs3l9SuDP1g6tSpBIAuXbpE48eP1+DRo0fp5s2bFBwcTOPHjycLCwu95NHJyYkiIyP1YluGJi5evEi9e/eW1igAvZOKBlO9NQ0NDXHhwgXwPI8ff/wRPM+D53ncunULlStXFsRGeXj16lXwPI8dO3ZIYq9GjRoAgMzMzFLTLFmyBD/99JNkZfAqmpmZwdzcHObm5jAwMNB7ft6Uffv2xfPnz8FxXLl448YNjB07Fo6OjpLm09XVFUlJSXovLyLC77//jpiYGHh6euo9L1LT3t4eT58+Re/evYU65/XyaGyF8eiNjIxo8eLF9MknnxARUXh4OKWkpBARkaurKw0fPpwUCl0DdsVD7dq1JbExf/584nmejI2NqU6dOjrT9evXj7p27UpmZmai56kstGnThv73v/9RZmYm5eTkUE5ODh0/flw0e5UrV6Z9+/YRAOJ5nkJCQsjW1law88fExFBGRka503/wwQe0YcMGmjFjhmB5+DfB3d2dzp49S9u2bSNLS8tS09WrV482bNhA48aNEy0v9vb25OnpSVu3bqUzZ87Q1KlTydPTk4yMxJoQ4NWwtbUlGxsb6Q3r25sXyqPv3Lkz8+Dv3LkDe3t7LFiwAIWFhWy7ra2tJE9tlUd//vx50W0tX76ceYuFhYXo0aOHVprBgwezNNWrV5fMe7GwsED79u3x3XffYceOHbh//77G9VCnWHmoVasWOI4Dz/N48OABdu3ahQYNGghqw8jICKNHj4aZmRmcnJywZcsWLcbHx2t49kuXLpXsOhBJ59EPGjQISqUSSqUSgYGBqFSpklaaq1evsjQBAQGC2m/QoAFWrlyJgIAAKJVKnXVtwYIFkpa9Ovv16wee5yX36PUu8kIIvYmJCc6ePcsu5Pbt29m+lStXsu2jRo2S5GJKJfSTJk1Ceno6Ew9fX1+d6Xbs2CGp0Ht4eGDt2rWIjY3VeaOVZHx8vGh5UQn97du38fHHH0ty/XWxbdu2ehX6bt26ITk5WXQ76kKvVCoxefJkjf1du3YFx3Fs/7x58wS1HxoaiuzsbDx58gT79u3D4sWL8dlnn8HFxQXz58/Hy5cv8fjxY1StWlUv9WDq1KngeR4eHh5CnfPdEXr1eHxQUBCqVKnC9o0cOZLtW7lypSQXUyqhP3fuHBOOqKgoWFlZ6UwXFBQEjuOQm5srWgWvVKkS+vfvj5CQEOZBl2Rqairi4uKwbt06eHh4oHr16nB3dy8130JQJfRSPeRLY7Vq1di1ysnJQYcOHSS1v2zZMsk9eqVSqfUWPXnyZCb0aWlpsLS0FNT+rl27MHjw4FL3HzlyBDzP44MPPtBLPZgxYwZ4nheyXapcQq+/YJVAcHNzo7Fjx7LfP/zwg0bM9Ny5cxQZGUmNGzeWJD9169alunXrim6nY8eOrD0iPz+ffHx86Pnz52Uec+rUKUpNTRUlP+PHj6dff/1Va/uZM2coMjKSjh07Rrdu3dKyn5ycLEp+VJg7d64kdsqCo6Mj7d69m/2+dOkSBQcH6y0/YkO9LSwzM1Nj3+TJk9l+juMoOztbUNtDhw4tdZ+FhQU1a9ZMUHtvAo7jJLf5nxZ6U1NTWrJkCdWsWZNyc3PJx8eHIiI0F7GKj4+nVatW0fr16yXJk4ODA9nb2xNRUUOdWOjZsycZGBS1pRcUFJChoSFZW1uTUqmk3Nxclq5BgwbUqlUrlk4M9OjRg1asWMF+p6am0tWrV2nx4sUUEhJCPM+LYvdVMDMzozFjxtCNGzfo5MmTesmDjY0N7dy5k9q2bUtERKGhoeTp6amXvEiF4rf0UveVeJuXDC1btiRnZ2dKS0sTzeEpDwICAiS/J/7TQt+2bVvq3LkzERGFhYXRypUr9ZofhUJBkydPJiKiFy9eiJqf1q3/fx0XS0tL2rZtGxERpaWl0dmzZ9mbTPfu3cnU1JSIiDZv3ix4PoYOHUpLlizR6MkQGBhI06dP1+vNRERUq1YtMjExobS0NL3Yt7GxoSNHjjCRJyLatGmTXspFoVBI0utszJgx5U778uVLEXOijXnz5hERkZ+fHyUlJUlqW9/4zwq9qakpu3BERAkJCXrMTRGsra1p5MiRRET04MEDuntXvAk7S/MIHBwcaMiQITRkyBCtfc2aNaMTJ04Img8TExNydHTU2DZkyBAaOHAgXbhwgQ4ePEgnTpwQ9e2mNJiamhLHcRQdHS25bUdHRw1PPj8/n6ZPn05//fWX5HkhIsk86EqVKpW6z9ramszNzdnvLVu2iJ4fFczNzcnFxYWIiG7fvi2ZXV1Qf+OWDPpuiH3TxlgfHx/WyOfl5VVm18ng4GBJGmNtbGxYnn744QdRbTVo0ACXLl3C1atXkZiYWK7BOg8fPsRvv/2G9957Dw4ODoLkw9DQEO7u7mX2qsnOzsbly5dRt25d0Ru71HngwAFwHIfjx49Lardy5coIDg7WKPuoqChJ81CSUVFR2LBhg+h2zp49yxpi1Xu/ERHc3d2hVCpZY+z8+fMl+/9Lly4Fz/O4e/cuTE1Ny3WMsbGx4PmYMWMGpk+fLuQ5K25jbJUqVViIJCcnhwICArQafVTo3r07i1GLjTZt2khih4jo/v37rDG2Ro0aVL16dSIqmn6hfv36ZG9vrzXMunbt2jR58mQaMWIEJSUlCdJA/cknn1CvXr3I19eXMjIyaMqUKdSoUSONNBYWFuTm5kYDBw6kZcuWvbXN8sDU1FSy664OOzs7OnTokEa4Rso2otJgY2NT6j0iJDIzM1mIqFmzZtSkSRO2r0ePHqRQKMjAwIB4nqfu3btT1apViYhoypQpxPM8ZWZm0kcffUTx8eVaIa9cqFGjBk2fPp2IiL799lvKz89/5TH9+/en77//nlq0eOUMwK8FNzc3Cg0NFfSc5YK+vfk38eg/++wz5ikOGDCg1HSdOnXCjRs3mGc5ceJEUb2G+fPnM1stWrSQzFspjfPmzWMeZVJSEsLDwxEZGYnnz5/j5s2bcHFxeeNzGxoawtraGq1atdLwkCwtLdGwYUP06tULfn5+Gp59eHg4LCwsJPnvHTt2ZH301bvbiklHR0cEBgZqdKP09/eHtbW13utCSkoKli1bJrqdkt0r1fvMl3ebkIOoLC0tERkZCZ7nsXPnzjK99IYNG6J///64du0aXr58iQkTJghePidPntSLR/824uxERIFEFElEd4loevF2XyJ6TES3itldSKG3trZGRkYGeJ5HdHS0zjSOjo4YOnQoHj16xETG29tb9DlV7t69C57nce/ePZiZmYlqqzy8du0aEx3VwBRjY2M0a9bsrc/dunVrVraRkZHYs2eP1iAQGxsbrF+/nqVLSkqSTHQ7duwIjuMQGhoqWXmPGDFCI1xT2mAgc3NzWFtbo2XLltizZw92796Nv//+G+3atRMtb8nJyRg6dKjoZdCkSRMcPXr0jYQ+PT0dsbGxgo15MDAwYAMmHz9+rDFY0MjICGZmZqhRowa+/vprXLt2DQUFBeB5Hv7+/hgxYoTgZaNQKBAWFvafE3pHInIr/m5JRDFE1ISKhP4bsTx6Ozs7Jhx79+7V2u/o6IgVK1ZoeJIxMTGie1UDBw5k9v7880/Rb6hXsXnz5igsLGSiU3KE4tvS399fKxZfWFiIoKAgzJ8/H/Pnz8eJEyfYQ5njOCxZsgSGhoaS/P9+/frh6dOn2LNnj2Rlri70EREROqdacHZ2xq1bt7TaT0ob1SwE27Rpg8LCQkEe8OWhiYkJBg4ciH379uHs2bMICAhAcHCwltC/fPkSAQEBCAgIwMCBA2Fvby9oPv744w9WN2fMmIFu3bph0qRJ2L17t8abvmow3/HjxwW/T9TZoUMH8Dz/3xJ6HWJ9mIg6k8hCrz4KVv2p6+HhgQULFiAlJUXjAu7evVuSUZFDhw4Fz/MAgJYtW0pyQ5XFFi1aaAiJ0KNPO3XqhKysrHJNcaCaQVTK/79mzRrRBbQk4+LiwHEc7t27p9HwbGdnh549eyIqKgoJCQka1+Xu3bsIDAwUtaG6S5cuACCZ0OuioaEhjh8/riH048aNE82eyvECoLM+Pnz4EFFRUfDx8UGTJk1gY2Mjehn07t0bPM/D3d1dyPNK1xirUCjqEFFzIrpKRG2JaKpCoRhJRNeJaDYArVYghUIxgYgmvK4tKysr9t3X15fq169PEydOJFtbWzIxMdFIu2vXLpo+fbqk/aiLH1x6h9CNSCVx5swZql27NnXp0oX69etH7u7urEG4JMLCwqhbt26i5qcknj17RkRFI6OlwMiRI6lmzZpERLRx40ZKTEwkY2NjMjAwoE2bNlGvXr000iuVSvruu+/owIEDFBcXJ2reFAoF3blzh9LT00W1UxY4jiN/f3/q2rUrGRgYUGhoKO3YsUM0e6puniqh4ziO8vLyaOfOnbRjxw66fv06vXjxQjT7ZSExMVFym28t9AqFojIR7SeiGQCeKxSKNUS0kIqeNguJaAURjS15HID1RLS++BzlVsdr166x73Xr1qX58+drpXn48CH9/PPPtGHDBslGoKl6WRQUFJSrVV9siC30RERZWVm0d+9e2rt3L9nZ2VHbtm21+u9HRUXRhg0b2JTRUiE1NZUUCgVFRUVJYu/ly5fsIb948WIaNGgQmZubU8OGDTXSJSQk0JEjR+js2bN0+PBhSfIGgKpXr06HDx/WS08kFcLCwggomi46Ly9P1HvzwoULNGLECCIiiouLo4sXL4pm6z+BtwzXGBPRSSKaVcr+OkQULlaMviR37tyJxo0bl7ufrJBULXhy5coVyW2XpIGBAa5cuSJq6ObfzsaNG4PjOMHGC5SHERERpY5hiIyMhLe3N+rUqSN5WahCN6tXr9brNXF2dkZycjIL3Qgdk/+3UxW66d+/v5DnFb0xVkFEfkT0S4ntjmrfZxLRLiGFnohga2uLGzduQIXdu3ejcePGKH4z0AsnTZoEnucxdepUvVcoY2NjDZG5ePGiKIM//s3csGEDOI4TfHbEstigQQN4eXlpifzMmTPh5OSkt7Lo0qULnj59ilq1aun9uoSFhTGhHz58uN7zIyU//vhj8DyPb7/9Vsjzii707YoN3SG1rpREtI2Iwoq3HyE14RdK6GW+ml9//TVSUlKQnp6O1q1b6z0/UrNdu3bgOA6dO3fWe15k/j/r1auH27dvQ6lUYvHixahXr57e8/QfZ7mEXvFvaDx8nRi9DBnlgYGBAf36668UHx9Py5cv13d2ZMgQC6EAXtkgJwu9DBkyZPx3US6hrzCLg8uQIUOGDN2QhV6GDBkyKjhkoZchQ4aMCg5Z6GXIkCGjgkMWehkyZMio4JCFXoYMGTIqOGShlyFDhowKDlnoZciQIaOCQxZ6GTJkyKjgkIVehgwZoqN+/foUHx9PPM/Tli1byMXFRd9ZeqdQIYV+1qxZ/5oFQGTIUMfDhw+J4zhJbF27do2OHDlCP/zwAzVt2lQSm6VhzJgxVKtWLQJAI0aMoClTpug1P1LD2dmZunXrRocOHaLo6GiKiYmh6OhomjVrFnXr1o0aN24sbgbeZj56oUgCz+i2Z88eAPhXTMsqU/9s37497ty5A47j9DoH+ujRo5GTk4PCwkLRbdWtWxcvXrwAz/N48eIFsrOzMX36dEmWzNNF9YXqV69eLdnawURFC4H7+PggKSkJwcHBGDBggGS2XV1dcezYMYSHh5e5QHpsbCyWLVsGZ2fn17Uh7Zqx/yahV0EfFfpd5ueff47Vq1cjKioKAJCbm4uOHTtKYvvgwYPo0KGDzn2TJ0/W+2IX1tbWCAgIQGFhoSRCryqTkovz/PPPPxg3bpxki/PUr18fv/76KwoKCpjQd+3aVbJyr1evHkJCQpjtjIwMZGRk4PLly2yNhhYtWqBFixai2D927JhOYS/td3p6Otq3b/86Nt5NoR88eLCoQm9oaIjly5dj06ZNcHZ2hpmZWalpzczMYGtrCxMTE0kqdaNGjTBz5kycO3cOPM+D4zjwPI99+/ahUaNGotk1NDTEmDFjkJGRobXoRkZGBjp16iTq/x4yZAgKCws1FotX56ZNm/Qu9Hv37mUiL5XQ29nZwcPDAykpKSgsLGRin5+fjxo1akiShyVLlmjUhz///FOy+4Ho/xdsT0xMhJeXF2xtbTFs2DC8fPkSLVu2ROXKlXH37l2kp6cLXiaNGzfGqlWrsGrVKvzyyy9YtWoVvLy8tNINGDAAgYGBGsL/GnbeTaFXhW3EEvqWLVtqeEihoaFYv369Tt64cQM8zyMgIABNmjQRtUJ7eXkhOzubCVrJz4cPH6JKlSqi2J42bRqzk5WVhWXLlqFVq1ZwdnbGoUOHkJWVJepKT2fPnoVSqURAQIDWPmtrazx48AAcx+HChQuSioyKbdu2RWJiouRCr6KVlRXu37/P6uw333wjid3g4GDk5+czkf/jjz8kXeZz+PDh4Hkez58/17j/mjdvjs2bN4OIcPz4cXAch5ycHL0s86jitGnT/t1CT0RxVLSi1C2VUSKyI6LTRHSv+NNWaqGfNWuWKBfE1tYWe/fuRW5ubqlr1+ri9u3bYWFhIXh+zM3N4enpCZ7nkZKSgpUrV6Jdu3Zsv729PR4+fAie57FmzRrB7Xfq1Ik9YH7//Xet/W3btgXHcUhKShJN7MsS+qlTp7IH3q5du0Sx/yqqljVUcdSoUZLYNTIyQqdOnRAcHKxRFwcNGiS6bV9fXybyjx49wsKFC1GpUiVJy33s2LHsvtC1f9y4ccjLywPHcUIv7/fadHV1RXJy8r9e6O1LbPuZiL4t/v4tES2VSugTEhIAAIMHDxb1wrRo0QKzZ8+Gv78/E9Lr16/D399fg4GBgeB5Hunp6XBxcRE8H56enlAqlUhJSUHz5s11plHFKJOTkwW3HxAQAI7j8PTpUzRs2FBrv0roOY4TxZM0NTXF1atXwXEczp49q7V/9+7dLIRVWgxfTH777bfgOI558tnZ2UIvDq1BR0dHDB48GJs2bWIL1quYmpqKH374AVWrVhX1P/v4+Gh48rpCd61atUKfPn3Qp08f0dbTdXFxAQC8ePECH3zwgca+vn37snL56aefJK8XJflf8ehLCn00Fa8VS0SORBQthdB//PHHLGwjttCrs0aNGnB1ddXpsfr6+rLwjdB227dvDwCleiwqhoSEgOd5BAUFCWrf3t4e9+7dA8dxmDRpks406kJ/5swZwcugSZMm7OZYt26d1v7du3czj15qoXd2dkZoaKiG0E+bNk00e3Z2djrfNJOSkrB//34tsRODw4YN02h4/f3337U8+T59+uDp06csTWxsrCgedbNmzVhbVUJCAjw8PDBq1CgsX74cycnJzH6bNm0krRcl2aFDBwBg+QkMDHyd48sl9Eb09gARnSpeDnAdgPVEVA1AEhERgCSFQlG15EEKhWICEU0QwD5D7dq12fdLly4Jeeoy8eTJE3ry5InWdnNzc5o9ezYREd24cUNwu40bNyae52nx4sWvTAuAIiMjBbVfrVo1qlevHr148YLWrl2rM03VqlqXXjScPn261H0cx1FhYaFkeSEi6tatG7m6umpsO3z4sGj2DAwMyMzMTGPbgwcPqEOHDpSUlCSaXXW0bt2aDA0N2e+IiAjKy8sjIiIbGxvq27cvrVixgqytrVkaZ2dn+vHHHwkALV26VLC8VKtWjX2vWbMmHTt2jIiIFAqFysGku3fv0tOnTwWz+bowNzenmTNnEs/zBIByc3Np5cqVwhsSwKOvUfxZlYhuE1EHInpWIk2mFB69Kj6/Z88evT6hVfTx8WEeRefOnQU//4QJE1ivmrLSqTx6oWP0TZs2BcdxyM7O1rl//PjxSE1NFdWjt7S0RFhYGJRKJQYOHKixz9jYmMXvw8LCJL32ffv2xbNnz1BYWKjh0deuXVs0m5UqVcL+/ftRUFDAvPnIyEgsWLCg1LCe0MzKymLXe/369axXmo2NDQ4dOsT2ZWdnIzg4GIcPH8aaNWvYNqF7vtjZ2WHChAkICwtjtnmex6NHj/Dhhx9K3m6gTnNzc/z5558anSYiIyNf9zzS97ohIl8i+ob0FLr5twn9ypUrwfM8Lly4IMr527Vrh7t370KpVOrstkX0/42xHMehS5cugtqvWrUqIiMjwXEcxo0bBxsbG8aJEydqdbdcuXKlKOWgEvOSQu/g4MDCOo8fP9YQkWHDhmHKlCmYPHkyQkJCMHv2bEHz5Ovry/43AGRlZaFPnz6S1LvOnTsjPz9fK4Qzb948Ue0OGjRII2yjGvyjLvJZWVmYOnWqRrikRo0a7Jg5c+aIkreDBw8yG7du3RL1gVtebtiwQasf/aJFi173POILPRFZEJGl2vdLRORBRMtIszH2ZymE/tKlSwCkjc+XxuHDh4PjOBQUFKB79+6i2WnUqBGys7PB8zyGDx/OtteuXRuLFi1ibxRiiayTk5PO/vMqBgUF4cqVK6J59ESEwMBA1rvj999/Z9y4caOG2JbMm/o2X19fQfPk4+PDvHiO43DlyhXJ6+BXX32F6OhoDbEfPny4aN1s586dy8rz5MmTMDMzg7W1NQ4fPswEtm3btlrHiS30o0ePRlxcHCsDsRwvZ2dnTJs2jdXHsLAwTJs2DdOmTcPXX3+N3r17o0+fPpg2bZpWfQTwpg9iSYS+HhWFa24T0V0i8ireXoWIAqioe2UAEdlJIfQq/BuEPjk5GTzP4+LFi6Lb6tevH/MIQkJCEBISgpSUFA1PYcKECaLZHzduHGJiYrSE9OLFi7C1tcXYsWNFFfrp06eXOeqwtG2ZmZnYtWsXvvnmG0EHUpUcBctx3OuOdhSMVlZWmDdvHpsOged5JCYmolevXoLbUhf6DRs2gKhoAKNq2/jx43UeJ6bQGxkZITo6GhzHITU1FXl5ecjNzUXv3r0FtbNkyRI8evRIq46p17/nz5/j+fPnOutjYGAgrKys3sT2uzVgSl89bnTx66+/Rn5+PgoLC0X15tXp5eXFYvHq/elVHr2YQk9E6NWrF3r37g1vb294e3vDxcWFDY5RCX1OTg4aNGgguG0TExM2l0lSUhKSk5ORlJTEHnaqG+uvv/5CrVq1YG9vD3t7e9HmffHw8NAYHKVPoVexXr16+O2335CTk8PmvxE6br948WINobewsMCUKVOQnZ0Nf39/nQ9TBwcHeHl5geM4pKWlCV5PfXx8wHEcLl26BGNjY9YLLiQkRFA76sKdlJSEyMhIvHz58pVTHqj/Pnv2LI4dO/a6D6F3S+jVR8TqczKzWrVq4fHjx+B5HpcuXZLcfvPmzTX6JUvh0b+KKqEXK3xUGtVj9FJ2rzx79qyG0J87d070vuvl5caNG5ln7+3tLei51Rveo6KiWAjj4MGDOtO7uLiw7rmFhYUYN26coPmxtrbGixcvwHEcGyT2/vvvs3EtQg5gVBfuqVOnokOHDsjIyHgtoVc9JJo1a/Y6tiXrXvmvQ2Jiot5sT5gwgRwdHYmIaMGCBZLbv3nzpsbvDRs20Pjx4yXPx78BT58+pVOnTlGXLl0kszl69Gj69NNPNbZt27aNUlNTJctDWVi9ejWNHTuWiIjc3NwEPffChQtp1apVpFAoyMXFhc057+rqSs2aNaMnT57QuHHjqE6dOkRE1LlzZ6pXrx7xPE/r1q2jv/76S9D8fPvtt1SpUiW6du0anTlzhoiIwsPD6ejRo9SjRw+aNGkSrVixQhBbBgYGxPM8ERH9+uuvZe4nIvr9999VTi4R/X+Xz7Vr11JUVJQgeVJHhRP6R48e6c12gwYNaPTo0UREdO7cOTp16pTe8qLCwYMH6csvv9RrHtq0aUNEpNWnXGxwHEfZ2dmkUCjIwEC6pRfUb+ilS5fSxo0bJbNNVNRffd68eRQTE0MKhYLq169P//zzD/Xp04dGjhzJ0l2/fl1Qu7/99hsplUpauHAh2drasu116tShW7du6TwGAN2/f5+mTp0qaF6qV69Oo0ePJoVCQYsWLaLMzEy2LzU1lRQKBbVr104woa9fvz6tXr2aPDw8dO5/+PAh3b17l9atW0cREREUGxsriN1yQ99hG6FDN/rsWnnkyBHwPA8A/5pX9bVr10oSoy+LN27cAMdxgndhLA+lHhl74MABjbDNyJEjJf/PpqamrL2mNF65cgV2dnai2P/ggw/w999/s3Ivyby8PLx8+RIJCQmi1cuGDRuyPvPvvfce296iRQtkZmaisLBQ8LY8e3t7dO3aFevXr0dkZCQiIyMxY8YMdO3aVczZY9+tGP2sWbP0KvSmpqYICgpiUw1IOUtfWVy7di2USqXk8XHA1jw0AAAgAElEQVR1qoS+X79+ktuWWujVB0elpaUJPnahvOzbty+uX7/OGl9VVCqVuH37tuhTRxMVTSj3999/M4Ffu3YtvLy8UK9ePdFtW1hY4Pjx42wO/j59+mDXrl1IS0src6Kz/yDfLaEnKprQTF9C37hxY3YzSdXTpjx0c3MDx3G4fv26XuzXrVuXzSuib6EfNmyY6PbWrVvHhF4Ke69i48aNER4ezua8+fzzz/WeJ6k4YMAA1utMnTdv3tTbA1gEvntCr0/+73//A8/zOHfunF6HVZekm5sb61+vD/tNmzbFs2fP9Cb048aNQ1paGgIDA9GyZUu9Xw+Z0rJfv35sArUrV65g7ty5/5qwqkCUhV5Kzpo1CzzP4/Dhw3rPizpVHr2/v7/e8jBv3jzk5+frRehlyqzglIVeSlatWhWBgYGSLSpRXpqZmSE8PFyvjbEyZcoUjeUSekWx0OoVxVMcy5AhQ4aM10MogBavSiRd52IZMmTIkKEXyEIvQ4YMGRUcstDLkCFDRgWHLPQyZMiQUcEhC70MGTJkVHDIQi9DhgwZFRxvPHulQqFoSES71TbVI6L5RGRDROOJKK14+/cAjr9xDmXIkCFDxlvhjT16ANEAPgTwIRF9REQviOhg8e5Vqn2yyMuQIUMGkZGREW3bto14nqd+/fpJaluo0M3nRPQAQLxA55MhQxTY2NhQaGgo5eTkUMuWLfWdHRnvCExMTGjDhg30xRdfEADat2+ftBkQaAqDTUQ0tfi7LxHFEdGd4u22pRwzgYiuF1Pfw4hlvgP88ssvkZ+fz2YxzM7Olic6kykJf//9d1bv/Pz8cOjQIaHOLc1cN0RkQkTpRFSt+Hc1IjKkoreFn4ho07sw142KRkZGGDlyJDZt2oSMjAy2EElERITgK88TEVxdXZGUlIRNmzZp7RszZgwyMzNFsfsmdHd3h6+vL3x9ffVif+bMmcjMzMS6devYTTdnzhy9l8u7TkdHRzajZI0aNWBsbKz3PAnJRo0asfV0d+zYAVNTUyEXIpFM6PsQ0alS9tUhovB3RegbNGiAxYsXl7qqT05ODpydnQW1+b///Q8cxyE3Nxd16tTR2Ldx40ZwHIctW7botVzc3d0RGBgIdUhlu3r16hg4cCCICAYGBrC2toZCocBff/0FjuOQkZEheXm0aNECLVu2RMuWLWFlZYUmTZq89UI1VlZW+O6775Cenq5R5wBgy5YtaNiwoV7rAFHRqk8nTpzQYlJSEuLj43HixAmkpKQIOle8k5MTPD09sXv3bgBgZfL3339j8eLFcHFxgYmJiWj/2cLCAjt27ADHcUhJSUGNGjWEtiGZ0O8iojFqvx3Vvs8kol1SCX316tXRu3dvbNiwgQmKlIs/zJw5k91g6enp2LlzJ/z8/PD555/j2LFj4HkeGzZsENTml19+yaYhLrmqvUroDxw4IFkZqFOXwKvg7u4uun1HR0dcvXoVeXl5WuJRp04d5tVPnz5dFPsNGzbE8OHDERAQgHXr1uGbb75BQECAxhJ7hw4dQlRUFLy9vd/YjouLC1s68MKFC5g4cSK6deuGLl264L333kNgYCAyMjIwe/Zs2Nra6qUuEBHWrFmjc2nBknz27Nlb2+rcuTPGjBmDjIwMjXNHRETg8OHDOHz4ME6dOgWO47BmzRoUT6woOAcOHMhs79q1Swwb4gs9EZkTUQYRWatt20ZEYVQUoz9CasIvptC///77SExM1Ko0CQkJklTiKlWqIDc3FzzPIy0tDZaWliAiVoGMjY0xfvx4mJubC2pX5dGfOXNGa9/HH3+Mly9fIjo6WnC7pVEVnikPxBb7f/75h9WDkuGrKlWqsH1eXl6C2161ahVSUlK06mNERAQ+/PBDuLq6Mi5fvhwPHz5E69at38hWhw4dwPM8Hjx4oDPsYWRkhBkzZiA7Oxu7d++GjY2NJHVBnTY2NuUSeY7j3mqVuCpVquD3339HQUEBOI6DUqnE+fPnMWfOHDg7O8Pa2pqlNTQ0xE8//YS8vDwYGRkJ/p8rV66MS5cugeM4PH78GK1atRKjbN+d+ej79Omj8eQuLCxEbGwsli1bhvnz50tSkZs1a4b8/HzwPI99+/aVefGXLFmCJk2aCGJXJfTnz5/XuT8iIgIcx+Hjjz8W9f/7+vqW6r2XhsDAQNHy89VXXyE3NxccxyE9PR0fffSRxn6xhH7o0KEIDw/XuTD23bt3S311v3fv3ht7sj179gTP8698c+vcuTP8/Pywc+dOGBoailofSnLs2LGsHO7du4d79+6xkOO+ffvg6+uLjRs3Yvny5XB0dHxjO/Pnz2cCHxAQIMnauKVxwIAB7D/36tVLLDvvhtCPGjUKT548Yd77nTt3NERt2LBhzLsWkx06dEBhYSF4nsfEiRN1phk0aBDu378PnucxfPhwQewuXbqUvbLr2h8ZGQme57Fs2TLR/ntpHryq4VXdc1dPK1aj7GeffYacnBxwHIe0tDR4eHhopTEzM8PFixfBcRy2bdv21jaNjY3h6+uLwsJCcByHvLw8cByHgoICLF26FAMGDMCXX36p81iFQoH79++jsLDwjWz/8ssv4HkeX3zxxSvTGhkZ4cWLFxgxYoRo4YqSHDBgAFtOsnv37rCysoKVlRVq1KiBatWqCWbn+++/B8dxePToEbp16ybJfyuLqnagR48eoW7duuU+btiwYRpvHq9gxRf6SpUq4fHjx+A4Djdu3NBYC7JKlSrw9vZGYWGhJEvYtWzZEgUFBeB5HosWLdLY17FjR6xdu5Y9CDiOw7Rp0wSx+yqPfuHChcybJCKMHj0aBgYGgv73kuJdWkim5ANBjNBN+/btcf78eeZJ/fXXXzrTqXv0HMe9sT2FQoEZM2aw8+Tm5uLo0aMYOnQotm3bhsaNG7/yHKo47pIlS94oD6rQTY8ePcqVfsmSJeB5Hg4ODoKXf0l+9NFHSEpKYt1ZhXqTLUlDQ0M8fPgQSqUS8+bNe61jzczMBA/dWFhYICQkBBzH4dy5c+U6xsnJCRERESgsLEROTk55daviC/3mzZuZ11S7dm0YGBjAzc0NO3bswJ9//sluPqmFPiQkBMbGxmjTpg1++eUXtp3neWRnZ2Pz5s2oV6+eIHZfJfRz587VEPpTp06hc+fOopeHOnWFdcQI21SrVg3x8fHsuu/evRtWVlY606oL/b17997YpouLCztPbGwsBgwYAKKiEE55zyGU0J89e7Zc6R0dHRETEwMHBwc0bdpUtLCCoaEhzp49y8onKysLX3/9tSjdJ3v27AmO43Dnzp3XOq59+/bIyMjA0aNHBc3PsGHDWA8fXW+UJens7Mze9lUhR57ny9O2VrGF3traGsnJyazHSevWrXH48GGtuGheXh4+++wzUSqyOtWF/vHjx5g8eTIyMzOZwBcUFGDfvn2CP3ReV+hv3779xoJSHpYnVi+GyE+aNAnh4eHsumdkZOD9998vNX3z5s0FidGryp/juDeuZ9OnT38roa9atSrCw8ORmJhY6oOtJB0cHLBv3z7WltCuXTvBr8mdO3d0NrampqZiyZIlgrYTqIQ+OzsbzZs3L9cxbdq0QVpaGjiOw4MHDwTLi4mJCW7evAmO4xAfH4/atWuXmb5OnTqIiYkBx3G4cuUKatWqhQcPHoDjOFno+/fvX2qr/e3bt3H69GlwHIeAgADBK7Auuri4IDw8XGf/+aCgIEH7BqvzdYRe1Wgnxk2tYnkgdGx+4sSJePHiBbv+Fy9ehJubW6npjY2NsXfvXuaFv83b1aNHj8r1YCmLV69eBcdxpcbwy8MhQ4aA53nExcWhfv36r0w/dOhQVj9v3boleOiiRYsWrJ2kNM6dO1cweyYmJuxtbteuXahUqZLOdAqFAo0bN8bff/+tkT8hhb5y5crsvKGhoWWmNTQ0ZHUxNzcXbm5uGDx4MDteFvoSQq9UKhEREYGmTZvCwsIC27dvB8dxsLOzE7QCl0YjIyPWvVJFjuNw7NgxUXs4zJ49m70ibtu2Tauf9Ny5czXyFBMTU26v700YGBiosxHW3d1dFKFv2rQpnj59qtE171XTGqgefhz39iNjATBP7E2Ob9WqFXJzcxEVFfVWXWCrVauGPXv2gOd5ZGZm4quvvioz/b59+8DzPE6ePPlaDYXl5c6dO8sUeY4rGkD0tgPF1Pnjjz+yc4eFhcHPzw9+fn74+uuv4e3tDT8/P+zZs4eliYuLw927d8FxHC5duiRYPsor9AYGBvDx8WEPGnd3d1haWuLgwYOsfakc2lGxhd7d3R2XL1/G5cuX4efnh8GDB7N977//vigVqTQ2a9YMV65c0RDUzMxMjBo1SnTbxsbG2LRpE3uwJCQk4PLly5g7dy4iIyNZeKugoADnz5+Hk5OT6HkqjaqQjlChGzs7O2RnZ7Obat++fahcuXKp6U1MTODt7Y0HDx6goKAAN2/exHvvvfdWeXjvvffAcRzy8/Nx6dIl2Nvbv9bx+/fvZw+Kt62rJiYmGDlyJJKTk8HzPO7cuQMfHx80adIEkydPxnfffYeTJ0/i8uXLUCqV4Hm+zDeft6Grqyu8vb3h7e2NRo0awdraGtbW1hg/fjyOHTvGrpmZmZlgNhUKBebNm4fAwMAyHzBBQUH49ttvUb16ddaWN3PmTMHyUV6hV0936NAhWFlZ4cCBA+A4Dk+ePCnv4LaKLfRlccGCBeA4Dl27dhWlEqtz0KBBePLkiVa45ubNm5KNQnRycsKGDRu0KrRK/FXtGFLkpTSqe/RCCf3ixYvZ/9u/f7/WyGB1urq6arThREdHC5KHSpUqsW6aHMfh1q1b6NevX5nD6k1MTGBvb4+1a9dCqVRi3759gvWEMjY2RrVq1ZCeno709HQm6DzPQ6lU4vr165g2bRry8vIQHh4uiSNUkg4ODqIIvYpGRkas111JOjg4aDQGqx60mzdvFsx+eYX+gw8+YOk2bNiAW7dugeOKBle9xmR7767QJycn4+LFi4J3IyxJQ0NDBAcH64zLr169WtKbx8jICHZ2dvjwww8xZ84czJkzB40bN2YevVCTd6l3kXzT44QI3fTq1Yv1WT948GCpgtG8eXNs3LgRcXFx7KbauHGjoHO/ODs7a8WjL1++DCcnJzg5OcHa2pp9HzBgAK5du4br168jLy8P586dK1evjDdlx44d8ccff8DX15fFxHfv3g2e50VtlC+LYgv961Al9FJ79I0aNWLeuzofP378unl5N4W+ffv2yMvLE2teCQ3OmzdPQ9wTEhL0JvSlUTUyViihf1OvPFCtJ44Q/edVDZi62mFMTExQpUoVfPfdd6yxlOOK5lD5448/RJngq23btjh06BDrGqdO9d5AKiYkJGDnzp2S1weFQoELFy6A53lB5/i5dOkShgwZotH+U716ddSsWRM1a9aEpaUljI2NMXPmTAQFBVVooTc1NWXX/OXLl4iJicH8+fMxduxYRlUUoGS9eIOuru+m0B89ehS5ublo0aKFqBXEyMgIFy9eZHPbzJo1C7NmzWJCP3r0aL1WYKKi2TRVXTyFFvryiLWueW+EaohVv0nWrFmDVatWMaqHaAoKCpjAN2jQQPQyb9OmDQ4ePKjRf1ydcXFx8PX11VtbiaenJ6ujbzq3TlnXIzo6Gvv378f+/fs1HnoRERFaZXL69GnJp2JQp5WVFaKiogQXeiKCpaUlZs+eXWZbgarMTp48iV9++QW//PLLmzSMl0vo33jN2H8r7O3tKSkpia5fvy6qHUtLS8rLyyMionPnztHq1avpt99+Y/vDwsJEtV8eJCYmUkZGBllZWZGpqakg5wwKCiJ3d3cKDAwkIqIff/xRZ7pPP/2U3N3dNY47d+4c+fr6CpIPf39/8vDwICKiCRMm6EwTHR1NixYtor///lsQm+XB5cuXqV+/fqRQKOijjz7S2p+SkkKPHj2SLD8lMWDAACIiiomJoTt37gh+/gYNGlCDBg20tjds2JAaNmzIft+8eZNGjRpFHMcJnofywtLSklxcXIiI6PHjx4KeOzs7mzZv3ky2trbk5OREnp6ebN/169fp9OnTFBwcTBcuXKCCggJSKpWC2teCvr15IT16S0tLREREvNWUr6/DdevW6YzPJycnv3GfaqGpCt2oBky9LdUbVcsLsea0Ue+9oc67d+/Cy8sLNWvW1Hv5/9uois8LfY/Y29sjMjKSzWlTGhMTE7Fjx45/xfz4NWvWZPkSY/ZKifjuefSNGzfW8BrExpMnTwgAKRQKti0lJYVWrFhB0dHRkuWjLGRlZQl6vqCgIFIoFOTr60uffvop2+7u7s68+6CgIAoKChLUri706NFDdBsVDceOHaNBgwbRp59+SosWLRLsvOnp6TRq1CjKycmhunXrEhHRmDFjyMLCgu7du0cnT54kIqILFy4IXidllAP69uaF9Oi3bNkCjuMkWdRCxXbt2mHt2rV48OAB/vrrr3IPv5aKqsFBAq5RKfM/zJEjR4LneZw+fVrvedE31T16KebDEonvVmOshYUF62HxGlN8ypT5TrFdu3YICgrCiBEj9J4XfdPa2pr1uvn888/1np83pHBCT0SbiCiV1NZ/JSI7IjpNRPeKP22LtyuIaDUR3aeiVabcpBB6MzMzREREwNfXV/T+8zJlypT5L2G5hF5RLLRlQqFQdCCiHCLyA/B+8bafiegpgP8pFIpvqUjo5ykUiu5ENI2IuhNRayL6FUDrV5z/1ZmQIUOGDBklEQqgxasSGZTnTACCiehpic19iGhr8fetRNRXbbsfinCFiGwUCoVj+fIsQ4YMGTKERrmEvhRUA5BERFT8WbV4e00iUu8onFi8TQMKhWKCQqG4rlAoxO3wLkOGDBnvOMToXqnQsU0rNANgPRGtJ5JDNzJkyJAhJt7Go09RhWSKP1OLtycSkZNaulpE9OQt7MiQIUOGjLfA2wj9ESIaVfx9FBEdVts+UlGEj4koSxXikSFDhgwZ0qNcoRuFQrGTiNyJyF6hUCQSkQ8R/Y+I9igUinFElEBEg4qTH6eiHjf3iegFEY0ROM8yZMiQIeM1UK7ulaJnQo7Ry5AhQ8abQLjulTJkyJAh478LWehlyJAho4JDFnqBYGVlRbNmzaKTJ0/SyZMn6ddff6VevXqRo6M8VuxdRps2bahnz54EgKZNmya5/YkTJ9Ljx48pNjaWnJ2dJbe/aNEimjNnDhkYyFKjV+h7QjOh5rp5HdasWROffPIJPvnkE9SvX1+QczZv3lzn3PSpqakYOXKk3ubCqF27NhYtWsRWs+nfv7/oNj/66CMsXLgQBw4c0CiLR48eYfLkybCxsRHUnmolK/XlCnVByllNiQidOnVCYGAglEollEolxo4dK6n9hg0bIikpCTzPAwDmz58PoqI1W6WYD97NzQ2JiYlQKpVwcHCQ9L+/Q3y3Zq9Up7m5OapUqYK5c+fCz88PERER+OGHHxAUFITIyEi2YHZBQQFWrVoliM0qVapg2bJlCAsLQ1hYGB49eqQhcj179pS0AkyYMAH+/v5ISUmBUqkEx3FQKpUICQkRzaaNjQ1WrVrFFu3Oy8vD8+fPsWrVKvzxxx/Izs6GUqlEbGwsqlSpIqjtV4k8IN4CKLrYpUsXZGZmQqlUIi0tDZ6eniAiGBsbo0GDBvDy8sJff/2FTz75BFWrVhXcfqNGjbTqoFKpRHZ2NnJzc/Hy5Uv88ssvopbB1q1b2UNOFnrR+O4J/YwZM3Dw4EFER0drrMmoztDQUBw4cABTpkxBx44dRbsA1apVw61bt9hNNm/ePFEvuLm5Odzc3LB27VqkpKSw/17yUyyhr1KlisaC3devX8d7772nlW7x4sXgOA7ffvutoPZV3rqvry9jYGCgxgNAzPJX54cffoiMjAwmcm5ubjAwMMCHH36IJUuWsO0q3r9/H2PGjBE0D6tXr9b5hllS+MUS+0qVKiE4OBhKpRIxMTGwtLSUrPzLqiOnT59mbzinTp2Ci4uL5PkYPXo07t+/jzt37misddyjR483Od+7J/SqZfM4jkNsbCwiIiJw48YNzJkzBxMmTICdnR1MTU0lu6B9+/aVROgbNWqE/fv3a3juqs99+/Zp/P7+++8Ft69QKLBz5072IG3YsKHOcjY2NsbWrVtFEfrSqBJ6Kb35bt26MRE/cOAADA0NMW7cOA1xf/r0KYKCghAUFIScnBw8ePBAsMXLly1bhsLCwlcKvUrsxQgt9uzZk/3XLVu2SFb2uli3bl1cunQJhYWFiIyMxLJly1gosU+fPpLkwcLCAl9++SUOHTrE7sWSvHXr1puc+90QemtrayxZsgQ9e/bEqFGjMHToULRp0+Zf4UE0adJEdKGfOXOmlueekpKClStXol27dnBzc9PYL0aMvnnz5uA4DikpKaW+ojds2BAXLlxgD+IvvvhC9PLXhzdPRJg6dSqUSiXi4uLY2sHx8fFQKpU4e/Ys2rZti2bNmrH00dHRUCqV+PvvvwWx/+TJE1bv/vnnH6Snp7PfW7duRevWrREXF8e2TZo0SfAyuHbtGjiOQ2RkJOzt7UtNV7lyZVSqVEm0a2Fubo7AwEBkZmZi9OjRqFSpEipVqgSe5zF79mzR60K9evWwdu1ado3Vna6SzMrKehMbFV/o7ezscPXqVfA8j/3794t+0V6XCxYsEF3o165dq+XBOzk5sf3qnv7evXtFycO0adPAcRwePXqk86b9/PPP2aLRR44cQcuWLVE8SE5UqiBlI+wXX3yBrKwsKJVKfPPNNyAi/Pzzz+A4DosXL4atra3WMY0aNWJtKULYV/fm33//fXh6eoLneWzZsgXm5uYgIvj4+LA0kZGRgpZB9+7dwXEccnJyymyA7t69O8LDw7F//35UrlxZlOtx/PhxcByHTz75hG0bMWIEeJ7HBx98IFo9qF27NjZu3IgXL15oCXpCQgI8PT1hZmYGZ2dntn3JkiVvYqtiC72NjY2Gh2hlZSXaRXtTLl26lN1MM2bMEMVG+/bt4e/vD39/f0yYMEGjou3fv18jNi90A6iK9vb27DpMnDhRY5+LiwsSExORl5eHFStWSBY6U0HKkA0R4dixYyzu7uDggJEjR7LG6XXr1pV63OPHjwUR+l9//ZXVucTERNSuXRvdunVDcHAwe7sgIpiYmODo0aPgeR5paWmC/X9HR0eEhoaC4zgcOHCgzLTffPMNc1A6d+4s+LVwdXUFz/NadXLNmjVIT08XvPeXig4ODsjNzdUS+LS0NPzxxx8abVfz5s1j+xcuXPgm9iq20KsWveY4DidPnhTlgr0pq1WrhgEDBiA/P5/ddGZmZpLm4dy5cxqevlgiT1QUo9+2bRs4jkN+fj5q1aoFIkLTpk2RkJCAnJwcbN26VbL/rh6ykdKbHz16NPPgVN58+/btcf78+Vcu0C6E0Ds7OyMrK4vVuc8++wxEBCsrK1hYWGil3759u+BCP2LECCiVSuTm5qJNmzalpuvZsydiYmLAcRzS0tLQtGlTwa/HggULwHGcxkNk8uTJePnyJU6cOCFaPfjss8+Qn5+PlJQUbN++Hdu3b8eYMWO0Gn779OmDgoICKJVKPHjw4E0bht8dod+zZ49oF+1V7NKlC1atWoWrV68yxsXFITs7W6PRq3nz5qLnxd7eHjNnztTZ66asOKkQtLW1xaZNm1iD7KhRo3DmzBlwHIeNGzdKdj1K62YZGBgoqujb2tri9u3bUCqV8Pf311i32NjYGObm5jAyMir1eCGEfsaMGay++fv7vzL23aFDB/A8j4KCAgwePPity8DZ2RkxMTEs3ty4cWMMGjQI33zzDeOuXbuwevVq5OXlMUekrDedt6FK6Hv16gUigpOTE8LDw8FxnKhCT1R0L5b1xmBgYIBr164xb/4t6ua7I/Qcx2H//v2ShW9MTEzQvXt3ZGZmlrt3w4sXL7B06VLRYpH9+/fHw4cPtXrdqPefF9OrJyJYWlri3LlzGtdlz549ksTjVXwVAgMDBbepUCjw1VdfgeM4REVFsTea8nLixImCiI+60Pv5+b0yfZ06dVj66dOnv3U5LFq0iF13AFpdm3VtS0pKEq0uODk5IT09HYmJifDx8cHRo0dx584dJCYmii70r+JPP/3ERP7y5ctvM86gYgs9EWHcuHEsPAIA27Zt09nYJSQrVarEXnnVmZ2djfPnz2v0ZtDF+/fvv7YQvIr9+/fX8uDj4uIQEhKC1NRUtr1r166iV2A7Ozs8f/6c3cgjRoyQ9AZS70evTjHFfuTIkeym9fDweO3jo6Oj8fz587cO7+lb6Ldt21Zmz5LExETcuHEDXl5e2Lt3L5RKJZ48eSJqffjhhx/Yf7x8+TLc3Nxw8uRJvYd7Dx06BKVSiRcvXqBdu3Zvc66KL/RERcOsw8LCmMBduHBBtEYWIs3eCjzP49ixY2jbti1rYGnQoIGGFz9o0CCtqQASExOxYMECfPzxx2+dH3t7ey1PfsGCBcx779q1K9u+Zs0aUSuvnZ0dAgMDkZOTwzx7pVIJLy8vvd5UKqoLvlCNtKampggLC4NSqURoaChq1KhR7mPNzc3Rq1cvxMfHw9vb+63zEhIS8lpC/8MPPwCAYEJvZ2eH6dOnw9/fH507d9ais7MzS3vz5k1JhN7AwACtW7fGF198AXNzc1SuXBmxsbF69ejnz5/PRokPGTLkbc/3bgi9iuohg7CwMDRq1EiUixQaGspupvXr12vEYs3MzHDs2DG2f9u2bSAiGBoawsfHBy9evNAQ/MzMzLfOT6NGjVh7wN27d7X+t4eHB/Pog4KCRK3As2fPRl5eHgYOHAiiovBaYWEhcnJyJBuYUhbFEPo5c+Ywb7VmzZrlPs7CwgLbt29Henq6YD1O1NuFXiX0JiYmOHLkCHieR1ZWlmCDtcpLqYS+JKtVqyZJjL40dujQAUBRCGvZsmVCnFMYoSeiTVS0Hmy42rZlRBRFRHeI6Lg4KiIAABo7SURBVCAR2RRvr0NEL4noVjHXiiH01tbWWtuMjIzwxx9/MLFXiazQTExMZDfTmTNnYGNjg9q1a6Njx444dOgQCyMFBgZqNb4NHDgQAQEBCAgIwMyZM+Hq6ipInvr164fhw4frfPVX730jhNdWGu3t7RESEoIVK1ZobFf1If/5559Fs11eqhpqhQzdqN5afvzxx3Klr1SpEtq1a4ft27dj7969gvblLq/QGxsbY/HixSztn3/+Kfm1eBeFvmXLlrh9+zY4jsPhw4eFaq8TTOg7EJEbaQp9FyIyKv6+lIiWqgl9eHkMv43QHz16FJ06ddLabmRkpOHZizGRWKNGjZCQkMBukocPHyI5OVnDU9+xY8dreXdi0cvLSyNmL+aMhT179sSjR4+0+snPnj2bDTfXZ1mI1eWS4zikpqa+cmKybt26YdCgQbh69SqUSiVmzZol+OjtrVu3sjq4b98+1KhRQ8vZcHV1xcGDB0WfAuFVvHXrluiNsbpYrVo18DwvehizJG1tbdncP8+ePRPMySMhQzdUhoATUT8i2iGl0AcHB+PWrVtajYtGRkYIDg5mlVis0bL169eHn5+fRp/lvLw8xMbGonPnzpL2MimN27ZtQ3Z2tiT96IkIPXr0QG5uLvr166ex/fr16+A4Drt27dJbWaiHbIRuiFXN0tm7d2+d+4cNG8YGEKl6WAwdOlSUKTomTZrEwnQq7t69G7NmzULLli2xZs0aDSclLS1Nb1Noq3qdvCse/aZNm1ijtMDdSSUT+n+IyFMtXS4R3SSic0TUvoxzTiCi68V8rT83ZcoUFBQUICsrC35+fpg7dy7OnTuHkJAQja5bw4cPF/Xivf/+++jZsyd69uz5ti3nr01zc3MsWrQIixYtYiEbXf3o4+LiJMmbsbExjhw5gqNHj7IHXaNGjdh86PoK3Ygp8kT/H7q5e/cuxowZwxgcHIzo6GgkJyez0Jmvr6/oczBt3LhRS+xLUjVAR+gZM1+H3t7eehH6ypUr48GDB5IKvWruIxUFPr/4Qk9EXlQUo1ctMm5KRFWKv39ERI+IyEpoj56IMH36dDx+/JiJukrYVFy/fr3eKrEUdHNzYwISHh6OkJAQrd43e/fulWSBCRVbt27N4o/29vaszSQ5ORlubm6S5MHd3Z0Junq4RqwBU56enjonqFIxNDQUq1atQrVq1SQbHV1aF9+CggI8ePAAo0eP1mvdJSLs2LEDSqUS+fn52L17t6S2T5w4IZnQ165dm61LEB8fjzlz5ghtQ1yhJ6JRRHSZiMzLOC6IiFqIIfRERU/nIUOG4O7duxpTEjs6OpY5CrGi0N/fHwC0Zq/U1ftGClauXBnHjh1DYWEhXr58iYKCglcO/ReaJfvMqyCmzZJiHx0djYULF2LhwoWij+sojdOmTYO3tzeSkpIQHR0Nb29vje6N+qabmxsCAgJEHRlbGn18fPD48WPRB1jWqFEDkZGRrF6INIJfPKEnIg8iiiAihxLpHIjIsPh7PSJ6TER2Ygn9u84qVapgzZo1bMWsvXv3onnz5pLPq1OSn3/+OZ48eQKO43Dnzh3R2wfU6e7uriHwYoRrZP63qZq9cty4caJOkdyjRw8m8jdu3ICjo6MYdgTrdbOTiJKIqJCIEoloHBHdp6KwjEY3SiIaQER3ieg2Ed0gol7lfJDo/eLLlCnz3aCbmxt4nkdycjKqV68uig0bGxs8efKEhafEmJ2zmO/WgCmZMmXKLC/Xrl0r1IAlnTQwMICvry+UStEXhS+X0KsaUfWK4l4aMmTIkCHj9RAKoMWrEhlIkRMZMmTIkKE/yEIvQ4YMGRUcstDLkCFDRgWHLPQyZMiQUcEhC70MGTJkVHDIQi9DhgwZFRyy0MuQIUNGBYcs9DJkyJBRwVHhhN7KyoomTZpE2dnZdPnyZX1nR4YMGTL0DiN9Z0BIKBQKmjNnDnl5eVFUVBSNGDFC8jxMmTKFVq9eTYaGhqLasbKyIp7nycLCgjZt2kQKhYLGjBlDRESjR4+muLg4IiJq0aIFpaWlERHR8uXLied5UfMlQxtmZmZUpUoV9nvYsGHk7e1NK1asoAULFghiw9fXl3x9fctMExgYSO7u7kRUdK/oEzNnzqS+fftS+/btCQBVq1aN0tPTRbVpY2NDEydOJHt7e7YtPj6e/Pz86Pnz56La1jv0Pc+NUHPdKBQKTJo0CTzPIyYmBubm5pLPn9GjRw88f/4cSqUSt2/fRv369UWz5efnh/DwcDbf+7Nnz/D9999rzMlfco5+MRea6NmzJ2bOnAkiwvjx4xEbG8vsAkVTKcfGxsLFxUXy66IvmpiYoH///rh48SJbI0B9OuOcnBz0799fkryUnNUTEHZJxfJywoQJ8Pf311g3QalUYuXKlaLZNDIyQqdOnUpdv2Lr1q16rytvwXdrUrNu3bqx1XO++OILyQt86NChOH/+PLuJOY5DixYtRLPXu3dvtsCKi4sLOnTogMqVK6Nz585aHD9+PE6cOCHavN9ubm7Izs6GUqlEQkIC8vPzNQRNXeBiYmIwaNAg0crFzc0NR48eBc/zuHDhArp06SJ5XVBx/fr1pZaD6ndkZCTs7OxEy4P64islIcUUzubm5vD09MT+/fu11k1Q/0xNTYWTk5MoeWjVqpWGw3P27Fl8//332LBhgyz0/yWhd3BwYOu37t27V/LCHjRoEAICArRu4qCgINFsDhs2DImJifDw8NBrRbOzs8ORI0d0illubi7Onz+P2NhYrZWXlixZAmNjY0Hz0rhxY6SkpGjc1Hl5efjoo48kL5erV6/qFHZdv1u3bi1KHkpbhEVKsd+/f7/Gf1X/XLBgAZo3b84o1joKy5YtY/VhwIABMDQ0BFHRQukcx2HevHmS1w8B+W4IvYGBARYvXsxCNhYWFpIXtrrQSSX0fn5+WLhwITp06KDXilbyv6tu5IiICHTv3h1EBBcXFwwcOBCzZ89moS3VjW5iYiJYXoYPH85u6F27dmH06NHw9PQEUdGatmJ5jOo0MTHB+vXrNUJWSUlJiI+PZ79LhrTEEBpdoZrAwED4+vpqeflilUVISAjz2HNychASEoIJEyZIuhANEWHIkCGszJcvX87CukZGRqhatSoT/v8o3w2hV4Vsnj17BldXV70U9tatWyUX+jlz5qBfv35o1qyZXitafHy8TqGfO3euzvTDhw/XSPfTTz8Jlpfg4GBwHIczZ87A1NRUY1+DBg2QkZEh+htQ//79NcoiICCAhWYSExOhVCqRkpKCCxcusDKIi4sTPHyjC6p9JR8CYpRDo0aNkJ2drZe1i3Vx7dq1bO3cwYMHl+sYBwcHvea5nBRshalNRJRKmksJ+lLRMoGqFaa6q+37jopWoIomoq5iCr2ZmRkePHiAwsJCfP3113orbCcnJ0mF3tzcHKdPn8bjx4+1hL5u3bpsebQmTZqIXlkTEhJ0eqllhSP69OnD0l24cEGwvKiEXldcvkGDBuA4DufOnROtLNq2bYvU1FRWB1atWqUh4E2bNkVoaChmzJgBov9fIFupVKJmzZqC5UMXfH192X4phD4iIoLF4BcuXChqHSwPXVxckJWVBY7jsHnzZp1pHB0d4erqioULF8LPz08vYeA3oGBC34GI3Ehb6L/RkbYJFS0jaEpEdYnoARWvISu00BsYGODnn38Gz/PYv39/uY6pWrUqatSoIXhhlyb0eXl5GDhwoCgX2NfXFyEhIejVqxcGDRqEU6dOYfv27bh+/TouXryIkydPIi8vD1FRUTh58iRWrVoleI+X8ePHl9rw2qpVqzKP/eKLL5CdnY2IiAjBRG7z5s3gOE5nbycPDw9wHIdNmzaJcj2Iiho+VeVw7do1VK5cWSuN+jb19EKVga64fMk4fMk0QpeD1/+1d8YxUV35Hv8eoBQjojuCYHB18cWm0oRau31xsw/Tbl/aQkzVvpo2lSrJivXJI6+NprUhPiFhSdHQ2mbNLO/FRmvKatUntTYvaPuUhsZXYFtECLjSxRZdwDKgFDAW7/3uH3Pn7sxwZxge986dmZxP8svcOXNm5ve75853zvzO794pLdWPgxMnTtj+H8Yeq6io0CcknkKJ+fPn86mnnmJpaSl7enr0xxsbG/nwww/b7nMIZumfg5fBWOjfBPCm1/16AL+yQugXL15MVVV58+bNoKWUs2bN4t69e3np0iW6XC7evn2bubm5pu7sxMTESekbz4G+Y8cOSwY4Pz+f9fX1dLlcvHHjhk+52OXLl3ns2DGWlJTw9OnTevuZM2e4adMmZmVlmeLDa6+95rPw2tvbq8f94YcfTvl8T9rHrFl2Xl6eodAnJSXx888/p6Ioegmo2VZXV6ePwddff20o8t6WnJzsk782679Ljaps/MsorRT6Rx991KeaxuzP2kzMW+gvXrzI2tpafvvttz4ll4ODgywvL7f0T8NNNsuF/hqANrhTOz/T2n8PoMCr30EAzwd4za0AWjSbdoDvvPMOJyYmuH379oB9MjMzefbsWaqqyt7eXtbW1lJVVZ46dcr0HV5ZWWko9C6Xy5IBzs7O5p07d/SDtKysjAcOHOBjjz3G2bNnU/t7Rn3B6fXXX+ePP/5IVVX55ZdfmuLDN998o8fb2trKnJwculwufUY71UzOI/Q9PT2m+OMR+rq6Op8PallZmb6f3nvvPdPHwuFwsK+vj4qi8NatW1y/fv2Uz/nss89MryE3ms0b1cpbKfQFBQU+x39PT49ti7DelpaWxq6uroDnmCiKwjVr1lha6mqRWSr06QDi4b6Ewu8AvK+1H8Bkof8Xs2f0mZmZHB0d5ZUrV4L2e/vtt6mqKg8fPsy0tDQ+88wzluUMKysrDXPVQ0NDlgxwdnY2R0ZGuHnz5mk9r7y8nKqqcu/evUxISJiRD97lgp4Ulfcsf6r6ZI/QDw8Pm1I9lJiYyLa2NiqKoi/y7ty5U18UVBTFksXY0tJSXdymSg05HA6WlpZyfHyciqKwtbV1ytl/qGYk9P59jKpxzNwXBQUFttXLG1laWppe/eNt/f39bGtr486dO/XqsCg164Q+0GMIQ+pGCMGDBw9yYmKCGzduDNgvNzeXP/30E+vr6xkfH8+UlBQ2NDRwaGiI8+bNM32HB5rR371717KF4uXLl0/7OQ6Hg52dnVQUhU8++eSM3t/zZaYois9ahGeB9rvvvgv6fE+/W7dumVYmmpSUxO7ubt69e5cjIyOTZm5mC/3KlSs5Pj7Oe/fu8fbt21NWfq1atcrnGHn33XdN88Vf6I1q5P37eC/SmmGzZs3ixo0b6XQ66XQ62dHRMal+vr29nQ8++KCp7+tvycnJrK6u5tjY2KRj4JNPPrH0vcNsls7oF3ptvwbgqLb9EHwXY/8Ckxdjk5KSqKpq0JnywoULeePGDV67do1paWlMSUlhVVUVVVW1bJD37NnDiYkJw5NjKioq7D4YfMxzoshMzwg0mtEDYHp6Ot966y0+/fTTAZ+7bNkyvULFrNSNt33wwQccGRnh6OgoT548yZaWFpI0Xeh37dqlC8iRI0eC9t22bZuP4EzVf7oWioj75/DDdRmE5557jj09PbroNjQ0WDazf+ONN9jS0qLv566uLr2cVVGUkMsro8RMq7r5I4A+ABMArgP4LYAjAC7DnaM/DV/hL4W72uYKgLwQv0hCDmzDhg1UVZXV1dUB+3z00UdUVZXbtm3zEfmGhgampqZattOHhoaiSui/+OKLGeUkvWOczpfGkiVLuH//fj23v2LFCkvjnTNnjn5dIDOFPjExkefOndP3QbCFXqfT6bO/ent7Ta+C8hd6fxG3Om0zlc2fP99nhm9FCjU3N5cul4uKovDq1assKSlhamqqfswrisLCwsKwxm2xxd4JU3Fxcezq6uKdO3e4aNEiwz4PPPAAJyYm2NraynXr1rGpqUkX+Tlz5li60wMJ/dGjRyNqFd9z0BcWFs7oTGLPwuu9e/f48ccfh/Sc+Ph41tTU6Ptn69atlsebkZFhSepm0aJFPuk6I6F3OBw8ceKEnpP39LXisgxTzej9MUrtWG2eE6lUVaXT6TT99UdHR6ko7gvoLV26VG/fsmWLfgwsWLAg7HFbaLEn9AkJCVRVlSMjIwH7LF++nKqq6mmUoaEh7tixw9RT7QNZRUVFwOuaVFZWmv5+S5Ys4e7du/nCCy+EfGJUeno6Ozo6qKoqS0pKZvT+3guvoaZfiouL9f3T0tJiWllhMLNK6OfOncvOzk59H3gvqhYVFbGuro59fX2Tjok9e/YwLi7O9Dj9Z+weIX/88ccNyy7Nzs+HYqmpqXq9en9/v+mv71l4ramp0X+9p6en89ChQ1RVlePj49G++OpvsSv0VVVVho/Pnj2bFRUVPqvrZp55OZW99NJLAYU+1BnvdC0rK4vDw8McHBxkTk6Obt4VNcnJyczJyeGWLVvY0dHB5uZm1tXVce3atTN679WrV+tnG7pcLj7yyCMB+xYVFXH//v0+VUlWnLxmZFYJPeBOE3pe+8KFC6yurtYvh+u/CDg8PMzdu3dbGqtReiYQVubna2pqfNZoFi9ezIKCAp8zZq2Y0Tc1NVFRFL1kNS0tjYcOHdLHYN++fWE55sJosSv0RUVFkx7Lz89nf3+/ft2bU6dOsby8nHPnzg3bTk9ISGBjY2NYhT47O5ttbW0kfS+Ydf78eZ49e5b19fU+C1MkWVdXZ9r7e1/rxuVyccOGDZPM6NLF27dvt2RWa2RWCv2mTZsCfrl7329qagrbtfiNZu/hFHnvM2Obm5vZ3NzMgYEBn30yMDAQtGru/2uez8EPP/zA+vp6dnV16W3Hjx83rZQ1giz2hD4uLo7Hjx9ncXExAfdi2Msvv6wL/MjICMvLy/nQQw/ZtuMbGhrCKvSA++fwE088wcHBQV6/fj3oSSGqqnJsbIyrVq0yNd5AYuffNjY2FvCCZ1aZlULvXT1kJPSNjY1ct26d6ZdknsqCib3VKZvm5mZdXI3q6Ts6OiwrrywpKWF3d7fPMT8+Ps59+/bFosgTsSj00WBGQv/pp58yLy/P0veNi4ujw+FgUVERz5w5o3+o+vv7WVxczPb2dhYWFnLNmjVcsGCBacKTmZnJixcvTin033//PS9cuGBLftRKoQfc6QHPpSY8MTudTj777LO2LsL7X5L4/PnzYSmnXL9+PQcGBtjf38+BgQG9pt7pdFp63XmPpaSksLi4mFVVVayqqoq1nLy/SaGXFh7LyMhgWVmZodCfPHmSr776qi3/E+CxuLg41tbWWib00qTZaCEJvdCE1la067JIJJaRkZGB2tpavPLKK7h69ard7kgkZvEnkr+cqpMUeolEIoleQhL6uHB4IpFIJBL7kEIvkUgkMY4UeolEIolxpNBLJBJJjCOFXiKRSGIcKfQSiUQS4yTY7YDGIIAx7TbaSYWMI5KQcUQWMg5zWRJKp4ioowcAIURLKPWgkY6MI7KQcUQWMg57kKkbiUQiiXGk0EskEkmME0lC/592O2ASMo7IQsYRWcg4bCBicvQSiUQisYZImtFLJBKJxAKk0EskEkmMY7vQCyGeEUJcEUJ0CyF22e3PdBBCXBNCXBZCtAohWrQ2hxDinBDiqnb7M7v99EcI8b4Q4qYQot2rzdBv4eY9bXzahBAr7fPclwBxlAkhbmhj0iqEyPd67E0tjitCiKft8XoyQoifCyHOCyE6hRAdQoh/19qjakyCxBFVYyKESBJCNAkhLmlxlGvtWUKIr7TxOCaESNTa79fud2uP/8JO/w2x+Z+l4gF8C2ApgEQAlwBk2/2PV9Pw/xqAVL+2vQB2adu7AFTZ7aeB36sBrATQPpXfAPIB/A8AAWAVgK/s9n+KOMoA7DTom60dX/cDyNKOu3i7Y9B8WwhgpbY9B8CfNX+jakyCxBFVY6Lt12Rt+z4AX2n7+SMAL2rtfwDwr9r2dgB/0LZfBHDM7hj8ze4Z/T8C6Cb5F5I/ATgKYK3NPs2UtQAOa9uHAayz0RdDSH4BYMivOZDfawF8QDf/B2CeEGJheDwNToA4ArEWwFGSd0n2AOiG+/izHZJ9JL/Wtn8E0AkgE1E2JkHiCEREjom2X0e1u/dpRgC/AXBCa/cfD884nQDwpBBChMndkLBb6DMB9Hrdv47gB0akQQBnhRB/EkJs1drSSfYB7gMfwALbvJsegfyOxjH6Ny2l8b5X6iwq4tB+9j8C9ywyasfELw4gysZECBEvhGgFcBPAObh/bdwieU/r4u2rHof2+G0A88PrcXDsFnqjb71oqvf8NcmVAPIAFAshVtvtkAVE2xg5AfwDgBUA+gBUa+0RH4cQIhnASQCvkhwJ1tWgLWJiMYgj6saEpEJyBYBFcP/KWG7UTbuN2Dg82C301wH83Ov+IgB/tcmXaUPyr9rtTQCn4D4gBjw/o7Xbm/Z5OC0C+R1VY0RyQPuQqgD+C39PBUR0HEKI++AWxw9J/rfWHHVjYhRHtI4JAJC8BeAC3Dn6eUIIz4UgvX3V49Aen4vQU4phwW6hbwawTFvNToR7IeO0zT6FhBBithBijmcbwFMA2uH2f7PWbTOAj+3xcNoE8vs0gE1apccqALc96YRIxC9XvR7uMQHccbyoVUhkAVgGoCnc/hmh5XMPAugk+bbXQ1E1JoHiiLYxEUKkCSHmaduzAPwz3OsN5wE8r3XzHw/POD0P4H+prcxGDHavBsNdQfBnuHNgpXb7Mw2/l8JdMXAJQIfHd7hzc58DuKrdOuz21cD3P8L9E3oC7tnIbwP5DffP0gPa+FwG8Eu7/Z8ijiOan21wfwAXevUv1eK4AiDPbv+9/PonuH/qtwFo1Sw/2sYkSBxRNSYAcgB8o/nbDuA/tPalcH8RdQM4DuB+rT1Ju9+tPb7U7hj8TV4CQSKRSGIcu1M3EolEIrEYKfQSiUQS40ihl0gkkhhHCr1EIpHEOFLoJRKJJMaRQi+RSCQxjhR6iUQiiXH+BsuIsOfwC4SQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision.datasets import MNIST\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "plt.ion()\n",
    "\n",
    "img_transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.5,), (0.5,))\n",
    "])\n",
    "\n",
    "train_dataset = MNIST(root='./data/MNIST', download=True, train=True, transform=img_transform)\n",
    "train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "test_dataset = MNIST(root='./data/MNIST', download=True, train=False, transform=img_transform)\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "# show a few MNIST digits in the training set\n",
    "images, _ = iter(test_dataloader).next()\n",
    "plt.imshow(np.transpose(torchvision.utils.make_grid((images[1:50]+1)*0.5,10,5).numpy(), (1, 2, 0)))\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MLP Definition\n",
    "-----------------------\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of parameters: 669706\n"
     ]
    }
   ],
   "source": [
    "class MultilayerPerceptron(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MultilayerPerceptron, self).__init__()\n",
    "        \n",
    "        self.fc1 = nn.Linear(in_features=28*28, out_features=512) # MNIST resolution is 28 x 28\n",
    "        self.fc2 = nn.Linear(in_features=512, out_features=512)\n",
    "        self.fc3 = nn.Linear(in_features=512, out_features=10) # 10 outputs: probability for each digit class\n",
    "            \n",
    "    def forward(self, x):\n",
    "        x = x.view(x.size(0), -1) # flatten MNIST image to a vector\n",
    "\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        \n",
    "        # Softmax transforms the 10 scores for each class to probabilities that sum up to 1.\n",
    "        # The cross-entropy between these class probabilities and the ground truth class probabilities\n",
    "        # will be used as a loss when comparing to the ground truth.\n",
    "        # Cross-entropy computes a logarithm over the class probabilities, and it is numerically more stable\n",
    "        # to combine the softmax computation with the logarithm, which is done below:\n",
    "        x = F.log_softmax(self.fc3(x), dim=1)\n",
    "\n",
    "        return x\n",
    "\n",
    "mlp = MultilayerPerceptron()\n",
    "\n",
    "device = torch.device(\"cuda:0\" if use_gpu and torch.cuda.is_available() else \"cpu\")\n",
    "mlp = mlp.to(device)\n",
    "\n",
    "num_params = sum(p.numel() for p in mlp.parameters() if p.requires_grad)\n",
    "print('Number of parameters: %d' % num_params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train MLP\n",
    "--------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training ...\n",
      "Epoch [1 / 20] average reconstruction error: 0.313913\n",
      "Epoch [2 / 20] average reconstruction error: 0.135210\n",
      "Epoch [3 / 20] average reconstruction error: 0.101179\n",
      "Epoch [4 / 20] average reconstruction error: 0.079853\n",
      "Epoch [5 / 20] average reconstruction error: 0.065790\n",
      "Epoch [6 / 20] average reconstruction error: 0.058902\n",
      "Epoch [7 / 20] average reconstruction error: 0.048009\n",
      "Epoch [8 / 20] average reconstruction error: 0.043329\n",
      "Epoch [9 / 20] average reconstruction error: 0.040112\n",
      "Epoch [10 / 20] average reconstruction error: 0.033950\n",
      "Epoch [11 / 20] average reconstruction error: 0.031638\n",
      "Epoch [12 / 20] average reconstruction error: 0.027545\n",
      "Epoch [13 / 20] average reconstruction error: 0.030184\n",
      "Epoch [14 / 20] average reconstruction error: 0.024364\n",
      "Epoch [15 / 20] average reconstruction error: 0.027150\n",
      "Epoch [16 / 20] average reconstruction error: 0.020547\n",
      "Epoch [17 / 20] average reconstruction error: 0.020322\n",
      "Epoch [18 / 20] average reconstruction error: 0.023620\n",
      "Epoch [19 / 20] average reconstruction error: 0.018684\n",
      "Epoch [20 / 20] average reconstruction error: 0.020664\n"
     ]
    }
   ],
   "source": [
    "optimizer = torch.optim.Adam(params=mlp.parameters(), lr=learning_rate)\n",
    "\n",
    "# set to training mode\n",
    "mlp.train()\n",
    "\n",
    "train_loss_avg = []\n",
    "\n",
    "print('Training ...')\n",
    "for epoch in range(num_epochs):\n",
    "    train_loss_avg.append(0)\n",
    "    num_batches = 0\n",
    "    \n",
    "    for image_batch, label_batch in train_dataloader:\n",
    "        \n",
    "        image_batch = image_batch.to(device)\n",
    "        label_batch = label_batch.to(device)\n",
    "        \n",
    "        # class predictions\n",
    "        prediction_batch = mlp(image_batch)\n",
    "        \n",
    "        # The cross-entropy loss.\n",
    "        # The first input are the predicted log class probabilities.\n",
    "        # The ground truth probabilites for each image are expected to be\n",
    "        # 1 for a single class and 0 for all other classes.\n",
    "        # This function expects as second input the index of the class with probability 1.\n",
    "        # (this function is not called cross-entropy, since this function assumes\n",
    "        # that the inputs are log probabilities, not probabilities).\n",
    "        loss = F.nll_loss(prediction_batch, label_batch)\n",
    "        \n",
    "        # backpropagation\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        \n",
    "        # one step of the optmizer (using the gradients from backpropagation)\n",
    "        optimizer.step()\n",
    "        \n",
    "        train_loss_avg[-1] += loss.item()\n",
    "        num_batches += 1\n",
    "        \n",
    "    train_loss_avg[-1] /= num_batches\n",
    "    print('Epoch [%d / %d] average reconstruction error: %f' % (epoch+1, num_epochs, train_loss_avg[-1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot Training Curve\n",
    "--------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xl8XXWd//HXJ/vWpLlpuia3Gy07ZQkFiiAIQhUHFJBFfIiIww8VlXFc+P3AjRlnEEfFBdGqOIwbCCpWB0VEFrWULtBCC3SlS7qmSZtmXz+/P+5JuQ1J7mmbm5vkvp+Px33k3LPdT0+TvPM953u+x9wdERGRgWSkugARERn+FBYiIpKQwkJERBJSWIiISEIKCxERSUhhISIiCSksREQkIYWFiIgkpLAQEZGEslJdwGAZN26cT5s2LdVliIiMKMuXL9/j7uWJ1hs1YTFt2jSWLVuW6jJEREYUM9scZj2dhhIRkYQUFiIikpDCQkREElJYiIhIQgoLERFJSGEhIiIJKSxERCShtA+L+pYOvvWXdazcui/VpYiIDFuj5qa8w2UG3/zLWvKyM5hTOTbV5YiIDEtp37IozstmbEE2m+uaU12KiMiwlfZhATA1UsBWhYWISL8UFkBlpIAtCgsRkX4pLIBopIBte1vo7OpOdSkiIsOSwgKYWlZAZ7ezo7411aWIiAxLSQ0LM5tvZmvMbL2Z3dbH8pvN7GUzW2Fmfzez4+KW/d9guzVmdnEy66yMFADoVJSISD+SFhZmlgncC7wDOA64Nj4MAr9w9xPd/WTgbuAbwbbHAdcAxwPzge8F+0uKqMJCRGRAyWxZzAXWu/tGd28HHgQui1/B3ffHvS0EPJi+DHjQ3dvc/XVgfbC/pJhUkk92piksRET6kcyb8qYAW+PeVwNn9F7JzD4GfArIAd4Wt+3iXttOSU6ZkJlhVJQWsKVWYSEi0pdktiysj3n+phnu97r7TOBzwB2Hsq2Z3WRmy8xsWU1NzREVq+6zIiL9S2ZYVAOVce8rgO0DrP8g8O5D2dbdF7h7lbtXlZcnfN74gKKRfIWFiEg/khkWS4FZZjbdzHKIXbBeGL+Cmc2Ke3sJsC6YXghcY2a5ZjYdmAUsSWKtTI0UUt/SQX1zRzI/RkRkREraNQt37zSzW4DHgUzgfndfbWZ3AsvcfSFwi5ldCHQAe4Hrg21Xm9mvgFeATuBj7t6VrFrh4O6zJxaUJPOjRERGnKSOOuvujwGP9Zr3hbjpTw6w7VeArySvuoPFd589sUJhISIST3dwB6JlutdCRKQ/CotAUW4WZYU5bKlrSnUpIiLDjsIijrrPioj0TWERJ6qwEBHpk8IiTjRSwPZ9rXRoqHIRkYMoLOJEywro6na272tJdSkiIsOKwiKORp8VEembwiKOwkJEpG8KizgTi/PIyczQ6LMiIr0oLOJkZBgVGlBQRORNFBa9qPusiMibKSx6iUZiD0Fyf9PjM0RE0pbCopdopICGtk72aahyEZEDFBa9qEeUiMibKSx60eizIiJvprDoRS0LEZE3U1j0UpCTxbiiXN1rISISR2HRh6jutRAROYjCog+610JE5GAKiz5EywrZUd9Ce6eGKhcRAYVFn6KRArodtmmochERQGHRJ/WIEhE5mMKiDwoLEZGDKSz6MH5MLrlZGWypbUp1KSIiw0JSw8LM5pvZGjNbb2a39bH8U2b2ipm9ZGZPmtnUuGVdZrYieC1MZp29ZWQYleoRJSJyQFaydmxmmcC9wNuBamCpmS1091fiVnsRqHL3ZjP7CHA3cHWwrMXdT05WfYnEus/qAreICCS3ZTEXWO/uG929HXgQuCx+BXd/yt17/nxfDFQksZ5DEo0UsLVOQ5WLiEByw2IKsDXufXUwrz83An+Me59nZsvMbLGZvTsZBQ4kGimgsa2Tuqb2of5oEZFhJ2mnoQDrY16ff6ab2fuBKuCtcbOj7r7dzGYAfzWzl919Q6/tbgJuAohGo4NTdc+Hx/WIKivKHdR9i4iMNMlsWVQDlXHvK4DtvVcyswuB24FL3b2tZ767bw++bgSeBk7pva27L3D3KnevKi8vH9TiNVS5iMgbkhkWS4FZZjbdzHKAa4CDejWZ2SnAD4gFxe64+aVmlhtMjwPOBuIvjCddZWkQFhp9VkQkeaeh3L3TzG4BHgcygfvdfbWZ3Qksc/eFwNeAIuBhMwPY4u6XAscCPzCzbmKBdlevXlRJl5+TyfgxuWpZiIiQ3GsWuPtjwGO95n0hbvrCfrZbBJyYzNrC0OizIiIxuoN7ANGyWPdZEZF0p7AYQDRSwI79rbR1dqW6FBGRlFJYDCAaKcAdqvfqTm4RSW8KiwFo9FkRkRiFxQB67rXQdQsRSXcKiwGUF+WSl53BZt1rISJpTmExADNT91kREUKEhZm918zGBNN3mNlvzOzU5Jc2PEQjhToNJSJpL0zL4vPu3mBmbwEuBh4A7ktuWcNHT8tCQ5WLSDoLExY9NxlcAtzn7r8DcpJX0vASjeTT3N7FnkYNVS4i6StMWGwzsx8AVwGPBQP8pc21Do0+KyIS7pf+VcQGA5zv7vuACPCZpFY1jEQjhYC6z4pIegszkOAk4H/dvc3MzgNOAv4nqVUNIxWl+QDqPisiaS1My+LXQJeZHQX8GJgO/CKpVQ0jedmZTCzO02koEUlrYcKi2907gcuBe9z9X4i1NtJGNKLRZ0UkvYUJiw4zuxb4APCHYF528koafqJlujFPRNJbmLC4ATgL+Iq7v25m04GfJbes4SUaKWDn/lZaOzRUuYikp4RhETzO9NPAy2Z2AlDt7nclvbJhpGf02eq9al2ISHoKM9zHecA64F7ge8BaMzs3yXUNK7rXQkTSXZius18HLnL3NQBmNhv4JXBaMgsbTnpaFuo+KyLpKsw1i+yeoABw97Wk2QXussIcCnIy1bIQkbQVpmWxzMx+DPw0eH8dsDx5JQ0/PUOVq/usiKSrMGHxEeBjwCcAA54ldu0irUQjBWyqbUp1GSIiKZEwLNy9DfhG8Epb0UgBz66rwd0xs1SXIyIypPoNCzN7Gej3IQ7uflJSKhqmomUFtHZ0U9PQxvjivFSXIyIypAZqWbzrSHduZvOBbwGZwI96359hZp8CPgx0AjXAh9x9c7DseuCOYNV/d/cHjrSeI9HTI2pLXbPCQkTSTr9h0fNL+3CZWSaxezPeDlQDS81sYXCTX48XgSp3bzazjwB3A1ebWQT4IlBFrHWzPNh275HUdCTiw6JqWiRVZYiIpEQyH2I0F1jv7hvdvR14ELgsfgV3f8rde7oYLQYqgumLgSfcvS4IiCeA+UmsNaEppfmY6V4LEUlPyQyLKcDWuPfVwbz+3Aj88TC3TbrcrEwmFeep+6yIpKUww328y8wOJ1T66jLU5wVzM3s/sVNOXzuUbc3sJjNbZmbLampqDqPEQ6PRZ0UkXYUJgWuAdWZ2t5kdewj7rgYq495XANt7r2RmFwK3A5cG3XRDb+vuC9y9yt2rysvLD6G0wxONFLBZYSEiaSjMqLPvB04BNgA/MbPngr/oxyTYdCkwy8ymm1kOsdBZGL+CmZ0C/IBYUOyOW/Q4cJGZlZpZKXBRMC+lopECahraaGnXUOUikl5CnV5y9/3EHq/6ILGn5L0HeMHMPj7ANp3ALcR+yb8K/MrdV5vZnWZ2abDa14Ai4GEzW2FmC4Nt64B/IxY4S4E7g3kpVRn0iNqqocpFJM0kvIPbzP4J+BAwk9j4UHPdfbeZFRALge/0t627PwY81mveF+KmLxxg2/uB+xPVN5SmlhUCsKW2mdkTEjWsRERGjzBjQ70X+Ka7Pxs/M7g34kPJKWt4OjBUua5biEiaCTM21AfMbGJw6siBpe6+M1j2ZLILHE5KC7Ipys1S91kRSTthus7eCCwBLgeuBBanW4uiR89Q5eo+KyLpJsxpqM8Cp7h7LYCZlQGLGGbXE4ZKNFLA+prGVJchIjKkwvSGqgYa4t43cPDd1Wml58a87u5+B+QVERl1wrQstgHPm9nviF2zuAxYEowYi7un1XMuKiMFtHd2s7uhjYklGn1WRNJDmLDYELx6/C74mpZ9R6fGjT6rsBCRdBGmN9SXAYI7tt3d0/qE/YHus7VNzJ2uocpFJD2E6Q11gpm9CKwCVpvZcjM7PvmlDU+Tx+aTYaj7rIiklTAXuBcAn3L3qe4+FfhX4IfJLWv4ysnKYPLYfHWfFZG0EiYsCt39qZ437v40UJi0ikYA3WshIukmTFhsNLPPm9m04HUH8HqyCxvOFBYikm7ChMWHgHLgN8FrHHBDMosa7iojBexpbKeprTPVpYiIDIkBe0OZWSbw/9z9E0NUz4gwteyNocqPmVic4mpERJJvwJaFu3cBpw1RLSNGT/fZLbU6FSUi6SHMTXkvBg8lehho6pnp7r9JWlXDXDTuxjwRkXQQJiwiQC3wtrh5Tuz6RVoqyc9mTF6WwkJE0kaYsPiRu/8jfoaZnZ2kekYEM2NqmXpEiUj6CNMbqq/Hpvb7KNV0EY0U6JqFiKSNflsWZnYWMA8o7xlhNlAMZCa7sOGuMlLAX17ZTVe3k5lhqS5HRCSpBmpZ5ABFxAJlTNxrP7En5qW1qZFC2ru62bW/NdWliIgkXb8tC3d/BnjGzP7b3TcPYU0jQnyPqMlj81NcjYhIcoW5wJ1rZguAafHru/vb+t0iDcTfa3HmjLIUVyMiklxhwuJh4PvAj4Cu5JYzckwam0dmhqlHlIikhTC9oTrd/T53X+Luy3teYXZuZvPNbI2ZrTez2/pYfq6ZvWBmnWZ2Za9lXWa2IngtDPnvGTLZmRlM0VDlIpImwrQsfm9mHwV+C7T1zHT3uoE2CsaVuhd4O1ANLDWzhe7+StxqW4APAp/uYxct7n5yiPpSRqPPiki6CBMW1wdfPxM3z4EZCbabC6x3940AZvYgcBlwICzcfVOwrDtkvcNKZaSAx1fvTHUZIiJJF+YZ3NMPc99TgK1x76uBMw5h+zwzWwZ0Ane5+6OHWUfSTC0roK6pnYbWDsbkZae6HBGRpAnzDO4CM7sj6BGFmc0ys3eF2Hdfd6r5IdQWdfcq4H3APWY2s4/abjKzZWa2rKam5hB2PTh6ekRtrWsZ8s8WERlKYS5w/wRoJ3Y3N8RaCP8eYrtqoDLufQWwPWxh7r49+LoReBo4pY91Frh7lbtXlZeXh931oHnjXoumBGuKiIxsYcJiprvfDXQAuHsLfbcaelsKzDKz6WaWA1wDhOrVZGalZpYbTI8DzibuWsdwUamhykUkTYQJi3Yzyyc4hRScDmobeBNw907gFuBx4FXgV+6+2szuNLNLg32dbmbVwHuBH5jZ6mDzY4FlZrYSeIrYNYthFxYl+dmMLchWWIjIqBemN9QXgT8BlWb2c2J/5X8wzM7d/THgsV7zvhA3vZTY6ane2y0CTgzzGakW6z6raxYiMrqF6Q31hJm9AJxJ7PTTJ919T9IrGyEqIwWs3laf6jJERJIqzGko3L3W3f8XqFJQHGxqpIDqvS10dR9KRy8RkZElVFjEuTQpVYxg0UgBnd3OjnqdihKR0etQw0JP+eklqh5RIpIGDjUsTktKFSNYT/fZ13Y0pLgSEZHkCXMH991mVmxm2cATZrbHzN4/BLWNCBWl+cypKOHbf13Hzno9NU9ERqcwLYuL3H0/8C5id2XP5uBBBdOamfHNq0+mraObzzyykm5d6BaRUShMWPSMkPdO4JeJhiZPRzPKi7jjXcfyt3V7+O9Fm1JdjojIoAsTFr83s9eAKuBJMysHdL6ll/fNjXLBMeO560+vsWanrl+IyOiSMCzc/TbgLGL3WHQATcSeSyFxzIyvXnkSxXlZfPLBF2nr1BNoRWT0CHOB+73EHq3aZWZ3AD8DJie9shFoXFEuX73iJF7b2cDX/7w21eWIiAyaMKehPu/uDWb2FuBi4AHgvuSWNXJdcOwErjsjyg//tpFF63Wzu4iMDmHCoud8yiXAfe7+OyAneSWNfLdfcizTywr514dXUt/ckepyRESOWJiw2GZmPwCuAh4LnjNxqDfzpZWCnCzuueZkahrauP3Rl3FXd1oRGdnC/NK/itgzKea7+z4ggu6zSOikirHceuEs/vDSDn63IvQDAkVEhqUwvaGagQ3AxWZ2CzDe3f+c9MpGgY+cdxRVU0v5/KOrqN6rsaNEZOQK0xvqk8DPgfHB62dm9vFkFzYaZGbE7u524FO/WqlhzEVkxApzGupG4Ax3/0LwlLszgX9OblmjR2WkgC9dejxLXq9jwbMbU12OiMhhCRMWxhs9ogimNVT5Ibji1Cm888SJfOOJNazSU/VEZAQKExY/AZ43sy+Z2ZeAxcCPk1rVKGNmfOXdJxIpzOHWh1bQ2qG7u0VkZAlzgfsbwA1AHbAXuMHd70l2YaNNaWEO//XeOazf3chdf3wt1eWIiBySrIEWmlkG8JK7nwC8MDQljV7nzCrnQ2dP5/5/vM55R5dz3tHjU12SiEgoA7Ys3L0bWGlm0SGqZ9T77PyjmT2hiM888hJ1Te2pLkdEJJQw1ywmAavN7EkzW9jzSnZho1Vedib3XH0K9c0d3Pbrl3R3t4iMCAOehgp8OelVpJnjJhfzmYuP5iuPvcrDy6q56vTKVJckIjKgflsWZnaUmZ3t7s/EvwAn9njVhMxsvpmtMbP1ZnZbH8vPNbMXzKzTzK7stex6M1sXvK4/1H/YcHfjW6Zz1owyvvT71WyubUp1OSIiAxroNNQ9QF+PfGsOlg3IzDKBe4F3AMcB15rZcb1W2wJ8EPhFr20jwBeBM4C5wBfNrDTRZ44kGRnG16+aQ1aGcetDK+js6k51SSIi/RooLKa5+0u9Z7r7MmBaiH3PBda7+0Z3bwcepNcT9tx9U/AZvX9TXgw84e517r4XeAKYH+IzR5TJY/P5yntO5MUt+7jtNy/TocAQkWFqoGsWeQMsyw+x7ynA1rj31cRaCmH0te2U3iuZ2U3ATQDR6MjssPVPcyazfncj33pyHbWNbdx73akU5IS5lCQiMnQGalksNbM3jQFlZjcCy0Psu68hQcJ2/Qm1rbsvcPcqd68qLy8Puevh51/ePpv/eM+JPLO2hmsXLGZPY1uqSxIROchAf8LeCvzWzK7jjXCoIvaUvPeE2Hc1EN/NpwII+2CHauC8Xts+HXLbEel9Z0QpH5PLx3/5Alfct4gHbpjLtHGFqS5LRAQYoGXh7rvcfR6xrrObgteX3f0sd98ZYt9LgVlmNt3McoBrgLD3ZzwOXGRmpcGF7YuCeaPa24+bwC/++Uz2t3RwxX2LWLl1X6pLEhEBwo0N9ZS7fyd4/TXsjt29E7iF2C/5V4FfuftqM7vTzC4FMLPTzawaeC/wAzNbHWxbB/wbscBZCtwZzBv1To2W8shH5pGfk8k1Cxbz1JrdqS5JRAQbLXcQV1VV+bJly1JdxqDZ3dDKDT9Zyms7G/jPy0/kqirduCcig8/Mlrt7VaL1wgz3ISkwfkweD/2fs5g3s4zPPvIS33lynYYGEZGUUVgMY0W5Wfz4+tO5/JQpfP2Jtdz+6CrdvCciKaEO/cNcTlYGX79qDhNL8vje0xvYvb+N71x7Cvk5makuTUTSiFoWI4CZ8dn5x/DlS4/nydd2cd2PFrNXw5uLyBBSWIwg18+bxn3Xncqq7fu54vuL2FrXnOqSRCRNKCxGmPknTOLnHz6D2sZ2Lr9vEau21ae6JBFJAwqLEej0aREeufkssjOMaxYs5m/ralJdkoiMcgqLEWrWhDH85qNnU1Gazw0/Wcr3n9lAe6d6SolIcigsRrCJJXn86uazOP+Y8dz1x9eY/61neWatWhkiMvgUFiNccV42P/xAFT/54Om4w/X3L+HDDyxjS60ufovI4FFYjBLnHzOex289l9vecQzPbdjDhd98hv96fA3N7Z2pLk1ERgGFxSiSk5XBzW+dyV8/fR6XnDiJ7z61ngu+/gy/X7ldQ4WIyBFRWIxCE4rz+ObVJ/PIzWcRKczh4798kWsWLObVHftTXZqIjFAKi1GsalqEhbe8hf94z4ms3dXAJd/+G1/83Sr2NevubxE5NAqLUS4zw3jfGVGe+vR5vP/Mqfx08WbO/6+n+fnzm+nq1qkpEQlHYZEmxhbkcOdlJ/C/nziHWRPGcPtvV3Hpd//Osk1p8UwpETlCCos0c+ykYh666Uy+c+0p1DW1c+X3n+NfHlrB6u31ugguIv3SEOVpyMz4pzmTueDY8XzvqQ0seHYjv31xGxOKc3nr7HLOP3o8Z88aR3FedqpLFZFhQo9VFfY0tvH0mhqeWrObZ9fW0NDaSVaGcdrUUs4/ZjznHz2e2ROKMLNUlyoigyzsY1UVFnKQzq5uXtiyj6fX7OapNTUHuttOKsnjvKPHc/7R5Zx91DgKc9UoFRkNFBYyKHbWt/LM2t089VoNf1+/h8a2TrIzjbnTI5w3ezznH1POzHK1OkRGKoWFDLr2zm6Wb94btDp2s3ZXIwAVpflcfmoF1581lbKi3BRXKSKHQmEhSbdtXwtPr9nNE6/s4uk1NeRmZXDFaRV8+C3TmVFelOryRCQEhYUMqQ01jfzobxv59Qvb6Ojq5u3HTuD/vHUGp02NpLo0ERlA2LBI6n0WZjbfzNaY2Xozu62P5blm9lCw/HkzmxbMn2ZmLWa2Inh9P5l1ypGbWV7Ef15+Ev/43Nu45fyjWLKpjivue47Lv/cP/rRqp+4WFxnhktayMLNMYC3wdqAaWApc6+6vxK3zUeAkd7/ZzK4B3uPuVweh8Qd3PyHs56llMbw0t3fy8LJqfvT3jWyta2H6uEJufMt0rjytgrzszFSXJyKB4dCymAusd/eN7t4OPAhc1mudy4AHgulHgAtM3WpGhYKcLK6fN42n/vU87n3fqRTnZXHHo6uYd9dfuecva6lr0mCGIiNJMsNiCrA17n11MK/Pddy9E6gHyoJl083sRTN7xszOSWKdkkRZmRlcctIkHv3Y2Tx405mcUjmWe/6yjnl3PcnnH13Fpj1NqS5RREJI5p1VfbUQep/z6m+dHUDU3WvN7DTgUTM73t0PeiCDmd0E3AQQjUYHoWRJFjPjzBllnDmjjHW7Gvjh3zby0NKt/Oz5zcw/fiLvOHESJ0wuZlpZIRkZalyKDDfJDItqoDLufQWwvZ91qs0sCygB6jx2IaUNwN2Xm9kGYDZw0EUJd18ALIDYNYtk/CNk8M2aMIa7r5zDpy86mgee28RPn9vMH1ftBKAwJ5PjJ5dw/JRiTphcwglTSphZXkhWpsa8FEmlZIbFUmCWmU0HtgHXAO/rtc5C4HrgOeBK4K/u7mZWTiw0usxsBjAL2JjEWiUFxhfn8ZmLj+HWC2ezblcjq7bXs3pbPau27+fBJVtp6dgEQF52BsdO6gmPYo6fXMLsCWPIyVKAiAyVpIWFu3ea2S3A40AmcL+7rzazO4Fl7r4Q+DHwUzNbD9QRCxSAc4E7zawT6AJudnc9eGGUys7M4LjJxRw3uRiqYo3Rrm7n9T2NvLytnlXb9rNqWz2PvriNny7eHGxjHD1xDCdMLuH4KSUcVV5EtKyAicV5ZOo0lsig0015MmJ0dztb6ppZtT0WIKu31/Pytnr2NXccWCcnM4OK0nyiZQVEI3Gv4H1BjgZAFIkXtuusfnJkxMjIMKaNK2TauELeddJkANydbfta2LSnmS11zWyua2JrXWx6+ea9NLR2HrSPcUW5RCP5TC0rpDIIkqllBRw7qZgijaQr0i/9dMiIZmZUlBZQUVrwpmXuTn1LB5trY+Gxpa6ZLcH0ktfreHTFNnoa1pkZxpyKEubNHMe8mWWcOrVUNw+KxNFpKElb7Z3dQaukieWb97Jowx5WVtfT1e3kZGVwWrSUs2aWMW9mGXMqx5KtHlkyCmkgQZHD0NjWydLX61i0YQ+LNtTyyo79uENBTianT4swb2YZ82aO47jJxbqQLqOCwkJkEOxtauf512tZtCH2Wr879gyP4rwszpxRFrQ8xnHU+CKFh4xIusAtMghKC3OYf8Ik5p8wCYDd+1t5bmMti9bXsmjjHv78yi4AMgzKinIZPyaX8jGxr+PH5DG+OH5eHuVjcnUtREYkhYXIIRhfnMdlJ0/hspNjw5xtrWtm8cZattQ1s3t/GzWNbexuaOWV7fvZ09hGXyOzF+dlHQiPnjCJlhVy9IQxzJ5QxNiCnCH+V4kkprAQOQKVkQIqI2/uiQWxGwvrmtrZ3dBKTUMbuxvaYl/3t8ZCZX8bL27Zx+6GVlo7ug9sN35MLkdPHMPsCWNiATJxDLPGF1E4SF17G1o72Fnfyo761gNfI0U5nDE9wqzxep669E1hIZIkmRlGeXAKaiDuzs79razZ2cDaXQ2s2dnIut0N/Pz5zQeFSEVp/oHwOHrCGGZNKGJmedGB01ruzv7WziAAWthxIBBaDgqGxrbO/kohUpjD6dNKmTu9jDOmRzh2ki7kS4wucIsMU13dTvXe5jdCZFcj63Y1sKGmkY6u2M9thsG0skLMYEd9K83tXQftwyzWUplYks+k4jwmluQxeWxe7H1JHhOL85hQnMfO+lYWv17LktfrWPJ6HVvqmgEYk5tFVRAec6dHOHFKyRGPydXV7exuaGX7vha274t9jRTmcM6sciaW5B3RvuXQqTeUyCjV0dXNpj1NrNnVwNqdDazd1UhGBkwsDgIgLhDGj8k9rPtDdtS3sOT1Op4PwqOnF1h+dianTh3L3Gmx8DglOvagC/Y9rZtYELSwvb71jekgHHbub+33MbuzJxRx7qxyzp1dztzpEXUGGAIKCxEZNHsa21gaFx6v7ozdf5KTmcGcyhIKcrLYUR8Lg96nubIyjElj85hcks+UsflMDl6TxuYxZWws4Kr3tvC3dTU8u3YPSzbV0d7ZTW5WBnOnR3jr7HLOmVXO7AnD73qKe+y6VHZWBsV52aku57AoLEQkaepbOli+uY7nN9Yd+OXeEwRvBEIek8fmM64o95Cue7S0d/H867U8u3YPz66rOdCqmVCcyzlBq+Oco8ZRWjg0vca6u50d+1vZXNvEltpmNtU2s6Wuic21zWyubT4QjjPGFXJSRQknVoxlTkUJx08uIT9c4KgPAAAI9ElEQVRn+LeMFBYiMips3/dGq+Pv6/dQ39KBGZw0peRAeMwsLyQzwzAzMjOMDIMMMzLi3g/UKmnv7Gbr3tjYYZtrm9hc1xyEQRNb97bQ3vlGR4PsTKOyNDaS8dRIAdGyQlraO1lZXc9L1fvYtb8NiHVwmDW+iDkVYzmpsoQ5FWMH/TksLe1d1Da10dnlTBtXeFj7UFiIyKjT1e28VL3vQKtjxdZ9/V7/6EtfQWIGTW2dB90TU5CTydSyQqYGoxJPLStkajDM/eSx+QO2lHbtb2Xl1n28vK3+QID0DKOfkxV7kNecihJOClogM8pjd/+7O83tXdQ1tVPb1E5dUxu1jT3T7dQ2xubVNbWzpzE2r6Uj1qHhlOhYfvvRsw/rmCosRGTUq2/p4LkNteza30q3O13djjt0udPtTne30+0E84PpPpYV52czNVLAtHEFRCOFjCvKGbTrI+7O1roWVlbv46XqfaysrmfVtvoDPdcKczIZW5BDbVPbQV2l4+VkZTCuMIdIUQ6RwlzKCnOIFOZQVpRDWWEOlaUFzDtq3GHVp7AQERmmurqdjTWNB1oejW2dQQDkHgiASGEOZYW5RIpyKMzJTNrFfY0NJSIyTGVmGLMmjGHWhDFceVpFqssJRQP0i4hIQgoLERFJSGEhIiIJKSxERCQhhYWIiCSksBARkYQUFiIikpDCQkREEho1d3CbWQ2w+Qh2MQ7YM0jlJIPqOzKq78ioviMznOub6u7liVYaNWFxpMxsWZhb3lNF9R0Z1XdkVN+RGe71haHTUCIikpDCQkREElJYvGFBqgtIQPUdGdV3ZFTfkRnu9SWkaxYiIpKQWhYiIpJQWoWFmc03szVmtt7Mbutjea6ZPRQsf97Mpg1hbZVm9pSZvWpmq83sk32sc56Z1ZvZiuD1haGqL66GTWb2cvD5b3ralMV8OziGL5nZqUNY29Fxx2aFme03s1t7rTOkx9DM7jez3Wa2Km5exMyeMLN1wdfSfra9PlhnnZldP4T1fc3MXgv+/35rZmP72XbA74Uk1vclM9sW93/4zn62HfDnPYn1PRRX2yYzW9HPtkk/foPK3dPiBWQCG4AZQA6wEjiu1zofBb4fTF8DPDSE9U0CTg2mxwBr+6jvPOAPKT6Om4BxAyx/J/BHwIAzgedT+P+9k1gf8pQdQ+Bc4FRgVdy8u4HbgunbgK/2sV0E2Bh8LQ2mS4eovouArGD6q33VF+Z7IYn1fQn4dIj//wF/3pNVX6/lXwe+kKrjN5ivdGpZzAXWu/tGd28HHgQu67XOZcADwfQjwAWWrGcZ9uLuO9z9hWC6AXgVmDIUnz3ILgP+x2MWA2PNbFIK6rgA2ODuR3Kj5hFz92eBul6z47/PHgDe3cemFwNPuHudu+8FngDmD0V97v5nd+8M3i4GUvYot36OXxhhft6P2ED1Bb87rgJ+OdifmwrpFBZTgK1x76t58y/jA+sEPyz1QNmQVBcnOP11CvB8H4vPMrOVZvZHMzt+SAuLceDPZrbczG7qY3mY4zwUrqH/H9JUH8MJ7r4DYn8kAOP7WGe4HMcPEWsp9iXR90Iy3RKcJru/n9N4w+H4nQPscvd1/SxP5fE7ZOkUFn21EHp3BQuzTlKZWRHwa+BWd9/fa/ELxE6rzAG+Azw6lLUFznb3U4F3AB8zs3N7LR8OxzAHuBR4uI/Fw+EYhjEcjuPtQCfw835WSfS9kCz3ATOBk4EdxE719Jby4wdcy8CtilQdv8OSTmFRDVTGva8Atve3jpllASUcXhP4sJhZNrGg+Lm7/6b3cnff7+6NwfRjQLaZjRuq+oLP3R583Q38llhzP16Y45xs7wBecPddvRcMh2MI7Oo5NRd83d3HOik9jsEF9XcB13lwgr23EN8LSeHuu9y9y927gR/287mpPn5ZwOXAQ/2tk6rjd7jSKSyWArPMbHrwl+c1wMJe6ywEenqdXAn8tb8flMEWnN/8MfCqu3+jn3Um9lxDMbO5xP7/aoeivuAzC81sTM80sQuhq3qtthD4QNAr6kygvueUyxDq9y+6VB/DQPz32fXA7/pY53HgIjMrDU6zXBTMSzozmw98DrjU3Zv7WSfM90Ky6ou/Bvaefj43zM97Ml0IvObu1X0tTOXxO2ypvsI+lC9iPXXWEuslcXsw705iPxQAecROXawHlgAzhrC2txBrJr8ErAhe7wRuBm4O1rkFWE2sZ8diYN4QH78ZwWevDOroOYbxNRpwb3CMXwaqhrjGAmK//Evi5qXsGBILrR1AB7G/dm8kdh3sSWBd8DUSrFsF/Chu2w8F34vrgRuGsL71xM7393wf9vQQnAw8NtD3whDV99Pge+slYgEwqXd9wfs3/bwPRX3B/P/u+Z6LW3fIj99gvnQHt4iIJJROp6FEROQwKSxERCQhhYWIiCSksBARkYQUFiIikpDCQiQBM+vqNZrtoI1gambT4kcsFRmuslJdgMgI0OLuJ6e6CJFUUstC5DAFzyP4qpktCV5HBfOnmtmTwUB3T5pZNJg/IXg+xMrgNS/YVaaZ/dBizzH5s5nlB+t/wsxeCfbzYIr+mSKAwkIkjPxep6Gujlu2393nAt8F7gnmfZfYMO0nERuE79vB/G8Dz3hsEMNTid25CzALuNfdjwf2AVcE828DTgn2c3Oy/nEiYegObpEEzKzR3Yv6mL8JeJu7bwwGgdzp7mVmtofYEBQdwfwd7j7OzGqACndvi9vHNGLPrZgVvP8ckO3u/25mfwIaiY2M+6gHAyCKpIJaFiJHxvuZ7m+dvrTFTXfxxrXES4iNs3UasDwYyVQkJRQWIkfm6rivzwXTi4iNcgpwHfD3YPpJ4CMAZpZpZsX97dTMMoBKd38K+CwwFnhT60ZkqOgvFZHE8s1sRdz7P7l7T/fZXDN7ntgfXtcG8z4B3G9mnwFqgBuC+Z8EFpjZjcRaEB8hNmJpXzKBn5lZCbGRfL/p7vsG7V8kcoh0zULkMAXXLKrcfU+qaxFJNp2GEhGRhNSyEBGRhNSyEBGRhBQWIiKSkMJCREQSUliIiEhCCgsREUlIYSEiIgn9fwtQVqI53YvSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.ion()\n",
    "\n",
    "fig = plt.figure()\n",
    "plt.plot(train_loss_avg)\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Cross-entropy loss')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Alternatively: Load Pre-Trained Model\n",
    "-----------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'multilayer_perceptron.pth'\n",
    "import urllib\n",
    "if not os.path.isdir('./pretrained'):\n",
    "    os.makedirs('./pretrained')\n",
    "print('downloading ...')\n",
    "urllib.request.urlretrieve (\"http://geometry.cs.ucl.ac.uk/creativeai/pretrained/\"+filename, \"./pretrained/\"+filename)\n",
    "mlp.load_state_dict(torch.load('./pretrained/'+filename))\n",
    "print('done')\n",
    "\n",
    "# this is how the model parameters can be saved:\n",
    "# torch.save(mlp.state_dict(), './pretrained/my_multilayer_perceptron.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate on the Test Set\n",
    "-------------------------\n",
    "\n",
    "The best current methods achieve a classification error percentage of around 0.21%. See [here](http://rodrigob.github.io/are_we_there_yet/build/classification_datasets_results.html#4d4e495354) for a leaderboard. Also notice that there is some overfitting: the average loss is significantly higher than for the training set. Overfitting can be reduced by adding dropout between the fully connected layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average loss: 0.079978\n",
      "classification error: 1.760000%\n"
     ]
    }
   ],
   "source": [
    "# set to evaluation mode\n",
    "mlp.eval()\n",
    "\n",
    "num_incorrect = 0\n",
    "test_loss_avg = 0\n",
    "num_batches = 0\n",
    "num_instances = 0\n",
    "for image_batch, label_batch in test_dataloader:\n",
    "    \n",
    "    image_batch = image_batch.to(device)\n",
    "    label_batch = label_batch.to(device)\n",
    "    \n",
    "    # class predictions\n",
    "    prediction_batch = mlp(image_batch)\n",
    "    \n",
    "    # get number of correct and incorrect class predictions\n",
    "    _, predicted_label = prediction_batch.max(dim=1)\n",
    "    num_incorrect += (predicted_label != label_batch).sum().item()\n",
    "\n",
    "    # cross-entropy loss\n",
    "    loss = F.nll_loss(prediction_batch, label_batch)\n",
    "\n",
    "    test_loss_avg += loss.item()\n",
    "    num_batches += 1\n",
    "    num_instances += image_batch.size(0)\n",
    "    \n",
    "test_loss_avg /= num_batches\n",
    "print('average loss: %f' % (test_loss_avg))\n",
    "print('classification error: %f%%' % ((num_incorrect / num_instances)*100))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}