MNIST-Linear-Experiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "better-orchestra",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn \n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import matplotlib.pyplot as plt "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ceramic-settle",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_size = 784 # 28x28\n",
    "hidden_sizes = [500,300] \n",
    "num_classes = 10\n",
    "num_epochs = 5\n",
    "batch_size = 100\n",
    "learning_rate = 0.001 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "tropical-lounge",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import MNIST dataset \n",
    "train_dataset = torchvision.datasets.MNIST(root='./data', \n",
    "                                        train=True, \n",
    "                                   transform=transforms.ToTensor(),  \n",
    "                                       download=True)\n",
    "test_dataset = torchvision.datasets.MNIST(root='./data', \n",
    "                                       train=False, \n",
    "                                       transform=transforms.ToTensor()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "compound-spray",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data loader\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n",
    "                                        batch_size=batch_size, \n",
    "                                        shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n",
    "                                       batch_size=batch_size, \n",
    "                                       shuffle=False) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "exterior-welcome",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD6CAYAAAC4RRw1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAbO0lEQVR4nO3de4xWxfkH8O8jrld+sYBAt0BBwKJbqqJgEfHSKnIRBC9U1Bi8xLUNWIwURbCxN1NCExpbEbuJBLQErYC6KhUIQaktGHYrKrgglwhsXAWKVVYlsjC/P/Z0mDnseffd9z23Oe/3k2z2mXfOvufRZxkO886ZI0opEBGRe05IOgEiIioMB3AiIkdxACcichQHcCIiR3EAJyJyFAdwIiJHFTWAi8gIEdkqIttFZHpYSVGyWNfsYm2zRQpdBy4i7QB8CGAYgHoAGwDcopT6ILz0KG6sa3axttlzYhE/ezGA7UqpnQAgIs8BGAsg8JdBRHjXUEoopSSgi3V1236lVOeAvjbVlnVNlRbrWswUSjcAe4x2vfeaRUQqRaRGRGqKOBfFh3V1264cfa3WlnVNrRbrWswVeEtXcMf9ja2UqgJQBfBvdEewrtnVam1ZV7cUcwVeD6CH0e4O4OPi0qEUYF2zi7XNmGIG8A0AzhaRs0TkJAATAFSHkxYliHXNLtY2YwqeQlFKNYnIZAArALQDMF8ptTm0zCgRrGt2sbbZU/AywoJOxjm11MixCqXNWNdUqVVKDQzjjVjXVGmxrrwTk4jIURzAiYgcxQGciMhRxawDJ0qdX/ziF1b71FNP1fF5551n9d10002B7zNv3jyrvW7dOh0/++yzxaRIFBpegRMROYoDOBGRo7iMsERlaRnh888/r+Nc0yLF2LFjh46vvvpqq2/37t2RnLNAXEbYBt/73vd0vGXLFqtvypQpOv7zn/8cW04BuIyQiChLOIATETmKAzgRkaO4jJCcY855A/nPe/vnOFesWKHj3r17W31jxoyx2n369NHxbbfdZvX9/ve/z+v8lD4DBgzQ8dGjR62++vr6uNNpM16BExE5igM4EZGjOIVCThg48NgKquuvvz7wuM2b7d1Rr7vuOh3v37/f6mtsbNTxSSedZPWtX7/eap9//vk67tSpUx4ZkwsuuOACHX/55ZdW34svvhhzNm3HK3AiIkdxACcichQHcCIiRzk/B+5fQnbPPffo+OOP7ee1Hjp0SMeLFi2y+j755BMdb9++PcwUKQTl5eU6FrF3ATDnvYcPH271NTQ05PX+U6dOtdoVFRWBx7722mt5vSelT//+/a325MmTdeziLpO8AicichQHcCIiRzk/hTJ79myr3atXr7x+7t5777XaBw8e1LF/KVoczLu+/P9NNTU1caeTOq+88oqO+/bta/WZtTtw4EBB7z9hwgSrXVZWVtD7ULqdc845Vvv000/Xsf8OXxfwCpyIyFEcwImIHMUBnIjIUc7PgZvLBgH7wbV1dXVW37nnnqvjCy+80Oq78sordTx48GCrb8+ePTru0aNH3rk1NTVZ7X379unYXBbn53/CC+fAbbt27QrlfaZNm6Zj88ksLXn77bdbjMktDz74oNU2f5dc/HPGK3AiIke1OoCLyHwR2Ssim4zXOorIKhHZ5n3vEG2aFDbWNbtY29LR6kONReRyAI0AnlFK9fdemw3ggFJqlohMB9BBKfVQqydL8UNSO3Q49vts7lAGALW1tToeNGhQ3u9p3vkJAB9++KGO/dM7HTt21PGkSZOsvnnz5uV9zja4AiVQV9Po0aOt9gsvvKBj/26Ee/futdrmMsM333wzguxCUwvgAYRQW1fqmot/WfHOnTuttvln0r/EMGUKe6ixUmotAP/i2rEAFnrxQgDjis2O4sW6ZhdrWzoKnQPvqpRqAADve5fwUqIEsa7ZxdpmUOSrUESkEkBl1OeheLGu2cS6uqXQAfxTESlXSjWISDmAvUEHKqWqAFQB6Z5T++yzz3S8Zs2awONWr15d8DluvPFGHZtz7gDw/vvv6zjBW3ozV1eT+VQf4Ph5b5O/Bimf985HXrV1sa65XHHFFTn7zaW9Lip0CqUawEQvngjg5XDSoYSxrtnF2mZQPssIFwNYB6CfiNSLyN0AZgEYJiLbAAzz2uQQ1jW7WNvS0eoUilLqloCuq0LOJXO6dLE/J3ryySd1fMIJ9t+dv/nNb3Rc6I56bVEqdX3ppZd0fM011wQe98wzz1jtRx55JKqUIlcqtc3HD37wg5z9/p0/XcM7MYmIHMUBnIjIURzAiYgc5fxuhGnmvyW+c+fOOjaXLQLA1q1bY8kp6/y7PA4ZMkTHJ598stW3f/9+Hf/ud7+z+hobGyPIjuJg7iZ65513Wn3vvPOO1V61alUsOUWFV+BERI7iAE5E5ChOoYTs0ksv1fH06dMDjxs3bpzV3rRpU8sHUpssXbrUanfq1Cnw2L/+9a863rFjR2Q5UbyuvvpqHZu7fALA66+/brX9O4a6hlfgRESO4gBOROQoDuBERI7iHHjIRo0apeOysjKrz9zJcN26dbHllHXXXXedjv0Pqza98cYbVvvRRx+NKiVK0Pnnn69j/xPHlixZEnc6keIVOBGRoziAExE5igM4EZGjOAdepFNPPdVqjxgxQsfffPON1WfOuR4+fDjaxDLMv7Z7xowZOvZ/7mDauHGj1ebt8tnw7W9/22pfdtllOvZvUfHiiy/GklNceAVOROQoDuBERI7iFEqRpk2bZrUHDBigY/9tu//6179iySnrpk6darUHDRoUeKz5RB4uG8ymO+64w2qbT8L6+9//HnM28eIVOBGRoziAExE5igM4EZGjOAfeRtdee63V/uUvf2m1v/jiCx2bT5qn8DzwwAN5Hzt58mQdc9lgNvXs2TOwz//kq6zhFTgRkaM4gBMROYpTKHkw7/z705/+ZPW1a9fOai9fvlzH69evjzYxapX5RJZi7n79/PPPA9/HvPvzjDPOCHyPb33rW1Y736mgI0eOWO2HHnpIx1999VVe75Flo0ePDux75ZVXYswkfrwCJyJyFAdwIiJHtTqAi0gPEVkjInUisllEpnivdxSRVSKyzfveIfp0KSysa2aVsa6lI5858CYAU5VS/xaR/wNQKyKrANwBYLVSapaITAcwHcBDOd7HGf55bfOW+LPOOsvq8z/N3L+sMMVKoq7vvfdeKO/zwgsv6LihocHq69q1q45vvvnmUM6XyyeffKLjxx57rKVDMl/XoUOH6ti/G2EpafUKXCnVoJT6txcfBFAHoBuAsQAWeoctBDAuohwpAqxrZh1mXUtHm1ahiEgvAAMAvA2gq1KqAWgeDESkS8DPVAKoLDJPihDrmk2sa/blPYCLSHsASwHcr5T6QkTy+jmlVBWAKu89VCuHp0KfPn2s9kUXXRR4rH8pmH9KJe1crKu5VBMAxo4dG/k5x48fX9DPNTU16fjo0aOBx1VXV1vtmpqawGP/8Y9/tHpeF+vaFtdff72O/VOe77zzjo7Xrl0bW05JyGsVioiUofmXYZFSapn38qciUu71lwPYG02KFBXWNZtY19KRzyoUAfA0gDql1ByjqxrARC+eCODl8NOjqLCumca6loh8plAuBXA7gPdFZKP32gwAswD8TUTuBrAbQGH/xqSksK7Z1B6sa8lodQBXSr0FIGgC7apw00mOuaPZypUrA4/zP4Hn1VdfjSynKLlc1xtuuMFqP/jggzrO9VBjv+9///s6bsvyv/nz51vtjz76KPDYpUuX6njLli15n6MIjUopJ+uay2mnnWa1R40aFXjskiVLdOzfhiBreCcmEZGjOIATETmKuxF6KiuPLX397ne/G3jcm2++abWVSu1Kq5Ixe/bsot/j1ltvDSETiop/B0jzQQ3+JZiPP/54LDmlAa/AiYgcxQGciMhRHMCJiBxVsnPg5m5mAHDfffcllAkRtcY/Bz5kyJCEMkkXXoETETmKAzgRkaNKdgrlsssus9rt27cPPNbcYbCxsTGynIiI2oJX4EREjuIATkTkKA7gRESOKtk58Fzeffddq33VVcc2cTtw4EDc6RARtYhX4EREjuIATkTkKIlzN700PyS11OTY9L/NWNdUqVVKDQzjjVjXVGmxrrwCJyJyFAdwIiJHcQAnInJU3MsI9wPYBeBML06DUsylZ+uHtAnrmlucuYRZW9Y1t8TrGuuHmPqkIjVhfdBSLOYSnjTlz1zCk6b8mYuNUyhERI7iAE5E5KikBvCqhM7bEuYSnjTlz1zCk6b8mYshkTlwIiIqHqdQiIgcxQGciMhRsQ7gIjJCRLaKyHYRmR7nub3zzxeRvSKyyXito4isEpFt3vcOMeTRQ0TWiEidiGwWkSlJ5RIG1tXKJTO1ZV2tXFJZ19gGcBFpB2AugJEAKgDcIiIVcZ3fswDACN9r0wGsVkqdDWC1145aE4CpSqlzAQwGMMn7f5FELkVhXY+TidqyrsdJZ12VUrF8AbgEwAqj/TCAh+M6v3HeXgA2Ge2tAMq9uBzA1gRyehnAsDTkwrqytqyrO3WNcwqlG4A9Rrveey1pXZVSDQDgfe8S58lFpBeAAQDeTjqXArGuARyvLesaIE11jXMAb2n/6ZJewygi7QEsBXC/UuqLpPMpEOvaggzUlnVtQdrqGucAXg+gh9HuDuDjGM8f5FMRKQcA7/veOE4qImVo/kVYpJRalmQuRWJdfTJSW9bVJ411jXMA3wDgbBE5S0ROAjABQHWM5w9SDWCiF09E89xWpEREADwNoE4pNSfJXELAuhoyVFvW1ZDausY88T8KwIcAdgCYmcAHD4sBNAA4jOYrjLsBdELzp8fbvO8dY8hjKJr/OfoegI3e16gkcmFdWVvW1d268lZ6IiJH8U5MIiJHcQAnInJUUQN40rfaUjRY1+xibTOmiEn9dmj+cKM3gJMAvAugopWfUfxKxxfrmtmvfWHVNgX/Lfxqpa7FXIFfDGC7UmqnUuobAM8BGFvE+1E6sK5u25Wjj7V1V4t1LWYAz+tWWxGpFJEaEakp4lwUH9Y1u1qtLevqlhOL+Nm8brVVSlXBe/SQiBzXT6nDumZXq7VlXd1SzBV4Wm+1peKwrtnF2mZMMQN4Wm+1peKwrtnF2mZMwVMoSqkmEZkMYAWaP92er5TaHFpmlAjWNbtY2+yJ9VZ6zqmlh1KqpfnQgrCuqVKrlBoYxhuxrqnSYl15JyYRkaM4gBMROYoDOBGRoziAExE5igM4EZGjOIATETmqmFvpM+v000+32n/4wx90fO+991p9tbW1Vnv8+PE63rUr175CRETF4RU4EZGjOIATETmKAzgRkaM4B96C8vJyq33PPffo+OjRo1bfRRddZLVHjx6t47lz50aQHeVy4YUXWu1ly5bpuFevXpGf/5prrrHadXV1Ot6zZ4//cErYmDFjdFxdbe/rNXnyZB0/9dRTVt+RI0eiTSxPvAInInIUB3AiIkdxCsXTuXNnHS9cuDDBTKgYw4cPt9onn3xyrOc3/0kOAHfddZeOJ0yYEGsudLxOnTpZ7SeffDLw2CeeeELH8+fPt/q+/vrrcBMrEK/AiYgcxQGciMhRHMCJiBxVsnPgP//5z632uHHjdHzxxRcX/L6XX365jk84wf778d1339Xx2rVrCz4H2U488div8ahRoxLM5PitFR544AEd+7do+PLLL2PJiY4x/3wCQPfu3QOPXbx4sY4PHToUWU7F4BU4EZGjOIATETmqZKdQ/vjHP1pt/x2WhbrhhhtajAF7d8Kbb77Z6vP/05vy96Mf/UjHl1xyidU3e/bsWHPp0KGD1a6oqNDxaaedZvVxCiV6/mWkM2fOzPtnn332WR3H+fD3tuAVOBGRoziAExE5igM4EZGjJM65HRFJdCJp+fLlOh45cqTVV+gc+H/+8x+r3djYqOOePXvm/T7t2rUr6PyFUkpJWO8Vd1379+9vtd944w0d++th7hZp1iYqZi4AMHToUB37d7nct29fFCnUKqUGhvFGSf95DcPAgfb/ig0bNgQe29TUZLXLysoiyalALdaVV+BERI5qdQAXkfkisldENhmvdRSRVSKyzfveIdd7UPqwrtnF2paOfJYRLgDwBIBnjNemA1itlJolItO99kPhp1ecK664wmr369dPx/4pk3ynUPwbu69cudJqf/755zr+8Y9/bPXlWsL0s5/9TMfz5s3LK5ciLYCjdX3kkUestnmH44gRI6y+OKZNOnbsqGP/71xYy1PbaAEcrW3YbrzxxryP9f9ZdkGrV+BKqbUADvheHgvgf3uuLgQwLty0KGqsa3axtqWj0Bt5uiqlGgBAKdUgIl2CDhSRSgCVBZ6H4sW6ZldetWVd3RL5nZhKqSoAVUA2PtWmZqxrNrGubil0AP9URMq9v8nLAewNM6limA+ufe6556y+M888M6/3MG95B4ClS5fq+Ne//rXV99VXX+X9PpWVxy5szCcAAfYt36eccorVZz4Z5PDhw4HnC0Fq63rTTTfp2L/j4Pbt23VcU1MTW07/Y3624Z/zNpcV/ve//40poxaltrZR8u8+6PfNN9/ouC232adFocsIqwFM9OKJAF4OJx1KGOuaXaxtBuWzjHAxgHUA+olIvYjcDWAWgGEisg3AMK9NDmFds4u1LR2ZuxOzb9++Oq6rqws8zv+whTVr1ujY//DZ/fv3h5Lbfffdp+M5c+YE5uP/Z/g555yj4x07doSSi2t3Yj7//PM69i8NM/+/xrEE05ymA4D169fr2FxSCNgPWTZ/xyJU8ndiDhkyRMf//Oc/cx772Wef6dhfu5ThnZhERFnCAZyIyFEcwImIHFWyT+TxLze76667dBzWnLdfdXW1jm+77Tarb9CgQZGc01VnnHGG1R48eHDgsTFtPaCZy0EBe3mq/3OXmOa9ydCWP0tx/+6EjVfgRESO4gBOROSoTE+h+JcKmn74wx/GmEkzkWMr9/y55cr1V7/6lY5vv/320PNKI//DaLt166bjxYsXx52OpU+fPoF9mzZtCuyjePgf4mDy3w3LKRQiIkoEB3AiIkdxACciclTm5sB/+tOf6jihp6EEGjNmjI4HDBhg9Zm5+vM258BLxcGDB632xo0bdXzeeedZfeYt0AcO+J9jEI4uXY5tn23ujOj31ltvRXJ+CmY+OBoAbr311sBjzSdmAUB9fX0kOcWFV+BERI7iAE5E5CgO4EREjsrcHLg5z5wE80k7FRUVVt+MGTPyeo99+/ZZ7YifwpNKX3/9tdU2t9H1byf72muv6di/TW+++vfvb7V79+5ttc0tZHNtwZy2z11KQadOnax2rnsqVq1aFXU6seIVOBGRoziAExE5KnNTKEkzH4w6adKkvH/uo48+0vHEiROtvt27dxedl+seffRRHZtbEgDAtddeq+NCb7P370DpnybJ94HYCxYsKOj8VLhcyzr9t87/5S9/iTibePEKnIjIURzAiYgcxQGciMhRnAMv0vLly612v379CnqfDz74QMe8Hft4W7Zs0fFPfvITq++CCy7Qcd++fQt6/yVLluTsX7hwoY79T1My+Zc/UjS6d++u41y3zvtvlfc/ict1vAInInIUB3AiIkdlbgol11NvTCNHjgzsq6qqstrf+c53Ao/1n6PQO/GSvoPUZeZOhWYcpp07d+Z1nP+OTj6hJxpDhgzRca4/5y+99FIM2SSHV+BERI5qdQAXkR4iskZE6kRks4hM8V7vKCKrRGSb971D9OlSWFjXzCpjXUtHPlfgTQCmKqXOBTAYwCQRqQAwHcBqpdTZAFZ7bXIH65pdrGuJaHUOXCnVAKDBiw+KSB2AbgDGArjSO2whgDcAPBRJlm1gPmV69uzZgce9+uqrVjvX3HVb5rXzPfapp57K+z2j4Fpdk2Z+tuK/ld+Ugjnvw0qpfwPZrqt/B0KTuS3C448/Hkc6iWnTh5gi0gvAAABvA+jqDQJQSjWISJeAn6kEUFlknhQh1jWbWNfsy3sAF5H2AJYCuF8p9UWuqxCTUqoKQJX3HsEbKVMiWNdsYl1LQ14DuIiUofmXYZFSapn38qciUu79bV4OYG9USbbFsmXLdDxt2jSrz3zYQlTMhzHU1dVZfZWVxy5sGhoaIs+lNS7VNWnm7oS5HuiQBqVQ1+HDhwf2mbt3+h9inDX5rEIRAE8DqFNKmY87qQbwv31PJwJ4Ofz0KCqsa6axriUinyvwSwHcDuB9EdnovTYDwCwAfxORuwHsBjA+kgwpKqxrNrUH61oy8lmF8haAoAm0q8JNh+LCumZWo1KKdS0RmbuVfteuXTqeMGGC1Tdu3DgdT5kyJZLzP/bYYzqeO3duJOeg+J1yyimBfdyBMHplZWVWu0+fPoHHHjp0SMdZfyA4b6UnInIUB3AiIkdlbgrFtHbt2sD2ypUrrT5ziZ9/Z8Dq6mod+3cq9K+vNR/MQNlx55136tj/oNzf/va3MWdTevx3OJsPZvDvALl9+/ZYckoDXoETETmKAzgRkaM4gBMROSrTc+C5vP766znbRKYNGzboeM6cOVbfmjVr4k6n5Bw5csRqz5w5U8f+rQ1qa2tjySkNeAVOROQoDuBERI6SOHdW4/aU6ZHjdus2Y11TpVYpNTCMN2JdU6XFuvIKnIjIURzAiYgcxQGciMhRHMCJiBzFAZyIyFEcwImIHMUBnIjIURzAiYgcxQGciMhRHMCJiBwV926E+wHsAnCmF6dBKebSM+T3Y11zizOXMGvLuuaWeF1j3QtFn1SkJqz9GorFXMKTpvyZS3jSlD9zsXEKhYjIURzAiYgcldQAXtX6IbFhLuFJU/7MJTxpyp+5GBKZAyciouJxCoWIyFEcwImIHBXrAC4iI0Rkq4hsF5HpcZ7bO/98EdkrIpuM1zqKyCoR2eZ97xBDHj1EZI2I1InIZhGZklQuYWBdrVwyU1vW1collXWNbQAXkXYA5gIYCaACwC0iUhHX+T0LAIzwvTYdwGql1NkAVnvtqDUBmKqUOhfAYACTvP8XSeRSFNb1OJmoLet6nHTWVSkVyxeASwCsMNoPA3g4rvMb5+0FYJPR3gqg3IvLAWxNIKeXAQxLQy6sK2vLurpT1zinULoB2GO0673XktZVKdUAAN73LnGeXER6ARgA4O2kcykQ6xrA8dqyrgHSVNc4B3Bp4bWSXsMoIu0BLAVwv1Lqi6TzKRDr2oIM1JZ1bUHa6hrnAF4PoIfR7g7g4xjPH+RTESkHAO/73jhOKiJlaP5FWKSUWpZkLkViXX0yUlvW1SeNdY1zAN8A4GwROUtETgIwAUB1jOcPUg1gohdPRPPcVqRERAA8DaBOKTUnyVxCwLoaMlRb1tWQ2rrGPPE/CsCHAHYAmJnABw+LATQAOIzmK4y7AXRC86fH27zvHWPIYyia/zn6HoCN3teoJHJhXVlb1tXduvJWeiIiR/FOTCIiR3EAJyJyFAdwIiJHcQAnInIUB3AiIkdxACcichQHcCIiR/0/enVffNtcK4YAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 6 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "examples = iter(test_loader)\n",
    "example_data, example_targets = examples.next()\n",
    "for i in range(6):\n",
    "    plt.subplot(2,3,i+1)\n",
    "    plt.imshow(example_data[i][0], cmap='gray')\n",
    "plt.show() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "indoor-custom",
   "metadata": {},
   "outputs": [],
   "source": [
    "class NeuralNet(nn.Module):\n",
    "    def __init__(self, input_size, hidden_sizes, num_classes):\n",
    "        super(NeuralNet, self).__init__()\n",
    "        self.input_size = input_size\n",
    "        Ns = [*hidden_sizes]\n",
    "        Ns.insert(0,input_size)\n",
    "        hidden_layers = []\n",
    "        for i in range(len(Ns) - 1):\n",
    "            hidden_layers.append(nn.Linear(Ns[i], Ns[i + 1]))\n",
    "            hidden_layers.append(nn.ReLU())\n",
    "        self.layers = nn.Sequential(*hidden_layers)\n",
    "\n",
    "        self.l2 = nn.Linear(Ns[-1], num_classes)   \n",
    "    def forward(self, x):\n",
    "        out = self.layers(x)\n",
    "        out = self.l2(out)\n",
    "        return out "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "distributed-joyce",
   "metadata": {},
   "outputs": [],
   "source": [
    "device='cpu'\n",
    "model = NeuralNet(input_size, hidden_sizes, num_classes).to(device)\n",
    "import copy\n",
    "model_i = copy.deepcopy(model)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fixed-rolling",
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "hearing-subsection",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/omri/anaconda3/envs/mc_docker/lib/python3.8/site-packages/torch/autograd/__init__.py:130: UserWarning: CUDA initialization: CUDA unknown error - this may be due to an incorrectly set up environment, e.g. changing env variable CUDA_VISIBLE_DEVICES after program start. Setting the available devices to be zero. (Triggered internally at  /pytorch/c10/cuda/CUDAFunctions.cpp:100.)\n",
      "  Variable._execution_engine.run_backward(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/5], Step[100/600], Loss: 0.5035\n",
      "Epoch [1/5], Step[200/600], Loss: 0.3764\n",
      "Epoch [1/5], Step[300/600], Loss: 0.2464\n",
      "Epoch [1/5], Step[400/600], Loss: 0.1450\n",
      "Epoch [1/5], Step[500/600], Loss: 0.1893\n",
      "Epoch [1/5], Step[600/600], Loss: 0.2311\n",
      "Epoch [2/5], Step[100/600], Loss: 0.0959\n",
      "Epoch [2/5], Step[200/600], Loss: 0.0693\n",
      "Epoch [2/5], Step[300/600], Loss: 0.1453\n",
      "Epoch [2/5], Step[400/600], Loss: 0.2012\n",
      "Epoch [2/5], Step[500/600], Loss: 0.0913\n",
      "Epoch [2/5], Step[600/600], Loss: 0.0338\n",
      "Epoch [3/5], Step[100/600], Loss: 0.0551\n",
      "Epoch [3/5], Step[200/600], Loss: 0.0283\n",
      "Epoch [3/5], Step[300/600], Loss: 0.0242\n",
      "Epoch [3/5], Step[400/600], Loss: 0.0421\n",
      "Epoch [3/5], Step[500/600], Loss: 0.0458\n",
      "Epoch [3/5], Step[600/600], Loss: 0.0774\n",
      "Epoch [4/5], Step[100/600], Loss: 0.0459\n",
      "Epoch [4/5], Step[200/600], Loss: 0.0030\n",
      "Epoch [4/5], Step[300/600], Loss: 0.0155\n",
      "Epoch [4/5], Step[400/600], Loss: 0.0688\n",
      "Epoch [4/5], Step[500/600], Loss: 0.0411\n",
      "Epoch [4/5], Step[600/600], Loss: 0.0441\n",
      "Epoch [5/5], Step[100/600], Loss: 0.0485\n",
      "Epoch [5/5], Step[200/600], Loss: 0.0513\n",
      "Epoch [5/5], Step[300/600], Loss: 0.0044\n",
      "Epoch [5/5], Step[400/600], Loss: 0.0058\n",
      "Epoch [5/5], Step[500/600], Loss: 0.0054\n",
      "Epoch [5/5], Step[600/600], Loss: 0.0188\n"
     ]
    }
   ],
   "source": [
    "n_total_steps = len(train_loader)\n",
    "model_t_list = [model_i]\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (images, labels) in enumerate(train_loader):  \n",
    "        images = images.reshape(-1, 28*28).to(device)\n",
    "        labels = labels.to(device)\n",
    "        # Forward pass\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "        # Backward and optimize\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        if (i+1)%20 == 0:\n",
    "            model_t_list.append(copy.deepcopy(model))\n",
    "        if (i+1) % 100 == 0:\n",
    "            print (f'Epoch [{epoch+1}/{num_epochs}], Step[{i+1}/{n_total_steps}], Loss: {loss.item():.4f}') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "instructional-toner",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_f = copy.deepcopy(model)\n",
    "model_t_list.append(model_f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "armed-location",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_model_loss(model, data_loader):\n",
    "    with torch.no_grad():\n",
    "        loss = 0\n",
    "        n_iters = 0\n",
    "        for images, labels in data_loader:\n",
    "            images = images.reshape(-1, 28*28).to(device)\n",
    "            labels = labels.to(device)\n",
    "            outputs = model(images)\n",
    "            curr_loss = criterion(outputs, labels).item()\n",
    "            loss+=curr_loss*images.size(0)\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            n_iters += 1\n",
    "        loss = loss/len(data_loader.sampler)\n",
    "#         print(f'Loss of the network: {loss} ') \n",
    "    return loss\n",
    "# loss = evaluate_model_loss(model, train_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "quick-destination",
   "metadata": {},
   "source": [
    "### Linear interploate between initialized and converged models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "comic-minimum",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "def linear_interpolation(model_i,model_f,alpha,data_loader):\n",
    "    curr_model = NeuralNet(input_size, hidden_sizes, num_classes).to(device)\n",
    "    state_dict = curr_model.state_dict()\n",
    "    i_state_dict = model_i.state_dict()\n",
    "    f_state_dict = model_f.state_dict()\n",
    "    for (_,i_param),(_,f_param),(name,param) in zip(i_state_dict.items(), f_state_dict.items(), state_dict.items()):\n",
    "        state_dict[name].copy_((1-alpha)*i_param+alpha*f_param)\n",
    "    train_loss = evaluate_model_loss(curr_model, data_loader)\n",
    "    return train_loss\n",
    "\n",
    "x=[]\n",
    "y_train=[]\n",
    "y_test=[]\n",
    "for i in np.linspace(0,2,10):\n",
    "    x.append(i)\n",
    "    y_train.append(linear_interpolation(model_i,model_f,i,train_loader))\n",
    "    y_test.append(linear_interpolation(model_i,model_f,i,test_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "productive-wilson",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Linear interpolation of ReLU on MNIST')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEWCAYAAACT7WsrAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA1oElEQVR4nO3deXwU9f3H8dcnm4scQLjvOyAQINweHOJBFRXBo4oi3sfPHtZWq61Ha6utvdTaaq1a2qpUURRLFW2r9QAVFRDkVE4JJhAIV+7z8/tjJmETNmED2Z1N9vN8PPaR2ZnvzL53Gb6fnWNnRFUxxhhjAonxOoAxxpjIZUXCGGNMvaxIGGOMqZcVCWOMMfWyImGMMaZeViSMMcbUy4pEMyciE0XkC69zAIjIj0Xkaa9zBENEfioizx3H/OtE5NSmSxTUa4qI/FVE9ovIJ+F8bRO9rEg0EyKyXUTOqDteVZeo6iAvMtWlqr9Q1euCaXu8nXQ4icjfROR+/3GqOlRV3w1zlAnAmUAPVR1Xd6KIXCUilSJSICKHRGS1iJwbzIJF5FQR2VnPMpcGGB9wfWxqIqIisltEYv3GxYpIroio37h3RaRERHr6jTtDRLYHyiwi8SLyOxHZ6X5e20TkYXdagd+jSkSK/Z5fHur3HGmsSJhj4v+fNhpf3yO9ge2qWthAm49UNQVoCzwOvCAibcOQLZQOAGf7PZ8G7A/QrhC4J8hl/ggYA4wDUoEpwGcAqppS/QB2AOf5jZt3bG+h+bIi0czV/Qboflu6TUQ+F5GDIjJfRBL9pp8rIqtE5ICIfCgiw/2m3SkiW0QkX0TWi8hMv2lXicgHIvKwiOwDfhogS83WgYj0cb8FXikiO0Rkr4jc5U47C/gxcIn77Wy1O76NiPxFRHJE5GsRuV9EfPW9vt+4P7jvdaOInO6Xp5uILBKRfSKyWUSub+BzfElEdrnLeV9EhrrjbwAuB37oZv2X3+dc/a00QUQeEZFs9/GIiCT4//uIyA/cb785InJ1AzkCZhaRa4GngZPcHPfVtwwAVa0CngWSgXS/nL91/z12i8gTItKqoeUci4Y+d3cdeVFEnnHXs3UiMuYoi3wWmOP3fA7wTIB2jwKzRGRAEDHHAgtVNVsd21U10DKjnhWJlumbwFlAX2A4cBWAiIwC5gI3Au2BPwOLqjs0YAswEWgD3Ac8JyJd/ZY7HtgKdAIeCDLLBGAQcDpwr4gMVtU3gV8A891vZyPctn8HKoABwEhgKuC/+yrQ61eP6wD8BHhFRNq5054HdgLdgIuAX/gXkTrewOlMOwErgXkAqvqkO/xrN+t5Aea9CzgRyARG4Hw7vdtvehecz7Q7cC3wmIik1ZMjYGZV/QtwE+6Wgqr+pJ75AXCL69VAOfCVO/pXwEA35wA3z70NLecYHe1znw68gLO1swj441GW9yowSUTaultFE4F/Bmj3NfAUAb7ABLAM+L6I3Cwiw0REgpgnKlmRaJkedb8h7QP+hdMpAFwP/FlVP1bVSlX9O1CK08Ghqi+581Wp6nxgE06HVy1bVf+gqhWqWhxklvtUtVhVVwOrcTrRI4hIZ5xdCt9T1UJVzQUeBi49yuvnAo+oarmb+QvgHHH2TU8A7lDVElVdhfNN/IpAr6+qc1U1X1VLcTqZESLSJsj3eDnwM1XNVdU9OAXW/3XK3enlqroYKMApnHU/g0ZlrseJInIAKAF+C8xW1Vy3E7weuFVV96lqPk6hvrT+RTVekO9hqaouVtVKnK2EgOuEnxKc9fgSN+8id1wgvwTOq94SbMAvcYrm5cBy4GsRufIo80QlKxIt0y6/4SIgxR3uDfzA3dV0wO1MeuJ840NE5vjtijoAZOB8Q6+W1YRZ6uoNxAE5fq//Z5xv9g29/tda+yqVX+G8n25AdWfoP6173QWIiE9EHnR3tR0CtruTOtRtW49uHP627p+hWp6qVvg9r+9zCDpzA5apalsgDaczneiO7wgkASv8Pt833fENqcD5d6krDqf41RXMe6i7TiTK0Y8xPYOzm6m+XU0AuEX6j8DPGlqY+yXpMVU9BWeL5gFgrogMPkqOqGNFIrpkAQ+oalu/R5KqPi8ivXE21b8NtHc7mrWA/2Z4U14yuO6ysnC2ajr4ZWutqkMbmAege51dBb2AbPfRTkRS60z7OsAyLgPOB87A2S3Uxx1fvdyjve9snCJXN0NjNSZzg1S1ALgZuEJERgJ7gWJgqN/n28Y9ONuQHUAv/89YRJJwivdXAdo32XuoYwnQFegMHHG2VR2/wTkQPTqYBbtbuo/hHAwfcjwhWyIrEs1LnIgk+j0ae4bPU8BNIjJeHMkico77HzoZpzPcA+AeXM1o2vi17Ab6iEgMgKrmAP8BficirUUkRkT6i8jkoyynE/BdEYkTkYuBwcBiVc0CPgR+6X5Ww3GOBwQ6OyUVp0Dl4Xzb/kWArP0ayPA8cLeIdBSRDjj7+Rt9em8jMwezvDycXT33ugeynwIeFpFOACLSXUS+4T9PnfUrEfgYZ9fOne64ZOBBnF00RxSJpn4PfstV4Dxgep0tx0BtDwC/A35YXxsR+Z57UkErcU6pvRJnPfjseHK2RFYkmpfFON8Gqx8/bczMqrocZ7/0H3G+NW3GPaitqutx/mN9hNMpDgM+aJrYAb3k/s0TkZXu8BwgHljv5luA8+2xIR/jHHDei7PL4CK3cwSYhbNVkA0sBH6iqv8NsIxncDq8r93XXlZn+l+AIe5umlcDzH8/Tqf5ObAG58D3/QHaBSPYzMF6BJjmdtZ34PybL3N3q71F7WMj3am9fhXj7I48BzgV52D0VpxdSt9soLNu6vcAgKquU9V1QTb/PVDZwPRinPV9F8668y3gQlXdenwpWx6xmw6Z5kpErgKuU9UJXmcxpqWyLQljjDH1siJhjDGmXra7yRhjTL1sS8IYY0y9WtRF0jp06KB9+vTxOoYxxjQrK1as2KuqAX9Y2aKKRJ8+fVi+fLnXMYwxplkRkUA/jARsd5MxxpgGWJEwxhhTLysSxhhj6tWijkkYYyJXeXk5O3fupKSkvqt8m1BLTEykR48exMUFurBvYFYkjDFhsXPnTlJTU+nTpw92j5/wU1Xy8vLYuXMnffv2DXo+291kjAmLkpIS2rdvbwXCIyJC+/btG70lZ0XCGBM2ViC8dSyfvxUJ4OC+PSx7/Ab278nxOooxxkQUKxLAvpytjN69gE3zvu91FGNMhDj55JO9jhARrEgAfYeOZ0X3yxl3YDFrl77mdRxjTAT48MMPvY4QEaxIuDJn/4Js6Uzrt2+npLjI6zjGGBMRrEi4EpNS2T/lQXppNivm3et1HGOMiQhWJPwMnXQBK1ufwdisv7Jtg90P3ZiWKCUlpWa4uLiYyZMnU1np3A47KyuL6dOnk56eTv/+/bnlllsoKysDoKysjEmTJlFRUdGkeQ4cOMDjjz9+zPOH+tiJFYk6+l7+CCWSQOEr36GqssrrOMaYEJo7dy4XXHABPp8PVWXmzJnMnDmTTZs28eWXX1JQUMBdd90FQHx8PKeffjrz589v0gwNFQlVpaqq4X4o1MdOrEjUkda5J5uH30ZG+Ro+WvgHr+MYY0Jo3rx5nH/++QC8/fbbJCUlcfXVVwPg8/l4+OGHmTt3LkVFznHKGTNmMG/evIDLeuihh8jIyCAjI4NHHnkEgO3btzN48GCuv/56hg4dytSpUykuLq4135133smWLVvIzMzk9ttvr5nn5ptvZtSoUWRlZTFjxgxGjx7N0KFDefLJJ2vNn5KSEtTrHDNVbTGP0aNHa1OoqqzQjQ+cpPvv7aa7srOaZJnGRLv169d7HUFVVZOTk1VVtbS0VDt37lwz/ve//71+73vf07PPPltHjBihI0aM0HvuuUczMzN19erVqqpaUVGhHTp0OGKZy5cv14yMDC0oKND8/HwdMmSIrly5Urdt26Y+n08/++wzVVW9+OKL9dlnn60177Zt23To0KG1nouIfvTRRzXj8vLyVFW1qKhIhw4dqnv37q31foJ5nWqB/h2A5VpPv2rXbgpAYnykXvRHkuedwYZ536PzbQu8jmRMi3Lfv9axPvtQky5zSLfW/OS8oUG337t3L23btq15rqqICIsXL67VbtGiRTW/VPb5fMTHx5Ofn09qampNm6VLlzJz5kySk5MBuOCCC1iyZAnTp0+nb9++ZGZmAjB69Gi2b99+1Gy9e/fmxBNPrHn+6KOPsnDhQsA5brJp0ybat29fa55jeZ1g2O6menQbOIrPe1/JSQX/5dN3FnodxxjTxFq1alXrOkYZGRlH3Nny0KFDZGVl0b9//5pxpaWlJCYm1mrnfBkPLCEhoWbY5/MFdeC7utgAvPvuu7z11lt89NFHrF69mpEjRwa8/tKxvE4wbEuiAcMvu5/sX79B5/d+RP7YM0hNST36TMaYo2rMN/5QSUtLo7KykpKSEhITEznttNO48847eeaZZ5gzZw6VlZXceuutXHPNNSQlJQGQl5dHx44dj7jU9qRJk7jqqqu48847UVUWLlzIs88+G1SO1NRU8vPz651+8OBB0tLSSEpKYuPGjSxbtuzY3/QxsC2JBsQlJlM09Tf0IocVz93tdRxjzHGqqKio9Y176tSpLF26FHAufrdw4UIWLFhAeno66enpJCcn88ADD9S0f+edd5g2bdoRyx01ahRXXXUV48aNY/z48Vx33XWMHDkyqEzt27fnlFNOISMjg9tvv/2I6WeddRYVFRUMHz6ce+65p9ZuqLCo72BFc3w01YHrulY/cpGW3pum61Z/EpLlGxMNIuHA9apVq3Ts2LE1z1euXKmzZ88Oev6ZM2fqxo0bQxEtbBp74Nq2JILQ/4pHKZFWVP7zFsqb+Ic0xpjweOKJJ5g1axb3339/zbiRI0cyZcqUmh/TNaSsrIwZM2YwaNCgUMaMOFYkgpDcritZY37EsMp1LHnxEa/jGGOOwU033cT69euZOnVqrfHXXHMNPp/vqPPHx8czZ86cUMWLWFYkgjR02s1sShzGqC8eYseOr7yOY4wxYWFFIlgxMaRd8jjJlLDjhVsbPOXNGGNaCisSjdCh73A2DLiOCUVvs/TfL3kdxxhjQs6KRCNlXHIfOb5u9F52N3n7D3gdxxhjQsqKRCPFxLeiYtpD9GI3K+23E8aYFs6KxDHoOfps1nWcxql7/8GKTz/wOo4xxoSMFYlj1P/yRyiSJOLe+AHFpeVexzHGmJCwInGMEtt2Zs9JdzO8agPvvfA7r+MYY0xIWJE4DgOm3siWpExO2vooX27Z7HUcY0wTCvVtQZsLT4qEiPQUkXdEZIOIrBORWwK0ERF5VEQ2i8jnIjLKi6wNEqHjrMdpJaXkvPh9KqvstxPGtBShvi1oc+HVlkQF8ANVHQycCHxLRIbUaXM2kO4+bgD+FN6IwWndcyhbB93A5NL3ePtf//A6jjHGNClPioSq5qjqSnc4H9gAdK/T7HzgGfcihcuAtiLSNcxRgzLoop+QE9uDwSt/Ss6ePK/jGGNMk/H8mISI9AFGAh/XmdQdyPJ7vpMjCwkicoOILBeR5Xv27AlZzoZIXCK+6b+np+Ty2XN32SU7jIlgKSkpNcPFxcVMnjy55iqwWVlZTJ8+nfT0dPr3788tt9xCWVkZ4FwFdtKkSU12x7dqBw4c4PHHH/ds/qPxtEiISArwMvA9Va17w1sJMMsRva+qPqmqY1R1TMeOHUMRMyidhp/BF13O48wDL/LBh+95lsMYE7y5c+dywQUX4PP5UFVmzpzJzJkz2bRpE19++SUFBQXcddddgHMV2NNPP5358+c3aQYrEvUQkTicAjFPVV8J0GQn0NPveQ8gOxzZjlX/yx+mKCaZ1m/9kEPFpV7HMcYcxbx58zj//PMBePvtt0lKSuLqq68GnPtEP/zww8ydO5eioiIAZsyYwbx58wIu66GHHiIjI4OMjAweeeQRALZv387gwYO5/vrrGTp0KFOnTqW4uLjWfHfeeSdbtmwhMzOz5s50zz33HOPGjSMzM5Mbb7yxZkunsLCQc845hxEjRpCRkcH8+fMDzt+UPLnHtYgI8Bdgg6o+VE+zRcC3ReQFYDxwUFVzwpXxWMSmduTgpJ8y/L3v8895v+b86+7xOpIxkemNO2HXmqZdZpdhcPaDQTcvKytj69at9OnTB4D169czevRopk2bRna28310+vTp9OrVi82bNzN8+HAyMjL49NNPj1jWihUr+Otf/8rHH3+MqjJ+/HgmT55MWloamzZt4vnnn+epp57im9/8Ji+//DKzZ8+umffBBx9k7dq1rFq1CoANGzYwf/58PvjgA+Li4rj55puZN28ec+bM4c0336Rbt268/vrrgHP/6/Hjx9eav6l5UiSAU4ArgDUissod92OgF4CqPgEsBqYBm4Ei4Orwx2y8Xqdew7aV85iS9Tir11/MiCEneB3JGBPA3r17adu2bc1zVUVEWLx4ca12ixYtwvle62xdxMfHk5+fT2pqak2bpUuXMnPmTJKTkwG44IILWLJkCdOnT6dv375kZmYCMHr0aLZv395grrfffpsVK1YwduxYwDlu0qlTJwCGDRvGbbfdxh133MG5557LxIkT2b9///F8DEflSZFQ1aUEPubg30aBb4UnURMSofOsx/E9OYF9r9xG2cBFxMd6fn6AMZGlEd/4Q6VVq1aUlJTUPM/IyODll1+u1ebQoUNkZWXRv3//mnGlpaUkJibWatfQySoJCQk1wz6f74jdTXWpKldeeSW//OUvj5g2cOBAVqxYweLFi/nRj37E1KlTQ363POu9QiCp2wl8nXEzUyqW8Oarz3gdxxgTQFpaGpWVlTWF4rTTTqO4uJhnnnH+z1ZWVnLrrbdyzTXXkJSUBEBeXh4dO3YkLi6u1rImTZrEq6++SlFREYWFhSxcuJCJEycGlSM1NZX8/Pya56effjoLFiwgNzcXgH379vHVV87dMLOzs0lKSmL27NncdtttrFy58oj5m5oViRDpN+MudsX3YtSan7MtO9frOMYYoKKiotY3+6lTp7J06VIARISFCxeyYMEC0tPTSU9PJzk5mQceeKCm/TvvvMO0adOOWO6oUaO46qqrGDduHOPHj+e6665j5MiRQWVq3749p5xyChkZGdx+++0MGTKE+++/n6lTpzJ8+HDOPPNMcnKcw7Fr1qypOaD9wAMPcPfddx8xf5NT1RbzGD16tEaSfev/p/qT1vrP31yrVVVVXscxxlPr16/3OoKuWrVKx44dW/N85cqVOnv27KDnnzlzpm7cuDEU0cIm0L8DsFzr6VdtSyKE0gZPYXOPC5iW/zL/eectr+MYE9WeeOIJZs2axf33318zbuTIkUyZMqXmFNOGlJWVMWPGDAYNGhTKmBFHtAX9OnjMmDG6fPlyr2PUUlW4j/zfZfJVVSe6ff99OrRO8jqSMZ7YsGEDgwcP9jpG1Av07yAiK1R1TKD2tiURYjHJ7Sie8jOGs4l35v3K6zjGGNMoViTCoMuEK/mqzTi+sevPfLRqrddxjDEmaFYkwkGELpc/TqJUULLoNorKmvYCYcYYEypWJMIkoVM6uzO/y5Sqj3j9pb96HccYT7SkY6DN0bF8/lYkwqjnuXeyO6EPJ3/5IOu2R/RlqIxpcomJieTl5Vmh8IiqkpeXd8SvxY/Gq2s3RafYeJIv/CMp/ziXT+b/iBNu/yu+mAavTmJMi9GjRw927tyJV/d9MU6h7tGjR6PmsSIRZikDJ/JVn4s5b9vL/OvNN5kx7WyvIxkTFnFxcfTt29frGKaRbHeTB3p98zcU+lrT7+O7+HpfgddxjDGmXlYkPCBJaVRM/QXDZQvvPvcL20drjIlYViQ80n78ZexsdxLT8+byv49XeR3HGGMCsiLhFRG6XPY48VKJ/PsODhaVe53IGGOOYEXCQ7Ed+rF/7K2cph/zz/lPex3HGGOOYEXCY13Oup3cVv04Y/uvWbFph9dxjDGmFisSXvPF0frix+gm+9j20t2UVhz9ksXGGBMuViQiQGK/k/l6wCxmli7ildde9zqOMcbUsCIRIbpf+CCFsW0Z9tm95Oy3304YYyKDFYlI0aotpVPuI0O28dm7C71OY4wxgBWJiNJx/CUUSDIJG1/xOooxxgBWJCJLXCJfdz2T8SUfsjXHLoJmjPGeFYkI0+nky0mREta/+5LXUYwxxopEpEkbcjr7Y9rRZvOrdk0nY4znrEhEmhgfub2mMa5iBeu32o/rjDHesiIRgbpPnEOCVLDt/ee9jmKMiXJWJCJQSr9x7I7tTucdr1FZZbucjDHesSIRiUQ4MOB8Rlet5bN1G7xOY4yJYlYkIlTvSXOIEWXXh/O8jmKMiWJWJCJUYrfBZCUMpE/OG5RVVHkdxxgTpaxIRLDSwReSwRaWr/jE6yjGmChlRSKC9Z48myqEA5/YWU7GGG94UiREZK6I5IrI2nqmnyoiB0Vklfu4N9wZI0FcWg+2pYxk8N5/U1hitzc1xoSfV1sSfwPOOkqbJaqa6T5+FoZMESlm2MX0lRw+XfaO11GMMVHIkyKhqu8D+7x47eam94RZlBNL6cr5XkcxxkShSD4mcZKIrBaRN0RkaH2NROQGEVkuIsv37Gl5V06NSU5jW9uTGXHwbfbnF3sdxxgTZSK1SKwEeqvqCOAPwKv1NVTVJ1V1jKqO6dixY7jyhVWr0ZfQRfazYsm/vI5ijIkyEVkkVPWQqha4w4uBOBHp4HEsz/QYP5MiEolZs8DrKMaYKBORRUJEuoiIuMPjcHLmeZvKOxKfzFedTmNM0RKy9x7wOo4xJop4dQrs88BHwCAR2Ski14rITSJyk9vkImCtiKwGHgUu1Si/uUK78ZfTWopY855tTRhjwifWixdV1VlHmf5H4I9hitMsdM78Bgdea0PSFwuB67yOY4yJEhG5u8kE4Isju/tZjC39mK1f53idxhgTJaxINCNdJlxBopTz5bsveB3FGBMlrEg0I+0GTSDX15l2W/9p9782xoSFFYnmRIS9fc5jVMVqNmza7HUaY0wUsCLRzPScfBWxUsWOpf/wOooxJgpYkWhmUnsNIyu+H92yXrf7XxtjQs6KRDOUnz6D4foFqz9f5XUUY0wLZ0WiGep36pUA7Flm9782xoSWFYlmKLFjH7a0GsaAXW9QVl7pdRxjTAtmRaKZqhhyIf3ZycpPl3odxRjTglmRaKb6nTqbcnzkL7f7XxtjQseKRDMVl9qRranjGJr3XwpLyryOY4xpoaxINGOxmd+km+xl5Qdveh3FGNNCWZFoxvqecjElxFOxyu5/bYwJDSsSzVhMYipb200m89C77D9U4HUcY0wLdMxFQkSSRcTXlGFM4yWPvZQ0KWDVewu9jmKMaYGCLhIiEiMil4nI6yKSC2wEckRknYj8RkTSQxfT1KfX2PM4RApx6172OooxpgVqzJbEO0B/4EdAF1XtqaqdgInAMuBBEZkdgoymARKbwFddzmRU8Yfs2rvX6zjGmBamMUXiDFX9uap+rqpV1SNVdZ+qvqyqFwJ2BNUDHU6aTZKUsv5/9vEbY5pW0EVCVcsBRCRRRDJEZKiIJAZqY8Kr67DT2BvTgZRNdlzCGNO0GnNMIlZEfg3sBP4OPAdkiciDIhIbqoAmCDExZPc8h5FlK9mWtcPrNMaYFqQxu5t+A6QBfYHXVHUkzjGKDsBvQ5DNNEL3iXOIk0q2vGs3IzLGNJ16i4SI1L1y3LnADaqaD5wHoKqHgBvdacZD7fuPZmdsLzpuW2T3vzbGNJl6i4SqTjhyVE3vI34jK4EqjLdE2NdvOiOq1vHFFxu9TmOMaSEas7tpg4jMcYezq0e6p71uaNJU5pj0nezcjGjn0mc9TmKMaSkaUyS+BXxHRN4FNorIb0XkPeA7wP+FIpxpnNTuA9maMJieXy+2+18bY5pEY06B3amqY4GfA9uBHcB9qjpeVbMbnNmETdGgmQzSbaz57GOvoxhjWoDGnAIrAKr6tqr+QVUfVdX/BWpjvNP/1CuoVGHfx3aWkzHm+DXqshwi8h0R6eU/UkTiReQ0Efk7cGXTxjON1apdNzYljyY99027/7Ux5rg1pkicBVQCz4tItoisF5FtwCZgFvCwqv4tBBlNI2nGRfRkN6uWve11FGNMM9eYYxIlqvq4qp4C9AZOB0aqam9VvV5VV4UqpGmcAZMvpZQ4ila84HUUY0wzd0z3k1DVclXNUdUDTZzHNIG45DQ2tTmFjP1vUVhc4nUcY0wz1pgD1/kicqiexx4RWSYip4cyrAlewshL6CAHWb1kkddRjDHNWGN2N6WqautAD6ALzuU5fh+ypKZR+p88k3yS0NUveR3FGNOMNck9rlW1UlVXA38Ipr2IzBWRXBFZW890EZFHRWSziHwuIqOaImc0iYlvxdYOpzOiYAn7Dxz0Oo4xpplqkiJRTVX/HGTTv+GcLVWfs4F093ED8KfjSxad2oyfRYoUs+bdF72OYoxpppq0SARLVd8H9jXQ5HzgGXUsA9qKSNfwpGs5eo/6BnmSRuKGV7yOYoxppjwpEkHoDmT5Pd/pjjONIL5YdnQ9ixEln7Br9y6v4xhjmqFILRKBLu8R8Ip1InKDiCwXkeV79uwJcazmp/MpV5AgFXzxjl2mwxjTeJFaJHYCPf2e98Dv8uT+VPVJVR2jqmM6duwYlnDNSbchJ5Md0402W171OooxphmK1CKxCJjjnuV0InBQVXO8DtUsibCr97kML/uc7du3eJ3GGNPMeFIkROR54CNgkIjsFJFrReQmEbnJbbIY2ApsBp4CbvYiZ0vRa/IcYkTZ/p7djMgY0zixXryoqs46ynTFucmRaQId+gxjW9wAOn/1Gqo/wa7obowJVqTubjJN7ED/GQyu2sSX61d7HcUY04xYkYgS/afMoUqFXR/YLidjTPCsSESJ1p1782Wr4fTOXkxlZZXXcYwxzYQViShSesIF9CGbdSuWeB3FGNNMWJGIIgOnzKZcfRz4xH5YZ4wJjhWJKNKqTQc2pp7ICXv/TVlZuddxjDHNgBWJKCPDv0kn9rPmw8VeRzHGNANWJKLMoEkXUUgipZ/N9zqKMaYZsCIRZeISU/gybTJDD7xLYWGh13GMMRHOikQUSho9izZSyNr3X/Y6ijEmwlmRiELp489hH62RNQu8jmKMiXBWJKJQTFw82zpNZXjhh+zf39ANAo0x0c6KRJRqe+JlJEo5G+xmRMaYBliRiFL9MqeQI51o9cVCr6MYYyKYFYkoJTEx7OxxDsNKVrI7Z4fXcYwxEcqKRBTrNuEKYqWKze/M8zqKMSZCWZGIYt0HjWa7rw9pW//pdRRjTISyIhHlcvucx5CKDXy1Zb3XUYwxEciKRJTre+ocALLet5sRGWOOZEUiynXsOZAv4ofSLet1nFuLG2PMYVYkDPnpM+hX9RWb1nzidRRjTISxImEYOGUO5epjz4e2y8kYU5sVCUPrDl3YkDSGvrvepLKy0us4xpgIYkXCAFAx9EK6sYcNn7zldRRjTASxImEAGHzqpRRrPAXL7VpOxpjDrEgYAFqltGFD6wkMynubstJSr+MYYyKEFQlTIzbzm6SRz7qlr3odxRgTIaxImBqDJ87kIMlUrHrR6yjGmAhhRcLUiItP5It2pzPk0BIK8w96HccYEwGsSJhaUsfOIllK2fCebU0YY6xImDoGjfsGubQjdp3d/9oYY0XC1BHj87G127kML/qYz9+y+0wYE+2sSJgjjLj8F2yKHcCAJbeyY90yr+MYYzxkRcIcoVVyKq2vWcAhSaHVgss4lGu3NzUmWlmRMAF17d6HPec+Q1JVAXlPX0BlSYHXkYwxHvCsSIjIWSLyhYhsFpE7A0w/VUQOisgq93GvFzmj2bAxE/hk9G/oXbqZzU9eDlVVXkcyxoSZJ0VCRHzAY8DZwBBglogMCdB0iapmuo+fhTWkAeDU8+awuNu3GbTvXTa98EOv4xhjwsyrLYlxwGZV3aqqZcALwPkeZTENEBGmXnMf/2l1NulfPsXX7zztdSRjTBh5VSS6A1l+z3e64+o6SURWi8gbIjI00IJE5AYRWS4iy/fs2ROKrFEvPs5H5o1P8akMp9N7d3BwwzteRzLGhIlXRUICjKt7g+WVQG9VHQH8AXg10IJU9UlVHaOqYzp27Ni0KU2NTm1TSZr9HFnaiZgXr6A8d5PXkYwxYeBVkdgJ9PR73gPI9m+gqodUtcAdXgzEiUiH8EU0dQ3t35vNZ/6F8irl4F9mQvF+ryMZY0LMqyLxKZAuIn1FJB64FFjk30BEuoiIuMPjcLLmhT2pqWXqhJN5bfCvaV2Sze6nL4HKcq8jGWNCyJMioaoVwLeBfwMbgBdVdZ2I3CQiN7nNLgLWishq4FHgUlWtu0vKeOCyi2cxt/336Zz3MXvmfxvsn8WYFktaUr87ZswYXb58udcxosKBojIWPfR/zKlYwKFJP6X1abd6HckYc4xEZIWqjgk0zX5xbY5J26R4TrzuIf6t40l5/z7K1r3mdSRjTAhYkTDHbGCXNsRc8GfWVPVFF1yH5qz2OpIxpolZkTDH5cwRffn4xMfIq2pF0d8ugkM5XkcyxjQhKxLmuF131kk83eNBKDlIwd8ugrIiryMZY5qIFQlz3GJihO/PuZBfJd9O0r51FM2/1i4GaEwLYUXCNImUhFiuvfZmHuIKkrYspuy/93kdyZiWTxX2boblc2HtKyF5idiQLNVEpd7tkxl/2T3849mdXPbRI1R1TCdm1GyvYxnTcqjC/u2wfQlsW+L8zXePAw48GzIuaPKXtCJhmtTEgZ34yxkPsvStqznpX7dAu77Q5xSvYxnTfB3Iql0UDrrXRk3uCH0mQt+J0GcStO8fkpe3ImGa3DWTBnLX17+i68br6fmPy4i/8X8hW4GNaXEO5bhF4X3n7/7tzvhW7aDPBDjlFqc4dBwEEuhaqU3LfnFtQqKkvJLvPv4Kv9p3K8ltOxJ/0/+gVZrXsYyJPAW5tbcU8jY74xPbQO8J7pbCROg0BGJCcxi5oV9c25aECYnEOB/3XXUuP3w0m8cP/oyK568g9sqF4IvzOpox3irMg6+WHi4KezY64+NToffJMPoqpyh0GQYxPk+jghUJE0Jd27Tixiuu4K6ns/n1jieoev0HxJz3+7BsIhsTMYr3w1cfHi4Ku9c64+OSodeJMOJS55hC1xHgi7wuOfISmRZlTJ92bJr+LR5blM23Vv4dOgyEk7/tdSxjQqfkEOz46PAxhZzPAYXYROg5Hk672ykK3Uc1iy1rKxIm5GaN68U9X9/G6yt3Me0/dyPt+sEJ07yOZUzTKCt0i4K7pZC9CrQSfPHQYyxMvgP6ToIeYyA2weu0jWZFwoTFPecN45pdP6bXru8zZMG1+K79N3Qd7nUsYxqnogxy10POKqcY5KyCXWugqgJiYqH7aJhwq3Owued4iGvlceDjZ2c3mbDZW1DK1Y8u4umyO+iQkoDvxncgtYvXsYwJrKIUdq+rXRB2r4cq926MCW2cLzrdR7tF4URISPEw8LGzs5tMROiQksAvr5zKjU/s5YXCnyL/uJSYqxdDfJLX0Uy0Ky92CkDOZ4cLQu4GZwsBILEtdMuEk26GrpnOcFrfqDgJw4qECauM7m24+sLpfOvFPTyV8xAsvBEu/nvIzv825ghlRUduIeRucI4jgPOjtW6ZcPIZhwtC295RURACsSJhwu78zO6sz7mYB5bu4p4N8+B/P4czfuJ1LNMSlRXCrrW1C8KeLw4XhKQOThEY+I3DBaFNz6gtCIFYkTCe+OE3TuCa7Ct5fvsuZi19CDqkQ+ZlXscyzVlpgXMQ2b8g7P0S1L1sfXInpwiccK7zm4RumdC6uxWEo7AiYTzhixEenTWKCx+7mX5FuYxb9F2kbW+7GKA5OlUo2uf8UrlWQdgEuCfipHRxisCQGc7friMgtasVhGNgZzcZT23OzWfOY/9hvu9eeiQUIte9bRcDNI7SfMjbAvu2OH/zNruPLVBy4HC71G5uIcj0Kwh21lxj2NlNJmIN6JTKfZdMYPaz32cxPyXpH5cg1/3XLgYYLSpKnauc1hSAzZC31flbsKt22zY9nS8QGRdC+wHOLsquIyClkyfRo4UVCeO5M4d0ZuMZk7j6rVt4Xn+J78U5MPuVZnHJAhOEqkrnHgj+BaD6cTDr8DEDcO6R0K4/DDjDKQjtBzh/2/VrET9Ma46sSJiI8O3TBrA+Zwp3bNjDb7c9AYtvg3MfsX3IzYWqc8lr/wKwb+vhv5Vlh9vGpzodf4+xzsXtagpBf2jV1rO3YAKzImEigojw24tHcOGfCnn6wC6uW/E3aJ9uFwOMJJUVUJgLh7L9jhX4HScoKzjc1pfg3JWw/QDn9NL2A5xHu/7O7iEr/s2GFQkTMZITYnlqzhim/6GIgb5cJv7nbkTEuexB217OGSv2o7umV1UJhXuceyXn7/J7uM8L3OcFudScPQQgMc6/S/sB0OskpwBU7yJq0yMi7oVgjp8VCRNRerZL4o+Xj+GGuTewKHU//f/948MTY+KczqdtT2jTy+mg2vZ0Dmi27eWc8x6B1+P3TFUlFO493MnXVwQKc2sfFwBAnOMDqV2cR/UppKldnL/t+kFan2Z5VdPmqqpKyS+pYF9RGfsKyzjg/t1fVMb+onL6tE/ikrG9mvx17X+UiTinDOjAD6ZlMvW1HzNnYAUTOhTRN24fXTSXVkXZyIEdsPmtI89+kRinULTp6RSPtr38hns7BaYldGpVVVCUd+Q3/bpFoGD34V8W+0vqcLjD7zLscCHwLwLJHe3EgRDy7/D3F5Wxv7B2h1/9/EBRudOmsIwDxeVUVgX+yUKcT5g2rKsVCRM9rj6lDzkHi3n+kyz++mUCkAb0Jy0pjvROqQzon8IJ7eMYkpxP//h9tC3bhRzMggNZcGCHcyewNS8d+Q05pUvtrY+6WyXxyaF5Q5XlziUiygqhvMjZf19W6FxHqHo40PjyosPzleY7u4UKdh++8Jy/pPZOB5/S2bkfck3n71cAkjtBbHxo3mOUqq/D9+/gG9vhpyXFO4/kOAZ2TqFtUjztkuJJS44nLSmOtGT3udsmJSHW2TUbAvZjOhPRVJVdh0rYtLuAzbkFbMotYHNuPl/uLuBgcXlNu9TEWNI7pZDeKZX0zikM6JTCgPYJdIvZT8yhnU7hqC4gB93hgzsPX/a5WlL7Olsf7nBMXJ1Ou8DtyAuP7MwDFQP/s3uORmIgPsUpWPHJEJd0+HlKJ6cI+H/rT+3sjGsJW0lhpqoUllWSX1JOfkkF+SXlHCqpqBku8BvOL6lwp7ltS53ph0oqGuzwD3fwcbRLjve0w69PQz+msyJhmiVVZW9BGZty853isbugZnhvweEOOSne5xSM6gLSKYX0zin0SEvCR5Xzrby6gBzcUaeYZEFFSf0hqjvzuKTDHbr/Iy7AuGDGxybY2T9BqKxSCkr9Om3/DryknPzSuuNqDx8qKaewtIJ6+vcaMQKpiXGkJsaSkhBLa3fYecTRulUs7ZITIqbDPxZWJExU2VdY5m515PttgeSz+1BpTZuE2Bj6d3QKRnqnFAa4WyC92yUR63PPoFJ1Dvwe3OEM1xSDFOceGLGJ1pk3oLyyiuLySorLnEdRWeXh5+WVFJVVUFJeZ/wRbWo/rz1v3YPtR4qNkZrO3L9jT008srNPSfDr+P3aJcX7mkVHfzysSBgDHCwuZ7O7u8rZ8nAKyNcHimvaxPmEfh1SDm99dHa2QNKS4hARYgRiRIgRQWKqh52/4jctRgh7x1JZpZRXVlFWWUV5RRXllX7PK6uoqNSjT/Ob7j/Nv21ZRRXF5VVuh11RUwBKyg937CXllZRXNq5vEYFWcT6S4n0kxvlqDSfF+2gV76NVXCyt4mNIio8lMc7ndub+ReBwx986MY6E2JgW38E3hYi8dpOInAX8HvABT6vqg3Wmizt9GlAEXKWqK8Me1LQYbVrFMbp3GqN7174uVGFpBVv2FPgVjnzWZh9k8docjvc7VK2i4g77YvwLSnWB8S9AToGJiTly3soqpazC7cwra3fmofq+FxsjxPliiPMJ8bExxMbE1Oq8kxNiaZ+S4HTkcdWduV9nH+/zmxZ75DT3r3XokcmTIiEiPuAx4ExgJ/CpiCxS1fV+zc4G0t3HeOBP7l9jmlRyQizDe7RleI+2tcaXlFeyZY+ztVHg7rtWVaqqlCqFKlXU/Xv4+eHhmvbV46q0Ttvqdn5tq+qZ123vczvs+Njqjtt5xPuE2JrnTmce5//cHY6tHo49clpcbAxx1QUh1pkWFxNDTIx13NHMqy2JccBmVd0KICIvAOcD/kXifOAZdfaHLRORtiLSVVVzwh/XRKPEOB9Du7VhaLc2XkcxxjNeXeOgO5Dl93ynO66xbRCRG0RkuYgs37NnT5MHNcaYaOZVkQi0/Vp3j2owbVDVJ1V1jKqO6dixY5OEM8YY4/CqSOwEevo97wFkH0MbY4wxIeRVkfgUSBeRviISD1wKLKrTZhEwRxwnAgfteIQxxoSXJweuVbVCRL4N/BvnFNi5qrpORG5ypz8BLMY5/XUzzimwV3uR1Rhjoplnv5NQ1cU4hcB/3BN+wwp8K9y5jDHGHGZ3cDHGGFMvKxLGGGPq1aKu3SQie4CvjmMRHYC9TRSnKVmuxrFcjWO5Gqcl5uqtqgF/Q9CiisTxEpHl9V3kykuWq3EsV+NYrsaJtly2u8kYY0y9rEgYY4yplxWJ2p70OkA9LFfjWK7GsVyNE1W57JiEMcaYetmWhDHGmHpZkTDGGFOvqCgSInKWiHwhIptF5M4A00VEHnWnfy4io4KdN8S5LnfzfC4iH4rICL9p20VkjYisEpEmvbF3ELlOFZGD7muvEpF7g503xLlu98u0VkQqRaSdOy2Un9dcEckVkbX1TPdq/TpaLq/Wr6Pl8mr9Olour9avniLyjohsEJF1InJLgDahW8fUveViS33gXEBwC9APiAdWA0PqtJkGvIFzD4sTgY+DnTfEuU4G0tzhs6tzuc+3Ax08+rxOBV47lnlDmatO+/OA/4X683KXPQkYBaytZ3rY168gc4V9/QoyV9jXr2Byebh+dQVGucOpwJfh7MOiYUui5lapqloGVN8q1V/NrVJVdRnQVkS6BjlvyHKp6oequt99ugznnhqhdjzv2dPPq45ZwPNN9NoNUtX3gX0NNPFi/TpqLo/Wr2A+r/p4+nnVEc71K0dVV7rD+cAGjrxLZ8jWsWgoEsdzq9SgbqEawlz+rsX5plBNgf+IyAoRuaGJMjUm10kislpE3hCRoY2cN5S5EJEk4CzgZb/Rofq8guHF+tVY4Vq/ghXu9StoXq5fItIHGAl8XGdSyNYxzy4VHkbHc6vUoG6heoyCXraITMH5TzzBb/QpqpotIp2A/4rIRvebUDhyrcS51kuBiEwDXgXSg5w3lLmqnQd8oKr+3wpD9XkFw4v1K2hhXr+C4cX61RierF8ikoJTmL6nqofqTg4wS5OsY9GwJXE8t0oN5S1Ug1q2iAwHngbOV9W86vGqmu3+zQUW4mxWhiWXqh5S1QJ3eDEQJyIdgpk3lLn8XEqdXQEh/LyC4cX6FRQP1q+j8mj9aoywr18iEodTIOap6isBmoRuHQvFgZZIeuBsLW0F+nL4wM3QOm3OofZBn0+CnTfEuXrh3Jnv5Drjk4FUv+EPgbPCmKsLh3+IOQ7Y4X52nn5ebrs2OPuVk8Pxefm9Rh/qPxAb9vUryFxhX7+CzBX29SuYXF6tX+57fwZ4pIE2IVvHWvzuJj2OW6XWN28Yc90LtAceFxGACnWu8tgZWOiOiwX+oapvhjHXRcD/iUgFUAxcqs4a6fXnBTAT+I+qFvrNHrLPC0BEnsc5I6eDiOwEfgLE+eUK+/oVZK6wr19B5gr7+hVkLvBg/QJOAa4A1ojIKnfcj3GKfMjXMbsshzHGmHpFwzEJY4wxx8iKhDHGmHpZkTDGGFMvKxLGGGPqZUXCGGNMvaxIGNOE3KuBdjjeNsZECisSxhhj6mVFwphjJCKvuhd0W1f3om4i0kdENorI393r+y9wLwxX7TsistK9B8EJ7jzjxLmvw2fu30FhfUPGBGBFwphjd42qjgbGAN8VkfZ1pg8CnlTV4cAh4Ga/aXtVdRTwJ+A2d9xGYJKqjsT5NfQvQpremCBYkTDm2H1XRFbj3IuhJ86VSv1lqeoH7vBz1L7KavVF2lbgXC8InOsCveTeGe1hYCjGeMyKhDHHQEROBc4ATlLVEcBnQGKdZnWveeP/vNT9W8nhS/b/HHhHVTNwLkddd3nGhJ0VCWOOTRtgv6oWuccUTgzQppeInOQOzwKWBrHMr93hq5okpTHHyYqEMcfmTSBWRD7H2QJYFqDNBuBKt007nOMPDfk18EsR+QDnip3GeM6uAmtMCLi3mXzN3XVkTLNlWxLGGGPqZVsSxhhj6mVbEsYYY+plRcIYY0y9rEgYY4yplxUJY4wx9bIiYYwxpl7/D3aS2WUHGikoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pylab\n",
    "pylab.plot(x, y_train, label=\"J($\\dot{\\Theta}$) on train\")\n",
    "pylab.plot(x, y_test, label=\"J($\\dot{\\Theta}$) on test\")\n",
    "pylab.legend()\n",
    "pylab.xlabel('alpha')\n",
    "pylab.ylabel(r\"J($\\dot{\\Theta}$)\")\n",
    "pylab.title('Linear interpolation of ReLU on MNIST')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "exposed-symphony",
   "metadata": {},
   "source": [
    "### How far is SGD from this monotonic decreasing path from init to final?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "cross-signal",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f78c06028b0>]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAArsklEQVR4nO3deXhU5d3G8e9vspKVJCSQkA1C2HfC7oaKQEVQcUNUXFpq1dbWWovt29f6trS2UmutW21FUVCLiuLCIiLIIothDwRIIBASkpAQkkDIPs/7R0aLNGyZJGeW3+e6vCZzJpO5ewp3Ds8553nEGINSSinPYrM6gFJKqZan5a6UUh5Iy10ppTyQlrtSSnkgLXellPJAvlYHAOjQoYNJTk62OoZSSrmVzZs3lxhjopt6zSXKPTk5mfT0dKtjKKWUWxGRQ2d7TYdllFLKA2m5K6WUB9JyV0opD6TlrpRSHkjLXSmlPJCWu1JKeSAtd6WU8kAucZ27Uq6kqraBkpM1FJ+soeRE42PZqToGxLdnVEoUNptYHVGp89JyV14jI7+cLbnHGdk1im4xIYj8p6Rr6huYvyGXl7/cz9ETNWf9GYmRQXz/0i7ckpZAoJ9PW8RWqlm03JXl8suqqKlrICTQF5sIWUUnySyoaPyvsIIGO/xuch/SkiOb9fM35ZQye9leNh0s/XZbQmQ7QgL8qK1voKbeTnlVHSeq6xnZNYrpo5KJDgkgOjSADiEBdAj1JyTAly/2HOWN9Yf430W7eGFlNvdfnsLUYYla8soliSusxJSWlmZ0+gHvc6K6jj8v3cubG5q+gzoq2J9esWEcKq3kSFk1P7s6lR9d0Q2fCxwWKTtVy99WZPH6VweJDQvkntFdGNMzhg0HjrEmq5gGOwT42gjwtRHo78OEvp24NLXJaTq+ZYxh/f5jPLsii005pUSHBhAf0Y6Kqjqu7tWR+y7tQkxo4EXvC6WaQ0Q2G2PSmnxNy121teq6Br7Yc5Tff7Kbgopq7h6VzID49pyoqae23k5KdDC9Y8OIDg1ARDhRXcevP8jgo+1H6NohmNuHJzJlcDyhgb6ICOVVdRwpq+Jw6SkyCyrYXVBBZsEJ8suqAJg+MolfTuhJkH/L/kN1/f5jvLr2ANV1dnxswpqsYvx8bNw9KpmfXJVKcID+w1i1Li13Zbn6Bjvvbs5jSUYhGw8co6beTmpMCH+6qT+DEyPO+35jDIt3FvLq2gNsyS076/fZBFKiQ+gVG0bvuDBGpUTRP759y/0POYeckkqe/yKb97fkERseyNM3DeCS1A5t8tnKO2m5K0vtyCvj8YU72XWkgq7RwVzRPYbLe0QzsmsU/r4XfzXu7iMVrNx7lAa7wRgIDfQlrn0gndsHkdoxxPIx8M2HjjPz/R0cKKnkyUl9uGNEkqV5lOfScleWKKqo5vkvspm/8RAdQgJ4clIfxvft9J2rVDzVieo6fvL2VlbuLSYhsh2hAX7cd0kXpgyJtzqa8iDnKncdFFQtrqiimpdW7eetTbk02A13jEji0XE9CAv0szpamwkN9ONf04fyzzUH2FNQQdbRk/z83e1kFlQwc0JPfH30/kHVurTcVYupqW/gn6sP8PzKbOoaDDcNjufBMd1IjAqyOpolfGzC/ZenAFDXYGfWp5n8a20O+46e5O9TBxHeznt+2am2p+WuWsTqfcU88dEuckoq+V6/Tswc38trS70pfj42fjupDz06hfK/izK44YV1/HN6GinRIVZHUx5K/22onFJZU8/M93dw15xNALxx7zBenDZEi/0spg5LZP73R1BWVcf1L6xj0bZ87Hbrz3spz6Plrppta+5xrn1uDf9OP8z9l6ew9KeXcln3c98EpGBYl0g+emg0yVHBPPzONia9sJaM/HKrYykPc95yF5E5InJURDKaeO1RETEi0uG0bY+LSLaI7BWRcS0dWFmvpr6BZz7by00vr6euwfD2D0Ywc0JPAnz1NvwLFR8RxKIHR/PXWwdQcqKWKS99xQdb86yOpTzIhRy5vw6MP3OjiCQAY4Hc07b1Bm4D+jje86KI6N94D7LtcBnX/X0tz32RzeQBcSx++FJGdI2yOpZbstmEGwbF88lPLmFgQnt+9u/tTJ+ziR15ZVZHUx7gvOVujFkNlDbx0l+Bx4DTBwwnA+8YY2qMMTlANjCsJYIqaxljeGb5Pm58cR0nqut57e6hPHPrQL3iowV0CAlg3veHM3NCT7bnlTHp+XX84I10MgsqrI6m3FizxtxFZBKQb4zZfsZLnYHDpz3Pc2xr6mfMEJF0EUkvLi5uTgzVRowx/HHJHp5bkcX1gzqz7GeXMaZnjNWxPIqfj437L09hzWNj+NnV3dmw/xgT/raGp5ftwRVuNFTu56LLXUSCgF8D/9vUy01sa/JPpjHmFWNMmjEmLTpaT8K5KmMMf1q6l1dWH+CukUn85eYBXnUzUlsLDfTj4atTWfvLK7klLZ4XVu7nkQXbadAratRFas6RewrQBdguIgeBeGCLiHSi8Ug94bTvjQeOOBtSWcMYw+zP9vLyl/uZNjyRJyf18YqpA1xBeJAff5rSn59encoHW/NZtC3f6kjKzVx0uRtjdhpjYowxycaYZBoLfbAxphD4CLhNRAJEpAuQCmxq0cSqzfx1+T5eWLmfqcMS+N3kvlrsbUxE+MmVqfTtHMZfP99Hbb3d6kjKjVzIpZBvA+uBHiKSJyL3ne17jTG7gAXAbmAp8KAxpqGlwqq28+zn+3jui2xuTUtg1vX9dN1Qi9hswqPX9OBwaRX//jr3/G9QyuG80w8YY6ae5/XkM57PAmY5F0tZ6e8rsnj28yxuGhLPH2/UYrfa5d2jGZYcybOfZ3FZ92iSooKtjqTcgN6hqr5ljOG5FVn8Zfk+bhzUmT9N6a/F7gJEhFk39KXBGG7/50byjp+yOpJyA1ruCgC73fDkx7sbr2Uf1Jmnbx5wwWuVqtaX2jGUefcN50R1Hbe8vJ6tucetjqRcnJa7oq7BziMLtvH6Vwe5d3QXZmuxu6S+ncN56wcjsNmEW/6xnoff2crsZXv5KrtET7aq/6IrMXm5ugY7M95IZ+XeYn4xrgcPXJGiV8W4uPJTdTz58S425pRSWFFNg91gE0iOCub+K1KYPDAOfx8b+WVVxIW306E1D6bL7Kmz+uOSTP7x5QF+d31f7tS1Pt3Oqdp6Vu8rYXdBBV/uPcr2vHKC/H1o386PI+XVPHBFCo+N72l1TNVKtNxVk77YU8S9r6czbXgis27oZ3Uc5SS73bA6q5gVmUcpOVlDRXUdm3JKWfrTy3RREA+l5a7+S35ZFdc+t4a48HYsfGAUgX46eaenKTlZw5jZq+jRMZR53x+u/x97oHOVu55Q9UJ1DXZ+/NYW6hsML0wbrH/pPVSHkABm3dCP9EPH+fHbW6lr0JOu3kTL3cvU1DfwwPwtbMkt44839qNLB70hxpNNGhDH/03uw/LdRfzgjXRO1tRbHUm1ES13L1Jd18CP5m1h+e4inpzUh+sGxFkdSbWBu0Ym89SN/Vi9r5iRf1jBbz7MoLC82upYqpWdd/oB5Rmq6xqY8eZmVu8rZtYNfZk2XK+M8Sa3DUukZ2wYb6w/yL+/Psy7mw/z8FXd+eFlXfVSSQ+lJ1S9QF2DnXte+5p1+0t46sZ+3Do00epIykKHS0/x+093s2xXEaNSovj5Nd0ZnBih9ze4IT2h6uX+viKLtdkl/OnG/lrsioTIIF6+Ywh/ntKfHXnlTHlpPVNe+oqSkzVWR1MtSMvdw23JPc7zK7OZMjieW4YmnP8NyiuICLcMTWDjr65i1g192V1QwS0vryf3mE5K5im03D1YZU09j/x7G7Hh7XhiUm+r4ygXFBzgy7ThScy7bzjHKmuZ+Pc1LNlZQL1eNun2tNw92KzFmRwqPcVfbtF1T9W5pSVH8vFDlxAfEcSP5m9hyO8/5/PdRVbHUk7QcvdQX+wp4q2Nucy4tCsjukZZHUe5gcSoIBY+MIrnbx9EbHggjyzYRn5ZldWxVDNdyDJ7c0TkqIhknLbtaRHZIyI7ROQDEWl/2muPi0i2iOwVkXGtlFudQ2llLY+9t5OenUJ55JruVsdRbiTQz4eJ/eP4x51DsBv48VtbqNQbn9zShRy5vw6MP2PbcqCvMaY/sA94HEBEegO3AX0c73lRRPTe9jb2xEe7KK+q5a+3DiTAV3e/unhJUcH8+ab+bDtcxp2vbqS8qs7qSOoinbfcjTGrgdIztn1mjPnm1/kGIN7x9WTgHWNMjTEmB8gGhrVgXnUen+0q5OPtR/jxlan0ig2zOo5yY9/rF8sLtw9mZ345P3wzXRcEcTMtMeZ+L7DE8XVn4PBpr+U5tv0XEZkhIukikl5cXNwCMVT5qTr+58MMesWG8aMrUqyOozzAhH6x/Pmm/mw4UMrM93fQYLf+pkd1YZyafkBEfg3UA/O/2dTEtzX5p8EY8wrwCjTeoepMDtXo95/u5lhlLXPuHoqfj54rVy3jhkHx5B+vYvZn+6iub+DvUwfrMoxuoNkNICLTgYnANPOfOQzygNPvlIkHjjQ/nrpQX+4r5t3Nefzwsq707RxudRzlYR66MpVfjOvB4p2FfLnvqNVx1AVoVrmLyHjgl8AkY8zpt7R9BNwmIgEi0gVIBTY5H1Ody8maen61cCcp0cH85KpUq+MoD/WDS7sSEeTHwi35VkdRF+BCLoV8G1gP9BCRPBG5D3geCAWWi8g2EXkZwBizC1gA7AaWAg8aYxpaLb0C4E9L9nCkvIo/3zRAF95Qrcbf18bE/nEs311ERbVePePqzjvmboyZ2sTmV8/x/bOAWc6EUhduw4FjvLnhEPdd0oUhSRFWx1Ee7obBnXlzwyEWbTuiC6q7OD3r5saqahuY+f4OEiODePSaHlbHUV5gUEJ7Bie257kVWXpzk4vTcndjzyzfy8Fjp3hqSj/a+etwjGp9IsL/TOxN8YkaXlq13+o46hy03N3UltzjvLo2h2nDExmV0sHqOMqLDE6M4PqBcTy/Mpt/rj5gdRx1FrrMnhuqqW/gsfd20CkskJkTelodR3mhp6b0p67BMGtxJl06BHN1745WR1Jn0CN3N/T8F9lkHz3JrBv7EapT+SoLBPr58NdbB9KjYyj/uyiDkzr+7nK03N3M7iMVvLhqP1MGxzOmR4zVcZQX8/e18Ycb+1FQUc0fFmdaHUedQcvdzcxavJvwdn78ZmIvq6MoxZCkCGZc2pW3NuayaJve3ORKtNzdyNqsEtZlH+OhMd1oH+RvdRylAHh0XA+GJEXw5Me7qa7TexZdhZa7mzDG8Kele+jcvh3TRiRaHUepb/n52Pj52O6UVtbyyY4Cq+MoBy13N7F4ZyE788v52djuugCHcjkjU6JIjQlh7lcH+c88gspKWu5uoK7BzuzP9tK9Ywg3DGpyenylLCUi3DUqmZ355XyeqbNGugItdzfwbnoeOSWV/GJcT51HW7msm4fE0ycujEcWbCOnpNLqOF5Py93FVdU28LcV+xiSFMHVvfTSR+W6Av18ePmOIfjahAfnb8GuqzZZSsvdxc1df5Ciihp+Ob4nInrUrlxbQmQQv5nYm90FFazYo8MzVtJyd2Hlp+p4cWU2Y3pEM6xLpNVxlLogkwbEER/RjhdXZevJVQtpubuwl1fv50RNPY+N1/ljlPvw9bEx47KubM0tY2NOqdVxvJaWu4sqqqjmtXU5TB4QR6/YMKvjKHVRbklLoEOIPy/qtMCWuZBl9uaIyFERyThtW6SILBeRLMdjxGmvPS4i2SKyV0TGtVZwT/e3FVk02A2PjNVFOJT7CfTz4Z7RXVi9r5iM/HKr43ilCzlyfx0Yf8a2mcAKY0wqsMLxHBHpDdwG9HG850UR0TtuLlJOSSX//vowtw9LJDEqyOo4SjXLHSOSCAnw5aUv9ejdCuctd2PMauDMgbPJwFzH13OB60/b/o4xpsYYkwNkA8NaJqr3mP3ZXgJ8bTx0ZarVUZRqtvB2ftwxIoklOwv0uncLNHfMvaMxpgDA8fjNBdidgcOnfV+eY9t/EZEZIpIuIunFxcXNjOF5duaV8+mOAr5/SReiQwOsjqOUU+69JBlfHxv/0KP3NtfSJ1SbuhC7yWuhjDGvGGPSjDFp0dHRLRzDff152R4igvz4wWVdrY6ilNNiQgO5eUg872/JI7+syuo4XqW55V4kIrEAjsdv7lbIAxJO+7544Ejz43mXddklrMkq4cEx3XSFJeUx7r88BT8fG3e9upGCci34ttLccv8ImO74ejqw6LTtt4lIgIh0AVKBTc5F9A4NdsPvP80kPqIdd4xIsjqOUi0mITKI1+4eSkF5NWNmr2JpRqHVkbzChVwK+TawHughInkich/wFDBWRLKAsY7nGGN2AQuA3cBS4EFjjM7efwHe35xHZkEFMyf0JNBPLzBSnmV41ygW/+RSusWE8PjCHRw7WWN1JI8nrnB7cFpamklPT7c6hmUqa+q5YvYqEiLa8f6PRukcMspj7Ss6wbXPrcHXZuOe0cn8YlwP/fPuBBHZbIxJa+o1vUPVBfzjy/0Un6jhfyb21j/oyqN17xjKvPuGc2WvGF5ctZ9ffbCTuga71bE8kq/VAbzdkbIqXllzgEkD4hicGHH+Nyjl5oZ3jWJYl0iSo4J4YeV+cktP8fzUwUQE67rALUmP3C02e9le7AYeG6/TDCjvISL8YlxPnr6pP1/nHOeaZ1ezUqcIblFa7hbakVfGwq353HdJF+IjdJoB5X1uTktg4QOjiAr2Z8ab6Ww7XGZ1JI+h5W6hPyzOJCrYnweuSLE6ilKW6ds5nHdmjKBjWCAPzt/C8cpaqyN5BC13i2w/XMaGA6U8oDcsKUX7IH9enDaY4hM1/GzBNl3kowVouVvktXU5hAb4cuvQhPN/s1JeoH98ex7/Xk9W7S1m2a4iq+O4PS13CxSWV/PJjgJuGZpASIBesKTUN+4ckURKdDBPL9tDvV4i6RQtdwu8ueEgdmO4e1Sy1VGUcim+PjZ+Ob4n+4srmf3ZPqvjuDUt9zZWVdvAWxtzGdu7IwmReoWMUme6pk8nbh+eyMtf7mfRtnyr47gtHRNoYx9uy+f4qTruHd3F6ihKuawnrutN9tGT/HzBduoaDOP7dtIhzIukR+5tyBjDnLU59O0cxrAukVbHUcplBfj68Or0NPrEhfHou9sZPutzdh3RtVgvhpZ7G1qTVULW0ZPcO7qLziGj1HmEBvqx4P6RzL13GAF+Pvz2o116ieRF0HJvQ3PW5RAdGsC1/WOtjqKUWwjw9eHy7tE8ek0Pvj54nE92FFgdyW1oubeR7KMnWbW3mDtHJBHgq/O1K3Uxbh2aQO/YMP64OJOqWl0i4kJoubeR17/Kwd/Xxu3DE62OopTb8bEJT1zXmyPl1bysi21fEC33NlB2qpb3N+dz/cA4OoQEWB1HKbc0vGsUE/vH8tKq/WzXCcbOy6lyF5GficguEckQkbdFJFBEIkVkuYhkOR69fpLytzcdpqqugXsv0csflXLG7yb3JTo0gB/N20ypTjB2Ts0udxHpDPwESDPG9AV8gNuAmcAKY0wqsMLx3GvVNdh5Y/1BRneLomenMKvjKOXWIoL9efmOIZRU1vLwO1tpsOvVM2fj7LCML9BORHyBIOAIMBmY63h9LnC9k5/h1pZmFFJQXq03LSnVQvrFh/PkpD6sySrh7U25VsdxWc0ud2NMPjAbyAUKgHJjzGdAR2NMgeN7CoCYpt4vIjNEJF1E0ouLi5sbw+XNWZdDclQQY3o0uRuUUs1w29AERqVE8fSyvTo8cxbODMtE0HiU3gWIA4JF5I4Lfb8x5hVjTJoxJi06Orq5MVzaltzjbM0t457RXbDZ9KYlpVqKiPDkpD5U1tTzl8/2Wh3HJTkzLHM1kGOMKTbG1AELgVFAkYjEAjgevXZhxDlrcwgN9OWmIfFWR1HK46R2DOW2YQksSD9M3vFTVsdxOc6Uey4wQkSCpPFe+quATOAjYLrje6YDi5yL6J6OlFWxJKOQqcMSCdYJj5RqFQ9c0Q2AF1ZmW5zE9Tgz5r4ReA/YAux0/KxXgKeAsSKSBYx1PPc6b6w/hDGGu0YmWR1FKY8V174d04Yn8famw3yy44jVcVyKU4eUxpgngCfO2FxD41G81zpVW8/bm3IZ37cT8RE6Z7tSrWnmhJ5k5JfzyILtRAb7Myqlg9WRXILeodoKFm7Jp7xK52xXqi0E+vnwjzuHkBQZxN1zvmbVXq89zfcdWu4tzG43zFmXQ//4cIYkef3NuUq1iaiQAN69fyRdo4P55fs7OFlTb3Uky2m5t7Avs4o5UFypc7Yr1cbaB/nzxxv7UVRRw3MrsqyOYzkt9xY2Z20OHcMC+F4/nbNdqbY2KDGCm4bE8/pXBzlSVmV1HEtpubegfUUnWJNVwl0jk/H31V2rlBV+enUqxhj+/oV3H71rA7WgOWtzCPSzMXWYztmulFXiI4KYNjyJBel55JRUWh3HMlruLaT4RA0Lt+YzZXA8kcH+VsdRyqs9MCYFfx8bf12+z+ooltFybyHzNhyitt6uc7Yr5QJiQgO5Z3QyH+84QmZBhdVxLKHl3gKq6xp4c8Mhru4VQ0p0iNVxlFLADy9LISTAl7985p1H71ruLeCDrfmUVtZy3yVdrY6ilHIID/Lj/stT+DyziC25x62O0+a03J1ktxv+teYAfTuHMaJrpNVxlFKnuXtUMh1C/Jm9zPumBdZyd9KX+4rZX1zJ9y/pqjctKeViggN8eeCKbny1/xjrskusjtOmtNyd9M81B+gUFsi1/fWmJaVc0e3DE4kLD+TpZXsxxnvWXNVyd8KuI+V8tf8Yd49Oxs9Hd6VSrijQz4eHr05l2+EyPs/0nknFtJGc8OqaHIL8fZg6VG9aUsqVTRkcT5cOwcxethe73TuO3rXcm6mwvJqPth/hlrQEwoP8rI6jlDoHXx8bj4ztzt6iE3zsJYt6aLk309z1B7Ebo3O2K+Umru0XS6/YMJ5Zvo+6BrvVcVqdU+UuIu1F5D0R2SMimSIyUkQiRWS5iGQ5Hj1uUvPKmnrmbzjEuD6dSIzSlZaUcgc2m/DoNd05dOwU76bnWR2n1Tl75P43YKkxpicwgMYFsmcCK4wxqcAKx3OP8t7mPCqq6/n+pXrUrpQ7ubJnDIMS2/PiqmzqPfzovdnlLiJhwGXAqwDGmFpjTBkwGZjr+La5wPXORXQ98zceYmBCe4Yk6U1LSrkTEeHBK7qRd7yKT3YUWB2nVTlz5N4VKAZeE5GtIvIvEQkGOhpjCgAcjzEtkNNlZB89yb6ik9wwqLPVUZRSzXBlzxi6dwzhmeX72H64zOo4rcaZcvcFBgMvGWMGAZVcxBCMiMwQkXQRSS8uLnYiRttamtH4235cn04WJ1FKNYfNJvx2Uh9O1TZww4vrePnL/VZHahXOlHsekGeM2eh4/h6NZV8kIrEAjscm7xowxrxijEkzxqRFR0c7EaNtLckoZHBiezqFB1odRSnVTKNSOrDy0cuZ0C+Wp5bs4f3NnneCtdnlbowpBA6LSA/HpquA3cBHwHTHtunAIqcSupDcY6fYdaSC8X31qF0pdxca6Meztw5kVEoUMxfu4O1NuR617qqvk+//MTBfRPyBA8A9NP7CWCAi9wG5wM1OfobLWLarEIAJfXUeGaU8gZ+PjZemDeGu1zbx+MKd+NiE/5vch2nDk6yO5jSnyt0Ysw1Ia+Klq5z5ua5qSUYBfeLCSIjUa9uV8hThQX588KNRbMk9zuzP9vLU4j1M7B9HeDv3vvNc71C9QIXl1WzJLWOCDsko5XFsNiEtOZLfTOzNiZp63vjqoNWRnKblfoG+GZIZr0MySnmsPnHhXNUzhlfX5VBZU291HKdouV+gJRkFdIsJoVuMrpGqlCd78MpulJ2qY96GQ1ZHcYqW+wU4drKGTTmlOiSjlBcYnBjBZd2jef6LbArK3ffqGS33C7B8dxF2g14CqZSX+N3kPtTZ7fz6gwy3Xb1Jy/0CLMkoJDEyiN6xYVZHUUq1gaSoYH4+tgdf7DnK+gPHrI7TLFru51FeVcdX+0uY0LeTLoCtlBe5c2QSHUICeGFlttVRmkXL/TxWZBZR12AYp0MySnmVQD8ffnBpF9ZlH2Nr7nGr41w0LffzWJJRSKewQAbGt7c6ilKqjU0bkUR4Oz9eWOl+k4tpuZ9DZU09q/cVM75vJ2w2HZJRytuEBPhyz+hkPs8sYk9hhdVxLoqW+zms2ltMTb1dr5JRyovdPSqZYH8fnv/CvcbetdzPYUlGAR1C/BmarCsuKeWt2gf5c/foZD7ZUcAGN7pyRsv9LKrrGli55yhje3fCR4dklPJqD41JJTEyiMcX7qSqtsHqOBdEy/0s1mSVUFnboEMySina+fvwxxv7cehYJQ+9tQW73fVvbNJyP4slGQWEBfoysmuU1VGUUi5gdLcO/Pra3qzYc5Qv97n+0qBa7k2orbfz+e4iru7dEX9f3UVKqUZ3jmi8sWn+RtefVEybqwkbDhyjorpeV1xSSn2Hv6+NW9Li+WLPUfJdfEk+LfcmLMkoJMjfh0tTO1gdRSnlYqaNSMImwr/WHLA6yjk5Xe4i4iMiW0XkE8fzSBFZLiJZjscI52O2nQa7YfnuQsb0jCHQz8fqOEopF9O5fTsmDYzjnU2HOV5Za3Wcs2qJI/eHgczTns8EVhhjUoEVjudu4+uDpZScrNW525VSZzXjsq5U1TXw3uY8q6OclVPlLiLxwLXAv07bPBmY6/h6LnC9M5/R1pZmFBLga2NMjxiroyilXFTPTmEMSYrg7U25Ljvfu7NH7s8CjwH207Z1NMYUADgem2xJEZkhIukikl5c7BqXFdnthqUZhVzWPZrgAF+r4yilXNjtwxI5UFLpsvO9N7vcRWQicNQYs7k57zfGvGKMSTPGpEVHRzc3RovanldGYUW1Dskopc7r2v6xhLfzY/7GXKujNMmZI/fRwCQROQi8A1wpIvOAIhGJBXA8HnU6ZRtZmlGIr024qmdHq6MopVxcoJ8PUwbH89muQkpO1lgd5780u9yNMY8bY+KNMcnAbcAXxpg7gI+A6Y5vmw4scjplGzDGsCSjkFHdOhAe5Gd1HKWUG7h9eAJ1DYZ3013vxGprXOf+FDBWRLKAsY7nLm93QQW5pad0SEYpdcG6xYSSlhTBB1s9tNyNMauMMRMdXx8zxlxljEl1PJa2xGe0tqUZhdgErumtQzJKqQs3aWAc+4pOsrfwhNVRvkPvUHVYklHIsC6RRIUEWB1FKeVGJvSNxSbwyY4jVkf5Di13IPvoCbKPntS5ZJRSFy06NIBRKR34ePsRl7rmXcsd+GRHAQDX9NEhGaXUxbtuQCwHj51iS+5xq6N8y+vLvb7BzjubDnNpagdiw9tZHUcp5YYm9o8jLNCXOWsPWh3lW15f7iv2HKWwopo7RiRZHUUp5aaCA3y5fXgSSzIKOFx6yuo4gJY78zYcIjY8kKt66lwySqnmmz6q8QDx3fTDFidp5NXlfrCkkjVZJdw2NBFfH6/eFUopJ8WGt+PS1Gje35LvEmusenWjvb0pFx+bcOvQBKujKKU8wJQh8eSXVbHBBSYT89pyr65rYEH6Ycb26kin8ECr4yilPMA1vTsSGuDLB1vzrY7iveW+JKOA46fq9ESqUqrFBPr5MLZPR5btKqS23n7+N7Qiry33eRty6dIhmFEpUVZHUUp5kIn9Y6mormdddomlObyy3DMLKth86DjThidis4nVcZRSHuSSbtGEBfry6c4CS3N4ZbnP23AIf18bUwbHWx1FKeVh/H1tXNEjhlV7j1p61YzXlfvJmno+3JrPxP6xRAT7Wx1HKeWBruwZQ8nJWnbml1uWwevK/cOt+VTWNuiJVKVUq7m8ezQ2gS/2WLcQnVeVuzGGeRsO0Ts2jEEJ7a2Oo5TyUBHB/gxKjGDFniLLMnhVuW/JPc6ewhPcMSIJET2RqpRqPeP6dCQjv4LcY9bMNdPscheRBBFZKSKZIrJLRB52bI8UkeUikuV4jGi5uM6ZtyGXkABfJg+MszqKUsrDfbM+xOIMa66acebIvR74uTGmFzACeFBEegMzgRXGmFRgheO55Uora/l0ZwE3DOpMcICv1XGUUh4uITKIAfHhLLbokshml7sxpsAYs8Xx9QkgE+gMTAbmOr5tLnC9kxlbxHubD1Nbb9cTqUqpNjO+byw78sopKK9q889ukTF3EUkGBgEbgY7GmAJo/AUANDmXrojMEJF0EUkvLi5uiRhnZbcb5m/MZWhyBD06hbbqZyml1Deu7tVYfysy2/6qGafLXURCgPeBnxpjKi70fcaYV4wxacaYtOjoaGdjnNPa7BIOHTulR+1KqTbVLSaExMggVmS2/VUzTpW7iPjRWOzzjTELHZuLRCTW8XosYN2Fng7zNhwiKtif8X07WR1FKeVFRISre3Vk3f5jVNbUt+lnO3O1jACvApnGmGdOe+kjYLrj6+nAoubHc15BeRUr9hzl5rQEAnx9rIyilPJC4/p0pLbezudtfPTuzJH7aOBO4EoR2eb473vAU8BYEckCxjqeW+adTYexG8PtwxKtjKGU8lJDkyOJCw9k0bYjbfq5zb4m0BizFjjbnUBXNffntqS6BjvvfJ3LZanRJEYFWR1HKeWFbDbhuoFxvLomh2Mna4gKCWibz22TT7HIiswiiipq9ESqUspSkwbEUW83fLa77YZmPLrc523IJS48kCt7Nnk1plJKtYnesWEkRLZj2a7CNvtMjy33nJJK1maXMHVYIj66IIdSykIiwrjenfgq+xgnquva5DM9ttzf2ngIX5tw69AEq6MopRTj+naitsHOyr2te9PmNzyy3KvrGnh3cx7X9OlITFig1XGUUorBiRFEBfuzso3mePfIcv90RwFlp+q4Y7ieSFVKuQYfmzCiaxQbDxzDmNZffs8jy33exkN0jQ5mZEqU1VGUUupbI7pGcqS8msOlrT+RmMeV+64j5WzNLWPacF2QQynlWkZ0bTzg3HDgWKt/lseV+/yNuQT62bhpcLzVUZRS6ju6xYQQFeyv5X6xTlTX8eHWfK7rH0d4kJ/VcZRS6jtEGsfdN7TBuLtHlfuHW/M5VdvANL0jVSnlooa30bi7x5S7MYZ5G3Lp2zmMAfHhVsdRSqkmfTvuntO6QzMeU+6bDx1nb9EJ7tATqUopF5YaE0JkG4y7e0y5z9twiNBAXyYNjLM6ilJKnVXjuHskGw+Utuq4u0eU+7GTNSzeWciUwfEE+Td7FmOllGoTI7pGkV9WRW7pqVb7DI8o93c351HbYGfacF2QQynl+i7p1gGA1ftab54Zty93u93w1sZchnWJJLVjqNVxlFLqvLp0CCYpKqhVJxFrtXIXkfEisldEskVkZmt9zprsEnJLT+mCHEoptyEijOkRw1f7S6iua2iVz2iVchcRH+AFYALQG5gqIr1b47PmbThEhxB/xvfp1Bo/XimlWsXlPaKprrOzMae0VX5+ax25DwOyjTEHjDG1wDvA5Jb+kCNlVazILOKWtAT8fd1+hEkp5UVGdo0iwNfWalMAt1YjdgYOn/Y8z7HtWyIyQ0TSRSS9uLh5406nauu5vHs0U4fpiVSllHsJ9PPh9uGJJEYGtcrPb63rBpu6i+g7F3QaY14BXgFIS0tr1sWe3WJCee2eYc15q1JKWe6J6/q02s9urSP3POD09e3igSOt9FlKKaXO0Frl/jWQKiJdRMQfuA34qJU+Syml1BlaZVjGGFMvIg8BywAfYI4xZldrfJZSSqn/1mr36htjFgOLW+vnK6WUOju9flAppTyQlrtSSnkgLXellPJAWu5KKeWBpLUXab2gECLFwCEnfkQHoKSF4rQGV88Hrp/R1fOBZmwJrp4PXCtjkjEmuqkXXKLcnSUi6caYNKtznI2r5wPXz+jq+UAztgRXzwfukRF0WEYppTySlrtSSnkgTyn3V6wOcB6ung9cP6Or5wPN2BJcPR+4R0bPGHNXSin1XZ5y5K6UUuo0Wu5KKeWB3Lrc22oRbmeIyEER2Ski20Qk3QXyzBGRoyKScdq2SBFZLiJZjscIF8z4WxHJd+zHbSLyPQvzJYjIShHJFJFdIvKwY7vL7MdzZHSl/RgoIptEZLsj45OO7S6xH8+Rz2X24bm47Zi7YxHufcBYGhcH+RqYaozZbWmwM4jIQSDNGOMSNz2IyGXASeANY0xfx7Y/A6XGmKccvyQjjDG/dLGMvwVOGmNmW5XrGyISC8QaY7aISCiwGbgeuBsX2Y/nyHgLrrMfBQg2xpwUET9gLfAwcCMusB/PkW88LrIPz8Wdj9zbZBFuT2OMWQ2cudz6ZGCu4+u5NJaAZc6S0WUYYwqMMVscX58AMmlcI9hl9uM5MroM0+ik46mf4z+Di+zHc+RzC+5c7uddhNtFGOAzEdksIjOsDnMWHY0xBdBYCkCMxXnO5iER2eEYtrF06OgbIpIMDAI24qL78YyM4EL7UUR8RGQbcBRYboxxqf14lnzgQvvwbNy53M+7CLeLGG2MGQxMAB50DDmoi/cSkAIMBAqAv1iaBhCREOB94KfGmAqr8zSliYwutR+NMQ3GmIE0rrM8TET6WpnnTGfJ51L78GzcudzdYhFuY8wRx+NR4AMah5NcTZFjjPabsdqjFuf5L8aYIsdfNDvwTyzej44x2PeB+caYhY7NLrUfm8roavvxG8aYMmAVjePZLrUf4bv5XHUfnsmdy93lF+EWkWDHySxEJBi4Bsg497ss8REw3fH1dGCRhVma9M1fdocbsHA/Ok60vQpkGmOeOe0ll9mPZ8voYvsxWkTaO75uB1wN7MFF9uPZ8rnSPjwXt71aBsBxCdKz/GcR7lnWJvouEelK49E6NK5X+5bVGUXkbeAKGqctLQKeAD4EFgCJQC5wszHGshOaZ8l4BY3/DDbAQeCH34zLWpDvEmANsBOwOzb/isYxbZfYj+fIOBXX2Y/9aTxh6kPjgeYCY8z/iUgULrAfz5HvTVxkH56LW5e7UkqpprnzsIxSSqmz0HJXSikPpOWulFIeSMtdKaU8kJa7Ukp5IC13pZTyQFruSinlgf4fzTJQ5YBwrI4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch.nn.functional as F\n",
    "from torch.nn.utils import parameters_to_vector\n",
    "\n",
    "def compute_u(model_i, model_f):\n",
    "    i_params = parameters_to_vector(model_i.parameters())\n",
    "    f_params = parameters_to_vector(model_f.parameters())\n",
    "    minus_params = f_params-i_params\n",
    "    return F.normalize(minus_params,dim=0)\n",
    "    \n",
    "def alpha_beta(model_t,u, model_i):\n",
    "    i_params = parameters_to_vector(model_i.parameters())\n",
    "    t_params = parameters_to_vector(model_t.parameters())\n",
    "    \n",
    "    alpha = torch.dot((t_params-i_params),u)\n",
    "    v = t_params-(i_params+alpha*u)\n",
    "    v_normalized = F.normalize(v,dim=0)\n",
    "    beta = torch.dot(v,v)\n",
    "    return alpha, beta\n",
    "\n",
    "u = compute_u(model_i,model_f)\n",
    "alpha_list = []\n",
    "beta_list = []\n",
    "for model_t in model_t_list:\n",
    "    alpha, beta = alpha_beta(model_t, u, model_i)\n",
    "    alpha_list.append(alpha)\n",
    "    beta_list.append(beta)\n",
    "\n",
    "pylab.plot(alpha_list, beta_list)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "steady-norwegian",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}