pyTorchExample.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linear Regression Example with pyTorch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, we implement a gradient descent algorithm for reference, using numpy only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 : 15.9938818182\n",
      "1 : 11.6346546383\n",
      "2 : 8.57455118197\n",
      "3 : 6.4263659935\n",
      "4 : 4.91830088281\n",
      "5 : 3.85956758638\n",
      "6 : 3.11624261595\n",
      "7 : 2.5943186121\n",
      "8 : 2.2278072717\n",
      "9 : 1.97038821883\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pylab as plt\n",
    "\n",
    "import time\n",
    "from IPython import display\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "#y = np.array([7.46, 6.77, 12.74, 7.11, 7.81, 8.84, 6.08, 5.39, 8.15, 6.42, 5.73])\n",
    "y = np.array([8.04, 6.95, 7.58, 8.81, 8.33, 9.96, 7.24, 4.26, 10.84, 4.82, 5.68])\n",
    "#y = np.array([9.14, 8.14, 8.74, 8.77, 9.26, 8.10, 6.13, 3.10, 9.13, 7.26, 4.74])\n",
    "x = np.array([10., 8., 13., 9., 11., 14., 6., 4., 12., 7., 5.])\n",
    "N = len(x)\n",
    "\n",
    "# Design matrix\n",
    "A = np.vstack((np.ones(N), x)).T\n",
    "\n",
    "# Learning rate\n",
    "\n",
    "eta = 0.01\n",
    "              \n",
    "# initial parameters\n",
    "w = np.array([2., 1.])\n",
    "\n",
    "for epoch in range(10):\n",
    "    # Error\n",
    "    err = y-A.dot(w)\n",
    "    \n",
    "    # Average error\n",
    "    E = np.sum(err**2)/N\n",
    "    \n",
    "    # Gradient\n",
    "    dE = -2.*A.T.dot(err)/N\n",
    "    \n",
    "    if epoch%1 == 0: \n",
    "        print(epoch,':',E)\n",
    "        # print(w)    \n",
    "\n",
    "    # Perfom one descent step\n",
    "    w = w - eta*dE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Animated Visualization of Gradient Descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAT8AAAEzCAYAAABHZATQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAGzFJREFUeJzt3Xl4VOXZBvD7CRGUHQ1LFCFABGRT\nMCKgBBFEEAutS91F0Y9at34utSCICCK4tFZbxaJE9FOxithSFSSANCibLLIIiGGRLcimyJ5lnu+P\nCcjJmZBk5syc5b1/1+U1zsPMnOdc6u1Z3ve8oqogIjJNktsNEBG5geFHREZi+BGRkRh+RGQkhh8R\nGYnhR0RGKjP8RCRLRHaKyKoIf/aIiKiIpMSnPSKi+CjPkd9EAL1LFkXkbACXA9jscE9ERHFXZvip\nag6AvRH+6AUAjwLgKGki8p2orvmJSD8A21R1ucP9EBElRHJFvyAiVQEMBdCrnJ8fBGAQAFSrVu2C\nli1bVnSTREQntWTJkt2qWrci36lw+AFoBqAJgOUiAgANASwVkY6quqPkh1V1PIDxAJCRkaGLFy+O\nYpNERKUTke8r+p0Kh5+qrgRQ74SNbgKQoaq7K/pbRERuKc9Ql0kA5gNoISJbReTO+LdFRBRfZR75\nqeqNZfx5mmPdEBElCGd4EJGRGH5EZCSGHxEZieFHREZi+BGRkRh+RGQkhh8RGYnhR0RGYvgRkZEY\nfkRkJIYfERmJ4UdERmL4EZGRGH5EZCSGHxEZieFHREZi+BGRkRh+RGQkhh8RGYnhR0RGYvgRkZEY\nfkRkJIYfERmJ4UdERmL4EZGRGH5EZCSGHxEZieFHREZi+BGRkRh+RGQkhh8RGYnhR0RGYvgRkZEY\nfkRkJIYfERmJ4UdERmL4EZGRGH5EZCSGHxEZieFHREZi+BGRkRh+RGQkhh8RGYnhR0RGYvgRkZEY\nfkRkpDLDT0SyRGSniKw6ofaciKwVkRUi8pGI1I5vm0REzirPkd9EAL1L1LIBtFHVdgDWARjicF9E\nRHFVZvipag6AvSVqM1S1sPjtAgAN49AbEVHcOHHNbyCAaaX9oYgMEpHFIrJ4165dDmyOiCh2MYWf\niAwFUAjgndI+o6rjVTVDVTPq1q0by+aIiByTHO0XRWQAgKsA9FBVda4lIqL4iyr8RKQ3gD8B6Kaq\nh5xtiYgo/soz1GUSgPkAWojIVhG5E8DfAdQAkC0iX4vIq3Huk4jIUWUe+anqjRHKE+LQCxFRwnCG\nBxEZieFHREZi+BGRkRh+RGQkhh8RGYnhR0RGYvgRkZEYfkRkJIYfERmJ4UdERmL4EZGRGH5EZCSG\nHxEZieFHREZi+BGRkRh+RGQkhh8RGYnhR0RGYvgRkZEYfkRkJIYfERmJ4UdERmL4EZGRyly3l4j8\nKxQCcnOBvDwgNRVITweSeMgDgOFHFFihEJCVBcyd+0uta1dg4EAGIMDTXqLAys0NB1/jxkCTJuHX\nuXPDdWL4EQVWXl749dhR3rHXHTvc6cdreNpLFFCpqeHXUCgcfKFQ+H2DBu715LTCohAmztsU1XcZ\nfkQBlZ4evsZX8ppferp7PTnpwyVb8fAHy6P+PsOPKKCSksI3NzIzw6e6DRoE427vjn1H0GnMrJh/\nh+FHFGBJSUDz5uG//E5Vcc87SzFtlfWi5bzBl+GsZyr+eww/IvK8z9fuxB0Tv7LUnr2mHX574dlR\n/ybDj4g8a9+hApw3coal1q5hLUz5fRckV4rt/J3hR0Se9OR/vsEbX26y1LIfzMQ59Ws48vsMPyLy\nlCXf/4hrxs2z1P54RQvc293Z29QMPyLyhCMFRej67OfYtf/o8VrdGlUw99HuOPWUSo5vj+FHRK57\nZU4unp3+raX24e+74ILGdeK2TYYfEbkmd+d+9PxLjqU2oHNjPNm/Tdy3zfAjooQrLArh6nHzsGLr\nPkt9+fBeqFX1lIT0wPAjooR6f/EWPDp5haWWdXsGLmtZP6F9MPyIKCHy9h1G5zGzLbU+bRrglZs7\nQEQS3g/Dj4jiSlUx6P+WIHv1D5b6/CGXIbXWaS51xfAjojiateYH3PnmYkvtuWvb4bqM6KelOYXh\nR0SO++lQPs4fmW2pnXd2bXx4d+eYp6U5heFHRI66YFQ29hzMt9RmPtQN6fWqu9RRZGVGsIhkichO\nEVl1Qu10EckWke+KX+M3EpGIfGF8znqkDf7EEnyD+7TEprF9PRd8QPmO/CYC+DuAt06oDQYwS1XH\nisjg4vd/cr49IvK6Hw/mo/2obFt97ajecZmW5pQyw09Vc0QkrUS5P4BLi//+TQBzwPAjMk7r4dNx\nML/IUhvxq1a4/eImCdn+sXWJgVoVPrSM9ppffVXNAwBVzRORelH+DhH50Ccr8nDvu0tt9U1j+yas\nB+u6xCkpFf1+3G94iMggAIMAoFGjRvHeHBHF0ZGCIrR8fLqtPvfR7jj79KoJ7eXEdYmBo/llfb6k\naMPvBxFJLT7qSwWws7QPqup4AOMBICMjQ6PcHhG57OpXvsTSzT9Zard3ScOIfq1d6afkusQVFW34\nTQUwAMDY4td/R/k7RORxizftxbWvzrfVN4650pVpacecuC5xNMoMPxGZhPDNjRQR2QrgCYRD730R\nuRPAZgDXRbd5IvKqUEjR9LFPbfX/3HcJ2jas5UJHVtZ1iatUruj3RTVxZ6IZGRm6ePHisj9IRK4a\n/OEKvPfVFksts3ldvDWwo0sdRXbsbm+LFrW+Vd3XsiLf5QwPIjpu4+6D6P78HFt93VN9UDnZG9PS\nTnRsXWLg5wMV/S7Dj4gAAGmDP7HVJgzIQI9zE/ucvURh+BEZbtyc9Xhm+lpLrX7NKlj4WE+XOkoM\nhh+RofYcOIoLnpppq68Y0Qs1T03Mo+TdxPAjMlDzodOQX2QdIzKqf2vc2jnNnYZcwPAjMsjU5dvx\nwKRltnoip6V5BcOPyAClTUv7cvBlOKu2e4+SdxPDjyjgIt3FveuSJhh2VSsXuvEOhh9RQP1r2Tb8\n7z+/ttXdnpbmFQw/ooApCimaRZiW9vH9l6DNWe5PS/MKhh9RgLQd8Rn2Hym01U28oVEWhh9RAJT2\n5JVvn+qNKsnefZS8mxh+RD4X6YbGE79qhTsS9Ch5v2L4EfnUbVmLkLNul63OU9zyYfgR+cyWvYfQ\n9dnPbfUlw3rijOpVXOjInxh+RD4S6RT3xo6NMObqti50428MPyIfGPPpGvwjZ4OtzlPc6DH8iDzs\n5yMFaDdihq2e/WAmzqlfw4WOgoPhR+RRkU5x25xVEx/f39WFboKH4UfkMR8s3oI/Tl5hq3NamrMY\nfkQeUVgUQvrQabb6WwM7IrN5XRc6CjaGH5EHNB82DfmF9gVoeUMjfhh+RC5auGEPrh+/wFb36mpp\nQcLwI3JJpBsao37dBrd2auxCN+Zh+JGnHFuEOi8PSE0F0tPDa7MGyQ3j52PBhr22Ok9xE4vhR54R\nCgFZWcDcub/UunYFBg4MRgBu2n0Ql0ZYEHzZ45ejTrXKiW/IcAw/8ozc3HDwNW4cDrtQKPw+MxNo\n3tzt7mIT6RT3ts6NMbJ/Gxe6IYDhRx6Slxd+PXaUd+x1xw7/ht+oj1djwhcbbXWe4rqP4UeekZoa\nfg2FfjnyA4AGDdzrKVr7DhXgvJH2aWmzHu6GZnWru9ARlcTwI89ITw9f4yt5zS893b2eohHpFLd9\no9r46J6LXeiGSsPwI89ISgrf3MjMDJ/qNmjgr7u9kxZtxpApK211TkvzJoYfeUpSUvj6np+u8RUU\nhXBOhGlp79x1ES5OT3GhIyoPhh9RDJoM+QSq9jpvaHgfw48oCvNyd+Om1xfa6t+N7oNTKvnkPN1w\nDD+iClBVNBliXxB8zNVtcWPHRi50RNFi+BGV0zXj5mHJ9z/a6jzF9SeGH1EZNuw6gMv+/F9bffnw\nXqhV9RQXOiInMPyITiLSmL2BFzfB8F+1iur3THhwg18w/IgiGP7vVXhr/ve2eiynuEF/cIPfMPyI\nTvDjwXy0H5Vtq8955FKkpVSL6beD/OAGP2L4ERWLdIp7UZPT8c/fdXbk94P44AY/Y/iR8V7+PBfP\nffatre70XdwgPbghCBh+ZKyjhUVoMWy6rf7eoE7o1PQMx7cXlAc3BAXDj4wU6RQXiO+YPb8/uCFo\nGH5klOmrduDut5fY6omalubHBzcEVUzhJyIPArgLgAJYCeAOVT3iRGNETiptWtpjfc7FpalNMe8L\njrszTdThJyJnAXgAQCtVPSwi7wO4AcBEh3ojcsT5I2fgp0MFtvqGp/siKwsYnfVLjePuzBHraW8y\ngNNEpABAVQDbY2+JyBnfbN+Hvi99Yat/Pfxy1K5aGevWcdydyaIOP1XdJiLPA9gM4DCAGapqX7SA\nyAWRbmhc06Eh/vzb846/57g7s8Vy2lsHQH8ATQD8BOADEblFVd8u8blBAAYBQKNGfOQPxdddb36F\nmWt22uqR7uJy3J3ZYjnt7Qlgo6ruAgARmQKgCwBL+KnqeADjASAjIyPCM2+JYrdz/xF0HD3LVp/5\nUDek14u8WhrH3ZktlvDbDKCTiFRF+LS3B4DFjnRFVAGRTnGbplTD7EcuPen3OO7ObLFc81soIpMB\nLAVQCGAZio/wiBLhhex1eHHWd7Z6RQYqc9yduWK626uqTwB4wqFeiMrlSEERWj5un5b21sCOyGxe\n14WOyI84w4N8xY1paRRMDD/yhY9XbMd97y6z1XNH90EyV0ujKDD8yNNKm5b2ZL/WGNAlLfENUWAw\n/MizWg2fjkP5RbY6T3G9y09rlDD8yHPW/bAfvV7IsdWXP9ELtU7jamle5bc1Shh+5CmRbmjccOHZ\nGHtNOxe6oYrw2xolDD/yhOc+W4uXP19vq/MU1z/8Nlea4UeuKm21tIWP9UD9mqe60BFFy29zpRl+\n5JpIp7g3X9QIo3/T1oVuKFZ+myvN8KOEm7xkKx75YLmtzlNcf/PbXGmGHyVMaaulffpAV7Q6s6YL\nHZHT/DRXmuFHCdFx9Ezs3H/UUrswrQ4+uLuLSx2R6Rh+FFfz1u/GTa8ttNU3PH0lkpLEhY6Iwhh+\nDvDTqPZEKW1a2ht3XIjuLeq50BGRFcMvRn4b1Z4Id7yxCJ9/u8tSq5ychHVP9XGpIyI7hl+M/Daq\nPZ5ydx5Az7/811ZfM7I3TqtcyYWOiErH8IuR30a1x0ukMXuj+rfGrZ3TEt8MUTkw/GLkt1HtThsz\nbQ3+8d8NtjrH7JHXMfxi5LdR7U7Zc+AoLnhqpq2+aGgP1KvBaWnkfQy/GPltVLsTIp3i3t4lDSP6\ntXahG6LoMPwc4KdR7bF4b9FmDJ6y0lbnKS75EcOPynS0IISm9+ag6EAVVKpeDcl1DkIE+Ox/M9Gi\nQQ232yOKCsOPTurJqavx0rgCHN3SDBAACrRqn48FbzYL9Kk9BR/DjyLK3bkfPf+Sg4K91XB0SzNU\nqnUYIsD93c/B5s2C3Nzgn+ZTsDH8yKLktLSiA1UAAW7r3BhnVK9yvG7aOEYKHoYfHTfhi40Y9fFq\nS+26zBT8fMrZqFM1/N60cYwUXAw/wg8/H8FFT8+y1dc91QfJSUnISjZvHCMFH8PPcN2fn4ONuw9a\nav8c1AkXNT3j+HvTxjGSGRh+hvpkRR7ufXeppdajZT1MuP1C22dNGcdIZmH4GWb/kQK0HTHDVl85\nohdqnMoFwckcDD+DDJz4FWav3WmpvXxTB/Rtl+pSR0TuYfgZYOGGPbh+/AJLrWlKNcx+5FJ3GiLy\nAIZfgOUXhtB82DRbfdFjPVCPC4KT4Rh+ATVi6jeYOG+TpTb8qlYYeEkTdxoi8hiGX8Cs+2E/er2Q\nY6tvHHMlRLhaGtExDL+AKG21tJkPdUN6veoudETkbQy/AHgtZwNGf7rGUrvrkiYYdlUrlzoi8j6G\nn4/t2HcEncbYp6V9N7oPTqnEKRhEJ8Pw86lLnpmNrT8ettQm390ZGWmnu9QRkb8w/Hxm6vLteGDS\nMkutV6v6GH9bhksdEfkTw88nfj5SgHYRpqV98+QVqFaF/xiJKor/1fjArRMWYu53uy21V2/pgN5t\nOC2NKFoMPw+bt343bnptoaV2Tr3qyH6om0sdEQUHw8+DjhYWocWw6bY6FwQncg7Dz2OG/Wsl3l6w\n2VJ7sl9rDOiS5k5DRAHF8POItTt+Ru+/zrXUKiUJckf34bQ0ojiIKfxEpDaA1wG0AaAABqrqfCca\nM0UopGj6mH1a2uyHu6FpXU5LI4qXWI/8XgQwXVWvFZHKAKo60JMxxs1Zj2emr7XUftetKYb0Odel\njojMEXX4iUhNAJkAbgcAVc0HkO9MW8G2/afD6DJ2tq2eO7oPkjktjSghYjnyawpgF4A3ROQ8AEsA\n/EFVLUuBicggAIMAoFGjRjFsLhg6PT0LO34+YqlNuacLOjSq41JHRGaK5TAjGUAHAONUtT2AgwAG\nl/yQqo5X1QxVzahbt24Mm/O3fy3bhrTBn1iC78q2DbBpbF8GH5ELYjny2wpgq6oeG4U7GRHCz3T7\nDhfgvCft09JWj7wCVSvzZjuRW6L+r09Vd4jIFhFpoarfAugBYLVzrfnfTa8twLz1eyy18bdegF6t\nG7jUEREdE+uhx/0A3im+07sBwB2xt+R/X+buxs2vW6elnZtaE9P+0NWljoiopJjCT1W/BsBnKRUr\nbVra4mE9kVK9igsdEVFpAnnRKRQCcnOBvDwgNRVITweS4jyCZMiUFZi0aIulNurXbXBrp8bx3TAR\nRSVw4RcKAVlZwNwTZop17QoMHBifAFy9/Wdc+ZJ1WlqV5CSsHdWb09KIPCxw4ZebGw6+xo3DYRcK\nhd9nZgLNmzu3ndKmpc155FKkpVRzbkNEFBeBm06Qlxd+PXaUd+x1xw7ntvHy57m24Lu3ezNsGtuX\nwUfkE4E78kstfrhxKPTLkR8ANHBgdMm2nw7jYk5LIwqEwIVfenr4Gl/Ja37p6bH9bsZTM7H7wFFL\n7aN7uqA9Z2cQ+VLgwi8pKXxzIzMzfKrboEFsd3s/XLIVD3+w3FLrd96ZeOnG9g50S0RuCVz4AeGg\na948thscPx3Kx/kjs231NSN747TKlWLojoi8IJDhF6vf/mM+Fm3ca6lNGJCBHufWd6kjInIaw+8E\nOet24basRZZau4a1MPW+S1zqiIjiheEH4EhBEVo+bp+WtvTxy3F6tcoudERE8WZ8+P3xg+X4YMlW\nS23M1W1xY0c+eJUoyIwNv1Xb9uGqv31hqVWrXAmrnryC09KIDGBc+BWFFM0iTEvL+WN3NDqD6y8R\nmcKo8Htx5nd4YeY6S+2By9LxUK8WLnVERG4xIvy27D2Ers9+bquvf/pKVEriKS6RiQIdfqqK80dm\nY9/hAkt96n0Xo13D2i51RUReENjwe/+rLXj0wxWW2m/an4UXrj/fpY6IyEsCF34HjhaizROf2epr\nR/XGqadwWhoRhQUq/P4841v8bXaupfbGHReie4t6LnVERF4ViPCLNGbv5osaYfRv2rrUERF5na/D\nL78whN5/zcGG3QeP1047pRK+GtYT1av4eteIKM58mxBvztuEJ6Z+Y6m9e9dF6JKe4lJHROQnvgu/\nzXsOIfM565i96y5oiGevbcdpaURUbr4Jv1BIccuEhZi3fo+lbuKC4G6sS0wUNL4Iv49XbMd97y6z\n1F65uQOubJvqUkfuSfS6xERB5enw27X/KC4cPdNS63pOCt68oyOSDJ2Wlqh1iYmCzpPhp6p4+P3l\nmLJsm6XOJ6+cfF1ihh9R+Xku/L7M3Y2bX19oqY3q3xq3dk5zpyGPiee6xEQm8Uz47T9SgAuemon8\nwtDx2jn1quOTB7qicjIvZh0Tr3WJiUzjifB7ZvpajJuz3lL79IGuaHVmTZc68i6n1yUmMpWr4Rdp\nWhofLlo2J9YlJjKdK+F3tLAIvV7Iwfd7Dh2v1aiSjAWP9UA1TksjogRIeNJkfbERIz9ebalN+p9O\n6NzsjES3QkQGS2j4rdy2zxJ812ecjWeubZfIFoiIALh4zW/JsJ44w7BpaUTkHQkPv1dv6YDebcyb\nlkZE3pLQARJtz6rF4CMiT+DoMCIyUkLDb/9+YN26X6ZkERG5JaHht2cPMHp0+JFMDEAiclNCw69y\n5fCjmObODT+aiYjILQm/5nfiI5iIiNyS8PDjI5iIyAsSGn75+cD33/MRTETkvpgHOYtIJQCLAWxT\n1atO9tmUFGDoUD6CiYjc50QE/QHAmvJ8sHr18GOYGHxE5LaYYkhEGgLoC+B1Z9ohIkqMWI/B/grg\nUQActUdEvhL1NT8RuQrATlVdIiKXnuRzgwAMKn57VERWRbtNH0gBsNvtJuIoyPsX5H0Dgr9/FX78\nu6hqVFsSkTEAbgVQCOBUADUBTFHVW07yncWqmhHVBn2A++dfQd43gPsXSdSnvao6RFUbqmoagBsA\nzD5Z8BEReQnvuxKRkRx5mKmqzgEwpxwfHe/E9jyM++dfQd43gPtnE/U1PyIiP+NpLxEZKWHhJyKV\nRGSZiHycqG0miojUFpHJIrJWRNaISGe3e3KSiDwoIt+IyCoRmSQip7rdUyxEJEtEdp447EpETheR\nbBH5rvi1jps9xqKU/Xuu+N/PFSLykYjUdrPHWETavxP+7BERURFJKet3EnnkV+5pcD70IoDpqtoS\nwHkI0H6KyFkAHgCQoaptAFRC+O6+n00E0LtEbTCAWap6DoBZxe/9aiLs+5cNoI2qtgOwDsCQRDfl\noImw7x9E5GwAlwPYXJ4fSUj4BXkanIjUBJAJYAIAqGq+qv7kbleOSwZwmogkA6gKYLvL/cREVXMA\n7C1R7g/gzeK/fxPArxPalIMi7Z+qzlDVwuK3CwA0THhjDinlnx8AvIDwjLNy3chI1JFfkKfBNQWw\nC8Abxaf1r4tINbebcoqqbgPwPML/N80DsE9VZ7jbVVzUV9U8ACh+redyP/E0EMA0t5twkoj0Q/jJ\nUsvL+524h9+J0+DivS2XJAPoAGCcqrYHcBD+PmWyKL721R9AEwBnAqgmIhzM7lMiMhThWVnvuN2L\nU0SkKoChAIZX5HuJOPK7GEA/EdkE4D0Al4nI2wnYbqJsBbBVVRcWv5+McBgGRU8AG1V1l6oWAJgC\noIvLPcXDDyKSCgDFrztd7sdxIjIAwFUAbtZgjXFrhvD/nJcX50xDAEtF5KTPi497+AV9Gpyq7gCw\nRUSOTazuAWC1iy05bTOATiJSVUQE4f0LzA2dE0wFMKD47wcA+LeLvThORHoD+BOAfqp6yO1+nKSq\nK1W1nqqmFefMVgAdiv/bLBXH+TnjfgDviMgKAOcDeNrlfhxTfEQ7GcBSACsR/nfG17MFRGQSgPkA\nWojIVhG5E8BYAJeLyHcI3zEc62aPsShl//4OoAaAbBH5WkRedbXJGJSyfxX/nWAd/RIRlQ+P/IjI\nSAw/IjISw4+IjMTwIyIjMfyIyEgMPyIyEsOPiIzE8CMiI/0/vPXDyWlyS0AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10b802dd8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAT8AAAEzCAYAAABHZATQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAGzFJREFUeJzt3Xl4VOXZBvD7CRGUHQ1LFCFABGRT\nMCKgBBFEEAutS91F0Y9at34utSCICCK4tFZbxaJE9FOxithSFSSANCibLLIIiGGRLcimyJ5lnu+P\nCcjJmZBk5syc5b1/1+U1zsPMnOdc6u1Z3ve8oqogIjJNktsNEBG5geFHREZi+BGRkRh+RGQkhh8R\nGYnhR0RGKjP8RCRLRHaKyKoIf/aIiKiIpMSnPSKi+CjPkd9EAL1LFkXkbACXA9jscE9ERHFXZvip\nag6AvRH+6AUAjwLgKGki8p2orvmJSD8A21R1ucP9EBElRHJFvyAiVQEMBdCrnJ8fBGAQAFSrVu2C\nli1bVnSTREQntWTJkt2qWrci36lw+AFoBqAJgOUiAgANASwVkY6quqPkh1V1PIDxAJCRkaGLFy+O\nYpNERKUTke8r+p0Kh5+qrgRQ74SNbgKQoaq7K/pbRERuKc9Ql0kA5gNoISJbReTO+LdFRBRfZR75\nqeqNZfx5mmPdEBElCGd4EJGRGH5EZCSGHxEZieFHREZi+BGRkRh+RGQkhh8RGYnhR0RGYvgRkZEY\nfkRkJIYfERmJ4UdERmL4EZGRGH5EZCSGHxEZieFHREZi+BGRkRh+RGQkhh8RGYnhR0RGYvgRkZEY\nfkRkJIYfERmJ4UdERmL4EZGRGH5EZCSGHxEZieFHREZi+BGRkRh+RGQkhh8RGYnhR0RGYvgRkZEY\nfkRkJIYfERmJ4UdERmL4EZGRGH5EZCSGHxEZieFHREZi+BGRkRh+RGQkhh8RGYnhR0RGYvgRkZEY\nfkRkpDLDT0SyRGSniKw6ofaciKwVkRUi8pGI1I5vm0REzirPkd9EAL1L1LIBtFHVdgDWARjicF9E\nRHFVZvipag6AvSVqM1S1sPjtAgAN49AbEVHcOHHNbyCAaaX9oYgMEpHFIrJ4165dDmyOiCh2MYWf\niAwFUAjgndI+o6rjVTVDVTPq1q0by+aIiByTHO0XRWQAgKsA9FBVda4lIqL4iyr8RKQ3gD8B6Kaq\nh5xtiYgo/soz1GUSgPkAWojIVhG5E8DfAdQAkC0iX4vIq3Huk4jIUWUe+anqjRHKE+LQCxFRwnCG\nBxEZieFHREZi+BGRkRh+RGQkhh8RGYnhR0RGYvgRkZEYfkRkJIYfERmJ4UdERmL4EZGRGH5EZCSG\nHxEZieFHREZi+BGRkRh+RGQkhh8RGYnhR0RGYvgRkZEYfkRkJIYfERmJ4UdERmL4EZGRyly3l4j8\nKxQCcnOBvDwgNRVITweSeMgDgOFHFFihEJCVBcyd+0uta1dg4EAGIMDTXqLAys0NB1/jxkCTJuHX\nuXPDdWL4EQVWXl749dhR3rHXHTvc6cdreNpLFFCpqeHXUCgcfKFQ+H2DBu715LTCohAmztsU1XcZ\nfkQBlZ4evsZX8ppferp7PTnpwyVb8fAHy6P+PsOPKKCSksI3NzIzw6e6DRoE427vjn1H0GnMrJh/\nh+FHFGBJSUDz5uG//E5Vcc87SzFtlfWi5bzBl+GsZyr+eww/IvK8z9fuxB0Tv7LUnr2mHX574dlR\n/ybDj4g8a9+hApw3coal1q5hLUz5fRckV4rt/J3hR0Se9OR/vsEbX26y1LIfzMQ59Ws48vsMPyLy\nlCXf/4hrxs2z1P54RQvc293Z29QMPyLyhCMFRej67OfYtf/o8VrdGlUw99HuOPWUSo5vj+FHRK57\nZU4unp3+raX24e+74ILGdeK2TYYfEbkmd+d+9PxLjqU2oHNjPNm/Tdy3zfAjooQrLArh6nHzsGLr\nPkt9+fBeqFX1lIT0wPAjooR6f/EWPDp5haWWdXsGLmtZP6F9MPyIKCHy9h1G5zGzLbU+bRrglZs7\nQEQS3g/Dj4jiSlUx6P+WIHv1D5b6/CGXIbXWaS51xfAjojiateYH3PnmYkvtuWvb4bqM6KelOYXh\nR0SO++lQPs4fmW2pnXd2bXx4d+eYp6U5heFHRI66YFQ29hzMt9RmPtQN6fWqu9RRZGVGsIhkichO\nEVl1Qu10EckWke+KX+M3EpGIfGF8znqkDf7EEnyD+7TEprF9PRd8QPmO/CYC+DuAt06oDQYwS1XH\nisjg4vd/cr49IvK6Hw/mo/2obFt97ajecZmW5pQyw09Vc0QkrUS5P4BLi//+TQBzwPAjMk7r4dNx\nML/IUhvxq1a4/eImCdn+sXWJgVoVPrSM9ppffVXNAwBVzRORelH+DhH50Ccr8nDvu0tt9U1j+yas\nB+u6xCkpFf1+3G94iMggAIMAoFGjRvHeHBHF0ZGCIrR8fLqtPvfR7jj79KoJ7eXEdYmBo/llfb6k\naMPvBxFJLT7qSwWws7QPqup4AOMBICMjQ6PcHhG57OpXvsTSzT9Zard3ScOIfq1d6afkusQVFW34\nTQUwAMDY4td/R/k7RORxizftxbWvzrfVN4650pVpacecuC5xNMoMPxGZhPDNjRQR2QrgCYRD730R\nuRPAZgDXRbd5IvKqUEjR9LFPbfX/3HcJ2jas5UJHVtZ1iatUruj3RTVxZ6IZGRm6ePHisj9IRK4a\n/OEKvPfVFksts3ldvDWwo0sdRXbsbm+LFrW+Vd3XsiLf5QwPIjpu4+6D6P78HFt93VN9UDnZG9PS\nTnRsXWLg5wMV/S7Dj4gAAGmDP7HVJgzIQI9zE/ucvURh+BEZbtyc9Xhm+lpLrX7NKlj4WE+XOkoM\nhh+RofYcOIoLnpppq68Y0Qs1T03Mo+TdxPAjMlDzodOQX2QdIzKqf2vc2jnNnYZcwPAjMsjU5dvx\nwKRltnoip6V5BcOPyAClTUv7cvBlOKu2e4+SdxPDjyjgIt3FveuSJhh2VSsXuvEOhh9RQP1r2Tb8\n7z+/ttXdnpbmFQw/ooApCimaRZiW9vH9l6DNWe5PS/MKhh9RgLQd8Rn2Hym01U28oVEWhh9RAJT2\n5JVvn+qNKsnefZS8mxh+RD4X6YbGE79qhTsS9Ch5v2L4EfnUbVmLkLNul63OU9zyYfgR+cyWvYfQ\n9dnPbfUlw3rijOpVXOjInxh+RD4S6RT3xo6NMObqti50428MPyIfGPPpGvwjZ4OtzlPc6DH8iDzs\n5yMFaDdihq2e/WAmzqlfw4WOgoPhR+RRkU5x25xVEx/f39WFboKH4UfkMR8s3oI/Tl5hq3NamrMY\nfkQeUVgUQvrQabb6WwM7IrN5XRc6CjaGH5EHNB82DfmF9gVoeUMjfhh+RC5auGEPrh+/wFb36mpp\nQcLwI3JJpBsao37dBrd2auxCN+Zh+JGnHFuEOi8PSE0F0tPDa7MGyQ3j52PBhr22Ok9xE4vhR54R\nCgFZWcDcub/UunYFBg4MRgBu2n0Ql0ZYEHzZ45ejTrXKiW/IcAw/8ozc3HDwNW4cDrtQKPw+MxNo\n3tzt7mIT6RT3ts6NMbJ/Gxe6IYDhRx6Slxd+PXaUd+x1xw7/ht+oj1djwhcbbXWe4rqP4UeekZoa\nfg2FfjnyA4AGDdzrKVr7DhXgvJH2aWmzHu6GZnWru9ARlcTwI89ITw9f4yt5zS893b2eohHpFLd9\no9r46J6LXeiGSsPwI89ISgrf3MjMDJ/qNmjgr7u9kxZtxpApK211TkvzJoYfeUpSUvj6np+u8RUU\nhXBOhGlp79x1ES5OT3GhIyoPhh9RDJoM+QSq9jpvaHgfw48oCvNyd+Om1xfa6t+N7oNTKvnkPN1w\nDD+iClBVNBliXxB8zNVtcWPHRi50RNFi+BGV0zXj5mHJ9z/a6jzF9SeGH1EZNuw6gMv+/F9bffnw\nXqhV9RQXOiInMPyITiLSmL2BFzfB8F+1iur3THhwg18w/IgiGP7vVXhr/ve2eiynuEF/cIPfMPyI\nTvDjwXy0H5Vtq8955FKkpVSL6beD/OAGP2L4ERWLdIp7UZPT8c/fdXbk94P44AY/Y/iR8V7+PBfP\nffatre70XdwgPbghCBh+ZKyjhUVoMWy6rf7eoE7o1PQMx7cXlAc3BAXDj4wU6RQXiO+YPb8/uCFo\nGH5klOmrduDut5fY6omalubHBzcEVUzhJyIPArgLgAJYCeAOVT3iRGNETiptWtpjfc7FpalNMe8L\njrszTdThJyJnAXgAQCtVPSwi7wO4AcBEh3ojcsT5I2fgp0MFtvqGp/siKwsYnfVLjePuzBHraW8y\ngNNEpABAVQDbY2+JyBnfbN+Hvi99Yat/Pfxy1K5aGevWcdydyaIOP1XdJiLPA9gM4DCAGapqX7SA\nyAWRbmhc06Eh/vzb846/57g7s8Vy2lsHQH8ATQD8BOADEblFVd8u8blBAAYBQKNGfOQPxdddb36F\nmWt22uqR7uJy3J3ZYjnt7Qlgo6ruAgARmQKgCwBL+KnqeADjASAjIyPCM2+JYrdz/xF0HD3LVp/5\nUDek14u8WhrH3ZktlvDbDKCTiFRF+LS3B4DFjnRFVAGRTnGbplTD7EcuPen3OO7ObLFc81soIpMB\nLAVQCGAZio/wiBLhhex1eHHWd7Z6RQYqc9yduWK626uqTwB4wqFeiMrlSEERWj5un5b21sCOyGxe\n14WOyI84w4N8xY1paRRMDD/yhY9XbMd97y6z1XNH90EyV0ujKDD8yNNKm5b2ZL/WGNAlLfENUWAw\n/MizWg2fjkP5RbY6T3G9y09rlDD8yHPW/bAfvV7IsdWXP9ELtU7jamle5bc1Shh+5CmRbmjccOHZ\nGHtNOxe6oYrw2xolDD/yhOc+W4uXP19vq/MU1z/8Nlea4UeuKm21tIWP9UD9mqe60BFFy29zpRl+\n5JpIp7g3X9QIo3/T1oVuKFZ+myvN8KOEm7xkKx75YLmtzlNcf/PbXGmGHyVMaaulffpAV7Q6s6YL\nHZHT/DRXmuFHCdFx9Ezs3H/UUrswrQ4+uLuLSx2R6Rh+FFfz1u/GTa8ttNU3PH0lkpLEhY6Iwhh+\nDvDTqPZEKW1a2ht3XIjuLeq50BGRFcMvRn4b1Z4Id7yxCJ9/u8tSq5ychHVP9XGpIyI7hl+M/Daq\nPZ5ydx5Az7/811ZfM7I3TqtcyYWOiErH8IuR30a1x0ukMXuj+rfGrZ3TEt8MUTkw/GLkt1HtThsz\nbQ3+8d8NtjrH7JHXMfxi5LdR7U7Zc+AoLnhqpq2+aGgP1KvBaWnkfQy/GPltVLsTIp3i3t4lDSP6\ntXahG6LoMPwc4KdR7bF4b9FmDJ6y0lbnKS75EcOPynS0IISm9+ag6EAVVKpeDcl1DkIE+Ox/M9Gi\nQQ232yOKCsOPTurJqavx0rgCHN3SDBAACrRqn48FbzYL9Kk9BR/DjyLK3bkfPf+Sg4K91XB0SzNU\nqnUYIsD93c/B5s2C3Nzgn+ZTsDH8yKLktLSiA1UAAW7r3BhnVK9yvG7aOEYKHoYfHTfhi40Y9fFq\nS+26zBT8fMrZqFM1/N60cYwUXAw/wg8/H8FFT8+y1dc91QfJSUnISjZvHCMFH8PPcN2fn4ONuw9a\nav8c1AkXNT3j+HvTxjGSGRh+hvpkRR7ufXeppdajZT1MuP1C22dNGcdIZmH4GWb/kQK0HTHDVl85\nohdqnMoFwckcDD+DDJz4FWav3WmpvXxTB/Rtl+pSR0TuYfgZYOGGPbh+/AJLrWlKNcx+5FJ3GiLy\nAIZfgOUXhtB82DRbfdFjPVCPC4KT4Rh+ATVi6jeYOG+TpTb8qlYYeEkTdxoi8hiGX8Cs+2E/er2Q\nY6tvHHMlRLhaGtExDL+AKG21tJkPdUN6veoudETkbQy/AHgtZwNGf7rGUrvrkiYYdlUrlzoi8j6G\nn4/t2HcEncbYp6V9N7oPTqnEKRhEJ8Pw86lLnpmNrT8ettQm390ZGWmnu9QRkb8w/Hxm6vLteGDS\nMkutV6v6GH9bhksdEfkTw88nfj5SgHYRpqV98+QVqFaF/xiJKor/1fjArRMWYu53uy21V2/pgN5t\nOC2NKFoMPw+bt343bnptoaV2Tr3qyH6om0sdEQUHw8+DjhYWocWw6bY6FwQncg7Dz2OG/Wsl3l6w\n2VJ7sl9rDOiS5k5DRAHF8POItTt+Ru+/zrXUKiUJckf34bQ0ojiIKfxEpDaA1wG0AaAABqrqfCca\nM0UopGj6mH1a2uyHu6FpXU5LI4qXWI/8XgQwXVWvFZHKAKo60JMxxs1Zj2emr7XUftetKYb0Odel\njojMEXX4iUhNAJkAbgcAVc0HkO9MW8G2/afD6DJ2tq2eO7oPkjktjSghYjnyawpgF4A3ROQ8AEsA\n/EFVLUuBicggAIMAoFGjRjFsLhg6PT0LO34+YqlNuacLOjSq41JHRGaK5TAjGUAHAONUtT2AgwAG\nl/yQqo5X1QxVzahbt24Mm/O3fy3bhrTBn1iC78q2DbBpbF8GH5ELYjny2wpgq6oeG4U7GRHCz3T7\nDhfgvCft09JWj7wCVSvzZjuRW6L+r09Vd4jIFhFpoarfAugBYLVzrfnfTa8twLz1eyy18bdegF6t\nG7jUEREdE+uhx/0A3im+07sBwB2xt+R/X+buxs2vW6elnZtaE9P+0NWljoiopJjCT1W/BsBnKRUr\nbVra4mE9kVK9igsdEVFpAnnRKRQCcnOBvDwgNRVITweS4jyCZMiUFZi0aIulNurXbXBrp8bx3TAR\nRSVw4RcKAVlZwNwTZop17QoMHBifAFy9/Wdc+ZJ1WlqV5CSsHdWb09KIPCxw4ZebGw6+xo3DYRcK\nhd9nZgLNmzu3ndKmpc155FKkpVRzbkNEFBeBm06Qlxd+PXaUd+x1xw7ntvHy57m24Lu3ezNsGtuX\nwUfkE4E78kstfrhxKPTLkR8ANHBgdMm2nw7jYk5LIwqEwIVfenr4Gl/Ja37p6bH9bsZTM7H7wFFL\n7aN7uqA9Z2cQ+VLgwi8pKXxzIzMzfKrboEFsd3s/XLIVD3+w3FLrd96ZeOnG9g50S0RuCVz4AeGg\na948thscPx3Kx/kjs231NSN747TKlWLojoi8IJDhF6vf/mM+Fm3ca6lNGJCBHufWd6kjInIaw+8E\nOet24basRZZau4a1MPW+S1zqiIjiheEH4EhBEVo+bp+WtvTxy3F6tcoudERE8WZ8+P3xg+X4YMlW\nS23M1W1xY0c+eJUoyIwNv1Xb9uGqv31hqVWrXAmrnryC09KIDGBc+BWFFM0iTEvL+WN3NDqD6y8R\nmcKo8Htx5nd4YeY6S+2By9LxUK8WLnVERG4xIvy27D2Ers9+bquvf/pKVEriKS6RiQIdfqqK80dm\nY9/hAkt96n0Xo13D2i51RUReENjwe/+rLXj0wxWW2m/an4UXrj/fpY6IyEsCF34HjhaizROf2epr\nR/XGqadwWhoRhQUq/P4841v8bXaupfbGHReie4t6LnVERF4ViPCLNGbv5osaYfRv2rrUERF5na/D\nL78whN5/zcGG3QeP1047pRK+GtYT1av4eteIKM58mxBvztuEJ6Z+Y6m9e9dF6JKe4lJHROQnvgu/\nzXsOIfM565i96y5oiGevbcdpaURUbr4Jv1BIccuEhZi3fo+lbuKC4G6sS0wUNL4Iv49XbMd97y6z\n1F65uQOubJvqUkfuSfS6xERB5enw27X/KC4cPdNS63pOCt68oyOSDJ2Wlqh1iYmCzpPhp6p4+P3l\nmLJsm6XOJ6+cfF1ihh9R+Xku/L7M3Y2bX19oqY3q3xq3dk5zpyGPiee6xEQm8Uz47T9SgAuemon8\nwtDx2jn1quOTB7qicjIvZh0Tr3WJiUzjifB7ZvpajJuz3lL79IGuaHVmTZc68i6n1yUmMpWr4Rdp\nWhofLlo2J9YlJjKdK+F3tLAIvV7Iwfd7Dh2v1aiSjAWP9UA1TksjogRIeNJkfbERIz9ebalN+p9O\n6NzsjES3QkQGS2j4rdy2zxJ812ecjWeubZfIFoiIALh4zW/JsJ44w7BpaUTkHQkPv1dv6YDebcyb\nlkZE3pLQARJtz6rF4CMiT+DoMCIyUkLDb/9+YN26X6ZkERG5JaHht2cPMHp0+JFMDEAiclNCw69y\n5fCjmObODT+aiYjILQm/5nfiI5iIiNyS8PDjI5iIyAsSGn75+cD33/MRTETkvpgHOYtIJQCLAWxT\n1atO9tmUFGDoUD6CiYjc50QE/QHAmvJ8sHr18GOYGHxE5LaYYkhEGgLoC+B1Z9ohIkqMWI/B/grg\nUQActUdEvhL1NT8RuQrATlVdIiKXnuRzgwAMKn57VERWRbtNH0gBsNvtJuIoyPsX5H0Dgr9/FX78\nu6hqVFsSkTEAbgVQCOBUADUBTFHVW07yncWqmhHVBn2A++dfQd43gPsXSdSnvao6RFUbqmoagBsA\nzD5Z8BEReQnvuxKRkRx5mKmqzgEwpxwfHe/E9jyM++dfQd43gPtnE/U1PyIiP+NpLxEZKWHhJyKV\nRGSZiHycqG0miojUFpHJIrJWRNaISGe3e3KSiDwoIt+IyCoRmSQip7rdUyxEJEtEdp447EpETheR\nbBH5rvi1jps9xqKU/Xuu+N/PFSLykYjUdrPHWETavxP+7BERURFJKet3EnnkV+5pcD70IoDpqtoS\nwHkI0H6KyFkAHgCQoaptAFRC+O6+n00E0LtEbTCAWap6DoBZxe/9aiLs+5cNoI2qtgOwDsCQRDfl\noImw7x9E5GwAlwPYXJ4fSUj4BXkanIjUBJAJYAIAqGq+qv7kbleOSwZwmogkA6gKYLvL/cREVXMA\n7C1R7g/gzeK/fxPArxPalIMi7Z+qzlDVwuK3CwA0THhjDinlnx8AvIDwjLNy3chI1JFfkKfBNQWw\nC8Abxaf1r4tINbebcoqqbgPwPML/N80DsE9VZ7jbVVzUV9U8ACh+redyP/E0EMA0t5twkoj0Q/jJ\nUsvL+524h9+J0+DivS2XJAPoAGCcqrYHcBD+PmWyKL721R9AEwBnAqgmIhzM7lMiMhThWVnvuN2L\nU0SkKoChAIZX5HuJOPK7GEA/EdkE4D0Al4nI2wnYbqJsBbBVVRcWv5+McBgGRU8AG1V1l6oWAJgC\noIvLPcXDDyKSCgDFrztd7sdxIjIAwFUAbtZgjXFrhvD/nJcX50xDAEtF5KTPi497+AV9Gpyq7gCw\nRUSOTazuAWC1iy05bTOATiJSVUQE4f0LzA2dE0wFMKD47wcA+LeLvThORHoD+BOAfqp6yO1+nKSq\nK1W1nqqmFefMVgAdiv/bLBXH+TnjfgDviMgKAOcDeNrlfhxTfEQ7GcBSACsR/nfG17MFRGQSgPkA\nWojIVhG5E8BYAJeLyHcI3zEc62aPsShl//4OoAaAbBH5WkRedbXJGJSyfxX/nWAd/RIRlQ+P/IjI\nSAw/IjISw4+IjMTwIyIjMfyIyEgMPyIyEsOPiIzE8CMiI/0/vPXDyWlyS0AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10b802dd8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "eta = 0.0005             \n",
    "w = np.array([2., 1.])\n",
    "\n",
    "f = A.dot(w)\n",
    "\n",
    "fig = plt.figure(figsize=(5,5))\n",
    "ax = fig.gca()\n",
    "ax.set_xlim((4,14))\n",
    "ax.set_ylim((4,14))\n",
    "ln = plt.Line2D(xdata=x, ydata=f, linestyle='-',linewidth=2)\n",
    "ax.add_line(ln)\n",
    "\n",
    "plt.plot(x,y,'bo', alpha=0.5, markersize=5)\n",
    "\n",
    "for epoch in range(30):\n",
    "    f = A.dot(w)\n",
    "    err = y-f\n",
    "    \n",
    "    ln.set_xdata(x)\n",
    "    ln.set_ydata(f)\n",
    "    \n",
    "    E = np.sum(err**2)/N\n",
    "    dE = -2.*A.T.dot(err)/N\n",
    "    \n",
    "    if epoch%1 == 0: \n",
    "        print(epoch,':',E)\n",
    "        # print(w)    \n",
    "        \n",
    "    w = w - eta*dE\n",
    "\n",
    "\n",
    "    display.clear_output(wait=True)\n",
    "    display.display(plt.gcf())\n",
    "    time.sleep(0.1)\n",
    "\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Implementation in pyTorch\n",
    "\n",
    "### Fitting a polynomial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 : 1492762.4126231002\n",
      "100 : 17.19655854820136\n",
      "200 : 17.10613625301379\n",
      "300 : 17.016297628823512\n",
      "400 : 16.927038907069008\n",
      "500 : 16.838356343551855\n",
      "600 : 16.75024621824876\n",
      "700 : 16.662704835154873\n",
      "800 : 16.575728522129136\n",
      "900 : 16.489313630739936\n",
      "999 : 16.404312357809356\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pylab as plt\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.autograd\n",
    "from torch.autograd import Variable\n",
    "\n",
    "x = np.array([10., 8., 13., 9., 11., 14., 6., 4., 12., 7., 5.])\n",
    "yy = np.array([8.04, 6.95, 7.58, 8.81, 8.33, 9.96, 7.24, 4.26, 10.84, 4.82, 5.68])\n",
    "y = Variable(torch.DoubleTensor(yy))\n",
    "\n",
    "# Setup the feature (vandermonde) matrix\n",
    "N = len(x)\n",
    "degree = 3\n",
    "xx = np.vstack((np.power(x,i) for i in range(degree+1))).T\n",
    "# Create the system matrix\n",
    "A = Variable(torch.from_numpy(xx).double())\n",
    "w = Variable(torch.randn(degree+1).double(), requires_grad=True)\n",
    "# learning rate\n",
    "eta = 0.00000005\n",
    "\n",
    "for epoch in range(1000):\n",
    "    ## Compute the forward pass\n",
    "    f = torch.matmul(A, w)\n",
    "    #print(f)\n",
    "    E = torch.sum((y-f)**2)/N\n",
    "\n",
    "    if epoch%10000 == 0: \n",
    "        print(epoch,':',E.data[0])\n",
    "\n",
    "    # Compute the gradients by automated differentiation\n",
    "    E.backward()\n",
    "    \n",
    "    # For each adjustable parameter \n",
    "    # Move along the negative gradient direction\n",
    "    w.data.add_(-eta * w.grad.data)\n",
    "    #print(w.grad.data)\n",
    "    \n",
    "    # Reset the gradients, as otherwise they are accumulated in param.grad\n",
    "    w.grad.zero_()\n",
    "    \n",
    "print(epoch,':',E.data[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl4lNX5xvHvwyICglHBKqCiYnFF\n0bQKaK1YwCVBXGoNWrFqUaGtda2IVBRbUFuX1hUR3BAtqCgiIGIVbQEFkU1wrbKJBCw/NxQw5/fH\nEywgkGUmc2beuT/XlWuSySTvM5Dceee85zzHQgiIiEjuqxW7ABERSQ8FuohIQijQRUQSQoEuIpIQ\nCnQRkYRQoIuIJIQCXUQkIRToIiIJUWGgm9lQM1tuZnM387nLzSyYWZOaKU9ERCqrTiUe8wBwB/DQ\nhnea2W5AJ2BhZQ/WpEmT0LJlyyqUJyIiM2bMWBFCaFrR4yoM9BDCZDNruZlP3QpcCTxd2aJatmzJ\n9OnTK/twEREBzOyjyjyuWmPoZtYVWBJCmFWdrxcRkfSrzJDLRsysAdAX6FzJx/cEegLsvvvuVT2c\niIhUUnXO0PcG9gRmmdmHQAvgDTPbZXMPDiEMDiEUhhAKmzatcAhIRESqqcpn6CGEOcDO6z8uD/XC\nEMKKNNYlIiJVVJlpiyOAKUBrM1tsZufVfFkiIlJVlZnlUlLB51umrRoREak2rRQVEUmIKo+hi0gl\nfPstrF4NX33lbxu+v6W3EKBtWzjiCNhhh9jPQHKQAl0kHb7+GiZPhgkT/G3evNS+3wEHQPv2/tah\nA7RqBWbpqVUSS4EuUh0hwIIF/wvwl17yUN9mG/jJT+Dkk6FxY2jQoPJva9fC66/Dv/8N//oXjBwJ\n993nx2vS5H/h3r49FBbCtttG/SeQ7KNAF6msVatg0iQP8PHjYdEiv791a+jZE7p0gaOPhoYNq/f9\n69WDY47xN4CyMpg/3wN+fcg/84x/rm5dOOwwD/fzzoP990/9+UnOsxBCxg5WWFgY1MtFcspbb8ET\nT3iIT53qY+ONGsHPfuYB3qULZLLhXGkpTJni4f7vf/sZ/Zo1cNZZ0L8/7LVX5mqRjDGzGSGEwgof\np0AX2YyZM+GGG+DJJ/3jwsL/BfgRR/gZcjZYuRJuvBH+/ndYtw5+/Wu45hpo1ix2ZZJGlQ10TVsU\n2dDrr0PXrnDooT680q8fLFvm999wAxx1VPaEOcBOO8FNN8H77/uwz333wd57wxVXwAot3s43CnQR\n8GGM44+HH/8YXn0Vrr8ePvzQb3/wg9jVVaxZM7jzTnj7bTj9dLjlFh9+6d8fPvssdnWSIQp0yW+T\nJ/t4ePv2MH06DBzoQd6vHxQUxK6u6vbaCx58EObMgc6d4brr/L6//MXnwkuiKdAl/4Tgwyk//anP\nSpk71wPvww/hqqt8umGu239/GDXK/0j96Ec+BNOqFdx9t19ElURSoEv+CMGnGx55pJ+Vv/su3H47\n/Oc/cNll1Z9umM0OOwzGjfNXInvtBb16wb77wpgxsSuTGqBAl/wwd64Pqxx/vM8fv+suv5D4u99B\n/fqxq6t5Rx3loT5unE+77NrVZ8dkcJab1DwFuiTbunUwaJCfqb7/PgweDO+9BxddlH8rLc3guON8\nPv0ZZ/jw0rnnaggmQbRSVJLr7behRw+YNg1OPdXPynfeueKvS7r69eHRR33opX9/+OADXzzVpEns\nyiRFOkOX5Ckrg1tvhUMOgXfe8fAaOVJhviEzuPZa/7eZNs0XSy1YELsqSZECXZLl/fd99sqll/qF\nz3nzoKREnQq3pKQE/vlP+PxzD/UXXohdkaRAgS7JUFbmQypt2sCsWTBsmDey2nXX2JVlv3bt4LXX\nYLfdfIz9nntiVyTVpECX3PfRR76Ipndvn5I4dy6cc47Oyqtijz284Vfnzn7B+JJLvBGZ5BQFuuSu\nEGDIEDjoIJ+5cc89Ps98t91iV5abGjf2VzUXXwy33QYnnaS2ATmmwkA3s6FmttzM5m5w381mtsDM\nZpvZU2aWg2ukJactWQInnujdBQ87zJe6X3CBzspTVaeOh/ndd/sfxw4d/BWQ5ITKnKE/ABy3yX0T\ngQNDCG2Ad4A+aa5LZMvGjYMDD/Rdgv72N1/Gv+eesatKlgsv9H/nRYu8YdnUqbErkkqoMNBDCJOB\nTze57/kQwrryD6cCLWqgNpGNheDTEYuKfFOJWbPgt7+FWho5rBGdOnkXyu2285lDjz8euyKpQDp+\nE84Fxm3pk2bW08ymm9n00tLSNBxO8tKaNT68cuml0K2bt7jdZ5/YVSXffvv5PPUf/QjOPBMmToxd\nkWxFSoFuZn2BdcDwLT0mhDA4hFAYQihs2rRpKoeTfLVihZ8t3n8/9O3ri4SS2EgrWzVpAs895x0c\nf/5zLUDKYtUOdDPrARQBZ4ZM7mMn+WXePB/DnTYNhg/3XYM0xJJ5jRp5h8Z69XzIa+XK2BXJZlTr\nN8PMjgP+AHQNIXyV3pJEyj33nC96Wb0aXn4ZunePXVF+22MPeOopv1B66qlq6pWFKjNtcQQwBWht\nZovN7DzgDqARMNHM3jQzLS2T9AnBt1ArLvb9MV97DQ4/PHZVAt6CeOhQ/wPbq5fa72aZCrsthhBK\nNnP3/TVQi4if9V10kYfGKafAQw9pvDzbnHkmzJ8Pf/qTj6tfemnsiqScBiMle5SWekOtoUPhmmt0\n8TObXX+9D7tcfjk8+2zsaqScAl2yw9y5fvHztdf84ueAAbr4mc1q1fLNqNu29Y6Ns2fHrkhQoEs2\nGDvWx2a//loXP3NJw4be+6VRI7/e8cknsSvKewp0ievuuz0MWrWC11/Xxc9c07y5h3ppKZx8sv9R\nlmgU6BLPX//qMyVOPBFeeQVaqINETios9IvXU6bA+edr5ktECnTJvBB8jPzyy33l4ZNP6uJnrjvt\nNP8/HT4cBg6MXU3e0ibRklkh+PL9gQPh7LN9OX8d/RgmQt++Pp2xb19o3dpnwUhG6QxdMicE3wln\n4EDvXT5smMI8Scz8D/QRR8AvfwkzZsSuKO8o0CUzysp8wdDtt/uOOHffrWmJSbTttjB6NF9tvyOl\nHbtweO+H6DDoRUbPXBK7sryg3yipeevWwa9+BffeC336eE9z7SyUWKOXrqOkuC8NVn/JfU8OoHTF\nZ/R5co5CPQMU6FKz1q71peIPPeSrC//0J4V5wt084W1m7bg7lxRdRptl79F7yj9YvfZbbp7wduzS\nEk+BLjXnm2989sM//gE33wz9+inM88DSVasBeP6H7Ri9/9FcNHUk+5R+9N39UnMU6FIzvvoKunb1\nRSd33OFTFCUvNCuo/9371x/bky/qNWDQ+L/TvHG9iFXlBwW6pN/nn/tioYkTfdZD796xK5IMuqJL\na+rXrQ3Apw2254aO53HY0gXc+dm0yJUlnwJd0mvVKujSxVd+Dh8O554buyLJsG5tmzPwlINoXlAf\nA6Z1OJHlR/yEg++6ERYvjl1eolkmd48rLCwM06dPz9jxJMNWroTOnWHOHHjsMe9nLgLwwQdw4IG+\nN+zo0bqWUkVmNiOEUFjR43SGLumxapX3Mp83z39hFeayob32guuu82sqTzwRu5rEUqBL6r780sfM\n582Dp5+GE06IXZFko0su8f7pv/0t/Pe/satJJAW6pOabb/xsfOpUGDHCx89FNqdOHRgyxFvtXnll\n7GoSqTKbRA81s+VmNneD+3Y0s4lm9m757Q41W6ZkpXXrfNHQ88/7L6qaMUlFDj3Uz9SHDPHNTCSt\nKnOG/gBw3Cb3XQVMCiHsA0wq/1jySVkZ/PrXPh56222+tF+kMq67DvbcE3r21IYYaVZhoIcQJgOf\nbnL3ScCD5e8/CHRLc12SzULwnd4feAD69/dmWyKV1aCB9/V55x244YbY1VTa6JlL6DDoRfa8amzW\nNhyr7hj6D0IIHwOU3+6cvpIk6113nXdN/P3v4Y9/jF2N5KJOnbzF7o03+jTXLDd65hL6PDmHJatW\nE4Alq1ZnZcOxGr8oamY9zWy6mU0vLS2t6cNJTbv1Vg/0X/3Kt5DTfGKprltugYICH7r79tvY1WzV\nzRPeZvXajWvMxoZj1Q30T8xsV4Dy2+VbemAIYXAIoTCEUNi0adNqHk6ywtChPtRy2mlw333qZy6p\nadLEr79MmwZ33hm7mq3aUmOxbGs4Vt3fyGeAHuXv9wCeTk85krVGjfIzqS5d4JFHoHbt2BVJEnTv\n7j9TV18NCxfGrmaLNmw4Vpn7Y6nMtMURwBSgtZktNrPzgEFAJzN7F+hU/rEk1fjx/ovXrp3Paqmn\nrnmSJmZwzz1+ob1XL7/NQhs2HFuvft3aXNGldaSKNq/CDR1DCCVb+NSxaa5FstGrr/rCoQMOgGef\nhYYNY1ckSdOyJQwYAJdd5r3zf/GL2BV9T7e2zQEfS1+6ajXNCupzRZfW392fLdScS7bsjTfgmGNg\n111h8mTYWZOZpIasW+evABcuhPnzYccdY1eUVdScS1KzYIGPbRYUeF9zhbnUpPVtAVauhCuuiF1N\nzlKgy/ctWeJtcGvXhhdegN12i12R5IODD/adrYYO9VeEUmUKdNnYZ595t8RVq/xi6D77xK5I8sm1\n10KzZnDNNVl7gTSbKdDlf9au9Tnmb73l0xQPOSR2RZJv6tf3KYyvvAKTJsWuJuco0MWFABdc4OPl\ngwf7kItIDOefDy1aeFsJnaVXiQJd3IABMGyYv+RV50SJqV496NsXpkyBCRNiV5NTFOjiXROvvRbO\nOcdvRWI791zYfXedpVeRAj3fTZzoS/o7dfKhFjXbkmywzTbQrx+8/jo891zsanKGFhbls9mz4cgj\nfbOBV16Bxo1jV5RVRs9ckvUrAxNt7Vpo3Rp22AGmT8/rkw0tLJKtW7zYpyc2bgxjxyrMN5Er/a8T\nrW5dH3J54w145pnY1eQEBXo++r//8zD//HMYN85nFMhGcqX/deKddRa0auXXdsrKYleT9RTo+WbN\nGp9rPn++d0486KDYFWWlXOl/nXh16vhZ+qxZ8NRTsavJegr0fBKCXwB94QXvm/Gzn8WuKGvlSv/r\nvFBS4mPp/fvrLL0CCvR80r8/PPQQXH899OhR4cPzWa70v84Lder4kMvcub6CWbZIs1zyxdChcN55\nPr93yJC8njFQWZrlkkW+/RbatPFXmXPm5N2OWZWd5aJAzwcTJsCJJ/oQy5gxPntA8lJO/5EaORJO\nPx2GD/cdtPKIAl3c7NnQoYPPFJg8GRo1il2RRLJ+KuaGs3fq163NwFMOyo1QLyvzhnHffAPz5vlQ\nTJ7QPHSBTz6B4mLYfnvfPk5hntdyfipmrVpw3XXwzjvw6KOxq8lKCvSkWr0aunWDFSt8UUbzHDgD\nkxqViKmY3br5Wfr11/u2dbKRlALdzC4xs3lmNtfMRpjZtukqTFIQgl8AnToVHnkEDj00dkWyFaNn\nLqHDoBfZ86qxdBj0Yo2tRk3EVEwzP0t//314+OHY1WSdage6mTUHfgcUhhAOBGoDZ6SrMEnBgAEw\nYgQMHAgnnxy7GtmKTLYYSMxUzOJiOOwwP0tfuzZ2NVkl1SGXOkB9M6sDNACWpl6SpOQf//A5u2ef\nDX/4Q+xqpAKZHNfu1rY5A085iOYF9TGgeUH93LkguiEzD/MPP/TWz/KdlGa5mNnFwJ+A1cDzIYQz\nt/Z4zXKpYa+9BkcfDYWFvhq0Xr3YFUkF9rxqLJv7DTTgP4NOzHQ5uSMEaNcOPv7YL5Im/Ge9xme5\nmNkOwEnAnkAzoKGZnbWZx/U0s+lmNr20tLS6h5OKLFoEJ50Eu+4KTz6Z+B/wpEjEuHYM68/SFy70\nRXMCpDbk8jPgPyGE0hDCWuBJoP2mDwohDA4hFIYQCps2bZrC4WSLvvjCxxW/+sqnJ+rfOWckZlw7\nhk6doH17+POf4euvY1eTFVIJ9IXAEWbWwMwMOBaYn56ypNLKyrzF6Jw58PjjsP/+sSuSKkjMuHYM\n68/SFy/2dhZCtZdahRCmmdko4A1gHTATGJyuwqSS+vSBp5+Gv/0NjjsudjVSDd3aNleAV1fHjqw4\n9HC4uj9H/qcZOzUtyK12BmmW0iyXEMK1IYR9QwgHhhB+GUL4Jl2FSSUMGwY33QS9esFvfhO7GpGM\nG/3mUi494BSafL6Sklnj835nKa0UzVUvvwwXXODjiLffru6JkpdunvA2k5sdwGst9uecGWOwUJZb\n7QzSTIGei95/H045Bfbe2+ed51GTIpENrW9b8HDbE9lj1TKO/uCNje7PNwr0XLNqFRQV+ftjxkBB\nQdx6RCJaP71zfOv2lDYs4KyZYze6P98o0HPJunXeD/r9932ueatWFX5JpvqEiMSwftrn2tp1GdGm\nCx3fn06rL0rzdtqnAj2XXHIJTJwI997rK0IrkMk+ISIxbDjtc8Qhx1NWy7jry9fzdpaLBl9zxb33\nwh13wGWXwa9+Vakv2VqfkHz9gZfk2Wja57Kn+eGYx+Hrv8K2+df8VWfoueCll3xa4gknwI03VvrL\nEtH/WqQqeveGlSt9u7o8pEDPdh98AKeeCvvs47u0VGFz3KT2CdF1Admijh2hdWu4887YlUShQM9m\nn30GXbt6Z7lnnvGt5KogiX1CdF1AtsrMF9pNmwYzZsSuJuMU6Nnq22/hzDNhwQIYNapSM1o2lcQ+\nITm/L6bUvLPPhgYN4K67YleScboomq2uvto7J955p7+MrKak9QnRdQGpUEGBN6x76CH4y19ghx1i\nV5QxOkPPRg8/7D1aLrrIXz7Kd5J6XUDSrFcvb6k7bFjsSjJKgZ5tpk6F88+HY47xHi2ykSReF5Aa\ncPDB0KED3H23t5jOEwr0bLJoEXTrBi1a+LSrunVjV5R1knhdQGpIr17w3nu+GC9PpLSnaFVpT9Gt\n+PJLOOoo/wGcOlUbVYik6ptvYPfd4YgjfM+AHFbje4pKGoXgqz/ffBNGjFCYi6RDvXo+fPnss/DR\nR7GryQgFejYYMMCHWG68EU7UTu8iaXPBBX57771x68gQBXpsTzwB117rc2cvvzx2NSLJsvvuvoH6\nkCE+BJNwCvSYZs70IG/Xzs8gtOuQSPr17g2lpb5AL+EU6LEsWwYnnQQ77ui9zfOwM5xIRhx7rPdC\nyoP+LikFupkVmNkoM1tgZvPNrF26Cku0r7/2LeRWrvQeLbvsErsikeSqVcunME6Z4q+KEyzVM/Tb\ngfEhhH2Bg4H5qZeUcCFAz57+w/Xgg9C2beyKRJKvRw+oXz/x/V2qHehm1hj4CXA/QAhhTQhhVboK\nS6ybbvKl/dddB6edFrsakfywww7e7G74cN+XN6FSOUPfCygFhpnZTDMbYmYNN32QmfU0s+lmNr20\ntDSFwyXAM89Anz7wi19Av36xqxHJL716werV8MADsSupMdVeKWpmhcBUoEMIYZqZ3Q58FkLYYlLl\n9UrROXOgfXvYd1+YPNlf/olIZrVvDytWeFvqWrkzJyQTK0UXA4tDCNPKPx4FHJrC90uu5ct9Lmyj\nRjB6NKMXfKodd0Ri6NUL3n0XJk2KXUmNqHaghxCWAYvMbH2bu2OBt9JSVZJ8841vIffJJ/D004xe\njnbcEYnl5z+HJk0SO4Ux1dccvwWGm9ls4BDgz6mXlCAheE/zV1/1cbsf/Ug77ojEtL6/y5gxsHBh\n7GrSLqVADyG8GUIoDCG0CSF0CyH8N12FJcItt3iD/X79/EIo2nFHJLoLL/STrQT2d8mdqwK5ZuxY\nuOIKH27p3/+7u7Xjjkhke+wBRUWJ7O+iQK8J8+ZBSQkccogvHtrgarp23BHJAr17+2SFJ56IXUla\nKdDTbcUKn9HSoIE31W+48dR87bgjkgU6dYJWrXyLugSpE7uARFmzxodYli6Fl1+G3Xbb7MO6tW2u\nABeJqVYt31Smb1/f/GKPPWJXlBY6Q0+XEPxl3OTJMHQoHH547IpEZGvOOMNvH3ssbh1ppEBPl7/9\nzS+yXH01dO8euxoRqchee/l+o48+GruStFGgp8P48XDppdCtm28nJyK5oaQEZs+Gt5KxJlKBnqp5\n83yO+UEHeRfFHOoPIZL3Tj/df2dHjIhdSVoofVLxySe+qXODBt5JcbvtYlcUzeiZS9SfRnLPLrtA\nx44+7FLNRoXZRIFeXatX+xZyy5f7MuLdd49dUTSjZy5RfxrJXd27wwcfwOuvx64kZQr06igr882d\nX3vNG+YXVtjVMtHUn0Zy2sknwzbbJOLiqAK9Ovr29R3Eb7rJfxjynPrTSE4rKPCh08cfh2+/rfjx\nWUyBXlVDh8KgQb4v6GWXxa4mK6g/jeS8khJYtgxeeil2JSlRoFfFpElwwQW+bPiOO8AsdkVZQf1p\nJOcVFfmkhhyf7aJAr6z5831Zf+vWMHIk1K0bu6Ksof40kvPq1/fh01GjcroDo3q5VEZpqY+x1asH\nzz4L228fu6Kso/40kvO6d/e1JOPH+wy2HKRA38DomUu4ecLbLF21mmYF9bmiS2u67beTrwD9+GMf\nX2vZMnaZIlITjj3Wt6d79NGcDXQNuZTb3Fzqq5+YxeJuZ8C//+1/udVwSyS56tb1laNjxsDnn8eu\nploU6OU2N5f6whcfosWEp2HgQDjttEiViUjGlJT4osGnn45dSbWkHOhmVtvMZprZs+koKJZN50yf\nOmcSv5vyOI+36Qx/+EOkqkQko9q391XfOTrbJR1n6BcD89PwfaLacM704QvnMHD83/nXHm248/TL\nND1RJF/UquV90p9/3ncfyzEpBbqZtQBOBIakp5x41s+l3vPTJdz71J9YWLALl552DZeecEDs0kQk\nk7p3h3XrfApjjkl1lsttwJVAozTUElW3ts3Z5r8rOej081lXqzZ9zhtIn5L2moonkm/atIH99vNh\nlwsvjF1NlVQ70M2sCFgeQphhZj/dyuN6Aj0Bds/mjoRffMEJV50PX66ESZMY2b597IpEJAYzP0vv\n1w8WLdri3sDZKJUhlw5AVzP7EHgM6Ghmj2z6oBDC4BBCYQihsGnTpikcrgatXeuzWGbM8AY9CnOR\n/LZ+v9HHH49bRxVVO9BDCH1CCC1CCC2BM4AXQwhnpa2yTCkrg/POgwkT4N57oWvX2BWJSGytWsGP\nf5xzLXU1D/2qq3zR0IABcP75sasRkWxRUgIzZ8KCBbErqbS0BHoI4aUQQlE6vldG3Xor3Hwz9O7t\nPc5FRNb7xS98PD2H5qTn7xn6o4/CpZd6B8Xbb9dccxHZ2K67wjHHeKDnyH6j+RnoEyfCOefA0UfD\nI49A7doVfomI5KHu3eHdd33CRA7Iv0CfMQNOOQX23RdGj4Ztt41dkYhkq1NO8aZdOTLskl+B/t57\ncPzxsNNO3vO4oCB2RSKSzXbYAU44AR57LCf2G82fQF+2DLp08WmKEyZAs2axKxKRXFBSAkuXwiuv\nxK6kQvkR6J9/7n9lly2DsWN9GzkRkcooLoaGDXNiTnryA33NGh8Hmz3bm+1okwoRqYoGDXzXslGj\nPE+yWLIDvazMZ7O88ALcf7+Pn4uIVFX37vDf//pwbRZLbqCHAJdd5lenBw2CHj1iVyQiuapTJ59M\nkeWzXZIb6DfcALfdBhdfDFdeGbsaEclldevCz3/uW9N9+WXsarYomYE+aBD88Y9w9tlwyy1aBSoi\nqSspga++gmeeiV3JFiUv0P/6V+jTx8e8hg71LaVERFJ15JHQokVWD7skK+1uvx0uvxxOPx0efFBL\n+kUkfdbvNzpuHKxaFbuazUpOoN91F/z+9z5F8ZFHoE6qu+uJiGzi5JN9v9Esne2SjEAfPNhb4BYX\n+8uhunVjVyQiSXT44dCkCYwZE7uSzcr9QB82DC64wFeCjhwJ22wTuyIRSaratT1rxo3zM/Usk9uB\n/vDDvn1c587wxBNQr17sikQk6YqL4dNPYerU2JV8T+4G+ogRvgq0Y0e1wRWRzOnc2Yd1s3DYJTcD\nfeRI+OUv4aijfE5o/fqxKxKRfNG4sW+O8+yzsSv5ntwL9NGjfY55u3b+D9qgQeyKRCTfFBXBW2/B\nBx/ErmQj1Q50M9vNzP5pZvPNbJ6ZXZzOwjbr2Wd9jnlhITz3HGy3XY0fUkTke4qK/DbLztJTOUNf\nB1wWQtgPOALobWb7p6eszRg/3jd0PuQQf79Roxo7lIjIVu29N+y3X9aNo1c70EMIH4cQ3ih//3Ng\nPtA8XYVtZOJE70d84IE+oX/77WvkMCIilVZcDC+/DJ99FruS76RlDN3MWgJtgWnp+H7fM2mSb+r8\n/PO+x5+ISGxFRbB2redSlkg50M1sO+AJ4PchhO/9qTKznmY23cyml5aWVu8gAwf6fn477ZRasSIi\n6dKuHey4Y1aNo6cU6GZWFw/z4SGEJzf3mBDC4BBCYQihsGnTptU9kMbMRSS71Knju6CNHQvffhu7\nGiC1WS4G3A/MDyHckr6SRERyRFERrFgBr70WuxIgtTP0DsAvgY5m9mb52wlpqktEJPsdd5z3d8mS\n2S6pzHJ5NYRgIYQ2IYRDyt+eS2dxIiJZraDAV6xnyTh67q0UFRHJJsXFMGcOfPRR7EoU6CIiKcmi\nVaMKdBGRVPzwh/6WBePoCnQRkVQVFcE//wlffBG1DAW6iEiqiothzRpvUxKRAl1EJFUdOniPqcjj\n6Ap0EZFU1a37v1WjZWXRylCgi4ikQ1ERfPIJTJ8erQQFuohIOhx/PNSqFXW2iwJdRCQddtzRx9Ij\njqMr0EVE0qW4GN58ExYtinJ4BbqISLqsXzU6dmyUwyvQRUTSZd99fb/RSOPoCnQRkXQx87P0SZPg\nyy8zfngFuohIOhUXwzffeKhnmAJdRCSdjjoKGjeOMttFgS4ikk7bbANdunigZ3jVqAJdRCTdiorg\n449h5syMHlaBLiKSbiec4BdIMzzbRYEuIpJuTZpAu3YZH0dPKdDN7Dgze9vM3jOzq9JVlIhIzisu\nhhkzYOnSjB2y2oFuZrWBO4Hjgf2BEjPbP12FiYjktAirRlM5Q/8x8F4I4YMQwhrgMeCk9JQlIpLj\nDjgA9tgjo+PoqQR6c2DDDjSLy+/biJn1NLPpZja9tLQ0hcOJiOQQMx92eeEFWL06I4dMJdBtM/eF\n790RwuAQQmEIobBp06YpHE66pYKOAAAEHklEQVREJMcUFXmYv/hiRg6XSqAvBnbb4OMWQOZG/0VE\nst1PfwoNG2Zstksqgf46sI+Z7Wlm2wBnAM+kpywRkQSoVw86d/ZAD98bwEi7agd6CGEd8BtgAjAf\n+EcIYV66ChMRSYTiYli8GGbNqvFD1Unli0MIzwHPpakWEZHkOeEE32903boaP1RKgS4iIhX4wQ/g\nucyc92rpv4hIQijQRUQSQoEuIpIQCnQRkYRQoIuIJIQCXUQkIRToIiIJoUAXEUkICxnoL/DdwcxK\ngY8ydsDqawKsiF1Emuk5Zb+kPR/Qc0qXPUIIFbarzWig5wozmx5CKIxdRzrpOWW/pD0f0HPKNA25\niIgkhAJdRCQhFOibNzh2ATVAzyn7Je35gJ5TRmkMXUQkIXSGLiKSEAr0TZhZbTObaWaZ2QSwhplZ\ngZmNMrMFZjbfzNrFrilVZnaJmc0zs7lmNsLMto1dU1WZ2VAzW25mcze4b0czm2hm75bf7hCzxqra\nwnO6ufxnb7aZPWVmBTFrrKrNPacNPne5mQUzaxKjts1RoH/fxfiWeklxOzA+hLAvcDA5/tzMrDnw\nO6AwhHAgUBvfzzbXPAAct8l9VwGTQgj7AJPKP84lD/D95zQRODCE0AZ4B+iT6aJS9ADff06Y2W5A\nJ2BhpgvaGgX6BsysBXAiMCR2LelgZo2BnwD3A4QQ1oQQVsWtKi3qAPXNrA7QAFgauZ4qCyFMBj7d\n5O6TgAfL338Q6JbRolK0uecUQni+fP9hgKlAi4wXloIt/D8B3ApcCWTVRUgF+sZuw/+TymIXkiZ7\nAaXAsPJhpCFm1jB2UakIISwB/oKfGX0M/F8I4fm4VaXND0IIHwOU3+4cuZ50OxcYF7uIVJlZV2BJ\nCKHmd32uIgV6OTMrApaHEGbEriWN6gCHAneHENoCX5J7L+M3Uj6ufBKwJ9AMaGhmZ8WtSipiZn2B\ndcDw2LWkwswaAH2BP8auZXMU6P/TAehqZh8CjwEdzeyRuCWlbDGwOIQwrfzjUXjA57KfAf8JIZSG\nENYCTwLtI9eULp+Y2a4A5bfLI9eTFmbWAygCzgy5P096b/xkYlZ5VrQA3jCzXaJWVU6BXi6E0CeE\n0CKE0BK/yPZiCCGnz/xCCMuARWbWuvyuY4G3IpaUDguBI8ysgZkZ/pxy+kLvBp4BepS/3wN4OmIt\naWFmxwF/ALqGEL6KXU+qQghzQgg7hxBalmfFYuDQ8t+16BToyfdbYLiZzQYOAf4cuZ6UlL/aGAW8\nAczBf4azduXelpjZCGAK0NrMFpvZecAgoJOZvYvPoBgUs8aq2sJzugNoBEw0szfN7J6oRVbRFp5T\n1tJKURGRhNAZuohIQijQRUQSQoEuIpIQCnQRkYRQoIuIJIQCXUQkIRToIiIJoUAXEUmI/wdN/AJ7\nmT92xwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x108fb8550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(x, y.data.numpy().squeeze(),'o')\n",
    "\n",
    "x2 = np.arange(3,15,0.5)\n",
    "xx = np.vstack((np.power(x2,i) for i in range(degree+1))).T\n",
    "AA = Variable(torch.from_numpy(xx).double())\n",
    "\n",
    "f = torch.matmul(AA, w)\n",
    "\n",
    "plt.plot(x2, f.data.numpy(),'r-')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fitting a line, using a linear unit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 : 15.993881225585938\n",
      "1 : 11.634653091430664\n",
      "2 : 8.574551582336426\n",
      "3 : 6.426366806030273\n",
      "4 : 4.918301105499268\n",
      "5 : 3.8595666885375977\n",
      "6 : 3.1162407398223877\n",
      "7 : 2.59431791305542\n",
      "8 : 2.2278072834014893\n",
      "9 : 1.970388412475586\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import torch\n",
    "import torch.autograd\n",
    "from torch.autograd import Variable\n",
    "\n",
    "## The rows correspond to examples and the columns to features.\n",
    "## There is only one feature so the Tensors are actually just vectors\n",
    "x = torch.FloatTensor([[10, 8, 13, 9, 11, 14, 6, 4, 12, 7, 5]]).transpose_(0,1)\n",
    "#yy = [7.46, 6.77, 12.74, 7.11, 7.81, 8.84, 6.08, 5.39, 8.15, 6.42, 5.73]\n",
    "yy = [8.04, 6.95, 7.58, 8.81, 8.33, 9.96, 7.24, 4.26, 10.84, 4.82, 5.68]\n",
    "y = torch.FloatTensor([yy]).transpose_(0,1)\n",
    "\n",
    "\n",
    "# This is a linear unit that implements the function f(x) = weight*x + bias\n",
    "f = torch.nn.Linear(1, 1, bias=True)\n",
    "\n",
    "# Set w_1\n",
    "f.weight.data = torch.FloatTensor([[1.]])\n",
    "# Set w_0\n",
    "f.bias.data = torch.FloatTensor([[2.]])\n",
    "\n",
    "# learning rate\n",
    "eta = 0.01\n",
    "\n",
    "# This is the error function E(x, y) = (1/N) \\sum_{i=1}^N (x_i-y_i)^2\n",
    "EuclidianLoss = torch.nn.MSELoss(size_average=True)\n",
    "\n",
    "for epoch in range(10):\n",
    "    ## Compute the forward pass\n",
    "    E = EuclidianLoss(f(Variable(x)), Variable(y))\n",
    "\n",
    "    if epoch%1 == 0: \n",
    "        print(epoch,':',E.data[0])\n",
    "#        print(f.bias.data.numpy())\n",
    "#        print(f.weight.data.numpy())\n",
    "\n",
    "    # Compute the gradients by automated differentiation\n",
    "    E.backward()\n",
    "    \n",
    "    # For each adjustable parameter \n",
    "    # Move along the negative gradient direction\n",
    "    for param in f.parameters():\n",
    "        param.data.add_(-eta * param.grad.data)\n",
    "\n",
    "    # Reset the gradients, as otherwise they are accumulated in param.grad\n",
    "    f.zero_grad()\n",
    "\n",
    "#print('Weights')\n",
    "#print(f.weight.data, f.bias.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fitting a polynomial, using a linear unit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 : 307.4685264777547\n",
      "1000 : 5.530656533266996\n",
      "2000 : 2.718923180051055\n",
      "3000 : 1.7239929818677913\n",
      "4000 : 1.3719311097351328\n",
      "5000 : 1.2473455678637868\n",
      "6000 : 1.203251612334874\n",
      "7000 : 1.187639297888522\n",
      "8000 : 1.182105119916113\n",
      "9000 : 1.1801370741775745\n",
      "9999 : 1.1794313232370146\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.autograd\n",
    "from torch.autograd import Variable\n",
    "\n",
    "## The rows correspond to examples and the columns to features.\n",
    "## There is only one feature so the Tensors are actually just vectors\n",
    "x = np.array([10., 8., 13., 9., 11., 14., 6., 4., 12., 7., 5.])\n",
    "#yy = [7.46, 6.77, 12.74, 7.11, 7.81, 8.84, 6.08, 5.39, 8.15, 6.42, 5.73]\n",
    "yy = [8.04, 6.95, 7.58, 8.81, 8.33, 9.96, 7.24, 4.26, 10.84, 4.82, 5.68]\n",
    "y = torch.DoubleTensor([yy]).transpose_(0,1)\n",
    "\n",
    "# Setup the feature (vandermonde) matrix\n",
    "N = len(x)\n",
    "degree = 2\n",
    "xx = np.vstack((np.power(x,i) for i in range(degree+1))).T\n",
    "A = torch.from_numpy(xx)\n",
    "\n",
    "# This is a linear unit that implements the function f(x) = weight*x + bias\n",
    "f = torch.nn.Linear(degree+1, 1, bias=False).double()\n",
    "\n",
    "\n",
    "# learning rate\n",
    "eta = 0.00005\n",
    "\n",
    "# This is the error function E(f, y) = (1/N) \\sum_{i=1}^N (f_i-y_i)^2\n",
    "EuclidianLoss = torch.nn.MSELoss(size_average=True)\n",
    "\n",
    "for epoch in range(10000):\n",
    "    ## Compute the forward pass\n",
    "    E = EuclidianLoss(f(Variable(A)), Variable(y))\n",
    "\n",
    "    if epoch%1000 == 0: \n",
    "        print(epoch,':',E.data[0])\n",
    "#        print(f.bias.data.numpy())\n",
    "#        print(f.weight.data.numpy())\n",
    "\n",
    "    # Compute the gradients by automated differentiation\n",
    "    E.backward()\n",
    "    \n",
    "    # For each adjustable parameter \n",
    "    # Move along the negative gradient direction\n",
    "    for param in f.parameters():\n",
    "        param.data.add_(-eta * param.grad.data)\n",
    "\n",
    "    # Reset the gradients, as otherwise they are accumulated in param.grad\n",
    "    f.zero_grad()\n",
    "    \n",
    "print(epoch,':',E.data[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAHehJREFUeJzt3Xt4VdW1/vHvEBEj2EYFbAlVKtDw\nU1ChkWprUbmIF6qIQLXqQUHxeC8eUakea6uVKl5PixeKilYqR2hIrReiIoii6AlGBKuB1msCSiyN\niKYSYPz+mEEIBkmyV/bae+X9PE+eJCs7e4+t8D6Tucac09wdERHJfjvFXYCIiERDgS4ikhAKdBGR\nhFCgi4gkhAJdRCQhFOgiIgmhQBcRSQgFuohIQijQRUQSYud0vlj79u29S5cu6XxJEZGst3jx4o/d\nvcOOHpfWQO/SpQslJSXpfEkRkaxnZu815HE7nHIxs/vMbLWZLdvq2ggze8PMNplZQSqFiohINBoy\nhz4NOGaba8uAYcCCqAsSEZGm2eGUi7svMLMu21x7E8DMmqcqERFpNHW5iIgkRLMHupmNNbMSMyup\nrKxs7pcTEWmxmr3Lxd2nAFMACgoKdJqGSAtQVFrBpOIyVlZV0yk3h/GD8xnaOy/ushIvrW2LIpJ8\nRaUVTChcSnXNRgAqqqqZULgUQKHezBrStvgw8BKQb2blZjbGzE4ys3LgMOBxMytu7kJFJDtMKi77\nMsw3q67ZyKTispgqajka0uVy6nZ+NDviWkQkAVZWVTfqukRHXS4iEqlOuTmNui7RUaCLSKTGD84n\np3WrOtdyWrdi/OD8mCpqOXRTVEQitfnGp7pc0k+BLiKRG9o7TwEeA025iIgkhAJdRCQhFOgiIgmh\nQBcRSQgFuohIQijQRUQSQoEuIpIQCnQRkYRQoIuIJIQCXUQkIRToIiIJoUAXEUkIBbqISEIo0EVE\nEqIhZ4reZ2arzWzZVtf2NLOnzWxF7ec9mrdMERHZkYaM0KcBx2xz7Upgrrt3B+bWfi8iIjFqyCHR\nC8ysyzaXTwSOrP36AWA+cEWEdYmIZJSi0oqMP4WpqScW7e3uqwDcfZWZdYywJhGRjFJUWsGEwqVU\n12wEoKKqmgmFSwEyKtSb/aaomY01sxIzK6msrGzulxMRidyk4rIvw3yz6pqNTCoui6mi+jU10D8y\ns28D1H5evb0HuvsUdy9w94IOHTo08eVEROKzsqq6Udfj0tRAfxQYVfv1KOAv0ZQjIpJ5OuXmNOp6\nXBrStvgw8BKQb2blZjYG+C0wyMxWAINqvxcRSaTxg/PJad2qzrWc1q0YPzg/porq15Aul1O386MB\nEdciIpKRNt/4TGqXi4hIizK0d17GBfi2tPRfRCQhFOgiIgmhQBcRSQgFuohIQijQRUQSQl0uIiLN\nZe1aWLkyfBx8MOy5Z7O+nAJdRKSxvvgCVq2CiootgV3f1+vWbfmdJ56AY49t1rIU6CIi23KHf/4T\nVqyAv/89fGz++p134OOPv/o7bdpAp07h46CD4LjjtnyflxeuNTMFuoi0TO5QWVk3rLf++pNPtjx2\np51g332hWzc4+WTo3LluWHfqFKZTzOJ7PyjQRaQlqK6GN96A118PH0uWhM9r1mx5zE47QZcu0L07\nHHpoCO/u3cPnLl3CCDzDKdBFJDncobx8S2BvDu/ly2HTpvCY3XaDXr1g+HDYf/8Q2t27hxH4LrvE\nW3+KFOgikp3c4b33YNGi8FFaGgK8qmrLY777XTjwQBgxIsxhH3ggdO0aRuMJpEAXkezw+edQUhLC\n+6WXwucPPww/y8kJgf3Tn4bQPugg6NkTvvnNeGtOMwW6iGQed/jHP+qG95IlsLH2GLhu3WDgwDDX\nfdhhYQqldet4a84ACnQRid+mTbB0KTz7LMybF0J8c2tgu3bQty9ccUUI7x/8AHScZb0U6CKSfu6h\nNXDu3C0hvjnAu3WDIUNCeB96KBxwALRq9fXPJ0CKgW5mlwDnAAb8wd1vj6QqEUme8vIQ3s8+G4K8\nvDxcz8sLi3D69w8f3/lOvHVmsSYHupn1JIR5X2A9MMfMHnf3FVEVJyJZrKoKnnlmyyh8+fJwfa+9\n4KijYMCAEODdu8e+ICcpUhmh/z9gkbt/DmBmzwEnATdFUZiIZKHly+Gxx+Cvf4Xnnw83Mdu1gyOO\ngHPPDSHeq1di2wbjlkqgLwN+Y2Z7AdXAcUBJJFWJSHaoqYGFC0OAP/bYllF4z55w+eVw/PHhhqY6\nUNKiyYHu7m+a2Y3A08A6YAmwYdvHmdlYYCzAPvvs09SXE5FMsWYNPPlkCPA5c8LUyi67hGmUiy4K\nNzS7dIm7yhbJ3D2aJzK7ASh39zu395iCggIvKdEgXiTrrFgBRUVhJL5wYWgz7NgxhPeQIaEnfPfd\n464yscxssbsX7OhxqXa5dHT31Wa2DzAMOCyV5xPJJEWlFUwqLmNlVTWdcnMYPzifob3z4i4rfd5+\nGx55JHyUloZrBx8Mv/gF/OQnUFCgufAMk2of+p9r59BrgAvc/V8R1CQSu6LSCiYULqW6JqxMrKiq\nZkLhUoBkh/p778HMmfC//xuW2UNYyHPrrWHbWE2bZrSUAt3dfxxVISKZZFJx2Zdhvll1zUYmFZcl\nL9DLy0OIP/JIWGIPYfR9001hUyvNh2cNrRQVqcfKqupGXc86q1bBrFlhJL5wYbh28MEwcWII8a5d\n461PmkSBLlKPTrk5VNQT3p1yc2KoJiKffQZ//jNMmwbz54fl9716wXXXwciR8L3vxV2hpEiBLlKP\n8YPz68yhA+S0bsX4wfkxVtUE7vDCCyHEH3kkHFrctStcc00I8f33j7tCiZACXaQem+fJs7bL5f33\n4cEHQ5D/4x9htebIkXDmmXD44Vpqn1AKdJHtGNo7L3sCHMIBEEVFcP/9Yf8UdzjyyDAaHzYshLok\nmgJdJJu5h86UadNgxgxYuzZ0pVxzDYwaFY5gkxZDgS6Sjaqq4IEH4O674a23wsHHw4fDWWdBv35a\n8NNCKdBFssmSJTB5MkyfHqZYDj0U7r03tBpq6X2Lp0AXyXRffBHaDSdPhhdfDAci/+xncP750KdP\n3NVJBlGgi2Sq99+He+6BqVNh9epwNNutt4ZOlT32iLs6yUAKdJFMsmlT6FCZPDnsbAhhN8MLLgg7\nGqY4N97iNxxLOAW6SCZYuxbuuw/uvDNsVduhQzjl/txzYd99I3mJFrvhWAuiW+EicSovh/Hjw8HI\n48ZB+/bw0EPwwQdwww2RhTl8/YZjkgwaoYvE4fXX4eab4eGHwzTLyJHwX/8VdjlsJonfcEwU6CJp\n4x7mx2++GYqLoW3b0KkyblxatqhN5IZjUoemXESaW01N6Bvv0wcGDYLXXoPf/CZ0sdxxR9r2Gx8/\nOJ+c1q3qXMvKDcdkuzRCF2kun34aWg5vvz2Ed48e4fvTToNdd017OVm/4ZjskAJdJGqrVoWR9913\nwyefwBFHhDbE446LfUl+1m04Jo2S6iHR44CzAQeWAme5+7+jKEwk61RUwI03wpQpYZrl5JPhssug\nb9+v/TX1hktUmjxcMLM84GKgwN17Aq2AU6IqTCRrfPABXHgh7Lcf3HVXmFIpKwsHSjQgzCcULqWi\nqhpnS294UWlFemqXREn13387AzlmtjOwG7Ay9ZJEssT778N554Ul+ffcE7arXb48bJbVrVuDnkK9\n4RKlJk+5uHuFmd0MvA9UA0+5+1ORVSaSqd59Nyz6mTYtfD9mDFx5ZZMWAak3XKKUypTLHsCJwHeB\nTkBbMzu9nseNNbMSMyuprKxseqUicXv7bTj7bOjePexFPnZsON7trruavKJzez3g6g2XpkhlymUg\n8I67V7p7DVAI/HDbB7n7FHcvcPeCDh06pPByIjFZsSIcHPG974V+8vPPD+H++9+HJfspUG+4RCmV\nLpf3gUPNbDfClMsAoCSSqkQywXvvwS9/CX/8I7RpAxdfHPZd+fa3I3sJ9YZLlFKZQ3/ZzGYBrwIb\ngFJgSlSFicSmsjKs5LzrrtA3/vOfw+WXw957N8vLqTdcopJSH7q7/xL4ZUS1iMTr00/DARI33wzV\n1TB6dDhsuXPnuCsTaRCtFBX54ovQdnj99WF0fvLJ4esePeKuTKRRtDmXtFwbN4a9x3v0gEsugZ49\n4eWXYdYshblkJQW6tDzu8Pjj0Ls3nHFGOJ+zuDhsbbuDlZ0imUxTLtKyLFwYFgG98EJYzTljBowY\nEfumWdJ02gtnCwW6tAzLl4eWw0cfhW99K3SwjBkDrVvHXZmkQOek1qVhiSRbVRVceikccADMmxfa\nEf/+d/jP/1SYJ4D2wqlLI3RJpg0bwmES//3f8M9/htH49dc3Wy+5xEN74dSlEbokz9y54YbneefB\n/vvD4sXwhz8ozBNIe+HUpUCX5FixAk48EQYOhHXrQvvh/Pkh3CWRtBdOXZpykez3ySdhOuWOO8Ke\nKxMnhuX6MZzbKemlvXDqUqBL9tq4MRwmcfXV8PHHYUfE66+PdPMsyXzaC2cLBbpkp3nzwij89dfh\n8MPhySfh+9+PuyqRWCnQEy5xiy5WroRx48J5nfvuGz4PHw5mcVcmEjsFeoIlatHFhg0weXJoQ1y/\nHq69Nmxpm9MyuxlE6qMulwRLzKKLRYvgkEPCFMsPfwjLloWDJxTmInUo0BMs6xddrFkD554bQnz1\napg5M8yVd+sWd2UiGUmBnmBZu+jCHaZNg/z80MUybhy89ZbmykV2oMmBbmb5ZvbaVh9rzeznURYn\nqcnKRRdvvAFHHBFaELt3D6s8b7kFdt897spEMl4qZ4qWAQcDmFkroAKYHVFdEoGsWnSxbh38+tdw\n223wjW+EpfqjR2tbW5FGiKrLZQDwD3d/L6Lnk4hk/KILdygqCicGffBBCPEbb4T27bf7K4lrxRSJ\nSFTDn1OAhyN6LmkpVq6EYcPCR25uOHTi3nt3GOYTCpdSUVWNs6UVs6i0In11i2SolAPdzHYBTgBm\nbufnY82sxMxKKisrU305SQL3ENz77w9z5oQR+eLF8KMf7fBXE9OKKdIMohihHwu86u4f1fdDd5/i\n7gXuXtChQ4cIXk6y2jvvwNFHw9lnw0EHhaX7l1/e4MMmsr4VU6QZRRHop6LpFtmRjRvhf/4HevYM\nC4XuvDPsx9K9e6OeJmtbMUXSIKVAN7PdgEFAYTTlSCK9+Sb06xdufPbrF1oTzzuvSR0sWdmKKZIm\nKQW6u3/u7nu5+ydRFSQJUlMDN9wABx8cFgY9+CA88QTss0+Tn3Jo7zwmDutFXm4OBuTl5jBxWC91\nuYigzbmkuZSWhhbE116DESPgd7+L7Ai4jG/FFImJVm1ItP79b/jFL8JmWh9+CIWFYYtbnecp0uw0\nQpfovPhiGJWXlYWl+7fcAnvsEXdVIi2GRuiSuvXrYcIE+PGPwwi9uBjuu09hLpJmGqFLapYuhTPO\ngCVLYMyYsBeLNtISiYVG6NI0GzfCpElQUACrVsGjj8LUqQpzkRhphC6N9847MGoUPP88nHQS3HMP\naBWwSOw0QpeG27wHy4EHhnbEadPgz39WmItkCI3QpWE++gjOOQf++lc48sgQ5vvuG3dVIrIVjdBl\nx2bPDnuwPPUU3HorzJ2rMBfJQAp02b5PPoEzzwz7lX/nO2GL23HjdIqQSIbS30yp3/z5Ya78j3+E\nq68OOyQecEDcVYnI11CgS101NWHpfv/+0KYNLFwI110Hu+wSd2UisgO6KSpbvPsunHpqGI2PHh32\nL2/bNu6qRKSBFOgSzJwZuljc4eGH4ZRT4q5IRBpJUy4t3eefhyAfORJ69Ajb3irMRbKSAr0le/31\nsHR/6lS44oqw8nO//eKuSkSaSIHeErnD5MnQty/861+hv/y3v23wQc0ikplSPVM018xmmdlbZvam\nmR0WVWHSTNasCX3lF14IRx0VdkkcNCjuqkQkAqneFL0DmOPuw81sF2C3CGqS5rJgAZx2WljGf8st\n8POfa5GQSII0+W+zmX0D6AfcC+Du6929KqrCJEIbNsC114YR+a67hpOFLr1UYS6SMKn8jd4PqATu\nN7NSM5tqZmpazjQVFTBgAPzqV/Czn8Grr4YboSKSOKkE+s5AH+Aud+8NfAZcue2DzGysmZWYWUll\nZWUKLyeN9uyz0KdP2IPlgQfCMn4dQCGSWKkEejlQ7u4v134/ixDwdbj7FHcvcPeCDto3Oz02bYIb\nbgg3O/faC155Bf7jP+KuSkSaWZMD3d0/BD4ws/zaSwOAv0VSlTTdmjVwwglw1VVhsdArr8D++8dd\nlYikQapdLhcB02s7XN4Gzkq9JGmykhIYMSLMm//ud3DBBWAWd1UikiYpBbq7vwboDlvc3GHKFLj4\nYth777Di8wc/iLsqEUkzbc4Vg6LSCiYVl7GyqppOuTmMH5zP0N55TXuyzz6D884LNzwHD4aHHoL2\n7aMtWESyggI9zYpKK5hQuJTqmo0AVFRVM6FwKUDjQ72sDIYPhzfeCG2JV1+t3nKRFkx/+9NsUnHZ\nl2G+WXXNRiYVlzXuiWbNgkMOgVWrYM4cuOYahblIC6cESLOVVdWNuv4VNTXhXM8RI8KRcKWlcPTR\nEVYoItlKgZ5mnXJzGnW9jooKOPJIuP32cAP0uefC4c0iIijQ02784HxyWreqcy2ndSvGD87fzm/U\nevHFsGR/yRKYMQPuuEPnfIpIHQr0NBvaO4+Jw3qRl5uDAXm5OUwc1uvrb4hOnRpG5m3bwssvw09/\nmq5yRSSLqMslBkN75zWso2X9+jBffuedYZ58xgzYY4/mL1BEspJG6Jlq9WoYODCE+fjx8MQTCnMR\n+VoaoWeixYvhpJOgshKmTw/b3oqI7IBG6Jlm+nQ4/PCwB8vChQpzEWkwBXqm2LABLrsMTj89HN78\nf/8X9jIXEWkgTblkgjVr4JRT4Omnww6Jt90GrVvHXZWIZBkFetyWLYMTT4Ty8tCeOGZM3BWJSJbS\nlEucCgvh0EOhuhrmz1eYi0hKFOhxcIdrr4WTT4aePcPBFIcdFndVIpLlNOWSbtXVMHp0WCQ0ahTc\nfTfsumvcVYlIAijQ0+mjj2DoUFi0CCZOhCuu0BFxIhKZlALdzN4FPgU2AhvcXcfRbc+yZTBkSFgB\nOmtWmG4REYlQFCP0o9z94wieJ7nmzIGRI6FdO1iwIOyaKCISMd0UbW6//z0cfzx07QqvvKIwF5Fm\nk2qgO/CUmS02s7H1PcDMxppZiZmVVFZWpvhyWWTDBrjwQrjoohDozz8PnTvHXZWIJFiqgf4jd+8D\nHAtcYGb9tn2Au09x9wJ3L+jQoUOKL5cl1q6Fn/wEJk+GSy+F2bPDdIuISDNKaQ7d3VfWfl5tZrOB\nvsCCKArLWu++G25+vvUW3HMPjK33Hy6JU1RawaTiMlZWVdMpN4fxg/Mbtue7iESmyYFuZm2Bndz9\n09qvjwZ+HVll2eill0Jb4hdfhBuhAwfGXVFaFJVWMKFwKdU1GwGoqKpmQuFSAIW6SBqlMuWyN/CC\nmS0BXgEed/c50ZSVhWbMgKOOClMrixa1mDAHmFRc9mWYb1Zds5FJxWUxVSTSMjV5hO7ubwMHRVhL\ndnKH66+Ha64J+5jPng3t28ddVVqtrKpu1HURaR5qW0xFTQ2cc04I8zPOgGeeaXFhDtApN6dR10Wk\neSjQm2rdurDt7b33wtVXwwMPQJs2cVcVi/GD88lp3arOtZzWrRg/OD+mikRaJu3l0hQffRR6y0tL\nw+Za554bd0Wx2nzjU10uIvFSoDfW8uVw7LGwahUUFYV+c2Fo7zwFuEjMFOiNsWhR6DE3CwdS9O0b\nd0UiIl/SHHpDPfoo9O8Pubmh31xhLiIZRoHeEHfdBSedFE4XevFF6NYt7opERL5Cgf513OGqq+D8\n8+G442DePOjYMe6qRETqpTn07Vm/PvSYP/hg2I9l8mTYWf+5RCRzKaHqs3YtDB8OTz8N110XRuk6\nKk5EMpwCfVsrV4Ye82XL4P774cwz465IRKRBNIe+laeLnmfVAX1Y97cyxp1xPUUHDYq7JBGRBtMI\nvda8P82hzzmnsAnjp6dO5I2O3ZijLWBFJItohA7w3HP0HX0y1Tu3YcRpN/LGt0JboraAFZFsokB/\n9FEYPJiKdu05+fSbeHfPuqNxbQErItmiZU+5PPggjB4N3/8+lxx9JR/V7PKVh2gLWBHJFi13hH7b\nbTBqVDhlaO5czh3WV1vAikhWSznQzayVmZWa2WNRFNTs3MP+5ZdeGnrNH3sM2rVjaO88Jg7rRV5u\nDgbk5eYwcVgv3RAVkawRxZTLJcCbwDcieK7mtXEjXHRR2Jvl7LPDXuattozKtQWsiGSzlEboZtYZ\nOB6YGk05zWj9ejjttBDmV1wBU6bUCXMRkWyX6gj9duByYPcIamk+n30WplfmzIGbboLx4+OuSEQk\nck0eoZvZEGC1uy/ewePGmlmJmZVUVlY29eWa7l//gqOPhqeegqlTFeYiklipTLn8CDjBzN4FZgD9\nzeyhbR/k7lPcvcDdCzp06JDCyzXBqlXQrx+UlMDMmTBmTHpfX0QkjZoc6O4+wd07u3sX4BTgWXc/\nPbLKUvX223D44fDOO/D44zBsWNwViYg0q2QuLCorgwEDoLoann1Wx8WJSIsQSaC7+3xgfhTPlbJl\ny2DgwNBvPm8eHHhg3BWJiKRFslaKvvoqHHlkaEd87jmFuYi0KMkJ9EWLoH9/aNsWFiyAHj3irkhE\nJK2SEegLFsCgQdC+PTz/PHTtGndFIiJpl/2B/swzcMwx0LlzCPZ99om7IhGRWGR3oD/2GAwZAt27\nhznzTp3irkhEJDbZG+iFhaG3vFev0M3SsWPcFYmIxCo7A/1Pf4KRI+GQQ8KUy557xl2RiEjssi/Q\n77sPTj8dfvxjKC6Gb34z7opERDJCdgX65MlhP5ajj4YnnoB27eKuSEQkY2RPoN9yC1x4IZx4Ivzl\nL5Cjsz5FRLaWHYE+cSJcdlmYN585E9q0ibsiEZGMkx2B3r07nHUWTJ8OrVvHXY2ISEbKjt0Whw8P\nHyIisl3ZMUIXEZEdUqCLiCSEAl1EJCEU6CIiCaFAFxFJiCYHupntamavmNkSM3vDzH4VZWEiItI4\nqbQtfgH0d/d1ZtYaeMHMnnT3RRHVJiIijdDkQHd3B9bVftu69sOjKEpERBovpYVFZtYKWAx0Aya7\n+8v1PGYsMLb223VmVpbKa6ZJe+DjuIuImN5T5kva+wG9p6js25AHWRhop8bMcoHZwEXuvizlJ4yZ\nmZW4e0HcdURJ7ynzJe39gN5TukXS5eLuVcB84Jgonk9ERBovlS6XDrUjc8wsBxgIvBVVYSIi0jip\nzKF/G3igdh59J+ARd38smrJiNyXuApqB3lPmS9r7Ab2ntIpkDl1EROKnlaIiIgmhQN+GmbUys1Iz\nS8T0kZnlmtksM3vLzN40s8PirilVZjaudnXyMjN72Mx2jbumxjKz+8xstZkt2+ranmb2tJmtqP28\nR5w1NtZ23tOk2j97r5vZ7M333bJFfe9pq59dZmZuZu3jqK0+CvSvugR4M+4iInQHMMfdewAHkeXv\nzczygIuBAnfvCbQCTom3qiaZxle7wq4E5rp7d2Bu7ffZZBpffU9PAz3d/UBgOTAh3UWlaBr1dO+Z\n2XeAQcD76S7o6yjQt2JmnYHjgalx1xIFM/sG0A+4F8Dd19e2mGa7nYEcM9sZ2A1YGXM9jebuC4A1\n21w+EXig9usHgKFpLSpF9b0nd3/K3TfUfrsI6Jz2wlKwnf9PALcBl5Nhq+MV6HXdTviftCnuQiKy\nH1AJ3F87jTTVzNrGXVQq3L0CuJkwMloFfOLuT8VbVWT2dvdVALWfO8ZcT9RGA0/GXUSqzOwEoMLd\nl8Rdy7YU6LXMbAiw2t0Xx11LhHYG+gB3uXtv4DOy75/xddTOK58IfBfoBLQ1s9PjrUp2xMyuAjYA\n0+OuJRVmthtwFXBN3LXUR4G+xY+AE8zsXWAG0N/MHoq3pJSVA+Vb7bEzixDw2Wwg8I67V7p7DVAI\n/DDmmqLykZl9G6D28+qY64mEmY0ChgCnefb3SXclDCaW1GZFZ+BVM/tWrFXVUqDXcvcJ7t7Z3bsQ\nbrI96+5ZPfJz9w+BD8wsv/bSAOBvMZYUhfeBQ81sNzMzwnvK6hu9W3kUGFX79SjgLzHWEgkzOwa4\nAjjB3T+Pu55UuftSd+/o7l1qs6Ic6FP7dy12CvTkuwiYbmavAwcDN8RcT0pq/7UxC3gVWEr4M5yx\nK/e2x8weBl4C8s2s3MzGAL8FBpnZCkIHxW/jrLGxtvOefg/sDjxtZq+Z2d2xFtlI23lPGUsrRUVE\nEkIjdBGRhFCgi4gkhAJdRCQhFOgiIgmhQBcRSQgFuohIQijQRUQSQoEuIpIQ/x8/EJCoiyCYkwAA\nAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x109010358>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "plt.plot(x, y.numpy(),'o')\n",
    "\n",
    "x2 = np.arange(3,15,0.5)\n",
    "xx = np.vstack((np.power(x2,i) for i in range(degree+1))).T\n",
    "A = torch.from_numpy(xx)\n",
    "\n",
    "plt.plot(x2, f(Variable(A)).data.numpy(),'r-')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Autograd : Automatic differentiation example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 1  1\n",
      " 1  1\n",
      "[torch.FloatTensor of size 2x2]\n",
      "\n",
      "Variable containing:\n",
      " 3  3\n",
      " 3  3\n",
      "[torch.FloatTensor of size 2x2]\n",
      "\n",
      "Variable containing:\n",
      " 27  27\n",
      " 27  27\n",
      "[torch.FloatTensor of size 2x2]\n",
      " Variable containing:\n",
      " 27\n",
      "[torch.FloatTensor of size 1]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "\n",
    "x = Variable(torch.ones(2, 2), requires_grad=True)\n",
    "print(x)\n",
    "\n",
    "y = x + 2\n",
    "print(y)\n",
    "\n",
    "z = y * y * 3\n",
    "out = z.mean()\n",
    "\n",
    "print(z, out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 4.5000  4.5000\n",
      " 4.5000  4.5000\n",
      "[torch.FloatTensor of size 2x2]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "out.backward()\n",
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculating and plotting the the derivative of a function using autodiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD8CAYAAABjAo9vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8nNV5L/DfmU2jGY1GGo02S5Y0WmxjbINtecGYJQSC\nsxRCAimBrKSBpOltaNPQpunt7e1tbpekafJpewOEtEBDEgKBkCYQloTNeJXwvmrfR9JIo5nRjGY/\n94/RKxQjy5Lm3d/n+/n48/Eyet8jS3rmvM95znMY5xyEEEL0w6T0AAghhIiLAjshhOgMBXZCCNEZ\nCuyEEKIzFNgJIURnKLATQojOUGAnhBCdocBOCCE6Q4GdEEJ0xqLETb1eL29oaFDi1oQQolnt7e0B\nznn5pV6nSGBvaGhAW1ubErcmhBDNYoz1LeV1lIohhBCdocBOCCE6Q4GdEEJ0hgI7IYToDAV2QgjR\nGQrshBCiM3kHdsaYnTF2iDF2jDF2ijH2v8UYGCGEkJURY8aeAHAD5/wKAFcC2MMY2ynCdckK9E1E\n8eND/ZhOpJUeCjGQvokoHt/fi3gqo/RQCETYoMRzh6ZOz/7ROvuLDlJVAOcc9z95FEf6p/APL5zF\nZ69uwGd2NaDEYVN6aESnzvkj+N5rnfjFsWFkORCJp/Gl9zQrPSzDE2XnKWPMDKAdQDOAf+ecH1zg\nNfcCuBcA6urqxLgtucCLp0ZxpH8K913biO5AFN95pQPff6Mbn9hZj89d40OFy670EIlOHBuYwr+/\n2omXTo/CYTPjc7t9OOuP4MHXunDX9jqUOmkyoSSWm3CLdDHGSgA8C+B/cM5PXux1ra2tnFoKiCud\nyeLm77wBDuCl+6+FxWzCWX8Y33utC/99bBgWswmf3dWAv3j/OjDGlB4u0ajxSAJ/+tOjeLMjAHeh\nFZ/ZlXsqLHXacM4fwfu/+wY+t9uHr39wvdJD1SXGWDvnvPVSrxO1KoZzPgXgVQB7xLwuubSn2wfR\nNR7FAzevg8Wc+7KuqyrGd+/cjN9+5XrcfHkVHnqjGwe6JxUeKdGyB1/vwoHuCXzt/evw1l/cgD+5\nac3c7HxtlQsf3VKLx/b1YTAYU3ikxiZGVUz57EwdjLFCADcBOJvvdcnSzSQz+M4rHdhcV4KbL698\n1783eJ345u2b4C604ocHl9RDiJB3iacyeLp9EDdfXoX7rmtCUcG7M7l/ctMagAH/8nKHAiMkAjFm\n7NUAXmWMHQdwGMDLnPNfinBdskSP7uuFPxzHn++5eJrFbjXjjq21ePGkH+ORhMwjJHrwy+MjCM2k\n8Imd9Rd9zaqSQnxmVwOeOTKIs/6wjKMj8+Ud2DnnxznnmznnmzjnGzjnfyvGwBby/IkR/PNL56S6\nvCZNxZL43mudeM/acuxsLFv0tXftqEM6y/HTtgGZRkf05IcH+tBcUYQdPs+ir/vD63Oz+W/+mn5W\n5xsJzeBvfnEK50cjkt9LUztP23qD+MHeHoi54Kt133utC5FEGg/sWXfJ1zaWF+Hq5jL86GA/Mln6\nPyRLd3IohKMDU7h7R90lF99LHDZ88fom/ObsGA710JqOoGc8ikf39SIwLf0Ts6YCe5W7ALFkhjbf\nzBqemsF/7uvFbZtrcFl18ZI+5u4d9RiamsHr58ckHh3RkycO9sNuNeEjW2qX9PrP7vKhsrgA//DC\nGZqIzRqNxAEAVcXSlx1rKrBXzv6HjIYpRwwA33nlPMCBP71pzZI/5qb1lahwFeCHB/olHBnRk0g8\nheeODuGWK1bBXWhd0scU2sy4/8Y1eLt/Ci+dHpV4hNogxK0KCuy/S9hgMxqOKzwS5XWMRvB0+yA+\neVU9aksdS/44q9mEO7etxqvnxqgkjSzJz48MIZbM4O4dF180XcgdW2vRWO7EN188h3QmK9HotMMf\niqOowLJgNZHYNBXYq9wU2AUPv9ENh82you3bd26vAwPw40M0ayeL45zjiYP92FjjxhWrS5b1sRaz\nCQ/cvBadY9P4zVlK/Y1F4qgsLpDlXpoK7MJ/ip8CO/Z3T2B3sxeeFWzdXlVSiBvWVeLJwwNIpmkm\nRS6uvS+Is/4I7t6xsjYgN6yrRKHVjP1dEyKPTHv8ofhcOllqmgrsDpsFLrsFYwbPsQ9NzWAwOIPt\nlyg7W8zdO+sQmE7ipdN+EUdG9OaHB/rgKrDglitXrejjbRYTttSXUHUMcjl2ORZOAY0FdiC3gOoP\nGXvGfnj2hySfwH5dSzlqSwvxwwO0E5UsbDKaxPMn/PjIlho4bCvPC29vKMMZfxihmZSIo9OWbJZj\nLBKXZeEU0GBgryq2z5UNGdXBnkm4CixLLnFciMnEcNeOOhzonkTnmPQbJoj2PNU2gGQmi7sX2Wm6\nFNt9HnAOtPcZd9YejCWRynBUUY59YRXFBRg1+oy9dxJbG0phNuXXpfFjrathNTM8cZAWUcnvymY5\nfnSoH9sbPFhT6crrWpvrSmA1MxzqCYo0Ou0R1gUpx34RVcV2jEUSyBp052RgOoHOsem80jACb1EB\n9myoxtPtg5hJ0sk35B1vdQXQNxHD3TvzPzvBbjVjU20JDvUYdwFVWBesdFNgX1BlsR3pLMdkLKn0\nUBTR1pt7nL1Uv46lun1rLSLxNA4a+IeOvNuLp/xw2szYs6FKlOtt93lwfDBk2AnEKM3YFzdX8mjQ\ndMzBnkkUWEzYWLO8muKL2eHzwGYxYW9HQJTrEX3Y2xHAzsYyFFjMolxvu8+DdJbjSL8x0zFCKqa8\niHLsCxLe8cYMuoB6qGcSW+pKYbOI86WzW83Y1lCKvZ0U2EnOwGQMvRMxXN3sFe2aW+tLYWK5iYkR\njYYT8BbZRPu5vRTNBnZ/yHi17OF4CmdGwtgmUhpGsLu5HGf9EcO+WZLf9dbsm/w1LeIF9mK7FZdV\nF+Nwr1EDe1zWM4c1F9jLXQVgzJhtBdr7gshy8fLrgt2zM7N9nZRnJ8CbnQFUFheguaJI1Otu93nw\ndn/QkLudR8PxuZYoctBcYLeaTShzFhgysB/qmYTFxLC5Tpz8uuDyVcUodVjxJuXZDS+b5djXGcDV\nzV7RDz3f4fMgnsrixFBI1OtqwWhYvj4xgAYDO5Dry27UwL6x1p3XLsCFmEwMu5q9eKszQL2zDe70\nSBjBWErUNIxgW0PuSdNo7QVSmSwC00nZKmIAjQb2SpcdfoP1i4mnMjg+OCVK/fpCdjd74Q/H0TU+\nLcn1iTYIT21iLpwKyopy6R2j1bOPzZ4xTIH9EiqK7Rgz2Iz9SP8UUhkuen5dIOTZKR1jbHs7x7G2\n0iXZQt92nwdtfUFDHc34Tg07pWIWVVVsx0Q0aahFmEM9k2AM2FovTWBf7XGgocxB9ewGFk9lcLg3\niN0SpGEE2xs8iMTTOOsPS3YPtRmTeXMSIEJgZ4ytZoy9yhg7zRg7xRj7shgDW4zwzmek8rxDvRNY\nV1W85KPJVuLqZi8OdE8gRafdGNLh3kkk01lpA7vPeHl2YTOlpgI7gDSAr3DO1wPYCeBLjLH1Ilz3\noirdxjr7NJnOor0vKFkaRnBNixfRZAZHB6YkvQ9Rp70dAVjNTNLvs1UlhagtLTRUYB+NJGA1M3gc\nyz8UZ6XyDuyc8xHO+duzv48AOAOgJt/rLqbSYGefnhwOIZ7KSrZwKriq0QsTozy7Ub3ZEcCWulLR\nq64utN3nwaGeScNUYI2GcpuTTHl2Y10OUXPsjLEGAJsBHFzg3+5ljLUxxtrGx8fzuo/Rzj4VDtYQ\nysWk4nZYsbG2ZG7nITGOiekETo+EJSlzvNAOnwcT0SS6A1HJ76UGozKedSoQLbAzxooA/AzA/Zzz\nd62McM4f5py3cs5by8vL87pXqcMKm9lkmLNPD/VMotHrRLlL+m+Oa5q9ODowhXDcuKfdGNFbs2eS\n7m7J72dzKYxWzy7nWacCUQI7Y8yKXFB/gnP+jBjXvMT9UFFcYIizTzNZjkO9k5KnYQS7W7zIZDkO\n0OHDhrK3YxzFdgs21rglv5fP64S3qMAwgX0snNBeYGe5fcc/AHCGc/7t/Ie0NEY5+/ScP4JIPC1b\nYN9cV4JCq5nSMQbCOcfejgB2NXnzPpVrKRjLLdAaIbBHE2lEEmntBXYAVwP4JIAbGGNHZ399QITr\nLqqyuMAQZ58Ku/TkCuwFFjN2NHrwJgV2w+gJRDEcikta5nih7T4PhqZmMBiMyXZPJSixOQkQpypm\nL+eccc43cc6vnP31vBiDW0xlsd0QqZjDfUGscttRW+qQ7Z67m73oHo9ieGpGtnsS5eyVoE3vpQh5\ndr238RVKsqs0OGNXRGWxHdOJNKYTaaWHIqkzI2FsrJU+7zmfMHOjwzeMYW9HALWlhajzyDd5WFNZ\nBJvFhLMjEdnuqQRhxl5BgX1phHdAPZc8xlMZ9AaieZ8Sv1xrK10odxVQewEDSGey2N81gWtaxG/T\nuxiL2YSm8iKcGzVGYJezFzug4cBeMZuzGtXxAmr3eBRZDrTIHNgZY9g928Y3a6BmTUZ0bDCESCKN\n3c3SlzleaE1lETpG9d1N1B+Ow2kzo6hA2k1fF9JsYJ+bset4AbVjLDebWStzYAdyfWMmokmc9et7\nRmV0b3UGwBiwq6lM9nuvqXRhaGoGER3vmRgLJ+ZaoMhJs4HdCGefnvNHYDEx+LxO2e991ewPutF6\nZxvNoZ5JrKsqRqlTvj4mAiHF2DGm31m7Pxyfa4EiJ80GdmeBBa4Ci65z7OdHp9Hgdcp2svl8NSWF\nqCkpxCGdVy0YWSojT3O5i1lTmTtTtUPHeXa5zzoVaDawA7k8u54De8dYZO6bXwk7DNasyWhODoUw\nk8rItkfiQqtLHbBbTTiv0zw75xxj4cTceqCcNB3YK4vtug3sM8kM+idjslfEzLfN50Fg2jjNmoxG\nqCGXurncxZhMDC0VLpzX6Yw9GEshmclSKma5qortuu3J3jk2Dc6haGAXZnKHDbD124jkbC53MS2V\nRboN7EqVOgIaD+wVxXaMReK6LMkTvtmVTMU0ep3wFtkM0dPDaLJZjsO9QcXSMII1lS6MhhMIzeiv\nMsavUDsBQOOBvaq4AKkMRzCWVHooojs/FoHNbEJ9mfwVMQLGGLY1eHCQArvunB+LIDSTUiwNI9Dz\nAqoSZ50KNB3Y50oedZhnP++PoLHcCatZ2S+R0KxpiPrG6IrwFKaGGTsAXe5AFUqxKyjHvjxC4b8e\nm4GdH52WfcfpQijPrk8HeyZnm8sVKjqOmpJCOG1mXe5AHY3EUea0KVKurO3ArtMZ+3QijaGpGaxV\nML8uWFdVDFeBhdIxOsI5x+GeSWzzeWTtD7MQxhiaK/VZGTMaisve/Eug6cBeMbuar7eSRyHfqIYZ\nu9nE0NpQSjtQdaRvIoaxSELxNIxgTYU+K2NGI3FUKbBwCmg8sFvNJniLbDoM7LnHUiVLHefb7itD\n13gUgWn9pbyMSMivK7Xj9EJrq1wITCcxGdVXEYQ/JP+ReAJNB3YgtzCht1r286MRFFhMsvbHXsx2\nXykAoI3aC+jCod5JeJw2NJUrn+oD3nky1dOsPZXJYiKaoFTMSlW59bf79NxoBM0VRbKcP7kUG2tK\nUGAxUZ5dJw71TGJbQ6ni+XWBUPKop8AemE6Ac/lPThJoPrBX6rBfTMfotGrSMABgs5iwpa5U98eY\nGYE/FEf/ZAzbffK36b2YqmI7XAUWXQV2f0i5zUmALgK7HYHpJFKZrNJDEUVoJgV/OI4WFVTEzLfN\n58Hp4TDCOu6dbQRCt87tCm9Mmo8xhjVVLl01AxPSw5RjXyHhP24soo88e6eCh2ssZofPgywH2vuC\nSg+F5OFQzwSKCiy4rFpd31+505QiuukkOqrgrlNApMDOGPsPxtgYY+ykGNdbDr2dfXrOr66KGMHm\nuhJYTIw2KmncoZ5JbK0vhUXhHc0XaqlwIRhLYVwnlVej4TgsJoYyBQ4wAcSbsT8KYI9I11oWvZ19\nen40gkKrGTUlyu4IvJDDZsGGGjc1BNOwYDSJ86PTqqlfn29t1expSjpJx/jDcVS4CmBSqABClMDO\nOX8DgCI/8ZU6m7ELh2so9Q2xmB0+D44NTiGeyig9FElEE2mcHArpJh1wIWHxW42BvUVnlTG5AzaU\nScMAMubYGWP3MsbaGGNt4+Pjol3X47DBamYY1UmO/ZxfHT1iFrLd50Eqw3Gkf0rpoYiKc45njwzi\nhn9+DR/617342EP7cXIopPSwRHeoZxI2iwmbat1KD+VdyosKUOKw6iawj4bjipU6AjIGds75w5zz\nVs55a3l5uWjXNZlYbpOSDlIxwWgSgemEoj3YF9Na7wFj0FXZ47GBKXz0e/vwJ08eQ2WxHX++Zx26\nx6P4vX/bi689cxwTOsn5Armv25WrS1BgMSs9lHdhjGFNhX4qY/zhuGKljgBgUezOIqosLsBoRPuB\n/byKesQsxO2wYm2lSxd59rFIHN/89Tk81T4Ib1EB/un2Tbh9Sy1MJoa7dtThX3/TgUf39eKXx0fw\n5fe24NO7GhRvoZyP6UQaJ4fD+MPrm5QeykWtqSrCc0eHwTlXzeaplYgl04jE03PdZ5Wg3e/UeSqL\n7XMbArRMCOxqK3Wcb4fPg/a+oKb3Dbx2bgw3fOt1/PzoEO67thGv/tl1+Fjr6rl1DXehFX/1ofX4\n9f3XYktdKf7uV2fwge++iXENp/ve7gsik+WqzK8L1lS6EImnNd+tda6GXYE+7AKxyh1/DGA/gLWM\nsUHG2OfEuO5SVRbbddGT/fzoNFwFFlQr+E5/Kdt8HsykMprNQYfjKTzw9HHUlBTixfuvxdc+cBlc\nduuCr22uKMKjn92GRz7Vir7JGP72l6dlHq14DvdOwmxi2FJXqvRQLqqlQugZo+10jJJnnQrEqor5\nOOe8mnNu5ZzXcs5/IMZ1l6rKbUckkcZ0Ii3nbUV3fjSC5soiVT+G7pjdir6vS5ttfP/p12cRmE7g\nW3dcgcYlNMFijOHG9ZX4o/c047+PDeO1c2MyjFJ8b3UGsLHGDWeBerOvejkmb1TBs04FukjFCDNc\nLadjOOc4PxpRdRoGAMpdBVhX5cJbnQGlh7Js7X1BPHGwH5/Z5cPGZVaGfOG6JjRXFOGvfn4SsaS2\nJhDheArHBkO4psWr9FAWVVZUAG+RTfOVMSMhYcau3F4UXQT2uZOUNBzYA9NJBGMp1S6czre72Yu2\n3iBmktqpZ0+ms/jLZ06gutiOr7xvzbI/3mYx4f/ethGDwRl895UOCUYonQNdE8hkOa5uVndgB3Lp\nmHMaT8X4Q3G4CiwoUvDpSBeBXZixj4S0e+Cy8Pip1lLH+Xa3eJHMZOcaSmnB99/sxrnRCP721g0r\nTkds93lw57bVeGRvD04Na2eNYW9nAA6bWdX5dcGayiJ0arxnzEhoRtH8OqCTwK6HGfv5ucCu/hn7\ndp8HNrNJM+mY3kAU3/1NB96/oQo3rq/M61pfe/9lKHVY8bVnTiCT1Ubw2dsRwA6fR5FDlZdrTZUL\n0WQGQ1PanaT5Q3EK7GKwW83wOG2aLpM6NzoNd6F17hxXNXPYLNhSX4I3O9Qf2Dnn+PrPT6DAbMLf\n3HJ53tdzO6z4nx9aj+ODITy+vzfv60ltaGoG3YGoJtIwwDsTGy3n2f3huOKVbboI7ECuy6OWZ+yd\nsz1i1FwRM981LeU4MxJW/Tmozx4ZwludE3hgz1rRWqjecsUqXLumHN968RyGVT6zfGv2zfeaFvF2\ne0tpjcZLHlOZLMYiCUUXTgEdBfZqt31uNVqLusejqjmDcimEGaCa0zGT0ST+7ldnsLmuBHfvqBft\nuowxfOPDG5DhHP/rF6dEu64U9nYGUO4q0MTaDZB7IvIW2dAzHlV6KCsyHskdiUczdpFUue2aTcWE\nYilMRJPweZ1KD2XJNta44S60Yq+K0zHfeeU8wjMp/P1HNoreLXO1x4H7b1yDl0+P4vXz4jW1E1M2\ny/FWZwC7m72aeRIEAJ/XiZ6ANgP7O6WOFNhFUVVsx2Q0qcmWsj0TuW9iLQV2s4lhV1MZ3uoMqLKC\nITCdwJOHB/DRLbVYV1UsyT3uudqHqmI7HnytS5Lr5+uMP4yJaBK7NZJfF/i8TnRrNLAL6WAlOzsC\negrsbu32Ze8J5PKJjeXaCexALh0zHIqr8ofw8f19SKSz+Py1jZLdw2Yx4Z7dDdjfPYHjg+prZSyk\nybSycCrweYsQmE4gosHzdYWSa0rFiKR6drFCi3n2nkAMJpZ7vNcSYSej2vLssWQaj+/vxU3rK9Fc\nIW1u+ePb6+AqsOChN7olvc9KvNkRQEtFkeJpgeUSnlx7AzGFR7J8/lAcdqsJ7sKF+w/JRTeBXdsz\n9ihqSgtV2Sd7MXUeB2pLC1VX9vjTwwOYiqVwn4SzdYHLbsVdO+vwwokR9E+oJxDFUxkc6pnU3Gwd\neCewdwe0VxmTK3UsVHxNQ3eBXZsz9mn4vNqoWpiPMYZrWrw40DWBtEra+KYzWTyytwdb60vR2iBP\ni9p7rvbBbGJ4ZK96Zu1v9wWRSGdV3x9mIfVlDjAGTS6g+kPKnpwk0E1gLyqwwGW3aK6WnXOOnvEo\nGjW0cDrf7uZyRBJpHBtUxxb750/6MRickWW2LqgstuO2zTX4aduAak5cerMzAIuJYUdjmdJDWTa7\n1YxV7kJNBvaRkPKbkwAdBXZAqGVX94aRC41HEogmM5qqiJlvV1MZGIMqyh4553jo9S40ljtx42X5\ntQ5YrnuvbUQ8lcXj+/tkve/F7O0IYHNdiaKNqPLRWK69ksdslufOOqXALi4tnqQkfPM2aDSwlzpt\n2LDKrYoF1H1dEzg1HMa91zSKXrd+Kc0VLtx4WQUe39+reNfLYDSJk8Mh7G7Wxm7ThQi17Gospb2Y\nQDSBdJZTYBebFnefCoFdq6kYIFdO93Z/UPGDTh58vQvlrgJ8eHONIve/77omBGMpPNU+oMj9Bfu6\nJsA5sLtFe2kYQUOZE5F4GhPRpNJDWTK11LADOgvsVe5CjE8nNHUeZ08gCpvZhFUlyvaWyMc1LV6k\nsxwHu5U7VenUcAhvdgTw2asbYLcqU13UWl+KLXUl+P6b3YouJu/tHIerwIIraksUG0O+fLN7OrSU\njhEmldUK94kBdBbYq912cA5NHTrcHYiivswBs8ypAzFtrS9FgcWEvQqmYx5+oxtOm1nUnjDLxRjD\nvdc2YWByBi+c9Cs2jr2dAexsKoPFrN0fb+EJVks9Y9Rw1qlAu1/5BWix5LE3ENVsfl1gt5qx3edR\nbAF1MBjDL4+P4OPb6xTfGHLT+ko0ep14+I1uRfLDfRNRDEzOaK6NwIVqSgphNbO5dhtaMBKKw2pm\nKHPalB6KvgK71s4+zWQ5+iZims6vC3Y3e9ExNq3IBrEf7O0BA3DPbp/s976Q2cTw+WsbcWIopMiB\n38JT024N1q/PZzGbUOdxaGrG7g/FUVlsl33hfiGiBHbG2B7G2DnGWCdj7C/EuOZKVBcLbQW0UfI4\nPDWDZCar2VLH+YRA8vo5eTsdBqNJ/OTQAG65YpVq1ilu21wDb1EBHnxd/uZgr58bR7XbrovJgta6\nPI6EZlRRww6IENgZY2YA/w7g/QDWA/g4Y2x9vtddieJCC+xWk2Zm7ELzLD0E9vXVxagvc+CZI4Oy\n3ve/DvRhJpXBfdc1yXrfxditZnz26ga82RGQ9WzUyWgSr54bwwc2Viu+pV0MPq8TvRNRZDVyBKEw\nY1cDMWbs2wF0cs67OedJAD8BcKsI1102xhiq3YUY0Ui/mF4hsGusq+NCGGO4fUstDnRPytYzZSaZ\nwaP7enHDugqsrVLXWbGf2FkPp82Mh16Xr83Ac0eHkMpw3L61VrZ7SsnnLUIindXEzzPnXDW7TgFx\nAnsNgPmFu4Ozf6cILR2R1xOIwmkzo7xI/eecLsVHt9aCMeDpt+WZtT/dPoDJaFLW9gFL5S604q4d\ndfjViREMTMrzRvdU2yA21BTjsmpp+s/LzaehypipWAqJdFbxI/EEsi2eMsbuZYy1Mcbaxsely8NW\nu7UT2LsDUfjKnbp4bAaAVSWF2N3sxc/aByV/fE5nsnj4zW5srivBdp88zb6W657dPpgY8Mib0s/a\nTw2HcHokjDu2rpb8XnJpnKtlV3+XR+H0Nj3N2IcAzP9uqp39u9/BOX+Yc97KOW8tL5duq3OV247R\ncFwTeTmtdnVczO1bazE0NYMDEm9WeuGkHwOTM7jv2ibVvjFWuwtx65U1eLIt92QhpafbB2Ezm3DL\nFaskvY+cKlwFcNjMqjzI5UJ+lRyJJxAjsB8G0MIY8zHGbADuBPALEa67ItVuO9JZjkBU3ZuUEukM\nhoIzulg4ne/my6vgslvwVLt06RjOOR6cbfb1vvXyNvtari9cl2sO9ti+XsnukUxn8dzRYdy4vgKl\nKqihFgtjDA1l2qiMeWfXqU4CO+c8DeCPALwI4AyAn3LOFTu6XchxqT0dMzAZQ5Zru0fMQuxWM265\nYhVeODmCsERHm+3tDODUcBj3XSt/s6/lyjUHq8Rj+3sRS0rTS+e3Z0cxGU3qKg0j8JU754oM1Mwf\nmoGJQTXrZaLk2Dnnz3PO13DOmzjn3xDjmislNOBR++7T7nFtd3VczO1baxFPZfGr4yOSXP+h17tR\noWCzr+X6wnWNmIql8NPD0jQHe6ptEBWuAk0eqnEpjV4nBoIzSKbV3f9pJBRHuatANW0c1DEKEVVp\nZPep8HjpK9NfYL9ydQmaK4rwtATpmBODIeztDOCe3T7NHCXY2uBBa30pvv9mj+gN6sYicbx2fhwf\n2VKrmqAiJp/XiUyWYyConmMHF+IPx1VTEQPoMLCXOW2wmpnqZ+w9gSjKnDa4Hcr2NpECYwx3bK1F\ne18QXePiVjQ89EYXXAUW3LWjTtTrSu2+65owNDWD50+I+xTz8yNDyGT1U7t+Ia2UPI6E4qhWyeYk\nQIeB3WRiqCy2q/5Q655AVHcLp/PdtqUGZhMTddbeNxHF8ydGcNfOOhTbtfWG+N51FWipKMKDr4vX\nHIxzjqfaBrG5LveEpEfCz0ivypuBjYbUcXKSQHeBHdDGEXk9OujquJgKlx3XrynHM28PIiNS6elD\nb3TDYjIQujemAAAYI0lEQVThnquVb/a1XCYTw73XNuLMSBivnhsT5ZrHBkPoGJvW5aKpoMRhQ6nD\nquqSx0g8hUgirZqKGECngb3KXajqHPt0Io2xSELXM3YAuKO1FqPhBN7oyH9DWnvfJH58qB8f375a\nNf04luvWK2vQUObAXz93ClERTpt6qm0AdqsJH7qiWoTRqZfP61R1KkZNfdgFugzswhF5aj0vsVcH\nx+EtxQ3rKlHqsOadjomnMvjqU8exyl2Ir+5ZJ9Lo5GezmPBPt1+BoakZ/OOvz+Z1rXgqg18cG8ae\ny6s0l5ZaLp+3SNW17Go6OUmgy8BeWWxHIp3FVEyaOup8deuo+ddibBYTbr2yBi+fGsVUbOU7L7/1\n4jl0B6L4p9s3oajAIuII5bfd58FndjXg8f192Ne18oNJXjo9ikg8jdt1nIYRNJY74Q/HJdsHkK8R\nFZ11KtBlYK9W+UlKwoy9QYeljhe6o7UWyUwWj+3rW9HHt/dN4gdv9eDuHXW4WuOnAgkeuHkdGsoc\neODp4ytKyaQyWfznWz2oKSnEribtHli9VMLPSW9AnSWPQtq3olgdm5MAnQb2uVr2sDoXUHsCUaxy\n2xU7dFlOl69y44ObqvGd35zHK6dHl/Wx81MwX/vAZRKNUH6FNjO+ecfKUjKcc/z1c6dwpH8KX3nf\nGtXvvBXDXMmjStMxI6E4ypw2Vf086zKwv3NEnjr7xQhdHY3iW7dfgY01bvzxT44s6+AJPaVgLrSt\nwYPP7vItOyXzg709+PGhfnzx+iZ8ZIs+a9cv1OB1AFBvl8fRsLpKHQGdBvbyogKYWK5/g9pwztEz\nPq37ipj5Cm1mPPKpVrgLrfiDx9owtoQ9BnpMwVzoqzevXVZK5pXTo/jG82ew5/IqfPV9a2UYoTo4\nbBZUu+2qLXlU0wEbAl0GdovZhAqXXZU59mAshXA8rbt2vZdSUWzHI59uRWgmhc8/3oaZZOair9Vr\nCuZCy0nJnB4O449/cgQbVrnx7d+/whApmPl8XvU2A/OHZmjGLpcqt32u+b2aCI+TvtnHSyO5fJUb\n371zM44PhfCVp46+q2c+5xwnh0L46tPHdZuCudD8lMz33+hecP/FWDiOP3jsMIrtVjzy6VY4bPr+\nP1lIg0oPto6nMgjGUqoqdQQA3X6HVBXb0SlynxIxCF0djTZjF9y0vhJ/+f7L8I3nz+Db3vP44/e2\n4ED3BF4+PYpXzoxiJBQHY8AXr2/SbQrmQl+9eS3e7g/iG8+fwTeeP4NNtW7ceFklblpfiYYyJz7/\neBuCsRSe+sJVmt2cla9GrxPBWArBaFJVPeeFN2K1fV30G9jdduztXHmdsFR6AlFYTAy1pep6h5fT\nH1zjQ3dgGv/2aif+860eRJMZ2K0mXNtSjj+5aQ3eu64CZSrpay2HQpsZz/7hLnSOTeOl2Te4b798\nHt9++TyKCiyIJtN46BNbsaHGrfRQFTNXGTMRVVVgV9sBGwLdBvZqtx3TiTQi8RRcKtqZ1zsRRZ3H\nAasOW6wuFWMMf3vrBmRnO9jetL4Su1u8qioXkxtjDC2VLrRUuvCl9zRjLBLHb8+M4bdnx3Dd2nK8\n7/IqpYeoqLlmYIEottSVKjyad6ixnQCg48Au/EePhuOqCuzd4/ru6rhUVrMJ/3j7JqWHoVoVLjvu\n3F6HO7drqz2xVFZ7HDCb2FwqUy3UuOsU0PHiqbCYoabKmGyW676rIyFSsJpNqPM4VLeA6g/NoNhu\ngVNli/w6DuzqayswNDWDRDqr297ZhEipqdwp+sEt+crVsKtvvUy3gV3o26Cm9r3dBunqSIgUGstz\nXR7F6u8vBr8Kd50COg7sBRYzypw2Vc3Yu8Zys40mmrETsmxN5U4k0lkMT6lnR/lIKK66/DqQZ2Bn\njN3BGDvFGMsyxlrFGpRYqtx2VbUV6A5Mo9huQZmKyrUI0YrG8tyESC3pmFQmi8B0Qpcz9pMAPgLg\nDRHGIrpqtx3+sHoagXWNRdFUUQTGjLUdnBAxNM0FdnUsoI5FEuBcfTXsQJ6BnXN+hnN+TqzBiE1t\nM/au8Wk0GnTHKSH58jhtKHFYVTNjF2KLHmfsqlbtLkQwlkI8dfGGU3KJxFMYiyTQVEELp4SsVFN5\nEbpVEtjVeCSe4JKBnTH2CmPs5AK/bl3OjRhj9zLG2hhjbePj+R9uvBSrSnLvpINB5WftwsYKmrET\nsnKNXqdqUjFCXBHijJpcsqqec36jGDfinD8M4GEAaG1tlaVeqc6Tmx33T0YVrx3vnu3q2EwzdkJW\nrKmiCE+1DyIcTyl+iHffRAwep01VO9sFuk7FNJTlWuOq4azErrEozCY292ZDCFk+YQ+IGloL9E1E\nUV+mzvbb+ZY73sYYGwRwFYBfMcZeFGdY4vA4bSgqsKB/UvnA3h2YRp3HAZtF1++lhEhK2AOihjx7\n30QM9R51Bva8Ghxwzp8F8KxIYxEdYwx1Hgf6JpR/d+8ai6LJQOecEiKFOo8DFhNTvDImkc5gJDSD\nujJ1njur++ljfZkDfRPKztgzWY6eiejcBgtCyMpYzSbUlTnQNabsZG0wOIMsh2pn7AYI7E4MBGOK\n9pcYCs4gmc7SjJ0QETSVF80VIyilf3ay2KDSIy4NENgdSGU4RhTcqCQ8NtKMnZD8NZY70RtQdrIm\npHfVWgyh/8A++6jUr2A6RgjsTRTYCclbU3kRkpksBoPK/Uz3TcbgsJnhLVJn3yfdB/Y6oeRR0cAe\nRYnDCg81/yIkb0JKU8kF1L6JGOo8DtX2fdJ9YK92F8JmNqFvUrnFlu7xaZqtEyISYfe2krXsfRNR\nNJSpMw0DGCCwm00MtZ5ChVMxUTpcgxCRlDpt8Dhtis3YM1mOgckZ1W5OAgwQ2IFcnl2pVExoJoXA\ndIIO1yBERLlj8pSZsfvDcSQz2bk0rxoZI7CXOdE/EQXn8q+id9PCKSGiU7LLo1ARQ6kYhdWXORBN\nZjARTcp+b2FW0Ug17ISIprHcicB0EqFYSvZ7C2ndOpVuTgIMFNgBKLIDtXt8GhYTU/U3ASFaM3ea\nkgIblfomY7CaGVaVqK8Pu8AQgV3YRKBEz5iu8WnUlTlgNRviv5oQWcydfzqmQGCfiKK21AGzSZ2l\njoBBAvtqTyEYU2rGHqX8OiEiW11aCKuZoTsg/2StbyKm6ooYwCCBvcBixip3oezte9OZLHonopRf\nJ0RkFrMJ9WVO2WfsnHP0q7hdr8AQgR2AIu17B4MzSGU4zdgJkUBTuVP2GXswlkIkkUadiitiAAMF\n9gav/O173+kRo+5vAkK0qLG8CH0TUaQyWdnu2TtX6kgzdlWo8zgxEU1iOpGW7Z5zXR3pAGtCRNdU\nXoRUhmNAxhSrUOpIOXaVeKfkUb5Ht+7xKDxOG0qp+RchohPWruTsGdM3EQNjQG0pBXZVEOrI5UzH\ndI1PUxqGEIk0zT4Jy9kzpm8iiqpiO+xWs2z3XAnDBHYlNilRqSMh0nE7rPAWFcg7Y59Uf6kjYKDA\n7rJbUea0oV+m9r1TsSQmokkqdSREQo3lTpln7DHUq/TUpPnyCuyMsW8yxs4yxo4zxp5ljJWINTAp\n1Ml4sLXQI4Zm7IRIJ3f+qTyTtWgijcB0QtVdHQX5zthfBrCBc74JwHkAX8t/SNJpKHPKGNjpnFNC\npNZU7sRkNIlJGRr8CbFDzV0dBXkFds75S5xzoX7wAIDa/IcknTqPA8OhGSTSGcnv1TU+DauZYXWp\nehsFEaJ1whOxHC18hTSu0XLs9wB4QcTria6+zAHOcztCpdY9HkV9mRMWav5FiGTkLHkUZuy6SMUw\nxl5hjJ1c4Net817zdQBpAE8scp17GWNtjLG28fFxcUa/TMI7rRzH5FGpIyHSqy11wGY2ybKA2jcZ\nQ6nDimK7VfJ75ctyqRdwzm9c7N8ZY58B8CEA7+WLHFHEOX8YwMMA0NraKv9RRsidpAS8sy1YKvFU\nBv0TMey5vErS+xBidGYTQ2O5E+dHI5Lfq28iOhdD1C7fqpg9AB4AcAvnXLnTopeozGmD02aWfAH1\nnD+CdJZjY41b0vsQQoDLV7lxcjgs+dGXWmjXK8g3AfxvAFwAXmaMHWWMPSjCmCTDGENdmVPy9r3H\nh0IAgI21FNgJkdrGmmKMRxIYDScku0cyncXw1Izq2/UKLpmKWQznvFmsgcil3uPA+TFpH9tODE6h\n1GFFjYqPziJELzbW5rbPHB+cQpVbmvTnYDCGLIfq2/UKDFeyUe91YHByBpmsdI9txwdD2FhbAsbU\ne3QWIXqxvroYZhPDidknZSn0TQo17NqYsRsvsHucSGay8Ifjklw/nsqgY2wamyi/TogsCm1mtFQU\nSRrY+zVU6ggYMbBL3L739EgYmSyn/DohMtpY48aJwZBkC6h9EzE4bGaUFxVIcn2xGTiwS7OAemJw\nduGUZuyEyGZTrRsT0SSGQ9I8ifdNRFHncWgmvWq4wF7tzp1uLlVgPz4YgrfIhmq3XZLrE0LeTVhA\nPTE4Jcn1tdKuV2C4wG42MawudUjWvvfkUAgba9yaeWcnRA/WVblgkWgBNZvl6J+MaWZzEmDAwA7k\nFkB6A+LP2GPJNDrGInOzB0KIPOxWM9ZUunB8UPzA7g/HkUxn505h0wJDBvaG2U1KYi+0nB4OI8tB\nFTGEKGBTrRsnhsRfQNVSu16BIQN7nceB6URa9B7OwmyBKmIIkd/GWjemYinRu7dqqV2vwJCBvaki\n18P5nF/cHagnh0KocBWgspgWTgmR26aa2QVUkfPsZ/0R2K0mTRVEGDKwX7m6BIwBbX1BUa97fCiE\nTTRbJ0QRa6qKYDUz0fPs7X1BXFFboqmzFbQzUhG5C61YW+nC4d5J0a45nUija3waG2to4ZQQJRRY\nzFhXVYwTQ+KVPEYTaZwaDmNbg0e0a8rBkIEdALbWl+JI/5RoPWNODYXAOWjGToiCNtaKuwP12EAu\nRmxtKBXlenIxbGDf1uDBdCItWp5dyOttoIoYQhSzqcaNcDwtWmvutr4gGAO21FFg14St9bkvVFuf\nOOmYE0MhVLvtKHdpo5cEIXokTKzEyrMf7p3E2koX3IXqPw5vPsMG9trSQlQV29HWK84C6onBEPWH\nIURhaypdsFlMolTGZLIcR/qn0KqxNAxg4MDOGMPWhlK0ibCAGo6n0B2IUn6dEIXZLCZcVl2M4yL0\njDnrD2M6kUZrvbYWTgEDB3YA2FZfiuFQHENT+W1oODUUBkD5dULUYFONG6eGwsjmWRjRPlsOTTN2\njWmdLWHKd9YulFdRKoYQ5W2scSOSSKM3zzMXDvcGUVVs1+QRl4YO7OuqXHDYzHPvzCt1fDCEmpJC\nlGmkCT8heia09Mg3z97eO4mtDaWa7NRq6MBuMZuwpa4Uh/NcQD1JO04JUY2WiiIUWExzh96sxNDU\nDIZDcWyr114aBsgzsDPG/g9j7Dhj7Chj7CXG2CqxBiaXrfWlOOcPIxxPrejjQ7EUeidi1PiLEJWw\nmE24fFUxjucxYxfSs60a23EqyHfG/k3O+SbO+ZUAfgngr0UYk6y2NXiQ5cCR/pWtop8cpqPwCFGb\njTVunBoKrXhneVtvEE6bGeuqXCKPTB55BXbOeXjeH50ApDlJVkJX1pXAxHL5tJU4TmecEqI6G2tL\nEE1m0BOYXtHHt/UFsbmuVFONv+bLe9SMsW8wxgYA3A0NztiLCixYv6p4xXn2k0Mh1HkcKHHYRB4Z\nIWSlNuWxgBqOp3DWH9ZkmaPgkoGdMfYKY+zkAr9uBQDO+dc556sBPAHgjxa5zr2MsTbGWNv4+Lh4\nn4EIWus9ODowhVQmu+yPPT40Rfl1QlSmqbwIhVbziloLHOmfAufQ5MYkwSUDO+f8Rs75hgV+PXfB\nS58A8NFFrvMw57yVc95aXl6e77hF1dpQiplUBqeHw5d+8TwDkzEMTM7gCgrshKiK2cSwscaN/V0T\ny+702NY7CbOJ4co67bbgzrcqpmXeH28FcDa/4ShDeGde7sEbPzzYB7OJ4feu0FwxECG69+HNNTjr\njyx7n0pbbxCXVbtQVGCRaGTSyzfH/g+zaZnjAN4H4MsijEl2VW47aksLl7UDNZ7K4MnDA7j58kpU\nu7W3M40Qvfvw5lUotlvw6L7eJX9MKpPFkYGgptMwAJDXWxLn/KKpF61prS/FW7OPbUvZafaLo8OY\niqXwqasapB8cIWTZHDYLPta6Go/u68VoOL6ks4hPD4cRT2U1vXAKGHzn6XytDR6MRxJLatDPOcej\n+3qxttKFHT5tv7MTomefvKoeGc7xo4P9S3q9cFym1mfsFNhnCe/QS+nP3t4XxOmRMD61q16TfSQI\nMYr6Mifes7YCPzrUj2T60lVv7X3B3FkN7kvP7tWMAvusNRUuuOyWJZ2o9Nj+PrjsFty2uUaGkRFC\n8vGpq+oxHknghZMji76Oc47DvUG0arQ/zHwU2GeZTAxb60svOWMfC8fxwokRfKx1NRw27a6aE2IU\n17aUw+d14vH9fYu+rn8yhsB0QrP9YeajwD5Pa30pOsamMRVLXvQ1PzrUj3SW45M762UcGSFkpUwm\nhk/urEd7XxAnF9mJKuw+1/rCKUCB/Xdc1eQFAPzlsycQT2Xe9e/JdBZPHOzH9WvL0eB1yj08QsgK\nfXRrLRw2Mx67SOlj30QU//rbDniLbFhToc3GX/NRYJ9na30pvv6By/D8CT8+/R+HEJr53Va+vz7l\nx3gkgU/valBmgISQFXEXWnHb5ho8d2wYwejvPpEfH5zCR/7fPoRmUnjok60wmbRfEEGB/QKfv7YR\n3/n9K/F2fxC//9B++EPxuX97fF8v6sscuK5FXS0RCCGX9uldDUims3iybWDu714/P447Hz4Au9WM\np7+wC1t1sHAKUGBf0Ic31+A/PrMNA5MxfPR7+9A5FsHJoRDa+oL45M56XbyjE2I0aypduKqxDP+1\nvw+ZLMczbw/ic48eRn2ZE8/84S40VxQpPUTRUGC/iGtayvHkfVchkc7g9gf34+9+dRqFVjPuaF2t\n9NAIISv06V31GJqawZeeeBt/+tNj2NbgwZP37VzSrlQtocC+iA01bjzzxatRUmjFge5J3LalBu5C\nq9LDIoSs0I2XVWKV245fn/Ljg5uq8eg921Bs19/PNBViX0JdmQM/++IuPPh6F+7Z7VN6OISQPFjM\nJvz9RzfhvD+Cz+326Tatypbbq1gMra2tvK2tTfb7EkKIljHG2jnnrZd6HaViCCFEZyiwE0KIzlBg\nJ4QQnaHATgghOkOBnRBCdIYCOyGE6AwFdkII0RkK7IQQojOKbFBijI0DWPw4E3XyAggoPQgZGe3z\nBehzNgqtfs71nPNLtpdVJLBrFWOsbSm7vvTCaJ8vQJ+zUej9c6ZUDCGE6AwFdkII0RkK7MvzsNID\nkJnRPl+APmej0PXnTDl2QgjRGZqxE0KIzlBgXwHG2FcYY5wx5lV6LFJjjH2TMXaWMXacMfYsY6xE\n6TFJhTG2hzF2jjHWyRj7C6XHIzXG2GrG2KuMsdOMsVOMsS8rPSY5MMbMjLEjjLFfKj0WqVBgXybG\n2GoA7wPQr/RYZPIygA2c800AzgP4msLjkQRjzAzg3wG8H8B6AB9njK1XdlSSSwP4Cud8PYCdAL5k\ngM8ZAL4M4IzSg5ASBfbl+xcADwAwxOIE5/wlznl69o8HANQqOR4JbQfQyTnv5pwnAfwEwK0Kj0lS\nnPMRzvnbs7+PIBfsapQdlbQYY7UAPgjgEaXHIiUK7MvAGLsVwBDn/JjSY1HIPQBeUHoQEqkBMDDv\nz4PQeZCbjzHWAGAzgIPKjkRy30FuYpZVeiBSosOsL8AYewVA1QL/9HUAf4lcGkZXFvucOefPzb7m\n68g9uj8h59iI9BhjRQB+BuB+znlY6fFIhTH2IQBjnPN2xtj1So9HShTYL8A5v3Ghv2eMbQTgA3CM\nMQbkUhJvM8a2c879Mg5RdBf7nAWMsc8A+BCA93L91scOAVg978+1s3+na4wxK3JB/QnO+TNKj0di\nVwO4hTH2AQB2AMWMsR9yzj+h8LhER3XsK8QY6wXQyjnXYiOhJWOM7QHwbQDXcc7HlR6PVBhjFuQW\nh9+LXEA/DOAuzvkpRQcmIZaboTwGYJJzfr/S45HT7Iz9zzjnH1J6LFKgHDu5lH8D4ALwMmPsKGPs\nQaUHJIXZBeI/AvAicouIP9VzUJ91NYBPArhh9mt7dHY2SzSOZuyEEKIzNGMnhBCdocBOCCE6Q4Gd\nEEJ0hgI7IYToDAV2QgjRGQrshBCiMxTYCSFEZyiwE0KIzvx/kVg1si3G2k0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1115fc668>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pylab as plt\n",
    "\n",
    "x = Variable(torch.arange(-5,5,0.2), requires_grad=True)\n",
    "#print(x)\n",
    "\n",
    "#y = torch.sum(torch.sigmoid(0.3*x))\n",
    "y = torch.sum(x*torch.cos(x)**2)\n",
    "#print(y)\n",
    "\n",
    "#plt.plot(x.data.numpy(), y.data.numpy() )\n",
    "#plt.show()\n",
    "\n",
    "y.backward()\n",
    "plt.plot(x.data.numpy(), x.grad.data.numpy() )\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\n",
    "f(x_1, x_2) = \\sin\\left(\\frac{1}{2} x_1^2 - \\frac{1}{4} x_2^2 + 3 \\right) \\cos\\left(2 x_1+1-e^{x_2}\\right)\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1.00000e-02 *\n",
      "  5.9661\n",
      "[torch.FloatTensor of size 1]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import torch \n",
    "from torch.autograd import Variable\n",
    "\n",
    "x_1 = Variable( torch.FloatTensor([0.5]) , requires_grad=True)\n",
    "x_2 = Variable( torch.FloatTensor([3.5]) , requires_grad=True)\n",
    "\n",
    "u1 = x_1 ** 2 \n",
    "u2 = 0.5 * u1\n",
    "u3 = x_2 ** 2\n",
    "u4 =-0.25 * u3\n",
    "u5 = u2 + u4\n",
    "u6 = u5 + 3\n",
    "u7 = tr.sin(u6)\n",
    "u8 = 2 * x_1\n",
    "u9 = u8 + 1\n",
    "u10 = tr.exp(x_2)\n",
    "u11 = -1 * u10\n",
    "u12 = u9 + u11\n",
    "u13 = tr.cos(u12)\n",
    "f = u7 * u13\n",
    "\n",
    "print(f.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1.00000e-02 *\n",
      "  5.9661\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "\n",
      " 0.4397\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "\n",
      "-1.0561\n",
      "[torch.FloatTensor of size 1]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "f.backward()\n",
    "\n",
    "print(x_1.grad.data)\n",
    "print(x_2.grad.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "u7.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 0.4990\n",
       "[torch.FloatTensor of size 1]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_1.grad.data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$f(x_1, x_2) = x_1^2$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 34\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "\n",
      "-8\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "\n",
      " 12\n",
      "[torch.FloatTensor of size 1]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "x_1 = Variable( tr.FloatTensor([-4]) , requires_grad=True)\n",
    "x_2 = Variable( tr.FloatTensor([3]) , requires_grad=True)\n",
    "\n",
    "f = x_1**2 + 2*x_2**2\n",
    "\n",
    "print(f.data)\n",
    "\n",
    "f.backward()\n",
    "\n",
    "print(x_1.grad.data)\n",
    "print(x_2.grad.data)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "A = torch.DoubleTensor([[1,2,3],[4,5,6]])\n",
    "T = torch.rand([3,5,2])\n",
    "u = torch.DoubleTensor([[7],[8]])\n",
    "w = torch.rand([5,3]).double()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PyTorch has several Loss function.\n",
    "\n",
    "Its good to know a few:\n",
    "\n",
    "$$E(f, y) = (1/N) \\sum_{i=1}^N (f_i-y_i)^2$$\n",
    "EuclidianLoss = torch.nn.MSELoss(size_average=True)\n",
    "\n",
    "CE_Loss = torch.nn.CrossEntropyLoss(reduce=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import scipy as sc\n",
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate sz[0] example points of dimension sz[1]\n",
    "# We have sz[1] classes\n",
    "\n",
    "sz = (3,5)\n",
    "th = np.random.randn(*sz)\n",
    "# Generate random targets\n",
    "c = np.random.choice(range(sz[1]),size=sz[0])\n",
    "\n",
    "inp = Variable(torch.FloatTensor(th), requires_grad=True)\n",
    "target = Variable(torch.LongTensor(c), requires_grad=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 2.7447\n",
      " 4.2792\n",
      " 2.6336\n",
      "[torch.FloatTensor of size 3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "CE_Loss = torch.nn.CrossEntropyLoss(reduce=False)\n",
    "E = CE_Loss(inp, target)\n",
    "print(E)\n",
    "#E.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.74465860561\n",
      "4.27917593403\n",
      "2.63359341109\n"
     ]
    }
   ],
   "source": [
    "from functools import reduce \n",
    "\n",
    "for i,j in enumerate(c):\n",
    "    res = -th[i,j] + reduce(np.logaddexp, th[i,:])\n",
    "    print(res)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (py36)",
   "language": "python",
   "name": "py36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}