part10-Theano.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# http://bogotobogo.com/python/scikit-learn/Artificial-Neural-Network-ANN-10-Deep-Learning-3-Theano-TensorFlow-Keras.php"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import theano"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(200.5)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# initialize\n",
    "x1 = theano.tensor.scalar()\n",
    "w1 = theano.tensor.scalar()\n",
    "w0 = theano.tensor.scalar()\n",
    "z1 = w1 * x1 + w0\n",
    "\n",
    "# compile\n",
    "net_input = theano.function(inputs=[w1, x1, w0], outputs=z1)\n",
    "\n",
    "# execute\n",
    "net_input(20.0, 10.0, 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'float64'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theano.config.floatX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('cpu', 'float64')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theano.config.device, theano.config.floatX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Column sum Python list:', array([  9.,  11.,  13.]))\n",
      "('Column sum NumPy array:', array([  9.,  11.,  13.]))\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "# initialize\n",
    "# if running Theano on 32 bit mode, \n",
    "# use fmatrix instead of dmatrix\n",
    "x = theano.tensor.dmatrix(name='x')\n",
    "x_sum = theano.tensor.sum(x, axis=0)\n",
    "\n",
    "# compile\n",
    "calc_sum = theano.function(inputs=[x], outputs=x_sum)\n",
    "\n",
    "# execute (Python list)\n",
    "parray = [[1, 2, 3], [8, 9, 10]]\n",
    "print('Column sum Python list:', calc_sum(parray))\n",
    "\n",
    "# execute (NumPy array)\n",
    "nparray = np.array([[1, 2, 3], [8, 9, 10]], dtype=theano.config.floatX)\n",
    "print('Column sum NumPy array:', calc_sum(nparray))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[<TensorType(float64, matrix)>, Elemwise{add,no_inplace}.0]]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w = theano.shared(np.asarray([[0.0, 0.0, 0.0]], \n",
    "                             dtype=theano.config.floatX))\n",
    "update = [[w, w + 1.0]]\n",
    "update"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('z0:', array([[ 0.]]))\n",
      "('z1:', array([[ 6.]]))\n",
      "('z2:', array([[ 12.]]))\n",
      "('z3:', array([[ 18.]]))\n",
      "('z4:', array([[ 24.]]))\n"
     ]
    }
   ],
   "source": [
    "from theano import tensor as T\n",
    "\n",
    "# initialize\n",
    "x = T.dmatrix(name='x')\n",
    "w = theano.shared(np.asarray([[0.0, 0.0, 0.0]], \n",
    "                             dtype=theano.config.floatX))\n",
    "z = x.dot(w.T)\n",
    "update = [[w, w + 1.0]]\n",
    "\n",
    "# compile\n",
    "net_input = theano.function(inputs=[x], updates=update, outputs=z)\n",
    "\n",
    "# execute\n",
    "data = np.array([[1, 2, 3]], dtype=theano.config.floatX)\n",
    "for i in range(5):\n",
    "    print('z%d:' % i, net_input(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('z:', array([[ 0.]]))\n",
      "('z:', array([[ 6.]]))\n",
      "('z:', array([[ 12.]]))\n",
      "('z:', array([[ 18.]]))\n",
      "('z:', array([[ 24.]]))\n"
     ]
    }
   ],
   "source": [
    "# initialize\n",
    "data = np.array([[1, 2, 3]], dtype=theano.config.floatX)\n",
    "x = T.dmatrix(name='x')\n",
    "w = theano.shared(np.asarray([[0.0, 0.0, 0.0]], \n",
    "                             dtype=theano.config.floatX))\n",
    "z = x.dot(w.T)\n",
    "update = [[w, w + 1.0]]\n",
    "\n",
    "# compile\n",
    "net_input = theano.function(inputs=[], \n",
    "                            updates=update, \n",
    "                            givens={x: data},\n",
    "                            outputs=z)\n",
    "\n",
    "# execute\n",
    "for i in range(5):\n",
    "    print('z:', net_input())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[ 0.],\n",
       "        [ 1.],\n",
       "        [ 2.],\n",
       "        [ 3.],\n",
       "        [ 4.],\n",
       "        [ 5.],\n",
       "        [ 6.],\n",
       "        [ 7.],\n",
       "        [ 8.],\n",
       "        [ 9.]]),\n",
       " array([ 1. ,  1.3,  3.1,  2. ,  5. ,  6.3,  6.6,  7.4,  8. ,  9. ]))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# creating one-dimensional dataset\n",
    "import numpy as np\n",
    "X_train = np.asarray([[0.0], [1.0], [2.0], [3.0], [4.0],\n",
    "                      [5.0], [6.0], [7.0], [8.0], [9.0]], \n",
    "                     dtype=theano.config.floatX)\n",
    "\n",
    "y_train = np.asarray([1.0, 1.3, 3.1, 2.0, 5.0, \n",
    "                      6.3, 6.6, 7.4, 8.0, 9.0], \n",
    "                     dtype=theano.config.floatX)\n",
    "X_train, y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import theano\n",
    "from theano import tensor as T\n",
    "import numpy as np\n",
    "\n",
    "def train_linreg(X_train, y_train, eta, epochs):\n",
    "\n",
    "    costs = []\n",
    "    # Initialize arrays\n",
    "    eta0 = T.dscalar('eta0')\n",
    "    y = T.dvector(name='y') \n",
    "    X = T.dmatrix(name='X')   \n",
    "    w = theano.shared(np.zeros(\n",
    "                      shape=(X_train.shape[1] + 1),\n",
    "                      dtype=theano.config.floatX),\n",
    "                      name='w')\n",
    "    \n",
    "    # calculate cost\n",
    "    net_input = T.dot(X, w[1:]) + w[0]\n",
    "    errors = y - net_input\n",
    "    cost = T.sum(T.pow(errors, 2)) \n",
    "\n",
    "    # perform gradient update\n",
    "    gradient = T.grad(cost, wrt=w)\n",
    "    update = [(w, w - eta0 * gradient)]\n",
    "\n",
    "    # compile model\n",
    "    train = theano.function(inputs=[eta0],\n",
    "                            outputs=cost,\n",
    "                            updates=update,\n",
    "                            givens={X: X_train,\n",
    "                                    y: y_train})      \n",
    "    \n",
    "    for _ in range(epochs):\n",
    "        costs.append(train(eta))\n",
    "    \n",
    "    return costs, w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk4AAAGGCAYAAACNCg6xAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3XucXWV99/3PLyRAjCRRNJDIQUDUSCSQ4RCyN+ItFkQr\narHVEayU21YsWk3rLY9PpSo+qMVboYK2tFoVxLFKH8tBJIoI3pBwykSOARUBI5BwDhhOIfndf6w9\nZTJMkj2TvWftvebzfr3Wa89ea+21fxcB5pvruta1IjORJEnS5k0ouwBJkqRuYXCSJElqksFJkiSp\nSQYnSZKkJhmcJEmSmmRwkiRJapLBSZIkqUkGJ0mSpCYZnCRJkppkcJIkSWpS6cEpIo6PiBsiYnVj\nWxwRbxp0/PKIWD9oWxcRXxtyjZ0j4kcRsSYiVkbEqRFRetskSVK1TCy7AGAFcCLwm8b7Y4HzI2Kf\nzFwOJPCvwElANM55YuDDjYB0MXAvMB+YBZwDPAN8cgzqlyRJ40R04kN+I+Ih4GOZ+c2I+DmwLDP/\ndiPnHgFcAMzMzAcb+z4AfAF4aWY+O1Z1S5Kkauuo4ayImBAR7wZeACwedOjoiHggIm6KiM9FxORB\nx+YDNw2EpoZFwDRgr/ZXLUmSxotOGKojIuYAS4BtgceBd2Tm7Y3D5wJ3UwzF7Q2cCrwSeGfj+I7A\nqiGXXDXo2A3tq1ySJI0nHRGcgNuAucB04Cjg7Ih4XWbelplfH3TeLRGxEvhZROyWmXdu5robHYeM\niO2Bw4G7gKe2qHpJktSptgVeDizKzIe29GIdEZwa85B+23jbHxEHAB8BPjjM6dc0Xl8B3AmsBPYf\ncs4OjdehPVGDHU7RmyVJkqrvaOC7W3qRjghOw5gAbLORY/tS9CTd13i/BPh/I+Ilg+Y5HQasBm7d\nxHfcBfCd73yH2bNnb3HBnWjhwoWcdtppZZfRVlVvY9XbB9VvY9XbB9VvY9XbB9Vu4/LlyznmmGOg\n8Xt/S5UenCLiFODHFMsSbEeRCA8BDouI3YH3UCw38BDFcN6XgSsy8+bGJX5CEZDOiYgTgZnAZ4Ez\nM3PtJr76KYDZs2czb968lrerE0ybNq2ybRtQ9TZWvX1Q/TZWvX1Q/TZWvX0wPtpIi6bllB6cKIbV\nzqYIPKuBG4HDMvOyiNgJeCPFsN0UinD1A+CUgQ9n5vqI+GPgnynuxFsDfAv41Bi2QZIkjQOlB6fM\nfP8mjv0eeH0T11gB/HELy5IkSXqejlrHSZIkqZMZnCqst7e37BLaruptrHr7oPptrHr7oPptrHr7\nYHy0sVU68pErYyEi5gFLly5dOh4mxEmSNC719/fT09MD0JOZ/Vt6PXucJEmSmmRwkiRJapLBSZIk\nqUkGJ0mSpCYZnCRJkppkcJIkSWqSwUmSJKlJBidJkqQmGZwkSZKaZHCSJElqksFJkiSpSQYnSZKk\nJhmcJEmSmmRwkiRJapLBSZIkqUkGJ0mSpCYZnCRJkpo07oPTo4+WXYEkSeoW4z443Xhj2RVIkqRu\nMe6D07JlZVcgSZK6xbgPTjfcUHYFkiSpW4z74HTLLfDUU2VXIUmSusG4D07PPgtLl5ZdhSRJ6gbj\nPjhtuy1cdVXZVUiSpG4w7oPTa18LV15ZdhWSJKkbjPvgtM8+sHgxZJZdiSRJ6nQGp33goYfg9tvL\nrkSSJHW6cR+cXvtamDDB4TpJkrR54z44TZkCe+/tBHFJkrR54z44AdTrBidJkrR5BiegVoNf/xpW\nrSq7EkmS1MkMThQ9TlDcXSdJkrQxpQeniDg+Im6IiNWNbXFEvGnQ8W0i4qsR8WBEPB4R50XEjCHX\n2DkifhQRayJiZUScGhFNt22nnWCXXRyukyRJm1Z6cAJWACcCPY3tMuD8iJjdOH468BbgKOB1wCzg\nPwc+3AhIFwMTgfnA+4BjgZNHUkSt5p11kiRp00oPTpn5o8y8JDN/09g+CfwBmB8RU4HjgIWZeUVm\nLgP+AqhFxAGNSxwOvBo4OjNvysxFwEnACRExsdk66nXo74cnn2xp8yRJUoWUHpwGi4gJEfFu4AXA\nEooeqInAzwbOyczbgd8BBzV2zQduyswHB11qETAN2KvZ767VYO1auO66LWuDJEmqro4IThExJyIe\nB54Gvga8IzNvA3YEnsnMx4Z8ZFXjGI3XoffDrRp0rClz5sDUqQ7XSZKkjWt6KKvNbgPmAtMp5jKd\nHRGv28T5ATTzdLmmn0C31VZw0EFOEJckSRvXEcEpM58Fftt429+Yv/QR4PvA1hExdUiv0wye61Va\nCew/5JI7NF43uzLTwoULmTZtGgB33QV33AHnntvL0Uf3jqotkiSpHH19ffT19W2wb/Xq1S39jshs\nulNmzETEz4C7gY8CDwDvzswfNo69kqKH6sDMvK6xdMGFwMyBeU4R8VfAPwIzMnPtRr5jHrB06dKl\nzJs3D4DLLoNDD4WbbiqG7iRJUnfr7++np6cHoCcz+7f0eqX3OEXEKcCPKZYl2A44GjgEOCwzH4uI\nbwBfjohHgMeBrwBXZebANO6fALcC50TEicBM4LPAmRsLTRtz4IHFkN1VVxmcJEnS83XC5PAdgLMp\nepEupbiT7rDMvKxxfCFwEXAecDlwL8U8KAAycz3wx8A6YHHjWt8CPjXSQqZMgX33dYK4JEkaXuk9\nTpn5/s0cfxr4cGPb2DkrKMLTFqvX4fzzW3ElSZJUNZ3Q49RRajW48064996yK5EkSZ3G4DRErVa8\nuiyBJEkayuA0xMyZsPvuBidJkvR8Bqdh1GoGJ0mS9HwGp2HUarBsGfzhD2VXIkmSOonBaRj1Oqxb\nB9deW3YlkiSpkxichjF7Nkyf7npOkiRpQwanYUyYAAsWOM9JkiRtyOC0EfU6LFlSDNlJkiSBwWmj\najV4/PHigb+SJElgcNqo/feHSZMcrpMkSc8xOG3E5MnQ02NwkiRJzzE4bUKt5p11kiTpOQanTajX\nYcWKYpMkSTI4bcKCBcWrw3WSJAkMTps0YwbsuafDdZIkqWBw2ox63R4nSZJUMDhtRq0GN94Ijz1W\ndiWSJKlsBqfNqNVg/Xq4+uqyK5EkSWUzOG3Gq14F22/vcJ0kSTI4bVaE6zlJkqSCwakJtRpccw08\n+2zZlUiSpDIZnJpQr8OaNXDDDWVXIkmSymRwakJPD2yzjcN1kiSNdwanJmyzDey/vxPEJUka7wxO\nTarViuCUWXYlkiSpLAanJtVqcO+9cNddZVciSZLKYnBqkg/8lSRJBqcmbb89zJ5tcJIkaTwzOI2A\nC2FKkjS+GZxGoF6HW26BRx8tuxJJklQGg9MI1GrFXXVLlpRdiSRJKoPBaQT22ANmzHC4TpKk8crg\nNAIRxXCdE8QlSRqfDE4jNPDA32eeKbsSSZI01koPThHxiYi4NiIei4hVEfHDiHjlkHMuj4j1g7Z1\nEfG1IefsHBE/iog1EbEyIk6NiJa3r16Hp56CZctafWVJktTpSg9OwMHAGcCBwBuBScBPImLyoHMS\n+FdgB2BHYCbw8YGDjYB0MTARmA+8DzgWOLnVxe67L0ye7HCdJEnjUenBKTPfnJnnZObyzLyJIvDs\nAvQMOfWJzHwgM+9vbH8YdOxw4NXA0Zl5U2YuAk4CToiIia2sd9IkOOAAJ4hLkjQelR6chjGdoofp\n4SH7j46IByLipoj43JAeqfnATZn54KB9i4BpwF6tLnBggrgP/JUkaXzpqOAUEQGcDlyZmbcOOnQu\ncAzweuBzwHuBcwYd3xFYNeRyqwYda6laDe6/H+64o9VXliRJnaylw1gt8DXgNUBt8M7M/Pqgt7dE\nxErgZxGxW2beuZlrtrxf6KCDiqUJrrwSXvGKVl9dkiR1qo4JThFxJvBm4ODMvG8zp1/TeH0FcCew\nEth/yDk7NF6H9kRtYOHChUybNm2Dfb29vfT29m70M9Onw5w5xXDdscduplJJkjQm+vr66Ovr22Df\n6tWrW/odkR0wUacRmt4GHJKZv23i/BrwC2BuZt4cEW8CLgRmDsxzioi/Av4RmJGZa4e5xjxg6dKl\nS5k3b96Ia/7gB+GKK+DWWzd/riRJKkd/fz89PT0APZnZv6XXK32OU2M9pqOB9wBrImKHxrZt4/ju\nEfHJiJgXEbtGxJHAt4ErMvPmxmV+AtwKnBMRe0fE4cBngTOHC02tUKvB8uXw0EPtuLokSepEpQcn\n4HhgKnA5cO+g7c8ax5+hWN9pEbAc+CLwA+DIgQtk5nrgj4F1wGLgbOBbwKfaVXS9XrwuXtyub5Ak\nSZ2m9DlOmbnJ8JaZv6e4m25z11lBEZ7GxK67wqxZxQTxt751rL5VkiSVqRN6nLqSD/yVJGn8MTht\ngVoNrruueHadJEmqPoPTFqjV4JlnYOnSsiuRJEljweC0BebOhSlTHK6TJGm8MDhtgYkTYf58g5Mk\nSeOFwWkL1Wo+8FeSpPHC4LSF6vViEczbby+7EkmS1G4Gpy00fz5MmOBwnSRJ44HBaQtttx3svXex\nEKYkSao2g1MLuBCmJEnjg8GpBWo1+PWv4f77y65EkiS1k8GpBWq14tVeJ0mSqs3g1AI77wy77GJw\nkiSp6gxOLVKrOUFckqSqMzi1SL0O/f3w5JNlVyJJktrF4NQitRqsXQvXXVd2JZIkqV0MTi0yZw5M\nnepwnSRJVWZwapGttoKDDnKCuCRJVWZwaqFaDRYvhvXry65EkiS1g8GphWo1ePRRuPXWsiuRJEnt\nYHBqoQMPLIbsHK6TJKmaDE4tNGUK7LuvwUmSpKoyOLWYC2FKklRdBqcWq9fhzjvhvvvKrkSSJLWa\nwanFfOCvJEnVZXBqsZkzYffdHa6TJKmKDE5tUKvZ4yRJUhUZnNqgVoNly+APfyi7EkmS1EoGpzao\n12HdOrj22rIrkSRJrWRwaoPZs2H6dIfrJEmqGoNTG0yYAAsWOEFckqSqMTi1Sb0OS5YUQ3aSJKka\nDE5tUqvB44/DzTeXXYkkSWoVg1Ob7L8/TJrkcJ0kSVVicGqTyZOhp8cJ4pIkVUnpwSkiPhER10bE\nYxGxKiJ+GBGvHHLONhHx1Yh4MCIej4jzImLGkHN2jogfRcSaiFgZEadGRKntcyFMSZKqpfTgBBwM\nnAEcCLwRmAT8JCImDzrndOAtwFHA64BZwH8OHGwEpIuBicB84H3AscDJ7S9/4+p1+N3vYMWKMquQ\nJEmtUnpwysw3Z+Y5mbk8M2+iCDy7AD0AETEVOA5YmJlXZOYy4C+AWkQc0LjM4cCrgaMz86bMXASc\nBJwQERPHuEn/bcGC4tVeJ0mSqqH04DSM6UACDzfe91D0JP1s4ITMvB34HXBQY9d84KbMfHDQdRYB\n04C92l3wxsyYAXvu6QRxSZKqoqOCU0QExbDclZl5a2P3jsAzmfnYkNNXNY4NnLNqmOMMOqcU9bo9\nTpIkVUVHBSfga8BrgN4mzg2KnqnNaeactqnV4MYb4bGhsU+SJHWd0ub/DBURZwJvBg7OzHsHHVoJ\nbB0RU4f0Os3guV6llcD+Qy65Q+N1aE/UBhYuXMi0adM22Nfb20tvbzPZbfNqNVi/Hq6+Gg47rCWX\nlCRJw+jr66Ovr2+DfatXr27pd0RmqR0yRRFFaHobcEhm/nbIsanAA8C7M/OHjX2vBG4DDszM6yLi\nTcCFwMyBeU4R8VfAPwIzMnPtMN85D1i6dOlS5s2b17a2ZcJLXwonnACf+UzbvkaSJA2jv7+fnp4e\ngJ7M7N/S65Xe4xQRX6MYmjsSWBMRAz1FqzPzqcx8LCK+AXw5Ih4BHge+AlyVmdc1zv0JcCtwTkSc\nCMwEPgucOVxoGksRruckSVJVdMIcp+OBqcDlwL2Dtj8bdM5C4CLgvEHnHTVwMDPXA38MrAMWA2cD\n3wI+1ebam1KrFUN1zz5bdiWSJGlLlN7jlJmbDW+Z+TTw4ca2sXNWUISnjlOvw5o1cMMNxWNYJElS\nd+qEHqfK6+mBbbZxuE6SpG5ncBoD22wD++3nQpiSJHU7g9MYGVgIswNuYpQkSaNkcBojtRrcey/c\nfXfZlUiSpNEyOI2RgQf+OlwnSVL3GlVwioh/iIgXDLN/ckT8w5aXVT3bbw+zZztBXJKkbjbaHqdP\nAS8cZv8L6JC1kzpRrWaPkyRJ3Wy0wWljD9idCzw8+nKqrV6HW26BRx8tuxJJkjQaIwpOEfFIRDxM\nEZp+FREPD9pWAz8Fvt+OQqugVivuqluypOxKJEnSaIx05fCPUvQ2/TvFkNzgRw4/A9yVmcaCjdhj\nD5gxoxiuO+KIsquRJEkjNaLglJnfBoiIOykesuvT10Yg4rn1nCRJUvcZ7Rynx4HZA28i4m0R8V8R\n8bmI2Lo1pVVTrQbXXgvPPFN2JZIkaaRGG5zOAl4JEBG7A/8BPAH8KXBqa0qrploNnnwSli0ruxJJ\nkjRSow1OrwR+2fj5T4ErMvM9wLHAUS2oq7L23RcmT3a4TpKkbrQlyxEMfPaNwMWNn1cAL9nSoqps\n663hgAMMTpIkdaPRBqfrgU9GxHuBQ4AfNfbvBqxqRWFVVq8Xd9b5wF9JkrrLaIPTR4F5wJnAKZn5\nm8b+dwKLW1FYldVqcP/9cMcdZVciSZJGYqTrOAGQmTcCrx3m0P8C1m1RRePAQQcVSxNcdRW84hVl\nVyNJkpo12h4nACKiJyKOiYijI2JeZj6VmWtbVVxVTZ8Oc+b43DpJkrrNqHqcImIGxRIEhwCPUkwW\nnxYRPwfenZkPtK7EaqrV4Ioryq5CkiSNxGh7nM4AtgP2yswXZ+aLgDnAVOArrSquymo1WL4cHnqo\n7EokSVKzRhuc3gR8MDOXD+zIzFuBEwCfwtaEer14XexUekmSusZog9MEYLi5TGu34Jrjyq67wqxZ\nruckSVI3GW3IuQz4p4iYNbAjIl4GnAb8rBWFVV1EMVznBHFJkrrHaIPThyjmON0VEXdExG+AOxv7\nPtyq4qquXofrr4enny67EkmS1IzRruO0ApgXEX8EvJrirrpbM/PSVhZXdbVaEZqWLoUFC8quRpIk\nbc6Iepwi4g0RcWtETAXIzJ9m5hmZ+RXguoi4JSIObkulFTR3LkyZ4nCdJEndYqRDdR8F/i0zHxt6\nIDNXA2cBf9uKwsaDiRNh/nwniEuS1C1GGpzmApds4vhPgJ7RlzP+1GpFcPKBv5Ikdb6RBqcdGH4Z\nggHPAi8dfTnjT71eLIJ5++1lVyJJkjZnpMHpHoZ/uO+AvYH7Rl/O+DN/PkyY4HCdJEndYKTB6WLg\n5IjYduiBiJgMfAa4qBWFjRfbbQd77+0EcUmSusFIlyP4/4A/AX4VEWcCtwMJzKZ43MpWwCktrXAc\nqNdh0aKyq5AkSZszoh6nzFwFLABuBj4P/BD4L+BzjX21xjkagVoNfv1ruP/+siuRJEmbMuKVwzPz\n7sx8M/AS4EBgPvCSzHxzZt7V4vrGhVqteHWekyRJnW3UD+TNzEcy87rMvDYzH9mSIiLi4Ii4ICLu\niYj1EXHkkOPfbOwfvF085JwXRcS5EbE6Ih6JiK9HxJQtqWus7Lwz7LKLwUmSpE436uDUYlOAX1LM\nk9rYikY/plgOYcfG1jvk+Hcp5lodCrwFeB3FgpxdYWA9J0mS1LlG9ay6VsvMS2gsrBkRsZHTns7M\nB4Y7EBGvBg4HejJzWWPfh4EfRcTHMnNlG8puqVoNzjsPnnwSJk8uuxpJkjScTulxasbrI2JVRNwW\nEV+LiBcPOnYQ8MhAaGq4lKL36sAxrXKU6nVYuxauu67sSiRJ0sZ0S3D6MfDnwBuAjwOHABcP6p3a\nEdjgnrTMXAc83DjW8ebMgalTHa6TJKmTdcRQ3eZk5vcHvb0lIm4C7gBeD/x8Ex8NNj5nqqNstRUc\ndJALYUqS1Mm6IjgNlZl3RsSDwCsogtNKYMbgcyJiK+BFwCbXlVq4cCHTpk3bYF9vby+9vUPnnrdf\nrQZf/jKsX188hkWSJDWvr6+Pvr6+DfatXr26pd8RmZ3VIRMR64G3Z+YFmzhnJ+Bu4G2ZeVFjcvgt\nwH6DJocfRvGImJ2GmxweEfOApUuXLmXevHntaMqIXXYZHHoo3Hwz7LVX2dVIktT9+vv76enpgeIG\nsv4tvV5H9GtExJSImBsR+zR27d54v3Pj2KkRcWBE7BoRh1KsVv4rYBFAZt7W+PnfImL/iKgBZwB9\n3XBH3YADDyyG7ByukySpM3VEcAL2A5YBSynmJH0J6Kd4aPA6YG/gfIpn4/0bcB3wusxcO+ga7wFu\no7ib7iLgF8AHxqj+lpgyBfbd1wnikiR1qo6Y45SZV7DpEPemJq7xKHBMy4oqSa0GF2x0kFKSJJWp\nU3qc1FCvw513wn33lV2JJEkayuDUYXzgryRJncvg1GFmzoTddnOCuCRJncjg1IHqdXucJEnqRAan\nDlSrwbJlsGZN2ZVIkqTBDE4dqF6HdevgmmvKrkSSJA1mcOpAs2fD9OkO10mS1GkMTh1owgRYsMDg\nJElSpzE4dah6HRYvLobsJElSZzA4dahaDR5/vHjgryRJ6gwGpw61//4waZLDdZIkdRKDU4eaPBl6\nelwIU5KkTmJw6mC1mj1OkiR1EoNTB6vV4He/gxUryq5EkiSBwamj+cBfSZI6i8Gpg82YAXvuaXCS\nJKlTGJw6XK3mBHFJkjqFwanD1etw443Fmk6SJKlcBqcOV6vB+vVw9dVlVyJJkgxOHe5Vr4Ltt3e4\nTpKkTmBw6nARruckSVKnMDh1gVqtGKp79tmyK5EkaXwzOHWBeh3WrIEbbii7EkmSxjeDUxfo6YFt\ntnG4TpKkshmcusA228B++zlBXJKkshmcukS9XvQ4ZZZdiSRJ45fBqUvUanDvvXD33WVXIknS+GVw\n6hILFhSvDtdJklQeg1OX2H57mD3bCeKSJJXJ4NRFXAhTkqRyGZy6SL0ON98Mjz5adiWSJI1PBqcu\nUqsVd9UtWVJ2JZIkjU8Gpy6yxx4wY4bDdZIklcXg1EUiiuE676yTJKkcBqcuU6vBtdfC2rVlVyJJ\n0vhjcOoytRo8+SQsW1Z2JZIkjT8dEZwi4uCIuCAi7omI9RFx5DDnnBwR90bEExHx04h4xZDjL4qI\ncyNidUQ8EhFfj4gpY9eKsbHvvjB5ssN1kiSVoSOCEzAF+CVwAvC8p7FFxInAh4APAAcAa4BFEbH1\noNO+C8wGDgXeArwOOKu9ZY+9rbeGAw5wgrgkSWWYWHYBAJl5CXAJQETEMKd8BPhsZl7YOOfPgVXA\n24HvR8Rs4HCgJzOXNc75MPCjiPhYZq4cg2aMmVoNvv71YmmCYf9pSZKktuiUHqeNiojdgB2Bnw3s\ny8zHgGuAgxq75gOPDISmhkspeq8OHKNSx0y9DvffD3fcUXYlkiSNLx0fnChCU1L0MA22qnFs4Jz7\nBx/MzHXAw4POqYyDDip6mhyukyRpbHXEUN0oBcPMhxrpOQsXLmTatGkb7Ovt7aW3t3fLqmuj6dNh\nzpxigvj73ld2NZIkdYa+vj76+vo22Ld69eqWfkc3BKeVFAFoBzbsdZoBLBt0zozBH4qIrYAX8fye\nqg2cdtppzJs3r2XFjpVaDa64ouwqJEnqHMN1fPT399PT09Oy7+j4obrMvJMiGB06sC8iplLMXVrc\n2LUEmB4R+w766KEUgeuaMSp1TNVqsHw5PPRQ2ZVIkjR+dERwiogpETE3IvZp7Nq98X7nxvvTgU9G\nxFsj4rXA2cDvgfMBMvM2YBHwbxGxf0TUgDOAvqrdUTegXi9eFy/e9HmSJKl1OiI4AftRDLstpZiT\n9CWgH/gMQGaeShGEzqLoQZoMHJGZzwy6xnuA2yjuprsI+AXFuk+VtOuuMGuWE8QlSRpLHTHHKTOv\nYDMhLjM/DXx6E8cfBY5paWEdLKIYrjM4SZI0djqlx0mjUK/DddfB00+XXYkkSeODwamL1WpFaFq6\ntOxKJEkaHwxOXWzuXJgyxeE6SZLGisGpi02cCPPnFwthSpKk9jM4dbmBCeK5uTXUJUnSFjM4dbla\nrVgE8/bby65EkqTqMzh1ufnzYcIE5zlJkjQWDE5dbupU2Htvg5MkSWPB4FQB9boTxCVJGgsGpwqo\n1eDXv4b77y+7EkmSqs3gVAG1WvHqA38lSWovg1MF7Lwz7LKLw3WSJLWbwakifOCvJEntZ3CqiFqt\neGbdk0+WXYkkSdVlcKqIeh3WroXrriu7EkmSqsvgVBFz5hRrOjlcJ0lS+xicKmKrrXzgryRJ7WZw\nqpB6vViSYP36siuRJKmaDE4VUqvBo4/C8uVlVyJJUjUZnCrkwAOLITuH6yRJag+DU4VMmQL77usE\ncUmS2sXgVDEuhClJUvsYnCqmXoff/hbuu6/sSiRJqh6DU8UMPPDXXidJklrP4FQxM2fCbrsZnCRJ\nageDUwXV695ZJ0lSOxicKqhWg2XLYM2asiuRJKlaDE4VVKvBunVw7bVlVyJJUrUYnCroNa+B6dMd\nrpMkqdUMThU0YQIsWOAEcUmSWs3gVFEDD/x98MGyK5EkqToMThX1znfC1lvDPvs4ZCdJUqsYnCpq\nzz3hl78s1nR6/evh85+H9evLrkqSpO5mcKqwnXaCn/8cTjwR/v7v4Ygj4P77y65KkqTuZXCquIkT\n4ZRT4JJLirWd9tkHLr+87KokSepOXRGcIuJTEbF+yHbroOPbRMRXI+LBiHg8Is6LiBll1txpDjus\nGLp71avg0EPh5JOLtZ4kSVLzuiI4NdwM7ADs2Njqg46dDrwFOAp4HTAL+M+xLrDTzZoFl14KJ50E\nn/50EaZWriy7KkmSukc3BadnM/OBzLy/sT0MEBFTgeOAhZl5RWYuA/4CqEXEAWUW3Im22qoITZde\nCrfeCnPnFj9LkqTN66bgtGdE3BMRd0TEdyJi58b+HmAi8LOBEzPzduB3wEEl1NkV3vCGYuhu772L\nnqeTToJnny27KkmSOlu3BKergWOBw4Hjgd2AX0TEFIphu2cy87Ehn1nVOKaN2GEHWLQIPvtZ+Nzn\nirlP99xTdlWSJHWuiWUX0IzMXDTo7c0RcS1wN/BnwFMb+VgAublrL1y4kGnTpm2wr7e3l97e3lFW\n210mTCixS5QlAAARAklEQVSWKjj4YOjtLe66O+cceNObyq5MkqSR6evro6+vb4N9q1evbul3ROZm\ns0VHaoSnnwKXNrYXDe51ioi7gNMy85828vl5wNKlS5cyb968Mai48z3wAPz5nxdLF5x4YtETNWlS\n2VVJkjR6/f399PT0APRkZv+WXq9bhuo2EBEvBPYA7gWWAs8Chw46/kpgF2BJKQV2qZe+FH70I/jH\nf4T//b+LFcdXrCi7KkmSOkdXBKeI+GJEvC4ido2IBcAPKcLS9xq9TN8AvhwRr4+IHuCbwFWZeW2J\nZXelCRPg4x+HX/yiCE377AMXXlh2VZIkdYauCE7ATsB3gduA7wEPAPMz86HG8YXARcB5wOUUPVFH\njX2Z1bFgQbHSeK0GRx4Jf/d38MwzZVclSVK5umVy+CZnamfm08CHG5taZPvt4fzz4fTTi16oK6+E\n//gPePnLy65MkqRydEuPk0oSAQsXwlVXFQ8I3ndf+OEPy65KkqRyGJzUlAMOKIbu3vAG+JM/gb/5\nG3j66bKrkiRpbBmc1LTp0+G88+CMM+Css4r5T3fcUXZVkiSNHYOTRiQCPvQhWLwYHn20GLr7/vfL\nrkqSpLFhcNKo9PRAfz8ccQS8613wwQ/CUxtbw12SpIowOGnUpk6F730P/uVf4JvfhPnz4Ve/Krsq\nSZLax+CkLRIBH/gAXHMNPPkkzJsH555bdlWSJLWHwUktMXcuXH89vP3tcMwx8P73wxNPlF2VJEmt\nZXBSy2y3HZxzDnzjG/Dd78KBB8Ly5WVXJUlS6xic1FIRcNxxcN11sG4d7LcffPvbZVclSVJrGJzU\nFnvtVYSnd70Ljj0W3vc+WLOm7KokSdoyBie1zZQp8O//DmefXSycud9+cNNNZVclSdLoGZzUdu99\nLyxdCpMmFY9u+frXIbPsqiRJGjmDk8bEq19dLFnw3vfCX/5lcefd44+XXZUkSSNjcNKYmTwZ/vVf\nizvuLrigWH38l78suypJkppncNKY6+0tHtcyZUqx2vg//7NDd5Kk7mBwUin23BOWLIH/+T/hr/+6\nuPtu9eqyq5IkadMMTirNttvCV78KP/gBLFpUPK7l+uvLrkqSpI0zOKl073wnLFsGL34xLFgAX/mK\nQ3eSpM5kcFJH2H13uPJKOOEE+MhH4Kij4JFHyq5KkqQNGZzUMbbZBk47Df7rv+DnP4d99y2WMJAk\nqVMYnNRx3va2YpmCmTOhXocvfcmhO0lSZzA4qSPtuiv84hfw0Y/Cxz4GRx4JDz1UdlWSpPHO4KSO\nNWkSfPGLcNFFxdIF++wDV11VdlWSpPHM4KSO95a3FEN3u+4KhxwCX/gCrF9fdlWSpPHI4KSusNNO\ncPnl8PGPwyc+UYSpBx4ouypJ0ngzsewCpGZNnAif+1zR6/Te9xZDd3/zN8VSBrvtVmwvfjFElF2p\nJKmqDE7qOocfXgzdfeADcMop8Pjjzx3bbrvnQtRw25Qp5dUtSep+Bid1pVmz4MILi2UKHn4Y7rzz\n+dvFF8Ndd8Ezzzz3uZe+dOOhapddYOutS2uSJKkLGJzU1SJg++2Lbb/9nn98/Xq4777hg9WSJfD7\n3z830XzCBHjZyzYerGbNKs6RJI1fBidV2kAYetnLisU0h1q7Fn73u+eHqttugx//GO6//7lzt966\nuLNvY8Fq++2dXyVJVWdw0rg2aRLssUexDWfNmmK4b2iwuvpq6OuDxx577twXvnDT86te+MIxaZIk\nqY0MTtImTJkCe+1VbENlFg8iHm4Y8JJLisD19NPPnf+Sl2w8VO26q/OrJKkbGJykUYoolj948Yuh\np+f5x9evh5Urhw9W11wDK1Y8N78qYvj5VTvtBJMnFz1jW2/93Dbc+0mTnIMlSe1mcJLaZMKEYkL5\nrFlQqz3/+Nq1RXgaGqp+9StYtAhWrRr5d06c2FzIatX7Lb3WVls5L0xSd6lUcIqIE4CPATsCNwAf\nzszryq2qPH19ffT29pZdRlt1cxsnTSoW79x99+GPP/EEnHVWH4cf3sszzxTLKqxdy3//PNL3mzq2\nevXIr90qEX1stVUvEZSyTZjQ3us/+GAfM2Y8/9/RoYFxuADZzL7Rfq6V17rvvj5mzRr5f4dbEppH\n+9nRfO6ee/rYaaex/f/MWP+FopVtbGXtrbjWo49u+TUGq0xwioh3AV8C/gq4FlgILIqIV2bmg6UW\nV5JuDhXNqnIbX/AC+PnP+1i4sPPalwnr1rUmwJ15Zh/HH99LJqVs69e39/p3393Hy1/e+7x/fsP9\nMx3tvnZccyT7Vq3qY489Rvbv6cba0c7PjvZz997bx847t+a/w2ZraOa8LflnOPQ6K1b0seOOY/P/\nmlbV3azBc01boTLBiSIonZWZZwNExPHAW4DjgFPLLEyqmohiWHDixCLgbYmLLoK//uvW1NWJjjwS\nzj677Cra68gj4T//s+wq2ufII+H888uuor2OPBIuuKDsKtqjv3/4eaijVYmppBExCegBfjawLzMT\nuBQ4qKy6JElStVQiOAEvAbYChk6nXUUx30mSJGmLVWmobjgBbGw0dVuA5cuXj101Y2z16tX09/eX\nXUZbVb2NVW8fVL+NVW8fVL+NVW8fVLuNg37Pb9uK60WO9SytNmgM1T0BHJWZFwza/y1gWma+Y5jP\nvAc4d8yKlCRJZTo6M7+7pRepRI9TZq6NiKXAocAFABERjfdf2cjHFgFHA3cBT41BmZIkaextC7yc\n4vf+FqtEjxNARPwZ8G3gAzy3HME7gVdn5gNl1iZJkqqhEj1OAJn5/Yh4CXAysAPwS+BwQ5MkSWqV\nyvQ4SZIktVtVliOQJElqu3EXnCLi4Ii4ICLuiYj1EXFk2TW1UkR8IiKujYjHImJVRPwwIl5Zdl2t\nFBHHR8QNEbG6sS2OiDeVXVe7NP5M10fEl8uupVUi4lONNg3ebi27rlaKiFkRcU5EPBgRTzT+nZ1X\ndl2tEhF3DvNnuD4izii7tlaIiAkR8dmI+G3jz+83EfHJsutqpYh4YUScHhF3Ndp4ZUTsV3Zdo9XM\n7/eIODki7m2096cR8YqRfs+4C07AFIr5Tyew8TWeutnBwBnAgcAbgUnATyJicqlVtdYK4ESK1eJ7\ngMuA8yNidqlVtUFE7A/8JcVDq6vmZor5iDs2tnq55bROREwHrgKeBg4HZgN/BzxSZl0tth/P/dnt\nCPwRxf9Tv19mUS30/1DcbPTXwKuBjwMfj4gPlVpVa32D4u7zo4E5wE+BSyNiZqlVjd4mf79HxInA\nhyj+XA8A1lA803brkXzJuJ7jFBHrgbcPXvupahoT5u8HXpeZV5ZdT7tExEPAxzLzm2XX0ioR8UJg\nKfBB4CRgWWb+bblVtUZEfAp4W2ZWpgdmsIj4AnBQZh5Sdi1jJSJOB96cmZXo4Y6IC4GVmfmXg/ad\nBzyRmX9eXmWtERHbAo8Db83MSwbtvx64ODP/obTiWmC43+8RcS/wxcw8rfF+KsUTRt6XmU0H/vHY\n4zTeTKdI3g+XXUg7NLrT3w28AFhSdj0t9lXgwsy8rOxC2mTPRpf6HRHxnYjYueyCWuitwPUR8f3G\nkHl/RLy/7KLapbEI8dEUPRhVsRg4NCL2BIiIuUANuLjUqlpnIsWjyp4esv9JKtT7OyAidqPoGR38\nTNvHgGsY4TNtK7McgZ6vsQjo6cCVmVm1+SNzKILSwN+a3pGZt5VbVes0wuA+FMMhVXQ1cCxwOzAT\n+DTwi4iYk5lrSqyrVXan6Cn8EnAKxdD5VyLiqcz8TqmVtcc7gGkUa+lVxReAqcBtEbGOoqPh7zPz\ne+WW1RqZ+YeIWAKcFBG3UfS8vIciRPy61OLaY0eKToQtfqatwanavga8huJvSVVzGzCXokftKODs\niHhdFcJTROxEEXj/KDPXll1PO2Tm4BV8b46Ia4G7gT8DqjDcOgG4NjNPary/ISL2oghTVQxOxwE/\nzsyVZRfSQu+iCBLvBm6l+IvMP0XEvZl5TqmVtc4xwL8D9wDPAv3Ad4FKDqFvxKaeaTssh+oqKiLO\nBN4MvD4z7yu7nlbLzGcz87eZ2Z+Zf08xefojZdfVIj3AS4GlEbE2ItYChwAfiYhnGj2JlZKZq4Ff\nASO+w6VD3QcMfYL4cmCXEmppq4jYheJGlH8ru5YWOxX4fGb+IDNvycxzgdOAT5RcV8tk5p2Z+T8o\nJlXvnJnzga2BO8utrC1WUoSkHYbsn8Hze6E2yeBUQY3Q9Dbgf2Tm78quZ4xMALYpu4gWuRR4LcXf\ncOc2tuspeirmZgXv6GhMhN+DInBUwVXAq4bsexVFr1rVHEfxi6cqc38GvIDn90Ssp4K/NzPzycxc\nFREvorgL9L/KrqnVMvNOivB06MC+xuTwAynmszVt3A3VRcQUir/VDvytfffGpL+HM3NFeZW1RkR8\nDegFjgTWRMRAul6dmZV4mHFEnAL8mGJZgu0oJqUeAhxWZl2t0pjjs8GctIhYAzyUmUN7MbpSRHwR\nuJAiSLwM+AzFUEFfmXW10GnAVRHxCYrb8w8E3k+xtERlNHo/jwW+lZnrSy6n1S4E/j4iVgC3UAxf\nLQS+XmpVLRQRh1H8Lrwd2JOil2058K0Syxq1Jn6/nw58MiJ+A9wFfBb4PXD+SL5n3AUnism2P6f4\nm0RSTN6EYlLjcWUV1ULHU7Tr8iH7/wI4e8yraY8dKNoyE1gN3AgcVuG7z6B6a47tRDGXYnvgAeBK\nYH5mPlRqVS2SmddHxDsoJhifRDH08ZGqTCwe5I3AzlRjXtpQH6L4xfpViuGce4F/buyrimnA5yn+\n8vIwcB7wycxcV2pVo7fJ3++ZeWpEvAA4i2J+7P8BjsjMZ0byJeN6HSdJkqSRqNxYrSRJUrsYnCRJ\nkppkcJIkSWqSwUmSJKlJBidJkqQmGZwkSZKaZHCSJElqksFJkiSpSQYnSZKkJhmcJGkzImJ9RBxZ\ndh2SymdwktTRIuKbjeCyrvE68PPFZdcmafwZjw/5ldR9fgwcy3NPPQd4upxSJI1n9jhJ6gZPZ+YD\nmXn/oG01/Pcw2vERcXFEPBERd0TEUYM/HBFzIuJnjeMPRsRZETFlyDnHRcTNEfFURNwTEV8ZUsNL\nI+L/j4g1EfGriHhrm9ssqQMZnCRVwcnAD4C9gXOB70XEqwAiYjJwCfAQ0AO8E3gjcMbAhyPig8CZ\nwL8Ac4Ajgd8M+Y5/AL4HvBa4GDg3Iqa3r0mSOlFkZtk1SNJGRcQ3gWOApwbtTuBzmfmFiFgPfC0z\nPzToM0uApZn5oYj4S+DzwE6Z+VTj+BHAhcDMzHwgIn4PfCMzP7WRGtYDJ2fmpxvvXwA8DhyRmT9p\ncZMldTDnOEnqBpcBx7PhHKeHB/189ZDzlwBzGz+/GrhhIDQ1XEXR4/6qiACY1fiOTblp4IfMfCIi\nHgdmNNsASdVgcJLUDdZk5p0j/MxAd3oM+nm4c55s8nprh/ms0x2kccb/6CVVwfxh3t/W+PlWYJ/G\nXKcBdWAdcHtm/gG4Czi03UVK6n72OEnqBttExA5D9j2bmQ81fv7TiFgKXEkxH2p/4LjGsXOBTwPf\njojPUAyvfQU4OzMfbJzzaeCfI+IBiqUPpgILMvPMNrVHUpcyOEnqBm8C7h2y73bgNY2fPwW8G/gq\ncB/w7sy8DSAzn4yIw4F/Aq4FngDOA/5u4EKZeXZEbAMsBL4IPNg4579PGaYm76yRxiHvqpPU1Rp3\nvL09My8ouxZJ1eccJ0mSpCYZnCR1O7vNJY0Zh+okSZKaZI+TJElSkwxOkiRJTTI4SZIkNcngJEmS\n1CSDkyRJUpMMTpIkSU0yOEmSJDXJ4CRJktQkg5MkSVKT/i84BTXQso+MawAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc7133301d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "costs, w = train_linreg(X_train, y_train, eta=0.001, epochs=10)\n",
    "   \n",
    "plot(range(1, len(costs)+1), costs)\n",
    "\n",
    "tight_layout()\n",
    "xlabel('Epoch')\n",
    "ylabel('Cost')\n",
    "tight_layout()\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk4AAAGGCAYAAACNCg6xAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3Xt0lPd97/vPT3cEwx0ENiDASFgIG3ThZoy4CxAapdlZ\nbULrnaZZTt1dN23pqZMmbXZzmtO903YnbpvLyQ453U7rhig5zT7RSIAQ9/tNkgGDQMIIIZABcxNC\n6K7f+UMyRp4xjJBmHs3o/VrLC+v5Ppr5ZGKkj3565vcYa60AAADwZBFOBwAAAAgVFCcAAAA/UZwA\nAAD8RHECAADwE8UJAADATxQnAAAAP1GcAAAA/ERxAgAA8BPFCQAAwE8UJwAAAD8NqOJkjFlijCkw\nxlw1xnQaY/J8nPM3xpg6Y8wDY0yJMWaGE1kBAMDgM6CKk6Shkt6R9Lokr5voGWO+KumPJL0mab6k\nRknFxpiYYIYEAACDkxmoN/k1xnRK+g1rbcEjx+ok/YO19s3uj4dLui7pd621v3AmKQAAGCwG2orT\nJzLGTJM0QdLOD49Za+9JOippkVO5AADA4BEyxUldpcmqa4XpUde7ZwAAAAEV5XSAfmDk43qoh0Nj\nxkhaI+mSpOYgZQIAAANDnKSpkoqttbf6+mChVJyuqaskJajnqtN4SeWP+bw1kv49gLkAAMDA9zuS\nftbXBwmZ4mStrTbGXJO0UtIp6eHF4Qsk/eAxn3pJkt5++22lpKQEOuagsnHjRr355ptOxwgrvKaB\nwesaGLyugcHr2r8qKir0yiuvSN19oK8GVHEyxgyVNENdK0uSNN0YM0fSbWttraR/lPRXxpgL6noB\nviXpiqRfP+ZhmyUpJSVF6enpgYo+KI0YMYLXtJ/xmgYGr2tg8LoGBq9rwPTL5ToDqjhJypS0W13X\nLFlJ3+k+/lNJX7TW/r0xJl7S/5Q0UtJ+Seusta1OhAUAAIPLgCpO1tq9esI7/ay135T0zWDkAQAA\neFQobUcAAADgKIoTntqGDRucjhB2eE0Dg9c1MHhdA4PXdWAbsLdc6S/GmHRJpaWlpVxsBwDAIFNW\nVqaMjAxJyrDWlvX18VhxAgAA8BPFCQAAwE8UJwAAAD9RnAAAQK91dHQo3K+T9oXiBAAA/NbQ0KD/\n+l//qxYvXqyNGzeqoaHB6UhBRXECAAB+uX37thYtWqRz585pyZIlqqur04oVKwZVeaI4AQCAJ7p0\n6ZJeffVVzZo1S6mpqRo2bJhSU1M1ffp0/ehHP3I6XtAMqFuuAACAgaW5uVklJSUqKyvTpUuXtHLl\nyh7zKVOm6OjRow6lCz6KEwAA8OncuXMqKirS/fv3JUkJCQmqqalRamrqw3Nqa2u1cOFCpyIGHcUJ\nAAD0cP/+fW3dulVnz57tcTwtLU35+fmSpMTERNXW1qq6ulqbNm1yIqYjKE4AAECSZK3VyZMnVVxc\nrObmZq+5y+XST37yE504cUInTpxQZmamNm3aJJfL5UBaZ1CcAACA7ty5o8LCQl28eNHnfMaMGcrN\nzdWIESO0fPnyIKcbOChOAAAMYp2dnTp69Kh2796ttrY2r/mQIUO0du1avfDCCzLGOJBwYKE4AQAw\nSF2/fl0FBQWqq6vzOZ89e7bWrl2ruro6lZeXP/HxXC6XkpKS+jvmgEJxAgBgkGlvb9e+fft08OBB\ndXZ2es2HDx+u9evXKzk5WVVVVUpOTvb7sSsrK8O6PFGcAAAYRGpra1VQUKCbN2/6nGdmZmrVqlWK\njY2VpEd2BX9bUspjHrlC0ithv4s4xQkAgEGgpaVFO3fu1PHjx33Ox4wZo7y8PE2ZMuUTHiFFUnrA\n8oUKihMAAGGuqqpKhYWFunfvntcsIiJCixcvVlZWlqKiqAVPwisEAECYevDggbZt26bTp0/7nD/z\nzDNyu92aMGFCkJOFLooTAABhxlqrd999V9u2bdODBw+85lFRUVq+fLkWLlyoiIgIBxKGLooTAABh\npL6+XkVFRaqqqvI5nzZtmnJzczV69OggJwsPFCcAAMKAtVbHjx/Xzp071dra6jWPi4tTdna25s6d\ny0aWfUBxAgAgxN28eVMFBQWqra31OU9JSdG6desG1T3lAoXiBABAiOro6NDBgwe1b98+dXR0eM2H\nDRumnJwcpaQ8bv8lf1X0cR4eKE4AAISgq1evqqCgQDdu3PA5T0tLU3Z2tuLi4vr0PB+tUr3Sy/PD\nE8UJAIAQ0traqt27d+vo0aOy1nrNR40aJbfbrWnTpvXL8yUlJamystKvHcG5Vx0AABgwLl68KI/H\no7t373rNjDFauHChli9frujo6H593nAvQ71BcQIAYIBramrS9u3b9c477/icJyQkKC8vT88880yQ\nkw0+FCcAAAaws2fPasuWLWpsbPSaRUZGaunSpXrppZcUGRnpQLrBh+IEAMAA1NDQoC1btujcuXM+\n51OmTJHb7dbYsWODnGxwozgBADCAWGtVXl6u7du3q6WlxWseExOjVatWKTMzk40sHUBxAgBgAGho\naNCbb76p4uJiDR8+XGlpaYqNje1xTnJysnJycjRixAiHUoI7+wEA4LD6+notXrxY7777rl566SW5\nXC7l5+c/XHGKj4/XZz7zGX3uc5+jNDmMFScAABx07do1/fEf/7Gef/55paamStLDP8vLy/X7v//7\nWrNmjeLj452MiW6sOAEA4ID29nbt3LlTP/7xj3XhwgUlJib2mCcmJioyMlKf/vSnKU0DCCtOAAAE\nWU1NjTwej27duiWpax+mmpqahytNklRbW6tly5Y5lBCfhOIEAECQtLS0qKSkRKWlpT2Op6WlKT8/\nX1LXSlNtba2qq6u1adMmJ2LiMShOAAAEwfnz51VUVOTznm9DhgzR9773PZ0+fVplZWXKzMzUpk2b\nwv6GuaGI4gQAQAA1NjZq69atOnPmjM/5s88+q7y8PI0fP145OTlBTofeojgBABAA1lqdOnVKxcXF\nampq8ppHR0drxYoVmj9/viIieK9WqAip4mSMiZD0f0r6HUkTJNVJesta+385GgwAgEfcvXtXhYWF\neu+993zOp0+frtzcXN28efMTb9z7KJfLpaSkpP6OiacQUsVJ0l9Iek3S5yWdlZQp6S1jzF1r7fcd\nTQYAGPQ6Ozt17Ngx7dq1S21tbV7zuLg4rVmzRnPmzNGFCxeUnJzs92NXVlZSngaAUCtOiyT92lq7\nrfvjy8aY35Y038FMAADoxo0b8ng8unLlis95amqq1q5dq2HDhknSIxeJvy0p5TGPXCHpFZ8XlSP4\nQq04HZL0JWNMkrW2yhgzR9JiSRsdzgUAGKTa29t14MAB7d+/X52dnV5zl8ul9evXa+bMmZ/wCCmS\n0gOaEf0n1IrTtyUNl3TOGNOhrp3P/9Ja+3NnYwEABqMrV66ooKBAH3zwgc95RkaGVq1apbi4uCAn\nQ6CEWnH6rKTflvQ5dV3jNFfSPxlj6qy1/+ZoMgDAoNHa2qqdO3fq2LFjPuejR4+W2+3W1KlTgxsM\nARdqxenvJf03a+0vuz8+Y4yZKulrkh5bnDZu3Oh1R+kNGzZow4YNAYgJAAhXFy5cUGFhoerr671m\nxhi99NJLWrp0qaKjox1IN7ht3rxZmzdv7nHM1/9PfRFqxSlekv3YsU75cbPiN998U+np/A4ZAPB0\nHjx4oOLiYp06dcrnfOLEiXK73Zo4cWKQk+FDvhZEysrKlJGR0W/PEWrFySPpL40xtZLOqOtquo2S\nfuJoKgBA2LLW6syZM9q6dasePHjgNY+KitKyZcu0aNEiNrIcBEKtOP2RpG9J+oGk8eraAPP/7j4G\nAEC/unfvnoqKilRZWelzPnXqVOXm5mrMmDFBTganhFRxstY2Svqz7n8AAAgIa61KS0tVUlKi1tZW\nr3lsbKxWr16t9PR0GWP6+GwVfZwjmEKqOAEAEGi3bt2Sx+NRTU2Nz/nzzz+vnJwcuVyuPj3PR5//\nSi/Ph5MoTgAASOro6NChQ4e0d+9edXR0eM2HDh2qnJwcpaSk9MMqk5SUlKTKykq/dgTnXnUDB8UJ\nADDo1dXVyePx6Nq1az7nc+fOVXZ2toYMGdKvz0sZCj0UJwDAoNXW1qY9e/bo8OHDsvbju91II0eO\nlNvt1vTp0x1Ih4GI4gQAGJSqq6vl8Xh0584dr5kxRgsWLNDy5csVExPjQDoMVBQnAEBYqaqqeux1\nQ21tbTp79qwuX77scz5+/Hjl5eXp2WefDVREhDCKEwAgbFRVVSk5OfkT588//7zWr1/v8x1qkZGR\nysrK0uLFixUZGRnImAhhFCcAQNj4aKXpbUkpD48PG9asnJx3NWvW+z4/b/LkyXK73Ro3blzgQyKk\nUZwAAGEoRV135bqnxMRvasSIfbp3b5xaWtIUGxv78KzIyEhlZ2dr3rx5/bLFAMIfN9UBAISlkSMv\na9Kk+Zo//4qys5fL5XIpPz9fLS0tkrp+rbds2TLNnz+f0gS/seIEAAgrxhgtXPieXK5vaMSIF5Wa\nmipJD/88duxd3bjxqk6f/qb+7M+4gxd6hxUnAEDYuHfvnl599VWtWXNWH3zwvhITE3vMExMTdeJE\njE6fnuRQQoQ6ihMAIOS1t7dr165d2rdv38NtBBISErzuN1ddXasHDxY6ERFhgl/VAQBC2uXLl+Xx\neHTz5s0ex9PS0pSfny9rjaZOnaLq6lpt316ttrZNkqqcCYuQR3ECAISklpYW7dixQydOnPA5v3fv\nWbW0ePTrXxcpNvaEWloyu0uT9x5OgL8oTgCAkFNZWamioiLdu3fPa9bR0aEDB8Zq//75am9vlrRS\nbW0ru6cfrjRVBCsqwgzFCQAQMhobG1VcXKzTp0/7nI8ePVp/+7d/q+vXr/v1eL52EAceh+IEABjw\nrLU6ffq0tm3bpqamJq95VFSUVqxYoQULFmjt2rWPvVfdh1wul5KSkgIRF2GM4gQAGNDq6+tVWFio\nCxcu+JxPmzZNbrdbo0aNkiTKEAKK4gQAGJCstTp+/Lh27typ1tZWr3lcXJyys7M1d+5cdv5G0FCc\nAAADzgcffCCPx6Pa2lqf81mzZmndunUaNmxYkJNhsKM4AQAGjK53xB3Q/v371dHR4TUfNmyYcnJy\nlJKS4kA6gOIEABggrl69qoKCAt24ccPnPD09XatXr1ZcXFyQkwEfoTgBABzV2tqq3bt36+jRo7LW\nes1Hjx6t3NxcTZs2zYF0QE8UJwCAYy5evCiPx6O7d+96zYwxWrRokZYtW6bo6GgH0gHeKE4AgKBr\namrS9u3b9c477/icT5gwQXl5eZo4cWKQkwGPR3ECADxRVVVVv2wqaa3V2bNntXXrVjU2NnrNIyMj\ntXTpUr300kuKjIzsU2YgEChOAIDHqqqqUnJyst/nV1ZW+ixPDQ0NKioq0vnz531+3pQpU+R2uzV2\n7NinzgoEGsUJAPBYH600vS3pcdsAVEh6xWtlylqrsrIylZSUqKWlxeuzYmJitHr1amVkZLCRJQY8\nihMAwE8pktJ79Rm3b9+Wx+PRpUuXfM6Tk5O1fv16DR8+vO/xgCCgOAEA+l1nZ6cOHz6sPXv2qL29\n3WseHx+vdevWKTU1lVUmhBSKEwCgX9XX1+snP/mJ3n//fZ/zOXPmKDs7W/Hx8UFOBvQdxQkA0C+i\nojq0dOlK7d+/3+dGliNGjFBubq5mzJjhQDqgf1CcAAB9lph4SXl5ezVmzBKfpWnBggVasWKFYmJi\nHEgH9B+KEwA4pL/2RnJSbGyzVq/eoczMUp/zcePGKS8vT5MmTQpyMiAwKE4A4ID+2hvJSTNnntf6\n9UUaPty7/EVERGjJkiVasmQJG1kirFCcAMABfd0byRkVkqT4+NuaOfPHki7o7NkEpaWlKTY29uFZ\no0aN0uc+9zmNHz/eoZxA4FCcAMBRvd8bKdhcLlf3v72i2bNn6/79+5oxY54SE1eqpqZG+fn5+uxn\nPytjjHbu3Kmf/vSnlCaELYoTAOCxkpKSVFZWpsOHD8vj8cjlcik1NVWSHv5ZWVmpP/3TP9UXvvCF\nAfcrRaA/UZwAAJ+os7NTx44d065du9TW1qbr169r9uzZPc5JTEzU5cuX9fLLLzuUEggeihMAwKcb\nN26ooKBAV69efXgsISFBNTU1D1eaJKm2tlbz5s1zIiIQdBQnAEAP7e3t2r9/vw4cOKDOzs4es7S0\nNP3yl79URESEJk+erNraWlVXV2vTpk0OpQWCK8LpAACAgaO2tlY//vGPtW/fPq/SJEmLFy/W4cOH\nlZGRocuXLysjI0M7dux45AJyILyF3IqTMeYZSX8naZ2keElVkn7PWlvmaDAACGGtra3auXOnjh07\n5nM+ZswYud1uJSYmSpLeeOONYMYDBoyQKk7GmJGSDkraKWmNpJuSkiTdcTIXADy9ij7O++7ChQsq\nLCxUfX2918wYo8WLF2vp0qWKigqpbxlAQITa34K/kHTZWvvqI8dqnAoDAE/r0b2Rend+/3nw4IGK\ni4t16tQpn/OJEycqLy9PEyZM6PfnBkJVqBUnt6RtxphfSFoq6aqkH1prf+JsLADonaSkJFVWVjpy\nrzprrd59911t27ZNDx488JpHRUVp2bJlWrRokSIiuBQWeFSoFafpkv6LpO9I+ltJCyT9szGm2Vr7\ntqPJAKCXnNgosr6+Xlu2bFFlZaXP+dSpU+V2uzV69OggJwNCQ6gVpwhJx6y13+j++KQxJlVdZeqx\nxWnjxo0aMWJEj2MbNmzQhg0bAhIUAAYSa61OnDihHTt2qLW11WseGxur7OxspaWlyRjjQEKg7zZv\n3qzNmzf3OObr2r2+MNbafn3AQDLGXJK03Vr7+48c+wNJf2mtnfwJn5MuqbS0tFTp6QP7flAAEAg3\nb96Ux+PR5cuXfc6ff/555eTksKUAwlJZWZkyMjIkKaM/3oEfaitOByXN/NixmeICcQDw0tHRoUOH\nDmnv3r3q6Ojwmg8dOlQ5OTmaNWuWA+mA0BRqxelNSQeNMV+T9At1XeP0qqQvOZoKAAaYuro6FRQU\n6Pr16z7nc+fOVXZ2toYMGRLkZEBoC6niZK09YYz5tKRvS/qGpGpJf2Kt/bmzyQBgYGhra9Pu3bt1\n5MgR+boUY9SoUcrNzdX06dMdSAeEvpAqTpJkrd0iaYvTOQBgoKmurpbH49GdO957AhtjtHDhQi1f\nvlzR0dEOpAPCQ8gVJwBAT01NTSopKVF5ebnPeUJCgtxut5599tkgJwPCD8UJAEJYRUWFtmzZovv3\n73vNIiMjlZWVpcWLFysyMtKBdED4oTgBQAhqaGjQ1q1bVVHh+152kydPVl5ensaOHRvkZEB4ozgB\nQAix1qq8vFwlJSVqbm72msfExGjVqlXKzMxkI0sgAChOABAibt++rcLCQlVXV/ucJyUlaf369V53\nSQDQfyhOADDAdXZ26siRI9q9e7fa29u95vHx8Vq7dq1mz57NKhMQYBQnABjArl27Jo/Ho7q6Op/z\nF154QWvXrlV8fHyQkwGDE8UJAAag9vZ27d27V4cOHVJnZ6fXfPjw4crNzVVSUpID6YDBi+IEAANM\nTU2NPB6Pbt265XM+f/58rVixQrGxsUFOBoDiBAADQENDg37wgx+ouLhYcXFxSktL8ypGY8eOVV5e\nniZPnuxQSgARTgcAgMGuoaFBWVlZKisrU2Zmplwul/Lz89XS0iJJioiIUFZWll577TVKE+AwVpwA\nwEGNjY368pe/rKSkJKWmpkrSwz/Ly8v1mc98Rm63WwkJCU7GBNCNFScAcIC1VidPntQPfvADnTp1\nSomJiT3miYmJ6ujo0Be/+EVKEzCAsOIEAEF29+5dFRUV6cKFC5K6bsJbU1PzcKVJkmpra5Wdna2I\nCH6+BQYSihMABElnZ6eOHz+unTt3qq2t7eHxtLQ05efnS+paaaqtrVV1dbU2bdrkVFQAn4DiBCCs\nVFVVqaGh4YnnuVyuoO6B9MEHH6igoEBXrlzxmsXGxuqv//qvVVtbq1OnTikzM1ObNm2Sy+UKWj4A\n/qE4AQgbVVVVSk5O9vv8ysrKgJenjo4OHThwQPv371dHR4fX3OVyKScnR88//3xAcwDoHxQnAGHj\no5WmtyWlPObMCkmv+LUy1RdXrlyRx+PRjRs3fM7T09O1evVqxcXFBTQHgP5DcQIQhlIkpTv27K2t\nrdq1a5eOHj3qcz569Gi53W5NnTo1uMEA9BnFCQD60XvvvafCwkLdvXvXa2aM0aJFi7Rs2TJFR0c7\nkA5AX1GcAKAfNDU1qbi4WCdPnvQ5nzBhgvLy8jRx4sQgJwPQnyhOANAH1lqdPXtWW7duVWNjo9c8\nKipKS5cu1aJFixQZGelAQgD9ieIEAE/p3r172rJli86fP+9znpiYKLfbrTFjxgQ5GYBAoTgBQC9Z\na1VaWqodO3Y8vBHvo2JjY7V69Wqlp6fLGONAQgCBQnECgF64deuWPB6PampqfM5nzpypnJwcDR8+\nPMjJAAQDxQlAGKro49xbR0eHDh8+rD179vjcyHLo0KFat26dZs2axSoTEMYoTgDCxke3KHmll+c/\n3vvvv6+CggJdu3bN53zOnDnKzs5WfHy8X48HIHRRnACEjaSkJFVWVvbbvera2tq0d+9eHTp0SNZa\nr/nIkSOVm5ur55577qkzAwgtFCcAYaW/7j136dIleTwe3b592+d8wYIFWrFihWJiYvrl+QCEBooT\nADyiublZJSUlKisr8zkfN26c8vLyNGnSpCAnAzAQUJwAoNu5c+dUVFSk+/fve80iIiKUlZWll19+\nmY0sgUGM4gRg0Lt//762bt2qs2fP+pxPmjRJbrdb48ePD3IyAAMNxQnAoGWt1cmTJ1VcXKzm5mav\neXR0tFauXKl58+YpIiLCgYQABhqKE4BB6c6dOyosLNTFixd9zp977jnl5uZq5MiRQU4GYCCjOAEY\nVDo7O3X06FHt3r1bbW1tXvMhQ4ZozZo1evHFF9nIEoAXihOAQeP69evyeDy6evWqz/ns2bO1du1a\nDR06NMjJAIQKihOAsNfe3q79+/frwIED6uzs9Jq7XC7l5uYqOTnZgXQAQgnFCUDYamho0N/93d+p\npKREo0ePVlpammJjY3uck5mZqVWrVnkdBwBfeJsIgLB08+ZNLVy4UJWVlcrKypLL5VJ+fr5aWlok\nSWPGjNEXvvAFrV+/ntIEwG+sOAEIO1VVVXrjjTeUmpqq1NRUSXr4Z3l5uf78z/9cS5cuVVQUXwIB\n9A4rTgDCxoMHD/SrX/1KP/vZz3T58mUlJib2mCcmJio2NlYrV66kNAF4KnzlABDyrLV69913tW3b\nNj148ECSlJCQoJqamocrTZJUW1url19+2amYAMJASK84GWO+ZozpNMZ81+ksAJxRX1+vzZs361e/\n+tXD0iRJaWlpKi0t1ZkzZ3T//n1VVFSourpar732moNpAYS6kF1xMsbMk/QlSSedzgIg+Ky1On78\nuHbu3KnW1lav+fDhw/XWW2/pyJEjOnHihDIzM7Vp0ya5XC4H0gIIFyFZnIwxwyS9LelVSd9wOA6A\nILt586YKCgpUW1vrc56SkqJ169bJ5XJpyZIlQU4HIJyFZHGS9ANJHmvtLmMMxQkYJDo6OnTw4EHt\n27dPHR0dXvNhw4Zp3bp1mjVrlgPpAAwGvS5Oxpi3JP2LtXZf/8fx6/k/J2mupEwnnh+AM+rq6lRQ\nUKDr16/7nM+dO1fZ2dkaMmRIkJMBGEyeZsVplKQSY8xlSf9L0k+ttb5v/NTPjDGTJP2jpNXWWu+7\ncwIIO21tbdq9e7eOHDkia63XfNSoUcrNzdX06dMdSAdgsDG+vhA98ZOMGSfpP0v6XUmzJO2Q9P9I\n+nUgC40x5lOSfiWpQ9KHty2PlGS7j8Xaj/0PMsakSyrNysrSiBEjejzehg0btGHDhkDFBdBHFy9e\nVGFhoe7cueM1M8Zo4cKFWr58uaKjox1IB2Cg2bx5szZv3tzjWH19vfbt2ydJGdbasr4+x1MVpx4P\n0FVMfk9dF2rfV9dF2z+01lb1NZyP5xoqKfFjh9+SVCHp29baik/IV1paWqr09PT+jgQgAJqamrR9\n+3a98847PucJCQlyu9169tlng5wMQKgpKytTRkaG1E/FqU8XhxtjJkpaLSlbXSs+WyS9IOmsMeYr\n1to3+xrwUdbaRklnP5ahUdItX6UJQOg5e/astmzZosbGRq9ZZGSksrKytHjxYkVGRjqQDsBg9zQX\nh0dLylPXKlO2pFOS3pT079bahu5zPi3pX7qPB1rflswADAgNDQ3asmWLzp0753M+efJk5eXlaezY\nsUFOBgAfeZoVp/fVteP4ZknzrbW+1tJ3S7rbl2D+stauCMbzAAgMa63Ky8u1fft2tbS0eM1jYmK0\natUqZWZmyhjj4xEAIHiepjhtlPRLa23zJ51grb0radpTpwIwKNy+fVsej0eXLl3yOU9KStL69eu9\n3tgBAE7pdXGy1v5bIIIAGDw6Ozt1+PBh7dmzR+3t7V7z+Ph4rV27VrNnz2aVCcCAEqo7hwMIUdeu\nXVNBQYHef/99n/MXX3xRa9asUXx8fJCTAcCTUZwABEV7e7v27t2rgwcP+tzIcsSIEVq/fr2SkpIc\nSAcA/qE4AQi4mpoaeTwe3bp1y+d8/vz5WrFihWJjY4OcDAB6h+IEIGBaWlpUUlKi0tJSn/OxY8cq\nLy9PkydPDnIyAHg6FCcAAXH+/HkVFRWpoaHBaxYREaGXX35ZS5YsUVQUX4YAhA6+YgHoV42Njdq6\ndavOnDnjc/7ss8/K7XYrISEhyMkAoO8oTgD6hbVWp06dUnFxsZqamrzm0dHRWr58uRYsWKCIiAgH\nEgJA31GcAPTZ3bt3VVhYqPfee8/nfPr06crNzdWoUaOCnAwA+hfFCcBT6+zs1LFjx7Rr1y61tbV5\nzePi4rRmzRrNmTOHjSwBhAWKE4CncuPGDXk8Hl25csXnfNasWVq3bp2GDRsW5GQAEDgUJwC9cvfu\nXX3961/XkSNHlJCQoLS0tB77L7lcLuXk5Oj55593MCUABAbFCXBIVVWVz7fqf5zL5Rowu2mfO3dO\neXl5mjs5+3d9AAAYf0lEQVR3rlauXKmamhrl5+frs5/9rGJjY5Wenq7Vq1crLi7O6agAEBAUJ8AB\nVVVVSk5O9vv8yspKR8tTa2urdu3apX/6p3/S3LlzlZqaKkkP/zx37py++93vaurUqY5lBIBgoDgB\nDvhopeltSSmPObNC0it+rUwFyoULF1RYWKj6+npdv35ds2fP7jFPTEzUpUuXKE0ABgWKE+CoFEnp\nTofw6cGDByouLtapU6ceHktISFBNTc3DlSZJqq2t1YIFC5yICABBR3EC0IO1VmfOnNHWrVv14MGD\nHrO0tDTl5+crIiJCkydPVm1traqrq7Vp0yaH0gJAcLF9L4CH7t27p5///Of6j//4D6/SJEnJycna\nuXOnMjIydPnyZWVkZGjHjh1yuVwOpAWA4GPFCYCstSotLVVJSYlaW1u95rGxsVq9erXS09NljNEb\nb7zhQEoAcB7FCRjkbt26JY/Ho5qaGp/zmTNnKicnR8OHDw9yMgAYeChOwCDV0dGhQ4cOae/evero\n6PCaDx06VOvWrdOsWbO4XQoAdKM4AYNQXV2dPB6Prl275nM+d+5cZWdna8iQIUFOBgADG8UJcFRF\nH+e909bWpj179ujw4cOy1nrNR44cqdzcXD333HP9+rwAEC4oToADPnoX2iu9PP/pVVdXy+Px6M6d\nOz7nCxcu1PLlyxUTE9Pn5wKAcEVxAhyQlJSkysrKoNyrrrm5Wdu3b1d5ebnP+fjx4+V2uzVp0qSn\nfg4AGCwoToBDgnHvuYqKCm3ZskX379/3mkVERCgrK0svv/yyIiMjA54FAMIBxQkIQ/fv39eWLVtU\nUeH7GqlJkyYpLy9P48aNC3IyAAhtFCcgjFhr9c4772j79u1qbm72mkdHR2vlypWaN2+eIiK4cQAA\n9BbFCQgTd+7ckcfjUXV1tc/5jBkztH79eo0cOTLIyQAgfFCcgBDX2dmpo0ePateuXWpvb/eaDxky\nRGvXrtULL7zARpYA0EcUJyCEXb9+XQUFBaqrq/M5nz17ttauXauhQ4cGORkAhCeKExCC2tvbtW/f\nPh08eFCdnZ1e8+HDh2v9+vVKTk52IB0AhC+KExBiLl++LI/Ho5s3b/qcZ2ZmatWqVYqNjQ1yMgAI\nfxQnIES0tLRo586dOn78uM/5mDFjlJeXpylTpgQ5GQAMHhQnIARUVVWpsLBQ9+7d85pFRERo8eLF\nysrKUlQUf6UBIJD4KgsMYI2NjSouLtbp06d9zp955hm53W5NmDAhyMkAYHCiOAEDkLVWp0+f1rZt\n29TU1OQ1j4qK0vLly7Vw4UI2sgSAIKI4AQNMfX29CgsLdeHCBZ/zadOmKTc3V6NHjw5yMgAAxQkY\nABoaGvSjH/1IJSUliomJ0Ysvvuj1rri4uDhlZ2dr7ty5bGQJAA5hjR9wWENDg5YvX67jx49rzpw5\nio+PV35+vlpaWh6ek5KSoj/8wz9UWloapQkAHMSKE+Cgjo4OffWrX9X06dOVmpoqSQ//LC8v16pV\nq5STk6OUlBQnYwIAurHiBDjk6tWr+vGPf6wjR44oMTGxxywxMVFNTU16/fXXKU0AMICE3IqTMeZr\nkj4t6XlJTZIOSfqqtbbS0WCAn1pbW7V7924dPXpU1lolJCSopqbm4UqTJNXW1io3N1dxcXEOJv1I\nVVWVGhoanniey+VSUlJSEBIBgDNCrjhJWiLpe5JOqCv/f5e03RiTYq31ft82MIBcvHhRHo9Hd+/e\nfXgsLS1N+fn5krpWmmpra1VdXa1NmzY5FbOHqqqqXt3zrrKykvIEIGyFXHGy1uY8+rEx5guSbkjK\nkHTAiUzAkzQ1NWn79u165513vGaxsbH68pe/rNu3b+v8+fPKzMzUpk2b5HK5HEjq7aOVprclPe7X\nhhWSXvFrZQoAQlXIFScfRkqykm47HQT4OGutzp49q61bt6qxsdFrHhkZqaVLl+qll15SZGSkAwl7\nI0VSutMhAMBRIV2cTNf7sv9R0gFr7Vmn8wCPamhoUFFRkc6fP+9zPmXKFLndbo0dOzbIyQAATyuk\ni5OkH0qaJWmx00GAD1lrVVZWppKSkh57MX0oJiZGq1atUmZmJnsyAUCICdniZIz5vqQcSUuste8/\n6fyNGzdqxIgRPY5t2LBBGzZsCFBCDEa3b9+Wx+PRpUuXfM6Tk5OVk5Pj9d8iAKDvNm/erM2bN/c4\nVl9f36/PEZLFqbs0fUrSUmvtZX8+580331R6OtdnIDA6Ozt1+PBh7dmzR+3t7V7z+Ph4rVu3Tqmp\nqawyAUCA+FoQKSsrU0ZGRr89R8gVJ2PMDyVtkJQnqdEYk9A9qrfWNjuXDIPVtWvXVFBQoPff973w\n+eKLL2rNmjWKj48PcjIAQH8LueIk6Q/U9S66PR87/nuS/jXoaTBotbW1ae/evTp06JCstV7zESNG\nKDc3VzNmzHAgHQAgEEKuOFlruU0MHFdTU6OCggLdvu17F4z58+dr5cqViomJCXKyQKro4xwAQl/I\nFSfASc3NzdqxY4dKS0t9zseNGye3263JkycHOVngfLQR5yu9PB8Awg/FCfDT+fPnVVRU5HNn7IiI\nCC1ZskQvv/yyoqLC669VUlKSKisruVcdAIjiBDzR/fv3tW3bNp05c8bn/Nlnn1VeXp7Gjx8f5GTB\nQxkCgC4UJ+ATWGt18uRJFRcXq7nZ+w2b0dHRWrFihebPn6+ICC69A4DBgOIE+HD37l0VFhbqvffe\n8zmfPn26cnNzNWrUqCAnAwA4ieIEPKKzs1PHjh3Trl271NbW5jUfMmSI1qxZoxdffJGNLAFgEKI4\nAd1u3LihgoICXb161ec8NTVVa9eu1bBhw4KcDAAwUFCcMOi1t7dr//79OnDggDo7O73mLpdL69ev\n18yZMx1IBwAYSChOCCtVVVW9ett8bW2tPB6PPvjgA5/nZWRkaNWqVYqLi+vvqACAEERxQtioqqpS\ncnKyX+fGxMTorbfeUmVlpc/56NGj5Xa7NXXq1H5MCAAIdRQnhI2PVprelpTyiec999wJud0VPkuT\nMUYvvfSSli5dqujo6MAEBQCELIoTwlCKpHSvo/HxD7RmTbHmzHlf0kiv+cSJE+V2uzVx4sTARwQA\nhCSKEwYBq9mz39W6dds0dOgDr2lUVJSWLVumRYsWsZElAOCxKE4IYw1yuf5REydu16RJwxQVlSYp\ntscZU6dOVW5ursaMGeNMRABASOHHa4Spexo//mWtXXtaeXkL5XK5lJ+fr5aWFklSc3OzXnzxRX3+\n85+nNAEA/MaKE8LOmDH3lZT0miZPnqnU1FRJevhneXm5Roz4DW3Z8oZ+8zd/k92/AQC9QnGCX3q7\nP5ITOjs7tWTJEi1duldFReeVmLiyxzwxMVFvv31W165lSnry/xYAAD6O4oQn6s3+SJJUWVkZ9PJU\nV1en/fv3a+XKlZI6lZCQoJqamocrTZJUXV2rW7eWS2KVCQDwdChOeCJ/90eSKiS94tfKVH9pa2vT\n7t27deTIEVlrHx5PS0tTfn6+pK6VpurqWm3fXqG2tte7cwIA0HsUJ/SC7/2RnFJdXS2Px6M7d+54\nzaKjozV16lR5PB5FRUWppaVFbW1tkrIenuNyuYKYFgAQDihOCDnNzc3avn27ysvLfc5dLpfmzJmj\nT33qU5/4GE5eiwUACF0UJ4SUiooKbdmyRffv3/eaRUZGKisrS4sXL1ZkZKQD6QAA4Y7ihJDQ0NCg\nrVu3qqLC9/VJkydPltvt1rhx44KcDAAwmFCcMKBZa1VeXq6SkhI1Nzd7zWNiYrRy5UrNmzePPZkA\nAAFHccKAdfv2bRUWFqq6utrnfMaMGcrNzdWIESOCnAwAMFhRnDDgdHZ26siRI9q9e7fa29u95kOG\nDNHatWv1wgsvsMoEAAgqihN64Un7H/V9f6Rr167J4/Gorq7O5/yFF17QmjVrNHTo0D4/FwAAvUVx\nwhN9tN/RK70833/t7e3au3evDh06pM7OTq/58OHDlZubyxYCAABHUZzwRElJSaqsrAzYveouX76s\ngoIC3bp1y+d83rx5WrlypWJjY3v1uAAA9DeKE/wSiJWelpYW7dixQydOnPA5Hzt2rNxut6ZMmdLv\nzw0AwNOgOMERlZWVKioq0r1797xmERERWrx4sbKyshQVxX+iAICBg+9KCKrGxkZt27ZN7777rs/5\nM888o7y8PCUkJAQ5GQAAT0ZxQlBYa3X69Glt27ZNTU1NXvOoqCitWLFCCxYsUEREhAMJAQB4MooT\nAu7u3bsqKirShQsXfM6nTZsmt9utUaNGBTkZAAC9Q3FCwHR2dur48ePauXOn2travOZxcXHKzs7W\n3Llz2cgSABASKE4IiA8++EAFBQW6cuWKz/msWbO0bt06DRs2LMjJAAB4ehQn9KuOjg4dOHBA+/fv\nV0dHh9d82LBhysnJUUpKigPpAADoG4oT+s2VK1fk8Xh048YNn/P09HStXr1acXFxQU4GAED/oDih\nz1pbW7Vr1y4dPXrU53z06NHKzc3VtGnTgpwMAID+RXFCn7z33nsqLCzU3bt3vWbGGC1atEjLli1T\ndHS0A+kAAOhfFCf0WkNDg77//e9r27Ztio+PV1pamtd95CZMmKC8vDxNnDjRoZQAAPQ/dhpEr9y7\nd09LlixReXm55s+fL5fLpfz8fLW0tEiSIiMjtWLFCr366quUJgBA2GHFCX5ramrS66+/ruTkZKWm\npkrSwz/Ly8v1W7/1W3K73Ro7dqyTMQEACJiQXHEyxrxujKk2xjQZY44YY+Y5nWkwiImJUVVVlRIT\nE3scT0xMlLVWX/jCFyhNAICwFnLFyRjzWUnfkfTXktIknZRUbIzhO3aARUZGavXq1aqpqelxvLa2\nVqtWrWL3bwBA2Au54iRpo6T/aa39V2vtOUl/IOmBpC86G2tw+MpXvqKzZ8/qzJkzun//vioqKlRd\nXa3XXnvN6WgAAARcSBUnY0y0pAxJOz88Zq21knZIWuRUrsHE5XJp7969mjt3ri5fvqyMjAzt2LFD\nLpfL6WgAAARcqF0cPlZSpKTrHzt+XdLM4McZnMaMGaOvf/3rTscAACDoQmrF6TGMJOt0CAAAEN5C\nbcXppqQOSQkfOz5e3qtQPWzcuFEjRozocWzDhg3asGFDvwYEAADO2Lx5szZv3tzjWH19fb8+h+m6\nRCh0GGOOSDpqrf2T7o+NpMuS/tla+w8+zk+XVFpaWqr09PTghgUAAI4qKytTRkaGJGVYa8v6+nih\ntuIkSd+V9FNjTKmkY+p6l128pLecDAUAAMJfyBUna+0vuvds+ht1/cruHUlrrLUfOJsMAACEu5Ar\nTpJkrf2hpB86nQMAAAwu4fKuOgAAgICjOAEAAPiJ4gQAAOAnihMAAICfKE4AAAB+ojgBAAD4ieIE\nAADgJ4oTAACAnyhOAAAAfqI4AQAA+IniBAAA4CeKEwAAgJ8oTgAAAH6iOAEAAPiJ4gQAAOAnihMA\nAICfKE4AAAB+ojgBAAD4ieIEAADgJ4oTAACAnyhOAAAAfqI4AQAA+IniBAAA4CeKEwAAgJ8oTgAA\nAH6iOAEAAPiJ4gQAAOAnihMAAICfKE4AAAB+ojgBAAD4ieIEAADgJ4oTAACAnyhOAAAAfqI4AQAA\n+IniBAAA4CeKEwAAgJ8oTgAAAH6iOAEAAPiJ4gQAAOAnihMAAICfKE4AAAB+ojgBAAD4ieIEAADg\np5ApTsaYRGPMT4wxF40xD4wxVcaYbxpjop3ONlht3rzZ6Qhhh9c0MHhdA4PXNTB4XQe2kClOkp6X\nZCR9SdIsSRsl/YGkv3Uy1GDGX+7+x2saGLyugcHrGhi8rgNblNMB/GWtLZZU/MihS8aY/6Gu8vQV\nZ1IBAIDBJJRWnHwZKem20yEAAMDgELLFyRgzQ9IfSfqR01kAAMDg4Piv6owx/13SVx9zipWUYq2t\nfORznpW0VVK+tfZfnvAUcZJUUVHR16j4mPr6epWVlTkdI6zwmgYGr2tg8LoGBq9r/3rk+39cfzye\nsdb2x+M8fQBjxkga84TTLlpr27vPf0bSbkmHrLW/58fj/7akf+9zUAAAEMp+x1r7s74+iOPFqTe6\nV5p2STou6T9bP8J3F7M1ki5Jag5oQAAAMNDESZoqqdhae6uvDxYyxckYM1HSPnUVoN+V1PHhzFp7\n3aFYAABgEHH8GqdeyJY0vfuf2u5jRl3XQEU6FQoAAAweIbPiBAAA4LSQ3Y4AAAAg2AZNceJed/3H\nGPO6MabaGNNkjDlijJnndKZQZoz5mjHmmDHmnjHmujHmfxtjkp3OFU66X+NOY8x3nc4S6owxzxhj\n/s0Yc7P7a+lJY0y607lCmTEmwhjzrUe+P10wxvyV07lCjTFmiTGmwBhztfvve56Pc/7GGFPX/TqX\ndO8J2SuDpjiJe931C2PMZyV9R9JfS0qTdFJSsTFmrKPBQtsSSd+TtEDSKknRkrYbY4Y4mipMdBf7\nL6nrv1X0gTFmpKSDklrU9W7lFEn/h6Q7TuYKA38h6TVJf6iu71VfkfQVY8wfOZoq9AyV9I6k19V1\n/XMPxpivqmvj7NckzZfUqK7vXzG9eZJBfY2TMebPJf2BtbbXjXOwMsYckXTUWvsn3R8bdV2s/8/W\n2r93NFyY6C6hNyRlWWsPOJ0nlBljhkkqlfRfJH1DUrm19s+cTRW6jDHflrTIWrvU6SzhxBjjkXTN\nWvulR479v5IeWGs/71yy0GWM6ZT0G9bagkeO1Un6B2vtm90fD5d0XdLvWmt/4e9jD6YVJ1+4110v\ndP9aM0PSzg+Pde+ltUPSIqdyhaGR6vppif82++4HkjzW2l1OBwkTbkknjDG/6P61cpkx5lWnQ4WB\nQ5JWGmOSJMkYM0fSYklbHE0VRowx0yRNUM/vX/ckHVUvv3+F0nYE/eqRe93x06f/xqpr64eP75t1\nXdLM4McJP90reP8o6YC19qzTeUKZMeZzkuZKynQ6SxiZrq7Vu++o6zKHBZL+2RjTbK1929Fkoe3b\nkoZLOmeM6VDXosZfWmt/7myssDJBXT+Q+vr+NaE3DxTyxSkI97rDk324nxb67ofqugZvsdNBQpkx\nZpK6Cuhqa22b03nCSISkY9bab3R/fNIYk6quMkVxenqflfTbkj4n6ay6Cv8/GWPqrLX/5miy8Nfr\n718hX5wk/Q9J/+sJ51z88F+673W3S10/0b8WyGBh6Ka6dmxP+Njx8fJu8eglY8z3JeVIWmKtfd/p\nPCEuQ9I4SaXdq3hS12ppVvcFt7H+3LIJXt6X9PE7pldI+k8OZAknfy/pv1lrf9n98RljzFRJX5NE\nceof19RVkhLU8/vVeEnlvXmgkC9O3fed8eveMx+7190XA5krHFlr24wxpZJWSiqQHv5qaaWkf3Yy\nW6jrLk2fkrTUWnvZ6TxhYIekFz527C11fZP/NqXpqR2U96/lZ0qqcSBLOImX96pHp7gOud9Ya6uN\nMdfU9f3qlPTw4vAF6roW0m8hX5z81X2vuz3qutfdVySN//AHUe511yvflfTT7gJ1TF3bOsSr65sS\nnoIx5oeSNkjKk9RojPlwRa/eWsuNqZ+CtbZRXb/yeMgY0yjplrX24ysm8N+bkg4aY74m6Rfq+qbz\nqrq2e8DT80j6S2NMraQzktLV9bX1J46mCjHGmKGSZqhrZUmSpndfaH/bWlurrl/f/5Ux5oK6usC3\nJF2R9OtePc9g+cHLGPO7kj5+PZNR1xvDuNddLxhj/lBd5TNBXXtmfNlae8LZVKGr+22zvv4i/p61\n9l+DnSdcGWN2SXqH7Qj6xhiTo66LmWdIqpb0Ha4V7Zvub/jfkvRpdf3qqE7SzyR9y1rb7mS2UGKM\nWSppt7y/nv7UWvvF7nO+Ken31fXu5f2SXrfWXujV8wyW4gQAANBX/P4UAADATxQnAAAAP1GcAAAA\n/ERxAgAA8BPFCQAAwE8UJwAAAD9RnAAAAPxEcQIAAPATxQkAAMBPFCcAAAA/UZwAAAD8RHECAADw\nE8UJQNgxxow1xrxvjPmLR44tMsa0GGOWO5kNQGgz1lqnMwBAvzPGrJP0/0laJOm8pJOS/re19g1H\ngwEIaRQnAGHLGPM9SaslnZA0W9I8a22bs6kAhDKKE4CwZYyJk/SupEmS0q21Zx2OBCDEcY0TgHD2\nnKRn1PW1bprDWQCEAVacAIQlY0y0pGOSytV1jdOfSZptrf3A0WAAQhrFCUBYMsb8g6T/JOlFSQ8k\n7ZF0z1rrdjIXgNDGr+oAhB1jzFJJfyzpFWtto+36CfHzkl42xrzmbDoAoYwVJwAAAD+x4gQAAOAn\nihMAAICfKE4AAAB+ojgBAAD4ieIEAADgJ4oTAACAnyhOAAAAfqI4AQAA+IniBAAA4CeKEwAAgJ8o\nTgAAAH6iOAEAAPjp/wfIh31aRy4oLAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc7127d6590>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# prediction\n",
    "\n",
    "def predict_linreg(X, w):\n",
    "    Xt = T.matrix(name='X')\n",
    "    net_input = T.dot(Xt, w[1:]) + w[0]\n",
    "    predict = theano.function(inputs=[Xt], givens={w: w}, \n",
    "                              outputs=net_input)\n",
    "    return predict(X)\n",
    "\n",
    "scatter(X_train, y_train, marker='s', s=50)\n",
    "plot(range(X_train.shape[0]), \n",
    "         predict_linreg(X_train, w), \n",
    "         color='gray', \n",
    "         marker='o', \n",
    "         markersize=4, \n",
    "         linewidth=3)\n",
    "\n",
    "xlabel('x')\n",
    "ylabel('y')\n",
    "\n",
    "tight_layout()\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import struct\n",
    "import numpy as np\n",
    " \n",
    "def load_mnist(path, kind='train'):\n",
    "    \"\"\"Load MNIST data from `path`\"\"\"\n",
    "    labels_path = os.path.join(path, \n",
    "                               '%s-labels-idx1-ubyte' % kind)\n",
    "    images_path = os.path.join(path, \n",
    "                               '%s-images-idx3-ubyte' % kind)\n",
    "        \n",
    "    with open(labels_path, 'rb') as lbpath:\n",
    "        magic, n = struct.unpack('>II', \n",
    "                                 lbpath.read(8))\n",
    "        labels = np.fromfile(lbpath, \n",
    "                             dtype=np.uint8)\n",
    "\n",
    "    with open(images_path, 'rb') as imgpath:\n",
    "        magic, num, rows, cols = struct.unpack(\">IIII\", \n",
    "                                               imgpath.read(16))\n",
    "        images = np.fromfile(imgpath, \n",
    "                        dtype=np.uint8).reshape(len(labels), 784)\n",
    " \n",
    "    return images, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 784)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, y_train = load_mnist('mnist', kind='train')\n",
    "X_train.shape[0], X_train.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 784)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test, y_test = load_mnist('mnist', kind='t10k')\n",
    "X_test.shape[0], X_test.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((60000, 784), (10000, 784))"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# image in 64-bit format\n",
    "\n",
    "import theano \n",
    "\n",
    "theano.config.floatX = 'float64'\n",
    "X_train = X_train.astype(theano.config.floatX)\n",
    "X_test = X_test.astype(theano.config.floatX)\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 0, 4, 1, 9, 2, 1, 3, 1, 4], dtype=uint8)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# one-hot encoding class labels\n",
    "\n",
    "from keras.utils import np_utils\n",
    "\n",
    "y_train[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.],\n",
       "       [ 1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  1.],\n",
       "       [ 0.,  0.,  1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.,  0.]])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_OneHotEncoding = np_utils.to_categorical(y_train) \n",
    "y_train_OneHotEncoding[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 54000 samples, validate on 6000 samples\n",
      "Epoch 1/50\n",
      "54000/54000 [==============================] - 13s - loss: 2.2460 - val_loss: 2.1508\n",
      "Epoch 2/50\n",
      "54000/54000 [==============================] - 12s - loss: 1.9574 - val_loss: 1.6992\n",
      "Epoch 3/50\n",
      "54000/54000 [==============================] - 12s - loss: 1.4466 - val_loss: 1.1753\n",
      "Epoch 4/50\n",
      "54000/54000 [==============================] - 12s - loss: 1.0397 - val_loss: 0.8597\n",
      "Epoch 5/50\n",
      "54000/54000 [==============================] - 12s - loss: 0.8057 - val_loss: 0.6754\n",
      "Epoch 6/50\n",
      "54000/54000 [==============================] - 12s - loss: 0.6599 - val_loss: 0.5551\n",
      "Epoch 7/50\n",
      "54000/54000 [==============================] - 12s - loss: 0.5638 - val_loss: 0.4743\n",
      "Epoch 8/50\n",
      "54000/54000 [==============================] - 13s - loss: 0.4981 - val_loss: 0.4194\n",
      "Epoch 9/50\n",
      "54000/54000 [==============================] - 13s - loss: 0.4505 - val_loss: 0.3785\n",
      "Epoch 10/50\n",
      "54000/54000 [==============================] - 13s - loss: 0.4180 - val_loss: 0.3641\n",
      "Epoch 11/50\n",
      "54000/54000 [==============================] - 13s - loss: 0.3967 - val_loss: 0.3374\n",
      "Epoch 12/50\n",
      "54000/54000 [==============================] - 13s - loss: 0.3726 - val_loss: 0.3183\n",
      "Epoch 13/50\n",
      "54000/54000 [==============================] - 13s - loss: 0.3525 - val_loss: 0.2942\n",
      "Epoch 14/50\n",
      "54000/54000 [==============================] - 13s - loss: 0.3392 - val_loss: 0.2930\n",
      "Epoch 15/50\n",
      "54000/54000 [==============================] - 13s - loss: 0.3313 - val_loss: 0.2798\n",
      "Epoch 16/50\n",
      "54000/54000 [==============================] - 13s - loss: 0.3163 - val_loss: 0.2684\n",
      "Epoch 17/50\n",
      "54000/54000 [==============================] - 13s - loss: 0.3102 - val_loss: 0.2545\n",
      "Epoch 18/50\n",
      "54000/54000 [==============================] - 13s - loss: 0.3019 - val_loss: 0.2593\n",
      "Epoch 19/50\n",
      "54000/54000 [==============================] - 13s - loss: 0.2900 - val_loss: 0.2447\n",
      "Epoch 20/50\n",
      "54000/54000 [==============================] - 13s - loss: 0.2846 - val_loss: 0.2530\n",
      "Epoch 21/50\n",
      "54000/54000 [==============================] - 13s - loss: 0.2849 - val_loss: 0.2439\n",
      "Epoch 22/50\n",
      "54000/54000 [==============================] - 13s - loss: 0.2727 - val_loss: 0.2373\n",
      "Epoch 23/50\n",
      "54000/54000 [==============================] - 13s - loss: 0.2666 - val_loss: 0.2351\n",
      "Epoch 24/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2587 - val_loss: 0.2285\n",
      "Epoch 25/50\n",
      "54000/54000 [==============================] - 16s - loss: 0.2588 - val_loss: 0.2221\n",
      "Epoch 26/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2534 - val_loss: 0.2222\n",
      "Epoch 27/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2564 - val_loss: 0.2147\n",
      "Epoch 28/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2477 - val_loss: 0.2194\n",
      "Epoch 29/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2481 - val_loss: 0.2096\n",
      "Epoch 30/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2393 - val_loss: 0.2152\n",
      "Epoch 31/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2406 - val_loss: 0.2104\n",
      "Epoch 32/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2348 - val_loss: 0.2105\n",
      "Epoch 33/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2322 - val_loss: 0.2033\n",
      "Epoch 34/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2276 - val_loss: 0.2071\n",
      "Epoch 35/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2280 - val_loss: 0.2079\n",
      "Epoch 36/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2230 - val_loss: 0.1955\n",
      "Epoch 37/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2225 - val_loss: 0.2010\n",
      "Epoch 38/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2258 - val_loss: 0.2067\n",
      "Epoch 39/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2146 - val_loss: 0.1904\n",
      "Epoch 40/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2085 - val_loss: 0.1845\n",
      "Epoch 41/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2073 - val_loss: 0.1798\n",
      "Epoch 42/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2080 - val_loss: 0.1781\n",
      "Epoch 43/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2016 - val_loss: 0.1773\n",
      "Epoch 44/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.1990 - val_loss: 0.1807\n",
      "Epoch 45/50\n",
      "54000/54000 [==============================] - 15s - loss: 0.1986 - val_loss: 0.1736\n",
      "Epoch 46/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2014 - val_loss: 0.1842\n",
      "Epoch 47/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.2094 - val_loss: 0.1806\n",
      "Epoch 48/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.1942 - val_loss: 0.1732\n",
      "Epoch 49/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.1929 - val_loss: 0.1745\n",
      "Epoch 50/50\n",
      "54000/54000 [==============================] - 14s - loss: 0.1884 - val_loss: 0.1689\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fc6e8865b90>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense\n",
    "from keras.optimizers import SGD\n",
    "\n",
    "np.random.seed(1) \n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(input_dim=X_train.shape[1], \n",
    "                output_dim=50, \n",
    "                init='uniform', \n",
    "                activation='tanh'))\n",
    "\n",
    "model.add(Dense(input_dim=50, \n",
    "                output_dim=50, \n",
    "                init='uniform', \n",
    "                activation='tanh'))\n",
    "\n",
    "model.add(Dense(input_dim=50, \n",
    "                output_dim=y_train_OneHotEncoding.shape[1], \n",
    "                init='uniform', \n",
    "                activation='softmax'))\n",
    "\n",
    "sgd = SGD(lr=0.001, decay=1e-7, momentum=.9)\n",
    "model.compile(loss='categorical_crossentropy', optimizer=sgd)\n",
    "\n",
    "model.fit(X_train, y_train_OneHotEncoding, nb_epoch=50, \n",
    "          batch_size=300, verbose=1, \n",
    "          validation_split=0.1, show_accuracy=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 0, 4, 1, 9, 2, 1, 3, 1, 4, 3, 5, 3, 6, 1, 7, 2, 8, 6, 9, 4, 0, 9,\n",
       "       1, 1, 2, 4, 3, 2, 7, 3, 8, 6, 9, 0, 5, 6, 0, 7, 6, 1, 8, 7, 9, 3, 9,\n",
       "       8, 5, 3, 3, 3, 0, 7, 4, 9, 8, 0, 9, 4, 1, 4, 4, 6, 0, 4, 5, 6, 1, 0,\n",
       "       0, 1, 7, 1, 6, 3, 0, 2, 1, 1, 7, 8, 0, 2, 6, 7, 8, 3, 9, 0, 4, 6, 7,\n",
       "       4, 6, 8, 0, 7, 8, 3, 1])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_pred = model.predict_classes(X_train, verbose=0)\n",
    "y_train_pred[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 0, 4, 1, 9, 2, 1, 3, 1, 4, 3, 5, 3, 6, 1, 7, 2, 8, 6, 9, 4, 0, 9,\n",
       "       1, 1, 2, 4, 3, 2, 7, 3, 8, 6, 9, 0, 5, 6, 0, 7, 6, 1, 8, 7, 9, 3, 9,\n",
       "       8, 5, 9, 3, 3, 0, 7, 4, 9, 8, 0, 9, 4, 1, 4, 4, 6, 0, 4, 5, 6, 1, 0,\n",
       "       0, 1, 7, 1, 6, 3, 0, 2, 1, 1, 7, 9, 0, 2, 6, 7, 8, 3, 9, 0, 4, 6, 7,\n",
       "       4, 6, 8, 0, 7, 8, 3, 1], dtype=uint8)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((60000,), (60000,))"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.shape, y_train_pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "94.808333333333323"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_acc = np.sum(y_train == y_train_pred, axis=0, dtype='float') / X_train.shape[0]\n",
    "train_acc*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}