Neural Nets .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "class Neural_Network(object):\n",
    "    def __init__(self, Lambda):        \n",
    "        #Define Hyperparameters\n",
    "        self.inputLayerSize = 2\n",
    "        self.outputLayerSize = 1\n",
    "        self.hiddenLayerSize = 3\n",
    "\n",
    "        \n",
    "        #Weights (parameters)\n",
    "        self.W1 = np.random.randn(self.inputLayerSize,self.hiddenLayerSize)\n",
    "        self.W2 = np.random.randn(self.hiddenLayerSize,self.outputLayerSize)\n",
    "        \n",
    "        self.Lambda = Lambda\n",
    "        \n",
    "    def forward(self, X):\n",
    "        #Propogate inputs though network\n",
    "        self.z2 = np.dot(X, self.W1)\n",
    "\n",
    "        self.a2 = self.sigmoid(self.z2)\n",
    "\n",
    "        self.z3 = np.dot(self.a2, self.W2)\n",
    "        yHat = self.sigmoid(self.z3) \n",
    "        return yHat\n",
    "        \n",
    "    def sigmoid(self, z):\n",
    "        #Apply sigmoid activation function to scalar, vector, or matrix\n",
    "        return 1/(1+np.exp(-z))\n",
    "    \n",
    "    def sigmoidPrime(self,z):\n",
    "        #Gradient of sigmoid\n",
    "        return np.exp(-z)/((1+np.exp(-z))**2)\n",
    "    \n",
    "    def costFunction(self, X, y):\n",
    "        #Compute cost for given X,y, use weights already stored in class.\n",
    "        self.yHat = self.forward(X)\n",
    "        J = 0.5*sum((y-self.yHat)**2)/X.shape[0] + (self.Lambda/2)*(sum(self.W1**2)+sum(self.W2**2))\n",
    "        return J\n",
    "        \n",
    "    def costFunctionPrime(self, X, y):\n",
    "        #Compute derivative with respect to W and W2 for a given X and y:\n",
    "        self.yHat = self.forward(X)\n",
    "        \n",
    "        delta3 = np.multiply(-(y-self.yHat), self.sigmoidPrime(self.z3))\n",
    "        \n",
    "        dJdW2 = np.dot(self.a2.T, delta3)/X.shape[0] + (self.Lambda)*self.W2\n",
    "        \n",
    "        delta2 = np.dot(delta3, self.W2.T)*self.sigmoidPrime(self.z2)\n",
    "        \n",
    "        dJdW1 = np.dot(X.T, delta2)/X.shape[0]  + (self.Lambda)*self.W1\n",
    "        \n",
    "        return dJdW1, dJdW2\n",
    "    \n",
    "    #Helper Functions for interacting with other classes:\n",
    "    def getParams(self):\n",
    "        #Get W1 and W2 unrolled into vector:\n",
    "        params = np.concatenate((self.W1.ravel(), self.W2.ravel()))\n",
    "        return params\n",
    "    \n",
    "    def setParams(self, params):\n",
    "        #Set W1 and W2 using single paramater vector.\n",
    "        W1_start = 0\n",
    "        W1_end = self.hiddenLayerSize * self.inputLayerSize\n",
    "        self.W1 = np.reshape(params[W1_start:W1_end], (self.inputLayerSize , self.hiddenLayerSize))\n",
    "        W2_end = W1_end + self.hiddenLayerSize*self.outputLayerSize\n",
    "        self.W2 = np.reshape(params[W1_end:W2_end], (self.hiddenLayerSize, self.outputLayerSize))\n",
    "        \n",
    "    def computeGradients(self, X, y):\n",
    "        dJdW1, dJdW2 = self.costFunctionPrime(X, y)\n",
    "        return np.concatenate((dJdW1.ravel(), dJdW2.ravel()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from scipy import optimize\n",
    "\n",
    "class trainer(object):\n",
    "    def __init__(self, N):\n",
    "        #Make Local reference to network:\n",
    "        self.N = N\n",
    "        \n",
    "    def callbackF(self, params):\n",
    "        self.N.setParams(params)\n",
    "        self.J.append(self.N.costFunction(self.X, self.y))\n",
    "        self.testJ.append(self.N.costFunction(self.testX, self.testY))\n",
    "        \n",
    "    def costFunctionWrapper(self, params, X, y):\n",
    "        print \"test\", self.N.costFunction(X, y)\n",
    "        print \"gradInit\", self.N.computeGradients(X,y)\n",
    "        self.N.setParams(params)\n",
    "        print \"test2\", self.N.costFunction(X, y)\n",
    "        print \"gradUpdate\", self.N.computeGradients(X,y)\n",
    "        cost = self.N.costFunction(X, y)\n",
    "        grad = self.N.computeGradients(X,y)\n",
    "        \n",
    "        return cost, grad\n",
    "        \n",
    "    def train(self, trainX, trainY, testX, testY):\n",
    "        #Make an internal variable for the callback function:\n",
    "        self.X = trainX\n",
    "        self.y = trainY\n",
    "        \n",
    "        self.testX = testX\n",
    "        self.testY = testY\n",
    "\n",
    "        #Make empty list to store training costs:\n",
    "        self.J = []\n",
    "        self.testJ = []\n",
    "        \n",
    "        params0 = self.N.getParams()\n",
    "        print params0\n",
    "        \n",
    "        \n",
    "        options = {'maxiter': 200, 'disp' : False}\n",
    "        _res = optimize.minimize(self.costFunctionWrapper, params0, jac=True, method='BFGS', \\\n",
    "                                 args=(trainX, trainY), options=options, callback=self.callbackF)\n",
    "        \n",
    "        print \"res x\", _res.x\n",
    "        \n",
    "        \n",
    "        self.N.setParams(_res.x)\n",
    "        self.optimizationResults = _res\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0001\n",
      "[ 0.33347653 -1.12975893 -0.86482241  0.34270454  2.87498379  0.13370021\n",
      "  0.79822677  0.3000269  -0.39329544]\n",
      "test 0.0165152717972\n",
      "gradInit [-0.00493429 -0.00195269  0.00220597 -0.00305975 -0.00058311  0.00149841\n",
      " -0.02272582 -0.02228662 -0.0144645 ]\n",
      "test2 0.0165152717972\n",
      "gradUpdate [-0.00493429 -0.00195269  0.00220597 -0.00305975 -0.00058311  0.00149841\n",
      " -0.02272582 -0.02228662 -0.0144645 ]\n",
      "test 0.0165152717972\n",
      "gradInit [-0.00493429 -0.00195269  0.00220597 -0.00305975 -0.00058311  0.00149841\n",
      " -0.02272582 -0.02228662 -0.0144645 ]\n",
      "test2 0.0165152717972\n",
      "gradUpdate [-0.00493429 -0.00195269  0.00220597 -0.00305975 -0.00058311  0.00149841\n",
      " -0.02272582 -0.02228662 -0.0144645 ]\n",
      "test 0.0165152717972\n",
      "gradInit [-0.00493429 -0.00195269  0.00220597 -0.00305975 -0.00058311  0.00149841\n",
      " -0.02272582 -0.02228662 -0.0144645 ]\n",
      "test2 0.0152843540232\n",
      "gradUpdate [-0.00481831 -0.00199757  0.00200931 -0.00293    -0.00059657  0.00134403\n",
      " -0.02148511 -0.02092195 -0.01359578]\n",
      "test 0.0152843540232\n",
      "gradInit [-0.00481831 -0.00199757  0.00200931 -0.00293    -0.00059657  0.00134403\n",
      " -0.02148511 -0.02092195 -0.01359578]\n",
      "test2 0.0110717539744\n",
      "gradUpdate [-0.00425523 -0.00206445  0.00132255 -0.00234566 -0.0005906   0.0008215\n",
      " -0.01669051 -0.01571083 -0.01029034]\n",
      "test 0.0110717539744\n",
      "gradInit [-0.00425523 -0.00206445  0.00132255 -0.00234566 -0.0005906   0.0008215\n",
      " -0.01669051 -0.01571083 -0.01029034]\n",
      "test2 0.00443151073528\n",
      "gradUpdate [ -2.00956175e-03  -1.42506757e-03   1.43074803e-04  -3.76123945e-04\n",
      "  -1.59212340e-04   5.09270617e-05  -4.81835454e-03  -3.34804739e-03\n",
      "  -2.49729322e-03]\n",
      "test 0.00443151073528\n",
      "gradInit [ -2.00956175e-03  -1.42506757e-03   1.43074803e-04  -3.76123945e-04\n",
      "  -1.59212340e-04   5.09270617e-05  -4.81835454e-03  -3.34804739e-03\n",
      "  -2.49729322e-03]\n",
      "test2 0.00381462630472\n",
      "gradUpdate [ -1.19474835e-03  -1.02627749e-03  -1.56126997e-05   2.39606226e-04\n",
      "   6.34823378e-05  -1.00604334e-05  -1.79357394e-03  -3.73744296e-04\n",
      "  -6.25386413e-04]\n",
      "test 0.00381462630472\n",
      "gradInit [ -1.19474835e-03  -1.02627749e-03  -1.56126997e-05   2.39606226e-04\n",
      "   6.34823378e-05  -1.00604334e-05  -1.79357394e-03  -3.73744296e-04\n",
      "  -6.25386413e-04]\n",
      "test2 0.00375068344562\n",
      "gradUpdate [ -8.84945955e-04  -8.57968739e-04  -5.19610638e-05   4.60455114e-04\n",
      "   1.53434959e-04  -1.52951062e-05  -7.62001684e-04   6.15520444e-04\n",
      "  -1.82541677e-06]\n",
      "test 0.00375068344562\n",
      "gradInit [ -8.84945955e-04  -8.57968739e-04  -5.19610638e-05   4.60455114e-04\n",
      "   1.53434959e-04  -1.52951062e-05  -7.62001684e-04   6.15520444e-04\n",
      "  -1.82541677e-06]\n",
      "test2 0.00374619733187\n",
      "gradUpdate [ -8.25978815e-04  -8.24065190e-04  -5.74096846e-05   5.01321693e-04\n",
      "   1.70855762e-04  -1.53453721e-05  -5.74294516e-04   7.92998045e-04\n",
      "   1.10702895e-04]\n",
      "test 0.00374619733187\n",
      "gradInit [ -8.25978815e-04  -8.24065190e-04  -5.74096846e-05   5.01321693e-04\n",
      "   1.70855762e-04  -1.53453721e-05  -5.74294516e-04   7.92998045e-04\n",
      "   1.10702895e-04]\n",
      "test2 0.0037401260234\n",
      "gradUpdate [ -7.72885370e-04  -7.91717543e-04  -6.17804621e-05   5.37308103e-04\n",
      "   1.86598165e-04  -1.52293055e-05  -4.10454055e-04   9.45806950e-04\n",
      "   2.08675163e-04]\n",
      "test 0.0037401260234\n",
      "gradInit [ -7.72885370e-04  -7.91717543e-04  -6.17804621e-05   5.37308103e-04\n",
      "   1.86598165e-04  -1.52293055e-05  -4.10454055e-04   9.45806950e-04\n",
      "   2.08675163e-04]\n",
      "test2 0.00372918093846\n",
      "gradUpdate [ -7.16947796e-04  -7.55301442e-04  -6.57917971e-05   5.74175753e-04\n",
      "   2.03114301e-04  -1.50332340e-05  -2.44735129e-04   1.09751958e-03\n",
      "   3.07496215e-04]\n",
      "test 0.00372918093846\n",
      "gradInit [ -7.16947796e-04  -7.55301442e-04  -6.57917971e-05   5.74175753e-04\n",
      "   2.03114301e-04  -1.50332340e-05  -2.44735129e-04   1.09751958e-03\n",
      "   3.07496215e-04]\n",
      "test2 0.00370891901692\n",
      "gradUpdate [ -6.48962327e-04  -7.08297945e-04  -6.99142938e-05   6.17593505e-04\n",
      "   2.22835604e-04  -1.47879690e-05  -5.31162721e-05   1.26906443e-03\n",
      "   4.21317637e-04]\n",
      "test 0.00370891901692\n",
      "gradInit [ -6.48962327e-04  -7.08297945e-04  -6.99142938e-05   6.17593505e-04\n",
      "   2.22835604e-04  -1.47879690e-05  -5.31162721e-05   1.26906443e-03\n",
      "   4.21317637e-04]\n",
      "test2 0.00367127266435\n",
      "gradUpdate [ -5.56335195e-04  -6.41449776e-04  -7.44668983e-05   6.74736143e-04\n",
      "   2.48618854e-04  -1.45392556e-05   1.92606835e-04   1.48346399e-03\n",
      "   5.66337920e-04]\n",
      "test 0.00367127266435\n",
      "gradInit [ -5.56335195e-04  -6.41449776e-04  -7.44668983e-05   6.74736143e-04\n",
      "   2.48618854e-04  -1.45392556e-05   1.92606835e-04   1.48346399e-03\n",
      "   5.66337920e-04]\n",
      "test2 0.0036020604847\n",
      "gradUpdate [ -4.20574460e-04  -5.42309157e-04  -7.95716493e-05   7.55227228e-04\n",
      "   2.83292415e-04  -1.44671112e-05   5.26425597e-04   1.76627959e-03\n",
      "   7.61030397e-04]\n",
      "test 0.0036020604847\n",
      "gradInit [ -4.20574460e-04  -5.42309157e-04  -7.95716493e-05   7.55227228e-04\n",
      "   2.83292415e-04  -1.44671112e-05   5.26425597e-04   1.76627959e-03\n",
      "   7.61030397e-04]\n",
      "test2 0.00347787747512\n",
      "gradUpdate [ -2.13621111e-04  -3.97602122e-04  -8.52101917e-05   8.71534507e-04\n",
      "   3.27411772e-04  -1.52038918e-05   9.86849026e-04   2.14259138e-03\n",
      "   1.02330822e-03]\n",
      "test 0.00347787747512\n",
      "gradInit [ -2.13621111e-04  -3.97602122e-04  -8.52101917e-05   8.71534507e-04\n",
      "   3.27411772e-04  -1.52038918e-05   9.86849026e-04   2.14259138e-03\n",
      "   1.02330822e-03]\n",
      "test2 0.00326399901647\n",
      "gradUpdate [  1.00665459e-04  -2.07918094e-04  -9.18314915e-05   1.03051921e-03\n",
      "   3.70898012e-04  -1.86246446e-05   1.59248688e-03   2.60938936e-03\n",
      "   1.35038206e-03]\n",
      "test 0.00326399901647\n",
      "gradInit [  1.00665459e-04  -2.07918094e-04  -9.18314915e-05   1.03051921e-03\n",
      "   3.70898012e-04  -1.86246446e-05   1.59248688e-03   2.60938936e-03\n",
      "   1.35038206e-03]\n",
      "test2 0.00291929224773\n",
      "gradUpdate [  5.28708027e-04  -3.01995045e-05  -1.02474644e-04   1.16890494e-03\n",
      "   3.73451286e-04  -2.80618001e-05   2.22437101e-03   2.99553129e-03\n",
      "   1.63325476e-03]\n",
      "test 0.00291929224773\n",
      "gradInit [  5.28708027e-04  -3.01995045e-05  -1.02474644e-04   1.16890494e-03\n",
      "   3.73451286e-04  -2.80618001e-05   2.22437101e-03   2.99553129e-03\n",
      "   1.63325476e-03]\n",
      "test2 0.00251639796015\n",
      "gradUpdate [  8.10334770e-04  -1.36932695e-06  -1.17180327e-04   7.78480898e-04\n",
      "   2.81971601e-04  -2.71776003e-05   2.14834067e-03   2.32955643e-03\n",
      "   1.33044747e-03]\n",
      "test 0.00251639796015\n",
      "gradInit [  8.10334770e-04  -1.36932695e-06  -1.17180327e-04   7.78480898e-04\n",
      "   2.81971601e-04  -2.71776003e-05   2.14834067e-03   2.32955643e-03\n",
      "   1.33044747e-03]\n",
      "test2 0.00241579702235\n",
      "gradUpdate [  6.13967284e-04  -1.45477555e-05  -1.09202786e-04   2.92756402e-04\n",
      "   2.60097634e-04  -6.30620470e-06   1.42820939e-03   1.29336933e-03\n",
      "   7.99587276e-04]\n",
      "test 0.00241579702235\n",
      "gradInit [  6.13967284e-04  -1.45477555e-05  -1.09202786e-04   2.92756402e-04\n",
      "   2.60097634e-04  -6.30620470e-06   1.42820939e-03   1.29336933e-03\n",
      "   7.99587276e-04]\n",
      "test2 0.00233381826431\n",
      "gradUpdate [  2.74467856e-04  -9.11449610e-06  -8.72532122e-05   1.92765680e-05\n",
      "   2.66280077e-04   1.09753547e-05   5.39675023e-04   3.51886760e-04\n",
      "   3.06291489e-04]\n",
      "test 0.00233381826431\n",
      "gradInit [  2.74467856e-04  -9.11449610e-06  -8.72532122e-05   1.92765680e-05\n",
      "   2.66280077e-04   1.09753547e-05   5.39675023e-04   3.51886760e-04\n",
      "   3.06291489e-04]\n",
      "test2 0.00230863731515\n",
      "gradUpdate [  6.24168581e-06  -2.00113590e-05  -6.73476489e-05  -1.40761649e-05\n",
      "   2.70548031e-04   1.46313723e-05  -6.20377993e-05  -1.23157459e-04\n",
      "   3.81164292e-05]\n",
      "test 0.00230863731515\n",
      "gradInit [  6.24168581e-06  -2.00113590e-05  -6.73476489e-05  -1.40761649e-05\n",
      "   2.70548031e-04   1.46313723e-05  -6.20377993e-05  -1.23157459e-04\n",
      "   3.81164292e-05]\n",
      "test2 0.00230840177442\n",
      "gradUpdate [  2.96518427e-05  -1.91986619e-05  -6.92830959e-05   6.58975947e-08\n",
      "   2.70515831e-04   1.34888689e-05  -2.04782204e-06  -6.40686210e-05\n",
      "   7.02918338e-05]\n",
      "test 0.00230840177442\n",
      "gradInit [  2.96518427e-05  -1.91986619e-05  -6.92830959e-05   6.58975947e-08\n",
      "   2.70515831e-04   1.34888689e-05  -2.04782204e-06  -6.40686210e-05\n",
      "   7.02918338e-05]\n",
      "test2 0.00230827729525\n",
      "gradUpdate [  3.55634827e-05  -1.90512489e-05  -6.97542779e-05   4.22155477e-06\n",
      "   2.70473090e-04   1.31598068e-05   1.35052075e-05  -4.81918204e-05\n",
      "   7.88345879e-05]\n",
      "test 0.00230827729525\n",
      "gradInit [  3.55634827e-05  -1.90512489e-05  -6.97542779e-05   4.22155477e-06\n",
      "   2.70473090e-04   1.31598068e-05   1.35052075e-05  -4.81918204e-05\n",
      "   7.88345879e-05]\n",
      "test2 0.00230803605327\n",
      "gradUpdate [  4.15341313e-05  -1.89061356e-05  -7.02051779e-05   8.43496208e-06\n",
      "   2.70387122e-04   1.28263612e-05   2.92318332e-05  -3.21279035e-05\n",
      "   8.74439764e-05]\n",
      "test 0.00230803605327\n",
      "gradInit [  4.15341313e-05  -1.89061356e-05  -7.02051779e-05   8.43496208e-06\n",
      "   2.70387122e-04   1.28263612e-05   2.92318332e-05  -3.21279035e-05\n",
      "   8.74439764e-05]\n",
      "test2 0.00230727480304\n",
      "gradUpdate [  6.53729344e-05  -1.83344397e-05  -7.20015189e-05   2.52753744e-05\n",
      "   2.70039000e-04   1.14969954e-05   9.19467452e-05   3.19326468e-05\n",
      "   1.21774382e-04]\n",
      "test 0.00230727480304\n",
      "gradInit [  6.53729344e-05  -1.83344397e-05  -7.20015189e-05   2.52753744e-05\n",
      "   2.70039000e-04   1.14969954e-05   9.19467452e-05   3.19326468e-05\n",
      "   1.21774382e-04]\n",
      "test2 0.00230554494704\n",
      "gradUpdate [  9.72226658e-05  -1.75910561e-05  -7.42078597e-05   4.78787477e-05\n",
      "   2.69188741e-04   9.69235616e-06   1.75699069e-04   1.17469841e-04\n",
      "   1.67368391e-04]\n",
      "test 0.00230554494704\n",
      "gradInit [  9.72226658e-05  -1.75910561e-05  -7.42078597e-05   4.78787477e-05\n",
      "   2.69188741e-04   9.69235616e-06   1.75699069e-04   1.17469841e-04\n",
      "   1.67368391e-04]\n",
      "test2 0.00230241775888\n",
      "gradUpdate [  1.32207396e-04  -1.67670905e-05  -7.64034389e-05   7.28986068e-05\n",
      "   2.67671116e-04   7.60975359e-06   2.67650177e-04   2.11329054e-04\n",
      "   2.17119167e-04]\n",
      "test 0.00230241775888\n",
      "gradInit [  1.32207396e-04  -1.67670905e-05  -7.64034389e-05   7.28986068e-05\n",
      "   2.67671116e-04   7.60975359e-06   2.67650177e-04   2.11329054e-04\n",
      "   2.17119167e-04]\n",
      "test2 0.00229666238664\n",
      "gradUpdate [  1.74332611e-04  -1.56934973e-05  -7.88259281e-05   1.03394580e-04\n",
      "   2.64965111e-04   4.84235407e-06   3.78282058e-04   3.24118561e-04\n",
      "   2.76663554e-04]\n",
      "test 0.00229666238664\n",
      "gradInit [  1.74332611e-04  -1.56934973e-05  -7.88259281e-05   1.03394580e-04\n",
      "   2.64965111e-04   4.84235407e-06   3.78282058e-04   3.24118561e-04\n",
      "   2.76663554e-04]\n",
      "test2 0.0022860565841\n",
      "gradUpdate [  2.28715835e-04  -1.40201151e-05  -8.18761692e-05   1.43519778e-04\n",
      "   2.60090827e-04   6.14507363e-07   5.20816467e-04   4.69036833e-04\n",
      "   3.53210757e-04]\n",
      "test 0.0022860565841\n",
      "gradInit [  2.28715835e-04  -1.40201151e-05  -8.18761692e-05   1.43519778e-04\n",
      "   2.60090827e-04   6.14507363e-07   5.20816467e-04   4.69036833e-04\n",
      "   3.53210757e-04]\n",
      "test2 0.0022667012094\n",
      "gradUpdate [  3.01143001e-04  -1.09051718e-05  -8.64531361e-05   1.98617177e-04\n",
      "   2.51308910e-04  -6.72330224e-06   7.09731534e-04   6.59922973e-04\n",
      "   4.55166912e-04]\n",
      "test 0.0022667012094\n",
      "gradInit [  3.01143001e-04  -1.09051718e-05  -8.64531361e-05   1.98617177e-04\n",
      "   2.51308910e-04  -6.72330224e-06   7.09731534e-04   6.59922973e-04\n",
      "   4.55166912e-04]\n",
      "test2 0.0022319781427\n",
      "gradUpdate [  3.95555156e-04  -4.18916426e-06  -9.47585675e-05   2.74270843e-04\n",
      "   2.35754059e-04  -2.08899149e-05   9.53457956e-04   9.02382864e-04\n",
      "   5.89349238e-04]\n",
      "test 0.0022319781427\n",
      "gradInit [  3.95555156e-04  -4.18916426e-06  -9.47585675e-05   2.74270843e-04\n",
      "   2.35754059e-04  -2.08899149e-05   9.53457956e-04   9.02382864e-04\n",
      "   5.89349238e-04]\n",
      "test2 0.0021705973433\n",
      "gradUpdate [  5.02026700e-04   1.12837913e-05  -1.10708900e-04   3.68011158e-04\n",
      "   2.09373745e-04  -4.94147121e-05   1.22224215e-03   1.15595625e-03\n",
      "   7.44933942e-04]\n",
      "test 0.0021705973433\n",
      "gradInit [  5.02026700e-04   1.12837913e-05  -1.10708900e-04   3.68011158e-04\n",
      "   2.09373745e-04  -4.94147121e-05   1.22224215e-03   1.15595625e-03\n",
      "   7.44933942e-04]\n",
      "test2 0.00206088844943\n",
      "gradUpdate [  5.39664400e-04   4.07063162e-05  -1.24755999e-04   4.09612968e-04\n",
      "   1.64750557e-04  -9.26376620e-05   1.30575880e-03   1.17315248e-03\n",
      "   7.99773617e-04]\n",
      "test 0.00206088844943\n",
      "gradInit [  5.39664400e-04   4.07063162e-05  -1.24755999e-04   4.09612968e-04\n",
      "   1.64750557e-04  -9.26376620e-05   1.30575880e-03   1.17315248e-03\n",
      "   7.99773617e-04]\n",
      "test2 0.00197285492227\n",
      "gradUpdate [ -7.83521905e-05  -7.49405487e-05   1.39689103e-04  -3.39752874e-04\n",
      "  -1.05927485e-04   1.46977134e-04  -1.97937513e-04  -2.33667623e-04\n",
      "  -3.32770064e-04]\n",
      "test 0.00197285492227\n",
      "gradInit [ -7.83521905e-05  -7.49405487e-05   1.39689103e-04  -3.39752874e-04\n",
      "  -1.05927485e-04   1.46977134e-04  -1.97937513e-04  -2.33667623e-04\n",
      "  -3.32770064e-04]\n",
      "test2 0.00196789695737\n",
      "gradUpdate [  3.65407764e-04   3.48033627e-05  -6.89920624e-05   2.33435237e-04\n",
      "   9.71897776e-05  -6.87695873e-05   8.78288981e-04   6.94235934e-04\n",
      "   5.06207179e-04]\n",
      "test 0.00196789695737\n",
      "gradInit [  3.65407764e-04   3.48033627e-05  -6.89920624e-05   2.33435237e-04\n",
      "   9.71897776e-05  -6.87695873e-05   8.78288981e-04   6.94235934e-04\n",
      "   5.06207179e-04]\n",
      "test2 0.00220870868682\n",
      "gradUpdate [-0.00085365 -0.00039532  0.00057589 -0.00110794 -0.00044313  0.000548\n",
      " -0.00190034 -0.00130213 -0.00169797]\n",
      "test 0.00220870868682\n",
      "gradInit [-0.00085365 -0.00039532  0.00057589 -0.00110794 -0.00044313  0.000548\n",
      " -0.00190034 -0.00130213 -0.00169797]\n",
      "test2 0.00194260906737\n",
      "gradUpdate [  1.68854335e-04  -7.94458486e-06   1.34112335e-05   4.42073413e-05\n",
      "   3.11788145e-05  -7.21847743e-06   4.26978155e-04   2.86498033e-04\n",
      "   1.79583720e-04]\n",
      "test 0.00194260906737\n",
      "gradInit [  1.68854335e-04  -7.94458486e-06   1.34112335e-05   4.42073413e-05\n",
      "   3.11788145e-05  -7.21847743e-06   4.26978155e-04   2.86498033e-04\n",
      "   1.79583720e-04]\n",
      "test2 0.0019369011319\n",
      "gradUpdate [  3.72808629e-05  -5.09449124e-05   6.81713179e-05   6.06939951e-06\n",
      "   1.47063387e-05   1.00760072e-05   1.74347020e-04   1.04624604e-04\n",
      "   1.75019282e-05]\n",
      "test 0.0019369011319\n",
      "gradInit [  3.72808629e-05  -5.09449124e-05   6.81713179e-05   6.06939951e-06\n",
      "   1.47063387e-05   1.00760072e-05   1.74347020e-04   1.04624604e-04\n",
      "   1.75019282e-05]\n",
      "test2 0.00193583801236\n",
      "gradUpdate [  4.42918217e-05  -5.06039518e-05   6.10838687e-05   5.63102402e-05\n",
      "   3.43697417e-05  -7.57429941e-06   2.11271827e-04   1.50514618e-04\n",
      "   5.89681787e-05]\n",
      "test 0.00193583801236\n",
      "gradInit [  4.42918217e-05  -5.06039518e-05   6.10838687e-05   5.63102402e-05\n",
      "   3.43697417e-05  -7.57429941e-06   2.11271827e-04   1.50514618e-04\n",
      "   5.89681787e-05]\n",
      "test2 0.00193536348991\n",
      "gradUpdate [  3.86654778e-05  -5.17747567e-05   6.29290910e-05   5.13818106e-05\n",
      "   3.42694075e-05  -5.61984311e-06   1.96033301e-04   1.37470299e-04\n",
      "   4.87017415e-05]\n",
      "test 0.00193536348991\n",
      "gradInit [  3.86654778e-05  -5.17747567e-05   6.29290910e-05   5.13818106e-05\n",
      "   3.42694075e-05  -5.61984311e-06   1.96033301e-04   1.37470299e-04\n",
      "   4.87017415e-05]\n",
      "test2 0.00193388751424\n",
      "gradUpdate [  1.51156233e-05  -5.64439321e-05   7.05997019e-05   3.00363669e-05\n",
      "   3.37193040e-05   2.69601816e-06   1.31979698e-04   8.19120236e-05\n",
      "   5.49107284e-06]\n",
      "test 0.00193388751424\n",
      "gradInit [  1.51156233e-05  -5.64439321e-05   7.05997019e-05   3.00363669e-05\n",
      "   3.37193040e-05   2.69601816e-06   1.31979698e-04   8.19120236e-05\n",
      "   5.49107284e-06]\n",
      "test2 0.00193275825064\n",
      "gradUpdate [ -1.26693723e-05  -5.64330478e-05   7.88053905e-05  -3.03382118e-05\n",
      "   2.97936854e-05   2.42324313e-05   2.76838689e-05  -2.61439062e-05\n",
      "  -7.28178530e-05]\n",
      "test 0.00193275825064\n",
      "gradInit [ -1.26693723e-05  -5.64330478e-05   7.88053905e-05  -3.03382118e-05\n",
      "   2.97936854e-05   2.42324313e-05   2.76838689e-05  -2.61439062e-05\n",
      "  -7.28178530e-05]\n",
      "test2 0.00193269993162\n",
      "gradUpdate [ -5.69302600e-06  -5.56409890e-05   7.68449844e-05  -2.23283424e-05\n",
      "   2.94366705e-05   2.13351128e-05   4.83571859e-05  -6.65799697e-06\n",
      "  -5.88331014e-05]\n",
      "test 0.00193269993162\n",
      "gradInit [ -5.69302600e-06  -5.56409890e-05   7.68449844e-05  -2.23283424e-05\n",
      "   2.94366705e-05   2.13351128e-05   4.83571859e-05  -6.65799697e-06\n",
      "  -5.88331014e-05]\n",
      "test2 0.00193267826233\n",
      "gradUpdate [ -4.80650758e-06  -5.54315669e-05   7.66148183e-05  -2.22890521e-05\n",
      "   2.91732999e-05   2.12935158e-05   5.03108921e-05  -5.19982533e-06\n",
      "  -5.77807000e-05]\n",
      "test 0.00193267826233\n",
      "gradInit [ -4.80650758e-06  -5.54315669e-05   7.66148183e-05  -2.22890521e-05\n",
      "   2.91732999e-05   2.12935158e-05   5.03108921e-05  -5.19982533e-06\n",
      "  -5.77807000e-05]\n",
      "test2 0.00193260784488\n",
      "gradUpdate [ -1.28558960e-06  -5.45950436e-05   7.56991474e-05  -2.21785809e-05\n",
      "   2.81098772e-05   2.11430422e-05   5.80492518e-05   5.61097764e-07\n",
      "  -5.36246730e-05]\n",
      "test 0.00193260784488\n",
      "gradInit [ -1.28558960e-06  -5.45950436e-05   7.56991474e-05  -2.21785809e-05\n",
      "   2.81098772e-05   2.11430422e-05   5.80492518e-05   5.61097764e-07\n",
      "  -5.36246730e-05]\n",
      "test2 0.00193243563458\n",
      "gradUpdate [  2.01036262e-06  -5.37459698e-05   7.48026747e-05  -2.25595411e-05\n",
      "   2.68348717e-05   2.11410482e-05   6.49211025e-05   5.45494978e-06\n",
      "  -5.00409830e-05]\n",
      "test 0.00193243563458\n",
      "gradInit [  2.01036262e-06  -5.37459698e-05   7.48026747e-05  -2.25595411e-05\n",
      "   2.68348717e-05   2.11410482e-05   6.49211025e-05   5.45494978e-06\n",
      "  -5.00409830e-05]\n",
      "test2 0.00193210586246\n",
      "gradUpdate [  5.69451106e-06  -5.27389541e-05   7.36805410e-05  -2.29595422e-05\n",
      "   2.53029946e-05   2.10960906e-05   7.24242850e-05   1.07882466e-05\n",
      "  -4.59836784e-05]\n",
      "test 0.00193210586246\n",
      "gradInit [  5.69451106e-06  -5.27389541e-05   7.36805410e-05  -2.29595422e-05\n",
      "   2.53029946e-05   2.10960906e-05   7.24242850e-05   1.07882466e-05\n",
      "  -4.59836784e-05]\n",
      "test2 0.00193147311935\n",
      "gradUpdate [  1.05875366e-05  -5.13130310e-05   7.19982535e-05  -2.33695096e-05\n",
      "   2.31583326e-05   2.09445320e-05   8.21617265e-05   1.77321640e-05\n",
      "  -4.04631532e-05]\n",
      "test 0.00193147311935\n",
      "gradInit [  1.05875366e-05  -5.13130310e-05   7.19982535e-05  -2.33695096e-05\n",
      "   2.31583326e-05   2.09445320e-05   8.21617265e-05   1.77321640e-05\n",
      "  -4.04631532e-05]\n",
      "test2 0.00193026995877\n",
      "gradUpdate [  1.76446245e-05  -4.91002244e-05   6.92539618e-05  -2.38092530e-05\n",
      "   1.98759462e-05   2.05903130e-05   9.58341116e-05   2.75210134e-05\n",
      "  -3.23067360e-05]\n",
      "test 0.00193026995877\n",
      "gradInit [  1.76446245e-05  -4.91002244e-05   6.92539618e-05  -2.38092530e-05\n",
      "   1.98759462e-05   2.05903130e-05   9.58341116e-05   2.75210134e-05\n",
      "  -3.23067360e-05]\n",
      "test2 0.00192803036655\n",
      "gradUpdate [  2.82248338e-05  -4.54740669e-05   6.45670473e-05  -2.43571054e-05\n",
      "   1.45704466e-05   1.98677141e-05   1.15668294e-04   4.17922370e-05\n",
      "  -1.98073481e-05]\n",
      "test 0.00192803036655\n",
      "gradInit [  2.82248338e-05  -4.54740669e-05   6.45670473e-05  -2.43571054e-05\n",
      "   1.45704466e-05   1.98677141e-05   1.15668294e-04   4.17922370e-05\n",
      "  -1.98073481e-05]\n",
      "test2 0.00192403392164\n",
      "gradUpdate [  4.41345720e-05  -3.93384966e-05   5.63824594e-05  -2.53687295e-05\n",
      "   5.70318702e-06   1.85311412e-05   1.44200163e-04   6.24948201e-05\n",
      "  -6.71620343e-07]\n",
      "test 0.00192403392164\n",
      "gradInit [  4.41345720e-05  -3.93384966e-05   5.63824594e-05  -2.53687295e-05\n",
      "   5.70318702e-06   1.85311412e-05   1.44200163e-04   6.24948201e-05\n",
      "  -6.71620343e-07]\n",
      "test2 0.00191747913476\n",
      "gradUpdate [  6.67234004e-05  -2.89269760e-05   4.21725188e-05  -2.80335025e-05\n",
      "  -9.22682999e-06   1.62925518e-05   1.81830538e-04   9.04070364e-05\n",
      "   2.68796432e-05]\n",
      "test 0.00191747913476\n",
      "gradInit [  6.67234004e-05  -2.89269760e-05   4.21725188e-05  -2.80335025e-05\n",
      "  -9.22682999e-06   1.62925518e-05   1.81830538e-04   9.04070364e-05\n",
      "   2.68796432e-05]\n",
      "test2 0.00190828662706\n",
      "gradUpdate [  9.11659944e-05  -1.29495362e-05   1.94198818e-05  -3.36592618e-05\n",
      "  -3.21153492e-05   1.24388888e-05   2.14397218e-04   1.16775773e-04\n",
      "   5.81035888e-05]\n",
      "test 0.00190828662706\n",
      "gradInit [  9.11659944e-05  -1.29495362e-05   1.94198818e-05  -3.36592618e-05\n",
      "  -3.21153492e-05   1.24388888e-05   2.14397218e-04   1.16775773e-04\n",
      "   5.81035888e-05]\n",
      "test2 0.00189991496171\n",
      "gradUpdate [  9.40965724e-05  -4.56523014e-06   3.41096927e-06  -2.40731146e-05\n",
      "  -3.94384510e-05   4.71536661e-06   2.07859754e-04   1.13814255e-04\n",
      "   7.23643626e-05]\n",
      "test 0.00189991496171\n",
      "gradInit [  9.40965724e-05  -4.56523014e-06   3.41096927e-06  -2.40731146e-05\n",
      "  -3.94384510e-05   4.71536661e-06   2.07859754e-04   1.13814255e-04\n",
      "   7.23643626e-05]\n",
      "test2 0.00188906810161\n",
      "gradUpdate [  5.11522524e-05  -1.00802194e-05  -7.79460686e-06  -8.58185201e-06\n",
      "  -3.85349745e-05  -4.97212462e-06   8.56201572e-05   1.56272056e-05\n",
      "   4.05967857e-05]\n",
      "test 0.00188906810161\n",
      "gradInit [  5.11522524e-05  -1.00802194e-05  -7.79460686e-06  -8.58185201e-06\n",
      "  -3.85349745e-05  -4.97212462e-06   8.56201572e-05   1.56272056e-05\n",
      "   4.05967857e-05]\n",
      "test2 0.0018862239084\n",
      "gradUpdate [  2.79361346e-05  -1.89229782e-05  -8.41185600e-06   2.14965897e-05\n",
      "  -2.19639553e-05  -1.32319338e-05   4.33391823e-05  -1.57662619e-05\n",
      "   3.76148993e-05]\n",
      "test 0.0018862239084\n",
      "gradInit [  2.79361346e-05  -1.89229782e-05  -8.41185600e-06   2.14965897e-05\n",
      "  -2.19639553e-05  -1.32319338e-05   4.33391823e-05  -1.57662619e-05\n",
      "   3.76148993e-05]\n",
      "test2 0.00188566418995\n",
      "gradUpdate [  1.92453774e-05  -2.55526101e-05  -4.13154487e-06   3.73555226e-05\n",
      "  -1.30497816e-05  -1.65067991e-05   4.15855117e-05  -1.19715184e-05\n",
      "   4.01516831e-05]\n",
      "test 0.00188566418995\n",
      "gradInit [  1.92453774e-05  -2.55526101e-05  -4.13154487e-06   3.73555226e-05\n",
      "  -1.30497816e-05  -1.65067991e-05   4.15855117e-05  -1.19715184e-05\n",
      "   4.01516831e-05]\n",
      "test2 0.00188559533036\n",
      "gradUpdate [  1.00054794e-05  -3.00314602e-05  -2.42739138e-07   3.32725746e-05\n",
      "  -1.43089413e-05  -1.50535669e-05   2.12117053e-05  -2.74056867e-05\n",
      "   2.77529615e-05]\n",
      "test 0.00188559533036\n",
      "gradInit [  1.00054794e-05  -3.00314602e-05  -2.42739138e-07   3.32725746e-05\n",
      "  -1.43089413e-05  -1.50535669e-05   2.12117053e-05  -2.74056867e-05\n",
      "   2.77529615e-05]\n",
      "test2 0.00188558373712\n",
      "gradUpdate [  6.83758900e-06  -3.14665676e-05   1.34670994e-06   2.98701202e-05\n",
      "  -1.56549875e-05  -1.40074366e-05   1.29777030e-05  -3.40514590e-05\n",
      "   2.19643594e-05]\n",
      "test 0.00188558373712\n",
      "gradInit [  6.83758900e-06  -3.14665676e-05   1.34670994e-06   2.98701202e-05\n",
      "  -1.56549875e-05  -1.40074366e-05   1.29777030e-05  -3.40514590e-05\n",
      "   2.19643594e-05]\n",
      "test2 0.00188557690979\n",
      "gradUpdate [  5.72839511e-06  -3.19041635e-05   1.92670087e-06   2.82201954e-05\n",
      "  -1.62736936e-05  -1.35134883e-05   9.67914522e-06  -3.68262592e-05\n",
      "   1.95553945e-05]\n",
      "test 0.00188557690979\n",
      "gradInit [  5.72839511e-06  -3.19041635e-05   1.92670087e-06   2.82201954e-05\n",
      "  -1.62736936e-05  -1.35134883e-05   9.67914522e-06  -3.68262592e-05\n",
      "   1.95553945e-05]\n",
      "test2 0.00188556421996\n",
      "gradUpdate [  4.62265808e-06  -3.23105440e-05   2.51548588e-06   2.64333826e-05\n",
      "  -1.68898207e-05  -1.29756209e-05   6.25041214e-06  -3.97461024e-05\n",
      "   1.70235864e-05]\n",
      "test 0.00188556421996\n",
      "gradInit [  4.62265808e-06  -3.23105440e-05   2.51548588e-06   2.64333826e-05\n",
      "  -1.68898207e-05  -1.29756209e-05   6.25041214e-06  -3.97461024e-05\n",
      "   1.70235864e-05]\n",
      "test2 0.00188554028913\n",
      "gradUpdate [  3.26070155e-06  -3.27900289e-05   3.23828596e-06   2.41519816e-05\n",
      "  -1.76267321e-05  -1.22853995e-05   1.95286746e-06  -4.34256581e-05\n",
      "   1.38426195e-05]\n",
      "test 0.00188554028913\n",
      "gradInit [  3.26070155e-06  -3.27900289e-05   3.23828596e-06   2.41519816e-05\n",
      "  -1.76267321e-05  -1.22853995e-05   1.95286746e-06  -4.34256581e-05\n",
      "   1.38426195e-05]\n",
      "test2 0.00188549521369\n",
      "gradUpdate [  1.38404952e-06  -3.34342702e-05   4.22139439e-06   2.09313069e-05\n",
      "  -1.86187530e-05  -1.13074915e-05  -4.00964232e-06  -4.85451839e-05\n",
      "   9.42724624e-06]\n",
      "test 0.00188549521369\n",
      "gradInit [  1.38404952e-06  -3.34342702e-05   4.22139439e-06   2.09313069e-05\n",
      "  -1.86187530e-05  -1.13074915e-05  -4.00964232e-06  -4.85451839e-05\n",
      "   9.42724624e-06]\n",
      "test2 0.00188541124422\n",
      "gradUpdate [ -1.37768640e-06  -3.43706793e-05   5.63850473e-06   1.61026545e-05\n",
      "  -2.00553458e-05  -9.83618036e-06  -1.27846986e-05  -5.60917743e-05\n",
      "   2.93256385e-06]\n",
      "test 0.00188541124422\n",
      "gradInit [ -1.37768640e-06  -3.43706793e-05   5.63850473e-06   1.61026545e-05\n",
      "  -2.00553458e-05  -9.83618036e-06  -1.27846986e-05  -5.60917743e-05\n",
      "   2.93256385e-06]\n",
      "test2 0.0018852580521\n",
      "gradUpdate [ -5.59663091e-06  -3.57984084e-05   7.74289196e-06   8.61142123e-06\n",
      "  -2.22287023e-05  -7.54191221e-06  -2.61196690e-05  -6.75761083e-05\n",
      "  -6.92626889e-06]\n",
      "test 0.0018852580521\n",
      "gradInit [ -5.59663091e-06  -3.57984084e-05   7.74289196e-06   8.61142123e-06\n",
      "  -2.22287023e-05  -7.54191221e-06  -2.61196690e-05  -6.75761083e-05\n",
      "  -6.92626889e-06]\n",
      "test2 0.00188498738436\n",
      "gradUpdate [ -1.21237078e-05  -3.80271404e-05   1.08783232e-05  -3.14013092e-06\n",
      "  -2.55835050e-05  -3.91164400e-06  -4.65412738e-05  -8.52059529e-05\n",
      "  -2.20039550e-05]\n",
      "test 0.00188498738436\n",
      "gradInit [ -1.21237078e-05  -3.80271404e-05   1.08783232e-05  -3.14013092e-06\n",
      "  -2.55835050e-05  -3.91164400e-06  -4.65412738e-05  -8.52059529e-05\n",
      "  -2.20039550e-05]\n",
      "test2 0.00188452782153\n",
      "gradUpdate [ -2.20065292e-05  -4.14853967e-05   1.53852632e-05  -2.11855583e-05\n",
      "  -3.07144476e-05   1.74436490e-06  -7.69790173e-05  -1.11622688e-04\n",
      "  -4.44459791e-05]\n",
      "test 0.00188452782153\n",
      "gradInit [ -2.20065292e-05  -4.14853967e-05   1.53852632e-05  -2.11855583e-05\n",
      "  -3.07144476e-05   1.74436490e-06  -7.69790173e-05  -1.11622688e-04\n",
      "  -4.44459791e-05]\n",
      "test2 0.00188376609836\n",
      "gradUpdate [ -3.58003376e-05  -4.65641182e-05   2.11936871e-05  -4.68179731e-05\n",
      "  -3.81135112e-05   9.96208767e-06  -1.18473288e-04  -1.48062444e-04\n",
      "  -7.50037393e-05]\n",
      "test 0.00188376609836\n",
      "gradInit [ -3.58003376e-05  -4.65641182e-05   2.11936871e-05  -4.68179731e-05\n",
      "  -3.81135112e-05   9.96208767e-06  -1.18473288e-04  -1.48062444e-04\n",
      "  -7.50037393e-05]\n",
      "test2 0.00188246632432\n",
      "gradUpdate [ -5.23304732e-05  -5.32556061e-05   2.72434549e-05  -7.84275717e-05\n",
      "  -4.76407251e-05   2.04425370e-05  -1.66357799e-04  -1.91218484e-04\n",
      "  -1.10297699e-04]\n",
      "test 0.00188246632432\n",
      "gradInit [ -5.23304732e-05  -5.32556061e-05   2.72434549e-05  -7.84275717e-05\n",
      "  -4.76407251e-05   2.04425370e-05  -1.66357799e-04  -1.91218484e-04\n",
      "  -1.10297699e-04]\n",
      "test2 0.00188009812871\n",
      "gradUpdate [ -6.89292169e-05  -6.11298644e-05   3.17536171e-05  -1.12425991e-04\n",
      "  -5.87459476e-05   3.23716411e-05  -2.11027306e-04  -2.34095507e-04\n",
      "  -1.43879717e-04]\n",
      "test 0.00188009812871\n",
      "gradInit [ -6.89292169e-05  -6.11298644e-05   3.17536171e-05  -1.12425991e-04\n",
      "  -5.87459476e-05   3.23716411e-05  -2.11027306e-04  -2.34095507e-04\n",
      "  -1.43879717e-04]\n",
      "test2 0.00187545101504\n",
      "gradUpdate [ -7.84681410e-05  -6.73197718e-05   3.08487252e-05  -1.40660942e-04\n",
      "  -6.89747237e-05   4.37228862e-05  -2.28184392e-04  -2.57293017e-04\n",
      "  -1.60657024e-04]\n",
      "test 0.00187545101504\n",
      "gradInit [ -7.84681410e-05  -6.73197718e-05   3.08487252e-05  -1.40660942e-04\n",
      "  -6.89747237e-05   4.37228862e-05  -2.28184392e-04  -2.57293017e-04\n",
      "  -1.60657024e-04]\n",
      "test2 0.00186742965976\n",
      "gradUpdate [  6.26222968e-05   4.41701837e-05  -5.13654186e-05  -4.23884969e-05\n",
      "   1.07375125e-05   7.73432431e-06   2.44471993e-04   2.07452587e-04\n",
      "   1.25769440e-04]\n",
      "test 0.00186742965976\n",
      "gradInit [  6.26222968e-05   4.41701837e-05  -5.13654186e-05  -4.23884969e-05\n",
      "   1.07375125e-05   7.73432431e-06   2.44471993e-04   2.07452587e-04\n",
      "   1.25769440e-04]\n",
      "test2 0.00186752882334\n",
      "gradUpdate [ -4.49484520e-05  -4.67665996e-05   9.00872820e-06  -1.32800958e-04\n",
      "  -6.13346258e-05   4.43669056e-05  -1.08460435e-04  -1.56855322e-04\n",
      "  -9.60687035e-05]\n",
      "test 0.00186752882334\n",
      "gradInit [ -4.49484520e-05  -4.67665996e-05   9.00872820e-06  -1.32800958e-04\n",
      "  -6.13346258e-05   4.43669056e-05  -1.08460435e-04  -1.56855322e-04\n",
      "  -9.60687035e-05]\n",
      "test2 0.00198822283398\n",
      "gradUpdate [ 0.00034644  0.00041638 -0.00025482  0.00028399  0.00041391 -0.00018963\n",
      "  0.00131174  0.00160829  0.00090801]\n",
      "test 0.00198822283398\n",
      "gradInit [ 0.00034644  0.00041638 -0.00025482  0.00028399  0.00041391 -0.00018963\n",
      "  0.00131174  0.00160829  0.00090801]\n",
      "test2 0.00186433858532\n",
      "gradUpdate [ -5.91553932e-06  -1.60500096e-05  -1.49244654e-05  -9.16936142e-05\n",
      "  -3.15961487e-05   2.89773543e-05   2.79100030e-05  -1.95022661e-05\n",
      "  -6.71527018e-06]\n",
      "test 0.00186433858532\n",
      "gradInit [ -5.91553932e-06  -1.60500096e-05  -1.49244654e-05  -9.16936142e-05\n",
      "  -3.15961487e-05   2.89773543e-05   2.79100030e-05  -1.95022661e-05\n",
      "  -6.71527018e-06]\n",
      "test2 0.00186248630859\n",
      "gradUpdate [  9.01805910e-06   2.49628048e-07  -3.27762139e-05  -4.35413188e-05\n",
      "   4.75560738e-06   7.74426576e-06   1.03795410e-04   7.48986877e-05\n",
      "   6.37405057e-05]\n",
      "test 0.00186248630859\n",
      "gradInit [  9.01805910e-06   2.49628048e-07  -3.27762139e-05  -4.35413188e-05\n",
      "   4.75560738e-06   7.74426576e-06   1.03795410e-04   7.48986877e-05\n",
      "   6.37405057e-05]\n",
      "test2 0.00186145790124\n",
      "gradUpdate [ -2.28251450e-05  -2.55445630e-05  -2.48327415e-05  -5.59843388e-05\n",
      "  -1.19203745e-05   1.17147325e-05   6.93007847e-06  -1.88971675e-05\n",
      "   1.72224730e-05]\n",
      "test 0.00186145790124\n",
      "gradInit [ -2.28251450e-05  -2.55445630e-05  -2.48327415e-05  -5.59843388e-05\n",
      "  -1.19203745e-05   1.17147325e-05   6.93007847e-06  -1.88971675e-05\n",
      "   1.72224730e-05]\n",
      "test2 0.00186136037732\n",
      "gradUpdate [ -2.10369771e-05  -2.39334199e-05  -2.79445483e-05  -5.51594973e-05\n",
      "  -1.36298865e-05   1.09266162e-05   1.46798875e-05  -9.05679906e-06\n",
      "   2.33422189e-05]\n",
      "test 0.00186136037732\n",
      "gradInit [ -2.10369771e-05  -2.39334199e-05  -2.79445483e-05  -5.51594973e-05\n",
      "  -1.36298865e-05   1.09266162e-05   1.46798875e-05  -9.05679906e-06\n",
      "   2.33422189e-05]\n",
      "test2 0.00186132848054\n",
      "gradUpdate [ -2.21851856e-05  -2.56222959e-05  -2.75968657e-05  -5.29249004e-05\n",
      "  -1.27026184e-05   1.01030074e-05   1.76347357e-05  -4.15130196e-06\n",
      "   2.60570472e-05]\n",
      "test 0.00186132848054\n",
      "gradInit [ -2.21851856e-05  -2.56222959e-05  -2.75968657e-05  -5.29249004e-05\n",
      "  -1.27026184e-05   1.01030074e-05   1.76347357e-05  -4.15130196e-06\n",
      "   2.60570472e-05]\n",
      "test2 0.00186131806121\n",
      "gradUpdate [ -2.26233045e-05  -2.62090604e-05  -2.71901943e-05  -5.35390560e-05\n",
      "  -1.34593413e-05   1.04237376e-05   1.73225098e-05  -4.30729428e-06\n",
      "   2.53597606e-05]\n",
      "test 0.00186131806121\n",
      "gradInit [ -2.26233045e-05  -2.62090604e-05  -2.71901943e-05  -5.35390560e-05\n",
      "  -1.34593413e-05   1.04237376e-05   1.73225098e-05  -4.30729428e-06\n",
      "   2.53597606e-05]\n",
      "test2 0.00186129934921\n",
      "gradUpdate [ -2.33051039e-05  -2.69361389e-05  -2.66166715e-05  -5.43852720e-05\n",
      "  -1.42859020e-05   1.08696285e-05   1.62932809e-05  -5.18219363e-06\n",
      "   2.41089243e-05]\n",
      "test 0.00186129934921\n",
      "gradInit [ -2.33051039e-05  -2.69361389e-05  -2.66166715e-05  -5.43852720e-05\n",
      "  -1.42859020e-05   1.08696285e-05   1.62932809e-05  -5.18219363e-06\n",
      "   2.41089243e-05]\n",
      "test2 0.00186126513024\n",
      "gradUpdate [ -2.43525375e-05  -2.78735917e-05  -2.58313019e-05  -5.57047211e-05\n",
      "  -1.53977697e-05   1.15520111e-05   1.42256945e-05  -7.05836923e-06\n",
      "   2.20008939e-05]\n",
      "test 0.00186126513024\n",
      "gradInit [ -2.43525375e-05  -2.78735917e-05  -2.58313019e-05  -5.57047211e-05\n",
      "  -1.53977697e-05   1.15520111e-05   1.42256945e-05  -7.05836923e-06\n",
      "   2.20008939e-05]\n",
      "test2 0.00186120308839\n",
      "gradUpdate [ -2.60571218e-05  -2.91943475e-05  -2.46638226e-05  -5.78287359e-05\n",
      "  -1.69968164e-05   1.26480807e-05   1.03186196e-05  -1.06928449e-05\n",
      "   1.83817589e-05]\n",
      "test 0.00186120308839\n",
      "gradInit [ -2.60571218e-05  -2.91943475e-05  -2.46638226e-05  -5.78287359e-05\n",
      "  -1.69968164e-05   1.26480807e-05   1.03186196e-05  -1.06928449e-05\n",
      "   1.83817589e-05]\n",
      "test2 0.00186109432541\n",
      "gradUpdate [ -2.88441149e-05  -3.10864279e-05  -2.28763682e-05  -6.12731243e-05\n",
      "  -1.93580837e-05   1.44478786e-05   3.22511289e-06  -1.73837386e-05\n",
      "   1.21716043e-05]\n",
      "test 0.00186109432541\n",
      "gradInit [ -2.88441149e-05  -3.10864279e-05  -2.28763682e-05  -6.12731243e-05\n",
      "  -1.93580837e-05   1.44478786e-05   3.22511289e-06  -1.73837386e-05\n",
      "   1.21716043e-05]\n",
      "test2 0.00186091682238\n",
      "gradUpdate [ -3.31823076e-05  -3.36041341e-05  -2.02218221e-05  -6.65816866e-05\n",
      "  -2.26710567e-05   1.73230685e-05  -8.88768003e-06  -2.89028766e-05\n",
      "   1.93931188e-06]\n",
      "test 0.00186091682238\n",
      "gradInit [ -3.31823076e-05  -3.36041341e-05  -2.02218221e-05  -6.65816866e-05\n",
      "  -2.26710567e-05   1.73230685e-05  -8.88768003e-06  -2.89028766e-05\n",
      "   1.93931188e-06]\n",
      "test2 0.00186066436272\n",
      "gradUpdate [ -3.88735261e-05  -3.59532364e-05  -1.69020930e-05  -7.33687807e-05\n",
      "  -2.62546763e-05   2.13093700e-05  -2.69446218e-05  -4.60766475e-05\n",
      "  -1.28237672e-05]\n",
      "test 0.00186066436272\n",
      "gradInit [ -3.88735261e-05  -3.59532364e-05  -1.69020930e-05  -7.33687807e-05\n",
      "  -2.62546763e-05   2.13093700e-05  -2.69446218e-05  -4.60766475e-05\n",
      "  -1.28237672e-05]\n",
      "test2 0.001860456297\n",
      "gradUpdate [ -4.20696060e-05  -3.63120622e-05  -1.54989589e-05  -7.64398025e-05\n",
      "  -2.71228272e-05   2.33039703e-05  -4.04355298e-05  -5.89619833e-05\n",
      "  -2.22491388e-05]\n",
      "test 0.001860456297\n",
      "gradInit [ -4.20696060e-05  -3.63120622e-05  -1.54989589e-05  -7.64398025e-05\n",
      "  -2.71228272e-05   2.33039703e-05  -4.04355298e-05  -5.89619833e-05\n",
      "  -2.22491388e-05]\n",
      "test2 0.00186019949744\n",
      "gradUpdate [ -4.44233620e-05  -3.54138499e-05  -1.50696404e-05  -7.77738144e-05\n",
      "  -2.63310325e-05   2.43002922e-05  -5.58067323e-05  -7.38979493e-05\n",
      "  -3.09460745e-05]\n",
      "test 0.00186019949744\n",
      "gradInit [ -4.44233620e-05  -3.54138499e-05  -1.50696404e-05  -7.77738144e-05\n",
      "  -2.63310325e-05   2.43002922e-05  -5.58067323e-05  -7.38979493e-05\n",
      "  -3.09460745e-05]\n",
      "test2 0.00185995667403\n",
      "gradUpdate [ -4.47311464e-05  -3.37068990e-05  -1.56217514e-05  -7.69346610e-05\n",
      "  -2.44785238e-05   2.39126510e-05  -6.53630285e-05  -8.33331059e-05\n",
      "  -3.48430887e-05]\n",
      "test 0.00185995667403\n",
      "gradInit [ -4.47311464e-05  -3.37068990e-05  -1.56217514e-05  -7.69346610e-05\n",
      "  -2.44785238e-05   2.39126510e-05  -6.53630285e-05  -8.33331059e-05\n",
      "  -3.48430887e-05]\n",
      "test2 0.00185955785096\n",
      "gradUpdate [ -4.28475473e-05  -3.04071596e-05  -1.69893003e-05  -7.36517943e-05\n",
      "  -2.09664060e-05   2.22187144e-05  -7.05012623e-05  -8.82227883e-05\n",
      "  -3.50308040e-05]\n",
      "test 0.00185955785096\n",
      "gradInit [ -4.28475473e-05  -3.04071596e-05  -1.69893003e-05  -7.36517943e-05\n",
      "  -2.09664060e-05   2.22187144e-05  -7.05012623e-05  -8.82227883e-05\n",
      "  -3.50308040e-05]\n",
      "test2 0.00185887511925\n",
      "gradUpdate [ -3.85928809e-05  -2.57313317e-05  -1.86209872e-05  -6.76629738e-05\n",
      "  -1.61071485e-05   1.92855241e-05  -7.11198433e-05  -8.83415870e-05\n",
      "  -3.19631427e-05]\n",
      "test 0.00185887511925\n",
      "gradInit [ -3.85928809e-05  -2.57313317e-05  -1.86209872e-05  -6.76629738e-05\n",
      "  -1.61071485e-05   1.92855241e-05  -7.11198433e-05  -8.83415870e-05\n",
      "  -3.19631427e-05]\n",
      "test2 0.00185770266628\n",
      "gradUpdate [ -3.00340543e-05  -1.84049158e-05  -2.06244876e-05  -5.66161187e-05\n",
      "  -8.77493551e-06   1.42721166e-05  -6.25539334e-05  -7.84501295e-05\n",
      "  -2.30954920e-05]\n",
      "test 0.00185770266628\n",
      "gradInit [ -3.00340543e-05  -1.84049158e-05  -2.06244876e-05  -5.66161187e-05\n",
      "  -8.77493551e-06   1.42721166e-05  -6.25539334e-05  -7.84501295e-05\n",
      "  -2.30954920e-05]\n",
      "test2 0.00185579422051\n",
      "gradUpdate [ -1.50129889e-05  -7.68977191e-06  -2.22551103e-05  -3.71372170e-05\n",
      "   1.74198312e-06   6.08465658e-06  -4.05786552e-05  -5.40417619e-05\n",
      "  -6.80033476e-06]\n",
      "test 0.00185579422051\n",
      "gradInit [ -1.50129889e-05  -7.68977191e-06  -2.22551103e-05  -3.71372170e-05\n",
      "   1.74198312e-06   6.08465658e-06  -4.05786552e-05  -5.40417619e-05\n",
      "  -6.80033476e-06]\n",
      "test2 0.00185312844071\n",
      "gradUpdate [  8.77130151e-06   7.66679083e-06  -2.19741237e-05  -5.82680115e-06\n",
      "   1.54118674e-05  -6.48561937e-06   2.09394304e-06  -7.21112094e-06\n",
      "   1.76117766e-05]\n",
      "test 0.00185312844071\n",
      "gradInit [  8.77130151e-06   7.66679083e-06  -2.19741237e-05  -5.82680115e-06\n",
      "   1.54118674e-05  -6.48561937e-06   2.09394304e-06  -7.21112094e-06\n",
      "   1.76117766e-05]\n",
      "test2 0.00185112536943\n",
      "gradUpdate [  1.55492970e-05   1.12522023e-05  -1.61950477e-05   2.01512146e-06\n",
      "   1.26509216e-05  -8.65013282e-06   6.93716212e-06   3.36823892e-06\n",
      "   1.26021034e-05]\n",
      "test 0.00185112536943\n",
      "gradInit [  1.55492970e-05   1.12522023e-05  -1.61950477e-05   2.01512146e-06\n",
      "   1.26509216e-05  -8.65013282e-06   6.93716212e-06   3.36823892e-06\n",
      "   1.26021034e-05]\n",
      "test2 0.00184988419307\n",
      "gradUpdate [  7.97898895e-06  -1.39449520e-06  -3.25532628e-06  -1.78677156e-06\n",
      "  -4.93530890e-07  -4.23515294e-06  -1.79472460e-05  -1.45420698e-05\n",
      "  -1.13661108e-05]\n",
      "test 0.00184988419307\n",
      "gradInit [  7.97898895e-06  -1.39449520e-06  -3.25532628e-06  -1.78677156e-06\n",
      "  -4.93530890e-07  -4.23515294e-06  -1.79472460e-05  -1.45420698e-05\n",
      "  -1.13661108e-05]\n",
      "test2 0.00184971443992\n",
      "gradUpdate [  9.05429801e-06  -1.57135917e-06  -3.88096736e-06   1.46695219e-06\n",
      "   3.24293092e-06  -6.12699965e-06   3.35272706e-06   8.63783117e-06\n",
      "   3.21252580e-07]\n",
      "res x [ 0.99281191  0.94557894 -0.73649493 -0.06483795 -0.03293946 -0.06416741\n",
      "  1.25274248  1.26885673 -0.75091844]\n",
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x10866c310>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEPCAYAAACHuClZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8FeX59/HPlQQIawLIJkuCbAWERkFAXEi1VhAVa6vg\njhu2imvr4/Jrf/i0Pr8Wbalaa9XqD8SlaG1V6sJSIW4oixD2ABFZwhKWsC8hJNfzx5nAMSRhTpLJ\n3Emu9+t1XsnMuefM9yzJdea+ZxFVxRhjjKmMuLADGGOMqfmsmBhjjKk0KybGGGMqzYqJMcaYSrNi\nYowxptKsmBhjjKm0wIuJiAwVkSwRWS0iD5XR5hkRWSMimSKS5s1rICJzRWSRiCwVkXFR7ceJSI6I\nLPRuQ4N+HsYYY8qWEOSDi0gc8CxwIbAZmC8i76lqVlSbYUAXVe0mIgOB54FBqpovIj9Q1YMiEg98\nISIfqeo8b9EJqjohyPzGGGP8CXrLZACwRlXXq2oBMAUYUaLNCGAygKrOBZJEpI03fdBr04BI4Ys+\nwlKCDG6MMca/oItJe2Bj1HSON6+8NpuK24hInIgsArYCM1V1flS7sV632EsiklT10Y0xxvjl9AC8\nqhap6hlAB2CgiPTy7noOOE1V04gUGuvuMsaYEAU6ZkJkK6NT1HQHb17JNh3La6Oqe0VkNjAUWKGq\n26Pu/hvw79JWLiJ24jFjjKkAVY1pKCHoLZP5QFcRSRGR+sAoYGqJNlOBGwFEZBCwW1VzReSU4u4r\nEWkIXARkedNto5a/ElhWVgBVdeo2bty40DPUhEyu5rJMlqku5KqIQLdMVLVQRMYCM4gUrpdVdaWI\n3BG5W19U1Q9F5BIRyQYOADd7i7cDXvH2CIsD3lTVD737nvB2IS4C1gF3BPk8qtK6devCjnACFzOB\nm7kskz+WyT9Xc8Uq6G4uVHUa0KPEvBdKTI8tZbmlwJllPOaNVZnRGGNM5Tg9AF8bjR49OuwIJ3Ax\nE7iZyzL5Y5n8czVXrKSi/WM1gYhobX5+xhgTBBFBHRuANyVkZGSEHeEELmYCN3NZJn8sk3+u5oqV\nFRNjjDGVZt1cxhhTw3yV8xVTV03l/13w/xCp+jNLWTeXMcZ52w5sI/9oftgxapyCwgKmLJvCoJcG\nce0/r6V149YUaVHYsY6xLZNqlpGRQXp6etgxvsPFTOBmLsvkT2mZDh89zLjZ43j+6+c5UniEzsmd\n6dumL+mp6YzpN4Y4Cfa7bVmv04EjB1i1cxXLty1n+fbILTsvmyItIl7iiY+Lp1mDZvRu1Zs+rfvQ\np00fzmh7BkmJVXNKQD/v34Y9G7j4tYtp07gN9w26j8u6X0Z8XHyVrL80FdkyCfw4E2OM+SrnK25+\n72b6tO5D9t3ZNGvQjKwdWSzJXcILX7/A1FVTefXHr9KyUcsqX3dBYQHbDmwja3sWO1bsYP3u9azf\ns57svGxWbF9B7oFcurXoRu/Wvendqjc3p91M95bdSYhLoLCokEItZNehXSzbtoyl25YyZfkUFm9d\nTOfmnRncYTCDOw7mgs4X0DGp48nDVEB2XjY/nPxD7h14L/effX8g66gKtmVijAnUc/Of4zef/IY/\nD/szV/W+6oT7CwoLeOTjR3h7xdv846p/cFb7s05oo6psP7idnL057Dm8h31H9rH/yH725e9jT/4e\n9ubvZW/+XnYd3sWuQ7vYdXgXeYfy2HZgG3vz93JKo1No26QtqcmppCSlkJKUQpcWXejVqhedkzvH\n/C2/oLCAxbmLmbNxDp9v+JxZ386idePWXNzlYoZ2HcqQ1CEkJiRW+DUrtnzbci5+7WL+e8h/M6bf\nmEo/nl8V2TKxYmKMCUxhUSGdnurE9Ounc3rr08tt+6+V/+Jn7/+M81POp1ALyT+az+Gjh9myfwsb\n9mwgMSGRjs06kpyYTNMGTWlaP3Jr1qAZzRo0o2mDpjRPbE7zhs1p0bAFzROb07pxa1o2ahl4F1ph\nUSELtyxk+jfTmZY9jSW5S0hPTWd4t+Fc0u2SCm21zNk4h5+89ROevOhJru97fQCpy1aRYhL6CcUC\nPlmZumb27NlhRzhBGJkmLZqkZzx/hj437znde3hvqW3stfLH5Uwfr/1Yz3j+DN/Lrc1bq28seUP/\nueKf+v6q93XmNzN1ae7SMj8jFclUHXYc2KGvL3ldr/3ntdpyfEvt81wffWjmQ/rJuk/0yNEj5eba\ntHeT3vTOTdruD+30vaz3qi1zNO9/Z0z/b23MxFS7Ii3id5//jjH9xvDxtx/zX7P+i1Gnj+LX5/+a\ndk3bhR3PVKE3lr7BdX2u892+c/POdG7eOcBE1aNlo5Zc2+daru1zLYVFhczbNI8P13zI/dPvZ/XO\n1Zzd4WyGpAxhcMfBrNy+koQNCRwtOsqcjXOY8OUEbjvzNrLGZtGsQbOwn4pv1s1lqt207Gk88vEj\nLByzEBFh095N/PHLP/Ju1rvMuGEGXVt0DTuiL4cKDrFg8wLmbJzDlzlfsvvwbi7sfCEXd72Yfu36\nBbq3TU1w+OhhTv3jqSz5+RI6NOsQdhxn7Dq0i882fMYn6z5h7qa5HC06SkJcAglxCXRK6sS4IePo\n0qJLqBltzKQEKyZuGvb6MEb2HsnotNHfmf/i1y/yWMZjfHjdh6S1TQsnXBkKiwpZt3sd8zfP58uN\nXzInZw4rtq+gd6veDO44mLM7nE2zBs2YuXYm07+ZTu7+XB5Lf4yxA044IXad8c7Kd3hm3jPMvml2\n2FFMjGzX4BqgphwTEJSsHVks2rKId0a+c8J9Y/qNoXlic3706o94++q32ZO1h65ndmXr/q0cKDhA\n68atadekHW2atKF+fP0qyVNQWMCe/D3sPryb3Yd3k3coj9z9uWzdv5Wt+7fy7e5vWb1zNd/s+oZW\njVrRaVcnLr/4cib8aAL9T+1Pw3oNv/N4w7oNA2DVjlVcPuVytuzbwuMXPB7IUcrFXP1MvbE9ti6u\noLn4OoG7uWJlxcRUq2fmPsOYfmPK3G3yqt5XkZyYzGV/v4zCtYV0WNOBtk3a0qheI7Yd2MaW/VvY\ndmAbbRq3OXZcQM9TegKw+/Bu9uTvYc/hPewv2M+BIwc4UHCAgwUHOVRwKPLzaOTngSOR+UVaRFJi\nEsmJySQnJtM8sTltm7SlbZO2tGvajoEdBtKjZQ+6tuhK4/qNI3/456Sf9Hn2OKUHn9/8OZe8cQnb\nDmzjr5f+lYS4uvPntv/IfmZ8M4MXL30x7Cimmlg3l6k2uw7t4rRnTmPFnStOOtCuqmV+my8sKmTD\nng0s376cFdtXkLUjiziJO1YQkhok0bh+YxrXa0zj+o1pVK8RDRMa0rBeQxomNDw2r3G9xtSPrx/o\nVsO+/H1c+daVNKnfhLd++hb14usFti6XTMqcxLtZ7/LuqHfDjmIqwMZMSrBi4pYnv3iSxbmLee3K\n18KOUq2OFB4hfVI6dw+4m2v6XBN2nGpx0asXMebMMaUepGjcZyd6rAFcvHZBdWQ6WnSUZ+c/y70D\n7/W9TG15rerH1+ehcx7iT1/9iSC+3Lj2Om3Zt4UvP/uSS7tfGnaU73DtdSrmaq5YWTEx1WLB5gU0\na9Cs1FNl1AWXdr+UHQd38FXOV2FHCVR2XjZXvHkFF3e5+ISdE0ztZt1cplqM/3w8m/Zt4plhz4Qd\nJTRPffUUX+V8xZSfTgk7SpVTVV5Z/AoPznyQX5//a8YOGBv4KUxMcGzXYOOsTzd8yi1pt4QdI1S3\nnHELv/nkN2zcszGwM8yGYd3udTw480GydmQx68ZZ9GnTJ+xIJgT21aGaudg/GnSmwqJCvtjwBeel\nnBfTcrXttWrWoBk39L2B5+Y/V3WBCO91WrVjFaPfHU2/F/vxvZbfY/7t848Vktr23gXJ1VyxCryY\niMhQEckSkdUi8lAZbZ4RkTUikikiad68BiIyV0QWichSERkX1b65iMwQkVUiMl1EquYqNSYQS3KX\n0K5pO1o3bh12lNDdPfBuXlr0EgcLDoYdpUL25e/jzWVvcuWbV3LexPPo0rwL2Xdn89sLflslp1w3\nNVegYyYiEgesBi4ENgPzgVGqmhXVZhgwVlWHi8hA4GlVHeTd10hVD4pIPPAFcI+qzhOR8cBOVX3C\nK1DNVfXhUtZvYyYOeOqrp1i5fSUvXPZC2FGcMGLKCIZ3G16t16eoqEMFh1iSu4QFmxcw/ZvpZKzL\n4NxO53JlzysZdfoomtRvEnZEEwAXx0wGAGtUdT2AiEwBRgBZUW1GAJMBVHWuiCSJSBtVzVXV4q9v\nDbysGrXMEO/3V4AM4IRiYtzw6fpP+Wmvn4Ydwxn3D7qfG965gd6tenNOp3NCyaCq5BfmHzuFTN6h\nPHYc3MHGPRtZvydyJcI1O9eweudqepzSg/7t+nN176uZ/OPJJCcmh5LZuC3oYtIe2Bg1nUOkwJTX\nZpM3L9fbsvka6AL8RVXne21aq2ougKpuFZEa03/i4nl4gsykqny6/lOeHvp0zMvW1tcqPTWdP138\nJ676x1Vc2fNKfnfh72jaoCkQOdPuktwlbN63+dj5wvYc3sOho4c4VHCIQ0cPkV+Yf+xysoVFhWxZ\nuoUWvVocm1dQWMDRoqMUFBVQUFhAfmE++UfzyS+MXGyq+FQy8XHxJCcm06JhC1o0bEHLhi3p2Kwj\nKckp9D+1P12ad6FPmz4V6r6qre9dEFzNFSun9+ZS1SLgDBFpBrwrIr1UdUVpTct6jNGjR5OamgpA\ncnIyaWlpx9644oGv6pzOzMwMdf2lTRcL4vHX7V5HswbN6JjUMeblMzMzq+X5h/H+/bTXT6m/sT5/\nnf9XTl99OsO7DWfW7Fms272Onmf1pFNSJw5nH6ZJvSb0HtCb5MRk9q/eT5P4Jpw94GwS4hJY9fUq\n4iSO3qf15vtp32fFvBXExcXR75x+1Iuvx9K5S4mPi2fweYNpEN+ARV8ton58fS664CIa12/M559+\nXnq+c45Pf7XmK6de/8pMu/h5ihZmnoyMDCZNmgRw7P9lrIIeMxkEPKaqQ73ph4lcwWt8VJvngdmq\n+qY3nQUMKd7yiGr3a+CAqk4QkZVAuqrmikhbb/mepazfxkxC9tf5f2XuprlMumJS2FGcNevbWWRu\nzWRA+wGc2e5MGtVrFHYkU8e5eDqV+UBXEUkRkfrAKGBqiTZTgRvhWPHZ7RWJU4r30hKRhsBFHB9r\nmQqM9n6/CXgv0GdhKuzTDZ8yJGXIyRvWYRd0voAHzn6Aczuda4XE1FiBFhNVLQTGAjOA5cAUVV0p\nIneIyBivzYfAtyKSDbwA3Okt3g6YLSKZwFxgutcWYDxwkYisIrKn2O+DfB5VqeSmrQuCylQ8XnJ+\nyvkVWr4uvVaVYZn8cTETuJsrVoGPmajqNKBHiXkvlJg+4XJ0qroUOLOMx8wDfliFMU0A1u5aC8Bp\nzU8LOYkxJmh2bi4TmImLJjJj7Qz+/pO/hx3FGBMDF8dMTB32yfpPbLzEmDrCikk1c7F/NIhMhUWF\nzFw7kx+k/qDCj1FXXqvKskz+uJgJ3M0VKysmJhAz187k1Kan0uOUHidvbIyp8WzMxARi5NsjGZIy\nhDvPuvPkjY0xTrExE+OEvEN5TM+ezjWn143rnRtjrJhUOxf7R6s60xtL32BYt2E0b9i8Uo9TF16r\nqmCZ/HExE7ibK1ZWTEyVm5g5kZvTbg47hjGmGtmYialSS3KXMPyN4ay7dx3xcfFhxzHGVICL1zMx\nMXh54ct8veVrNu7dSM7eHNo3bc/Ua6YSJzVnA3LioomM/v5oKyTG1DE1579ULVFW/+jRoqPc9eFd\n9DylJ7edcRsvXfYSeYfyeH3J66FlitWRwiO8vvR1RqeNrpLHc7Ev2TL5Y5n8czVXrGzLxBHrd6+n\nTZM23D3w7mPznrjoCa7/1/Vc1fuqGnF97fdXv0/PVj3p0qJL2FGMMdXMxkwcMS17Gn/88o/MvGHm\nd+aPmDKC8zudzy8G/yKkZP4UFBbQ/2/9+dV5v+Kq3leFHccYUwl2nEkNtmbnGrq16HbC/N9d+DvG\nfzGeXYd2hZDKvwlfTqBdk3Z2rXdj6igrJtWsrP7R1TtXl1pMerXqxYgeI/j958FdsqWyfbbZedk8\nOedJ/jr8r4jE9GWmXC72JVsmfyyTf67mipUVE0esyVtD95bdS73vsfTHeGnRS2zcs7GaU52cqvKz\n93/GI+c+QufmncOOY4wJiY2ZOKLLM1346LqPyiwoj378KNsObOOly1+q5mTlm7x4Mk999RTzbp9H\nQpztz2FMbWBjJjXUkcIj5OzNoXNy2d/sfzn4l7yb9e6xqxe6YOv+rTw480H+dtnfrJAYU8dZMalm\npfWPrt21lo7NOlIvvl6Zy7Vo2II7z7qTxz99vFoynUzO3hzSJ6Vz78B76XdqvyrPBG72JVsmfyyT\nf67mipUVEwes2Vn2eEm0+wfdz9RVU8nOy66GVGVbu2st5088n1vPuJVHz3s01CzGGDfYmIkDJnw5\ngfW71/P0sKdP2vaxjMdYt3sdk66YFHywUqzcvpIfvfYjHj33UX5+1s9DyWCMCZaNmdRQq3euplvL\nE3cLLs19g+7j/dXvs2bnmoBTfVeRFjFx0UTSX0nnfy74HyskxpjvsGJSzUrrHy1vt+CSkhOTuWfg\nPfz2098GminavE3zOPvls3nh6xf44NoPuOH7N1TZuiuTKwyWyR/L5J+ruWIVeDERkaEikiUiq0Xk\noTLaPCMia0QkU0TSvHkdRGSWiCwXkaUick9U+3EikiMiC73b0KCfR5DKOvq9LPcOvJePsj9iSe6S\nwDKpKp+t/4zr/nUdI6aM4Of9f86cW+fQ/9T+ga3TGFNzBTpmIiJxwGrgQmAzMB8YpapZUW2GAWNV\ndbiIDASeVtVBItIWaKuqmSLSBPgaGKGqWSIyDtinqhNOsn7nx0wOFhykxfgWHHj0QEynbX9tyWs8\nlvEYc2+bS8tGLassz/rd63lz+Zu8vOhl4iSOW8+4ldvPvJ2kxKQqW4cxxm0uXs9kALBGVdcDiMgU\nYASQFdVmBDAZQFXnikiSiLRR1a3AVm/+fhFZCbSPWrbqztsRom/yvqFz884xX//j+r7Xs3jrYka+\nPZJp10+r8HEeew7vYc7GOUz/ZjrTv5nOzoM7uaz7ZUwcMZGzO5xdpadHMcbUXkF3c7UHos8BkuPN\nK6/NppJtRCQVSAPmRs0e63WLvSQiNeZrc8n+0VjGS0r6/Q9/T734evxiur8zCh8+epiFWxby8sKX\nuX3q7Zz+3Om0n9CeR15+hNaNW/Paj19j6y+38vKIlxnccXDohcTFvmTL5I9l8s/VXLFy/rBlr4vr\nbeBeVd3vzX4O+I2qqog8DkwAbi1t+dGjR5OamgpAcnIyaWlppKenA8ffxOqczszM/M70tKXT6Naj\nW4Ue77NPP+OuVnfxi9W/4Jm5z6DfKlv3b6Vht4Zs3b+VtYvWcqTwCE17NOXbXd+yasEq2jdrzznn\nn8PA9gPpf6Q/XXp0ISE+gfTz0snIyODT1Z+G+vpET2dmZoa6fj/vX9h5ormSx9VpFz9P0cLMk5GR\nwaRJkwCO/b+MVdBjJoOAx1R1qDf9MKCqOj6qzfPAbFV905vOAoaoaq6IJADvAx+paqkHYYhICvBv\nVe1byn3Oj5nc+t6tDGg/gDv631Hhx1i1YxVXv301Tes3JTU5ldTkVNo2aUujeo1omNCQhvUa0imp\nE71a9aoRF9kyxoTLxTGT+UBX7x/+FmAUcE2JNlOBu4A3veKzW1Vzvfv+F1hRspCISFtvTAXgSmBZ\nUE8gaGvy1nB93+sr9Rg9TunB4p8trqJExhgTu0DHTFS1EBgLzACWA1NUdaWI3CEiY7w2HwLfikg2\n8ALwcwAROQe4DrhARBaV2AX4CRFZIiKZwBDg/iCfR1UquWm7Jm+N7wMWg1IykytczGWZ/LFM/rma\nK1aBj5mo6jSgR4l5L5SYHlvKcl8Ape7ipKo3VmXGsOzN38ve/L2c2vTUsKMYY0yl2Lm5QrRwy0JG\nvzuaJT8P7uBDY4yJlZ2bq4bxe7ZgY4xxnRWTahbdP7omL7bTqATF1T5bF3NZJn8sk3+u5oqVFZMQ\nxXK2YGOMcZmNmYSo/4v9efaSZxnUYVDYUYwx5hgbM6lBirSIlTtW0vOUnmFHMcaYSrNiUs2K+0c3\n7NlA88TmTpyN19U+WxdzWSZ/LJN/ruaKlRWTkKzYvoJerXqFHcMYY6qEjZmE5A9z/kDO3hyeGvpU\n2FGMMeY7bMykBrEtE2NMbWLFpJoV94+6VExc7bN1MZdl8scy+edqrlhZMQmBqrJi+wrbk8sYU2vY\nmEkIcvbm0P/F/mz95daTNzbGmGpmYyY1hEtdXMYYUxWsmFSzjIwM54qJq322LuayTP5YJv9czRUr\nKyYhcK2YGGNMZdmYSQjO/d9zefyCx0lPTQ87ijHGnMDGTGqA4j25bMvEGFObWDGpZu9Me4c4iaNV\no1ZhRznG1T5bF3NZJn8sk3+u5oqVFZNqtm73Onq16oVITFuQxhjjNBszqWbPznuWZduW8fylz4cd\nxRhjSmVjJjWAjZcYY2ojKybVbM5nc5wrJq722bqYyzL5Y5n8czVXrAIvJiIyVESyRGS1iDxURptn\nRGSNiGSKSJo3r4OIzBKR5SKyVETuiWrfXERmiMgqEZkuIuFfYcqn4jETY4ypTQIdMxGROGA1cCGw\nGZgPjFLVrKg2w4CxqjpcRAYCT6vqIBFpC7RV1UwRaQJ8DYxQ1SwRGQ/sVNUnvALVXFUfLmX9To2Z\nbD+wnW5/7sauh3bZALwxxlkujpkMANao6npVLQCmACNKtBkBTAZQ1blAkoi0UdWtqprpzd8PrATa\nRy3zivf7K8AVwT6NqrFyx0rbk8sYUysFXUzaAxujpnM4XhDKarOpZBsRSQXSgK+8Wa1VNRdAVbcC\nrasscYC+3Pglrbe5F9XVPlsXc1kmfyyTf67milVC2AFOxuviehu4V1UPlNGszL6s0aNHk5qaCkBy\ncjJpaWmkp6cDx9/E6pp+beprDK43+Fi26l5/WdOu5SmezszMdCpPRkYGmZmZTuWJ5koeV6dd/DxF\nCzNPRkYGkyZNAjj2/zJWQY+ZDAIeU9Wh3vTDgKrq+Kg2zwOzVfVNbzoLGKKquSKSALwPfKSqT0ct\nsxJI99q09ZY/4UpTLo2ZbDuwjW5/7sa2X26jQUKDsOMYY0yZXBwzmQ90FZEUEakPjAKmlmgzFbgR\njhWf3cVdWMD/AiuiC0nUMqO9328C3gsge5X6YPUHXHTaRVZIjDG1UqDFRFULgbHADGA5MEVVV4rI\nHSIyxmvzIfCtiGQDLwA/BxCRc4DrgAtEZJGILBSRod5DjwcuEpFVRPYU+32Qz6MqTF09lct7XH7C\npq0LXMwEbuayTP5YJv9czRWrwMdMVHUa0KPEvBdKTI8tZbkvgPgyHjMP+GEVxgzUoYJDfLz2Y/52\n2d9YNm9Z2HGMMabK2bm5qsEHqz/giTlP8MnoT8KOYowxJ+XimIkBpq6ayuXdLw87hjHGBMaKScCK\ntIh/r/43l/eIFBMX+0ddzARu5rJM/lgm/1zNFStfxUREXvUzz5zo681fk5SYRLeW3cKOYowxgfE1\nZiIiC1X1zKjpeGCpqjp9xkIXxkx+NetXFBQWMP6i8SdvbIwxDqjyMRMReURE9gF9RWSvd9sHbKMG\nHNvhgqmrph7r4jLGmNqq3GKiqr9T1abAk6razLs1VdWWqvpINWWsseZtmsfOQzsZ1GHQsXku9o+6\nmAnczGWZ/LFM/rmaK1Z+B+DfF5HGACJyvYhMEJGUAHPVeKrKfdPu4/EfPE58XKmHyxhjTK3hd8xk\nCfB9oC8wCXgJuFpVhwSarpLCHDOZsmwKT855kvm3zydObKc5Y0zNEeRxJke9/8ojgGdV9S9A01gD\n1hUHCw7y0H8e4qmLn7JCYoypE/z+p9snIo8ANwAfeFdQrBdcrJrtj3P+yMD2Azkv5bwT7nOxf9TF\nTOBmLsvkj2Xyz9VcsfJ7bq6RwLXALaq6VUQ6AU8GF6vm2rR3E0/NfYoFty8IO4oxxlQb3+fmEpE2\nwFne5DxV3RZYqipS3WMmqsqof46iS/Mu/M+F/1Nt6zXGmKoU2JiJiFwNzAOuAq4G5orIT2OPWHup\nKnd/dDcb9mzg0fMeDTuOMcZUK79jJv8FnKWqN6nqjcAA4NfBxapZVJV7PrqHBZsXMO26aTSp36TM\nti72j7qYCdzMZZn8sUz+uZorVn6LSVyJbq2dMSxbqxUXkvmb5zP9+ukkJSaFHckYY6qd3+NMniRy\njMnfvVkjgSWq+lCA2Sot6DGT7Qe2c8+0e1i7ay0zrp9hhcQYUysEcW6uriJyjqo+SOSSun2925fA\nixVOWsOpKhMXTeT0v55O+6bt+fjGj62QGGPqtJN1VT0F7AVQ1X+p6gOq+gDwjndfnTM3Zy4XTr6Q\nv8z/C9Oum8YffvSHcsdISnKxf9TFTOBmLsvkj2Xyz9VcsTrZcSZtVHVpyZmqulREUgNJ5KDCokLe\nzXqXCV9NYPO+zTw4+EHu6HeHnXPLGGM85Y6ZiMgaVS31qk4ikq2qXQNLVgUqO2aSnZfNq4tf5ZXF\nr9C+WXseGPQAV3zvCisixpharSJjJifbMlkgIrer6t9KrOg24OtYA9YEuw7t4q3lbzF5yWSy87K5\n5vRr+NfIf3FmuzNPvrAxxtRRJxszuQ+4WUQyROSP3u0T4Fbg3uDjVY+jRUd5L+s9fvLWT0h9OpX/\nfPsfHj7nYXLuz+GpoU9VaSFxsX/UxUzgZi7L5I9l8s/VXLEqd8tEVXOBwSLyA+B0b/YHqjrL7wpE\nZCiRwfo44GVVPeH6tSLyDDAMOADcrKqLvPkvA5cCuaraN6r9OOB2Ild8BHhUVaf5zVTSox8/ysy1\nM7mz/528fPnLJCcmV/ShjDGmTvJ9bq4KPXjk7MKrgQuBzcB8YJSqZkW1GQaMVdXhIjIQeFpVB3n3\nnQvsByaXUkz2qeqEk6z/pGMm2XnZDHppEMvuXEbbJm0r9DyNMaY2CfJ6JhU1AFijqutVtQCYQuSa\nKNFGAJNdnAw+AAAWF0lEQVQBVHUukOSdVBJV/RzYVcZjx/REy/LgzAf55eBfWiExxphKCLqYtAc2\nRk3nePPKa7OplDalGSsimSLykohU6IjB2d/OJnNrJvcNuq8ii1eIi/2jLmYCN3NZJn8sk3+u5oqV\n3+uZuOY54DeqqiLyODCByE4BJxg9ejSpqakAJCcnk5aWRnp6OoVFhdz+59sZ3Xc0iQmJwPE3NT09\nPbDpzMzMQB+/ItPFXMlTPJ2ZmelUHnv/ava0i5+naGHmycjIYNKkSQDH/l/GKugxk0HAY6o61Jt+\nGNDoQXgReR6YrapvetNZwBBv8B8RSQH+HT1mUmIdZd5f3pjJSwtfYvLiyXwy+hNEqqTHzBhjagUX\nx0zmA11FJEVE6gOjgKkl2kwFboRjxWd3cSHxCCXGR0QkeoDjSmBZLKEOFhzk17N/zZ8u/pMVEmOM\nqQKBFhNVLQTGAjOA5cAUVV0pIneIyBivzYfAtyKSTeRkkncWLy8ibwBzgO4iskFEbvbuekJElohI\nJjAEuD+WXJ9v+JyuLbrS79R+lX2KMSu5aesCFzOBm7kskz+WyT9Xc8Uq8DET7/iPHiXmvVBiemwZ\ny15bxvwbK5Pps/WfcX6n8yvzEMYYY6IEOmYStrLGTNInpfPwuQ8ztOvQEFIZY4zbXBwzcU7+0XwW\nbF7A4I6Dw45ijDG1Rp0rJgs2L6B7y+40a9AslPW72D/qYiZwM5dl8scy+edqrljVuWLy2YbPOK/T\neWHHMMaYWqXOjZkMf2M4t6Tdwk96/SSkVMYY4zYbMzmJwqJC5mycw7mdzg07ijHG1Cp1qpgs27aM\n1o1b06ZJm9AyuNg/6mImcDOXZfLHMvnnaq5Y1aliYuMlxhgTjDo1ZjLy7ZFc0vUSbkq7KcRUxhjj\nNhszKYeq8un6TzkvxbZMjDGmqtWZYvLNrm+Il3g6J3cONYeL/aMuZgI3c1kmfyyTf67milWdKSaf\nrf+M81LOs7MEG2NMAOrMmMkt791C/1P7c+dZd55kKWOMqdtszKQcczfN5ewOZ4cdwxhjaqU6UUxU\nlfW713Na89PCjuJk/6iLmcDNXJbJH8vkn6u5YlUnisme/D3ESRxJiUlhRzHGmFqpToyZLM1dysi3\nR7LirhVhRzLGGOfZmEkZcvbm0DGpY9gxjDGm1qoTxWTj3o10aNoh7BiAm/2jLmYCN3NZJn8sk3+u\n5opVnSgmtmVijDHBqhNjJje/dzPndDyH2868LexIxhjjPBszKUPO3hw6NrMtE2OMCUqdKCYb92x0\nppvLxf5RFzOBm7kskz+WyT9Xc8Uq8GIiIkNFJEtEVovIQ2W0eUZE1ohIpoicETX/ZRHJFZElJdo3\nF5EZIrJKRKaLSJkHkKgqOXtz6NDMjQF4Y4ypjQIdMxGROGA1cCGwGZgPjFLVrKg2w4CxqjpcRAYC\nT6vqIO++c4H9wGRV7Ru1zHhgp6o+4RWo5qr6cCnr17yDeaQ+ncqeh/cE9jyNMaY2cXHMZACwRlXX\nq2oBMAUYUaLNCGAygKrOBZJEpI03/Tmwq5THHQG84v3+CnBFWQFsq8QYY4IXdDFpD2yMms7x5pXX\nZlMpbUpqraq5AKq6FWhdVsN1uzY6NfjuYv+oi5nAzVyWyR/L5J+ruWKVEHaAKlJmX93/vfe3FLXI\n57Hsx0hOTiYtLY309HTg+JtYndOZmZmhrr+06WKu5CmezszMdCqPvX81e9rFz1O0MPNkZGQwadIk\nAFJTU6mIoMdMBgGPqepQb/phQFV1fFSb54HZqvqmN50FDCne8hCRFODfJcZMVgLpqporIm295XuW\nsn69/Y1f0b5dAuPSxwX2PI0xpjZxccxkPtBVRFJEpD4wCphaos1U4EY4Vnx2FxcSj3i3ksuM9n6/\nCXivrADrd9mYiTHGBC3QYqKqhcBYYAawHJiiqitF5A4RGeO1+RD4VkSygReAY5dCFJE3gDlAdxHZ\nICI3e3eNBy4SkVVE9hT7fVkZNu135xgTcLN/1MVM4GYuy+SPZfLP1VyxCnzMRFWnAT1KzHuhxPTY\nMpa9toz5ecAP/ax/W75tmRhjTNBq/bm56o1rxM5HttK0QdOw4xhjTI3g4phJ+DTBCokxxgSs1heT\nBofdGS8BN/tHXcwEbuayTP5YJv9czRWrWl9M4vbbeIkxxgSt1o+ZNL7mNva/8bewoxhjTI1hYyal\nOJzbkVpcL40xxgm1vpjUO9SBvXvDTnGci/2jLmYCN3NZJn8sk3+u5opVrS8mzeM7kpt78nbGGGMq\nrtaPmaRdtII//3dPzj037DTGGFMz2JhJKdo37WBbJsYYE7BaX0w6tGrKtm1hpzjOxf5RFzOBm7ks\nkz+WyT9Xc8Wq1heT1q2xLRNjjAlYrR8zefZZZflyeO65sNMYY0zNYGMmpbAtE2OMCV6tLyZt2mBj\nJifhYiZwM5dl8scy+edqrljV+mJiWybGGBO8Wj9mkpendO4Mu3eHncYYY2oGGzMpRXIyHDwIhw+H\nncQYY2qvWl9MRCJdXdu3h50kwsX+URczgZu5LJM/lsk/V3PFqtYXE4gUE5cG4Y0xprap9WMmqsqw\nYXD33XDJJWEnMsYY99mYSRlsy8QYY4JVZ4qJK7sHu9g/6mImcDOXZfLHMvnnaq5YBV5MRGSoiGSJ\nyGoReaiMNs+IyBoRyRSRtJMtKyLjRCRHRBZ6t6HlZXDtwEVjjKltAh0zEZE4YDVwIbAZmA+MUtWs\nqDbDgLGqOlxEBgJPq+qg8pYVkXHAPlWdcJL1q6oyeTLMmAGvvRbI0zTGmFrFxTGTAcAaVV2vqgXA\nFGBEiTYjgMkAqjoXSBKRNj6W9f1EbcvEGGOCFXQxaQ9sjJrO8eb5aXOyZcd63WIviUhSeSFszKR8\nLmYCN3NZJn8sk3+u5opVQtgBSuFni+M54DeqqiLyODABuLW0hqNHj6Zly1S++QaeeiqZtLQ00tPT\ngeNvYnVOZ2Zmhrr+0qaLuZKneDozM9OpPPb+1expFz9P0cLMk5GRwaRJkwBITU2lIoIeMxkEPKaq\nQ73phwFV1fFRbZ4HZqvqm950FjAE6HyyZb35KcC/VbVvKetXVaWgABo1gvx8iKsT+68ZY0zFuThm\nMh/oKiIpIlIfGAVMLdFmKnAjHCs+u1U1t7xlRaRt1PJXAsvKC1GvHjRrBnl5VfGUjDHGlBRoMVHV\nQmAsMANYDkxR1ZUicoeIjPHafAh8KyLZwAvAneUt6z30EyKyREQyiWzF3H+yLCkp8NFHVfv8KqLk\npq0LXMwEbuayTP5YJv9czRWrwMdMVHUa0KPEvBdKTI/1u6w3/8ZYc0yaBBddBC1b2mlVjDGmqtWJ\nc3MVmzsXLrsM3noLvDEoY4wxJbg4ZuKUgQPhzTfh6qsjhcUYY0zVqFNbJsU++ABGjoTGjaFDh8it\nZcvInl7Rt/j44z/r1YOEhMjP+vWhYcPjtyZNoHnzyIW4mjeHdu0gMbH0TBkZGcd2zXOFi5nAzVyW\nyR/L5J+LuSqyZeLicSaBGz4c9u6NHMiYkxO55eWBKhQVQWFh5Gfx74WFUFAAR49Gfu7bFzmi/tCh\nyG3fvshlgXftijxObm6kqKSkQOfO0K9fZKvozDPDfubGGBOMOrllErTCQti6FTZsgOxsWLAg0q22\ndCn07QvXXw+jRkW2howxxjUV2TKxYlKNjhyBWbPg1VcjXW3p6fB//g8MHhx2MmOMOc4G4B1Xvz4k\nJmbw+uuRrZbhwyM7A9xwA2zZEl4uV/dzdzGXZfLHMvnnaq5YWTEJSbNmcPvtkJUV2QGgTx/4wx8i\n4zLGGFPTWDeXI1avhrvuiuwY8Oqr0L172ImMMXWVdXPVYN27w/TpcOONkTGUv/wlsneZMcbUBFZM\nqll5/aNxcZGtky++gMmTI7sTP/AATJwI8+dHdkGu7kxhcjGXZfLHMvnnaq5Y1cnjTFzXo0ekoMyY\nEdmdeNYsePppWLMGmjaFrl2hS5fIrsWNG0dujRr5O72+yPGfxb+vWRPpZiueV/KAzeKDNotviYmR\ngzUTEyPrTUqK3Bo0CO41Mca4zcZMahDVyF5f2dmR265dcOBA5Hbw4Mm7xYrvVz3x9+IDNot/Rh+w\nWXzQZkFBZPfm/Hw4fDhywOaBA7BnT+SWkACtWkH79pFbhw7QqxekpcHpp0cKjzHGfXacSQm1rZi4\nTDVS0LZtg02bYPNm2LgRli2DzExYtQpSU6Fnz+O3lJTjW1WNG0d2nY7eIpISH+XoLafS7itvujxl\nLVtyKy6WxzSmJrNiUoKLxcTF8/BUR6YjRyK7Qa9cefyWk3N8y+rAgciWT/QW0dGjGSQkRHKV3IKK\nVvItjuUtL2vZ0rbiIjKIj08v9RxuCQkn3uLjvztd3FWYkPDdIlWyWMVSuHbtyqBFi3T/C1RQWcU9\n+vfiW15eBqeemk79+pHnW/zloPiLQGnLRq8jiMKdk5NBhw7pZd5/snVW9gtKWV9SNm7MICUl/YTX\nMPpLU2mvS2IiPPSQ/0yxsHNzGWfVrx85lUzfEy6uXLaMDLcuFaAKs2fD+eeXfg63o0eP/ywo+O68\n6HO7Ff8eXSCjC1asxXDxYvj+96v++ZZcT8np0rpKi+ctXhwZ+ztyJHIrLDyxO7W0xyltXVX5HLp2\nLfu+ky0by3pKTpf1JUU1cl6/pKQTX8fiL01lfbFx7HuybZkYY4z5LjvOxBhjTCismFQzF/cpdzET\nuJnLMvljmfxzNVesrJgYY4ypNBszMcYY8x02ZmKMMSYUgRcTERkqIlkislpESt0rWkSeEZE1IpIp\nImknW1ZEmovIDBFZJSLTRSQp6OdRVVzsH3UxE7iZyzL5Y5n8czVXrAItJiISBzwLXAz0Bq4Rke+V\naDMM6KKq3YA7gOd9LPsw8B9V7QHMAh4J8nlUpczMzLAjnMDFTOBmLsvkj2Xyz9VcsQp6y2QAsEZV\n16tqATAFGFGizQhgMoCqzgWSRKTNSZYdAbzi/f4KcEWwT6Pq7N69O+wIJ3AxE7iZyzL5Y5n8czVX\nrIIuJu2BjVHTOd48P23KW7aNquYCqOpWoHUVZjbGGBMjFwfgK3JWnhqzy9a6devCjnACFzOBm7ks\nkz+WyT9Xc8VMVQO7AYOAaVHTDwMPlWjzPDAyajoLaFPessBKIlsnAG2BlWWsX+1mN7vZzW6x32L9\nfx/0iR7nA11FJAXYAowCrinRZipwF/CmiAwCdqtqrojsKGfZqcBoYDxwE/BeaSuPdT9pY4wxFRNo\nMVHVQhEZC8wg0qX2sqquFJE7Infri6r6oYhcIiLZwAHg5vKW9R56PPCWiNwCrAeuDvJ5GGOMKV+t\nPgLeGGNM9XBxAL7S/BwoWU05XhaRXBFZEjUv1AMuRaSDiMwSkeUislRE7gk7l4g0EJG5IrLIyzQu\n7ExR2eJEZKGITHUhk4isE5HF3ms1z4VMXoYkEfmHiKz0PlsDQ/5Mdfdeo4Xezz0ick/Yr5WI3C8i\ny0RkiYi8LiL1Hch0r/d3V6n/B7WumPg5ULIaTfRyRAv7gMujwAOq2hs4G7jLe31Cy6Wq+cAPVPUM\nIA0YJiIDwswU5V5gRdR02JmKgHRVPUNVBziSCeBp4ENV7Ql8n8iONGF+plZ7r9GZQD8iXejvhJlJ\nRE4F7gbOVNW+RIYZrgk5U2/gVqA/kb+9S0WkS4UyBbk3Vxg3InuBfVTeHmTVnCcFWBI1ncV390TL\nCvn1ehf4oSu5gEbAAuCssDMBHYCZQDow1YX3D/gWaFliXtiZmgHflDLflc/Uj4DPws4EnEpkjLc5\nkUIyNey/PeCnwN+ipn8FPMiJe8yeNFOt2zLB34GSYWqtjhxwKSKpRL6NfEXIB4J63UmLgK3ATFWd\nH3Ym4E9E/rCiBxbDzqTATBGZLyK3OZKpM7BDRCZ63UovikgjB3IVGwm84f0eWiZV3Qz8EdgAbAL2\nqOp/wswELAPO87q1GgGXAB0rkqk2FpOaJpQ9IESkCfA2cK+q7i8lR7XmUtUijXRzdQAGeJvfoWUS\nkeFArqpmUv6BtNX9/p2jka6bS4h0UZ5XSobqzpQAnAn8xct2gEiPQNi5EJF6wOXAP8rIUJ2fqWQi\np4JKIbKV0lhErgszk6pmEdk7dibwIbAIKCyt6ckeqzYWk01Ap6jpDt48V+R65x5DRNoC26o7gIgk\nECkkr6pq8TE6oecCUNW9QAYwNORM5wCXi8ha4O/ABSLyKrA1zNdJVbd4P7cT6aIcQPjvXQ6wUVUX\neNP/JFJcws4FMAz4WlV3eNNhZvohsFZV81S1kMgYzuCQM6GqE1W1v6qmA7uBVRXJVBuLybEDJUWk\nPpGDHaeGmEf47jfb4gMuoZwDLgP2v8AKVX06al5ouUTklOK9RUSkIXARkT7b0DKp6qOq2klVTyPy\nGZqlqjcA/w4rk4g08rYoEZHGRMYClhLyZ8rrDtkoIt29WRcCy8PO5bmGyJeBYmFm2gAMEpFEEREi\nr9OKkDMhIq28n52AHxPpEow9U3UN9FTnjci32lXAGuDhEHO8AWwG8ol8kG4mMvj2Hy/fDCC5mjOd\nQ2QzNpPIJu1C7/VqEVYuoI+XIxNYAvyXNz+0TCXyDeH4AHyYr1PnqPdtafFn24XXicgeXPO9fP8C\nksLORWRnju1A06h5YWcaR+SL0hIiZzyv50CmT4mMnSwisqdghV4nO2jRGGNMpdXGbi5jjDHVzIqJ\nMcaYSrNiYowxptKsmBhjjKk0KybGGGMqzYqJMcaYSrNiYkw5RGSf9zNFREpeJbSyj/1IienPq/Lx\njalOVkyMKV/xgVidgWtjWVBE4k/S5NHvrEj13Fge3xiXWDExxp/fAed6Z8W91zvL8RMSuahXpojc\nDiAiQ0TkUxF5j8gpRRCRd7yz/C4tPtOviPwOaOg93qvevH3FKxORJ732i0Xk6qjHni3HL0L1alT7\n33sXXcoUkSeq7VUxxhPoNeCNqUUeBn6hqpcDeMVjt6oO9M4B94WIzPDangH0VtUN3vTNqrpbRBKB\n+SLyT1V9RETu0shZdoup99g/Afqqah8Rae0t84nXJg3oReRU/V+IyGAi18O4QlW/5y3fLKgXwZiy\n2JaJMRXzI+BG7xosc4mcy6ibd9+8qEICcJ+IZBK5bkyHqHZlOQfv5ISquo3IWZTPinrsLRo5D1Im\nkArsAQ6JyEsi8mPgUCWfmzExs2JiTMUIcLdGLg17hqp20ciFjiByPY9II5EhwAXAQFVNI1IAEqMe\nw++6iuVH/V4IJGjkdOYDiFxW4FJgWszPxphKsmJiTPmK/5HvA5pGzZ8O3OldGwYR6eZdqa6kJGCX\nquaLyPeIXFa62JHi5Uus6zNgpDcu0wo4D5hXZsDIepNVdRrwANDX/9MzpmrYmIkx5Svem2sJUOR1\na01S1ae9yx4v9K5NsQ24opTlpwE/E5HlRE7n/WXUfS8CS0Tka41cK0UBVPUdERkELAaKgAdVdZuI\n9CwjWzPgPW9MBuD+ij9dYyrGTkFvjDGm0qybyxhjTKVZMTHGGFNpVkyMMcZUmhUTY4wxlWbFxBhj\nTKVZMTHGGFNpVkyMMcZUmhUTY4wxlfb/AbFYnyZdMh9SAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x108308a50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "\n",
    "#Training Data:\n",
    "X = np.array(([3,5], [5,1], [10,2], [6,1.5]), dtype=float)\n",
    "Y = np.array(([75], [82], [88], [70]), dtype=float)\n",
    "\n",
    "#Testing Data:\n",
    "testX = np.array(([4, 5.5], [4.5,1], [9,2.5], [6, 2]), dtype=float)\n",
    "testY = np.array(([70], [77], [52], [75]), dtype=float)\n",
    "\n",
    "#Normalize:\n",
    "X = X/np.amax(X, axis=0)\n",
    "Y = Y/100 #Max test score is 100\n",
    "\n",
    "#Normalize by max of training data:\n",
    "testX = testX/np.amax(X, axis=0)\n",
    "testY = testY/100 #Max test score is 100\n",
    "\n",
    "#Regularization Parameter\n",
    "Lambda = 0.0001\n",
    "print Lambda\n",
    "N = Neural_Network(Lambda)\n",
    "T = trainer(N)\n",
    "T.train(X, Y, testX, testY)\n",
    "\n",
    "%pylab inline\n",
    "plot(T.J)\n",
    "plot(T.testJ)\n",
    "grid(1)\n",
    "xlabel('Iterations')\n",
    "ylabel('Cost')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x1067effd0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEPCAYAAAB/WNKuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGupJREFUeJzt3X+Q3PV93/HnS8g4OAYJ25HkQaCLQQZMSw6mCCU41aWO\ngyQyiKYuBk+CDxpHbVBMwOMiSD1yOu0Y4aksGNWVCcRCNAm4xAQ5pkJm4OzaU4QwnMEggWgtFSlI\n2EkVYznj8uPdP76fk767c3fau9uPdvej12Nmh/1+9/Pd7+fFSvfWft773VNEYGZm1m7TOj0BMzMr\nkwuMmZll4QJjZmZZuMCYmVkWLjBmZpaFC4yZmWWRvcBIWixph6QXJd04xpjbJe2UNCzpvNr+XZK+\nJ+lpSU/U9p8saYukFyQ9LGlG7hxmZjYxWQuMpGnAOuBi4BzgSklnNY1ZApweEfOB5cB/qT38FjAQ\nEedFxILa/pXAIxFxJvAocFPGGGZmNgm538EsAHZGxO6IeB24F1jWNGYZsBEgIrYCMyTNTo9pjDku\nA+5O9+8GLmv3xM3MbGpyF5hTgJdr23vSvvHG7K2NCeAbkrZJ+kRtzKyI2A8QEfuAWW2dtZmZTdn0\nTk/gCC6KiFck/QJVodkeEd8eZZy/78bMrMvkLjB7gdNq23PTvuYxp442JiJeSf/9oaQHqJbcvg3s\nlzQ7IvZLmgO8OtrJJbnwmJlNQkRoqs+Re4lsG3CGpHmSjgeuADY1jdkEXAUgaSFwIBWOd0h6Z9r/\n88BvAN+vHTOY7n8ceHCsCUREsbdVq1Z1fA7O52zOV96tXbK+g4mINyWtALZQFbO7ImK7pOXVw3FH\nRDwkaamkl4CDwNXp8NnAA+ldyHTgzyJiS3psNfAVSdcAu4HLc+boVrt27er0FLIqOV/J2cD5rJK9\nBxMRm4Ezm/Z9qWl7xSjH/QDoH+M5/w749TZO08zM2sxX8vewwcHBTk8hq5LzlZwNnM8qaud6W7eR\nFCXnMzPLQRLRA01+y2hoaKjTU8iq5HwlZwPns4oLjJmZZeElMjMza+AlMjMz62ouMD2s9HXgkvOV\nnA2czyouMGZmloV7MGZm1sA9GDMz62ouMD2s9HXgkvOVnA2czyouMGZmloV7MGZm1sA9GDMz62ou\nMD2s9HXgkvOVnA2czyouMGZmloV7MGZm1sA9GDMz62ouMD2s9HXgkvOVnA2czyouMGZmloV7MGZm\n1sA9GDMz62ouMD2s9HXgkvOVnA2czyouMGZmloV7MGZm1sA9GDMz62ouMD2s9HXgkvOVnA2czyou\nMGZmlkX2HoykxcBaqmJ2V0SsHmXM7cAS4CAwGBHDtcemAU8CeyLi0rRvFfAJ4NU07OaI2DzK87oH\nY2Y2Qe3qwUxvx2TGkorDOuBDwN8A2yQ9GBE7amOWAKdHxHxJFwLrgYW1p7kOeB44qenp10TEmpzz\nNzOzycu9RLYA2BkRuyPideBeYFnTmGXARoCI2ArMkDQbQNJcYClw5yjPPeXq2utKXwcuOV/J2cD5\nrJK7wJwCvFzb3pP2jTdmb23MF4BPA6Otc62QNCzpTkkz2jRfMzNrk65t8ku6BNif+jGi8R3LF4H3\nRUQ/sA84JpfKBgYGOj2FrErOV3I2cD6rZO3BUL0bOa22PTftax5z6ihjPgJcKmkpcAJwoqSNEXFV\nRPywNv5PgK+NNYHBwUH6+voAmDlzJv39/Yf+cIy8zfW2t73t7WN5e2hoiA0bNgAc+nnZFhGR7QYc\nB7wEzAOOB4aBs5vGLAW+nu4vBB4f5XkWAZtq23Nq968H/nyM80fJHnvssU5PIauS85WcLcL5el36\n2TnlGpD1HUxEvClpBbCFwx9T3i5peQpwR0Q8JGmppJeoPqZ8dQtPfaukfuAtYBewPFMEMzObJH8X\nmZmZNfB3kZmZWVdzgelhI026UpWcr+Rs4HxWcYExM7Ms3IMxM7MG7sGYmVlXc4HpYaWvA5ecr+Rs\n4HxWcYExM7Ms3IMxM7MG7sGYmVlXc4HpYaWvA5ecr+Rs4HxWcYExM7Ms3IMxM7MG7sGYmVlXc4Hp\nYaWvA5ecr+Rs4HxWcYExM7Ms3IMxM7MG7sGYmVlXc4HpYaWvA5ecr+Rs4HxWcYExM7Ms3IMxM7MG\n7sGYmVlXc4HpYaWvA5ecr+Rs4HxWcYExM7Ms3IMxM7MG7sGYmVlXc4HpYaWvA5ecr+Rs4HxWcYEx\nM7Ms3IMxM7MG7sGYmVlXy15gJC2WtEPSi5JuHGPM7ZJ2ShqW1N/02DRJT0naVNt3sqQtkl6Q9LCk\nGblzdKPS14FLzldyNnA+q2QtMJKmAeuAi4FzgCslndU0ZglwekTMB5YD65ue5jrg+aZ9K4FHIuJM\n4FHgpgzTNzOzKcjag5G0EFgVEUvS9kogImJ1bcx64LGIuC9tbwcGImK/pLnAl4H/CNwQEZemMTuA\nRWnMHGAoIhoKVxrnHoyZ2QT1Sg/mFODl2vaetG+8MXtrY74AfBporhKzImI/QETsA2a1a8JmZtYe\n0zs9gbFIugTYHxHDkgaA8arpmG9TBgcH6evrA2DmzJn09/czMDAAHF5H7dXttWvXFpXnWMpXX8Pv\nhvk437Gdb2hoiA0bNgAc+nnZDkdjieyzEbE4bbeyRLYDWETVe/lt4A3gBOBE4KsRcVXTMtqcdPzZ\no5y/6CWyoaGhQ39YSlRyvpKzgfP1unYtkeUuMMcBLwAfAl4BngCujIjttTFLgWsj4pJUkNZGxMKm\n51kEfKrWg1kN/F1ErE6fTDs5IlaOcv6iC4yZWQ7tKjBZl8gi4k1JK4AtVP2euyJiu6Tl1cNxR0Q8\nJGmppJeAg8DVLTz1auArkq4BdgOX58pgZmaT4yv5e1jpb9NLzldyNnC+XtcrnyIzM7NjlN/BmJlZ\nA7+DMTOzruYC08Pqn8UvUcn5Ss4GzmcVFxgzM8vCPRgzM2vgHoyZmXU1F5geVvo6cMn5Ss4GzmcV\nFxgzM8vCPRgzM2vgHoyZmXU1F5geVvo6cMn5Ss4GzmcVFxgzM8vCPRgzM2vgHoyZmXU1F5geVvo6\ncMn5Ss4GzmcVFxgzM8vCPRgzM2vgHoyZmXU1F5geVvo6cMn5Ss4GzmcVFxgzM8vCPRgzM2vgHoyZ\nmXU1F5geVvo6cMn5Ss4GzmeVlgqMpHta2WdmZjaipR6MpKci4vza9nHAsxHxgZyTmyr3YMzMJu6o\n9GAk3STpNeBcST9Ot9eAV4EHp3pyMzMr17gFJiI+FxEnAp+PiJPS7cSIeHdE3HSU5mhjKH0duOR8\nJWcD57NKq03+v5b08wCSflvSGknzWjlQ0mJJOyS9KOnGMcbcLmmnpGFJ/Wnf2yVtlfS0pGclraqN\nXyVpj6Sn0m1xiznMzOwoabUH8wzwS8C5wAbgTuDyiFh0hOOmAS8CHwL+BtgGXBERO2pjlgArIuIS\nSRcCt0XEwvTYOyLip6nn8x3gkxHxRCo2r0XEmiOc3z0YM7MJOtrXwbyRflIvA9ZFxH8GTmzhuAXA\nzojYHRGvA/em56hbBmwEiIitwAxJs9P2T9OYtwPTgXq1mHJ4MzPLp9UC85qkm4DfAb6e3pm8rYXj\nTgFerm3vSfvGG7N3ZIykaZKeBvYB34iIbbVxK9KS2p2SZrSYoyilrwOXnK/kbOB8Vpne4riPAh8D\nromIfZJOAz6fb1qViHgLOE/SScBfSfpARDwPfBH49xERkv4DsAb4V6M9x+DgIH19fQDMnDmT/v5+\nBgYGgMN/SHp1e3h4uKvm43ze9nZvbg8NDbFhwwaAQz8v26Hl7yJLy1YXpM0nIuLVFo5ZCHw2Ihan\n7ZVARMTq2pj1wGMRcV/a3gEsioj9Tc/1GeBgc98lfdjgaxFx7ijndw/GzGyCjmoPRtLlwBPAvwQu\nB7ZK+kgLh24DzpA0T9LxwBXApqYxm4Cr0nkWAgciYr+k94wsfUk6AfgwsCNtz6kd/1vA91vJYWZm\nR0+rPZg/Ai6IiI9HxFVUzfvPHOmgiHgTWAFsAZ4D7o2I7ZKWS/q9NOYh4AeSXgK+BPx+Ovy9wGOS\nhoGtwMNpLMCtkp5Jjy0Crm8xR1FG3uKWquR8JWcD57NKqz2YaU1LYn9Li8UpIjYDZzbt+1LT9opR\njnsWOL95f3rsqlbObWZmndPqdTCfp7oG5i/Sro8Cz0TEqBdOdgv3YMzMJq5dPZhxC4ykM4DZEfEd\nSb8FfDA9dAD4s4j4X1OdQE4uMGZmE3e0mvxrgR8DRMRXI+KGiLgBeCA9Zh1U+jpwyflKzgbOZ5Uj\nFZjZqRfSIO3ryzIjMzMrwpGWyHZGxPwxHnspIs7INrM28BKZmdnEHa0lsiclfWKUk/8u8N2pntzM\nzMp1pALzh8DVkoYk/ad0+ybV17Jcl396Np7S14FLzldyNnA+q4x7HUz6upZfkfRrwD9Ku78eEY9m\nn5mZmfW0lr+LrBe5B2NmNnFH+/fBmJmZTYgLTA8rfR245HwlZwPns4oLjJmZZeEejJmZNXAPxszM\nupoLTA8rfR245HwlZwPns4oLjJmZZeEejJmZNXAPxszMupoLTA8rfR245HwlZwPns4oLjJmZZeEe\njJmZNXAPxszMupoLTA8rfR245HwlZwPns4oLjJmZZeEejJmZNXAPxszMupoLTA8rfR245HwlZwPn\ns4oLjJmZZZG9ByNpMbCWqpjdFRGrRxlzO7AEOAgMRsSwpLcD3wKOB6YD90fEH6fxJwP3AfOAXcDl\nEfH3ozyvezBmZhPUEz0YSdOAdcDFwDnAlZLOahqzBDg9IuYDy4H1ABHxM+DXIuI8oB9YImlBOmwl\n8EhEnAk8CtyUM4eZmU1c7iWyBcDOiNgdEa8D9wLLmsYsAzYCRMRWYIak2Wn7p2nM26nexUTtmLvT\n/buBy7Il6GKlrwOXnK/kbOB8VsldYE4BXq5t70n7xhuzd2SMpGmSngb2Ad+IiG1pzKyI2A8QEfuA\nWRnmbmZmUzC90xMYT0S8BZwn6STgryR9ICKeH23oWM8xODhIX18fADNnzqS/v5+BgQHg8L9CenV7\nZF+3zMf5Wt8eGBjoqvk437Gdb2hoiA0bNgAc+nnZDlmb/JIWAp+NiMVpeyUQ9Ua/pPXAYxFxX9re\nASwaeYdSG/cZ4GBErJG0HRiIiP2S5qTjzx7l/G7ym5lNUE80+YFtwBmS5kk6HrgC2NQ0ZhNwFRwq\nSAdS4XiPpBlp/wnAh4EdtWMG0/2PAw9mTdGlRv4FUqqS85WcDZzPKlmXyCLiTUkrgC0c/pjydknL\nq4fjjoh4SNJSSS9RfUz56nT4e4G70yfRpgH3RcRD6bHVwFckXQPsBi7PmcPMzCbO30VmZmYNemWJ\nzMzMjlEuMD2s9HXgkvOVnA2czyouMGZmloV7MGZm1sA9GDMz62ouMD2s9HXgkvOVnA2czyouMGZm\nloV7MGZm1sA9GDMz62ouMD2s9HXgkvOVnA2czyouMGZmloV7MGZm1sA9GDMz62ouMD2s9HXgkvOV\nnA2czyouMGZmloV7MGZm1sA9GDMz62ouMD2s9HXgkvOVnA2czyouMGZmloV7MGZm1sA9GDMz62ou\nMD2s9HXgkvOVnA2czyouMGZmloV7MGZm1sA9GDMz62ouMD2s9HXgkvOVnA2czyouMGZmloV7MGZm\n1qBnejCSFkvaIelFSTeOMeZ2STslDUvqT/vmSnpU0nOSnpX0ydr4VZL2SHoq3RbnzmFmZhOTtcBI\nmgasAy4GzgGulHRW05glwOkRMR9YDqxPD70B3BAR5wC/DFzbdOyaiDg/3TbnzNGtSl8HLjlfydnA\n+ayS+x3MAmBnROyOiNeBe4FlTWOWARsBImIrMEPS7IjYFxHDaf9PgO3AKbXjpvz2zczM8sldYE4B\nXq5t76GxSIw2Zm/zGEl9QD+wtbZ7RVpSu1PSjHZNuJcMDAx0egpZlZyv5GzgfFbp+k+RSXoncD9w\nXXonA/BF4H0R0Q/sA9Z0an5mZja66Zmffy9wWm17btrXPObU0cZImk5VXO6JiAdHBkTED2vj/wT4\n2lgTGBwcpK+vD4CZM2fS399/6F8fI+uovbq9du3aovIcS/nqa/jdMB/nO7bzDQ0NsWHDBoBDPy/b\nIevHlCUdB7wAfAh4BXgCuDIittfGLAWujYhLJC0E1kbEwvTYRuBHEXFD0/POiYh96f71wAUR8bFR\nzl/0x5SHhoYO/WEpUcn5Ss4Gztfr2vUx5ezXwaSPEN9GtRx3V0TcImk5EBFxRxqzDlgMHAQGI+Jp\nSRcB3wKeBSLdbo6Izanw9ANvAbuA5RGxf5RzF11gzMxy6JkC00kuMGZmE9czF1paPvV14BKVnK/k\nbOB8VnGBMTOzLLxEZmZmDbxEZmZmXc0FpoeVvg5ccr6Ss4HzWcUFxszMsnAPxszMGrgHY2ZmXc0F\npoeVvg5ccr6Ss4HzWcUFxszMsnAPxszMGrgHY2ZmXc0FpoeVvg5ccr6Ss4HzWcUFxszMsnAPxszM\nGrgHY2ZmXc0FpoeVvg5ccr6Ss4HzWcUFxszMsnAPxszMGrgHY2ZmXc0FpoeVvg5ccr6Ss4HzWcUF\nxszMsnAPxszMGrgHY2ZmXc0FpoeVvg5ccr6Ss4HzWcUFxszMsnAPxszMGrgHY2ZmXS17gZG0WNIO\nSS9KunGMMbdL2ilpWFJ/2jdX0qOSnpP0rKRP1safLGmLpBckPSxpRu4c3aj0deCS85WcDZzPKlkL\njKRpwDrgYuAc4EpJZzWNWQKcHhHzgeXA+vTQG8ANEXEO8MvAtbVjVwKPRMSZwKPATTlzdKvh4eFO\nTyGrkvOVnA2czyq538EsAHZGxO6IeB24F1jWNGYZsBEgIrYCMyTNjoh9ETGc9v8E2A6cUjvm7nT/\nbuCyvDG604EDBzo9haxKzldyNnA+q+QuMKcAL9e293C4SIw1Zm/zGEl9QD/weNo1KyL2A0TEPmBW\n22ZsZmZt0fVNfknvBO4HrouIg2MMOyY/KrZr165OTyGrkvOVnA2cz5KIyHYDFgKba9srgRubxqwH\nPlrb3gHMTvenA5upikv9mO21MXOA7WOcP3zzzTfffJv4rR01YDp5bQPOkDQPeAW4Ariyacwm4Frg\nPkkLgQMjy1/AnwLPR8RtoxwzCKwGPg48ONrJ2/E5bjMzm5zsF1pKWgzcRrUcd1dE3CJpOVWFvCON\nWQcsBg4CgxHxtKSLgG8Bz3K4qt4cEZslvQv4CnAqsBu4PCLcdTMz6yJFX8lvZmad0/VN/iNp9aLL\n8S74lPQHkranCzpvOTozP7J2ZEuPf0rSW+mdX9eYaj5Jt6bXbVjSX0o66ejNfmyTvbi41WM7LcfF\n091iKq9demyapKckbTo6M56YKf7ZnCHpv6W/c89JuvCIJ8zZ5D8aN6o+zL9N928EbhllzDTgJWAe\n8DZgGDgrPTYAbAGmp+33dDpTu7Klx+dSfVDiB8C7Op2pza/drwPT0v1bgM91QaZxX480Zgnw9XT/\nQuDxVo/t9G2K+eYA/en+O4EXuinfVLLVHr8e+K/Apk7naXc+YANwdbo/HTjpSOfs+XcwtHbR5XgX\nfP4bqh9sbwBExI8yz3cippoN4AvAp7POcvKmlC8iHomIt9K4x6mKaadN+uLiFo/ttFwXT3eDqbx2\nSJoLLAXuPHpTnpBJ50urA78aEV9Oj70RET8+0glLKDCtXHQ53gWf7wf+qaTHJT0m6Z9kne3ETCmb\npEuBlyPi2dwTnaSpvnZ11wD/ve0znLjJXFw8MqbVrJ3U7ount7Z9hpM31Wwj/5jr1sb2VPL9IvAj\nSV9OS4B3SDrhSCfM/THltpD0DWB2fRfVi/jvRhk+0Rd3OnByRCyUdAHVp9PeN6mJTkKubOnFvxn4\ncNNzH1WZX7uRc/wR8HpE/Plkju8Cx9TH6Zsunv5Jp+fTDpIuAfZHxLCkAcp7TacD5wPXRsSTktZS\nXde46kgHdb2I+PBYj0nan95+75c0B3h1lGF7gdNq23PTPqiq+FfTebalZvi7I+Jv2zT9cWXMdjrQ\nB3xPktL+70paEBGjPU8WmV87JA1SLUv8s/bMeMrGnW9tzKmjjDm+hWM7bSr5kDSdqrjcExGjXr/W\nQVPJ9hHgUklLgROAEyVtjIirMs53oqb02lGthjyZ7t9P1TcdX6cbT21oXK0mfTsAYzeKj+Nwc+t4\nqubW2emx5cAfp/vvB3Z3OlO7sjWN+wHVO7WO52rja7cYeA54d6ezTOT1oCqII43UhRxugrf0WvZq\nvrS9EVjT6Rw5stXGLKI7m/xTfe2+Cbw/3V8FrD7iOTsdug3/094FPEL1iZQtwMy0/73AX9fGLU5j\ndgIra/vfBtxDdUHnk8CiTmdqV7am5/rfdN+nyKb62u2kutD2qXT7YqczjTVfqn/I/F5tzLr0l/17\nwPkTeS07fZtEvvPSvouAN9MPtqfTa7a403na9drVHu/KAtOGP5u/RPXtLMNUqz4zjnQ+X2hpZmZZ\nlPApMjMz60IuMGZmloULjJmZZeECY2ZmWbjAmJlZFi4wZmaWhQuM2TgkvZb+O09S829jnepz39S0\n/e12Pr9Zp7nAmI1v5EKxXwQ+NpEDJR13hCE3N5wo4oMTeX6zbucCY9aazwEfTN8ke136xVK3Stqa\nfjHTJwAkLZL0LUkPUn2NDZIekLQt/ZKt3037PgeckJ7vnrTvtZGTSfp8Gv89SZfXnvux2i99uqc2\n/hZJ309zufWo/V8xG0dPfNmlWRdYCXwqIi4FSAXlQERcKOl44DuStqSx5wHnRMT/SdtXR8QBST8H\nbJP0lxFxk6RrI+L82jkiPfe/AM6NiH8saVY65ptpTD/wAWBfOuevADuAyyLirHR8V/xmTzO/gzGb\nnN8ArpL0NNXvNHkXMD899kStuAD8oaRhDv9StPmM7yLgLwCi+ubrIeCC2nO/EtV3PA1TfWP23wP/\nIOlOSf8c+IcpZjNrCxcYs8kR8AcRcV66nR4Rj6THDh4aJC2i+lUCF0ZEP1VR+Lnac7R6rhE/q91/\nk+pXfb9J9dsK7wd+k+pXZJt1nAuM2fhGfri/BpxY2/8w8Pvp95sgab6kd4xy/Azg/0bEzySdRfUV\n6CP+38jxTef6H8BHU5/nF4BfBZ4Yc4LVeWdGxGbgBuDc1uOZ5eMejNn4Rj5F9gzwVloS2xARt6Vf\n+/tU+oVurwKXjXL8ZuBfS3qO6mvS/2ftsTuAZyR9NyJ+Z+RcEfGApIVUX5f+FvDpiHhV0tljzO0k\n4MHU4wG4fvJxzdrHX9dvZmZZeInMzMyycIExM7MsXGDMzCwLFxgzM8vCBcbMzLJwgTEzsyxcYMzM\nLAsXGDMzy+L/A6KVvkGIayMyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1048bf7d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Plot cost during training:\n",
    "%pylab inline\n",
    "plot(T.J)\n",
    "plot(T.testJ)\n",
    "grid(1)\n",
    "xlabel('Iterations')\n",
    "ylabel('Cost')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Test network for various combinations of sleep/study:\n",
    "hoursSleep = linspace(0, 10, 100)\n",
    "hoursStudy = linspace(0, 5, 100)\n",
    "\n",
    "#Normalize data (same way training data way normalized)\n",
    "hoursSleepNorm = hoursSleep/10.\n",
    "hoursStudyNorm = hoursStudy/5.\n",
    "\n",
    "#Create 2-d versions of input for plotting\n",
    "a, b  = meshgrid(hoursSleepNorm, hoursStudyNorm)\n",
    "\n",
    "#Join into a single input matrix:\n",
    "allInputs = np.zeros((a.size, 2))\n",
    "allInputs[:, 0] = a.ravel()\n",
    "allInputs[:, 1] = b.ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "allOutputs = NN.forward(allInputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Contour Plot:\n",
    "yy = np.dot(hoursStudy.reshape(100,1), np.ones((1,100)))\n",
    "xx = np.dot(hoursSleep.reshape(100,1), np.ones((1,100))).T\n",
    "\n",
    "CS = contour(xx,yy,100*allOutputs.reshape(100, 100))\n",
    "clabel(CS, inline=1, fontsize=10)\n",
    "xlabel('Hours Sleep')\n",
    "ylabel('Hours Study')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#3D plot:\n",
    "#Uncomment to plot out-of-notebook (you'll be able to rotate)\n",
    "%matplotlib qt\n",
    "\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "fig = plt.figure()\n",
    "ax = fig.gca(projection='3d')\n",
    "\n",
    "#Scatter training examples:\n",
    "ax.scatter(10*X[:,0], 5*X[:,1], 100*Y, c='k', alpha = 1, s=30)\n",
    "\n",
    "\n",
    "surf = ax.plot_surface(xx, yy, 100*allOutputs.reshape(100, 100), \\\n",
    "                       cmap=cm.jet, alpha = 0.5)\n",
    "\n",
    "\n",
    "ax.set_xlabel('Hours Sleep')\n",
    "ax.set_ylabel('Hours Study')\n",
    "ax.set_zlabel('Test Score')"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}