Whats in the weights of a randomly initialized nn/Paper Implementation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# What is hidden in a randomly weighted neural network?\n",
    "\n",
    "This is a small implementation of a fully connected neural network in numpy/cupy ( i switched to the latter since it took a long time on CPU, pretty interchangeable save for some parts of the code where the cupy array has to be converted to numpy array)\n",
    "\n",
    "TL ; DR of the paper : Chonky neural nets contains wisdom. \n",
    "\n",
    "The paper explores the posibility of finding a good subnetwork inside a randomly initialized neural network that could perform well on a certain task. How the authors approach finding this subnetwork is by assigning a score corresponding to each weights, and during feedforward, only weights with a top K % score is considered. Optimization is done on the score, while weights are kept constant.\n",
    "\n",
    "In this implementation, i observe how both approaches perform on a similar task : Digits classification in MNIST."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import numpy as np\n",
    "import cupy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(69420)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implementation : Baseline\n",
    "\n",
    "Since we need a comparison, we will first see how would a fully connected NN trained with normal backprop perform."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def glorot_init(shape):\n",
    "    fan_out,fan_in = shape\n",
    "    limit = np.sqrt(6/(fan_in+fan_out))\n",
    "    params = np.random.uniform(low=limit*-1,high=limit,size=shape)\n",
    "    return params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def kaiming_uniform(shape):\n",
    "    fan_out, fan_in = shape\n",
    "    params = np.random.normal(loc=0,scale=np.sqrt(2/fan_in),size=shape)\n",
    "    return params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def relu(a,prime=False):\n",
    "    if not prime:\n",
    "        mask = a > 0\n",
    "        return a * mask\n",
    "    else:\n",
    "        mask = a > 0\n",
    "        return mask.astype(np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(a,prime=False):\n",
    "    a = np.exp(a)\n",
    "    a = a / np.sum(a,axis=0)\n",
    "    return a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def targets_to_onehot(targets,numclass):\n",
    "    onehot = np.zeros((numclass,len(targets)))\n",
    "    onehot[targets,np.arange(len(targets))] = 1\n",
    "    return onehot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def crossentropy_with_logits(logits_out, y,prime=False):\n",
    "    num_classes, num_samples = logits_out.shape\n",
    "    y_onehot = targets_to_onehot(y,num_classes)\n",
    "    assert logits_out.shape == y_onehot.shape\n",
    "    softmaxed = softmax(logits_out)\n",
    "    if prime:\n",
    "        softmaxed[y,np.arange(num_samples)] -=1\n",
    "        return softmaxed\n",
    "    else:\n",
    "        return np.sum(np.nan_to_num(y_onehot * np.log(softmaxed) + (1-y_onehot) * np.log((1-softmaxed))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SGDOptimizer(gradients,parameters,learningrate):\n",
    "    parameters = [p - learningrate * g for p,g in zip(parameters,gradients)]\n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def MomentumSGD(lr,m):\n",
    "    vl = None\n",
    "    def optimizer(grads,params):\n",
    "        nonlocal vl\n",
    "        if vl is None:\n",
    "            vl = [np.zeros(p.shape) for p in params]\n",
    "        vl = [m * v + lr * g for v,g in zip(vl,grads)]\n",
    "        params = [p-v for p,v in zip(params,vl)]\n",
    "        return params    \n",
    "    return optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FCNN(object):\n",
    "    \n",
    "    def __init__(self, layers_size,initializer_weight,initializer_bias,activation,loss_function,optimizer_w,optimizer_b):\n",
    "        self.layers_size = layers_size\n",
    "        self.weights = [initializer_weight((k,j)) for j,k in zip(layers_size[:-1],layers_size[1:])]\n",
    "        self.biases = [initializer_bias((k,1)) for k in layers_size[1:]]\n",
    "        self.activation = activation\n",
    "        self.loss_function = loss_function\n",
    "        self.optimizer_w = optimizer_w\n",
    "        self.optimizer_b = optimizer_b\n",
    "    \n",
    "    def feedforward(self,X):\n",
    "        a = X\n",
    "        zs = []\n",
    "        acs = [a]\n",
    "        for w,b in zip(self.weights,self.biases):\n",
    "            a = np.dot(w,a) + b\n",
    "#             print(a.shape)\n",
    "            zs.append(a)\n",
    "            a = self.activation(a)\n",
    "            acs.append(a)\n",
    "        return a, zs, acs\n",
    "    \n",
    "    def feedforward_softmax(self,X):\n",
    "        final_a, _, _ = self.feedforward(X)\n",
    "        final_a = np.exp(final_a)\n",
    "#         print(final_a.shape)\n",
    "        final_a = final_a / np.sum(final_a,axis=0)\n",
    "        return final_a\n",
    "    \n",
    "    def backpropagation(self,X,y):\n",
    "        input_dim, batch_size = X.shape\n",
    "        final_activation, zs, acs = self.feedforward(X)\n",
    "        delta_w = [np.zeros(w.shape) for w in self.weights]\n",
    "        delta_b = [np.zeros(b.shape) for b in self.biases]\n",
    "        error = self.loss_function(final_activation,y,prime=True) * self.activation(zs[-1],True)\n",
    "        delta_w[-1] += np.einsum('ik,jk->ijk',error,acs[-2]).mean(axis=2)\n",
    "        delta_b[-1] += np.expand_dims(error,1).mean(axis=2)\n",
    "        for i in range(2,len(self.layers_size)):\n",
    "            error = np.einsum('ik,ij->jk',error,self.weights[-i+1]) * self.activation(zs[-i],prime=True)\n",
    "            delta_w[-i] += np.einsum('ik,jk->ijk',error,acs[-i-1]).mean(axis=2)\n",
    "            delta_b[-i] += np.expand_dims(error,1).mean(axis=2) \n",
    "        return delta_w, delta_b\n",
    "    \n",
    "    def fit(self,X,y,batch_size,epochs):\n",
    "        # expecting X and y to be already numpy array with the num_samples as \n",
    "        # 1st dim\n",
    "        history = np.zeros(epochs + 1)\n",
    "        history -= 1\n",
    "        index = 0\n",
    "        for epoch in range(epochs):\n",
    "            indices = np.arange(X.shape[0],dtype=np.int32)\n",
    "            np.random.shuffle(indices)\n",
    "            X = X.copy()[indices,:]\n",
    "            y = y.copy()[indices]\n",
    "            i = 0\n",
    "            batches_X = [X[k:k+batch_size] for k in range(0,X.shape[0],batch_size)]\n",
    "            batches_y = [y[k:k+batch_size] for k in range(0,y.shape[0],batch_size)]\n",
    "            losses = 0\n",
    "            if index == 0:\n",
    "                for Xb, yb in zip(batches_X,batches_y):\n",
    "                    logits,_,_ = self.feedforward(Xb.T)\n",
    "                    history[0] += self.loss_function(logits,yb.T) / (Xb.shape[0] * len(batches_X))\n",
    "            for Xb, yb in zip(batches_X,batches_y):\n",
    "                delta_w, delta_b = self.backpropagation(Xb.T, yb.T)\n",
    "#                 print(delta_w)\n",
    "                self.weights = self.optimizer_w(delta_w,self.weights)\n",
    "                self.biases = self.optimizer_b(delta_b,self.biases)\n",
    "                logits,_,_ = self.feedforward(Xb.T)\n",
    "                losses += self.loss_function(logits,yb.T) / (Xb.shape[0] * len(batches_X))\n",
    "#                 print((Xb.shape[0] * len(batches_X)))\n",
    "            history[index] = losses\n",
    "            index += 1\n",
    "            print(f\"Epoch {epoch} done. Loss : {losses}\")\n",
    "        return history\n",
    "    \n",
    "    def predict(self,X):\n",
    "        softmaxed = self.feedforward_softmax(X.T)\n",
    "        preds = np.argmax(softmaxed,axis=0)\n",
    "        return preds\n",
    "    \n",
    "    def score(self, X, y):\n",
    "        preds = self.predict(X)\n",
    "        assert preds.shape == y.shape\n",
    "        return np.mean(preds == y) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist_data = np.load('../Dataset/Mnist/train.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist_label = mnist_data[:,0]\n",
    "mnist_pixels = mnist_data[:,1:] / 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "smol_batch_pixels = mnist_pixels[:32,:]\n",
    "smol_batch_label = mnist_label[:32].astype(np.int32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_optimizer = MomentumSGD(0.001,0.9)\n",
    "bias_optimizer = MomentumSGD(0.001,0.9)\n",
    "nn_overfit = FCNN([784,100,100,10],kaiming_uniform,kaiming_uniform,relu,crossentropy_with_logits,weight_optimizer,bias_optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 done. Loss : -3.749897180894854\n",
      "Epoch 1 done. Loss : -3.682271576986854\n",
      "Epoch 2 done. Loss : -3.6035557455315725\n",
      "Epoch 3 done. Loss : -3.524958694301793\n",
      "Epoch 4 done. Loss : -3.4521233438296495\n",
      "Epoch 5 done. Loss : -3.3863179595712936\n",
      "Epoch 6 done. Loss : -3.326984844858233\n",
      "Epoch 7 done. Loss : -3.272507119815727\n",
      "Epoch 8 done. Loss : -3.2225475436691395\n",
      "Epoch 9 done. Loss : -3.1742886049148593\n",
      "Epoch 10 done. Loss : -3.1295937742786153\n",
      "Epoch 11 done. Loss : -3.090815039681315\n",
      "Epoch 12 done. Loss : -3.0583340538066777\n",
      "Epoch 13 done. Loss : -3.0299311680301093\n",
      "Epoch 14 done. Loss : -3.0065050339187893\n",
      "Epoch 15 done. Loss : -2.9877819564389\n",
      "Epoch 16 done. Loss : -2.9730377988787353\n",
      "Epoch 17 done. Loss : -2.9607982252199583\n",
      "Epoch 18 done. Loss : -2.9515256536750374\n",
      "Epoch 19 done. Loss : -2.944579446683294\n",
      "Epoch 20 done. Loss : -2.93923338426746\n",
      "Epoch 21 done. Loss : -2.932552790172003\n",
      "Epoch 22 done. Loss : -2.922423843219667\n",
      "Epoch 23 done. Loss : -2.90843088878648\n",
      "Epoch 24 done. Loss : -2.8909462988632404\n",
      "Epoch 25 done. Loss : -2.8710966714481536\n",
      "Epoch 26 done. Loss : -2.850119969598677\n",
      "Epoch 27 done. Loss : -2.8297825866825805\n",
      "Epoch 28 done. Loss : -2.810511507589215\n",
      "Epoch 29 done. Loss : -2.7906826594448524\n",
      "Epoch 30 done. Loss : -2.77229375844108\n",
      "Epoch 31 done. Loss : -2.7552891830881965\n",
      "Epoch 32 done. Loss : -2.739940104924739\n",
      "Epoch 33 done. Loss : -2.7261564933495137\n",
      "Epoch 34 done. Loss : -2.713815785426376\n",
      "Epoch 35 done. Loss : -2.7020775157332193\n",
      "Epoch 36 done. Loss : -2.690935949169175\n",
      "Epoch 37 done. Loss : -2.6801027914635753\n",
      "Epoch 38 done. Loss : -2.6695324993687723\n",
      "Epoch 39 done. Loss : -2.6590850366538428\n",
      "Epoch 40 done. Loss : -2.648748829863668\n",
      "Epoch 41 done. Loss : -2.638568004964945\n",
      "Epoch 42 done. Loss : -2.628558046570406\n",
      "Epoch 43 done. Loss : -2.618958086882685\n",
      "Epoch 44 done. Loss : -2.6097763854353015\n",
      "Epoch 45 done. Loss : -2.600895531726861\n",
      "Epoch 46 done. Loss : -2.5921377734273525\n",
      "Epoch 47 done. Loss : -2.5834768370827588\n",
      "Epoch 48 done. Loss : -2.5749047785508514\n",
      "Epoch 49 done. Loss : -2.5663823773205596\n",
      "Epoch 50 done. Loss : -2.558015904660372\n",
      "Epoch 51 done. Loss : -2.5495576196170076\n",
      "Epoch 52 done. Loss : -2.5409952995971636\n",
      "Epoch 53 done. Loss : -2.532444321061992\n",
      "Epoch 54 done. Loss : -2.5238455598952267\n",
      "Epoch 55 done. Loss : -2.5152722784375756\n",
      "Epoch 56 done. Loss : -2.5067094383161717\n",
      "Epoch 57 done. Loss : -2.498184141218182\n",
      "Epoch 58 done. Loss : -2.489801632740854\n",
      "Epoch 59 done. Loss : -2.481572138455064\n",
      "Epoch 60 done. Loss : -2.4735170333566234\n",
      "Epoch 61 done. Loss : -2.465593230477081\n",
      "Epoch 62 done. Loss : -2.4577860636856323\n",
      "Epoch 63 done. Loss : -2.4501007111815327\n",
      "Epoch 64 done. Loss : -2.4425124644644214\n",
      "Epoch 65 done. Loss : -2.4349689085086137\n",
      "Epoch 66 done. Loss : -2.427545283001365\n",
      "Epoch 67 done. Loss : -2.4202716760317786\n",
      "Epoch 68 done. Loss : -2.413162617312104\n",
      "Epoch 69 done. Loss : -2.4061622521973636\n",
      "Epoch 70 done. Loss : -2.3992921906700766\n",
      "Epoch 71 done. Loss : -2.3925295042732464\n",
      "Epoch 72 done. Loss : -2.3858701272651524\n",
      "Epoch 73 done. Loss : -2.379394441509174\n",
      "Epoch 74 done. Loss : -2.3729992157750432\n",
      "Epoch 75 done. Loss : -2.3666351018918514\n",
      "Epoch 76 done. Loss : -2.3603234225735106\n",
      "Epoch 77 done. Loss : -2.3540756822270152\n",
      "Epoch 78 done. Loss : -2.34785219919553\n",
      "Epoch 79 done. Loss : -2.341668397380067\n",
      "Epoch 80 done. Loss : -2.335538427199446\n",
      "Epoch 81 done. Loss : -2.3294603647092598\n",
      "Epoch 82 done. Loss : -2.323453528515503\n",
      "Epoch 83 done. Loss : -2.3175435484226075\n",
      "Epoch 84 done. Loss : -2.3116772803409873\n",
      "Epoch 85 done. Loss : -2.305893689688614\n",
      "Epoch 86 done. Loss : -2.3001511457859034\n",
      "Epoch 87 done. Loss : -2.294420267210267\n",
      "Epoch 88 done. Loss : -2.288699168714569\n",
      "Epoch 89 done. Loss : -2.28302749153894\n",
      "Epoch 90 done. Loss : -2.2774114774614826\n",
      "Epoch 91 done. Loss : -2.271835578905523\n",
      "Epoch 92 done. Loss : -2.266267181991543\n",
      "Epoch 93 done. Loss : -2.2607358115366623\n",
      "Epoch 94 done. Loss : -2.2552444620977234\n",
      "Epoch 95 done. Loss : -2.249822558442805\n",
      "Epoch 96 done. Loss : -2.2444664941855503\n",
      "Epoch 97 done. Loss : -2.2391655490229168\n",
      "Epoch 98 done. Loss : -2.2339177889757416\n",
      "Epoch 99 done. Loss : -2.2287484920555958\n",
      "Epoch 100 done. Loss : -2.223654500320441\n",
      "Epoch 101 done. Loss : -2.218622905458489\n",
      "Epoch 102 done. Loss : -2.2136795145752894\n",
      "Epoch 103 done. Loss : -2.2087810464752202\n",
      "Epoch 104 done. Loss : -2.203919578806706\n",
      "Epoch 105 done. Loss : -2.199110930239043\n",
      "Epoch 106 done. Loss : -2.194354473105374\n",
      "Epoch 107 done. Loss : -2.189643123928097\n",
      "Epoch 108 done. Loss : -2.184975532259253\n",
      "Epoch 109 done. Loss : -2.1803770672825653\n",
      "Epoch 110 done. Loss : -2.175845306228579\n",
      "Epoch 111 done. Loss : -2.1713784118082584\n",
      "Epoch 112 done. Loss : -2.166978252103824\n",
      "Epoch 113 done. Loss : -2.162587506942403\n",
      "Epoch 114 done. Loss : -2.158205648881474\n",
      "Epoch 115 done. Loss : -2.1538445827620736\n",
      "Epoch 116 done. Loss : -2.1495203905865545\n",
      "Epoch 117 done. Loss : -2.145220905796042\n",
      "Epoch 118 done. Loss : -2.140969934768669\n",
      "Epoch 119 done. Loss : -2.136769421377502\n",
      "Epoch 120 done. Loss : -2.1326440955545585\n",
      "Epoch 121 done. Loss : -2.1285826851928977\n",
      "Epoch 122 done. Loss : -2.1245338415137365\n",
      "Epoch 123 done. Loss : -2.120511659062371\n",
      "Epoch 124 done. Loss : -2.1165366865376094\n",
      "Epoch 125 done. Loss : -2.1125688285705624\n",
      "Epoch 126 done. Loss : -2.1086459823465677\n",
      "Epoch 127 done. Loss : -2.104757516465712\n",
      "Epoch 128 done. Loss : -2.100904858227392\n",
      "Epoch 129 done. Loss : -2.0970764115814355\n",
      "Epoch 130 done. Loss : -2.0932864454715303\n",
      "Epoch 131 done. Loss : -2.0895255282452236\n",
      "Epoch 132 done. Loss : -2.0858140450812037\n",
      "Epoch 133 done. Loss : -2.0821500770355827\n",
      "Epoch 134 done. Loss : -2.0785234521218334\n",
      "Epoch 135 done. Loss : -2.074939668495971\n",
      "Epoch 136 done. Loss : -2.071408230295349\n",
      "Epoch 137 done. Loss : -2.0679194311402873\n",
      "Epoch 138 done. Loss : -2.0644929894582473\n",
      "Epoch 139 done. Loss : -2.061144272992395\n",
      "Epoch 140 done. Loss : -2.05786512657359\n",
      "Epoch 141 done. Loss : -2.054550205737628\n",
      "Epoch 142 done. Loss : -2.0512120354600887\n",
      "Epoch 143 done. Loss : -2.0479067409377505\n",
      "Epoch 144 done. Loss : -2.044699662738425\n",
      "Epoch 145 done. Loss : -2.0415597003211188\n",
      "Epoch 146 done. Loss : -2.0384932487973475\n",
      "Epoch 147 done. Loss : -2.0354817170713693\n",
      "Epoch 148 done. Loss : -2.032511307819553\n",
      "Epoch 149 done. Loss : -2.0295998171544603\n",
      "Epoch 150 done. Loss : -2.0267305179904973\n",
      "Epoch 151 done. Loss : -2.023881168574858\n",
      "Epoch 152 done. Loss : -2.0210416308686634\n",
      "Epoch 153 done. Loss : -2.0182305974142\n",
      "Epoch 154 done. Loss : -2.015464618369066\n",
      "Epoch 155 done. Loss : -2.0127310783505\n",
      "Epoch 156 done. Loss : -2.0100440917851934\n",
      "Epoch 157 done. Loss : -2.0073977168710573\n",
      "Epoch 158 done. Loss : -2.0047850313042255\n",
      "Epoch 159 done. Loss : -2.002218324881305\n",
      "Epoch 160 done. Loss : -1.9996817409077472\n",
      "Epoch 161 done. Loss : -1.9971551020537746\n",
      "Epoch 162 done. Loss : -1.9946480086704526\n",
      "Epoch 163 done. Loss : -1.9921289846639993\n",
      "Epoch 164 done. Loss : -1.9896120796398893\n",
      "Epoch 165 done. Loss : -1.9870989381482151\n",
      "Epoch 166 done. Loss : -1.984616208205762\n",
      "Epoch 167 done. Loss : -1.9821494096308498\n",
      "Epoch 168 done. Loss : -1.9797046724275833\n",
      "Epoch 169 done. Loss : -1.977297543651376\n",
      "Epoch 170 done. Loss : -1.9749173298462694\n",
      "Epoch 171 done. Loss : -1.9725781488535288\n",
      "Epoch 172 done. Loss : -1.9702751715794893\n",
      "Epoch 173 done. Loss : -1.9679940373064237\n",
      "Epoch 174 done. Loss : -1.9657388516858658\n",
      "Epoch 175 done. Loss : -1.9635274210452365\n",
      "Epoch 176 done. Loss : -1.9613775849484294\n",
      "Epoch 177 done. Loss : -1.9592743108323258\n",
      "Epoch 178 done. Loss : -1.9571989445716458\n",
      "Epoch 179 done. Loss : -1.9551564325245852\n",
      "Epoch 180 done. Loss : -1.9531255646510464\n",
      "Epoch 181 done. Loss : -1.9511115210552012\n",
      "Epoch 182 done. Loss : -1.9491066522239493\n",
      "Epoch 183 done. Loss : -1.9471331771679659\n",
      "Epoch 184 done. Loss : -1.9451952116498794\n",
      "Epoch 185 done. Loss : -1.943303303338614\n",
      "Epoch 186 done. Loss : -1.9414452296006508\n",
      "Epoch 187 done. Loss : -1.9396279513050765\n",
      "Epoch 188 done. Loss : -1.9378126557037878\n",
      "Epoch 189 done. Loss : -1.935999784658776\n",
      "Epoch 190 done. Loss : -1.9342232825568857\n",
      "Epoch 191 done. Loss : -1.9324779260655398\n",
      "Epoch 192 done. Loss : -1.930734516074708\n",
      "Epoch 193 done. Loss : -1.9290161757472704\n",
      "Epoch 194 done. Loss : -1.927305591135447\n",
      "Epoch 195 done. Loss : -1.9256043147659434\n",
      "Epoch 196 done. Loss : -1.923918951830359\n",
      "Epoch 197 done. Loss : -1.9222475949324216\n",
      "Epoch 198 done. Loss : -1.9205797471684543\n",
      "Epoch 199 done. Loss : -1.9189262113006056\n",
      "Epoch 200 done. Loss : -1.9172727147677735\n",
      "Epoch 201 done. Loss : -1.915630880184416\n",
      "Epoch 202 done. Loss : -1.9140164435285016\n",
      "Epoch 203 done. Loss : -1.9124413208593891\n",
      "Epoch 204 done. Loss : -1.9108928595719963\n",
      "Epoch 205 done. Loss : -1.9093982543879469\n",
      "Epoch 206 done. Loss : -1.9079175844145346\n",
      "Epoch 207 done. Loss : -1.9064320711593115\n",
      "Epoch 208 done. Loss : -1.9049490971329903\n",
      "Epoch 209 done. Loss : -1.9034931924587708\n",
      "Epoch 210 done. Loss : -1.9020580932628344\n",
      "Epoch 211 done. Loss : -1.900657780582374\n",
      "Epoch 212 done. Loss : -1.899281333268342\n",
      "Epoch 213 done. Loss : -1.897923289737569\n",
      "Epoch 214 done. Loss : -1.8965702563674265\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 215 done. Loss : -1.8952339064433565\n",
      "Epoch 216 done. Loss : -1.8939455899668347\n",
      "Epoch 217 done. Loss : -1.8926414683822967\n",
      "Epoch 218 done. Loss : -1.891330622304981\n",
      "Epoch 219 done. Loss : -1.890029269149288\n",
      "Epoch 220 done. Loss : -1.8887399371788973\n",
      "Epoch 221 done. Loss : -1.8874870521921034\n",
      "Epoch 222 done. Loss : -1.8862615884719216\n",
      "Epoch 223 done. Loss : -1.885066548822278\n",
      "Epoch 224 done. Loss : -1.8838812459478917\n",
      "Epoch 225 done. Loss : -1.882702086325273\n",
      "Epoch 226 done. Loss : -1.8815376465421618\n",
      "Epoch 227 done. Loss : -1.8803926742377537\n",
      "Epoch 228 done. Loss : -1.879265933016892\n",
      "Epoch 229 done. Loss : -1.8781801818224377\n",
      "Epoch 230 done. Loss : -1.8770536271469291\n",
      "Epoch 231 done. Loss : -1.8759139125378028\n",
      "Epoch 232 done. Loss : -1.874771445388541\n",
      "Epoch 233 done. Loss : -1.8736605366412609\n",
      "Epoch 234 done. Loss : -1.8725554848473556\n",
      "Epoch 235 done. Loss : -1.871424262898949\n",
      "Epoch 236 done. Loss : -1.8702809265444356\n",
      "Epoch 237 done. Loss : -1.8691527675742723\n",
      "Epoch 238 done. Loss : -1.868037419164664\n",
      "Epoch 239 done. Loss : -1.8669280107130353\n",
      "Epoch 240 done. Loss : -1.8658457581840402\n",
      "Epoch 241 done. Loss : -1.8648003786938485\n",
      "Epoch 242 done. Loss : -1.8637344072852824\n",
      "Epoch 243 done. Loss : -1.8626796033712187\n",
      "Epoch 244 done. Loss : -1.861617941124476\n",
      "Epoch 245 done. Loss : -1.8605636100678742\n",
      "Epoch 246 done. Loss : -1.8595393759837529\n",
      "Epoch 247 done. Loss : -1.8585617286993283\n",
      "Epoch 248 done. Loss : -1.857609978910545\n",
      "Epoch 249 done. Loss : -1.8566776203691213\n",
      "Epoch 250 done. Loss : -1.8557531384123682\n",
      "Epoch 251 done. Loss : -1.85483602906774\n",
      "Epoch 252 done. Loss : -1.8539305364305019\n",
      "Epoch 253 done. Loss : -1.853030789894039\n",
      "Epoch 254 done. Loss : -1.8521680938275613\n",
      "Epoch 255 done. Loss : -1.8512944386048245\n",
      "Epoch 256 done. Loss : -1.850417541476021\n",
      "Epoch 257 done. Loss : -1.8495648830671094\n",
      "Epoch 258 done. Loss : -1.8487397529767797\n",
      "Epoch 259 done. Loss : -1.8479097648075027\n",
      "Epoch 260 done. Loss : -1.8470943274709999\n",
      "Epoch 261 done. Loss : -1.8463168638408665\n",
      "Epoch 262 done. Loss : -1.8454943963383674\n",
      "Epoch 263 done. Loss : -1.844686037529089\n",
      "Epoch 264 done. Loss : -1.843896852055197\n",
      "Epoch 265 done. Loss : -1.8431080222073413\n",
      "Epoch 266 done. Loss : -1.84232901859979\n",
      "Epoch 267 done. Loss : -1.8415760629957139\n",
      "Epoch 268 done. Loss : -1.8408453909667752\n",
      "Epoch 269 done. Loss : -1.8401277180967004\n",
      "Epoch 270 done. Loss : -1.8393813976685713\n",
      "Epoch 271 done. Loss : -1.8386558003183509\n",
      "Epoch 272 done. Loss : -1.8379283090520708\n",
      "Epoch 273 done. Loss : -1.8371996461887394\n",
      "Epoch 274 done. Loss : -1.8364702072577153\n",
      "Epoch 275 done. Loss : -1.835742135139161\n",
      "Epoch 276 done. Loss : -1.835017704558839\n",
      "Epoch 277 done. Loss : -1.8343017092043867\n",
      "Epoch 278 done. Loss : -1.8335987660449802\n",
      "Epoch 279 done. Loss : -1.8329010912696384\n",
      "Epoch 280 done. Loss : -1.8322270647754448\n",
      "Epoch 281 done. Loss : -1.8315490998545876\n",
      "Epoch 282 done. Loss : -1.8308766817824997\n",
      "Epoch 283 done. Loss : -1.8302069638749234\n",
      "Epoch 284 done. Loss : -1.8295606756369684\n",
      "Epoch 285 done. Loss : -1.8289133163085498\n",
      "Epoch 286 done. Loss : -1.8282410643708664\n",
      "Epoch 287 done. Loss : -1.8275807387391965\n",
      "Epoch 288 done. Loss : -1.8268953207535552\n",
      "Epoch 289 done. Loss : -1.826248361433714\n",
      "Epoch 290 done. Loss : -1.8255743681646335\n",
      "Epoch 291 done. Loss : -1.8248674498481876\n",
      "Epoch 292 done. Loss : -1.8241919620666311\n",
      "Epoch 293 done. Loss : -1.8235082976527845\n",
      "Epoch 294 done. Loss : -1.822843302367602\n",
      "Epoch 295 done. Loss : -1.8221866090916155\n",
      "Epoch 296 done. Loss : -1.8215095009519457\n",
      "Epoch 297 done. Loss : -1.8208777958974747\n",
      "Epoch 298 done. Loss : -1.8202608893425896\n",
      "Epoch 299 done. Loss : -1.8196663219981704\n",
      "Epoch 300 done. Loss : -1.8190893143690041\n",
      "Epoch 301 done. Loss : -1.818534332503557\n",
      "Epoch 302 done. Loss : -1.8179755045567156\n",
      "Epoch 303 done. Loss : -1.817413549126467\n",
      "Epoch 304 done. Loss : -1.816855288146035\n",
      "Epoch 305 done. Loss : -1.8163103752824203\n",
      "Epoch 306 done. Loss : -1.8157700856157164\n",
      "Epoch 307 done. Loss : -1.8152213828074366\n",
      "Epoch 308 done. Loss : -1.8146764024754503\n",
      "Epoch 309 done. Loss : -1.81413928555657\n",
      "Epoch 310 done. Loss : -1.8136139276245213\n",
      "Epoch 311 done. Loss : -1.8130781556763704\n",
      "Epoch 312 done. Loss : -1.8125309745373368\n",
      "Epoch 313 done. Loss : -1.8120071431619138\n",
      "Epoch 314 done. Loss : -1.8114794456564853\n",
      "Epoch 315 done. Loss : -1.8109420836359829\n",
      "Epoch 316 done. Loss : -1.8104379769768661\n",
      "Epoch 317 done. Loss : -1.8099281329329195\n",
      "Epoch 318 done. Loss : -1.8094471495651436\n",
      "Epoch 319 done. Loss : -1.8089407503934116\n",
      "Epoch 320 done. Loss : -1.8084332234058698\n",
      "Epoch 321 done. Loss : -1.807930891978053\n",
      "Epoch 322 done. Loss : -1.807437753329571\n",
      "Epoch 323 done. Loss : -1.8069698778512622\n",
      "Epoch 324 done. Loss : -1.8064951726445162\n",
      "Epoch 325 done. Loss : -1.806027833623062\n",
      "Epoch 326 done. Loss : -1.8055973654319002\n",
      "Epoch 327 done. Loss : -1.805146991196342\n",
      "Epoch 328 done. Loss : -1.8047096716852138\n",
      "Epoch 329 done. Loss : -1.8042650273338032\n",
      "Epoch 330 done. Loss : -1.8038167771265554\n",
      "Epoch 331 done. Loss : -1.8033754244531717\n",
      "Epoch 332 done. Loss : -1.8029007793309066\n",
      "Epoch 333 done. Loss : -1.8024745280294578\n",
      "Epoch 334 done. Loss : -1.8020637878055736\n",
      "Epoch 335 done. Loss : -1.8016399461519939\n",
      "Epoch 336 done. Loss : -1.8012024179599715\n",
      "Epoch 337 done. Loss : -1.8008014586464207\n",
      "Epoch 338 done. Loss : -1.8003684234614572\n",
      "Epoch 339 done. Loss : -1.7999783967448766\n",
      "Epoch 340 done. Loss : -1.7995926378439646\n",
      "Epoch 341 done. Loss : -1.7992012614022306\n",
      "Epoch 342 done. Loss : -1.7988083662918444\n",
      "Epoch 343 done. Loss : -1.798392242274681\n",
      "Epoch 344 done. Loss : -1.7980149673419392\n",
      "Epoch 345 done. Loss : -1.7976001666793515\n",
      "Epoch 346 done. Loss : -1.7972686623959064\n",
      "Epoch 347 done. Loss : -1.796929200923766\n",
      "Epoch 348 done. Loss : -1.7965776700604676\n",
      "Epoch 349 done. Loss : -1.7962369189017826\n",
      "Epoch 350 done. Loss : -1.7958863744425306\n",
      "Epoch 351 done. Loss : -1.7955291792713868\n",
      "Epoch 352 done. Loss : -1.795139007792805\n",
      "Epoch 353 done. Loss : -1.7947645239621877\n",
      "Epoch 354 done. Loss : -1.794416181949602\n",
      "Epoch 355 done. Loss : -1.794080323038825\n",
      "Epoch 356 done. Loss : -1.7936842773775412\n",
      "Epoch 357 done. Loss : -1.7933387147839943\n",
      "Epoch 358 done. Loss : -1.7930129992660881\n",
      "Epoch 359 done. Loss : -1.7926906512107332\n",
      "Epoch 360 done. Loss : -1.7923264550773346\n",
      "Epoch 361 done. Loss : -1.7919950554329545\n",
      "Epoch 362 done. Loss : -1.7916882715493307\n",
      "Epoch 363 done. Loss : -1.7913579932100072\n",
      "Epoch 364 done. Loss : -1.7910275367108568\n",
      "Epoch 365 done. Loss : -1.7907071770728926\n",
      "Epoch 366 done. Loss : -1.7903974769796254\n",
      "Epoch 367 done. Loss : -1.7900723963556295\n",
      "Epoch 368 done. Loss : -1.7897291315644874\n",
      "Epoch 369 done. Loss : -1.7894158704915828\n",
      "Epoch 370 done. Loss : -1.7891068100254708\n",
      "Epoch 371 done. Loss : -1.788789007635536\n",
      "Epoch 372 done. Loss : -1.7884645713774119\n",
      "Epoch 373 done. Loss : -1.7881714041804433\n",
      "Epoch 374 done. Loss : -1.78786531533065\n",
      "Epoch 375 done. Loss : -1.787584696905208\n",
      "Epoch 376 done. Loss : -1.7872909130818575\n",
      "Epoch 377 done. Loss : -1.7869920796402057\n",
      "Epoch 378 done. Loss : -1.7866751210682068\n",
      "Epoch 379 done. Loss : -1.7863864720876816\n",
      "Epoch 380 done. Loss : -1.7860788514563994\n",
      "Epoch 381 done. Loss : -1.78580458614578\n",
      "Epoch 382 done. Loss : -1.785534706206089\n",
      "Epoch 383 done. Loss : -1.7852560788343723\n",
      "Epoch 384 done. Loss : -1.7849776337250804\n",
      "Epoch 385 done. Loss : -1.784701218662589\n",
      "Epoch 386 done. Loss : -1.7844085390049593\n",
      "Epoch 387 done. Loss : -1.7841291986475043\n",
      "Epoch 388 done. Loss : -1.7838524789197288\n",
      "Epoch 389 done. Loss : -1.7835798568870556\n",
      "Epoch 390 done. Loss : -1.7832990658691725\n",
      "Epoch 391 done. Loss : -1.783033992986859\n",
      "Epoch 392 done. Loss : -1.7827638704273407\n",
      "Epoch 393 done. Loss : -1.7824908023420944\n",
      "Epoch 394 done. Loss : -1.782229275330545\n",
      "Epoch 395 done. Loss : -1.7819855287760051\n",
      "Epoch 396 done. Loss : -1.781739728144716\n",
      "Epoch 397 done. Loss : -1.7815007885186618\n",
      "Epoch 398 done. Loss : -1.7812538852428774\n",
      "Epoch 399 done. Loss : -1.7810710028925918\n",
      "Epoch 400 done. Loss : -1.7807770588141962\n",
      "Epoch 401 done. Loss : -1.7805501035118558\n",
      "Epoch 402 done. Loss : -1.7802982899595885\n",
      "Epoch 403 done. Loss : -1.7800297576352089\n",
      "Epoch 404 done. Loss : -1.7797545784221385\n",
      "Epoch 405 done. Loss : -1.7794815546547946\n",
      "Epoch 406 done. Loss : -1.7792209567294304\n",
      "Epoch 407 done. Loss : -1.7789883428636284\n",
      "Epoch 408 done. Loss : -1.7787573041995581\n",
      "Epoch 409 done. Loss : -1.7785521324006417\n",
      "Epoch 410 done. Loss : -1.7783211553490998\n",
      "Epoch 411 done. Loss : -1.7780917509273717\n",
      "Epoch 412 done. Loss : -1.777849516770512\n",
      "Epoch 413 done. Loss : -1.7775988310480528\n",
      "Epoch 414 done. Loss : -1.7773744054826053\n",
      "Epoch 415 done. Loss : -1.7771474166853418\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 416 done. Loss : -1.7769216229833087\n",
      "Epoch 417 done. Loss : -1.7767083556962506\n",
      "Epoch 418 done. Loss : -1.776489120082767\n",
      "Epoch 419 done. Loss : -1.7762620596481773\n",
      "Epoch 420 done. Loss : -1.7760155383253697\n",
      "Epoch 421 done. Loss : -1.7758060176911128\n",
      "Epoch 422 done. Loss : -1.7755919220836562\n",
      "Epoch 423 done. Loss : -1.7754003200488453\n",
      "Epoch 424 done. Loss : -1.77520291246819\n",
      "Epoch 425 done. Loss : -1.774980302424369\n",
      "Epoch 426 done. Loss : -1.77474800452253\n",
      "Epoch 427 done. Loss : -1.7745360546064886\n",
      "Epoch 428 done. Loss : -1.7743255465285592\n",
      "Epoch 429 done. Loss : -1.774144848509482\n",
      "Epoch 430 done. Loss : -1.7739645408758538\n",
      "Epoch 431 done. Loss : -1.7737618838608773\n",
      "Epoch 432 done. Loss : -1.773543876780049\n",
      "Epoch 433 done. Loss : -1.7733208322389378\n",
      "Epoch 434 done. Loss : -1.773117866268997\n",
      "Epoch 435 done. Loss : -1.7729321105534228\n",
      "Epoch 436 done. Loss : -1.7727466425356673\n",
      "Epoch 437 done. Loss : -1.7725638443321778\n",
      "Epoch 438 done. Loss : -1.7723912475670915\n",
      "Epoch 439 done. Loss : -1.7721955148235502\n",
      "Epoch 440 done. Loss : -1.772002377421315\n",
      "Epoch 441 done. Loss : -1.7718185319609665\n",
      "Epoch 442 done. Loss : -1.7716275210893722\n",
      "Epoch 443 done. Loss : -1.771435320699448\n",
      "Epoch 444 done. Loss : -1.7712306859500682\n",
      "Epoch 445 done. Loss : -1.7710686379254712\n",
      "Epoch 446 done. Loss : -1.770857114370046\n",
      "Epoch 447 done. Loss : -1.7706854108910846\n",
      "Epoch 448 done. Loss : -1.7705340357761945\n",
      "Epoch 449 done. Loss : -1.7703596833109851\n",
      "Epoch 450 done. Loss : -1.7701777119024935\n",
      "Epoch 451 done. Loss : -1.769998943027678\n",
      "Epoch 452 done. Loss : -1.7698103311392077\n",
      "Epoch 453 done. Loss : -1.769614708302106\n",
      "Epoch 454 done. Loss : -1.7694383860247387\n",
      "Epoch 455 done. Loss : -1.769240058188525\n",
      "Epoch 456 done. Loss : -1.7690665999169992\n",
      "Epoch 457 done. Loss : -1.7689168231533876\n",
      "Epoch 458 done. Loss : -1.7687495975053449\n",
      "Epoch 459 done. Loss : -1.7685763705626798\n",
      "Epoch 460 done. Loss : -1.7683969218786009\n",
      "Epoch 461 done. Loss : -1.7682252224952315\n",
      "Epoch 462 done. Loss : -1.7680652475281093\n",
      "Epoch 463 done. Loss : -1.7679125193698835\n",
      "Epoch 464 done. Loss : -1.767741370419078\n",
      "Epoch 465 done. Loss : -1.7675628418965137\n",
      "Epoch 466 done. Loss : -1.767403742817852\n",
      "Epoch 467 done. Loss : -1.7672268265735616\n",
      "Epoch 468 done. Loss : -1.7670724987670334\n",
      "Epoch 469 done. Loss : -1.7669163810730644\n",
      "Epoch 470 done. Loss : -1.766759402273321\n",
      "Epoch 471 done. Loss : -1.766600395212874\n",
      "Epoch 472 done. Loss : -1.7664431233184708\n",
      "Epoch 473 done. Loss : -1.7662795840489598\n",
      "Epoch 474 done. Loss : -1.7661205857213473\n",
      "Epoch 475 done. Loss : -1.7659762726706534\n",
      "Epoch 476 done. Loss : -1.765815270624408\n",
      "Epoch 477 done. Loss : -1.7656533855635048\n",
      "Epoch 478 done. Loss : -1.7655006715677044\n",
      "Epoch 479 done. Loss : -1.7653442446824756\n",
      "Epoch 480 done. Loss : -1.7651999663590239\n",
      "Epoch 481 done. Loss : -1.7650513117680422\n",
      "Epoch 482 done. Loss : -1.7648979153056301\n",
      "Epoch 483 done. Loss : -1.7647448960186938\n",
      "Epoch 484 done. Loss : -1.7645988230421419\n",
      "Epoch 485 done. Loss : -1.764453512117711\n",
      "Epoch 486 done. Loss : -1.7643161842139836\n",
      "Epoch 487 done. Loss : -1.7641824430705189\n",
      "Epoch 488 done. Loss : -1.7640387622188105\n",
      "Epoch 489 done. Loss : -1.7638856612084561\n",
      "Epoch 490 done. Loss : -1.7637504278238338\n",
      "Epoch 491 done. Loss : -1.7636070272512785\n",
      "Epoch 492 done. Loss : -1.7634763065258015\n",
      "Epoch 493 done. Loss : -1.7633456850540452\n",
      "Epoch 494 done. Loss : -1.7632075552392228\n",
      "Epoch 495 done. Loss : -1.7630661846744935\n",
      "Epoch 496 done. Loss : -1.7629479734782325\n",
      "Epoch 497 done. Loss : -1.7627853772207853\n",
      "Epoch 498 done. Loss : -1.7626553346467198\n",
      "Epoch 499 done. Loss : -1.7625363364098001\n",
      "Epoch 500 done. Loss : -1.762418400634361\n",
      "Epoch 501 done. Loss : -1.7622834235436682\n",
      "Epoch 502 done. Loss : -1.7621351743837173\n",
      "Epoch 503 done. Loss : -1.7620032172675792\n",
      "Epoch 504 done. Loss : -1.7618663194959685\n",
      "Epoch 505 done. Loss : -1.7617351802775745\n",
      "Epoch 506 done. Loss : -1.761599789491742\n",
      "Epoch 507 done. Loss : -1.7614715622199137\n",
      "Epoch 508 done. Loss : -1.7613387832251672\n",
      "Epoch 509 done. Loss : -1.7612085009143073\n",
      "Epoch 510 done. Loss : -1.761094383194961\n",
      "Epoch 511 done. Loss : -1.7609821285326954\n",
      "Epoch 512 done. Loss : -1.760863346650567\n",
      "Epoch 513 done. Loss : -1.760735953323611\n",
      "Epoch 514 done. Loss : -1.760598829271463\n",
      "Epoch 515 done. Loss : -1.7604937227072064\n",
      "Epoch 516 done. Loss : -1.7603543586747203\n",
      "Epoch 517 done. Loss : -1.7602483743952568\n",
      "Epoch 518 done. Loss : -1.7601490257509844\n",
      "Epoch 519 done. Loss : -1.760049209946244\n",
      "Epoch 520 done. Loss : -1.7599284135102193\n",
      "Epoch 521 done. Loss : -1.7598004017693205\n",
      "Epoch 522 done. Loss : -1.7597046116102901\n",
      "Epoch 523 done. Loss : -1.75956112593913\n",
      "Epoch 524 done. Loss : -1.7594361912421235\n",
      "Epoch 525 done. Loss : -1.7593123886735829\n",
      "Epoch 526 done. Loss : -1.759195141716614\n",
      "Epoch 527 done. Loss : -1.7590769357187213\n",
      "Epoch 528 done. Loss : -1.7589694370293416\n",
      "Epoch 529 done. Loss : -1.7588691208980858\n",
      "Epoch 530 done. Loss : -1.7587517206330419\n",
      "Epoch 531 done. Loss : -1.7586449699989828\n",
      "Epoch 532 done. Loss : -1.7585334542989834\n",
      "Epoch 533 done. Loss : -1.7584300729058384\n",
      "Epoch 534 done. Loss : -1.7583161795843514\n",
      "Epoch 535 done. Loss : -1.7582035700287881\n",
      "Epoch 536 done. Loss : -1.7580984319379769\n",
      "Epoch 537 done. Loss : -1.7579774404869495\n",
      "Epoch 538 done. Loss : -1.7578723813026913\n",
      "Epoch 539 done. Loss : -1.757753072675464\n",
      "Epoch 540 done. Loss : -1.7576619676298866\n",
      "Epoch 541 done. Loss : -1.7575495771888399\n",
      "Epoch 542 done. Loss : -1.7574833819172735\n",
      "Epoch 543 done. Loss : -1.7573611194947154\n",
      "Epoch 544 done. Loss : -1.7572576926917305\n",
      "Epoch 545 done. Loss : -1.757168477441744\n",
      "Epoch 546 done. Loss : -1.7570614588908964\n",
      "Epoch 547 done. Loss : -1.7569788990086885\n",
      "Epoch 548 done. Loss : -1.7568891220612706\n",
      "Epoch 549 done. Loss : -1.7567858991046161\n",
      "Epoch 550 done. Loss : -1.7566946976306932\n",
      "Epoch 551 done. Loss : -1.7565928656603738\n",
      "Epoch 552 done. Loss : -1.7564875777130893\n",
      "Epoch 553 done. Loss : -1.7563924836138498\n",
      "Epoch 554 done. Loss : -1.7562856265811781\n",
      "Epoch 555 done. Loss : -1.7561767941825976\n",
      "Epoch 556 done. Loss : -1.756066640617952\n",
      "Epoch 557 done. Loss : -1.7559588109632538\n",
      "Epoch 558 done. Loss : -1.7558636226770377\n",
      "Epoch 559 done. Loss : -1.7557670991538612\n",
      "Epoch 560 done. Loss : -1.7556751044754586\n",
      "Epoch 561 done. Loss : -1.755582264237116\n",
      "Epoch 562 done. Loss : -1.7554923755167406\n",
      "Epoch 563 done. Loss : -1.755392220510265\n",
      "Epoch 564 done. Loss : -1.7552844122584108\n",
      "Epoch 565 done. Loss : -1.7551951695661507\n",
      "Epoch 566 done. Loss : -1.7551013895818017\n",
      "Epoch 567 done. Loss : -1.7550200413064294\n",
      "Epoch 568 done. Loss : -1.7549144361388673\n",
      "Epoch 569 done. Loss : -1.754831762574061\n",
      "Epoch 570 done. Loss : -1.7547431460337553\n",
      "Epoch 571 done. Loss : -1.754652608163584\n",
      "Epoch 572 done. Loss : -1.7545545144884953\n",
      "Epoch 573 done. Loss : -1.7544730840165377\n",
      "Epoch 574 done. Loss : -1.7543878543082938\n",
      "Epoch 575 done. Loss : -1.754290992760182\n",
      "Epoch 576 done. Loss : -1.7542037500224292\n",
      "Epoch 577 done. Loss : -1.7541164244778633\n",
      "Epoch 578 done. Loss : -1.7540311016360899\n",
      "Epoch 579 done. Loss : -1.7539511654407507\n",
      "Epoch 580 done. Loss : -1.7538675838845514\n",
      "Epoch 581 done. Loss : -1.7537763332374858\n",
      "Epoch 582 done. Loss : -1.7536831712110361\n",
      "Epoch 583 done. Loss : -1.753589037233308\n",
      "Epoch 584 done. Loss : -1.753531429218829\n",
      "Epoch 585 done. Loss : -1.75345460566547\n",
      "Epoch 586 done. Loss : -1.7533699341606899\n",
      "Epoch 587 done. Loss : -1.7532963420821837\n",
      "Epoch 588 done. Loss : -1.753212853949478\n",
      "Epoch 589 done. Loss : -1.7531185901914095\n",
      "Epoch 590 done. Loss : -1.753030640771265\n",
      "Epoch 591 done. Loss : -1.7529388520898077\n",
      "Epoch 592 done. Loss : -1.7528694182656872\n",
      "Epoch 593 done. Loss : -1.7527887097305923\n",
      "Epoch 594 done. Loss : -1.752714397286849\n",
      "Epoch 595 done. Loss : -1.75262925352546\n",
      "Epoch 596 done. Loss : -1.7525338295171604\n",
      "Epoch 597 done. Loss : -1.7524451727595491\n",
      "Epoch 598 done. Loss : -1.752355334519571\n",
      "Epoch 599 done. Loss : -1.752288970461839\n",
      "Epoch 600 done. Loss : -1.7522015142357437\n",
      "Epoch 601 done. Loss : -1.7521333856784238\n",
      "Epoch 602 done. Loss : -1.7520687115909452\n",
      "Epoch 603 done. Loss : -1.752008509523884\n",
      "Epoch 604 done. Loss : -1.7519394177082686\n",
      "Epoch 605 done. Loss : -1.7518604162580822\n",
      "Epoch 606 done. Loss : -1.751776474158951\n",
      "Epoch 607 done. Loss : -1.7516920833713656\n",
      "Epoch 608 done. Loss : -1.7516208249990337\n",
      "Epoch 609 done. Loss : -1.7515438349123835\n",
      "Epoch 610 done. Loss : -1.7514686205620766\n",
      "Epoch 611 done. Loss : -1.751409995263244\n",
      "Epoch 612 done. Loss : -1.751349618867699\n",
      "Epoch 613 done. Loss : -1.7512832370655351\n",
      "Epoch 614 done. Loss : -1.7512219047336497\n",
      "Epoch 615 done. Loss : -1.7511777530123458\n",
      "Epoch 616 done. Loss : -1.7510868743958952\n",
      "Epoch 617 done. Loss : -1.751020317650156\n",
      "Epoch 618 done. Loss : -1.7509522441774439\n",
      "Epoch 619 done. Loss : -1.750874615206949\n",
      "Epoch 620 done. Loss : -1.750807985712163\n",
      "Epoch 621 done. Loss : -1.7507104467681702\n",
      "Epoch 622 done. Loss : -1.7506254044427543\n",
      "Epoch 623 done. Loss : -1.7505538558877833\n",
      "Epoch 624 done. Loss : -1.7504715717238883\n",
      "Epoch 625 done. Loss : -1.7504399423378867\n",
      "Epoch 626 done. Loss : -1.750348780523177\n",
      "Epoch 627 done. Loss : -1.7502952347223206\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 628 done. Loss : -1.750246990235238\n",
      "Epoch 629 done. Loss : -1.7501942357555187\n",
      "Epoch 630 done. Loss : -1.7501184092420965\n",
      "Epoch 631 done. Loss : -1.7500400582381062\n",
      "Epoch 632 done. Loss : -1.7499770205336436\n",
      "Epoch 633 done. Loss : -1.7499020502999934\n",
      "Epoch 634 done. Loss : -1.7498243792374921\n",
      "Epoch 635 done. Loss : -1.749766024086258\n",
      "Epoch 636 done. Loss : -1.7496883129403893\n",
      "Epoch 637 done. Loss : -1.749601867860132\n",
      "Epoch 638 done. Loss : -1.7495096514191717\n",
      "Epoch 639 done. Loss : -1.7494740728426115\n",
      "Epoch 640 done. Loss : -1.7493842425409176\n",
      "Epoch 641 done. Loss : -1.7493646040422175\n",
      "Epoch 642 done. Loss : -1.7493490707613042\n",
      "Epoch 643 done. Loss : -1.7493091098694942\n",
      "Epoch 644 done. Loss : -1.749252371205976\n",
      "Epoch 645 done. Loss : -1.7491775056820182\n",
      "Epoch 646 done. Loss : -1.749093219886229\n",
      "Epoch 647 done. Loss : -1.749046187384429\n",
      "Epoch 648 done. Loss : -1.7489660300413097\n",
      "Epoch 649 done. Loss : -1.748909384153961\n",
      "Epoch 650 done. Loss : -1.7488381999274771\n",
      "Epoch 651 done. Loss : -1.748759079956127\n",
      "Epoch 652 done. Loss : -1.7486858018492653\n",
      "Epoch 653 done. Loss : -1.7486229175366215\n",
      "Epoch 654 done. Loss : -1.7485600047190804\n",
      "Epoch 655 done. Loss : -1.7485135684325899\n",
      "Epoch 656 done. Loss : -1.7484272076630716\n",
      "Epoch 657 done. Loss : -1.7483657846456342\n",
      "Epoch 658 done. Loss : -1.7483002725133259\n",
      "Epoch 659 done. Loss : -1.7482515190215353\n",
      "Epoch 660 done. Loss : -1.7481976624277507\n",
      "Epoch 661 done. Loss : -1.7481318507035501\n",
      "Epoch 662 done. Loss : -1.7480786560373591\n",
      "Epoch 663 done. Loss : -1.7480283290951983\n",
      "Epoch 664 done. Loss : -1.747968212678206\n",
      "Epoch 665 done. Loss : -1.747917832823103\n",
      "Epoch 666 done. Loss : -1.7478679114447986\n",
      "Epoch 667 done. Loss : -1.747806735587767\n",
      "Epoch 668 done. Loss : -1.7477490822867505\n",
      "Epoch 669 done. Loss : -1.747689229920586\n",
      "Epoch 670 done. Loss : -1.7476334871453099\n",
      "Epoch 671 done. Loss : -1.7475693535894392\n",
      "Epoch 672 done. Loss : -1.7475077078913257\n",
      "Epoch 673 done. Loss : -1.7474505968915337\n",
      "Epoch 674 done. Loss : -1.7474036641292758\n",
      "Epoch 675 done. Loss : -1.7473505079588936\n",
      "Epoch 676 done. Loss : -1.7472949416774928\n",
      "Epoch 677 done. Loss : -1.7472313089479994\n",
      "Epoch 678 done. Loss : -1.7471972716859663\n",
      "Epoch 679 done. Loss : -1.7471618947432335\n",
      "Epoch 680 done. Loss : -1.7471204313931221\n",
      "Epoch 681 done. Loss : -1.7470531330310657\n",
      "Epoch 682 done. Loss : -1.747001227779347\n",
      "Epoch 683 done. Loss : -1.746969229887895\n",
      "Epoch 684 done. Loss : -1.7469254262887444\n",
      "Epoch 685 done. Loss : -1.7468875372925976\n",
      "Epoch 686 done. Loss : -1.7468362746334503\n",
      "Epoch 687 done. Loss : -1.7467742141747573\n",
      "Epoch 688 done. Loss : -1.7467082400915097\n",
      "Epoch 689 done. Loss : -1.7466543170784012\n",
      "Epoch 690 done. Loss : -1.7465906494552903\n",
      "Epoch 691 done. Loss : -1.7465354868019225\n",
      "Epoch 692 done. Loss : -1.7464864157642466\n",
      "Epoch 693 done. Loss : -1.7464318112999542\n",
      "Epoch 694 done. Loss : -1.746372209714586\n",
      "Epoch 695 done. Loss : -1.746329717264787\n",
      "Epoch 696 done. Loss : -1.7462878562408872\n",
      "Epoch 697 done. Loss : -1.7462509351557054\n",
      "Epoch 698 done. Loss : -1.74620852088877\n",
      "Epoch 699 done. Loss : -1.7461443969291284\n",
      "Epoch 700 done. Loss : -1.7460931122435182\n",
      "Epoch 701 done. Loss : -1.7460469731163704\n",
      "Epoch 702 done. Loss : -1.746022645349022\n",
      "Epoch 703 done. Loss : -1.745989210478824\n",
      "Epoch 704 done. Loss : -1.7459444949526068\n",
      "Epoch 705 done. Loss : -1.7458857926708578\n",
      "Epoch 706 done. Loss : -1.7458319783942187\n",
      "Epoch 707 done. Loss : -1.7457723498638953\n",
      "Epoch 708 done. Loss : -1.7457253103446821\n",
      "Epoch 709 done. Loss : -1.745670872762199\n",
      "Epoch 710 done. Loss : -1.745626583745262\n",
      "Epoch 711 done. Loss : -1.7455874630466457\n",
      "Epoch 712 done. Loss : -1.7455551889154264\n",
      "Epoch 713 done. Loss : -1.7455273411822172\n",
      "Epoch 714 done. Loss : -1.7454875609183909\n",
      "Epoch 715 done. Loss : -1.7454343877034875\n",
      "Epoch 716 done. Loss : -1.745378374793542\n",
      "Epoch 717 done. Loss : -1.7453274907227758\n",
      "Epoch 718 done. Loss : -1.745278112102289\n",
      "Epoch 719 done. Loss : -1.7452445917989734\n",
      "Epoch 720 done. Loss : -1.7451973896688366\n",
      "Epoch 721 done. Loss : -1.7451680027206002\n",
      "Epoch 722 done. Loss : -1.7451260265573487\n",
      "Epoch 723 done. Loss : -1.7450722501346854\n",
      "Epoch 724 done. Loss : -1.7450103617913864\n",
      "Epoch 725 done. Loss : -1.7449707721880197\n",
      "Epoch 726 done. Loss : -1.7449346447707084\n",
      "Epoch 727 done. Loss : -1.7448971777932911\n",
      "Epoch 728 done. Loss : -1.7448698522282522\n",
      "Epoch 729 done. Loss : -1.7448289788809512\n",
      "Epoch 730 done. Loss : -1.744782000058158\n",
      "Epoch 731 done. Loss : -1.7447333093923818\n",
      "Epoch 732 done. Loss : -1.7446874349697317\n",
      "Epoch 733 done. Loss : -1.7446355736223982\n",
      "Epoch 734 done. Loss : -1.7446075880122716\n",
      "Epoch 735 done. Loss : -1.744564235664313\n",
      "Epoch 736 done. Loss : -1.7445218929517978\n",
      "Epoch 737 done. Loss : -1.7445031904597448\n",
      "Epoch 738 done. Loss : -1.7444740025933512\n",
      "Epoch 739 done. Loss : -1.7444324166996186\n",
      "Epoch 740 done. Loss : -1.7443795293707498\n",
      "Epoch 741 done. Loss : -1.7443240001292792\n",
      "Epoch 742 done. Loss : -1.7442843097700906\n",
      "Epoch 743 done. Loss : -1.7442626377339094\n",
      "Epoch 744 done. Loss : -1.744214181044108\n",
      "Epoch 745 done. Loss : -1.7441958913892668\n",
      "Epoch 746 done. Loss : -1.7441701992057872\n",
      "Epoch 747 done. Loss : -1.7441410579865138\n",
      "Epoch 748 done. Loss : -1.744110356470566\n",
      "Epoch 749 done. Loss : -1.7440784374268081\n",
      "Epoch 750 done. Loss : -1.7440408959127414\n",
      "Epoch 751 done. Loss : -1.744002569146748\n",
      "Epoch 752 done. Loss : -1.7439657487633757\n",
      "Epoch 753 done. Loss : -1.7439281955184174\n",
      "Epoch 754 done. Loss : -1.7438812055076527\n",
      "Epoch 755 done. Loss : -1.7438357157805395\n",
      "Epoch 756 done. Loss : -1.7437868494636422\n",
      "Epoch 757 done. Loss : -1.743747201519351\n",
      "Epoch 758 done. Loss : -1.7436887266907286\n",
      "Epoch 759 done. Loss : -1.74365175385375\n",
      "Epoch 760 done. Loss : -1.7436261348050612\n",
      "Epoch 761 done. Loss : -1.7435712861025272\n",
      "Epoch 762 done. Loss : -1.7435450402766184\n",
      "Epoch 763 done. Loss : -1.7435140225987644\n",
      "Epoch 764 done. Loss : -1.7434806445342679\n",
      "Epoch 765 done. Loss : -1.7434464859561014\n",
      "Epoch 766 done. Loss : -1.743412118152247\n",
      "Epoch 767 done. Loss : -1.7433694233417838\n",
      "Epoch 768 done. Loss : -1.743325983843828\n",
      "Epoch 769 done. Loss : -1.7432868007908564\n",
      "Epoch 770 done. Loss : -1.7432521736467332\n",
      "Epoch 771 done. Loss : -1.7432073379441861\n",
      "Epoch 772 done. Loss : -1.7431908980220545\n",
      "Epoch 773 done. Loss : -1.743138646539244\n",
      "Epoch 774 done. Loss : -1.7431200457315168\n",
      "Epoch 775 done. Loss : -1.7430894572744107\n",
      "Epoch 776 done. Loss : -1.7430463864299024\n",
      "Epoch 777 done. Loss : -1.7430107385209381\n",
      "Epoch 778 done. Loss : -1.742981483699778\n",
      "Epoch 779 done. Loss : -1.742955233945489\n",
      "Epoch 780 done. Loss : -1.7429382156594975\n",
      "Epoch 781 done. Loss : -1.7429010297042096\n",
      "Epoch 782 done. Loss : -1.7428685745606094\n",
      "Epoch 783 done. Loss : -1.7428489760411725\n",
      "Epoch 784 done. Loss : -1.7428097947783583\n",
      "Epoch 785 done. Loss : -1.742787999841531\n",
      "Epoch 786 done. Loss : -1.7427563809947522\n",
      "Epoch 787 done. Loss : -1.7427154080691727\n",
      "Epoch 788 done. Loss : -1.7426750086915146\n",
      "Epoch 789 done. Loss : -1.7426262892281037\n",
      "Epoch 790 done. Loss : -1.7425853896931316\n",
      "Epoch 791 done. Loss : -1.7425664415018758\n",
      "Epoch 792 done. Loss : -1.742535526938445\n",
      "Epoch 793 done. Loss : -1.7424931628810638\n",
      "Epoch 794 done. Loss : -1.7424703567604807\n",
      "Epoch 795 done. Loss : -1.7424634463286588\n",
      "Epoch 796 done. Loss : -1.742416827048049\n",
      "Epoch 797 done. Loss : -1.7423898006866674\n",
      "Epoch 798 done. Loss : -1.7423495690799227\n",
      "Epoch 799 done. Loss : -1.7423136573511568\n",
      "Epoch 800 done. Loss : -1.7423013784300039\n",
      "Epoch 801 done. Loss : -1.742250138561289\n",
      "Epoch 802 done. Loss : -1.7422270589059083\n",
      "Epoch 803 done. Loss : -1.7421959693674123\n",
      "Epoch 804 done. Loss : -1.7421646605558403\n",
      "Epoch 805 done. Loss : -1.7421301956679682\n",
      "Epoch 806 done. Loss : -1.7420904344756805\n",
      "Epoch 807 done. Loss : -1.7420624744534898\n",
      "Epoch 808 done. Loss : -1.7420254983328023\n",
      "Epoch 809 done. Loss : -1.7420136313679988\n",
      "Epoch 810 done. Loss : -1.741985264780432\n",
      "Epoch 811 done. Loss : -1.7419599440996598\n",
      "Epoch 812 done. Loss : -1.7419277892451948\n",
      "Epoch 813 done. Loss : -1.741887842818885\n",
      "Epoch 814 done. Loss : -1.7418478059325442\n",
      "Epoch 815 done. Loss : -1.7418259514992973\n",
      "Epoch 816 done. Loss : -1.7417885777135502\n",
      "Epoch 817 done. Loss : -1.7417621226457338\n",
      "Epoch 818 done. Loss : -1.7417413106484783\n",
      "Epoch 819 done. Loss : -1.7417117361625505\n",
      "Epoch 820 done. Loss : -1.7416791044690774\n",
      "Epoch 821 done. Loss : -1.7416374249919475\n",
      "Epoch 822 done. Loss : -1.7416050711473625\n",
      "Epoch 823 done. Loss : -1.7415772121804647\n",
      "Epoch 824 done. Loss : -1.7415498562968157\n",
      "Epoch 825 done. Loss : -1.7415365160578316\n",
      "Epoch 826 done. Loss : -1.7415169889352868\n",
      "Epoch 827 done. Loss : -1.7414888919442664\n",
      "Epoch 828 done. Loss : -1.7414597341417577\n",
      "Epoch 829 done. Loss : -1.741418847928216\n",
      "Epoch 830 done. Loss : -1.7413957206721817\n",
      "Epoch 831 done. Loss : -1.7413659027764772\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 832 done. Loss : -1.7413544249358646\n",
      "Epoch 833 done. Loss : -1.7413322504982003\n",
      "Epoch 834 done. Loss : -1.7413096108730222\n",
      "Epoch 835 done. Loss : -1.7412823057583102\n",
      "Epoch 836 done. Loss : -1.7412475466093955\n",
      "Epoch 837 done. Loss : -1.741206758692646\n",
      "Epoch 838 done. Loss : -1.7411698017013775\n",
      "Epoch 839 done. Loss : -1.7411429816718205\n",
      "Epoch 840 done. Loss : -1.7411144774669542\n",
      "Epoch 841 done. Loss : -1.7410966617943735\n",
      "Epoch 842 done. Loss : -1.7410726175926203\n",
      "Epoch 843 done. Loss : -1.741038546126815\n",
      "Epoch 844 done. Loss : -1.741007738587649\n",
      "Epoch 845 done. Loss : -1.7409839080830385\n",
      "Epoch 846 done. Loss : -1.740962609232453\n",
      "Epoch 847 done. Loss : -1.74092799533366\n",
      "Epoch 848 done. Loss : -1.740908410556358\n",
      "Epoch 849 done. Loss : -1.7408854545969163\n",
      "Epoch 850 done. Loss : -1.7408564641686257\n",
      "Epoch 851 done. Loss : -1.7408180690019672\n",
      "Epoch 852 done. Loss : -1.7407942045558735\n",
      "Epoch 853 done. Loss : -1.7407598067061687\n",
      "Epoch 854 done. Loss : -1.7407560533267588\n",
      "Epoch 855 done. Loss : -1.7407301366996242\n",
      "Epoch 856 done. Loss : -1.7407152643504515\n",
      "Epoch 857 done. Loss : -1.7406925022509756\n",
      "Epoch 858 done. Loss : -1.740662726915927\n",
      "Epoch 859 done. Loss : -1.740640647873741\n",
      "Epoch 860 done. Loss : -1.7406143769049147\n",
      "Epoch 861 done. Loss : -1.7405949991306544\n",
      "Epoch 862 done. Loss : -1.7405572525303268\n",
      "Epoch 863 done. Loss : -1.7405403556469587\n",
      "Epoch 864 done. Loss : -1.7405169349584617\n",
      "Epoch 865 done. Loss : -1.7404866480452863\n",
      "Epoch 866 done. Loss : -1.740452516229813\n",
      "Epoch 867 done. Loss : -1.7404183040459336\n",
      "Epoch 868 done. Loss : -1.7403954612488344\n",
      "Epoch 869 done. Loss : -1.7403663850577427\n",
      "Epoch 870 done. Loss : -1.7403472062736047\n",
      "Epoch 871 done. Loss : -1.74032689869535\n",
      "Epoch 872 done. Loss : -1.7403153875995572\n",
      "Epoch 873 done. Loss : -1.740282282599547\n",
      "Epoch 874 done. Loss : -1.7402618987054046\n",
      "Epoch 875 done. Loss : -1.7402595104789378\n",
      "Epoch 876 done. Loss : -1.7402278818810526\n",
      "Epoch 877 done. Loss : -1.7402164881359852\n",
      "Epoch 878 done. Loss : -1.74020417596469\n",
      "Epoch 879 done. Loss : -1.7401826832808265\n",
      "Epoch 880 done. Loss : -1.7401560041459632\n",
      "Epoch 881 done. Loss : -1.740123235100406\n",
      "Epoch 882 done. Loss : -1.7400910448041067\n",
      "Epoch 883 done. Loss : -1.740048841634088\n",
      "Epoch 884 done. Loss : -1.7400205891158893\n",
      "Epoch 885 done. Loss : -1.7399982081062455\n",
      "Epoch 886 done. Loss : -1.7399693154779388\n",
      "Epoch 887 done. Loss : -1.7399370092508577\n",
      "Epoch 888 done. Loss : -1.7399122130361317\n",
      "Epoch 889 done. Loss : -1.7398993214644065\n",
      "Epoch 890 done. Loss : -1.739880777284652\n",
      "Epoch 891 done. Loss : -1.7398618030165176\n",
      "Epoch 892 done. Loss : -1.7398368288852946\n",
      "Epoch 893 done. Loss : -1.7398123656773459\n",
      "Epoch 894 done. Loss : -1.73979056992816\n",
      "Epoch 895 done. Loss : -1.7397646223470968\n",
      "Epoch 896 done. Loss : -1.739735997864564\n",
      "Epoch 897 done. Loss : -1.739733429493296\n",
      "Epoch 898 done. Loss : -1.739700211794617\n",
      "Epoch 899 done. Loss : -1.7396869644889157\n",
      "Epoch 900 done. Loss : -1.7396703011919632\n",
      "Epoch 901 done. Loss : -1.7396523915284572\n",
      "Epoch 902 done. Loss : -1.7396283703523514\n",
      "Epoch 903 done. Loss : -1.7396071981044634\n",
      "Epoch 904 done. Loss : -1.7395775114127137\n",
      "Epoch 905 done. Loss : -1.7395550817867775\n",
      "Epoch 906 done. Loss : -1.739531217427274\n",
      "Epoch 907 done. Loss : -1.7395135517337756\n",
      "Epoch 908 done. Loss : -1.739491377875555\n",
      "Epoch 909 done. Loss : -1.7394670304126918\n",
      "Epoch 910 done. Loss : -1.73943726011691\n",
      "Epoch 911 done. Loss : -1.739425320963144\n",
      "Epoch 912 done. Loss : -1.739396540365912\n",
      "Epoch 913 done. Loss : -1.739381604202443\n",
      "Epoch 914 done. Loss : -1.739390830867404\n",
      "Epoch 915 done. Loss : -1.7393608155674767\n",
      "Epoch 916 done. Loss : -1.739333635594788\n",
      "Epoch 917 done. Loss : -1.7393188705722484\n",
      "Epoch 918 done. Loss : -1.7392992690105893\n",
      "Epoch 919 done. Loss : -1.7392789833383866\n",
      "Epoch 920 done. Loss : -1.7392590407120196\n",
      "Epoch 921 done. Loss : -1.739246414105545\n",
      "Epoch 922 done. Loss : -1.7392376851486175\n",
      "Epoch 923 done. Loss : -1.7392108677978513\n",
      "Epoch 924 done. Loss : -1.7391866553225175\n",
      "Epoch 925 done. Loss : -1.7391577074081\n",
      "Epoch 926 done. Loss : -1.739124856624998\n",
      "Epoch 927 done. Loss : -1.739094988558643\n",
      "Epoch 928 done. Loss : -1.7390765662994423\n",
      "Epoch 929 done. Loss : -1.7390594989793458\n",
      "Epoch 930 done. Loss : -1.7390540489249617\n",
      "Epoch 931 done. Loss : -1.7390390932716455\n",
      "Epoch 932 done. Loss : -1.739030822469549\n",
      "Epoch 933 done. Loss : -1.7390160506598336\n",
      "Epoch 934 done. Loss : -1.7389955233064165\n",
      "Epoch 935 done. Loss : -1.738970021919906\n",
      "Epoch 936 done. Loss : -1.7389400303733193\n",
      "Epoch 937 done. Loss : -1.7389196605538912\n",
      "Epoch 938 done. Loss : -1.7388867182198724\n",
      "Epoch 939 done. Loss : -1.7388696355905284\n",
      "Epoch 940 done. Loss : -1.7388415036737133\n",
      "Epoch 941 done. Loss : -1.7388178871080693\n",
      "Epoch 942 done. Loss : -1.738793179520988\n",
      "Epoch 943 done. Loss : -1.7387678036612741\n",
      "Epoch 944 done. Loss : -1.7387492744419928\n",
      "Epoch 945 done. Loss : -1.7387272352013325\n",
      "Epoch 946 done. Loss : -1.7387088431229076\n",
      "Epoch 947 done. Loss : -1.7386939895397742\n",
      "Epoch 948 done. Loss : -1.7386793518779706\n",
      "Epoch 949 done. Loss : -1.7386618678949788\n",
      "Epoch 950 done. Loss : -1.7386474857373486\n",
      "Epoch 951 done. Loss : -1.7386302860750562\n",
      "Epoch 952 done. Loss : -1.738617626481056\n",
      "Epoch 953 done. Loss : -1.7385969955213048\n",
      "Epoch 954 done. Loss : -1.7385785337671469\n",
      "Epoch 955 done. Loss : -1.7385554913577317\n",
      "Epoch 956 done. Loss : -1.738536639655445\n",
      "Epoch 957 done. Loss : -1.738511526794635\n",
      "Epoch 958 done. Loss : -1.7385004158951927\n",
      "Epoch 959 done. Loss : -1.7384806940240687\n",
      "Epoch 960 done. Loss : -1.7384721032067714\n",
      "Epoch 961 done. Loss : -1.7384590267719737\n",
      "Epoch 962 done. Loss : -1.7384407188360478\n",
      "Epoch 963 done. Loss : -1.7384177300981816\n",
      "Epoch 964 done. Loss : -1.7383905141961336\n",
      "Epoch 965 done. Loss : -1.7383634706039626\n",
      "Epoch 966 done. Loss : -1.7383406614124852\n",
      "Epoch 967 done. Loss : -1.7383388445956072\n",
      "Epoch 968 done. Loss : -1.738303042230199\n",
      "Epoch 969 done. Loss : -1.738291312477233\n",
      "Epoch 970 done. Loss : -1.7382775430865616\n",
      "Epoch 971 done. Loss : -1.7382669133790447\n",
      "Epoch 972 done. Loss : -1.738255383984587\n",
      "Epoch 973 done. Loss : -1.7382414486092022\n",
      "Epoch 974 done. Loss : -1.738223302830385\n",
      "Epoch 975 done. Loss : -1.7382060089066482\n",
      "Epoch 976 done. Loss : -1.7381880462643862\n",
      "Epoch 977 done. Loss : -1.738166703685242\n",
      "Epoch 978 done. Loss : -1.7381477108966013\n",
      "Epoch 979 done. Loss : -1.7381255045343509\n",
      "Epoch 980 done. Loss : -1.7381042866221208\n",
      "Epoch 981 done. Loss : -1.7380872986191496\n",
      "Epoch 982 done. Loss : -1.7380697558448934\n",
      "Epoch 983 done. Loss : -1.7380479996193592\n",
      "Epoch 984 done. Loss : -1.738031488587673\n",
      "Epoch 985 done. Loss : -1.7380297479456732\n",
      "Epoch 986 done. Loss : -1.7380016785653076\n",
      "Epoch 987 done. Loss : -1.7379911828379648\n",
      "Epoch 988 done. Loss : -1.7379756373091433\n",
      "Epoch 989 done. Loss : -1.7379651839538468\n",
      "Epoch 990 done. Loss : -1.737951934382817\n",
      "Epoch 991 done. Loss : -1.7379364344912915\n",
      "Epoch 992 done. Loss : -1.7379171658370098\n",
      "Epoch 993 done. Loss : -1.7379011438429994\n",
      "Epoch 994 done. Loss : -1.7378783842186505\n",
      "Epoch 995 done. Loss : -1.7378585063214853\n",
      "Epoch 996 done. Loss : -1.7378375847108856\n",
      "Epoch 997 done. Loss : -1.7378117777535314\n",
      "Epoch 998 done. Loss : -1.737793038260624\n",
      "Epoch 999 done. Loss : -1.7377799547918542\n"
     ]
    }
   ],
   "source": [
    "hist = nn_overfit.fit(smol_batch_pixels,smol_batch_label,32,1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "samples = mnist_pixels[:120,:]\n",
    "samples_y =  mnist_label[:120]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(46.875)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn_overfit.score(smol_batch_pixels,smol_batch_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x20385f942e8>]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmYFfWd7/H392y9N4t0I9IgKGhQFNAeosElUYPoNRon3qhZxsmYyziP3izjTG6SyWNunMk8mZlcdZJMEknMZJnEjHFJGMeNGPcEtCHIrqBsLQiNDTRN0/v3/nGq4dic030aTvfprvN5Pc95TtWvfnXqW5R+q/pXv6qfuTsiIlI4IvkOQEREhpYSv4hIgVHiFxEpMEr8IiIFRolfRKTAKPGLiBQYJX4RkQKjxC8iUmCU+EVECkws3wGkM27cOJ8yZUq+wxARGTGWL1++x92rsqk7LBP/lClTqKury3cYIiIjhpltzbaumnpERAqMEr+ISIFR4hcRKTBK/CIiBUaJX0SkwPTbq8fMioHngaKg/oPu/tVede4GPhDMlgLV7j46WNYFrA6WbXP3q3MUu4iIHINsunO2AZe4e7OZxYEXzexxd1/aU8HdP98zbWb/G5iTsv4hd5+ds4hFROS49NvU40nNwWw8+PQ1XuONwP05iG3AvvX0Rp57vSEfmxYRGTGyauM3s6iZrQR2A0vcfVmGeicDU4HfpRQXm1mdmS01sw/3sY2FQb26hoZjS973PvcGzyvxi4j0KavE7+5dQXNNDTDXzGZmqHoDyXsAXSllk929FvgYcI+ZnZphG4vcvdbda6uqsnrq+CgliSiHOrr6rygiUsAG1KvH3fcBzwILMlS5gV7NPO6+I/h+M1h3ztGr5UZJIsqhdiV+EZG+9Jv4zazKzHp66JQAlwEb0tQ7HRgD/CGlbIyZFQXT44B5wLrchH60kniUlvbOwfp5EZFQyKZXzwTgJ2YWJXmieMDdHzWzO4E6d18c1LsR+KW7p974nQHca2bdwbrfcPfBS/yJGIc6ugfr50VEQqHfxO/uq0jTPOPud/Sa/79p6vweOOs44huQkniEQ7riFxHpU6ie3C1NxHRzV0SkH6FK/CWJKC26uSsi0qdwJf64evWIiPQnVIm/VP34RUT6FarEn+zOqcQvItKXcCX+RJT2zm66uvt6lZCISGELVeIvTUQB1NwjItKHUCX+kniQ+NXcIyKSUbgSfyL5PJoSv4hIZuFK/MEVf0uHnt4VEckkVIn/cBu/rvhFRDIKVeIvUeIXEelXuBJ/XL16RET6E6rE39PUo4e4REQyC1XiL1Z3ThGRfmUzAlexmb1sZq+a2Voz+1qaOn9uZg1mtjL4fDpl2U1mtjH43JTrHUilB7hERPqXzQhcbcAl7t5sZnHgRTN73N2X9qr3n+5+W2qBmY0FvgrUAg4sN7PF7r43F8H3Vhr04z+owVhERDLq94rfk5qD2XjwyfZlOJcDS9y9MUj2S8g8UPtxK45HMFNTj4hIX7Jq4zezqJmtBHaTTOTL0lT7iJmtMrMHzWxSUDYR2J5Spz4oGxRmRlkixsE2JX4RkUyySvzu3uXus4EaYK6ZzexV5b+AKe5+NvBb4CdBuaX7uXTbMLOFZlZnZnUNDQ3ZRZ9GaSJKi5p6REQyGlCvHnffBzxLr+Yad3/H3duC2R8A5wbT9cCklKo1wI4Mv73I3WvdvbaqqmogYb1LWVGMg2rqERHJKJtePVVmNjqYLgEuAzb0qjMhZfZqYH0w/SQw38zGmNkYYH5QNmhKE1Fa2nTFLyKSSTa9eiYAPzGzKMkTxQPu/qiZ3QnUufti4DNmdjXQCTQCfw7g7o1m9vfAK8Fv3enujbneiVRliZh69YiI9KHfxO/uq4A5acrvSJn+EvClDOv/CPjRccQ4IKVFURoPtg/V5kRERpxQPbkLwRW/mnpERDIKXeJP9urRzV0RkUxCl/jLinTFLyLSl9Al/p4rfvdsHy4WESksoUv8ZUUxOrud9q7ufIciIjIshS7xH34nv17bICKSVugSf5ne0Cki0qfQJf7SIo3CJSLSl9Al/sNX/OrZIyKSVugSv8bdFRHpW+gSf1mRrvhFRPoSusTfc8Wvm7siIumFLvFXFMcBaG5V4hcRSSeEiT/Z1NOkxC8iklboEn9xPEoiFqHpUEe+QxERGZZCl/gBKotjuuIXEckgm6EXi83sZTN71czWmtnX0tT5azNbZ2arzOxpMzs5ZVmXma0MPotzvQPpVBbHaWrVFb+ISDrZDL3YBlzi7s1mFgdeNLPH3X1pSp0/ArXu3mJmfwX8M3B9sOyQu8/Obdh9qyiJc0BX/CIiafV7xe9JzcFsPPh4rzrPuHtLMLsUqMlplANUWRxTG7+ISAZZtfGbWdTMVgK7gSXuvqyP6jcDj6fMF5tZnZktNbMP97GNhUG9uoaGhqyCz0RNPSIimWWV+N29K2iuqQHmmtnMdPXM7BNALfAvKcWT3b0W+Bhwj5mdmmEbi9y91t1rq6qqBrQTvVUUx9TUIyKSwYB69bj7PuBZYEHvZWZ2GfB3wNXu3payzo7g+81g3TnHHm52KkviauoREckgm149VWY2OpguAS4DNvSqMwe4l2TS351SPsbMioLpccA8YF3uwk+vsjhGW2c3bZ16UZuISG/Z9OqZAPzEzKIkTxQPuPujZnYnUOfui0k27ZQDvzIzgG3ufjUwA7jXzLqDdb/h7oOe+Hte23CgtZOi8uhgb05EZETpN/G7+yrSNM+4+x0p05dlWPf3wFnHE+CxqCwJXttwqINx5UVDvXkRkWEtlE/uVhQdueIXEZF3C2XiryxJJv79usErInKUUCb+0aVK/CIimYQ68e9rac9zJCIiw084E39JAoC9LbriFxHpLZSJPxGLUF4UY6+u+EVEjhLKxA/J5p59uuIXETlKaBP/mNKErvhFRNIIbeIfXRpXG7+ISBqhTfxjShPq1SMikkZoE/+48iL2HGjrv6KISIEJbeIfX1nEwfYumtv02gYRkVQhTvzFAOxqas1zJCIiw0toE391ZfKtnEr8IiLvFtrE33PFv7tJ7fwiIqmyGYGr2MxeNrNXzWytmX0tTZ0iM/tPM9tkZsvMbErKsi8F5a+Z2eW5DT+znsT/tq74RUTeJZsr/jbgEnefBcwGFpjZeb3q3AzsdfdpwN3APwGY2RnADcCZJMfp/W4wktegKy+KMaokzvbGlqHYnIjIiNFv4vek5mA2Hny8V7VrgJ8E0w8Cl1pyDMZrgF+6e5u7bwY2AXNzEnkWpowrY/Oeg0O1ORGRESGrNn4zi5rZSmA3sMTdl/WqMhHYDuDuncB+4ITU8kB9UDYkThlXxhYlfhGRd8kq8bt7l7vPBmqAuWY2s1cVS7daH+VHMbOFZlZnZnUNDQ3ZhNWvqePK2LG/lUPtXTn5PRGRMBhQrx533wc8S7K9PlU9MAnAzGLAKKAxtTxQA+zI8NuL3L3W3WurqqoGElZGp40vB2Ddzqac/J6ISBhk06unysxGB9MlwGXAhl7VFgM3BdPXAb9zdw/Kbwh6/UwFpgMv5yr4/pxz8hgAVmzdO1SbFBEZ9rK54p8APGNmq4BXSLbxP2pmd5rZ1UGd+4ATzGwT8NfAFwHcfS3wALAOeAK41d2HrN2luqKYk08oZdnmd4ZqkyIiw16svwruvgqYk6b8jpTpVuB/Zlj/68DXjyPG4/LBGeP5yR+28E5zGyeUF+UrDBGRYSO0T+72uP5PJtHR5fz0D1vzHYqIyLAQ+sQ/fXwFl585nh+9tJn2zu58hyMiknehT/wAHzmnhgOtnazYppu8IiIFkfh7evesrt+f50hERPKvIBL/uPIiJowqZs0OJX4RkYJI/AAzJ45izVtK/CIiBZP4TxtfzpZ3WnSDV0QKXsEk/mnV5XR1O1vf0UvbRKSwFU7ir6oAYNPu5n5qioiEW8Ek/lOrywAlfhGRgkn8pYkYE0eXsKlBiV9EClvBJH6AU6vLdcUvIgWvoBL/tKpy3mhoprs77VgwIiIFobASf3U5rR3dvLXvUL5DERHJm4JL/IDa+UWkoBVk4n9D7fwiUsD6HYjFzCYBPwVOBLqBRe7+r73q/C3w8ZTfnAFUuXujmW0BDgBdQKe71+Yu/IEZW5ZgbFlCN3hFpKD1m/iBTuB2d19hZhXAcjNb4u7reiq4+78A/wJgZh8CPu/ujSm/8QF335PLwI/VtCr17BGRwtZvU4+773T3FcH0AWA9MLGPVW4E7s9NeLl3anU5mxqaSY4FLyJSeAbUxm9mU0iOv7ssw/JSYAHwUEqxA0+Z2XIzW9jHby80szozq2toaBhIWANyxoQK9rV0UL9XPXtEpDBlnfjNrJxkQv+cuzdlqPYh4KVezTzz3P0c4ArgVjO7KN2K7r7I3WvdvbaqqirbsAasZ1CW5Vs1GpeIFKasEr+ZxUkm/Z+7+8N9VL2BXs087r4j+N4NPALMPbZQc+P08RWUJaIahlFECla/id/MDLgPWO/ud/VRbxRwMfCblLKy4IYwZlYGzAfWHG/QxyMWjTBr0mhd8YtIwcrmin8e8EngEjNbGXyuNLNbzOyWlHrXAk+5e+oL78cDL5rZq8DLwH+7+xM5i/4YzZ06lnU7m9h7sD3foYiIDLl+u3O6+4uAZVHvx8CPe5W9Ccw6xtgGzcWnVXHPbzfy/MYGrpndVwclEZHwKagnd3ucXTOaMaVxnntt8HoPiYgMVwWZ+KMR48LpVTy/sUFv6hSRglOQiR/g/adXsae5nbU7MvVMFREJp4JN/BedVoUZPL1hV75DEREZUgWb+MeVF3Hu5DE8uVaJX0QKS8EmfoAFM09k/c4mtr5zsP/KIiIhUdCJ//IzTwTgybVv5zkSEZGhU9CJf9LYUs48qZIn1ijxi0jhKOjED8mr/hXb9rGrqTXfoYiIDImCT/xXnpVs7vmvV3fkORIRkaFR8Il/WnUFZ00cxSN/fCvfoYiIDImCT/wA186ZyNodTby+60C+QxERGXRK/MDVs08iGjEeXqGrfhEJPyV+kg9zXTR9HL9Z+Zbe3SMioafEH/jTc2rYub+VpW++k+9QREQGVTYjcE0ys2fMbL2ZrTWzz6ap834z258yUMsdKcsWmNlrZrbJzL6Y6x3IlQ+eMZ6KohgP6yaviIRcNlf8ncDt7j4DOI/kgOlnpKn3grvPDj53AphZFPg3kgOtnwHcmGHdvCuOR7nirBN5fPVODrV35TscEZFB02/id/ed7r4imD4ArAeyHbZqLrDJ3d9093bgl8A1xxrsYLt2Tg0H27t4ap2e5BWR8BpQG7+ZTQHmAMvSLD7fzF41s8fN7MygbCKwPaVOPdmfNIbce6eOZeLoEvXuEZFQyzrxm1k58BDwOXfvPXrJCuBkd58FfBv4dc9qaX4qbbcZM1toZnVmVtfQkJ8hESMR48NzTuKFjQ3sPqBXOIhIOGWV+M0sTjLp/9zdH+693N2b3L05mH4MiJvZOJJX+JNSqtYAad+N4O6L3L3W3WurqqoGuBu5c+2cGrodFq/UKxxEJJyy6dVjwH3Aene/K0OdE4N6mNnc4HffAV4BppvZVDNLADcAi3MV/GCYVl3OnMmjuf/lbbirT7+IhE82V/zzgE8Cl6R017zSzG4xs1uCOtcBa8zsVeBbwA2e1AncBjxJ8qbwA+6+dhD2I6c+/t6TeaPhIMs2N+Y7FBGRnLPheFVbW1vrdXV1edt+a0cXc7/+W95/ejXfunFO3uIQEcmWmS1399ps6urJ3TSK41E+cm4Nj6/ZyZ7mtnyHIyKSU0r8GXz8vZPp6HIeXF6f71BERHJKiT+DadUVzJ06ll8s20aXXtwmIiGixN+Hm86fwrbGFpboSV4RCREl/j4smHkik8eW8v3n3lTXThEJDSX+PkQjxv+6cCort+/jlS178x2OiEhOKPH347pzJzG2LMG9z72R71BERHJCib8fJYkoN50/hac37Gb9zt6vKBIRGXmU+LPw5++bQkVxjHt++3q+QxEROW5K/FkYVRrn5gum8uTaXazdsT/f4YiIHBcl/iz9xQVTqSyOcc9vN+Y7FBGR46LEn6XK4jifvvAUlqzbxep6XfWLyMilxD8An5o3hVElcbX1i8iIpsQ/ABXFcRZedApPb9jN8q16ZbOIjExK/AP0qXlTqK4o4uv/vV5P84rIiKTEP0CliRi3zz+NFdv28fgavcNHREaebIZenGRmz5jZejNba2afTVPn42a2Kvj83sxmpSzbYmarg5G78je6Sg5dd+4kTh9fwTce30B7Z3e+wxERGZBsrvg7gdvdfQZwHnCrmZ3Rq85m4GJ3Pxv4e2BRr+UfcPfZ2Y4OM9xFI8aX/8cMtjW28LOlW/MdjojIgPSb+N19p7uvCKYPkBw7d2KvOr939563mC0FanId6HBz8WlVXDh9HN96eiP7WzryHY6ISNYG1MZvZlOAOcCyPqrdDDyeMu/AU2a23MwWDjTA4ezLV86gqbWD7zyjh7pEZOTIOvGbWTnwEPA5d0/7tjIz+wDJxP9/Uornufs5wBUkm4kuyrDuQjOrM7O6hoaGrHcgn2ZMqOSj507i31/awsZdB/IdjohIVrJK/GYWJ5n0f+7uD2eoczbwQ+Aad3+np9zddwTfu4FHgLnp1nf3Re5e6+61VVVVA9uLPPrCgtMpK4rxlV+vUfdOERkRsunVY8B9wHp3vytDncnAw8An3f31lPIyM6vomQbmA2tyEfhwcUJ5EV+84j0s29zIwyveync4IiL9imVRZx7wSWC1ma0Myr4MTAZw9+8DdwAnAN9NnifoDHrwjAceCcpiwC/c/Ymc7sEwcH3tJB6o284/PraeS2dUM7o0ke+QREQysuHYPFFbW+t1dSOry/+6HU1c/Z0X+dCsk7j7+tn5DkdECoyZLc+2y7ye3M2RM06q5NYPTOORP77Fk2v1RK+IDF9K/Dl02yXTOPOkSv7ukdU0HmzPdzgiImkp8edQPBrh/310FvsPdfCVX69WLx8RGZaU+HPsPSdW8vkPnsZjq9/mV3X1+Q5HROQoSvyD4C8vOpV5007gjsVr9GCXiAw7SvyDIBox7v7obMoSMW77xR9p7ejKd0giIocp8Q+S6spi7rp+Nq/tOsDX/mtdvsMRETlMiX8QXXxaFbdcfCr3v7yNB17Znu9wREQAJf5B9zfzT+PC6eP4yq/XsHzr3v5XEBEZZEr8gywWjfDtG+cwYXQxt/zHct7e35rvkESkwCnxD4HRpQl+8Ge1tLR18pc/q9PNXhHJKyX+IXLa+Aruvn42r9bv53O/XElXtx7uEpH8UOIfQvPPPJE7rjqDJ9a+rff3i0jeZPNaZsmhv7hgKnua2/jus28wrjzB7fNPz3dIIlJglPjz4G8vP53Gg+18+3ebiEcjfObS6fkOSUQKSDYjcE0ys2fMbL2ZrTWzz6apY2b2LTPbZGarzOyclGU3mdnG4HNTrndgJDIzvn7tWfzpORO5a8nrfPPJ19TsIyJDJpsr/k7gdndfEQyjuNzMlrh76uOoVwDTg897ge8B7zWzscBXgVrAg3UXu3vBd2iPRoxvXjeLRDTCd57ZxN6Wdr529ZnEorrtIiKDq9/E7+47gZ3B9AEzWw9MBFIT/zXATz152brUzEab2QTg/cASd28EMLMlwALg/pzuxQgViRj/eO1ZjC5N8P3n3mDHvkN852PnUFakFjgRGTwDurw0synAHGBZr0UTgdR3EtQHZZnKJRCJGF+84j38w4dn8tzrDVy/6A96yEtEBlXWid/MyoGHgM+5e1PvxWlW8T7K0/3+QjOrM7O6hoaGbMMKjU+cdzI/vKmWzQ0HuerbL/CHN97Jd0giElJZJX4zi5NM+j9394fTVKkHJqXM1wA7+ig/irsvcvdad6+tqqrKJqzQueQ94/nNbfMYVRLnE/ctY9Hzb+imr4jkXDa9egy4D1jv7ndlqLYY+LOgd895wP7g3sCTwHwzG2NmY4D5QZlkMK26gt/cdgGXnzmef3xsA7f+YgXNbZ35DktEQiSbu4jzgE8Cq81sZVD2ZWAygLt/H3gMuBLYBLQAnwqWNZrZ3wOvBOvd2XOjVzIrL4rxbx87hx+88CbfeHwDq+r3888fOZv3TRuX79BEJARsODYl1NbWel1dXb7DGBaWb23kb361is17DvKJ8ybzhQXvobI4nu+wRGSYMbPl7l6bTV11Gh/mzj15LI995kI+fcFUfr5sG5d881keeGU73XrJm4gcIyX+EaAkEeUrV53B4lsvYPLYUr7w0Cqu/e5LLH1TPX9EZOCU+EeQs2pG8dBfvY+7r5/F202t3LBoKZ+8bxmr6vflOzQRGUHUxj9CtXZ08bM/bOW7z25ib0sHC848kVs/MI2zakblOzQRyYOBtPEr8Y9wB1o7uO/Fzfzwhc00t3Uyd+pYPn3BVC6dMZ5oJN3zcyISRkr8BaiptYMHXtnOv7+0hbf2HWLy2FI+WlvDdedO4sRRxfkOT0QGmRJ/Aevs6uaJtW/zH0u3svTNRiIGF59WxYfnTOSyGeP1AjiRkBpI4lcWCJlYNMJVZ5/EVWefxJY9B3lweT0PLq/nmddWUhKPcumMaq6edRIXn15FUSya73BFJA90xV8AurudV7Y0svjVHTy2eid7WzqoLI6xYOaJXDpjPOefeoIeChMZ4dTUIxl1dHXz0qY9LH51B0+t3UVzWyfRiHHO5NFcOL2KC6aPY+ZJo0jE1NNXZCRR4pestHd2s2LbXl7Y2MALG/ew+q39uENxPMLsSaP5kyljqZ0yljmTR+svApFhTolfjsk7zW0s29xI3Za91G1tZO2OJrq6HTM4ZVwZZ9eM5qyJozirZhRnnlRJaUK3iESGCyV+yYmDbZ2s3L6P5Vv3sqp+P6vf2seupjYAIgbTqss5a+JoZk6sZHp1BdOqyxlfWUTyTd4iMpTUq0dyoqwoxrxp45iX8jro3U2trH5rf3Ai2M9zr+/moRX1h5eXF8U4tbqcaVXlnFpdxqlV5UweW0rNmBIq1FwkMizoil+Oi7vTcKCNTbub2dTQzKbdzbwRfPf8ddBjdGmcmjElTBqTPBHUjCllfGUR1ZXFjK8spqq8SDeVRY6RrvhlyJgZ1ZXFVFcWHzVQTFNrB5sbDlK/9xD1e1vYvreF+r2HeH3XAX63YTdtnd1H/d4JZYngRFBEdUVR8oRQUcTYsgRjyxKcUJacHlMaJxbVSULkWPSb+M3sR8BVwG53n5lm+d8CH0/5vRlAVTD61hbgANAFdGZ7NpJwqCyOM2vSaGZNGn3UMnfnnYPt7GpqZXdTG7uaWtnV1MauA63sDqbX7WhiT3Mb6YYeMINRJfHgZJBgVEmCypIYo0riVBbHqSyJB9OxI9PBfHlRTPchpKD129RjZhcBzcBP0yX+XnU/BHze3S8J5rcAte6+ZyBBqalHenR2ddPY0k7jwXYam9t552ByOvndlpxubqeptZOmQx00HergQD9jFEcMKkvilCVilCailCailCSilKbMlyZilCSilCWilPQqP1I/Smk8RmlRcro4FiWiF+NJnuS0qcfdnzezKVlu+0bg/izrivQrFo1QXVFMdUX2L5rr6naaWzvZf6iDptbkyaCptSM5f6jz8PTBti4OdXQmv9u72H2glZb2Llraumhp7+RQRxcdXQO7B1YSj1JWFJwY4jGK4xESseQnHo2QiAbz0V7lsWyWGYloNCg3ErEIRWnW79mOTkKSSc7a+M2sFFgA3JZS7MBTZubAve6+KFfbE8kkGjFGlcYZVXr8vYjaO7s51N5FS0fnu04KLR3Jk8XBtuQJoqU9+LS9e1lbZzftnd20dnRzoLWT9mC+vevId0fP9wBPMv2JR414NEI0YsQiRjRiRCw5HYkc+Y5aclk0Q9nhT5ZlPdtIVxbpHUv03cuiGcqi0WBZT1mG7Ucsed8pYsE0hhlEIkfmj9TJ/B2x5Ho99Q/Ph6SJMJc3dz8EvOTujSll89x9h5lVA0vMbIO7P59uZTNbCCwEmDx5cg7DEjl2PVfeoxj8rqjd3R6cAFJPCk57VxdtnckTQ8+Jo6OrO3lSSTlxvKs8payr2+l2p7Pb6e52uno+Gcq6UubbO7uPKktXL3Ub6cqGYefBY9ZzIogkzwxHTgwcOUFEIqnzPSeVI+tZunnghLIiHrjl/EHfh1wm/hvo1czj7juC791m9ggwF0ib+IO/BhZBso0/h3GJjAiRiFEciVIcD99bU7tTThapJ6HDJx53Ort6naDSlfVz0ur2ZMcBd+jONM+R8u5ux+HI/OG6BNNp5jnyW93u4O+e98O/FczTs60j890ZYqwoHpqOljnZipmNAi4GPpFSVgZE3P1AMD0fuDMX2xORkSUSMSIYITynjUjZdOe8H3g/MM7M6oGvQvLvXnf/flDtWuApdz+Ysup44JGgTSwG/MLdn8hd6CIiciyy6dVzYxZ1fgz8uFfZm8CsYw1MREQGhx59FBEpMEr8IiIFRolfRKTAKPGLiBQYJX4RkQKjxC8iUmCG5UAsZtYAbD3G1ccBA3obaAhonwuD9jn8jmd/T3b3qmwqDsvEfzzMrK7Q3vuvfS4M2ufwG6r9VVOPiEiBUeIXESkwYUz8hfjOf+1zYdA+h9+Q7G/o2vhFRKRvYbziFxGRPoQm8ZvZAjN7zcw2mdkX8x1PrpjZJDN7xszWm9laM/tsUD7WzJaY2cbge0xQbmb2reDfYZWZnZPfPTh2ZhY1sz+a2aPB/FQzWxbs83+aWSIoLwrmNwXLp+Qz7mNlZqPN7EEz2xAc7/PDfpzN7PPBf9drzOx+MysO23E2sx+Z2W4zW5NSNuDjamY3BfU3mtlNxxNTKBK/mUWBfwOuAM4AbjSzM/IbVc50Are7+wzgPODWYN++CDzt7tOBp4N5SP4bTA8+C4HvDX3IOfNZYH3K/D8Bdwf7vBe4OSi/Gdjr7tOAu4N6I9G/Ak+4+3tIvtJ8PSE+zmY2EfgMUOvuM4EoyZH8wnacf0xyPPJUAzquZjaW5Fgo7yU5kuFXe04Wx8QPDy82cj/A+cCTKfNfAr6U77gGaV9/A3wQeA2YEJRNAF4Lpu8Fbkypf7jeSPoANcH/EJefcBZAAAACrElEQVQAj5IcknQPEOt9zIEngfOD6VhQz/K9DwPc30pgc++4w3ycgYnAdmBscNweBS4P43EGpgBrjvW4AjcC96aUv6veQD+huOLnyH9APeqDslAJ/rSdAywDxrv7ToDguzqoFpZ/i3uALwDdwfwJwD537wzmU/fr8D4Hy/cH9UeSU4AG4N+D5q0fBkOWhvY4u/tbwDeBbcBOksdtOeE+zj0GelxzerzDkvgtTVmouiuZWTnwEPA5d2/qq2qashH1b2FmVwG73X15anGaqp7FspEiBpwDfM/d5wAHOfLnfzojfp+DpoprgKnASUAZyaaO3sJ0nPuTaR9zuu9hSfz1wKSU+RpgR55iyTkzi5NM+j9394eD4l1mNiFYPgHYHZSH4d9iHnC1mW0BfkmyueceYLSZ9QwXmrpfh/c5WD4KaBzKgHOgHqh392XB/IMkTwRhPs6XAZvdvcHdO4CHgfcR7uPcY6DHNafHOyyJ/xVgetAbIEHyBtHiPMeUE5Ycrf4+YL2735WyaDHQc2f/JpJt/z3lfxb0DjgP2N/zJ+VI4e5fcvcad59C8lj+zt0/DjwDXBdU673PPf8W1wX1R9SVoLu/DWw3s9ODokuBdYT4OJNs4jnPzEqD/8579jm0xznFQI/rk8B8MxsT/KU0Pyg7Nvm+6ZHDmydXAq8DbwB/l+94crhfF5D8k24VsDL4XEmybfNpYGPwPTaobyR7OL0BrCbZYyLv+3Ec+/9+4NFg+hTgZWAT8CugKCgvDuY3BctPyXfcx7ivs4G64Fj/GhgT9uMMfA3YAKwBfgYUhe04A/eTvIfRQfLK/eZjOa7AXwT7vgn41PHEpCd3RUQKTFiaekREJEtK/CIiBUaJX0SkwCjxi4gUGCV+EZECo8QvIlJglPhFRAqMEr+ISIH5/623nf9jGU2PAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plt.plot(-1 * np.asnumpy(hist[hist != -1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now for the actual training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_optimizer = MomentumSGD(0.001,0.9)\n",
    "bias_optimizer = MomentumSGD(0.001,0.9)\n",
    "nn = FCNN([784,100,100,10],kaiming_uniform,kaiming_uniform,relu,crossentropy_with_logits,weight_optimizer,bias_optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "indices = np.arange(len(mnist_label),dtype=np.int32)\n",
    "np.random.shuffle(indices)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Data is p big lets save 10% for test (around 420 for each classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "cutoff = int(len(indices) * 0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "indices_train = indices[:cutoff]\n",
    "indices_test = indices[cutoff:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pix, train_lab = mnist_pixels[indices_train], mnist_label[indices_train].astype(np.int32)\n",
    "test_pix, test_lab = mnist_pixels[indices_test], mnist_label[indices_test].astype(np.int32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 done. Loss : -2.4802696535794917\n",
      "Epoch 1 done. Loss : -2.009661014390803\n",
      "Epoch 2 done. Loss : -1.7308973423293676\n",
      "Epoch 3 done. Loss : -1.386876585273928\n",
      "Epoch 4 done. Loss : -1.0678495731406326\n",
      "Epoch 5 done. Loss : -1.0122846991639363\n",
      "Epoch 6 done. Loss : -0.980201576513601\n",
      "Epoch 7 done. Loss : -0.9551785397439634\n",
      "Epoch 8 done. Loss : -0.9352621267967882\n",
      "Epoch 9 done. Loss : -0.9174188928238542\n",
      "Epoch 10 done. Loss : -0.9037583537324133\n",
      "Epoch 11 done. Loss : -0.890639400793885\n",
      "Epoch 12 done. Loss : -0.8784535629246161\n",
      "Epoch 13 done. Loss : -0.8685471745774762\n",
      "Epoch 14 done. Loss : -0.8577273466565674\n",
      "Epoch 15 done. Loss : -0.8493724076044776\n",
      "Epoch 16 done. Loss : -0.8404698227420342\n",
      "Epoch 17 done. Loss : -0.8319384617234646\n",
      "Epoch 18 done. Loss : -0.8255263771825812\n",
      "Epoch 19 done. Loss : -0.8181351910809391\n",
      "Epoch 20 done. Loss : -0.6688311741366602\n",
      "Epoch 21 done. Loss : -0.34378259614937046\n",
      "Epoch 22 done. Loss : -0.23832538189819832\n",
      "Epoch 23 done. Loss : -0.2203512101311073\n",
      "Epoch 24 done. Loss : -0.20890273595405834\n",
      "Epoch 25 done. Loss : -0.19745975923382394\n",
      "Epoch 26 done. Loss : -0.18647647816950697\n",
      "Epoch 27 done. Loss : -0.1783658002565789\n",
      "Epoch 28 done. Loss : -0.17013312349013332\n",
      "Epoch 29 done. Loss : -0.16325732226173523\n"
     ]
    }
   ],
   "source": [
    "fit_history = nn.fit(train_pix,train_lab,32,30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x20386029b38>]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHsRJREFUeJzt3Xl4HPWd5/H3t7vV3VLrsk7bkmz5IBw2xsKKAZskJBsSwmYDhDOzISFLQhiSmWR3Zp/MzrPPTDbP5nlms7vZmQw5hiTshh0mhCtA8sBOjiEBbDDIFwYMxrdlyzqt+2pJv/2jW44sS9bVcqmrP6/nqaerq6vV33I9/lT1r371a3POISIi/hLwugAREUk9hbuIiA8p3EVEfEjhLiLiQwp3EREfUriLiPiQwl1ExIcU7iIiPqRwFxHxoZBXH1xSUuKqq6u9+ngRkbS0ffv2Fudc6VTreRbu1dXV1NXVefXxIiJpycyOTGc9NcuIiPiQwl1ExIcU7iIiPjRluJtZlZk9b2Z7zexNM/vKBOtcY2YdZrYrOf3V/JQrIiLTMZ0LqkPAnznndphZHrDdzH7tnHtr3HovOuc+nvoSRURkpqY8c3fONTjndiTnu4C9QMV8FyYiIrM3ozZ3M6sGaoBtE7x8lZntNrPnzGxNCmoTEZFZmna4m1ku8ATwVedc57iXdwDLnXOXAX8PPDXJ37jHzOrMrK65uXlWBe9r7OK//vIt+uPDs3q/iEgmmFa4m1kWiWB/2Dn35PjXnXOdzrnu5PyzQJaZlUyw3gPOuVrnXG1p6ZQ3WE2o/lQvP3rpEDuOnJrV+0VEMsF0essY8GNgr3Pu25Osszi5Hma2Mfl3W1NZ6KiNK4oJBYwtB1rm48+LiPjCdM7cNwN3Ah8a09XxejO718zuTa5zC/CGme0GvgPc4Zxz81FwbiTEZVWFbNk/L8cOERFfmLIrpHPuJcCmWOd+4P5UFTWVzauKuf/5/XT2x8mPZp2vjxURSRtpeYfqptUljDjYdrDN61JERBaktAz3mmWFRLMCbNmvdncRkYmkZbhHQkHeW13EVl1UFRGZUFqGO8CmVSXsa+ymqavf61JERBactA33zauLAXj5gHrNiIiMl7bhvmZpAfnRkNrdRUQmkLbhHgwYV60qZsv+VuapS72ISNpK23AH2Ly6hOPtfRxt6/W6FBGRBSWtw33TqsTwNbpbVUTkTGkd7qtKY5TnRzTOjIjIOGkd7mbG5lUlvHKglZERtbuLiIxK63CHxFAErT2DvNPY5XUpIiILRtqH+2h/d3WJFBH5g7QP9yUF2awsibFVNzOJiJyW9uEOsGl1MdsOthIfHvG6FBGRBcEf4b6qhJ7BYV6vb/e6FBGRBcEX4X7VymLM1N9dRGSUL8J9USzMJUvydVFVRCTJF+EOiaEIdh5tp29w2OtSREQ855tw37SqmMHhEV47rJ/eExHxTbhvXFFEVtA0FIGICD4K95xwiJqqRWzVRVUREf+EOyT6u79xooP23kGvSxER8ZSvwn3z6hKcg1cO6uxdRDKbr8L9sspCcsJB9XcXkYznq3APhwJsXFHEVl1UFZEM56twh0SXyAPNPZzs6Pe6FBERz/gw3BM/vaezdxHJZL4L90uW5LMoJ0vt7iKS0XwX7oGAcdWqYrYeaME5/fSeiGQm34U7JJpmGjr6OdTS43UpIiKe8GW4b16daHffol9nEpEM5ctwry7OYWlBlK0aAlhEMpQvw93M2LS6hJcPtjIyonZ3Eck8U4a7mVWZ2fNmttfM3jSzr0ywjpnZd8xsv5m9bmaXz0+507d5dTHtvXHeauj0uhQRkfNuOmfuQ8CfOecuBq4EvmRml4xb52PABcnpHuD7Ka1yFkb7u+vXmUQkE00Z7s65BufcjuR8F7AXqBi32g3AQy7hFaDQzJakvNoZKM+PcmF5Hs++cdLLMkREPDGjNnczqwZqgG3jXqoAjo15Xs/ZBwDM7B4zqzOzuubm5plVOgt/dMUydh9rZ/uRU/P+WSIiC8m0w93McoEngK8658Y3ZNsEbznrSqZz7gHnXK1zrra0tHRmlc7CLRsqyY+GePClQ/P+WSIiC8m0wt3MskgE+8POuScnWKUeqBrzvBI4Mffy5iYWCfGpjct47o0G6k/1el2OiMh5M53eMgb8GNjrnPv2JKs9A3wm2WvmSqDDOdeQwjpn7bObqjEzfrL1sNeliIicN9M5c98M3Al8yMx2JafrzexeM7s3uc6zwEFgP/BD4L75KXfmlhZm87G1i3nk1WN0Dwx5XY6IyHkRmmoF59xLTNymPnYdB3wpVUWl2t1Xr+CXrzfwWN0xPrd5hdfliIjMO1/eoTpezbJFXL6skP+95TDDumNVRDJARoQ7wN1Xr+RoWy+/2dvodSkiIvMuY8L9o2vKqSjM5sfqFikiGSBjwj0UDHDXpmpePdTGG8c7vC5HRGReZUy4A9y+sYpYOKizdxHxvYwK9/xoFrfWVvGL3Sdo7Oz3uhwRkXmTUeEO8LnN1Qw7x/99+YjXpYiIzJuMC/flxTGuvbich7cdoT8+7HU5IiLzIuPCHRI3NZ3qjfPkjuNelyIiMi8yMtw3rihibUU+D245ROLmWhERf8nIcDcz/t3mFexv6ub3++Z/XHkRkfMtI8Md4OPrllKWF1G3SBHxpYwN93AowGeuWs6L77awr7HL63JERFIqY8Md4I+uWE4kFNAvNYmI72R0uBfFwnzy8kqe3Hmc1u4Br8sREUmZjA53gLuvrmZwaISHtx31uhQRkZTJ+HBfXZbHB95TykMvH2FgSDc1iYg/ZHy4A9y1uZqW7gGef7vJ61JERFJC4Q68b3UJJbkRntp5wutSRERSQuFOYqz3T1y2lH95u4mO3rjX5YiIzJnCPenGmqUMDo/w3BsNXpciIjJnCvekSysKWFka4+c7NZiYiKQ/hXuSmXHj+gq2HWrjeHuf1+WIiMyJwn2MG9dXAPDMLl1YFZH0pnAfY1lxDpcvK+QpNc2ISJpTuI9zU00F7zR2sbeh0+tSRERmTeE+zr9et5RQwHT2LiJpTeE+TlEszAfeU8rTu04wMqJfaRKR9KRwn8ANNRWc7OznlUOtXpciIjIrCvcJXHtxObFwkKc1HIGIpCmF+wSyw0GuW7uEZ/c00B/XSJEikn4U7pO4sWYpXQNDGilSRNKSwn0Sm1aVUJoX0XAEIpKWpgx3M3vQzJrM7I1JXr/GzDrMbFdy+qvUl3n+BQPGJy5byu/eaaa9d9DrckREZmQ6Z+7/B7huinVedM6tT07fmHtZC8NNNRUMDo/w7J6TXpciIjIjU4a7c+4FoO081LLgrFmaz6rSGE/tUtOMiKSXVLW5X2Vmu83sOTNbk6K/6Tkz46aaCl491Eb9qV6vyxERmbZUhPsOYLlz7jLg74GnJlvRzO4xszozq2tubk7BR8+/G0ZHitytPu8ikj7mHO7OuU7nXHdy/lkgy8xKJln3AedcrXOutrS0dK4ffV5UFeVQu3wRT+08jnMajkBE0sOcw93MFpuZJec3Jv+mr+7bv6Gmgn2N3ext6PK6FBGRaZlOV8ifAi8DF5pZvZndbWb3mtm9yVVuAd4ws93Ad4A7nM9OcT9+6ZLESJG6sCoiaSI01QrOuU9N8fr9wP0pq2gBWhQLc82FpTy96zhfu+4iggHzuiQRkXPSHarTdGNNBY2dA2w76KsWJxHxKYX7NH344nJyIyENRyAiaUHhPk3RrCDXrV3M/3vjpEaKFJEFT+E+AzfVVNA1MMRv92qkSBFZ2BTuM3DlymLK8yM8uaPe61JERM5J4T4DwYDxycsr+d2+Zpo6+70uR0RkUgr3Gbp1QyXDI47HdfYuIguYwn2GVpbmsrG6iMfq6jUcgYgsWAr3WbjtvVUcaunhtcOnvC5FRGRCCvdZuP7SxeRGQvzstWNelyIiMiGF+yzkhEP8m8uW8OyeBrr6416XIyJyFoX7LN1WW0VffJhfvt7gdSkiImdRuM/S+qpC3lOeq6YZEVmQFO6zZGbcVlvFrmPt7GvUOO8isrAo3OfgppoKsoLGozp7F5EFRuE+B8W5ET58cTlP7jzO4NCI1+WIiJymcJ+j295bRVvPIL/d2+h1KSIipync5+j9F5SyOD/Kz+rUNCMiC4fCfY6CAeOWDZW8sK+Zho4+r8sREQEU7ilxW20VIw6e2K7BxERkYVC4p8Cy4hyuWlnMo3X1jIxoMDER8Z7CPUVue28lR9t6eeWQfkBbRLyncE+Rj61dQl40xGN1apoREe8p3FMkmhXkhvVLeXZPAx19GkxMRLylcE+h22uXMTA0wjO7T3hdiohkOIV7Cq2tyOeixXkajkBEPKdwTyEz4/b3VrHneAdvnej0uhwRyWAK9xS7cX0F4WCAR3XHqoh4SOGeYotiYa5dU85Tu44zMDTsdTkikqEU7vPg9toq2nvj/OpNDSYmIt5QuM+Dq1eXUFGYraYZEfGMwn0eBJKDib20v4X//NQeDrX0eF2SiGSYkNcF+NXn37eCkx39PPpaPQ9vO8q1F5dzz/tXsmH5IszM6/JExOfMOW8GuqqtrXV1dXWefPb51NTVz0Nbj/CP247Q3hunZlkhX3jfSj66ZjHBgEJeRGbGzLY752qnXG+qcDezB4GPA03OubUTvG7A3wHXA73AXc65HVN9cKaE+6jewSEe317Pj148xNG2XpYV5XD31Su4tbaSnLC+QInI9KQy3N8PdAMPTRLu1wN/QiLcrwD+zjl3xVQfnGnhPmp4xPHrt07ywAsH2XG0nYLsLD595TJuq61iWVGOmmxE5JymG+5TnjI6514ws+pzrHIDieB3wCtmVmhmS5xzDdOuNoMEA8Z1a5dw3dolbD/Sxg9fOMT3fneA7z5/gIrCbDatKmbz6hI2rSqmLD/qdbkikqZS0R5QAYzt81efXKZwn8KG5UVsuLOIY229/O6dJrbsb+VXbzXyWPIXnS4oy2XTqmI2rS7hypXFFGRneVyxiKSLVIT7RO0IE7b1mNk9wD0Ay5YtS8FH+0NVUQ53XlXNnVdVMzzi2NvQyZb9LWw50MqjdfX85OUjBAwurSjgylXF1FQVcmllIUsLomrGEZEJpSLc64GqMc8rgQnHvHXOPQA8AIk29xR8tu8EA8baigLWVhTwxQ+sYnBohF3H2tmyv4WtB1p48KVDxIcT/3TFsTCXVhawrqKASysLWVdZQLmackSE1IT7M8CXzewREhdUO9TenjrhUICNK4rYuKKIf3/te+iPD/P2yS721Lfzen0He4538MK+ZkZ/urUsL8K6ygIurSjk4iV5XFCeR9WibEJB3a8mkkmmDHcz+ylwDVBiZvXAXwNZAM65HwDPkugps59EV8jPzVexkvjFp/VVhayvKjy9rG9wmLcaOhJhX9/B68c7+O3bTYx2hAoHA6woibG6PJfVpblcUJ7L6rJcVpTEiISCHm2JiMwn3cTkU90DQ7zb2MX+pu7T07tN3Rw71Xs69AMGy4tjrCrNZUVJDitKcqkuyWFFSYzyvCgB3WQlsuCkrCukpKfcSIiaZYuoWbbojOX98WEONHefEfr7m7p54d1mBodGTq8XzQpQXRxLTCWx0+FfuSibsryImnlEFjiFe4aJZgVZs7SANUsLzlg+POJo6OjjcEsvh1p7ONySmPY1dfHbtxtPX8SFxBl/WV6UJYVRlhREWZyfzdLCKIsLoiwpyGZJQVQHABGPKdwFSPTSqVyUQ+WiHK6+oOSM14aGRzje3sehlh5OtPfT0NFHQ0fi8e2TXTz/djN98TN/mCRgUJoXYXF+lPL8xEGgvCDK4vzkVJCYNPSCyPzQ/yyZUigYYHlxjOXFsQlfd87R2TdEQ2cfDe39p4P/ZEc/Jzv7Odzaw8sHW+nqHzrrvXnREOX5UcrzI5TlJc74y/ITj+X5o88jOgiIzJD+x8icmRkFOVkU5GRx0eL8SdfrHRw6Hfijj40d/TR2DtDU1c+rh9po7hpgcHjkrPfmRUKU5kUoioUpioUpzg2zKOfM+eJYhKLcMEU5YbLD6gUkmU3hLudNTjjEytJcVpbmTrqOc4723jhNXYnAHw3+ps4BmrsGaO0Z4EhrLzuOtnOqd5DhkYl7e+WEg5TkRijODVOSG6Ek+VgcC1OSF6E4FqE0L0xRLEJBdpaGXxbfUbjLgmJmLIqFWRQLc+HivHOuO9oc1NY7SFvPAK3dg5zqHaS1Z5DW7kFauwdo6R7kWFsvO4+209YzwCTHAvIiocS3j+wsCpOPiSl8er4wJzEtygmffoxm6RuCLEwKd0lbY5uDVpRMfD1grOERR3vvIC3J4G/uThwQOvridPTF6eyL056cb+zspr03sWyiZqJRkVDgdNiPDf6C7PDpg0RhdqLGwuxw8jGLnHBQ4wLJvFK4S8YIBozi3AjFuRHg3N8KRjnn6I+P0N43SHtvPDkNcqo3PmZZ8nnvIPubujnVG6ejb/CM7qPjZQWNguws8qJZ5EZCxCJBciOh5Hzo7PloiLxoiPxoFnnREHnRLPKzQ7rDWCalcBc5BzMjOxwkO5zNkoLsab/POUdffJj23sQ3gfZk4I/OtycfuweG6O6P0zMwzIn2froHhugZGKJ7YIiBocm/MYwKhwLkjwn9/OwsPre5mg9dVD6XzRYfULiLzAMzIyccIiccYmnh9A8KY8WHR04HfffAEF39Q3T2xROP/fHTzzvHPD/Y3M0f/+MOHv3iVVw2ZvwhyTwKd5EFKisYoDAnTGFOeNrvae0e4IbvbuELD9Xxiz+5WkNAZzDdHy7iI8W5EX702Vp6Boa456E6+sfdOSyZQ+Eu4jMXLc7nf92+ntePd/C1J17Hq5FfxVsKdxEf+siaxfz5Ry7k6V0n+P7vD3hdjnhAbe4iPnXfNat452QX//2f3+E9ZXl8+BL1oMkkOnMX8Skz41u3rOPSigK+8shO3jnZ5XVJch4p3EV8LJoV5IE7a4lFQnz+oddo6xn0uiQ5TxTuIj63uCDKA5+ppbFzgPse3k78HMMpiH8o3EUywPqqQr518zpeOdjG15950+ty5DzQBVWRDHFjTQVvn+ziB78/wEVL8rnzyuVelyTzSGfuIhnkP370Qv7VRWV8/Zk32XqgxetyZB4p3EUySDBg/O0d61lVGuO+h3fQ2R/3uiSZJwp3kQyTF83iv928jvbeOL/c3eB1OTJPFO4iGWh9VSEXlOXy2PZjXpci80ThLpKBzIxbayvZebSd/U26ucmPFO4iGerGmgqCAeOx7fVelyLzQOEukqHK8qJ88MJSfr7jOEO6scl3FO4iGeyWDVU0dQ3w4rvqFuk3CneRDPahi8ooioV1YdWHFO4iGSwcCnDj+gp+81YTpzSomK8o3EUy3C0bKhkcHuHpXce9LkVSSOEukuEuWZrPmqX56jXjMwp3EeHWDZW8eaKTt050el2KpMi0wt3MrjOzd8xsv5n9xQSv32VmzWa2Kzl9PvWlish8uWF9BeFgQBdWfWTKcDezIPBd4GPAJcCnzOySCVb9mXNufXL6UYrrFJF5tCgW5sOXlPH0rhMMDqnPux9M58x9I7DfOXfQOTcIPALcML9licj5duuGKtp6BvmXt5u8LkVSYDrhXgGM/a5Wn1w23s1m9rqZPW5mVRP9ITO7x8zqzKyuubl5FuWKyHx53wUllOVFeFxNM74wnXC3CZa5cc9/AVQ759YBvwF+MtEfcs494Jyrdc7VlpaWzqxSEZlXoWCAT15eyfPvNNPU1e91OTJH0wn3emDsmXglcGLsCs65VufcQPLpD4ENqSlPRM6nWzZUMjzieGqn+rynu+mE+2vABWa2wszCwB3AM2NXMLMlY55+AtibuhJF5HxZXZZLzbJCHqurx7nxX9AlnUwZ7s65IeDLwD+TCO1HnXNvmtk3zOwTydX+1MzeNLPdwJ8Cd81XwSIyv27dUMW7Td3sru/wuhSZA/Pq6FxbW+vq6uo8+WwRmVxnf5yN3/wNN19eyTdvutTrcmQcM9vunKudaj3doSoiZ8iPZnHdmsU8s/sE/fFhr8uRWVK4i8hZbq2toqt/iF+91eh1KTJLCncROctVK4upKMzmsTr1eU9XCncROUsgYNx8eQUv7W/hRHuf1+XILCjcRWRCt2yowjl4coeGAk5HCncRmdCy4hyuWFHE49vV5z0dKdxFZFK31lZxuLWX1w6f8roUmSGFu4hM6vpLF5MXCfGXP9/DsbZer8uRGVC4i8ikcsIh/uEzG2jq7Oem721h51GdwacLhbuInNOmVSU8ed9mssNB7njgFZ7b0+B1STINCncRmdLqslyeum8za5bm88cP7+AHvz+gi6wLnMJdRKalODfCP33hSj6+bgl/89zb/Kcn9xAf1k/yLVQhrwsQkfQRzQrynTtqqC6Ocf/z+6k/1cf3Pn05+dEsr0uTcXTmLiIzEggYf/7RC/nWLet45WArN39vq3rSLEAKdxGZldtqq3jo7o00JnvS7DrW7nVJMobCXURmbWxPmtv/4WWe2F7PqZ5Br8sS9GMdIpICrd0DfOGhOnYcTZy9F2RnUV0SY2VJjOriGCtKY6wojlFdkkOe2ufnZLo/1qELqiIyZ8W5EX56z5Vs3d/KgeZuDrX0cLi1h1cPtfHzcT+2XZIbYUVJDksLsynNjVCaF6Ek+Tg6FeWECQTMo63xB4W7iKREJBTkgxeV8cGLys5Y3h8f5khrL4daehKh39LDodYedh1rp6lzgL4Jfu0pGDCKY+Gzgz83QknycXRZfjSEmQ4E4yncRWReRbOCXLg4jwsX5034es/AEM1dAzR3D9DcNUBL8nF0auoaYF9jFy3dA8SHz25GDgcDiYNAXoTS3DDFsQgleWFKciOnp9Lk84LsrIw5ECjcRcRTsUiIWCREdUnsnOs55+joi/8h+LvHPXYNcLy9n931HbT1DDI8cvaBIBQwipMHgILsrD9MOYnH/DHL8qOhM9YJBdOr/4nCXUTSgplRmBOmMCfMBeUTfwsYNTLiONU7SEv3IC3dA8kpOd81QFvPIB19cQ40d9PRF6ejL87A0Lnvts2NhMYcABLzhdnhMw4MeZEQuZEQedEQudEQeZGs0/NZ5/ngoHAXEd8JBIzi3AjFuREu5NwHglH98WE6++N0JsP+9NQbp6NvaMyyxIHhUEsPHX3tdPTF6Y9PPQxDJBQgL5oI+397xTI+/76Vc93Mc1K4i4iQuDYQzQpSlhed8XsHhobp6IvT3T9E98AQ3f1DdJ6ej9OVnO8aGKKrf4iS3Mg8bMGZFO4iInMUCQUpywtSNr0vCedFel0hEBGRaVG4i4j4kMJdRMSHFO4iIj6kcBcR8SGFu4iIDyncRUR8SOEuIuJDnv1Yh5k1A0dm+fYSoCWF5SwEftsmv20P+G+b/LY94L9tmmh7ljvnSqd6o2fhPhdmVjedXyJJJ37bJr9tD/hvm/y2PeC/bZrL9qhZRkTEhxTuIiI+lK7h/oDXBcwDv22T37YH/LdNftse8N82zXp70rLNXUREzi1dz9xFROQc0i7czew6M3vHzPab2V94XU8qmNlhM9tjZrvMrM7rembKzB40syYze2PMsiIz+7WZvZt8XORljTM1yTZ93cyOJ/fTLjO73ssaZ8LMqszseTPba2ZvmtlXksvTcj+dY3vSeR9FzexVM9ud3Kb/kly+wsy2JffRz8wsPK2/l07NMmYWBPYB1wL1wGvAp5xzb3la2ByZ2WGg1jmXlv1zzez9QDfwkHNubXLZt4A259zfJA/Ci5xzX/OyzpmYZJu+DnQ75/6Hl7XNhpktAZY453aYWR6wHbgRuIs03E/n2J7bSN99ZEDMOddtZlnAS8BXgP8APOmce8TMfgDsds59f6q/l25n7huB/c65g865QeAR4AaPa8p4zrkXgLZxi28AfpKc/wmJ/3hpY5JtSlvOuQbn3I7kfBewF6ggTffTObYnbbmE7uTTrOTkgA8BjyeXT3sfpVu4VwDHxjyvJ813aJIDfmVm283sHq+LSZFy51wDJP4jAmUe15MqXzaz15PNNmnRhDGemVUDNcA2fLCfxm0PpPE+MrOgme0CmoBfAweAdufcUHKVaWdeuoW7TbAsfdqVJrfZOXc58DHgS8kmAVl4vg+sAtYDDcD/9LacmTOzXOAJ4KvOuU6v65mrCbYnrfeRc27YObceqCTRUnHxRKtN52+lW7jXA1VjnlcCJzyqJWWccyeSj03Az0ns1HTXmGwXHW0fbfK4njlzzjUm//ONAD8kzfZTsh33CeBh59yTycVpu58m2p5030ejnHPtwO+AK4FCMwslX5p25qVbuL8GXJC8ehwG7gCe8bimOTGzWPKCEGYWAz4CvHHud6WFZ4DPJuc/CzztYS0pMRqCSTeRRvspebHux8Be59y3x7yUlvtpsu1J831UamaFyfls4MMkriU8D9ySXG3a+yitessAJLs2/S0QBB50zn3T45LmxMxWkjhbBwgB/5Ru22RmPwWuITGCXSPw18BTwKPAMuAocKtzLm0uUE6yTdeQ+LrvgMPAF0fbqxc6M7saeBHYA4wkF/8liXbqtNtP59ieT5G++2gdiQumQRIn3o86576RzIhHgCJgJ/Bp59zAlH8v3cJdRESmlm7NMiIiMg0KdxERH1K4i4j4kMJdRMSHFO4iIj6kcBcR8SGFu4iIDyncRUR86P8DKGNx/8fQOpsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plt.plot(np.asnumpy(fit_history[fit_history != -1]) * -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Performance of FCNN trained with normal backpropagation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is our performance using normal FCNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance on training data : 97.47619047619047 %\n"
     ]
    }
   ],
   "source": [
    "print(f\"Performance on training data : {nn.score(train_pix,train_lab)} %\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance on test data : 95.54761904761905 %\n"
     ]
    }
   ],
   "source": [
    "print(f\"Performance on test data : {nn.score(test_pix,test_lab)} %\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Not good, but eh, not our main goal.**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Score-trained NN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Just remember to make it extra wide."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ScoreFCNN(object):\n",
    "    \n",
    "    def __init__(self, layers_size,topKPercentage,initializer_weight,initializer_score,activation,loss_function,optimizer):\n",
    "#         assert len(layers_size) == (len(topKPercentage)+1)\n",
    "        self.layers_size = layers_size\n",
    "        self.weights = [initializer_weight((k,j)) for j,k in zip(layers_size[:-1],layers_size[1:])]\n",
    "        self.scores = [initializer_score(w.shape) for w in self.weights]\n",
    "        self.topK = [int(j*k*topKPercentage) for j,k in zip(layers_size[:-1],layers_size[1:])]\n",
    "        assert len(self.weights) == len(self.scores)\n",
    "        self.activation = activation\n",
    "        self.loss_function = loss_function\n",
    "        self.optimizer = optimizer\n",
    "    \n",
    "    def feedforward(self,X):\n",
    "        a = X\n",
    "        zs = []\n",
    "        acs = [a]\n",
    "        for w,s,k in zip(self.weights,self.scores,self.topK):\n",
    "            ss = s.copy()\n",
    "            ww = w.copy()\n",
    "#             assert not np.shares_memory(ss,s)\n",
    "#             assert not np.shares_memory(ww,w)\n",
    "#             ss = np.abs(ss)\n",
    "            ss_flat = ss.ravel()\n",
    "            sort_idx = ss_flat.argsort()\n",
    "            ss_flat[sort_idx[-k:]] = 1\n",
    "            ss_flat[sort_idx[:-k]] = 0\n",
    "            assert ss.shape == w.shape\n",
    "            ww = ww * ss\n",
    "#             print(ww)\n",
    "#             print(ww.shape)\n",
    "            a = np.dot(ww,a)\n",
    "            zs.append(a)\n",
    "            a = self.activation(a)\n",
    "            acs.append(a)\n",
    "        return a, zs, acs\n",
    "    \n",
    "    def feedforward_softmax(self,X):\n",
    "        final_a, _, _ = self.feedforward(X)\n",
    "        final_a = np.exp(final_a)\n",
    "        final_a = final_a / np.sum(final_a,axis=0)\n",
    "        return final_a\n",
    "    \n",
    "    def backpropagation(self,X,y):\n",
    "        input_dim, batch_size = X.shape\n",
    "        final_activation, zs, acs = self.feedforward(X)\n",
    "        delta_s = [np.zeros(s.shape) for s in self.scores]\n",
    "        error = self.loss_function(final_activation,y,prime=True) * self.activation(zs[-1],True)\n",
    "        delta_s[-1] += np.einsum('ik,jk->ijk',error,acs[-2]).mean(axis=2)\n",
    "        for i in range(2,len(self.layers_size)):\n",
    "            error = np.einsum('ik,ij->jk',error,self.weights[-i+1]) * self.activation(zs[-i],prime=True)\n",
    "            delta_s[-i] += np.einsum('ik,jk->ijk',error,acs[-i-1]).mean(axis=2) * self.weights[-i]\n",
    "        return delta_s\n",
    "    \n",
    "    def fit(self,X,y,batch_size,epochs):\n",
    "        # expecting X and y to be already numpy array with the num_samples as \n",
    "        # 1st dim\n",
    "        history = np.zeros(epochs + 1)\n",
    "        history -= 1\n",
    "        index = 1\n",
    "        for epoch in range(epochs):\n",
    "            indices = np.arange(X.shape[0],dtype=np.int32)\n",
    "            np.random.shuffle(indices)\n",
    "            X = X.copy()[indices,:]\n",
    "            y = y.copy()[indices]\n",
    "            i = 0\n",
    "            batches_X = [X[k:k+batch_size] for k in range(0,X.shape[0],batch_size)]\n",
    "            batches_y = [y[k:k+batch_size] for k in range(0,y.shape[0],batch_size)]\n",
    "            losses = 0\n",
    "            if index == 0:\n",
    "                    for Xb, yb in zip(batches_X,batches_y):\n",
    "                        logits,_,_ = self.feedforward(Xb.T)\n",
    "                        history[0] += self.loss_function(logits,yb.T) / (Xb.shape[0] * len(batches_X))\n",
    "            for Xb, yb in zip(batches_X,batches_y):\n",
    "                delta_s = self.backpropagation(Xb.T, yb.T)\n",
    "#                 print(delta_w)\n",
    "                self.scores = self.optimizer(delta_s,self.scores)\n",
    "                logits,_,_ = self.feedforward(Xb.T)\n",
    "                losses += self.loss_function(logits,yb.T) / (Xb.shape[0] * len(batches_X))\n",
    "#                 print((Xb.shape[0] * len(batches_X)))\n",
    "            history[index] = losses\n",
    "            index += 1\n",
    "            print(f\"Epoch {epoch} done. Loss : {losses}\")\n",
    "        return history\n",
    "    \n",
    "    def predict(self,X):\n",
    "        softmaxed = self.feedforward_softmax(X.T)\n",
    "        preds = np.argmax(softmaxed,axis=0)\n",
    "        return preds\n",
    "    \n",
    "    def score(self, X, y):\n",
    "        preds = self.predict(X)\n",
    "        assert preds.shape == y.shape\n",
    "        return np.mean(preds == y) * 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**First i overfit the model to make sure everything works**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = MomentumSGD(0.1,0.9)\n",
    "scorenn = ScoreFCNN([784,200,200,10],0.5,kaiming_uniform,kaiming_uniform,relu,crossentropy_with_logits,optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 done. Loss : -3.2425714835989865\n",
      "Epoch 1 done. Loss : -3.2354897343219866\n",
      "Epoch 2 done. Loss : -3.2337572799238345\n",
      "Epoch 3 done. Loss : -3.227333682029196\n",
      "Epoch 4 done. Loss : -3.222114689995363\n",
      "Epoch 5 done. Loss : -3.207706201705143\n",
      "Epoch 6 done. Loss : -3.1894967775216188\n",
      "Epoch 7 done. Loss : -3.177335603944957\n",
      "Epoch 8 done. Loss : -3.168136392260462\n",
      "Epoch 9 done. Loss : -3.155187095102973\n",
      "Epoch 10 done. Loss : -3.1393059641762067\n",
      "Epoch 11 done. Loss : -3.1370253041790495\n",
      "Epoch 12 done. Loss : -3.1288555899404544\n",
      "Epoch 13 done. Loss : -3.1157260003711262\n",
      "Epoch 14 done. Loss : -3.1056316341944807\n",
      "Epoch 15 done. Loss : -3.0955540912766804\n",
      "Epoch 16 done. Loss : -3.077880703934052\n",
      "Epoch 17 done. Loss : -3.06199027221588\n",
      "Epoch 18 done. Loss : -3.057015508268693\n",
      "Epoch 19 done. Loss : -3.048093664666508\n",
      "Epoch 20 done. Loss : -3.041575996415699\n",
      "Epoch 21 done. Loss : -3.0316802154521936\n",
      "Epoch 22 done. Loss : -3.023976337046328\n",
      "Epoch 23 done. Loss : -3.016236083805995\n",
      "Epoch 24 done. Loss : -3.0083860901113786\n",
      "Epoch 25 done. Loss : -2.9985973830442205\n",
      "Epoch 26 done. Loss : -2.987223848283601\n",
      "Epoch 27 done. Loss : -2.9819681876349557\n",
      "Epoch 28 done. Loss : -2.9727329241812583\n",
      "Epoch 29 done. Loss : -2.9698529497188435\n",
      "Epoch 30 done. Loss : -2.960549956654095\n",
      "Epoch 31 done. Loss : -2.9426337281940236\n",
      "Epoch 32 done. Loss : -2.93111744722647\n",
      "Epoch 33 done. Loss : -2.913578533375831\n",
      "Epoch 34 done. Loss : -2.9055913734930767\n",
      "Epoch 35 done. Loss : -2.886533397398618\n",
      "Epoch 36 done. Loss : -2.8751085748201053\n",
      "Epoch 37 done. Loss : -2.870420165753404\n",
      "Epoch 38 done. Loss : -2.8625615251341716\n",
      "Epoch 39 done. Loss : -2.846740186238559\n",
      "Epoch 40 done. Loss : -2.8329180790904105\n",
      "Epoch 41 done. Loss : -2.8221633668469774\n",
      "Epoch 42 done. Loss : -2.804793841243492\n",
      "Epoch 43 done. Loss : -2.7874808870201715\n",
      "Epoch 44 done. Loss : -2.773101754122993\n",
      "Epoch 45 done. Loss : -2.7513446840329223\n",
      "Epoch 46 done. Loss : -2.7392186451656744\n",
      "Epoch 47 done. Loss : -2.710675781097792\n",
      "Epoch 48 done. Loss : -2.7076020020399287\n",
      "Epoch 49 done. Loss : -2.7009304646728385\n",
      "Epoch 50 done. Loss : -2.687038323380577\n",
      "Epoch 51 done. Loss : -2.679132316167579\n",
      "Epoch 52 done. Loss : -2.6729613108497565\n",
      "Epoch 53 done. Loss : -2.66019335867133\n",
      "Epoch 54 done. Loss : -2.651210858165932\n",
      "Epoch 55 done. Loss : -2.640369497776317\n",
      "Epoch 56 done. Loss : -2.631133107481687\n",
      "Epoch 57 done. Loss : -2.6179083645534273\n",
      "Epoch 58 done. Loss : -2.610190659835224\n",
      "Epoch 59 done. Loss : -2.5978358174927747\n",
      "Epoch 60 done. Loss : -2.5822231200796053\n",
      "Epoch 61 done. Loss : -2.5732603815765893\n",
      "Epoch 62 done. Loss : -2.557488240970728\n",
      "Epoch 63 done. Loss : -2.561146944975169\n",
      "Epoch 64 done. Loss : -2.546696462577379\n",
      "Epoch 65 done. Loss : -2.5367362756545644\n",
      "Epoch 66 done. Loss : -2.5305993416437573\n",
      "Epoch 67 done. Loss : -2.512201074233883\n",
      "Epoch 68 done. Loss : -2.5053457707153886\n",
      "Epoch 69 done. Loss : -2.4899498171225742\n",
      "Epoch 70 done. Loss : -2.4750806786317816\n",
      "Epoch 71 done. Loss : -2.4578069196192227\n",
      "Epoch 72 done. Loss : -2.4339198986486394\n",
      "Epoch 73 done. Loss : -2.4256435210048997\n",
      "Epoch 74 done. Loss : -2.424315854005899\n",
      "Epoch 75 done. Loss : -2.4122766626279875\n",
      "Epoch 76 done. Loss : -2.3939982595248064\n",
      "Epoch 77 done. Loss : -2.386188286097288\n",
      "Epoch 78 done. Loss : -2.375143318792415\n",
      "Epoch 79 done. Loss : -2.369840814064695\n",
      "Epoch 80 done. Loss : -2.370878379668471\n",
      "Epoch 81 done. Loss : -2.3599404931871995\n",
      "Epoch 82 done. Loss : -2.349004955161064\n",
      "Epoch 83 done. Loss : -2.3420735791921046\n",
      "Epoch 84 done. Loss : -2.3297419792413523\n",
      "Epoch 85 done. Loss : -2.319681662888886\n",
      "Epoch 86 done. Loss : -2.3109828610943395\n",
      "Epoch 87 done. Loss : -2.3081703449743376\n",
      "Epoch 88 done. Loss : -2.307639939629273\n",
      "Epoch 89 done. Loss : -2.305120111815581\n",
      "Epoch 90 done. Loss : -2.3095307596685295\n",
      "Epoch 91 done. Loss : -2.299724579744129\n",
      "Epoch 92 done. Loss : -2.2851383617315957\n",
      "Epoch 93 done. Loss : -2.286245184556896\n",
      "Epoch 94 done. Loss : -2.2750141791550984\n",
      "Epoch 95 done. Loss : -2.2790931679213484\n",
      "Epoch 96 done. Loss : -2.2717033283285497\n",
      "Epoch 97 done. Loss : -2.258568359695486\n",
      "Epoch 98 done. Loss : -2.254512064033243\n",
      "Epoch 99 done. Loss : -2.2442299771691725\n",
      "Epoch 100 done. Loss : -2.238288267870943\n",
      "Epoch 101 done. Loss : -2.229001596403639\n",
      "Epoch 102 done. Loss : -2.221783907883057\n",
      "Epoch 103 done. Loss : -2.2188266002104555\n",
      "Epoch 104 done. Loss : -2.228307359244539\n",
      "Epoch 105 done. Loss : -2.221119774519641\n",
      "Epoch 106 done. Loss : -2.2228759934130125\n",
      "Epoch 107 done. Loss : -2.2150257205759227\n",
      "Epoch 108 done. Loss : -2.201245398766435\n",
      "Epoch 109 done. Loss : -2.1959428050243117\n",
      "Epoch 110 done. Loss : -2.1919477992980765\n",
      "Epoch 111 done. Loss : -2.19377093371314\n",
      "Epoch 112 done. Loss : -2.189018378106329\n",
      "Epoch 113 done. Loss : -2.175353599799471\n",
      "Epoch 114 done. Loss : -2.167566001238815\n",
      "Epoch 115 done. Loss : -2.163336260870161\n",
      "Epoch 116 done. Loss : -2.1576955620954275\n",
      "Epoch 117 done. Loss : -2.157606901865008\n",
      "Epoch 118 done. Loss : -2.1497821531840655\n",
      "Epoch 119 done. Loss : -2.152435729825465\n",
      "Epoch 120 done. Loss : -2.1446669792405255\n",
      "Epoch 121 done. Loss : -2.1388858489146827\n",
      "Epoch 122 done. Loss : -2.1364298345747867\n",
      "Epoch 123 done. Loss : -2.1251485339713643\n",
      "Epoch 124 done. Loss : -2.1240867073671588\n",
      "Epoch 125 done. Loss : -2.112409811432541\n",
      "Epoch 126 done. Loss : -2.111312793923772\n",
      "Epoch 127 done. Loss : -2.099788436801163\n",
      "Epoch 128 done. Loss : -2.089673847554609\n",
      "Epoch 129 done. Loss : -2.0855553244852416\n",
      "Epoch 130 done. Loss : -2.081336048765727\n",
      "Epoch 131 done. Loss : -2.079637392827578\n",
      "Epoch 132 done. Loss : -2.079405950555878\n",
      "Epoch 133 done. Loss : -2.0768984528045027\n",
      "Epoch 134 done. Loss : -2.0699585796353155\n",
      "Epoch 135 done. Loss : -2.0678991033332874\n",
      "Epoch 136 done. Loss : -2.064238660694685\n",
      "Epoch 137 done. Loss : -2.0617581737917337\n",
      "Epoch 138 done. Loss : -2.0538434621828534\n",
      "Epoch 139 done. Loss : -2.0534699972006014\n",
      "Epoch 140 done. Loss : -2.0467356569270443\n",
      "Epoch 141 done. Loss : -2.0445943352754323\n",
      "Epoch 142 done. Loss : -2.0450163367853804\n",
      "Epoch 143 done. Loss : -2.034724847278416\n",
      "Epoch 144 done. Loss : -2.026268778841299\n",
      "Epoch 145 done. Loss : -2.0182777209512874\n",
      "Epoch 146 done. Loss : -2.017312209773744\n",
      "Epoch 147 done. Loss : -2.0143813478586776\n",
      "Epoch 148 done. Loss : -2.01346146545222\n",
      "Epoch 149 done. Loss : -2.008400380028583\n",
      "Epoch 150 done. Loss : -2.0028066764816033\n",
      "Epoch 151 done. Loss : -2.0003839731593627\n",
      "Epoch 152 done. Loss : -2.002456565522766\n",
      "Epoch 153 done. Loss : -2.0016390043139825\n",
      "Epoch 154 done. Loss : -1.995998606005118\n",
      "Epoch 155 done. Loss : -1.9879373159400653\n",
      "Epoch 156 done. Loss : -1.9841978633250674\n",
      "Epoch 157 done. Loss : -1.9835225130796266\n",
      "Epoch 158 done. Loss : -1.9758076283668178\n",
      "Epoch 159 done. Loss : -1.9628605134067607\n",
      "Epoch 160 done. Loss : -1.9583412911687008\n",
      "Epoch 161 done. Loss : -1.9518134261751492\n",
      "Epoch 162 done. Loss : -1.9506046619334803\n",
      "Epoch 163 done. Loss : -1.953686353871152\n",
      "Epoch 164 done. Loss : -1.9439786052449597\n",
      "Epoch 165 done. Loss : -1.9382487315922359\n",
      "Epoch 166 done. Loss : -1.9378129764580065\n",
      "Epoch 167 done. Loss : -1.9401393427190112\n",
      "Epoch 168 done. Loss : -1.9280556828385103\n",
      "Epoch 169 done. Loss : -1.9255102455149622\n",
      "Epoch 170 done. Loss : -1.9154018510596997\n",
      "Epoch 171 done. Loss : -1.9077422431191864\n",
      "Epoch 172 done. Loss : -1.9031577487749545\n",
      "Epoch 173 done. Loss : -1.9010275316923728\n",
      "Epoch 174 done. Loss : -1.9060879106189028\n",
      "Epoch 175 done. Loss : -1.9056806216168174\n",
      "Epoch 176 done. Loss : -1.9066678394080903\n",
      "Epoch 177 done. Loss : -1.9078725253353745\n",
      "Epoch 178 done. Loss : -1.9096419929456006\n",
      "Epoch 179 done. Loss : -1.915411646068474\n",
      "Epoch 180 done. Loss : -1.908577664400661\n",
      "Epoch 181 done. Loss : -1.9074629619424779\n",
      "Epoch 182 done. Loss : -1.9022765524834286\n",
      "Epoch 183 done. Loss : -1.9085162866731609\n",
      "Epoch 184 done. Loss : -1.9065084559178629\n",
      "Epoch 185 done. Loss : -1.9133979432322976\n",
      "Epoch 186 done. Loss : -1.913689188654839\n",
      "Epoch 187 done. Loss : -1.9157164388962706\n",
      "Epoch 188 done. Loss : -1.9105827190009685\n",
      "Epoch 189 done. Loss : -1.9064144649425583\n",
      "Epoch 190 done. Loss : -1.9053210334072208\n",
      "Epoch 191 done. Loss : -1.8956969088631748\n",
      "Epoch 192 done. Loss : -1.892605282555751\n",
      "Epoch 193 done. Loss : -1.8914772577646413\n",
      "Epoch 194 done. Loss : -1.886236590988759\n",
      "Epoch 195 done. Loss : -1.880947262336555\n",
      "Epoch 196 done. Loss : -1.8795655220443503\n",
      "Epoch 197 done. Loss : -1.8744626028918865\n",
      "Epoch 198 done. Loss : -1.8698188157649422\n",
      "Epoch 199 done. Loss : -1.8707512422632084\n",
      "Epoch 200 done. Loss : -1.8700805095183486\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 201 done. Loss : -1.8764023938655194\n",
      "Epoch 202 done. Loss : -1.8779028247232512\n",
      "Epoch 203 done. Loss : -1.8737513367060363\n",
      "Epoch 204 done. Loss : -1.8679204774255571\n",
      "Epoch 205 done. Loss : -1.8654483334746776\n",
      "Epoch 206 done. Loss : -1.8628010457345485\n",
      "Epoch 207 done. Loss : -1.8547658783836174\n",
      "Epoch 208 done. Loss : -1.8536864235760648\n",
      "Epoch 209 done. Loss : -1.8511545456850111\n",
      "Epoch 210 done. Loss : -1.8492703586718715\n",
      "Epoch 211 done. Loss : -1.8487244534465206\n",
      "Epoch 212 done. Loss : -1.8492230387654394\n",
      "Epoch 213 done. Loss : -1.8450819065644206\n",
      "Epoch 214 done. Loss : -1.8398250455455307\n",
      "Epoch 215 done. Loss : -1.8369457380362468\n",
      "Epoch 216 done. Loss : -1.8334412764224948\n",
      "Epoch 217 done. Loss : -1.8326291881053174\n",
      "Epoch 218 done. Loss : -1.831424875074343\n",
      "Epoch 219 done. Loss : -1.8300040106094375\n",
      "Epoch 220 done. Loss : -1.825511245410273\n",
      "Epoch 221 done. Loss : -1.8221484076939665\n",
      "Epoch 222 done. Loss : -1.8229731550338069\n",
      "Epoch 223 done. Loss : -1.8290104584994706\n",
      "Epoch 224 done. Loss : -1.8264270355107024\n",
      "Epoch 225 done. Loss : -1.8205836353104246\n",
      "Epoch 226 done. Loss : -1.8203958424618523\n",
      "Epoch 227 done. Loss : -1.817725655955401\n",
      "Epoch 228 done. Loss : -1.8130695181278762\n",
      "Epoch 229 done. Loss : -1.8148030399504416\n",
      "Epoch 230 done. Loss : -1.8108150575125936\n",
      "Epoch 231 done. Loss : -1.8056517045540132\n",
      "Epoch 232 done. Loss : -1.802945089226226\n",
      "Epoch 233 done. Loss : -1.7984648237325334\n",
      "Epoch 234 done. Loss : -1.7901855430924511\n",
      "Epoch 235 done. Loss : -1.7877617237360934\n",
      "Epoch 236 done. Loss : -1.7838697472388345\n",
      "Epoch 237 done. Loss : -1.7761809794444412\n",
      "Epoch 238 done. Loss : -1.7717459109805707\n",
      "Epoch 239 done. Loss : -1.7677932273809522\n",
      "Epoch 240 done. Loss : -1.7800676874771597\n",
      "Epoch 241 done. Loss : -1.7641510358556005\n",
      "Epoch 242 done. Loss : -1.76187210030454\n",
      "Epoch 243 done. Loss : -1.7647850221035033\n",
      "Epoch 244 done. Loss : -1.7586101752424295\n",
      "Epoch 245 done. Loss : -1.7559179468997335\n",
      "Epoch 246 done. Loss : -1.7572397187790352\n",
      "Epoch 247 done. Loss : -1.757448623959332\n",
      "Epoch 248 done. Loss : -1.7624880486940786\n",
      "Epoch 249 done. Loss : -1.7608430738287644\n",
      "Epoch 250 done. Loss : -1.764839765320192\n",
      "Epoch 251 done. Loss : -1.758777644767765\n",
      "Epoch 252 done. Loss : -1.7620117383586194\n",
      "Epoch 253 done. Loss : -1.7603724060032315\n",
      "Epoch 254 done. Loss : -1.7674745450156684\n",
      "Epoch 255 done. Loss : -1.7734263444882874\n",
      "Epoch 256 done. Loss : -1.7712785056548106\n",
      "Epoch 257 done. Loss : -1.7690820033911243\n",
      "Epoch 258 done. Loss : -1.7696156881087273\n",
      "Epoch 259 done. Loss : -1.7681653831558517\n",
      "Epoch 260 done. Loss : -1.7642332423531009\n",
      "Epoch 261 done. Loss : -1.7616141720817833\n",
      "Epoch 262 done. Loss : -1.7458355012082198\n",
      "Epoch 263 done. Loss : -1.746933110784524\n",
      "Epoch 264 done. Loss : -1.7431663319981192\n",
      "Epoch 265 done. Loss : -1.7389793732628451\n",
      "Epoch 266 done. Loss : -1.742776243351393\n",
      "Epoch 267 done. Loss : -1.734755330511203\n",
      "Epoch 268 done. Loss : -1.731842667840462\n",
      "Epoch 269 done. Loss : -1.7317448290186426\n",
      "Epoch 270 done. Loss : -1.727194280817248\n",
      "Epoch 271 done. Loss : -1.7303974551998238\n",
      "Epoch 272 done. Loss : -1.72897659329264\n",
      "Epoch 273 done. Loss : -1.725025787019674\n",
      "Epoch 274 done. Loss : -1.7226520488361807\n",
      "Epoch 275 done. Loss : -1.7151962261291873\n",
      "Epoch 276 done. Loss : -1.7128371910268965\n",
      "Epoch 277 done. Loss : -1.7109522837212303\n",
      "Epoch 278 done. Loss : -1.7043628278363903\n",
      "Epoch 279 done. Loss : -1.6984455384443278\n",
      "Epoch 280 done. Loss : -1.6988092081176007\n",
      "Epoch 281 done. Loss : -1.7005267912629738\n",
      "Epoch 282 done. Loss : -1.7021306714694482\n",
      "Epoch 283 done. Loss : -1.7016621928003222\n",
      "Epoch 284 done. Loss : -1.6949880042712677\n",
      "Epoch 285 done. Loss : -1.6927981932035059\n",
      "Epoch 286 done. Loss : -1.690583210921261\n",
      "Epoch 287 done. Loss : -1.6890507589908372\n",
      "Epoch 288 done. Loss : -1.6852221252037516\n",
      "Epoch 289 done. Loss : -1.68421939962837\n",
      "Epoch 290 done. Loss : -1.691811541688626\n",
      "Epoch 291 done. Loss : -1.6908945064888488\n",
      "Epoch 292 done. Loss : -1.6867113409761785\n",
      "Epoch 293 done. Loss : -1.6844507228078274\n",
      "Epoch 294 done. Loss : -1.6811305684064166\n",
      "Epoch 295 done. Loss : -1.6777756805069666\n",
      "Epoch 296 done. Loss : -1.6751653215600129\n",
      "Epoch 297 done. Loss : -1.6752523413032656\n",
      "Epoch 298 done. Loss : -1.664721462573079\n",
      "Epoch 299 done. Loss : -1.665282289050567\n",
      "Epoch 300 done. Loss : -1.6629031155670115\n",
      "Epoch 301 done. Loss : -1.6594682710146638\n",
      "Epoch 302 done. Loss : -1.659532303365292\n",
      "Epoch 303 done. Loss : -1.6684374287378358\n",
      "Epoch 304 done. Loss : -1.6702099864205788\n",
      "Epoch 305 done. Loss : -1.6709316819057185\n",
      "Epoch 306 done. Loss : -1.6682569913078864\n",
      "Epoch 307 done. Loss : -1.6589391607699349\n",
      "Epoch 308 done. Loss : -1.6549364542795888\n",
      "Epoch 309 done. Loss : -1.6522860168703515\n",
      "Epoch 310 done. Loss : -1.6574967684494932\n",
      "Epoch 311 done. Loss : -1.6571706425639188\n",
      "Epoch 312 done. Loss : -1.659055207049004\n",
      "Epoch 313 done. Loss : -1.6520134792447503\n",
      "Epoch 314 done. Loss : -1.6462441633553868\n",
      "Epoch 315 done. Loss : -1.6523018981257018\n",
      "Epoch 316 done. Loss : -1.6527689835807604\n",
      "Epoch 317 done. Loss : -1.650516644629029\n",
      "Epoch 318 done. Loss : -1.6464879866701012\n",
      "Epoch 319 done. Loss : -1.6475493908585683\n",
      "Epoch 320 done. Loss : -1.6407469605234504\n",
      "Epoch 321 done. Loss : -1.6419779535349752\n",
      "Epoch 322 done. Loss : -1.635246776817914\n",
      "Epoch 323 done. Loss : -1.6331215224875404\n",
      "Epoch 324 done. Loss : -1.6355755354795076\n",
      "Epoch 325 done. Loss : -1.6349343252742168\n",
      "Epoch 326 done. Loss : -1.6346452502872542\n",
      "Epoch 327 done. Loss : -1.630688764790933\n",
      "Epoch 328 done. Loss : -1.6308607452698467\n",
      "Epoch 329 done. Loss : -1.62574599076394\n",
      "Epoch 330 done. Loss : -1.6203943800469855\n",
      "Epoch 331 done. Loss : -1.6243785306845004\n",
      "Epoch 332 done. Loss : -1.6227139506141186\n",
      "Epoch 333 done. Loss : -1.6156811157043895\n",
      "Epoch 334 done. Loss : -1.6174376123102303\n",
      "Epoch 335 done. Loss : -1.6175173227174695\n",
      "Epoch 336 done. Loss : -1.614370415665234\n",
      "Epoch 337 done. Loss : -1.6152201139926496\n",
      "Epoch 338 done. Loss : -1.6173979279162882\n",
      "Epoch 339 done. Loss : -1.6137530907765618\n",
      "Epoch 340 done. Loss : -1.6194169027995489\n",
      "Epoch 341 done. Loss : -1.6233541427654496\n",
      "Epoch 342 done. Loss : -1.6125031537491092\n",
      "Epoch 343 done. Loss : -1.6114786731245108\n",
      "Epoch 344 done. Loss : -1.6102742033426374\n",
      "Epoch 345 done. Loss : -1.6113075141460147\n",
      "Epoch 346 done. Loss : -1.6063384307115014\n",
      "Epoch 347 done. Loss : -1.6057597883662034\n",
      "Epoch 348 done. Loss : -1.6057631977109126\n",
      "Epoch 349 done. Loss : -1.6053020957220272\n",
      "Epoch 350 done. Loss : -1.605987934510721\n",
      "Epoch 351 done. Loss : -1.6001116785647131\n",
      "Epoch 352 done. Loss : -1.599859988560726\n",
      "Epoch 353 done. Loss : -1.5998399149190894\n",
      "Epoch 354 done. Loss : -1.596041794398964\n",
      "Epoch 355 done. Loss : -1.5966539269750453\n",
      "Epoch 356 done. Loss : -1.595991483938795\n",
      "Epoch 357 done. Loss : -1.5926952034798598\n",
      "Epoch 358 done. Loss : -1.589742465128532\n",
      "Epoch 359 done. Loss : -1.5893105725692978\n",
      "Epoch 360 done. Loss : -1.5902384892794061\n",
      "Epoch 361 done. Loss : -1.590786691164217\n",
      "Epoch 362 done. Loss : -1.5928356259117296\n",
      "Epoch 363 done. Loss : -1.584253609970593\n",
      "Epoch 364 done. Loss : -1.5828929780090193\n",
      "Epoch 365 done. Loss : -1.5852070614753853\n",
      "Epoch 366 done. Loss : -1.5851047024762552\n",
      "Epoch 367 done. Loss : -1.5823158529784254\n",
      "Epoch 368 done. Loss : -1.5814690009855812\n",
      "Epoch 369 done. Loss : -1.577063077703238\n",
      "Epoch 370 done. Loss : -1.575510700290479\n",
      "Epoch 371 done. Loss : -1.5716451773733886\n",
      "Epoch 372 done. Loss : -1.57096276896479\n",
      "Epoch 373 done. Loss : -1.569880791706944\n",
      "Epoch 374 done. Loss : -1.5674939403717392\n",
      "Epoch 375 done. Loss : -1.5635588894276344\n",
      "Epoch 376 done. Loss : -1.577518417667958\n",
      "Epoch 377 done. Loss : -1.5583568225740203\n",
      "Epoch 378 done. Loss : -1.5582305329648878\n",
      "Epoch 379 done. Loss : -1.5647588605407585\n",
      "Epoch 380 done. Loss : -1.562215105064778\n",
      "Epoch 381 done. Loss : -1.5616333228378951\n",
      "Epoch 382 done. Loss : -1.5633811278107843\n",
      "Epoch 383 done. Loss : -1.5645084670800138\n",
      "Epoch 384 done. Loss : -1.5613495536741602\n",
      "Epoch 385 done. Loss : -1.558746674455183\n",
      "Epoch 386 done. Loss : -1.5628988614195953\n",
      "Epoch 387 done. Loss : -1.5620586291408394\n",
      "Epoch 388 done. Loss : -1.5621112608533314\n",
      "Epoch 389 done. Loss : -1.5625464103096598\n",
      "Epoch 390 done. Loss : -1.5643672273138134\n",
      "Epoch 391 done. Loss : -1.563333585532805\n",
      "Epoch 392 done. Loss : -1.5609113991961707\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 393 done. Loss : -1.5563992261319095\n",
      "Epoch 394 done. Loss : -1.5561375896189706\n",
      "Epoch 395 done. Loss : -1.559239203088886\n",
      "Epoch 396 done. Loss : -1.562947142276892\n",
      "Epoch 397 done. Loss : -1.5603923970884197\n",
      "Epoch 398 done. Loss : -1.5523531712574954\n",
      "Epoch 399 done. Loss : -1.5480319923232526\n",
      "Epoch 400 done. Loss : -1.5475465326242495\n",
      "Epoch 401 done. Loss : -1.5484004924060697\n",
      "Epoch 402 done. Loss : -1.545554521506536\n",
      "Epoch 403 done. Loss : -1.5431150110605032\n",
      "Epoch 404 done. Loss : -1.545666541555746\n",
      "Epoch 405 done. Loss : -1.5458647011186208\n",
      "Epoch 406 done. Loss : -1.5446743388928499\n",
      "Epoch 407 done. Loss : -1.5440373420340685\n",
      "Epoch 408 done. Loss : -1.544105388105959\n",
      "Epoch 409 done. Loss : -1.5438138096964815\n",
      "Epoch 410 done. Loss : -1.5394760318954341\n",
      "Epoch 411 done. Loss : -1.5387203354904413\n",
      "Epoch 412 done. Loss : -1.5385466652129383\n",
      "Epoch 413 done. Loss : -1.5327188640856289\n",
      "Epoch 414 done. Loss : -1.5382802082912779\n",
      "Epoch 415 done. Loss : -1.5333895119033412\n",
      "Epoch 416 done. Loss : -1.5322805481014778\n",
      "Epoch 417 done. Loss : -1.5312749762509577\n",
      "Epoch 418 done. Loss : -1.5296670982023088\n",
      "Epoch 419 done. Loss : -1.527145845500339\n",
      "Epoch 420 done. Loss : -1.5242098740162406\n",
      "Epoch 421 done. Loss : -1.5320455783330398\n",
      "Epoch 422 done. Loss : -1.537132658198988\n",
      "Epoch 423 done. Loss : -1.5339800642391506\n",
      "Epoch 424 done. Loss : -1.525472141812994\n",
      "Epoch 425 done. Loss : -1.5176191949932694\n",
      "Epoch 426 done. Loss : -1.5109728803799336\n",
      "Epoch 427 done. Loss : -1.5083687214065167\n",
      "Epoch 428 done. Loss : -1.513814387074863\n",
      "Epoch 429 done. Loss : -1.514335872566872\n",
      "Epoch 430 done. Loss : -1.5076781373714154\n",
      "Epoch 431 done. Loss : -1.5108262341441905\n",
      "Epoch 432 done. Loss : -1.5192691435965697\n",
      "Epoch 433 done. Loss : -1.518804996909605\n",
      "Epoch 434 done. Loss : -1.518499365588702\n",
      "Epoch 435 done. Loss : -1.5175439624340892\n",
      "Epoch 436 done. Loss : -1.5089997658019216\n",
      "Epoch 437 done. Loss : -1.5013596592577545\n",
      "Epoch 438 done. Loss : -1.4982848459005322\n",
      "Epoch 439 done. Loss : -1.496898207846749\n",
      "Epoch 440 done. Loss : -1.5005716881643494\n",
      "Epoch 441 done. Loss : -1.4981326365624135\n",
      "Epoch 442 done. Loss : -1.501626519830837\n",
      "Epoch 443 done. Loss : -1.5048563642043207\n",
      "Epoch 444 done. Loss : -1.4968075301427342\n",
      "Epoch 445 done. Loss : -1.4925488106092135\n",
      "Epoch 446 done. Loss : -1.4981567509810798\n",
      "Epoch 447 done. Loss : -1.5028557220150003\n",
      "Epoch 448 done. Loss : -1.5023894187129154\n",
      "Epoch 449 done. Loss : -1.501027040450008\n",
      "Epoch 450 done. Loss : -1.499077061170555\n",
      "Epoch 451 done. Loss : -1.4943318847424554\n",
      "Epoch 452 done. Loss : -1.4900797920944986\n",
      "Epoch 453 done. Loss : -1.4950872418543797\n",
      "Epoch 454 done. Loss : -1.4972559824260485\n",
      "Epoch 455 done. Loss : -1.4995442295914616\n",
      "Epoch 456 done. Loss : -1.4937317744947687\n",
      "Epoch 457 done. Loss : -1.4887085833919649\n",
      "Epoch 458 done. Loss : -1.4939610579346083\n",
      "Epoch 459 done. Loss : -1.495410769357589\n",
      "Epoch 460 done. Loss : -1.4977712297183672\n",
      "Epoch 461 done. Loss : -1.4980811873448783\n",
      "Epoch 462 done. Loss : -1.4916752787880425\n",
      "Epoch 463 done. Loss : -1.4882291547553448\n",
      "Epoch 464 done. Loss : -1.4842243637513355\n",
      "Epoch 465 done. Loss : -1.4818813485323838\n",
      "Epoch 466 done. Loss : -1.485607024313636\n",
      "Epoch 467 done. Loss : -1.4804574412777693\n",
      "Epoch 468 done. Loss : -1.4846543623605069\n",
      "Epoch 469 done. Loss : -1.4790214084090947\n",
      "Epoch 470 done. Loss : -1.4749380580316718\n",
      "Epoch 471 done. Loss : -1.4801923841832694\n",
      "Epoch 472 done. Loss : -1.474638091359016\n",
      "Epoch 473 done. Loss : -1.4751455932830955\n",
      "Epoch 474 done. Loss : -1.477002507736136\n",
      "Epoch 475 done. Loss : -1.4789476793796987\n",
      "Epoch 476 done. Loss : -1.4783960943342516\n",
      "Epoch 477 done. Loss : -1.474442331577153\n",
      "Epoch 478 done. Loss : -1.4717324904250226\n",
      "Epoch 479 done. Loss : -1.4655213179615343\n",
      "Epoch 480 done. Loss : -1.4679373638384599\n",
      "Epoch 481 done. Loss : -1.4717287912383625\n",
      "Epoch 482 done. Loss : -1.4675208565089348\n",
      "Epoch 483 done. Loss : -1.4717696853541078\n",
      "Epoch 484 done. Loss : -1.4682954673286601\n",
      "Epoch 485 done. Loss : -1.467081903291791\n",
      "Epoch 486 done. Loss : -1.4660263349879061\n",
      "Epoch 487 done. Loss : -1.4644091592111415\n",
      "Epoch 488 done. Loss : -1.466429927205926\n",
      "Epoch 489 done. Loss : -1.4642817646513522\n",
      "Epoch 490 done. Loss : -1.4618564497699933\n",
      "Epoch 491 done. Loss : -1.4543998730164098\n",
      "Epoch 492 done. Loss : -1.455292141357757\n",
      "Epoch 493 done. Loss : -1.454806719998701\n",
      "Epoch 494 done. Loss : -1.454135044085305\n",
      "Epoch 495 done. Loss : -1.4556841858613487\n",
      "Epoch 496 done. Loss : -1.4528450368567223\n",
      "Epoch 497 done. Loss : -1.452510562089849\n",
      "Epoch 498 done. Loss : -1.4559037309101146\n",
      "Epoch 499 done. Loss : -1.4566963526426848\n",
      "Epoch 500 done. Loss : -1.455596582799356\n",
      "Epoch 501 done. Loss : -1.456902643168442\n",
      "Epoch 502 done. Loss : -1.4562139552542095\n",
      "Epoch 503 done. Loss : -1.452141571321063\n",
      "Epoch 504 done. Loss : -1.4522506877405328\n",
      "Epoch 505 done. Loss : -1.4489085805896826\n",
      "Epoch 506 done. Loss : -1.4521663623036019\n",
      "Epoch 507 done. Loss : -1.4495435402417622\n",
      "Epoch 508 done. Loss : -1.4587844496031002\n",
      "Epoch 509 done. Loss : -1.4532840839453742\n",
      "Epoch 510 done. Loss : -1.451838423653476\n",
      "Epoch 511 done. Loss : -1.4492723169210504\n",
      "Epoch 512 done. Loss : -1.4475359559406256\n",
      "Epoch 513 done. Loss : -1.450068178453873\n",
      "Epoch 514 done. Loss : -1.4499909412668683\n",
      "Epoch 515 done. Loss : -1.4492710316751434\n",
      "Epoch 516 done. Loss : -1.4506288354973866\n",
      "Epoch 517 done. Loss : -1.4493116082237205\n",
      "Epoch 518 done. Loss : -1.4533461190255226\n",
      "Epoch 519 done. Loss : -1.4446847627646364\n",
      "Epoch 520 done. Loss : -1.4421657895076376\n",
      "Epoch 521 done. Loss : -1.446693588885176\n",
      "Epoch 522 done. Loss : -1.4424718584828737\n",
      "Epoch 523 done. Loss : -1.4474485345401966\n",
      "Epoch 524 done. Loss : -1.4482714241081829\n",
      "Epoch 525 done. Loss : -1.4456954375653686\n",
      "Epoch 526 done. Loss : -1.4439050967300435\n",
      "Epoch 527 done. Loss : -1.4416821786918235\n",
      "Epoch 528 done. Loss : -1.4394795626610388\n",
      "Epoch 529 done. Loss : -1.4370774526238852\n",
      "Epoch 530 done. Loss : -1.4376025980455573\n",
      "Epoch 531 done. Loss : -1.4352776621583532\n",
      "Epoch 532 done. Loss : -1.4375382057156583\n",
      "Epoch 533 done. Loss : -1.438296970566466\n",
      "Epoch 534 done. Loss : -1.4391181409793834\n",
      "Epoch 535 done. Loss : -1.4403225867262353\n",
      "Epoch 536 done. Loss : -1.4400021795414195\n",
      "Epoch 537 done. Loss : -1.4392616857638252\n",
      "Epoch 538 done. Loss : -1.435673715308571\n",
      "Epoch 539 done. Loss : -1.4381448388121163\n",
      "Epoch 540 done. Loss : -1.4370517712122166\n",
      "Epoch 541 done. Loss : -1.4372085749665195\n",
      "Epoch 542 done. Loss : -1.4300567780263016\n",
      "Epoch 543 done. Loss : -1.433933459363783\n",
      "Epoch 544 done. Loss : -1.4310524799145559\n",
      "Epoch 545 done. Loss : -1.433134262557545\n",
      "Epoch 546 done. Loss : -1.4310476470011615\n",
      "Epoch 547 done. Loss : -1.4284186690589507\n",
      "Epoch 548 done. Loss : -1.4277478785403366\n",
      "Epoch 549 done. Loss : -1.4301675057579417\n",
      "Epoch 550 done. Loss : -1.4290295427549544\n",
      "Epoch 551 done. Loss : -1.4281336988823194\n",
      "Epoch 552 done. Loss : -1.425597796542295\n",
      "Epoch 553 done. Loss : -1.4241497722092737\n",
      "Epoch 554 done. Loss : -1.4219507215093385\n",
      "Epoch 555 done. Loss : -1.4240557264317295\n",
      "Epoch 556 done. Loss : -1.4293368383708025\n",
      "Epoch 557 done. Loss : -1.4255266063096474\n",
      "Epoch 558 done. Loss : -1.4213243102002098\n",
      "Epoch 559 done. Loss : -1.4280114465295473\n",
      "Epoch 560 done. Loss : -1.4243009247879996\n",
      "Epoch 561 done. Loss : -1.4181286069584487\n",
      "Epoch 562 done. Loss : -1.4170441713557906\n",
      "Epoch 563 done. Loss : -1.4160708924658085\n",
      "Epoch 564 done. Loss : -1.416435240210098\n",
      "Epoch 565 done. Loss : -1.4158467742258887\n",
      "Epoch 566 done. Loss : -1.4155895950192372\n",
      "Epoch 567 done. Loss : -1.4101709818004642\n",
      "Epoch 568 done. Loss : -1.4130432847748151\n",
      "Epoch 569 done. Loss : -1.4101195214668558\n",
      "Epoch 570 done. Loss : -1.407973872819662\n",
      "Epoch 571 done. Loss : -1.4116639418286265\n",
      "Epoch 572 done. Loss : -1.4129459109792808\n",
      "Epoch 573 done. Loss : -1.4093372819621264\n",
      "Epoch 574 done. Loss : -1.4088970813805946\n",
      "Epoch 575 done. Loss : -1.4051992083298164\n",
      "Epoch 576 done. Loss : -1.4030785663470215\n",
      "Epoch 577 done. Loss : -1.4034174287160583\n",
      "Epoch 578 done. Loss : -1.403956456050242\n",
      "Epoch 579 done. Loss : -1.403254342259594\n",
      "Epoch 580 done. Loss : -1.401575588525873\n",
      "Epoch 581 done. Loss : -1.4067637127406376\n",
      "Epoch 582 done. Loss : -1.404743475030672\n",
      "Epoch 583 done. Loss : -1.408523991843541\n",
      "Epoch 584 done. Loss : -1.4081019773439833\n",
      "Epoch 585 done. Loss : -1.4064186291238796\n",
      "Epoch 586 done. Loss : -1.4071641017390353\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 587 done. Loss : -1.4057816697900816\n",
      "Epoch 588 done. Loss : -1.3996003343825851\n",
      "Epoch 589 done. Loss : -1.4001979035057843\n",
      "Epoch 590 done. Loss : -1.3978980796088682\n",
      "Epoch 591 done. Loss : -1.3983984893111074\n",
      "Epoch 592 done. Loss : -1.396549387808087\n",
      "Epoch 593 done. Loss : -1.404857850422685\n",
      "Epoch 594 done. Loss : -1.4047105131851694\n",
      "Epoch 595 done. Loss : -1.400243538984951\n",
      "Epoch 596 done. Loss : -1.4002511108587274\n",
      "Epoch 597 done. Loss : -1.4017407026373283\n",
      "Epoch 598 done. Loss : -1.400182500310732\n",
      "Epoch 599 done. Loss : -1.3992608935107134\n",
      "Epoch 600 done. Loss : -1.4008894493313242\n",
      "Epoch 601 done. Loss : -1.3952646361953964\n",
      "Epoch 602 done. Loss : -1.4038782121632025\n",
      "Epoch 603 done. Loss : -1.3999498828213366\n",
      "Epoch 604 done. Loss : -1.4010480702037653\n",
      "Epoch 605 done. Loss : -1.4015457991387317\n",
      "Epoch 606 done. Loss : -1.402503008593699\n",
      "Epoch 607 done. Loss : -1.3983336068578258\n",
      "Epoch 608 done. Loss : -1.395919323903604\n",
      "Epoch 609 done. Loss : -1.3937090303286044\n",
      "Epoch 610 done. Loss : -1.3905103602824962\n",
      "Epoch 611 done. Loss : -1.4160555478506796\n",
      "Epoch 612 done. Loss : -1.4159548440832057\n",
      "Epoch 613 done. Loss : -1.4163502438514435\n",
      "Epoch 614 done. Loss : -1.3944410271571193\n",
      "Epoch 615 done. Loss : -1.3935429659913638\n",
      "Epoch 616 done. Loss : -1.3903006804076496\n",
      "Epoch 617 done. Loss : -1.3885587494114433\n",
      "Epoch 618 done. Loss : -1.389838895098591\n",
      "Epoch 619 done. Loss : -1.3905385522650868\n",
      "Epoch 620 done. Loss : -1.3859871942194606\n",
      "Epoch 621 done. Loss : -1.3864492078036972\n",
      "Epoch 622 done. Loss : -1.384742016814152\n",
      "Epoch 623 done. Loss : -1.391508998201425\n",
      "Epoch 624 done. Loss : -1.389889105517245\n",
      "Epoch 625 done. Loss : -1.3837275048534914\n",
      "Epoch 626 done. Loss : -1.382818908531422\n",
      "Epoch 627 done. Loss : -1.3827321382976672\n",
      "Epoch 628 done. Loss : -1.3755637334208999\n",
      "Epoch 629 done. Loss : -1.4169478814286007\n",
      "Epoch 630 done. Loss : -1.3739224411359785\n",
      "Epoch 631 done. Loss : -1.3800660501278585\n",
      "Epoch 632 done. Loss : -1.3805853679065778\n",
      "Epoch 633 done. Loss : -1.381898412526331\n",
      "Epoch 634 done. Loss : -1.3862172107307438\n",
      "Epoch 635 done. Loss : -1.3801887747380917\n",
      "Epoch 636 done. Loss : -1.3963853823528667\n",
      "Epoch 637 done. Loss : -1.398555692413966\n",
      "Epoch 638 done. Loss : -1.3749556515334955\n",
      "Epoch 639 done. Loss : -1.3755161513363259\n",
      "Epoch 640 done. Loss : -1.3768875097796924\n",
      "Epoch 641 done. Loss : -1.3750737634944803\n",
      "Epoch 642 done. Loss : -1.3705249632899907\n",
      "Epoch 643 done. Loss : -1.3702344243704578\n",
      "Epoch 644 done. Loss : -1.3648423685235598\n",
      "Epoch 645 done. Loss : -1.3694296637374594\n",
      "Epoch 646 done. Loss : -1.3710726038294245\n",
      "Epoch 647 done. Loss : -1.4003115899207292\n",
      "Epoch 648 done. Loss : -1.3712515651011437\n",
      "Epoch 649 done. Loss : -1.3703194179884766\n",
      "Epoch 650 done. Loss : -1.3641966444586804\n",
      "Epoch 651 done. Loss : -1.3659994372233701\n",
      "Epoch 652 done. Loss : -1.37086586907393\n",
      "Epoch 653 done. Loss : -1.3974852144597074\n",
      "Epoch 654 done. Loss : -1.3723252223428735\n",
      "Epoch 655 done. Loss : -1.3760757809914337\n",
      "Epoch 656 done. Loss : -1.3757566119263689\n",
      "Epoch 657 done. Loss : -1.3882784107438924\n",
      "Epoch 658 done. Loss : -1.3902681509343726\n",
      "Epoch 659 done. Loss : -1.3659979283942327\n",
      "Epoch 660 done. Loss : -1.3711798076709676\n",
      "Epoch 661 done. Loss : -1.3675956060778613\n",
      "Epoch 662 done. Loss : -1.3669408411607193\n",
      "Epoch 663 done. Loss : -1.3663017502285375\n",
      "Epoch 664 done. Loss : -1.3699199369238986\n",
      "Epoch 665 done. Loss : -1.390197901173032\n",
      "Epoch 666 done. Loss : -1.393470480887594\n",
      "Epoch 667 done. Loss : -1.3775900559440846\n",
      "Epoch 668 done. Loss : -1.3765044031693376\n",
      "Epoch 669 done. Loss : -1.3818090400164782\n",
      "Epoch 670 done. Loss : -1.3755302369219329\n",
      "Epoch 671 done. Loss : -1.3862522463401439\n",
      "Epoch 672 done. Loss : -1.3940690205882267\n",
      "Epoch 673 done. Loss : -1.3966787229158917\n",
      "Epoch 674 done. Loss : -1.3808362571120272\n",
      "Epoch 675 done. Loss : -1.3764893940494254\n",
      "Epoch 676 done. Loss : -1.3707709213191448\n",
      "Epoch 677 done. Loss : -1.382359658896518\n",
      "Epoch 678 done. Loss : -1.3877825323511588\n",
      "Epoch 679 done. Loss : -1.3747381098581903\n",
      "Epoch 680 done. Loss : -1.3719885598264412\n",
      "Epoch 681 done. Loss : -1.3762727990631638\n",
      "Epoch 682 done. Loss : -1.4035427727143863\n",
      "Epoch 683 done. Loss : -1.3781658890704107\n",
      "Epoch 684 done. Loss : -1.39348001662272\n",
      "Epoch 685 done. Loss : -1.3702711786680835\n",
      "Epoch 686 done. Loss : -1.369078779851268\n",
      "Epoch 687 done. Loss : -1.3738297282537348\n",
      "Epoch 688 done. Loss : -1.3920963823831989\n",
      "Epoch 689 done. Loss : -1.3887555844532535\n",
      "Epoch 690 done. Loss : -1.3758835835020564\n",
      "Epoch 691 done. Loss : -1.3711853594934083\n",
      "Epoch 692 done. Loss : -1.3643314128338608\n",
      "Epoch 693 done. Loss : -1.365084116813085\n",
      "Epoch 694 done. Loss : -1.385292496872321\n",
      "Epoch 695 done. Loss : -1.3896107175642736\n",
      "Epoch 696 done. Loss : -1.3817064862961468\n",
      "Epoch 697 done. Loss : -1.3805013475134202\n",
      "Epoch 698 done. Loss : -1.3706759127450265\n",
      "Epoch 699 done. Loss : -1.3703347784718773\n",
      "Epoch 700 done. Loss : -1.3665390531738177\n",
      "Epoch 701 done. Loss : -1.3832329588491528\n",
      "Epoch 702 done. Loss : -1.3805752352213974\n",
      "Epoch 703 done. Loss : -1.3748581009601706\n",
      "Epoch 704 done. Loss : -1.371521109876427\n",
      "Epoch 705 done. Loss : -1.3701745282310238\n",
      "Epoch 706 done. Loss : -1.3725654039009827\n",
      "Epoch 707 done. Loss : -1.3598266853613583\n",
      "Epoch 708 done. Loss : -1.3611892194423598\n",
      "Epoch 709 done. Loss : -1.3751454199273967\n",
      "Epoch 710 done. Loss : -1.3633870021965402\n",
      "Epoch 711 done. Loss : -1.3794582880860253\n",
      "Epoch 712 done. Loss : -1.360236998064069\n",
      "Epoch 713 done. Loss : -1.37653187488711\n",
      "Epoch 714 done. Loss : -1.358023543323518\n",
      "Epoch 715 done. Loss : -1.3615656118419566\n",
      "Epoch 716 done. Loss : -1.3712562118172182\n",
      "Epoch 717 done. Loss : -1.3707718176769363\n",
      "Epoch 718 done. Loss : -1.3678366485180162\n",
      "Epoch 719 done. Loss : -1.3617408328925165\n",
      "Epoch 720 done. Loss : -1.3636673426606314\n",
      "Epoch 721 done. Loss : -1.3647819268723704\n",
      "Epoch 722 done. Loss : -1.367222564898293\n",
      "Epoch 723 done. Loss : -1.3704777097952134\n",
      "Epoch 724 done. Loss : -1.370322484672468\n",
      "Epoch 725 done. Loss : -1.3656421730966959\n",
      "Epoch 726 done. Loss : -1.3561838558465908\n",
      "Epoch 727 done. Loss : -1.362573324736307\n",
      "Epoch 728 done. Loss : -1.3642570321465204\n",
      "Epoch 729 done. Loss : -1.3601788555605299\n",
      "Epoch 730 done. Loss : -1.3584383795671515\n",
      "Epoch 731 done. Loss : -1.3554945495183792\n",
      "Epoch 732 done. Loss : -1.3547341254593452\n",
      "Epoch 733 done. Loss : -1.3545473846079386\n",
      "Epoch 734 done. Loss : -1.358130729639236\n",
      "Epoch 735 done. Loss : -1.3639055018568973\n",
      "Epoch 736 done. Loss : -1.3662102614913225\n",
      "Epoch 737 done. Loss : -1.3584661026006954\n",
      "Epoch 738 done. Loss : -1.363188300746101\n",
      "Epoch 739 done. Loss : -1.3535633711030117\n",
      "Epoch 740 done. Loss : -1.354884147124679\n",
      "Epoch 741 done. Loss : -1.3544922261854455\n",
      "Epoch 742 done. Loss : -1.3503437562565064\n",
      "Epoch 743 done. Loss : -1.3521071639667288\n",
      "Epoch 744 done. Loss : -1.3531029674545598\n",
      "Epoch 745 done. Loss : -1.350832018086705\n",
      "Epoch 746 done. Loss : -1.3524298304379736\n",
      "Epoch 747 done. Loss : -1.3533909207036452\n",
      "Epoch 748 done. Loss : -1.356558947778427\n",
      "Epoch 749 done. Loss : -1.3545440424928137\n",
      "Epoch 750 done. Loss : -1.3545965275456104\n",
      "Epoch 751 done. Loss : -1.355474761022383\n",
      "Epoch 752 done. Loss : -1.348549330894128\n",
      "Epoch 753 done. Loss : -1.3544090184967563\n",
      "Epoch 754 done. Loss : -1.3661948832961666\n",
      "Epoch 755 done. Loss : -1.3525170284511754\n",
      "Epoch 756 done. Loss : -1.3565046875362154\n",
      "Epoch 757 done. Loss : -1.3514539512242218\n",
      "Epoch 758 done. Loss : -1.3472911642542134\n",
      "Epoch 759 done. Loss : -1.3542777733318934\n",
      "Epoch 760 done. Loss : -1.3573272302707586\n",
      "Epoch 761 done. Loss : -1.3557634035606914\n",
      "Epoch 762 done. Loss : -1.3468974380664118\n",
      "Epoch 763 done. Loss : -1.349730406743316\n",
      "Epoch 764 done. Loss : -1.3545794454055333\n",
      "Epoch 765 done. Loss : -1.3548882884729676\n",
      "Epoch 766 done. Loss : -1.3540704782842052\n",
      "Epoch 767 done. Loss : -1.3509296584940582\n",
      "Epoch 768 done. Loss : -1.344774955135419\n",
      "Epoch 769 done. Loss : -1.3551850304778705\n",
      "Epoch 770 done. Loss : -1.3512597498805956\n",
      "Epoch 771 done. Loss : -1.349525065701151\n",
      "Epoch 772 done. Loss : -1.349073148743436\n",
      "Epoch 773 done. Loss : -1.347771807748684\n",
      "Epoch 774 done. Loss : -1.3449938697767339\n",
      "Epoch 775 done. Loss : -1.348108138769768\n",
      "Epoch 776 done. Loss : -1.3468727882846845\n",
      "Epoch 777 done. Loss : -1.350228351145319\n",
      "Epoch 778 done. Loss : -1.3468225639609597\n",
      "Epoch 779 done. Loss : -1.3403770636385968\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 780 done. Loss : -1.3560599701099425\n",
      "Epoch 781 done. Loss : -1.346357365336869\n",
      "Epoch 782 done. Loss : -1.348753718612059\n",
      "Epoch 783 done. Loss : -1.3460443325248597\n",
      "Epoch 784 done. Loss : -1.3431452440279616\n",
      "Epoch 785 done. Loss : -1.342496754075457\n",
      "Epoch 786 done. Loss : -1.3369385022697344\n",
      "Epoch 787 done. Loss : -1.3390466539141421\n",
      "Epoch 788 done. Loss : -1.3494897634865923\n",
      "Epoch 789 done. Loss : -1.337964821231451\n",
      "Epoch 790 done. Loss : -1.3370830622050258\n",
      "Epoch 791 done. Loss : -1.3335767867565154\n",
      "Epoch 792 done. Loss : -1.3317290472650778\n",
      "Epoch 793 done. Loss : -1.332404460770316\n",
      "Epoch 794 done. Loss : -1.3321405504479464\n",
      "Epoch 795 done. Loss : -1.3324008122372941\n",
      "Epoch 796 done. Loss : -1.332552598904384\n",
      "Epoch 797 done. Loss : -1.336541857044837\n",
      "Epoch 798 done. Loss : -1.3297362691618377\n",
      "Epoch 799 done. Loss : -1.333322359630523\n",
      "Epoch 800 done. Loss : -1.330121842469771\n",
      "Epoch 801 done. Loss : -1.337087979055522\n",
      "Epoch 802 done. Loss : -1.325410160473349\n",
      "Epoch 803 done. Loss : -1.3204322508319377\n",
      "Epoch 804 done. Loss : -1.3210616347907664\n",
      "Epoch 805 done. Loss : -1.318323413737646\n",
      "Epoch 806 done. Loss : -1.320970854891312\n",
      "Epoch 807 done. Loss : -1.324051110942874\n",
      "Epoch 808 done. Loss : -1.3243764339525834\n",
      "Epoch 809 done. Loss : -1.3197454794755321\n",
      "Epoch 810 done. Loss : -1.323934995718877\n",
      "Epoch 811 done. Loss : -1.3258873016394177\n",
      "Epoch 812 done. Loss : -1.321830727173809\n",
      "Epoch 813 done. Loss : -1.3197262163830032\n",
      "Epoch 814 done. Loss : -1.3154804166880059\n",
      "Epoch 815 done. Loss : -1.3134078346648996\n",
      "Epoch 816 done. Loss : -1.317907481649634\n",
      "Epoch 817 done. Loss : -1.3130939314512369\n",
      "Epoch 818 done. Loss : -1.3242664895355123\n",
      "Epoch 819 done. Loss : -1.3221965947730543\n",
      "Epoch 820 done. Loss : -1.3233184757794598\n",
      "Epoch 821 done. Loss : -1.3200387124314958\n",
      "Epoch 822 done. Loss : -1.3117251477193173\n",
      "Epoch 823 done. Loss : -1.3162831731813553\n",
      "Epoch 824 done. Loss : -1.3186790725775481\n",
      "Epoch 825 done. Loss : -1.3161329916090359\n",
      "Epoch 826 done. Loss : -1.311174706645408\n",
      "Epoch 827 done. Loss : -1.316962850211756\n",
      "Epoch 828 done. Loss : -1.3231881372335756\n",
      "Epoch 829 done. Loss : -1.3125830235406877\n",
      "Epoch 830 done. Loss : -1.3140520261594995\n",
      "Epoch 831 done. Loss : -1.3048828395650285\n",
      "Epoch 832 done. Loss : -1.3076843208838984\n",
      "Epoch 833 done. Loss : -1.3058331204205686\n",
      "Epoch 834 done. Loss : -1.3081688134014964\n",
      "Epoch 835 done. Loss : -1.3150231060641706\n",
      "Epoch 836 done. Loss : -1.3193200540133936\n",
      "Epoch 837 done. Loss : -1.3188395887997553\n",
      "Epoch 838 done. Loss : -1.3105158019662033\n",
      "Epoch 839 done. Loss : -1.30740081724694\n",
      "Epoch 840 done. Loss : -1.3105092006400176\n",
      "Epoch 841 done. Loss : -1.3054295450988227\n",
      "Epoch 842 done. Loss : -1.307243856691744\n",
      "Epoch 843 done. Loss : -1.3095952616044386\n",
      "Epoch 844 done. Loss : -1.3079404541302297\n",
      "Epoch 845 done. Loss : -1.3067420711844515\n",
      "Epoch 846 done. Loss : -1.3068883883328524\n",
      "Epoch 847 done. Loss : -1.3061051866060476\n",
      "Epoch 848 done. Loss : -1.3057651925382896\n",
      "Epoch 849 done. Loss : -1.3069850680595563\n",
      "Epoch 850 done. Loss : -1.3069436586895211\n",
      "Epoch 851 done. Loss : -1.3063702295807629\n",
      "Epoch 852 done. Loss : -1.3070531490470843\n",
      "Epoch 853 done. Loss : -1.3083923900481202\n",
      "Epoch 854 done. Loss : -1.3062059983386316\n",
      "Epoch 855 done. Loss : -1.308826210795372\n",
      "Epoch 856 done. Loss : -1.3115023794637073\n",
      "Epoch 857 done. Loss : -1.3060752017110135\n",
      "Epoch 858 done. Loss : -1.3069135319489011\n",
      "Epoch 859 done. Loss : -1.3110243911068895\n",
      "Epoch 860 done. Loss : -1.3106502596069838\n",
      "Epoch 861 done. Loss : -1.3039490896629056\n",
      "Epoch 862 done. Loss : -1.3045660717784529\n",
      "Epoch 863 done. Loss : -1.307057632626579\n",
      "Epoch 864 done. Loss : -1.3101638595501863\n",
      "Epoch 865 done. Loss : -1.3059312087465924\n",
      "Epoch 866 done. Loss : -1.3017777291533226\n",
      "Epoch 867 done. Loss : -1.298791271947866\n",
      "Epoch 868 done. Loss : -1.3036925460957758\n",
      "Epoch 869 done. Loss : -1.2983067891103577\n",
      "Epoch 870 done. Loss : -1.298547925054069\n",
      "Epoch 871 done. Loss : -1.3034082543391445\n",
      "Epoch 872 done. Loss : -1.3030118631038268\n",
      "Epoch 873 done. Loss : -1.3032130831318427\n",
      "Epoch 874 done. Loss : -1.291334558040753\n",
      "Epoch 875 done. Loss : -1.2900042114763057\n",
      "Epoch 876 done. Loss : -1.2988085622417074\n",
      "Epoch 877 done. Loss : -1.303855667566212\n",
      "Epoch 878 done. Loss : -1.3001502780305834\n",
      "Epoch 879 done. Loss : -1.2908966077024777\n",
      "Epoch 880 done. Loss : -1.2949000673078648\n",
      "Epoch 881 done. Loss : -1.2905988412284999\n",
      "Epoch 882 done. Loss : -1.2941691631864403\n",
      "Epoch 883 done. Loss : -1.299507270017687\n",
      "Epoch 884 done. Loss : -1.2977929914444342\n",
      "Epoch 885 done. Loss : -1.3016782633914767\n",
      "Epoch 886 done. Loss : -1.2920399900432855\n",
      "Epoch 887 done. Loss : -1.2902394831534703\n",
      "Epoch 888 done. Loss : -1.2863474671333008\n",
      "Epoch 889 done. Loss : -1.2896160556345735\n",
      "Epoch 890 done. Loss : -1.2920783598901524\n",
      "Epoch 891 done. Loss : -1.2916914492872364\n",
      "Epoch 892 done. Loss : -1.2946261225510411\n",
      "Epoch 893 done. Loss : -1.3002084979781001\n",
      "Epoch 894 done. Loss : -1.29559990979625\n",
      "Epoch 895 done. Loss : -1.2977031354944917\n",
      "Epoch 896 done. Loss : -1.2948915976766082\n",
      "Epoch 897 done. Loss : -1.290466979771654\n",
      "Epoch 898 done. Loss : -1.2950678632870782\n",
      "Epoch 899 done. Loss : -1.2927284665615604\n",
      "Epoch 900 done. Loss : -1.2992290073617503\n",
      "Epoch 901 done. Loss : -1.2990072062463085\n",
      "Epoch 902 done. Loss : -1.2945551552050425\n",
      "Epoch 903 done. Loss : -1.2968649009479296\n",
      "Epoch 904 done. Loss : -1.299696735413793\n",
      "Epoch 905 done. Loss : -1.292394041454496\n",
      "Epoch 906 done. Loss : -1.3009538634515607\n",
      "Epoch 907 done. Loss : -1.293203621841142\n",
      "Epoch 908 done. Loss : -1.2945399442099004\n",
      "Epoch 909 done. Loss : -1.2930943776316608\n",
      "Epoch 910 done. Loss : -1.2926749754931706\n",
      "Epoch 911 done. Loss : -1.290516800534045\n",
      "Epoch 912 done. Loss : -1.28633522727218\n",
      "Epoch 913 done. Loss : -1.2896064012436441\n",
      "Epoch 914 done. Loss : -1.285354312606482\n",
      "Epoch 915 done. Loss : -1.2899331478175577\n",
      "Epoch 916 done. Loss : -1.28506069894203\n",
      "Epoch 917 done. Loss : -1.2833286165904925\n",
      "Epoch 918 done. Loss : -1.2837051091908638\n",
      "Epoch 919 done. Loss : -1.2849437394706684\n",
      "Epoch 920 done. Loss : -1.2818245645667412\n",
      "Epoch 921 done. Loss : -1.278220593434661\n",
      "Epoch 922 done. Loss : -1.2802256934217886\n",
      "Epoch 923 done. Loss : -1.287409509534082\n",
      "Epoch 924 done. Loss : -1.2897705480254023\n",
      "Epoch 925 done. Loss : -1.2841738509238492\n",
      "Epoch 926 done. Loss : -1.2849152750716848\n",
      "Epoch 927 done. Loss : -1.2822858788140659\n",
      "Epoch 928 done. Loss : -1.2821310179024499\n",
      "Epoch 929 done. Loss : -1.2860950171183934\n",
      "Epoch 930 done. Loss : -1.2792095472669232\n",
      "Epoch 931 done. Loss : -1.2817457910195147\n",
      "Epoch 932 done. Loss : -1.2806280086554875\n",
      "Epoch 933 done. Loss : -1.2841482939991127\n",
      "Epoch 934 done. Loss : -1.2855980494910915\n",
      "Epoch 935 done. Loss : -1.2826938154506358\n",
      "Epoch 936 done. Loss : -1.2779215953832712\n",
      "Epoch 937 done. Loss : -1.2836766796277124\n",
      "Epoch 938 done. Loss : -1.2791863685142297\n",
      "Epoch 939 done. Loss : -1.2764054232424948\n",
      "Epoch 940 done. Loss : -1.2780461086872745\n",
      "Epoch 941 done. Loss : -1.2836190893863\n",
      "Epoch 942 done. Loss : -1.279076563260619\n",
      "Epoch 943 done. Loss : -1.2813037527484714\n",
      "Epoch 944 done. Loss : -1.2793666364996696\n",
      "Epoch 945 done. Loss : -1.2770809254843898\n",
      "Epoch 946 done. Loss : -1.2826774316287477\n",
      "Epoch 947 done. Loss : -1.2800258559765112\n",
      "Epoch 948 done. Loss : -1.2822685715909459\n",
      "Epoch 949 done. Loss : -1.2805360111936261\n",
      "Epoch 950 done. Loss : -1.2763512577197202\n",
      "Epoch 951 done. Loss : -1.2778755647454025\n",
      "Epoch 952 done. Loss : -1.2812486522175885\n",
      "Epoch 953 done. Loss : -1.2841676476157788\n",
      "Epoch 954 done. Loss : -1.2815363944702465\n",
      "Epoch 955 done. Loss : -1.2794384053988805\n",
      "Epoch 956 done. Loss : -1.270492468518364\n",
      "Epoch 957 done. Loss : -1.2740239822638095\n",
      "Epoch 958 done. Loss : -1.2781789691663628\n",
      "Epoch 959 done. Loss : -1.280028339028704\n",
      "Epoch 960 done. Loss : -1.2783574100006756\n",
      "Epoch 961 done. Loss : -1.278380735164595\n",
      "Epoch 962 done. Loss : -1.281890904020183\n",
      "Epoch 963 done. Loss : -1.286603248481138\n",
      "Epoch 964 done. Loss : -1.2839496840155382\n",
      "Epoch 965 done. Loss : -1.2802242963045525\n",
      "Epoch 966 done. Loss : -1.2761624784957348\n",
      "Epoch 967 done. Loss : -1.2834602150892707\n",
      "Epoch 968 done. Loss : -1.2832116976452381\n",
      "Epoch 969 done. Loss : -1.2854628950123477\n",
      "Epoch 970 done. Loss : -1.2836588319790998\n",
      "Epoch 971 done. Loss : -1.28400358886265\n",
      "Epoch 972 done. Loss : -1.2765836715534575\n",
      "Epoch 973 done. Loss : -1.2755736068644432\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 974 done. Loss : -1.2694584121812649\n",
      "Epoch 975 done. Loss : -1.2716328429250965\n",
      "Epoch 976 done. Loss : -1.2751260558401825\n",
      "Epoch 977 done. Loss : -1.2848384306762037\n",
      "Epoch 978 done. Loss : -1.2817246953913393\n",
      "Epoch 979 done. Loss : -1.277890910374864\n",
      "Epoch 980 done. Loss : -1.2740045523751822\n",
      "Epoch 981 done. Loss : -1.2782231919915843\n",
      "Epoch 982 done. Loss : -1.2763029358003783\n",
      "Epoch 983 done. Loss : -1.2780528402210165\n",
      "Epoch 984 done. Loss : -1.2700738881868106\n",
      "Epoch 985 done. Loss : -1.2736531052177695\n",
      "Epoch 986 done. Loss : -1.2775600112464474\n",
      "Epoch 987 done. Loss : -1.2752901550423257\n",
      "Epoch 988 done. Loss : -1.2767761536044437\n",
      "Epoch 989 done. Loss : -1.2775299914630258\n",
      "Epoch 990 done. Loss : -1.277118419545296\n",
      "Epoch 991 done. Loss : -1.275615288789976\n",
      "Epoch 992 done. Loss : -1.2744329663268248\n",
      "Epoch 993 done. Loss : -1.276630684186657\n",
      "Epoch 994 done. Loss : -1.2780516440174443\n",
      "Epoch 995 done. Loss : -1.2733308986228602\n",
      "Epoch 996 done. Loss : -1.2720692503250821\n",
      "Epoch 997 done. Loss : -1.271285220399605\n",
      "Epoch 998 done. Loss : -1.2731573568846553\n",
      "Epoch 999 done. Loss : -1.2701621940398704\n"
     ]
    }
   ],
   "source": [
    "hist = scorenn.fit(smol_batch_pixels,smol_batch_label,32,1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Seems nice, not all the way to 0 but still"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ignore the first epoch error i messed up something for sure xd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x203860560b8>]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VeWdx/HP72ZfIHsgBEgIq8gqEXBp64KK1q2tdrStuy9ebe2o09rWTqd1tPbVaadTl1prHR2tU0dtlapFqxY3cAMCIrKHnUCAECAh+/bMH/cmhpCQEG5y77n5vl+vvLj33Cf3/k6OfvPkOc95jjnnEBGRyOILdQEiIhJ8CncRkQikcBcRiUAKdxGRCKRwFxGJQAp3EZEIpHAXEYlACncRkQikcBcRiUDRofrgzMxMl5+fH6qPFxHxpOXLl+93zmV11y5k4Z6fn09RUVGoPl5ExJPMbHtP2mlYRkQkAincRUQikMJdRCQCKdxFRCKQwl1EJAIp3EVEIpDCXUQkAnky3F/+ZDd7K+tCXYaISNgK2UVMvdXQ1MKtz3wMwPqfzSU+JirEFYmIhB/P9dxb2t3Q+/H3toawEhGR8OW5cG/v5ZW7Q12CiEhY8my4D4qPZsPew5RX1Ye6FBGRsOPZcJ9dkAHA0q0HQlyJiEj48Vy4tw65T8lNISEmio+2lIe2IBGRMOS5cG8VE+2jMD+Nj7ao5y4i0pHnwt3x2WyZ2QUZGncXEemE58K9lQGnjfaPuy8u3h/aYkREwoxnwx1gcm4KmclxPPBmMS0trvtvEBEZIDwX7u2uYSImysePLpzA1v3VfLzzYOiKEhEJM54L91Zm/n/nTBxClM94c92+0BYkIhJGPBfuHQdfUhJimJmfzutr9uCchmZERMCD4d7KsLbH5588hM1l1eyu0EqRIiLg4XBvb9qIVAA+LTkU4kpERMKD58K9s6GXk3IGE+0zPt6pcBcRAQ+Geyv7bFSG+JgoThudwXPLdlLT0BS6okREwoRnw72jfz5nLIdqGrUMsIgIPQh3M4s3s6Vm9omZrTGzuztpE2dmz5nZJjNbYmb5fVEsHD1bptWp+WlMGDqIpz7crlkzIjLg9aTnXg+c45ybCkwD5prZ7A5tbgIOOufGAPcBvwxumd0zM645LY+1pZWs2KELmkRkYOs23J1fVeBpTOCrY9f4MuCPgcfPA+eatR8VD55jdcovn5ZLclw0T3+0oy8+WkTEM3o05m5mUWa2EtgH/MM5t6RDk1xgJ4BzrgmoADKCWWgnNR21LSkumi9Nz2XBp6VU1Db25ceLiIS1HoW7c67ZOTcNGA7MNLNJHZp01ks/qo9tZvPMrMjMisrKyo6/2h64fPowGppaWLBKJ1ZFZOA6rtkyzrlDwDvA3A4vlQAjAMwsGkgBjrqLhnPuUedcoXOuMCsrq1cFd3lGNWD6iDSmDE/h9+9s1rRIERmwejJbJsvMUgOPE4A5wPoOzV4Grgs8vgJ4y/XxlJWuBvR9PuPWc8ZScrCW55bt7MsSRETCVk967jnA22a2CliGf8x9gZndY2aXBto8DmSY2Sbgu8CdfVPukXdi6sqciUMYk53MOxv6ZuhHRCTcRXfXwDm3CpjeyfaftntcB1wZ3NKOrbu5OLNGpfPyyt00tziifH0ycUdEJGxFzBWqHZ2an87h+iY27Dkc6lJERPqd58K9pyP5U1tXitylxcREZODxXLi36m6gJS89kczkWBbqDk0iMgB5Ltx7OgXH5zPOmziUJVvKtdaMiAw4ngv3Vj1Z3WDisMFU1jWx80BtP1QkIhI+PBvuPXHGaP8KCM8Vaa0ZERlYPBfuxzPEUpCVzBcn5/Dk+9uoa2zuw6pERMKL58K9VU/XnLxs2jCqG5r5dFdF3xYkIhJGPBfux3tqdEZeGgDLth211I2ISMTyXLi36uk1pxnJcYzNTub9Tfv7tB4RkXDi2XA/HudMyGbJlgMadxeRAcNz4d6bKevTR6bR1OJYW1oZ/IJERMKQ58K9zXHcxW/qiBQAVu3UUgQiMjB4Ltx7suRvR0MHx5M1KI5VJZoxIyIDg+fCvdXxLOJrZkwdnsrKEvXcRWRg8Gy4H6+pw1PYUlZNZZ1unC0ikc974d7LNcCmBJYAXq2hGREZALwX7gHHcT4V8PfcAV2pKiIDgmfD/XilJsaSmhjDzoM1oS5FRKTPeS7cT2Rl9mEpCWzcUxW0WkREwpXnwr2VHdd8Gb8zx2aydNsBVuw42AcViYiED8+F+4ncVOm2c8cSE2W8vmZP8AoSEQlDngv3Vsd7QhUgKS6awrx03lizV7feE5GI5tlw760rZgxn6/5qlm7VEsAiErk8F+69WX6gvYsm5zAoLpo/F5UEqSIRkfDjuXBv1YtRGQASYqOYO2kor6/ZoyWARSRieS7cgzFU/qXpuVTVN7Fw3d4TfzMRkTDkuXBv1ZsTqq1mFWSQkRTLm+v2Ba8gEZEw4tlwPxFRPuPz47JYuG4vtQ0amhGRyOO5cA/WBMYvTc/lcF0TH20tD9I7ioiED8+Fe6veXKHa3qn56cRG+fhAN84WkQjkuXAP1sVHCbFRnJKXyvub1HMXkcjjuXBvc2IddwDOGJ3J2tJKVukOTSISYbwb7kHwuXFZAFz/xLIQVyIiElyeC/dgLgkzbUQqp4xM5UB1AzUNTcF7YxGREOs23M1shJm9bWbrzGyNmd3WSZuzzKzCzFYGvn7aN+W2+8wgvc8dF4wHYMGq0iC9o4hI6PWk594EfM85dxIwG7jFzCZ20m6xc25a4OueoFbZh04ryCA3NYF3N5SFuhQRkaDpNtydc6XOuRWBx4eBdUBuXxfWHTuRS1Q7vM/UESkUbT9AQ1NLUN5TRCTUjmvM3czygenAkk5ePs3MPjGzv5vZyUGord9cMmUYeyvrWb5dd2gSkcjQ43A3s2TgBeB251xlh5dXAHnOuanAb4EXu3iPeWZWZGZFZWW9Gwbpi3tszCrIAGDlTk2JFJHI0KNwN7MY/MH+tHNufsfXnXOVzrmqwONXgRgzy+yk3aPOuULnXGFWVtYJFR6sE6oA6Umx5GcksnKneu4iEhl6MlvGgMeBdc6533TRZmigHWY2M/C+fXLp54nerKMr00emsWLHId1+T0QiQnQP2pwBXAN8amYrA9v+FRgJ4Jx7BLgC+JaZNQG1wFWuj1MySOdT20wdnsJfP97FvsP1DBkcH9w3FxHpZ92Gu3PuPboZBXHOPQQ8FKyiQqEgKxmArfurFe4i4nkD+grV9kZn+8N90UbNdxcR7/NcuLcK9rBMbmoCM/PTeXTRFg5WNwT3zUVE+plnw70v/PDCCTS1OD7aomWARcTbPBfufXmW9uRhg4nyGat3V/Thp4iI9D3PhXurE70TU2fiY6KYkZfGglWlmhIpIp7muXDv69D9yim5bC+v4b6FxX36OSIifclz4d4q2CdUW10ydRgAD75ZTHW91ngXEW/ybLj3lcTYaJ6dNxuAt9bvC3E1IiK947lw74+R8FPz08keFMcruoGHiHiU58K9P0T5jIsm5/D2hn1UaWhGRDzIc+HeX5NYLp6SQ31TC88t20nZ4fr++VARkSDxXLi3CtadmLoyIy+NcUOS+dmCtcz+xZs0t2hqpIh4h2fDva+ZGU/fPJsx2ck0tzgWFWvNGRHxDg+Ge//1oLMGxbXNnLnhiWUanhERz/BguPv17aDMZzKT4zh/4hAAfvHqun76VBGRE+O5cA/FqgCPfGMGU4ansKi4jLrG5v4vQETkOHku3Fv18fnUI/h8xr+cN479VQ3MX7Gr/z5YRKSXPBvu/e2scVmMG5LM4+9tUe9dRMKe58I9VBMSzYxbzh7D5rJqJvzkNZ58f2uIKhER6Z7nwr1VXyz5250LJ+W0Pf75q+toaGrp9xpERHrCc+EeymXWY6N9rL3nAn71lSk0NjuWbTsQumJERI7Bc+Heqj9PqLaXGBvNhZOHkpEUy9cfW8I7G7RypIiEH8+GeygNio/h8etPBeD/luwIcTUiIkfzXLi7kJ1SPdK0Eal8bdZIFhfvZ3+VrlwVkfDiuXBvFaJRmSPceEY+zS2OwnsXsnqXbqotIuHDc+EeTvetHpM9iH+/9GQArnzkQxqbNXtGRMKD58K9VahOqHb0tVkj+d5546htbOa3b+qm2iISHjwb7uHkO+eM4fTRGTz09iZKK2pDXY6IiPfCPZyGZVqZGd+/YDwtDn712oZQlyMi4r1w/0yYjMsETB+ZxgUnD2HBqt1U1jWGuhwRGeA8F+7hMhWyM/M+X0Bjs+PFj7VypIiElufCPZxNH5HG7IJ0fvrSGm55egUuHMeQRGRA8Gy4h8tsmfZ8PuPRawu5euYIXvm0lDtf+JQW3VhbREIgOtQFHK9w7wwPjo/h55dPJik2msfe28qQlHi+e964UJclIgOMd3vuoS7gGHw+484LJ3DKyFQefLOY//j7+lCXJCIDTLfhbmYjzOxtM1tnZmvM7LZO2piZPWhmm8xslZmd0jflekd0lI8nb5xJTko8T324jer6plCXJCIDSE967k3A95xzJwGzgVvMbGKHNhcCYwNf84DfB7VKjxocH8Nvr55OTUMz//n6Bpo1/i4i/aTbcHfOlTrnVgQeHwbWAbkdml0GPOX8PgJSzSyHPmTheEa1EzPy0jh7fBZPfrCNm/64jAPVDaEuSUQGgOMaczezfGA6sKTDS7nAznbPSzj6F0BQhPsJ1Y7MjMevO5Vbzh7NOxvKmPnzhew8UBPqskQkwvU43M0sGXgBuN05V9nx5U6+5agYNrN5ZlZkZkVlZWXHV2kPPjBc+XzGHeeP565LJtLU4vi3F1dT19gc6rJEJIL1KNzNLAZ/sD/tnJvfSZMSYES758OB3R0bOecedc4VOucKs7KyelOvZ5kZN5wxijvOH8e7G8v400fbQ12SiESwnsyWMeBxYJ1z7jddNHsZuDYwa2Y2UOGcKw1inW3CefmBnvjOOWM5KWcwr35aqitYRaTP9KTnfgZwDXCOma0MfF1kZt80s28G2rwKbAE2Af8NfLtvyv2MR86nduqrhcNZseMQU+5+Q1MkRaRPdHuFqnPuPboZ4nb+LugtwSrq2J/VH5/St64/PZ+F6/by/qZy/unRD3ni+plkDYoLdVkiEkG8e4Wqh3vuZsafbprF58ZmsnpXJU+8vzXUJYlIhPFsuHudmfG/gYB//L2tVNQ2sqO8htdW72FvZR2/f2czeyvrQl2miHiU9xYOC3UBQfbDuRO4+LfvMfXuN4567cWPd/HsvNmkJcWGoDIR8TLP9tzNUzPduzYpN4Wzx/unhZ4yMpVBcf7ft1edOoINew9z7yvraGpuCWWJIuJB3uu5R8IZ1Q4evbaQ3YdqyctIwjlHfVML8TFRmMEzS3eSOSiWW88ZS1V9E0MGx4e6XBHxAM+Fe5vI6LgDEBPlIy8jCfCPxcfHRAHwiy9Poaq+mT+8u4U/vLuF+Bgff7ppFoX56aEsV0Q8wLPDMgPF/f80rW3Ypq6xhRufXMauQ7UhrkpEwp3neu6RNyhzbFE+44kbZgKwuayK8+9bxMUPLuYrpwznB3MnEBut388icjTPJkMEjcr02OisZJ6+eRajMpN47L2t3Dl/FbUNWoBMRI7mvZ77QOu6dzC7IIP53z6DBxYWc9/CjcxfsYv0pFjuOH88X5s1MtTliUiY8G7P3cuXqAbBreeO4dl5sxmelsCB6gb+9a+fcuEDi6moaQx1aSISBjwb7gOdmTG7IINXbv0ci39wNpdMHca60kpu+b8VNGpevMiA58FwH+DjMh2kJMQwIj2R3149nf+8YgrvbdrPHz/YFuqyRCTEPBjufgN7UKZzVxaO4PTRGfzytfX88PlVbNp3ONQliUiIeC7cB/oJ1e48/PVTaGx2PFe0kzm/WURLi2NPRR0vLC+h8N6F/PD5VZQdrg91mSLSxzw3W6bVAD+f2qXUxFgeu7aQm58qAmDK3W9Q1e6GIM+vKGH9nkpe+s6ZoSpRRPqB53ru0r05E4fw/p3nMCguuu1G3KOzknjt9s9xx/nj+aSkgvsXbtSCZCIRzHM9d43K9ExuagKf3n0Bzrkjpo3GRPn45WvruX9hMcX7qvj++ePJz0wKYaUi0hc823OPlCV/+1rH6wFGZyWz9MfnAvDKqlLO+vU7uhOUSATybLhL72UPimfrLy7iP748mYLMJO59ZR0L1+4NdVkiEkSeC3fNlgkOM+OqmSN5dt5sRmclcfNTRVzz+BLeK94f6tJEJAg8N+beSrNlgiN7cDwPf30GX3xwMYuL97N8+0FSEmKYkZfGzy+fTEpiTKhLFJFe8GDPXV33YBuTnczyn5zHu98/i8m5KZRW1LFgVSlfeeQDVu48FOryRKQXvNtzD3UBESY5LprkuGienTebJVsPUNvQzA1PLuPy373PxVNy+HjHIe68cAKXTB0W6lJFpAc813OXvtW6INnZE7J55BunALBgVSm7DtXyz898zMyfL+Tuv61h2/7qEFcqIsfiuXDXoEz/mTsph6U/PpefXjyRR74xg/gYH/sO1/PE+9s469fvMO+pIlpadEREwpFnh2U0LtM/sgfFc+OZowBYOmYOOw/U8Pjircz/eBdvrN3L9J/9gytnDOeOC8a33dhbRELPez13dRRDZnB8DCcPS+E3/zSNdffMZc5JQxgUH81j721lwk9e46G3inltdSk3/7GIvZV1oS5XZEDzbM9dV6iGVkJsFI9dVwjAr15bz8PvbObXb2xse33t7gpunzOOWQXp5GVoeQOR/ubZcJfw8YO5E7jq1JG8t2k/H20p51BtI4s2lvGDF1YBcOWM4dxwxijGDkkmygyfT7+YRfqa58Ld6ZRqWBqZkcjXMka23aS7qr6JhWv38sT7W/nL8hKeX1FCTJSP/IxE/v2SkzltdMaAvw+uSF/yXLi3Ui6Et+S4aC6fnsvl03PZUV7D79/dxDNLd7JxbxVfe2wJX56eyzknZbN5XzW3zRkb6nJFIo73wl0dd88ZmZHIL748hZ9dNolPSip4aeUunvpwO/M/3gVATUMTh+ubKMhMorK2kVkFGRyqaeSiyUPVuxfpJe+Fe4D+l/ee6CgfM/LSmJGXxiVTh3HlIx8C8IdFW45s+NYmAH568USuOz2fKI3Rixw3z4a7eNup+elsuHcufy4qYWLOYJpbHDFRRkJsFH/6aDvPLt3JPQvWcs+CtYzKTGLq8BS+MmM4swsyMKC2sZlB8VrUTKQr3Ya7mf0PcDGwzzk3qZPXzwJeAlrv+DDfOXdPMItsT6MykSMuOoprZucdtf3eyydz+5xxPP3RDlbsOEhjcwsvrtzNiyt3k5eRSE1DM4frGnn99s9rmqVIF3rSc38SeAh46hhtFjvnLg5KRT2ksdjIlpkcd8SJ1k9LKrjuiaWUVzW03fD7d29v4sLJObxfvJ/zTx7KzFHpoSpXJOx0G+7OuUVmlt/3pfSMrlAdmCYPT2HFT86jrrGZ19fs4a31+/hzUQl/LioB4E9LtvPRj85lW3kNeemJJMRGaTkEGdCCNeZ+mpl9AuwG7nDOrQnS+3ZJHfeBKT4misum5TIjL421uyvZVl7Nt84aw4NvFjPtnn+0tUuIiWJ4WgKx0T6mjkilIDOJmz9XEMLKRfpXMMJ9BZDnnKsys4uAF4FOJy6b2TxgHsDIkSOD8NEyUA1PS+Qf3/1C2/O4aB8PvllMfVMLAC3OUbyvCoA1uysBuPeVddx54QSuPz2fmCifZuFIRLOe3NkoMCyzoLMTqp203QYUOueOeTPOwsJCV1RU1LMq21lcXMY1jy/lL988jVPzNcYqR6prbKaythEzY/n2g1TWNlKQlcR9Czfy/qbytnYxUcZ3zh6Lw/H1WXlkDYoLYdUiPWdmy51zhd21O+Geu5kNBfY655yZzcS/0mR5N992wtTnks7Ex3w21j530tC27U/dOIvl2w/y7adXsL+qnuxB8dy30L/Q2f0LiynITOK/vjqV6SPT2k7YxkX72F5ezZjsQf2/IyInqCdTIZ8BzgIyzawEuAuIAXDOPQJcAXzLzJqAWuAq14c3OtUJVemNKJ8xc1Q6Rf82B4AD1Q08sHAj1Q3NGPDCihK+9PAHnX7vV04ZTlpiDPsO13PVqSNIiI1i+si0fqxe5Pj1ZLbM1d28/hD+qZL9SidU5USkJ8Vy92WfjTJeNXMk81eUsGZ3JSflDGbHgeq2YZwXVpS0tXv5k90AzDlpCIX5aeypqGNvZR1XzBjO2eOzteKlhA1doSoCbcsitFff1ExslI9t5TXkpMTz8DubefL9rVw6bRh/XlbCwnV78Rm0OPj76j0ATModzBfGZREbFcXa0gq+ddYYpo1IDcUuyQDnuXDXqIz0l7ho/9j9qEz/VbDfPW8c3z1vHAC3zxnHnoo6TsoZzKqSQ9y3sJjt5dVsKatm9a7Ktvd4fc1ebj1nDGOGDGJybgpDB8eTENuz+ff/t2QHLc7xjXZX8a7fU8nGvVVcOnVYsHZTIpTnwv0z+vNXQiczOY7MZP8Mm+kj03jqxpltr20pq2LZtgOsKz3MG2v28GBgITSA7EFxXHXqCF5YsYvM5Fimj0zjysLhLNq4n5mj0qhpaKamoZnUhBj+9a+fAnDexCEMGRwPwNz7FwMcFe6tf2V0d+X24bpG1u6uZFZBxon/ECSseTjcRcJTQVYyBVnJAPz7pSdTWlHLL15dz8GaBraVV7eFfUVtI5+UVPDkB9uO+X6PvLuZa0/Lp7Sitm3bh5vL+dH8Vfz26lMYmhLPqT9fyDe/MJolW8v5zyumkDUonqbmFt7eUMaU4Sn8+vUN3HXpydz10hoWrtvLA1dNY0x2MhNzBlNV36RF2CJQj+a594XeznN/Z8M+rn9iGS986/SjxkhFwl1DUwslB2vwmZGfmcTzy0tYsqWc9ORYUhJiePjtzVwyNYdFG/ez61Bt25h+V75/wXhmjkpvWz651dnjs6hvauGDzceelTwjL401uyv42WWTaGx2XD1zBJV1TVTXNzEsNSEYuyxB1tN57p4N9/nfPp1TNB1NIlRTcwtlVfWkJcby7NIdfLzzEC+t3N3nnzsiPYGdB2rbHv/yy1NoceAzOH1MJtD5EFBLi6OpxREb7evzGge6fruIqb/phKoMBNFRPnJS/D3n688YxfXAA1dN538/2s4Ly0t44KppLNl6gL8U7WTZtoNdvs/M/HRSEmO4eEoOtz27kuxBcRyua6K2sbmtTVpiDAdrGgHagr318dceW9L2fFBcNNeensfv3t4MwIqfnEdyXDSby6p4+J3N/O2T3bzxL58HYFhqAtE+a7ugrLahmdrGZtKTYoPy85Huea7n/vaGfdzwxDL++u3TdSGJDHjOOf62qpTJuSm8tnoPF00eSmpiLK+tLiV7UDxnjs0kJuro3nRFTSO1jc38uWgn18zOY1t5NaMyk7jr5TVH/IXwhXFZnDxsMA+/s7lX9Y3OSmJYagKLi/2rkTw7bzbvbChj3ucLFPS9FLHDMgp3kb7X3OJobjfMsmlfFcNS41lcvJ+JOYN5e8M+3lizF5/PGJWRyMJ1+9h16LNef3JcdNsyDp2ZOiKVey+bxJKt5dQ2NHPz5wp6PEV0oIvYYRmNy4j0vSifHbFq5phs/+yfC072r9dz7Wn5XHtaftvrd1/m/7elxeHwj9Fv2HuYUZlJbC+vYeG6veSmJvDcsp18sLmcT3Ye4pKH3mv7/sq6RoYMjudAdQM3njmKtbsrSYqLblvaOS8jkaQ478VVKHn2p6U7MYmEn/bLL0wYOhiAcUMGMW6If/G1y6bl4pzjrfX7KK2o45VVpXy4pZz/Xry17fs6GwJKjI3i5jNH8cKKXdx/1TQm56a0jedX1zex82ANMVE+0hJjSUuMUT7gwXB36rqLeJqZce5JQwD4xuw8tu2v5ukl28lIjiM2ysc9C9Ye9T01Dc1t1we0Tvscm51MVX0TpRV1R7SN8hmXTh3G0JR4YqN8XDptGDsO1JA9KI6JOYMp3lfFqMwkSg/Vta37P+ek7Ij7heC5cG8VWYdBZODKz0zix1+c2Pb8hjPyaXFQ29hMlBn7q+r5yUurSY6L5tyTsrn7b2s5Z3w2VfVNJMdHs6+ynt2HajlQ08ChmkaaWxx//XhX2/s98GZx2+PPj8ti0cayo2oYlhLPxVOHcc6EbGYXZLB2dyXJcdEMT0vo0WJwDU0txERZWP2C8Gy4i0hkMjOizH9SFmBEeiJP3vDZ8g5fmj68y+91zvH+pnIq6xr5j7+vZ8eBmiNe7yzYAcqrG3h00RYeXbSFxNgoahqaj2ozdHA81fVNxMX4OClnMGWH6zl9dCbpSTE8+cE29lc1kJIQw7ghycwuyGDp1gNcPHUYo7OSqG9qITMpjhbnmNpPC8l5Lty1nruIdMXMOHOs/2Kr8yYOwQCfGT6fsXpXBc75b7be3OKOOGF8sLqBe19Zx44D1WzaV0VNQzMFWUlsKatua7On0j/8c7ietqmd6/ccPuLzK2obWbbtYNu1B0u2HjiqxuFpCfzs8kmcPT47qPvekefCvVUY/fUjImGo4/z+SbkpbY873j83LSmW//rqVMB/BW5LCyTERuGcw8yorGukeG8V44Ykc6imkcbmFqJ9Pn71+nrOHJPJJVOHUXKwlvzMRB5bvJVFG8v4auEInlm6g8q6Rm4+s4Cq+iY+2FzOoo1lbNtfDeP7dv89N8994dq93PxUES9/5wymDNc62SLiLfVNzW3LSfdGT+e5e3YhCNMpVRHxoBMJ9uPh2XAXEZGueS7cdT5VRKR7ngv3VjqhKiLSNc+Fe6hOAIuIeInnwl1ERLqncBcRiUCeC3cNyoiIdM9z4d5KJ1RFRLrmuXDX+VQRke55Ltxb6QpVEZGueTbcRUSkax4Md43LiIh0x4Ph7qcTqiIiXfNsuIuISNc8F+6aLSMi0j3PhXsrDcuIiHTNc+GujruISPe6DXcz+x8z22dmq7t43czsQTPbZGarzOyU4JfZyedqnruISJd60nN/Eph7jNcvBMYGvuYBvz/xskRE5ER0G+7OuUXAgWM0uQx4yvl9BKSaWU6wCjy6nr56ZxGRyBGMMfdcYGcbU2IXAAAFCUlEQVS75yWBbX1KJ1RFRLoWjHDvLGY77V+b2TwzKzKzorKysl592NCUeL44OYfkuOhefb+IyEAQjIQsAUa0ez4c2N1ZQ+fco8CjAIWFhb0aYJmRl8aMvLTefKuIyIARjJ77y8C1gVkzs4EK51xpEN5XRER6qdueu5k9A5wFZJpZCXAXEAPgnHsEeBW4CNgE1AA39FWxIiLSM92Gu3Pu6m5ed8AtQatIREROmOeuUBURke4p3EVEIpDCXUQkAincRUQikMJdRCQCmQvRYi1mVgZs7+W3ZwL7g1iOF2ifBwbt88BwIvuc55zL6q5RyML9RJhZkXOuMNR19Cft88CgfR4Y+mOfNSwjIhKBFO4iIhHIq+H+aKgLCAHt88CgfR4Y+nyfPTnmLiIix+bVnruIiByD58LdzOaa2YbADbnvDHU9wWJmI8zsbTNbZ2ZrzOy2wPZ0M/uHmRUH/k0LbA/JjcmDzcyizOxjM1sQeD7KzJYE9vc5M4sNbI8LPN8UeD0/lHX3lpmlmtnzZrY+cKxPGwDH+F8C/02vNrNnzCw+0o6zmf2Pme0zs9Xtth33cTWz6wLti83suhOpyVPhbmZRwO/w35R7InC1mU0MbVVB0wR8zzl3EjAbuCWwb3cCbzrnxgJvBp5D5NyY/DZgXbvnvwTuC+zvQeCmwPabgIPOuTHAfYF2XvQA8JpzbgIwFf++R+wxNrNc4Fag0Dk3CYgCriLyjvOTwNwO247ruJpZOv4l1WcBM4G7Wn8h9IpzzjNfwGnA6+2e/wj4Uajr6qN9fQk4D9gA5AS25QAbAo//AFzdrn1bO6984b9r15vAOcAC/Lds3A9EdzzewOvAaYHH0YF2Fup9OM79HQxs7Vh3hB/j1nsspweO2wLggkg8zkA+sLq3xxW4GvhDu+1HtDveL0/13AnRzbj7W+BP0enAEmCIC9zZKvBvdqBZJPws7gd+ALQEnmcAh5xzTYHn7fepbX8Dr1cE2ntJAVAGPBEYinrMzJKI4GPsnNsF/BrYAZTiP27Liezj3Op4j2tQj7fXwr3HN+P2KjNLBl4AbnfOVR6raSfbPPOzMLOLgX3OueXtN3fS1PXgNa+IBk4Bfu+cmw5U89mf6p3x/D4HhhUuA0YBw4Ak/MMSHUXSce5OV/sY1H33Wrj3+GbcXmRmMfiD/Wnn3PzA5r1mlhN4PQfYF9ju9Z/FGcClZrYNeBb/0Mz9QKqZtd4hrP0+te1v4PUU4EB/FhwEJUCJc25J4Pnz+MM+Uo8xwBxgq3OuzDnXCMwHTieyj3Or4z2uQT3eXgv3ZcDYwJn2WPwnZl4OcU1BYWYGPA6sc879pt1LLwOtZ82vwz8W37rdszcmd879yDk33DmXj/84vuWc+zrwNnBFoFnH/W39OVwRaO+pHp1zbg+w08zGBzadC6wlQo9xwA5gtpklBv4bb93niD3O7RzvcX0dON/M0gJ/8Zwf2NY7oT4J0YuTFhcBG4HNwI9DXU8Q9+tM/H+CrQJWBr4uwj/e+CZQHPg3PdDe8M8c2gx8in82Qsj3o5f7fhawIPC4AFiK/4brfwHiAtvjA883BV4vCHXdvdzXaUBR4Di/CKRF+jEG7gbWA6uB/wXiIu04A8/gP6fQiL8HflNvjitwY2DfNwE3nEhNukJVRCQCeW1YRkREekDhLiISgRTuIiIRSOEuIhKBFO4iIhFI4S4iEoEU7iIiEUjhLiISgf4f0EvSyxDWhHcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plt.plot(np.asnumpy(hist) * -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Probs have something to do with the depth / width, at least we know it works"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(68.75)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scorenn.score(smol_batch_pixels,smol_batch_label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Testing on a deeper and wider model.\n",
    "\n",
    "According to the paper, wider is better, even at constant parameter count.\n",
    "\n",
    "For starters, lets keep the 3 layers model, and make the hidden layers 1.25 times larger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "newoptimizer = MomentumSGD(0.1,0.9)\n",
    "newscorenn = ScoreFCNN([784,250,250,10],0.5,kaiming_uniform,kaiming_uniform,relu,crossentropy_with_logits,newoptimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 done. Loss : -1.5410837720706656\n",
      "Epoch 1 done. Loss : -0.8331827868285081\n",
      "Epoch 2 done. Loss : -0.72061430549481\n",
      "Epoch 3 done. Loss : -0.664491091224655\n",
      "Epoch 4 done. Loss : -0.6331079629079583\n",
      "Epoch 5 done. Loss : -0.6129379573998177\n",
      "Epoch 6 done. Loss : -0.5966897745681198\n",
      "Epoch 7 done. Loss : -0.5833934950389326\n",
      "Epoch 8 done. Loss : -0.5688775422496612\n",
      "Epoch 9 done. Loss : -0.5595980873396752\n",
      "Epoch 10 done. Loss : -0.551634420776636\n",
      "Epoch 11 done. Loss : -0.5436108081496245\n",
      "Epoch 12 done. Loss : -0.5369025916453636\n",
      "Epoch 13 done. Loss : -0.5310137006902936\n",
      "Epoch 14 done. Loss : -0.5263946790909554\n",
      "Epoch 15 done. Loss : -0.5227101023723674\n",
      "Epoch 16 done. Loss : -0.5184713351217556\n",
      "Epoch 17 done. Loss : -0.5137013486994436\n",
      "Epoch 18 done. Loss : -0.510345059280892\n",
      "Epoch 19 done. Loss : -0.5073897322844743\n",
      "Epoch 20 done. Loss : -0.5043291947273818\n",
      "Epoch 21 done. Loss : -0.501224583621243\n",
      "Epoch 22 done. Loss : -0.4988008502092243\n",
      "Epoch 23 done. Loss : -0.4966628609282628\n",
      "Epoch 24 done. Loss : -0.49420542095701664\n",
      "Epoch 25 done. Loss : -0.49186291106104113\n",
      "Epoch 26 done. Loss : -0.48883986780135785\n",
      "Epoch 27 done. Loss : -0.4861787182208831\n",
      "Epoch 28 done. Loss : -0.48503646632818725\n",
      "Epoch 29 done. Loss : -0.48402316312957594\n"
     ]
    }
   ],
   "source": [
    "history = newscorenn.fit(train_pix,train_lab,32,30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ignore the first epoch error i messed up something for sure xd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x2038627eb00>]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHbBJREFUeJzt3XmUXGd95vHvr9buqt6q1YtsLZYsyfs4xm4MxjYIswmGieHgMCiBiTOAh4BDMpmZkJmTCQ45OQPJMGEmLD4O4+MkJ5h4xmA4JMAAljHGQVhe8SprsXHLUlcvkrp6qbXf+aNutVqtXqq7q1Vd9z6fc/pU1a3bVe9VHT319nvf93fNOYeIiPhLqN4NEBGR2lO4i4j4kMJdRMSHFO4iIj6kcBcR8SGFu4iIDyncRUR8SOEuIuJDCncRER+K1OuNu7q63JYtW+r19iIiDenRRx8dcs51L7Zf3cJ9y5Yt7Nu3r15vLyLSkMzs5Wr207CMiIgPKdxFRHxI4S4i4kMKdxERH1K4i4j4kMJdRMSHFO4iIj7k23B3znHPI6+QLZTq3RQRkbPOt+H+ZP9J/uDep7j/+XS9myIictb5NtyPnZwEYHg8X+eWiIicfb4N94HRHADHFe4iEkC+Dfd0JgvAiMJdRALIt+Fe6bmfmFC4i0jw+DjcvZ77RKHOLREROft8G+6DGY25i0hw+TbcKz334xqWEZEA8mW454oljnvDMeq5i0gQ+TLcK0MyG1PNjOdL5IpapSoiweLLcK/MlLlofSsAJ3RSVUQCxpfhnvbG2y9a3wZorruIBI8/w90blrnQ67nrpKqIBI0vw31gNEs4ZGzrbgHg+LiGZUQkWHwZ7ulMjp7WOF0tMUA9dxEJHl+G+8Bolp7WOB0JL9w15i4iAePLcE+P5uhpayIWCdESjzCinruIBIw/wz2TpbctDkAqGdVUSBEJHN+Fe2V1ak9rEwCpRExTIUUkcHwX7mlvAdN0zz0R0wlVEQkc/4W7N8e9p63cc+9MKtxFJHj8F+7e6tSe1nLPvSMR1Tx3EQkc34V7pdRvb6XnnogxliuSL07Vs1kiImfVouFuZneaWdrMnl5kv9eaWcnMbqpd85YunckRCRmd3hz3VLJ8q8vtiUiQVNNzvwvYtdAOZhYGPgd8vwZtWpGB0RzdrXFCIQPKJ1QBzXUXkUBZNNydcw8CI4vs9jvAvUC6Fo1aiXQmO30yFcrz3EH1ZUQkWFY85m5mG4D3ArevvDkrlx7N0eudTIVTPXfNmBGRIKnFCdUvAJ9yzi16uSMzu8XM9pnZvsHBwRq89ZkGMll62k6Fe2dS4S4iwROpwWv0AV83M4Au4F1mVnTO3Td7R+fcHcAdAH19fa4G732abKHEiYkCva2nhmU6EpVhGYW7iATHisPdObe1ct/M7gK+M1ewnw2Va6f2zhhzj0fCJGNhRjTmLiIBsmi4m9ndwE6gy8z6gU8DUQDn3JoYZ69IZ8pz3LtnDMtAeTqkpkKKSJAsGu7Oud3Vvphz7uYVtWaFKhfGnjksA17xMIW7iASIr1aopqdXp57Zc9eYu4gEia/CfcBbnVqZ/ljRmYhyXDXdRSRA/BXu3uX1KqtTKzoS6rmLSLD4KtwHM7nTVqdWdCZjZHJFCiUVDxORYPBVuFd67rOltJBJRALGZ+GeO22Oe0UqofoyIhIsvgn3bKHEycnCGTNlgOnyv+q5i0hQ+CbcB2ddXm+mjkq466SqiASEb8J9YNbl9WY6VTxMwzIiEgy+Cff0HHVlKqaLh2lYRkQCwjfhPvvaqTM1RcMkYmFGNCwjIgHho3DPEQ3b9MyY2VKJmHruIhIYvgn3dCZLT2sTXl35M6SSUZ1QFZHA8E+4j+ZOuwLTbOXKkDqhKiLB4Jtwn291akWnarqLSID4JtzTmblXp1akEjGdUBWRwPBFuJ9anbpwuGeyKh4mIsHgi3BPe1dg6l5wWKY8i+aExt1FJAD8Ee6Z+ee4V3SovoyIBIgvwn362qkLzJaZLkGgcXcRCQCfhHulrsxCPXeVIBCR4PBFuKczC69OBRUPE5Fg8Ue4jy68OhWYvmi2pkOKSBD4ItwHMtkFV6dCuXhYczSsMXcRCQRfhHt6NEfvAuPtFZ3JmIZlRCQQfBHuA6PZBWfKVHQkojqhKiKB0PDhni2UGM0W57y83mydSZUgEJFgaPhwr6xOXahoWEUqoeJhIhIMi4a7md1pZmkze3qe53/DzJ7yfh42s1+pfTPnN1DF6tSKVCKqnruIBEI1Pfe7gF0LPH8YeJNz7nLgT4E7atCuqk0vYKpizD2VjDGaLVJU8TAR8blFw9059yAwssDzDzvnjnsPfwZsrFHbqlIZlql2tgzAiUnNmBERf6v1mPuHge/W+DUXNJDJEguHpssLLGS6eJiGZkTE5yK1eiEzezPlcL9ugX1uAW4B2Lx5c03eNz2ao7s1vuDq1IrOhEoQiEgw1KTnbmaXA18FbnTODc+3n3PuDudcn3Our7u7uxZvTTpT3Rx3OFU8TCdVRcTvVhzuZrYZ+AbwIefc/pU3aWkGRhe+vN5M02Pumg4pIj636LCMmd0N7AS6zKwf+DQQBXDO3Q78MbAO+LI3NFJ0zvWtVoNnGxjNcu22dVXtO108TOEuIj63aLg753Yv8vxHgI/UrEVLMJkvkalydSpAcyxMUzSkE6oi4nsNvUK1msvrzdaZUPEwEfG/hg73gSWUHqhIJWPquYuI7zV0uC+n555KxDTmLiK+19DhXs2FsWdLJWOc0LCMiPhcQ4d7ejRLLBKivXnx1akVKh4mIkHQ2OGeydFT5erUilQixmi2oOJhIuJrDR3u5SswVT/eDuWFTM7BSRUPExEfa/hwX8pMGThVgkCX2xMRP2vocE9nqi89UFEpQaC57iLiZw0b7hP5orc6dWk99+kSBDqpKiI+1rDhfuraqUvruadUPExEAqBxwz2z9DnucKqm+8i4hmVExL8aNtwr105d6ph7cyxMPBLSCVUR8bWGD/elzpaB8klV1ZcRET9r2HAfzOSWvDq1IpWIqecuIr7WsOFeXsC0tNWpFamkShCIiL81cLjnljxTpiKVUPEwEfG3hg33pVwYezaV/RURv2vccF9Jzz0Z4+RkgdKUq3GrRETWhoYM94l8kUxu6atTKzoTURUPExFfa8hwr6xO7V1Bzx1UgkBE/Kshw325C5gqKvVlVIJARPyqMcPdKz2w3GEZFQ8TEb9ryHBPV3ruyx6WUU13EfG3xgz3TI54JERbc2RZv6+a7iLidw0Z7gOjWXqWuToVoDkaJhYJqb6MiPhWQ4Z7ejS37CEZADOjU/VlRMTHGjLcBzJLvzD2bKlkTDXdRcS3GjLc06M5updR6nemVCKqnruI+Nai4W5md5pZ2syenud5M7P/ZWYHzOwpM7uy9s08ZTxXZCxXrEnPXeEuIn5VTc/9LmDXAs+/E9jh/dwCfGXlzZrfci+vN1sqEdUJVRHxrUXD3Tn3IDCywC43An/ryn4GdJjZObVq4GynrsC0sp57Z0LFw0TEv2ox5r4BeGXG435v2xnM7BYz22dm+wYHB5f1ZjXruSdjTDkYVfEwEfGhWoT7XJPN5+wOO+fucM71Oef6uru7l/Vmb7moh3/85HVsXpdY1u9XTJcg0Li7iPjQ8pZ4nq4f2DTj8Ubg1Rq87pyS8QiXntu+4tepVIZU8TAR8aNa9Ny/Dfwbb9bM64GTzrmjNXjdVdU5XTxMwzIi4j+L9tzN7G5gJ9BlZv3Ap4EogHPuduCfgHcBB4AJ4LdWq7G11JHwiodpxoyI+NCi4e6c273I8w74RM1adJacKh6mcBcR/2nIFaq1kIiFiYVDOqEqIr4U2HA3M1LJKCc05i4iPhTYcIfydEj13EXEjwIf7jqhKiJ+FOhw71TxMBHxqUCHe0ciqkvtiYgvBTrcO5MxTkzkmVLxMBHxmUCHeyrhFQ/LqvcuIv4S7HBPllepjuikqoj4TLDDPVFZpaqeu4j4S6DDfboEgXruIuIzgQ531XQXEb8KdrirpruI+FSgwz0ZCxMNm2q6i4jvBDrczYxUIqaeu4j4TqDDHconVTUVUkT8JvDhXi5BoHAXEX8JfLiXi4dpzF1E/CXw4d6hsr8i4kOBD/fORIwTkwUVDxMRXwl8uKeSMUpTjky2WO+miIjUjMI94RUP00lVEfERhXulvozCXUR8JPDh3plQ8TAR8Z/Ah/t08TCFu4j4SODDvbOlHO4vDY/XuSUiIrUT+HBviUe44aIevrb3l0zkNWNGRPwh8OEO8Ik3b+f4RIGv7f1lvZsiIlITVYW7me0ysxfM7ICZ/eEcz282sz1m9riZPWVm76p9U1fPVeeleMO2ddzx4CGyhVK9myMismKLhruZhYEvAe8ELgF2m9kls3b7I+Ae59xrgA8AX651Q1fbrW/eTjqT4/882l/vpoiIrFg1PfergQPOuUPOuTzwdeDGWfs4oM273w68Wrsmnh3XbFvHlZs7uP2BgxRKU/VujojIilQT7huAV2Y87ve2zXQb8EEz6wf+CfidmrTuLDIzbr1hO0dOTPKtJxruu0lE5DTVhLvNsW12la3dwF3OuY3Au4C/M7MzXtvMbjGzfWa2b3BwcOmtXWVvvrCHS85p48t7DlBSITERaWDVhHs/sGnG442cOezyYeAeAOfcPwNNQNfsF3LO3eGc63PO9XV3dy+vxauo0ns/NDTOd58+Wu/miIgsWzXh/giww8y2mlmM8gnTb8/a55fAWwDM7GLK4b72uuZV2HXperZ1J/ni/QdwTr13EWlMi4a7c64I3Ap8H3iO8qyYZ8zsM2b2q95u/wH4qJk9CdwN3OwaNBlDIePjO7fz/LEMP3ouXe/miIgsi9Urg/v6+ty+ffvq8t6LKZSmuOHzD7AuGeebH38DZnOddhAROfvM7FHnXN9i+2mF6hyi4RAfe9M2nnjlBD89MFzv5oiILJnCfR43XbWR3rY4X9zzYr2bIiKyZAr3ecQjYW554zZ+dmiEfS+N1Ls5IiJLonBfwO6rN9GZjPHFPQfq3RQRkSVRuC8gEYvw4eu28sALgzx95GS9myMiUjWF+yI+dM15tDZF+OL96r2LSONQuC+irSnKzW/YwveeOcaLA5l6N0dEpCoK9yr81rVbScTCfPmBg/VuiohIVRTuVehMxviN123mW08c4WVda1VEGoDCvUofvf58IuEQf3Tf0+SKulqTiKxtCvcq9bQ18ZlfvZSfvDjEJ+9+XBf0EJE1TeG+BB+4ejO3/atL+P4zA/z+PU+q5ruIrFmRejeg0dx87VayxSk++93niYVD/MVNlxMKqbCYiKwtCvdl+NibtpErTPGXP9xPPBriz95zmSpHisiaonBfpk++ZTvZYomvPHCQeCTEH7/7EgW8iKwZCvdlMjP+4B0XkitMcedPDxOPhPnUrgsV8CKyJijcV8DM+K/vvphcscTtPz5IUzTE7731gno3S0RE4b5SZsaf3ngZueIUX/jhi8QjYX5757Z6N0tEAk7hXgOhkPG5911OvjjF5773PPFIiH973dZ6N0tEAkzhXiPhkPH59/8KuWKJz3znWfKlKT5y3VYiYS0lEJGzT8lTQ9FwiL/afSXvuLSXz373ed79Vw/x8MGhejdLRAJI4V5jsUiI2z94Fbd/8ErGckV+/a/38rG/e5RXRibq3TQRCRANy6wCM2PXZeew88IevvqTQ3xpz0HufyHNR6/fysd3bicZ1z+7iKwu9dxXUVM0zK037GDPf9zJv/wX5/ClPQe54fMP8M3H+5lSXRoRWUUK97NgfXsTf/mvr+De334D69ua+Pf/8CTvu/1hnnjlRL2bJiI+pXA/i646L8U3P34tf3HT5bwyMsl7vvRTbv3aY/z0wJB68iJSUxr8PctCIePX+jax67L1fGnPQf5+78t856mjnNvexHuv3MD7rtzI+d0t9W6miDQ4c64+Pca+vj63b9++urz3WpItlPjBswPc+1g/D+4fZMrBazZ3cNNVG3n35efS3hytdxNFZA0xs0edc32L7qdwXzsGRrPc9/gR7n2sn/0DY8QiId52SS83XbmR63d0aUGUiNQ23M1sF/A/gTDwVefcZ+fY5/3AbYADnnTO/fpCr6lwn59zjqePjHLvY/1864kjHJ8o0NUS44aLerjhol6u39Gl6ZQiAVWzcDezMLAfeBvQDzwC7HbOPTtjnx3APcANzrnjZtbjnEsv9LoK9+rki1Pc/3ya7zz1Kj/eP0gmWyQWCXHN+et4y8U9vOXiXjZ0NNe7mSJyltQy3K8BbnPOvcN7/J8BnHP/bcY+fw7sd859tdoGKtyXrlCa4pGXRrj/uTQ/ej7N4aFxAC5a38pbL+7lhot7uGJjhy77J+JjtQz3m4BdzrmPeI8/BLzOOXfrjH3uo9y7v5by0M1tzrnvzfFatwC3AGzevPmql19+ufojkjMcHBzj/ufS/PC5Afa9fJzSlGNdMsbrz1/H1Vs7ed35nVzQ06qwF/GRasO9moHbuZJh9jdCBNgB7AQ2Aj8xs8ucc6et0nHO3QHcAeWeexXvLQvY1t3Ctu4WPvrG8zkxkefH+wf58QuD7D08wj/+4igAHYkor93Syeu2dvK6reu45Nw2wgp7Ed+rJtz7gU0zHm8EXp1jn5855wrAYTN7gXLYP1KTVsqiOhIxbrxiAzdesQGAV0Ym2Ht4hJ8fHmbv4RF+8OwAAK3xCH1bUly9dR3Xbl/Hpee2K+xFfKiacH8E2GFmW4EjwAeA2TNh7gN2A3eZWRdwAXColg2VpdnUmWBTZ4KbrtoIwLGTWfZ6Qb/30DB7XhgEyj37a7d1ce32Lq7f0cWmzkQ9my0iNbJouDvnimZ2K/B9yuPpdzrnnjGzzwD7nHPf9p57u5k9C5SA/+ScG17NhsvSrG9vOq1nn85kefjAMA8dGOKhF4emh3E2dya4bkcX12/v4ppt6+hIxOrZbBFZJi1iEpxzHBwc56EXB3nowDA/OzTMWK6IGVy+oZ3LN3ZwwfpWLuxt5YLeFgW+SB3V8oSq+JyZsb2nhe09Ldx87VYKpSme6j/BT14c4uEDw9z3+BEyueL0/j2tcS5c38oFXthf0NvKjt5WWrSwSmTNUM9dFuWc4+jJLPsHMuwfyPDCsTH2D2R4MZ0hW5ia3u+c9ia2diVP+9nSlWRTKkEsotIJIrWgnrvUjJlxbkcz53Y0s/PCnuntpSlH//EJXjhWDv1Dg+McHh7nO08d5eRkYXq/cMjYlGpmixf453uhv7UrybntzZqHL7IKFO6ybOGQcd66JOetS/L2S9ef9tzx8TyHh8c5PDjO4aHx6fs/PzzCRL40vV88EmLLOq+n330q/Ld2JelMxjBT8Issh8JdVkUqGSOVjHHl5tRp251zpDO5ci9/aJyXhsc5NDjOi+kMP3p+gELp1DBhazzChlQzG1PNbEwl2NBRvr/Be5xKRBX+IvNQuMtZZWb0tjXR29bENdvWnfZcsTTFkROTHBoa56WhcV4enqD/+CT9xyfYe2jktJO6AM3R8HTYn9PexPo277a9iXPam+htb6I1HtEXgASSwl3WjEg4ND3Mw4VnPn9yskD/8XLgHzk+OR38R05M8vSRkwyN5c/4nWQs7IV9M71tTXS1xliXjLEuGWddS4yulvJtZzJGPBI+C0cpcnYo3KVhtDdHaW9u59Jz2+d8PlcskR7NcWw0y9GTWY6dnOToySwD3uN/PjjE0HiefHFqzt9vjUemgz6ViJXfLxH13jdKx4z77c2x6W1RXURF1iCFu/hGPBKeLrswH+ccY7kiw2N5hsdz3m2e4bEcQzPuHxvN8vyxDKOThTOGg2Zra4rQmYzRkTj1xdCZjJJKxuhMlM89dLWU/1roao2TjIU1VCSrTuEugWJmtDZFaW2KsqUrWdXvFEtTjGaLnJwscHKywImJ/PT94+MFjk/kGRnPc3wiz8BoluePjjI8nic3z18I8UiIrpZ4OfBPu43TmYzS1hSlrblyG6G1KaovBFkyhbvIIiLhEJ3Jcq98KSbzJUYm8ozM+CthaCzH8HieoUyOofE8x05meebVkwyP5SlOzb+gMBwyWpsipwI/HqWlKUJrPEJLU4QW77byOBmrPI7S2uTdb4rovEKAKNxFVklzLMyGWHNVl0GcmnLlvwomC4xOFhjNFshki9P3RyeL3q23PVvglZEJxnLF8k+2uOCXQ0UsHDot7Fvi5b8MWuORGduj3vbyl8nsfZOxsC7W3gAU7iJrQChk02sDlsM5R644RSZ7KuwzufIXwXiuOL09ky2SyRam749li/QfnyTjfZmM5YqUqvySSMTDJKJhEvEIiVjY+4nQHAuT9O43RcM0R8M0x0I0R8Plx96+lecSsdN/vyka0hBUDSjcRXzAzGjywrO7Nb7s13HOMVkoeV8CxdNCv3J/Il9iPF9kMl9iPFdislDeNpErkc5kmciVmMiXmCyUf+abnTT/sTDrSyNCMnbqS6HyBZKIlh83xyIk4zO+KOJh7y+R6Kkhq3gkcBelUbiLyDQz83rSEXrbavOapSlH1gv6yfzct+O5ondbYjJfZDxfYiLvfZHkyvcz2SLp0RwThfIXy4T3U61ELDw9vNTiDUUlYmFa4uUvhGS8fK6ifFt+3BIvf5HEIiGioRCRsBENh4h6t5GwEQuHiIRDRELlbWvlS0ThLiKrKhyycmCuQknoqSlHtuj9peCF/Xi+PNxUGZ4a9YahxrIzhqe8v0QGMznG8+Whq/H80v/KmIsZREPlkK98GcwM/kjY2P3azXz0jefX4F9gfgp3EWlYodCpvzRqoVCaYiJXYixfZMI7WT2RL1EoTVEoOYqlKfKlKYolR3Fqiry37dTz5e2FkqPk3RanKvt7+065FQ2dVUvhLiLiiYZDtCdCtCei9W7Kimk+k4iIDyncRUR8SOEuIuJDCncRER9SuIuI+JDCXUTEhxTuIiI+pHAXEfEhc27xCnCr8sZmg8DLy/z1LmCohs2pJx3L2uSXY/HLcYCOpeI851z3YjvVLdxXwsz2Oef66t2OWtCxrE1+ORa/HAfoWJZKwzIiIj6kcBcR8aFGDfc76t2AGtKxrE1+ORa/HAfoWJakIcfcRURkYY3acxcRkQU0XLib2S4ze8HMDpjZH9a7PSthZi+Z2S/M7Akz21fv9iyFmd1pZmkze3rGtk4z+4GZvejdpurZxmrMcxy3mdkR73N5wszeVc82VsvMNpnZHjN7zsyeMbPf9bY31OeywHE03OdiZk1m9nMze9I7lj/xtm81s73eZ/IPZra8K6Mv9N6NNCxjZmFgP/A2oB94BNjtnHu2rg1bJjN7CehzzjXc3F0zeyMwBvytc+4yb9ufAyPOuc96X7wp59yn6tnOxcxzHLcBY865/17Pti2VmZ0DnOOce8zMWoFHgfcAN9NAn8sCx/F+GuxzMTMDks65MTOLAg8Bvwv8PvAN59zXzex24Enn3Fdq+d6N1nO/GjjgnDvknMsDXwdurHObAsk59yAwMmvzjcDfePf/hvJ/yDVtnuNoSM65o865x7z7GeA5YAMN9rkscBwNx5WNeQ+j3o8DbgD+r7d9VT6TRgv3DcArMx7306AfuscB/8/MHjWzW+rdmBrodc4dhfJ/UKCnzu1ZiVvN7Clv2GZND2PMxcy2AK8B9tLAn8us44AG/FzMLGxmTwBp4AfAQeCEc67o7bIqOdZo4W5zbGuccaUzXeucuxJ4J/AJb4hA6u8rwDbgCuAo8Pn6NmdpzKwFuBf4PefcaL3bs1xzHEdDfi7OuZJz7gpgI+XRh4vn2q3W79to4d4PbJrxeCPwap3asmLOuVe92zTwTcoffCMb8MZLK+Om6Tq3Z1mccwPef8gp4K9poM/FG9e9F/h759w3vM0N97nMdRyN/LkAOOdOAA8Arwc6zCziPbUqOdZo4f4IsMM70xwDPgB8u85tWhYzS3onizCzJPB24OmFf2vN+zbwm9793wS+Vce2LFslCD3vpUE+F+/k3f8GnnPO/Y8ZTzXU5zLfcTTi52Jm3WbW4d1vBt5K+RzCHuAmb7dV+UwaarYMgDf96QtAGLjTOfdndW7SspjZ+ZR76wAR4GuNdCxmdjewk3J1uwHg08B9wD3AZuCXwK8559b0ycp5jmMn5T/9HfAS8O8qY9ZrmZldB/wE+AUw5W3+L5THqxvmc1ngOHbTYJ+LmV1O+YRpmHJn+h7n3Ge8//9fBzqBx4EPOudyNX3vRgt3ERFZXKMNy4iISBUU7iIiPqRwFxHxIYW7iIgPKdxFRHxI4S4i4kMKdxERH1K4i4j40P8HP18ykFPvmYIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plt.plot(np.asnumpy(history) * -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance of Score NN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is our performance using scoreNN. Pretty good, huh?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance on training data : 93.35978835978837 %\n"
     ]
    }
   ],
   "source": [
    "print(f\"Performance on training data : {newscorenn.score(train_pix,train_lab)} %\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance on training data : 92.26190476190477 %\n"
     ]
    }
   ],
   "source": [
    "print(f\"Performance on training data : {newscorenn.score(test_pix,test_lab) } %\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}