1. Basics of Neural Networks - Fine Tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Workspace\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.datasets import fashion_mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "(trainX, trainY), (testX,testY) = fashion_mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28, 28)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainX[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x2625d2d6320>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEfJJREFUeJzt3W2M1eWZx/HfJfjEg6AigsiKVlzZGBfXEY1PUStGN41atVhfbDDW0piabJOarPFNTcxGott2+8I0odZUY2vbpFI1PtWYTdwNqIyEAHW2LSrWERxUFHl0GLj2BYfNiPO/rsM5Z8459P5+EjMz55p7zj1n+HnOzPW/79vcXQDKc1inJwCgMwg/UCjCDxSK8AOFIvxAoQg/UCjCDxSK8AOFIvxAoca2887MjMsJgVHm7lbP5zX1zG9mV5vZn8xsnZnd3czXAtBe1ui1/WY2RtKfJc2X1C9phaRb3P3NYAzP/MAoa8cz/zxJ69z9bXcflPRrSdc18fUAtFEz4Z8h6b1hH/fXbvsCM1tkZr1m1tvEfQFosWb+4DfSS4svvax39yWSlki87Ae6STPP/P2SZg77+GRJG5qbDoB2aSb8KyTNNrNTzewISd+U9HRrpgVgtDX8st/dh8zsTkkvShoj6RF3/2PLZgZgVDXc6mvozvidHxh1bbnIB8Chi/ADhSL8QKEIP1Aowg8UivADhSL8QKEIP1Aowg8UivADhSL8QKEIP1Aowg8Uqq1bd6P9zOIFXs2u6pw4cWJYv/jiiytrzz//fFP3nX1vY8aMqawNDQ01dd/NyuYeadVKXJ75gUIRfqBQhB8oFOEHCkX4gUIRfqBQhB8oFH3+v3GHHRb//33Pnj1h/fTTTw/rt99+e1jfuXNnZW379u3h2F27doX1119/Paw308vP+vDZ45qNb2Zu0fUL2c9zOJ75gUIRfqBQhB8oFOEHCkX4gUIRfqBQhB8oVFN9fjNbL2mrpD2Shty9pxWTQutEPWEp7wtfccUVYf3KK68M6/39/ZW1I488Mhw7bty4sD5//vyw/vDDD1fWBgYGwrHZmvmD6aePZMKECZW1vXv3hmN37NjR1H3v14qLfC53949a8HUAtBEv+4FCNRt+l/QHM3vDzBa1YkIA2qPZl/0XufsGM5sq6SUz+193f2X4J9T+p8D/GIAu09Qzv7tvqL3dJGmppHkjfM4Sd+/hj4FAd2k4/GY23swm7n9f0lWS1rZqYgBGVzMv+0+UtLS2dHGspF+5+wstmRWAUddw+N39bUn/2MK5YBQMDg42Nf68884L67NmzQrr0XUG2Zr4F198Mayfc845Yf2BBx6orPX29oZj16xZE9b7+vrC+rx5X/oN+Auix3XZsmXh2OXLl1fWtm3bFo4djlYfUCjCDxSK8AOFIvxAoQg/UCjCDxTKWnXcb113Zta+OytItE109vPNlsVG7TJJmjx5cljfvXt3ZS1buppZsWJFWF+3bl1lrdkW6PTp08N69H1L8dxvuummcOxDDz1UWevt7dVnn31W1/nfPPMDhSL8QKEIP1Aowg8UivADhSL8QKEIP1Ao+vxdIDvOuRnZz/fVV18N69mS3Uz0vWXHVDfbi4+O+M6uMVi5cmVYj64hkPLv7eqrr66snXbaaeHYGTNmhHV3p88PoBrhBwpF+IFCEX6gUIQfKBThBwpF+IFCteKUXjSpnddaHOiTTz4J69m69Z07d4b16BjusWPjf37RMdZS3MeXpKOPPrqylvX5L7nkkrB+4YUXhvVsW/KpU6dW1l54oT3HX/DMDxSK8AOFIvxAoQg/UCjCDxSK8AOFIvxAodI+v5k9Iulrkja5+1m1246T9BtJsyStl7TA3eOGMbrSuHHjwnrWr87qO3bsqKxt2bIlHPvxxx+H9Wyvgej6iWwPhez7yh63PXv2hPXoOoOZM2eGY1ulnmf+X0g6cOeBuyW97O6zJb1c+xjAISQNv7u/ImnzATdfJ+nR2vuPSrq+xfMCMMoa/Z3/RHffKEm1t9XXKgLoSqN+bb+ZLZK0aLTvB8DBafSZf8DMpktS7e2mqk909yXu3uPuPQ3eF4BR0Gj4n5a0sPb+QklPtWY6ANolDb+ZPSFpuaS/N7N+M/uWpMWS5pvZXyTNr30M4BCS/s7v7rdUlL7a4rkUq9mec9RTztbEn3TSSWH9888/b6oerefP9uWPrhGQpMmTJ4f16DqBrE9/xBFHhPWtW7eG9UmTJoX11atXV9ayn1lPT/Vv0G+++WY4djiu8AMKRfiBQhF+oFCEHygU4QcKRfiBQrF1dxfItu4eM2ZMWI9afTfffHM4dtq0aWH9ww8/DOvR9thSvHR1/Pjx4dhsaWvWKozajLt37w7HZtuKZ9/38ccfH9YfeuihytrcuXPDsdHcDua4d575gUIRfqBQhB8oFOEHCkX4gUIRfqBQhB8olLXzeGgz69xZ1F0s6ykPDQ01/LXPP//8sP7ss8+G9ewI7mauQZg4cWI4NjuCO9va+/DDD2+oJuXXIGRHm2ei7+3BBx8Mxz7++ONh3d3ravbzzA8UivADhSL8QKEIP1Aowg8UivADhSL8QKEOqfX80VrlrN+cbX+drYOO1n9Ha9br0UwfP/Pcc8+F9e3bt4f1rM+fbXEdXUeS7RWQ/UyPOuqosJ6t2W9mbPYzz+Z+9tlnV9ayo8tbhWd+oFCEHygU4QcKRfiBQhF+oFCEHygU4QcKlfb5zewRSV+TtMndz6rddq+kb0va36i9x93jhnIdmlkbPpq98tF26aWXhvUbb7wxrF900UWVteyY62xNfNbHz/YiiH5m2dyyfw/RvvxSfB1Ato9FNrdM9rht27atsnbDDTeEY5955pmG5nSgep75fyHp6hFu/7G7z63913TwAbRXGn53f0XS5jbMBUAbNfM7/51mttrMHjGzY1s2IwBt0Wj4fyrpK5LmStoo6YdVn2hmi8ys18x6G7wvAKOgofC7+4C773H3vZJ+Jmle8LlL3L3H3XsanSSA1mso/GY2fdiHX5e0tjXTAdAu9bT6npB0maQpZtYv6QeSLjOzuZJc0npJ3xnFOQIYBcXs23/ccceF9ZNOOimsz549u+GxWd/2jDPOCOuff/55WI/2KsjWpWfnzG/YsCGsZ/vfR/3u7Az7wcHBsD5u3LiwvmzZssrahAkTwrHZtRfZev5sTX70uA0MDIRj58yZE9bZtx9AiPADhSL8QKEIP1Aowg8UivADheqqVt8FF1wQjr/vvvsqayeccEI4dvLkyWE9WnoqxctLP/3003Bsttw4a1llLa9o2/Fs6+2+vr6wvmDBgrDe2xtftR0dw33ssfGSkFmzZoX1zNtvv11Zy44H37p1a1jPlvxmLdSo1XjMMceEY7N/L7T6AIQIP1Aowg8UivADhSL8QKEIP1Aowg8Uqu19/qhfvnz58nD89OnTK2tZnz6rN7NVc7bFdNZrb9akSZMqa1OmTAnH3nrrrWH9qquuCut33HFHWI+WBO/atSsc+84774T1qI8vxcuwm11OnC1lzq4jiMZny4VPOeWUsE6fH0CI8AOFIvxAoQg/UCjCDxSK8AOFIvxAodra558yZYpfe+21lfXFixeH4996663KWrYVc1bPjnuOZD3fqA8vSe+9915Yz7bPjvYyiLb1lqRp06aF9euvvz6sR8dgS/Ga/Oxncu655zZVj773rI+fPW7ZEdyZaA+G7N9TtO/FBx98oMHBQfr8AKoRfqBQhB8oFOEHCkX4gUIRfqBQhB8o1NjsE8xspqTHJE2TtFfSEnf/iZkdJ+k3kmZJWi9pgbt/En2toaEhbdq0qbKe9bujNdLZMdbZ1856zlFfN9tnffPmzWH93XffDevZ3KL9ArI189mZAkuXLg3ra9asCetRnz87Nj3rxWfnJUTHk2ffd7amPuvFZ+OjPn92DUF0pHv2mAxXzzP/kKTvu/scSRdI+q6Z/YOkuyW97O6zJb1c+xjAISINv7tvdPeVtfe3SuqTNEPSdZIerX3ao5LiS8EAdJWD+p3fzGZJOkfSa5JOdPeN0r7/QUia2urJARg9dYffzCZI+p2k77n7ZwcxbpGZ9ZpZb/Y7HID2qSv8Zna49gX/l+7+ZO3mATObXqtPlzTiX/LcfYm797h7T7OLIQC0Thp+2/dnyZ9L6nP3Hw0rPS1pYe39hZKeav30AIyWtNUn6SJJ/yJpjZmtqt12j6TFkn5rZt+S9FdJ38i+0ODgoN5///3Kera8uL+/v7I2fvz4cGy2hXXWIvnoo48qax9++GE4duzY+GHOlhNnbaVoWW22hXS2dDX6viVpzpw5YX379u2Vtaz9+sknYec4fdyiuUdtQClvBWbjsyO6o6XUW7ZsCcfOnTu3srZ27dpw7HBp+N39fyRVNSW/Wvc9AegqXOEHFIrwA4Ui/EChCD9QKMIPFIrwA4Wqp8/fMjt37tSqVasq608++WRlTZJuu+22ylq2vXV2nHO29DVaVpv14bOeb3blY3YEeLScOTuaPLu2Iju6fOPGjQ1//Wxu2fURzfzMml0u3MxyYim+juDUU08Nxw4MDDR8v8PxzA8UivADhSL8QKEIP1Aowg8UivADhSL8QKHaekS3mTV1Z9dcc01l7a677grHTp0abzGYrVuP+rpZvzrr02d9/qzfHX39aItoKe/zZ9cwZPXoe8vGZnPPROOjXnk9sp9ZtnV3tJ5/9erV4dgFCxaEdXfniG4A1Qg/UCjCDxSK8AOFIvxAoQg/UCjCDxSq7X3+aJ/4rDfajMsvvzys33///WE9uk5g0qRJ4dhsb/zsOoCsz59dZxCJjkyX8usAonMYpPhnum3btnBs9rhkorln696zfQyyn+lLL70U1vv6+ipry5YtC8dm6PMDCBF+oFCEHygU4QcKRfiBQhF+oFCEHyhU2uc3s5mSHpM0TdJeSUvc/Sdmdq+kb0vafzj9Pe7+XPK12ndRQRudeeaZYX3KlClhPdsD/uSTTw7r69evr6xl/ey33norrOPQU2+fv55DO4Ykfd/dV5rZRElvmNn+Kxh+7O7/0egkAXROGn533yhpY+39rWbWJ2nGaE8MwOg6qN/5zWyWpHMkvVa76U4zW21mj5jZsRVjFplZr5n1NjVTAC1Vd/jNbIKk30n6nrt/Jumnkr4iaa72vTL44Ujj3H2Ju/e4e08L5gugReoKv5kdrn3B/6W7PylJ7j7g7nvcfa+kn0maN3rTBNBqafht3xaoP5fU5+4/Gnb79GGf9nVJa1s/PQCjpZ5W38WS/lvSGu1r9UnSPZJu0b6X/C5pvaTv1P44GH2tv8lWH9BN6m31HVL79gPIsZ4fQIjwA4Ui/EChCD9QKMIPFIrwA4Ui/EChCD9QKMIPFIrwA4Ui/EChCD9QKMIPFIrwA4WqZ/feVvpI0rvDPp5Su60bdevcunVeEnNrVCvndkq9n9jW9fxfunOz3m7d269b59at85KYW6M6NTde9gOFIvxAoTod/iUdvv9It86tW+clMbdGdWRuHf2dH0DndPqZH0CHdCT8Zna1mf3JzNaZ2d2dmEMVM1tvZmvMbFWnjxirHYO2yczWDrvtODN7ycz+Uns74jFpHZrbvWb2fu2xW2Vm/9yhuc00s/8ysz4z+6OZ/Wvt9o4+dsG8OvK4tf1lv5mNkfRnSfMl9UtaIekWd3+zrROpYGbrJfW4e8d7wmZ2qaRtkh5z97Nqtz0gabO7L679j/NYd/+3LpnbvZK2dfrk5tqBMtOHnywt6XpJt6qDj10wrwXqwOPWiWf+eZLWufvb7j4o6deSruvAPLqeu78iafMBN18n6dHa+49q3z+etquYW1dw943uvrL2/lZJ+0+W7uhjF8yrIzoR/hmS3hv2cb+668hvl/QHM3vDzBZ1ejIjOHH/yUi1t1M7PJ8DpSc3t9MBJ0t3zWPXyInXrdaJ8I90mkg3tRwucvd/knSNpO/WXt6iPnWd3NwuI5ws3RUaPfG61ToR/n5JM4d9fLKkDR2Yx4jcfUPt7SZJS9V9pw8P7D8ktfZ2U4fn8/+66eTmkU6WVhc8dt104nUnwr9C0mwzO9XMjpD0TUlPd2AeX2Jm42t/iJGZjZd0lbrv9OGnJS2svb9Q0lMdnMsXdMvJzVUnS6vDj123nXjdkYt8aq2M/5Q0RtIj7v7vbZ/ECMzsNO17tpf2rXj8VSfnZmZPSLpM+1Z9DUj6gaTfS/qtpL+T9FdJ33D3tv/hrWJul+kgT24epblVnSz9mjr42LXyxOuWzIcr/IAycYUfUCjCDxSK8AOFIvxAoQg/UCjCDxSK8AOFIvxAof4PYwQAhKEd7F8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(trainX[0], cmap='gray')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Noramlizing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainX = trainX/255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainY = to_categorical(trainY)\n",
    "testY = to_categorical(testY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Dense, Dropout, BatchNormalization\n",
    "from keras.models import Sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model(model_config):\n",
    "    models = []\n",
    "    for name,config in model_config.items():\n",
    "        model = Sequential()\n",
    "        model.add(Dense(config[0][1], input_dim=784, activation='relu'))\n",
    "        for num,nodes in config[1:]:\n",
    "            model.add(Dense(nodes, activation='relu'))\n",
    "            #model.add(Dropout(.25))\n",
    "            model.add(BatchNormalization())\n",
    "        model.add(Dense(10, activation='softmax'))\n",
    "        models.append(model)\n",
    "    return models\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_config = {\n",
    "    'A':[(1,700),(2,128)],\n",
    "    'B':[(1,256),(2,128),(3,64),(4,32)]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = create_model(model_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_17 (Dense)             (None, 700)               549500    \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (None, 128)               89728     \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 641,030\n",
      "Trainable params: 640,774\n",
      "Non-trainable params: 256\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "models[0].summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_20 (Dense)             (None, 256)               200960    \n",
      "_________________________________________________________________\n",
      "dense_21 (Dense)             (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "dense_22 (Dense)             (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 64)                256       \n",
      "_________________________________________________________________\n",
      "dense_23 (Dense)             (None, 32)                2080      \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 32)                128       \n",
      "_________________________________________________________________\n",
      "dense_24 (Dense)             (None, 10)                330       \n",
      "=================================================================\n",
      "Total params: 245,418\n",
      "Trainable params: 244,970\n",
      "Non-trainable params: 448\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "models[1].summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainX = trainX.reshape(60000,784)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "testX = testX.reshape(10000,784)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/10\n",
      " - 5s - loss: 0.4969 - acc: 0.8289 - val_loss: 0.5917 - val_acc: 0.7954\n",
      "Epoch 2/10\n",
      " - 4s - loss: 0.3792 - acc: 0.8669 - val_loss: 0.4488 - val_acc: 0.8470\n",
      "Epoch 3/10\n",
      " - 4s - loss: 0.3396 - acc: 0.8770 - val_loss: 0.4691 - val_acc: 0.8393\n",
      "Epoch 4/10\n",
      " - 4s - loss: 0.3507 - acc: 0.8776 - val_loss: 0.4760 - val_acc: 0.8445\n",
      "Epoch 5/10\n",
      " - 4s - loss: 0.3143 - acc: 0.8860 - val_loss: 0.3863 - val_acc: 0.8657\n",
      "Epoch 6/10\n",
      " - 5s - loss: 0.3000 - acc: 0.8914 - val_loss: 0.3928 - val_acc: 0.8619\n",
      "Epoch 7/10\n",
      " - 4s - loss: 0.2809 - acc: 0.8975 - val_loss: 0.3419 - val_acc: 0.8813\n",
      "Epoch 8/10\n",
      " - 4s - loss: 0.2695 - acc: 0.9012 - val_loss: 0.3489 - val_acc: 0.8803\n",
      "Epoch 9/10\n",
      " - 4s - loss: 0.2651 - acc: 0.9037 - val_loss: 0.3684 - val_acc: 0.8673\n",
      "Epoch 10/10\n",
      " - 4s - loss: 0.2634 - acc: 0.9044 - val_loss: 0.3424 - val_acc: 0.8790\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/10\n",
      " - 3s - loss: 0.5741 - acc: 0.8094 - val_loss: 0.5005 - val_acc: 0.8226\n",
      "Epoch 2/10\n",
      " - 2s - loss: 0.3814 - acc: 0.8634 - val_loss: 0.5373 - val_acc: 0.8010\n",
      "Epoch 3/10\n",
      " - 2s - loss: 0.3349 - acc: 0.8785 - val_loss: 0.5216 - val_acc: 0.8248\n",
      "Epoch 4/10\n",
      " - 2s - loss: 0.3133 - acc: 0.8854 - val_loss: 0.3837 - val_acc: 0.8575\n",
      "Epoch 5/10\n",
      " - 2s - loss: 0.3068 - acc: 0.8873 - val_loss: 0.3990 - val_acc: 0.8552\n",
      "Epoch 6/10\n",
      " - 2s - loss: 0.2914 - acc: 0.8938 - val_loss: 0.3774 - val_acc: 0.8609\n",
      "Epoch 7/10\n",
      " - 2s - loss: 0.2735 - acc: 0.8987 - val_loss: 0.3651 - val_acc: 0.8663\n",
      "Epoch 8/10\n",
      " - 2s - loss: 0.2594 - acc: 0.9045 - val_loss: 0.3358 - val_acc: 0.8792\n",
      "Epoch 9/10\n",
      " - 2s - loss: 0.2480 - acc: 0.9087 - val_loss: 0.3375 - val_acc: 0.8794\n",
      "Epoch 10/10\n",
      " - 2s - loss: 0.2418 - acc: 0.9094 - val_loss: 0.3590 - val_acc: 0.8694\n"
     ]
    }
   ],
   "source": [
    "all_history = []\n",
    "for model in models:\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    history = model.fit(trainX,trainY,validation_split=.2, epochs=10, batch_size=256, verbose=2)\n",
    "    all_history.append(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([9, 2, 1, ..., 8, 1, 5], dtype=int64)"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.argmax(models[1].predict(testX.reshape(10000,784)),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 0s 25us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.4089719118833542, 0.8659]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(testX.reshape(-1,784),testY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_acc = all_history[1].history['val_acc']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_acc = all_history[1].history['acc']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "h = all_history[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['val_loss', 'val_acc', 'loss', 'acc'])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h.history.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x262050ace48>"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd8leX9//HXJ3sSQhaQEBJmCKAgAUFQERQZzjqKirulzq9SraNf2lprW6v92tpfHVWL1i0CjgoioCBVkBKQETIgzISVQUhCBhnn+v1xn0CIQA5Z98k5n+fjkcdZ9zn5nCO+z5XrvoYYY1BKKeUdfOwuQCmlVMfR0FdKKS+ioa+UUl5EQ18ppbyIhr5SSnkRDX2llPIiGvpKKeVFNPSVUsqLaOgrpZQX8bO7gKaio6NNUlKS3WUopVSnsm7duiJjTExzx7ld6CclJZGenm53GUop1amIyG5XjtPuHaWU8iIa+kop5UU09JVSyou4XZ/+ydTW1pKfn091dbXdpbS7oKAgEhIS8Pf3t7sUpZQH6hShn5+fT3h4OElJSYiI3eW0G2MMxcXF5Ofnk5ycbHc5SikP1Cm6d6qrq4mKivLowAcQEaKiorziLxqllD06RegDHh/4DbzlfSql7NEpuneUUqrTcDigrgpqq09+WXcUaqugrvqHl9EDYMiP2rU8Df12EhYWxpEjR+wuQyl1OnU1cGATlOz6YQDXVZ86uE8a5M7r9TUtr2fINRr6SinVZqpKIG8t7FkNeWtg7zor3E9GfME/GPyCGl0GgV+wdRkUAX6BJznmZJeBx593yssg8PFt949AQ99Fjz76KL179+aee+4B4IknnkBEWLlyJSUlJdTW1vLUU09x5ZVX2lypUgoAY6wWfN4a2POd9VOYZT3m4wfdz4K0O6DXuRA7yBnOjQLY1zOHTXe60P/tv7eQua+sTV8ztWcXfnP54NMeM336dB588MFjoT937lwWL17MrFmz6NKlC0VFRYwePZorrrhCT8YqZYf6WqurZs+a4y35IwetxwIjoNdIq/skcTTEnwMBofbWa5NOF/p2GT58OAUFBezbt4/CwkIiIyPp0aMHs2bNYuXKlfj4+LB3714OHjxI9+7d7S5XKc9XXWp11eQ5W/F710FtpfVY10RIvhASz4XEMRCT0iFdJ51Bpwv95lrk7enaa69l3rx5HDhwgOnTp/POO+9QWFjIunXr8Pf3JykpScfYK9UejIHDe07sqinIBIzV9959KJxzi9VVkzgauvS0u2K31elC307Tp0/npz/9KUVFRXz99dfMnTuX2NhY/P39Wb58Obt3u7SyqVKqOfV1cHCz1VXT0JIv3289FhBuddWkXmm15OPTIDDM3no7EQ39MzB48GDKy8uJj4+nR48e3HTTTVx++eWkpaUxbNgwUlJS7C5Rqc6pugzy1x5vyeenQ22F9ViXBOg91mrB9zoX4gZrV00raOifoc2bNx+7Hh0dzerVq096nI7RV+o0aqtg+3LY/pXVkj+4BYwDxAfihsDwm4531UQk2F2tR9HQV0p1jIpi2LoYchZB7pfWRCb/UKur5oJHrK6ahJEQGG53pR5NQ18p1X6Kt1shn73IatEbB3SJh+EzIGUq9B4HfgF2V+lVNPSVUm3H4YB96yF7oRX2hdnW/XFD4IJfwMCp0ONs0LksttHQV0q1Tt1R2LnSGfSfw5ED1jDK3ufBiNtg4BSITLK7SuWkoa+UOnNVJbB1CeQstPrna45AQBj0mwgDp0H/SyCkm91VqpNwKfRFZDLwPOALvGaMebrJ472BOUAMcAiYYYzJdz52KzDbeehTxph/tVHtSqmOVLLb2T+/EHavAlMPYXEw9FpIuQySzrfWrVFurdnQFxFf4AXgEiAfWCsinxpjMhsd9mfgTWPMv0RkAvBH4GYR6Qb8BkgDDLDO+dyStn4j7e3w4cO8++67x9becdXUqVN599136dq1aztVplQ7MQb2bzx+Ivagc7hyTAqMfQBSpkHPc8Cn0+zFpHCtpT8KyDXG7AAQkfeBK4HGoZ8KzHJeXw587Lx+KbDUGHPI+dylwGTgvdaX3rEOHz7Miy+++IPQr6+vx9f31BNFFi1a1N6lKdV26mpg9zdWyOd8DmX51tj5XqNh0lPWidiovnZXqVrBldCPB/Ia3c4Hzm1yzEbgGqwuoKuBcBGJOsVz45v+AhGZCcwESExMdLX2DvXYY4+xfft2hg0bhr+/P2FhYfTo0YMNGzaQmZnJVVddRV5eHtXV1TzwwAPMnDkTgKSkJNLT0zly5AhTpkxh3LhxrFq1ivj4eD755BOCg4NtfmfK61WXQe5Sq9tm2zI4WmotMdx3Alz0SxhwKYRG212laiOuhP7JxlaZJrcfBv4uIrcBK4G9QJ2Lz8UY8wrwCkBaWtoPHj/B54/Bgc2nPeSMdR8KU54+7SFPP/00GRkZbNiwgRUrVjBt2jQyMjJITk4GYM6cOXTr1o2qqipGjhzJNddcQ1RU1AmvsW3bNt577z1effVVrr/+eubPn8+MGTPa9r0o5YrSvVa3Tc4i2PkfcNRCSDSkXm6diO0zHgJC7K5StQNXQj8f6NXodgKwr/EBxph9wI8ARCQMuMYYUyoi+cD4Js9d0Yp63caoUaOOBT7A3/72Nz766CMA8vLy2LZt2w9CPzk5mWHDhgEwYsQIdu3a1WH1Ki93tBx2fQs7Vlg/DZuJRPWD0Xdb/fMJI3VNGy/gSuivBfqLSDJWC346cGPjA0QkGjhkjHEAj2ON5AH4AviDiEQ6b09yPt5yzbTIO0po6PENGFasWMGyZctYvXo1ISEhjB8//qRLLAcGBh677uvrS1VVVYfUqrxQfa21vnxDyOevBUedtSNU4hgYdgMMmAIxA+yuVHWwZkPfGFMnIvdhBbgvMMcYs0VEngTSjTGfYrXm/ygiBqt7517ncw+JyO+wvjgAnmw4qdvZhIeHU15eftLHSktLiYyMJCQkhOzsbL777rsOrk55PWOs2a8NIb/rG2vsPAI9h8N5/2N12fQ6V4dVejmXxukbYxYBi5rc9+tG1+cB807x3Dkcb/l3WlFRUYwdO5YhQ4YQHBxMXFzcsccmT57Myy+/zFlnncXAgQMZPXq0jZUqr1G2D3Z8fTzojxyw7u/WB866HvpcBEnjdJKUOoEYc/rzph0tLS3NpKenn3BfVlYWgwYNsqmijudt71e5qLrMasE3hHxRjnV/SJTViu8z3toiMLK3XRUqG4nIOmNMWnPH6TIMSrmruhrYm96oXz7dmgXrF2ytazN8hhX0cUN0gpRymYa+Uu7CGGvf12P98t9au0eJjzXzddwsZ7/8KPALPP1rKXUKnSb0jTGIFyzH6m7dbaqdle49HvI7VkBFgXV/VD9rhE2f8Va/fHDkKV9CqTPRKUI/KCiI4uJioqKiPDr4jTEUFxcTFKSjKzxWdak1Gaoh5Iu3WfeHRFsB3/ciq1++a69Tv4ZSrdApQj8hIYH8/HwKCwvtLqXdBQUFkZCge4J6lKNHrHVsNn8I27+0xsv7h1ibfY+4zQr72FTtl1cdolOEvr+//wmzX5Vye3U1VsBvnmctdVBbaW0TOPpuGDAZEkbpNoHKFp0i9JXqFBwO2LPKatFv+RiqD1t98WdPhyHXWjNhtTWvbKahr1RrNKw5v/lDyFgA5fusrpuUaTD0OmuClLbolRvR0FeqJYpyIWOeFfbFueDjD/0uhkm/s/aEDQht/jWUsoGGvlKuKttnteYz5sG+7wGxhlOedz8MukKXO1Cdgoa+UqdTVQKZn1ot+l3fAAZ6DLN2kRr8I4j4wZ5ASrk1DX2lmqqphK2fWyNvti21Nhjp1hcufNTaBDy6v90VKtViGvpKgbX+/PblVos+e6G1/EF4Dzj3Z1bQ9xgGHjwxUHkPDX3lvRwOyPvOatFv+QiqDkFQVyvkh15nLWqmO0kpD6Ohr7yLMXAww2rRb54PZfnWqpUpU62g7ztRh1gqj6ahrzybMdbJ2NJ82PqFNfKmMBt8/KyAv/g3MHAqBIbZXalSHUJDX3VODWFefgDK98ORg9ZlecPlAWsnqfKDUH/0+PMSz4Npz0HqVRAaderXV8pDaegr99LSMG8QGAHhcRDe3Vr2ICzOOiEbHmetd6OrVyovp6GvOsYpw/zA8Z+Whnl4D+v+sO4QENLx702pTkRDX7WPuhrYtgQ2vgcHNlmhXl/zw+MCI6zADo9rEubdj/9omCvVZjT0VdsxBvZvgA3vWaNjqg5BaCz0udAZ5BrmStlNQ1+1XvkB2DQXNrwLhVngG2CNiBl2ozVCxlf/mSnlLvT/RtUytdWQs9Bq1W//EowDEkZaI2OG/Ej3dFXKTWnoK9cZA/lrrRZ9xgI4WmrtBjX2QatVr2vSKOX2NPRV8w7nwab3YeP71trxfsGQegWcfQMkX6BLFSjViWjoq5OrqYCsf1ut+p0rAWNt5D1uFqReCYHhdleolGoBl0JfRCYDzwO+wGvGmKebPJ4I/Avo6jzmMWPMIhHxB14DznH+rjeNMX9sw/pVW2rY43XDu5D5CdQcgcgkGP8YnPVj6Kab0yvV2TUb+iLiC7wAXALkA2tF5FNjTGajw2YDc40xL4lIKrAISAKuAwKNMUNFJATIFJH3jDG72vh9qNY4tMPqutn4HhzeAwHhMPgqGHaTNXZelxRWymO40tIfBeQaY3YAiMj7wJVA49A3QBfn9QhgX6P7Q0XEDwgGaoCyNqhbtVZ1GWR+bI2+2bMKEOgzHib8ClIu0/HzSnkoV0I/HshrdDsfOLfJMU8AS0TkfiAUuNh5/zysL4j9QAgwyxhzqDUFq1Zw1MOOFVaLPuszqKuCqP4w8TdW941u/aeUx3Ml9E/2t71pcvsG4A1jzP+JyBjgLREZgvVXQj3QE4gE/iMiyxr+ajj2C0RmAjMBEhMTz/AtqGYVboWN78LGD6B8n7VRyLAbrZ/4Edp9o2xRWH6UFTkF1NYb6h0O6h2GeoPzepNLY6hzGByO45f1xljPcTR5zBjq6p2XjuPH1DuO39dwbMP9SdGhPHhxfwb3jLD7Y2l3roR+PtB4acIEjnffNLgTmAxgjFktIkFANHAjsNgYUwsUiMi3QBpwQugbY14BXgFIS0tr+oWiWuJoOWz6wDopu3cdiC/0uxgm/xEGTgG/QLsrVF7s8837+eVHmymprHXpeD8fwcdH8PMRfOX49WOXIvj5Wo/5+lg/Dff5yPFj/X19CPJ3HuN8HV8RVu8oZtrfvuGqYT15aNJAenXz3O5NV0J/LdBfRJKBvcB0rDBvbA8wEXhDRAYBQUCh8/4JIvI2VvfOaOCvbVS7OpWibfDedGtMfdwQmPR7a1eo8Di7K1Nerqy6lic+3cKC9XsZGh/BG7cPoXtE0AnB7Ns4yJ33tbfSqlpe/no7c77ZyaLNB5gxujf3TehHt1DP20VNjGm+YS0iU7HC2heYY4z5vYg8CaQbYz51jth5FQjD6vp5xBizRETCgNeBVKxuoteNMc+e7nelpaWZ9PT0Vr0pr7ZtKcy7E3z94do51mJnSrmB1duLefjDjRwoq+bei/px/4R++Pv62F3WCfaXVvH8sm3MTc8jJMCPuy7swx3jkgkJcP8pTSKyzhiT1uxxroR+R9LQbyFjYNXfYOlvoPtQmP6ubhii3EJ1bT1//iKH177ZSXJ0KM9dfzbDE917babcgnL+tDiHpZkHiQkP5MGL+3N9Wi+3+5JqTEPfm9RWwaf/A5vnwuCr4coXdcilcgsZe0v5+dwNbD14hJtH9+bxqSmdotXcIH3XIZ7+PJv03SX0iQ7lF5cOZPKQ7ogbDn7Q0PcWpXvhg5tg3waYMBvOf0hH4yjb1dU7+MfKHfx12VYiQwJ49rqzuXBAjN1ltYgxhmVZBTyzOJttBUc4u1dXHp+Swug+7rXHsoa+N9izBj6YYbX0r3nVGpWjlM12FVXw87kbWL/nMJed1YOnrhpC15DOf0K0rt7BgvV7eW7pVg6UVXPRwBgemZzCoB5dmn9yB9DQ93Tr34KFP7eWNr7hfYhNsbsi5eWMMbz73z38fmEWfj7C764awpXDPG/CX3VtPW+s2sWLy3MpP1rH1cPj+fklA0iItLdLVUPfU9XXwZL/hTUvQ5+LrBE6Id3srkp5uYKyah6dv4nlOYWM6xfNs9edRY+IYLvLaleHK2t4acV2Xl+1CwzcMqY3917Uj0ibhnlq6HuiykPw4a3WUsej74VLntStCJXtGiZaVdbU8/iUFG4Zk9QhY+vdxb7DVfxl6Vbmr88nNMCPu8b35Y6xyQQHdOw+Exr6nuZgJrx/A5Ttg8uft5ZQUMpGZdW1PPHJFhZ8b020+suPh9EvNszusmyTc6CcZ7/IZllWAXFdApl18QCuHZGAXwcN89TQ9yTZC2HBTAgIhR+/A71G2l2R8nKrcot4+MONHCw/6rYTrezy352HePrzLNbvOUzfmFAemZzCpNS4dh/mqaHvCYyBlc/C8t9Dz3Ng+jvQpafdVSkvVl1bzzOLc5jz7U76RIfy3I+HMaxXV7vLcjvGGJZkHuSZxdlsL6zgnMSuPD51ECOT2u/8m4Z+Z1dTAR/fbe1gddZ0q0vHP8juqpQXy9hbyqwPNrCt4Ai3jOnN41MGdXi/dWdTV+9g3rp8/rJsKwfLjnLxoFgemZzCgLi2325UQ78zK9kN798IBZnWydox9+mEK2WbunoHL3+9nb8u20ZUWADPXns2F3TSiVZ2qaqp5/VVO3lpxXYqjtZxzTkJzLpkAD27tt0IJw39zmrXNzD3Fmto5nVzrOWQlbLJrqIKZs3dwPd7DnP52T353ZWDPWKilV1KKmp4YXkub67eDQK3n5fEPeP7ERHi3+rX1tDvjNa+Bp8/Ct36wPT3ILqf3RUpL2WM4Z011kQrf1/PnWhll/ySSp5bupWPvt9LeKAf917Uj1vPSyLIv+XdZRr6nUldDXz+CKx7Hfpfai2pEOT5O/go91RQVs0j8zexIqeQ8/tH88y1nj/Ryi5Z+8t4ZnE2y3MK6RERxMOTBnLNiIQWvZaroa8ze+x2pNDqztmzCsb93Fo0zUdPjil7LNy0n//9eDNVNfX89orB3Dy6t1dNtOpog3p04fXbR7F6ezFPL85m897SFoe+qzT07bR/I7x/E1QUwjX/hKHX2l2R8lKlVbX85pMMPt6wj7MSInjueu+eaNXRxvSN4uN7zuNonaPdf5eGvl0y5sPH91rr5tyxGHoOt7si5aW+dU60Kig/ygMT+3OfTrSyhYi0qk/fVRr6Hc3hsCZb/efP0Gs0/PgtCIu1uyrlhapr6/nT4mxe/3YXfaJDmX/3eTrRygto6Hek6jJrOYWtn8M5t8DUP4NfoN1VKS+0Ob+UWXM3kFtwhFvH9OYxnWjlNTT0O0rxdmvCVdE2K+xH/kQnXKkOY4xhR1EFy7MLWJ5TwHc7DhEdFsCbd4zSiVZeRkO/I2z/Cj68HcQHbvkYki+wuyLlBapr6/nvzkN85Qz63cWVAPSPDWPmBX342QV9dKKVF9LQb0/GwHcvwpLZEDMIbngXIpPsrkp5sP2lVSzPLuSr7AK+zS2iqraeQD8fzusbxU/GJTN+YCy9utm7w5Oyl4Z+e6mths9mwcZ3IeUyuPofEKhD4FTbqncYvt9T4mzNF5K1vwyA+K7BXDMingkpsYzpE6399eoYDf32UH7AGn+/Nx3GPw4XPAI+OgROtY2SihpWbrNa819vLeRwZS2+PsKI3pE8NiWFCSmx9I8Na/f121XnpKHf1oq3wxvTrJE6178JqVfaXZHq5IwxZO0vZ3lOAV9lF/D9nhIcBqJCA5iQEsuElFjO7xfTJot2Kc+nod/W1rwMVSXwky+h+xC7q1GdVMXROr7NLWJ5TgHLsws5UFYNwND4CO6b0J+LBsZwdkJXXSJBnTEN/bZUXwdbPoIBl2rgqzO2q6jiWGt+zY5D1NQ7CAv0Y1y/aCakxDJ+YAyxXXQjHdU6Gvptafc31jo6Q66xuxLVCdTUOVi7yzmkMruAHUUVAPSJCeWWMb2ZkBJLWlI3Avz0fJBqOy6FvohMBp4HfIHXjDFPN3k8EfgX0NV5zGPGmEXOx84C/gF0ARzASGNMdZu9A3eSMR8CwqD/JLsrUW6qoKz6WGv+m21FVNTUE+Drw+i+UdwypjcXpcTSOyrU7jKVB2s29EXEF3gBuATIB9aKyKfGmMxGh80G5hpjXhKRVGARkCQifsDbwM3GmI0iEgXUtvm7cAd1NZD5KaRMA39de1xZjDHkFhxhSeZBlmQeZGPeYQB6RARxxTBrSOXYflGEBOgf3apjuPIvbRSQa4zZASAi7wNXAo1D32C15AEigH3O65OATcaYjQDGmOK2KNot7VgB1Ye1a0dR7zCs31PCki0HWJp5kF3OmbBnJ0Tw8KQBTBwUR0r3cB1SqWzhSujHA3mNbucD5zY55glgiYjcD4QCDRu7DgCMiHwBxADvG2OeafoLRGQmMBMgMTHxTOp3HxnzIagr9LnI7kqUDapq6vkmt4glWw7wVXYBxRU1+PsKY/pGc+f5fbhkUBzdI/QkrLKfK6F/suZI0z0WbwDeMMb8n4iMAd4SkSHO1x8HjAQqgS+dW3p9ecKLGfMK8ApY2yWe4XuwX20VZC+EwVeBn65l4i0OVdTwZdZBlmYeZOW2QqprHYQH+XHRwFguSY1j/MAYwoN07LxyL66Efj7Qq9HtBI533zS4E5gMYIxZLSJBQLTzuV8bY4oARGQRcA7wJZ5k21KoKdeuHS+wu7iCpc7++fRdh3AYq3/++rReXJIax7nJUTraRrk1V0J/LdBfRJKBvcB04MYmx+wBJgJviMggIAgoBL4AHhGREKAGuBD4SxvV7j4y5kNoDCSdb3clqo0ZY9iUX8rSTKtFn3OwHICU7uHcd1E/LkntzpD4Lto/rzqNZkPfGFMnIvdhBbgvMMcYs0VEngTSjTGfAg8Br4rILKyun9uMMQYoEZHnsL44DLDIGLOwvd6MLY6Ww9YvYPgM8NURGE099Vkm324vJiEy2PkTQkJkML0iQ0joFkwXN+z+qKlzsHpHMUszD7Ass4ADZdX4CIxK7savLktlUmqcrlSpOi2XUso55n5Rk/t+3eh6JjD2FM99G2vYpmfKWQx1Vdq1cxLrdh/itW92ktqjC3uKK/k2t4jKmvoTjukS5EdCZAi9uh3/Qjh+GdxhfeJl1bUszy5gaeZBvs4ppPxoHcH+vlwwIJpfpA5kQkoskaF6vkZ1fto0ba2M+dAlHno1HdDk3RwOw5OfZREbHsiHd40hNNAPYwwllbXkl1SSX1J17DLvUCU7CitYudVa/72xriH+1hdAV+dfCN1O/GIIDWz5P+H9pVUsc/bPf7ejmNp6Q3RYAFOH9mDS4DjG9ovukI2qlepIGvqtUVUCucvg3J/p0slNfLpxHxvzDvPn684+FswiQrfQALqFBnBWwg834DbGcKiihrxGXwj5JZXkHapiW4G1yuTROscJz4kM8f/BF0FD91F8ZPAJk56MMeQcLGfpFivoN+8tBaBPdCh3jE1m0uA4hvWKxFcXMVMeTEO/NbI+A0etdu00UVVTz58WZzM0PoIfDY93+XkiQlRYIFFhgQzrdfIvhaIjNdYXwQlfDFVk7y9nWVYBNU2+FKJCA0iIDKZHRDCZ+8vYc8iaKDU8sSuPTB7IpNTu9IvVzW2U99DQb42M+RCZDD2H212JW3ll5Q72l1bz/PThbbr0r4gQEx5ITHggwxMjf/C4w2EoOnL0B38p5JdUsfVgOX1iQrnrwr5cPChWV6tUXktDv6WOFMLOr2Hcz0GH6x1zoLSal7/eztSh3RmV3K1Df7ePjxDbJYjYLkGM6P3DLwWlFGhHdEtlfQLGoV07TTz7RQ71DsNjkwfZXYpS6iQ09FsqYwHEDIK4VLsrcRub8g8zf30+t49LIjFKx7Er5Y409FuidC/sXgVDfmR3JW7DGMPvPsskOiyA+y7qZ3c5SqlT0NBvicyPAQODNfQbfJ5xgLW7Svj5JQN1kTGl3JiGfktkzIceZ0O0tmgBqmvr+cOiLFK6h/Pjkb2af4JSyjYa+mfq0E7Yu05P4Dby+re7yC+pYva0VJ3YpJSb09A/U1sWWJeDr7a3DjdRWH6UF5bncvGgWMb1j7a7HKVUMzT0z1TGAmudna6ddIevNvbc0hyqa+v55VQdoqlUZ6ChfyYKsuFghnbtOGXtL+ODtXncPKY3fWJ0KQOlOgMN/TOxZQGID6ReZXcltjPG8NTCTLoE+/PAxP52l6OUcpGGvquMsUbtJI2D8Di7q7HdsqwCvs0t5sGJ/ekaouvMK9VZaOi76sBmKM7Vrh2snaX+sCiLvjGh3DS6t93lKKXOgIa+qzLmg48fDLrC7kps99Z3u9lZVMHsaan4++o/IaU6E/0/1hXGWKN2+k6AkI5dOdLdlFTU8PyyrZzfP5rxA2PsLkcpdYY09F2Rnw6le3TZBeCvy7Zy5Ggds6elIrqktFKdjoa+KzLmg28gpEy1uxJb5RaU8/aaPdx4biIDu4fbXY5SqgU09JvjqIctH0H/SyAowu5qbPXUwixCAnyZdfEAu0tRSrWQhn5zdq+CIwe8ftTO11sLWZFTyP0T+hEVFmh3OUqpFtLQb07GfPAPhQGX2l2JberqHTz1WSa9o0K49bwku8tRSrWChv7p1NdC5icwcAoEhNpdjW3eW5vHtoIjPD5lEIF+vnaXo5RqBQ3909nxNVQd8uqundKqWp5bksO5yd24dLDORFaqs9PQP52M+RAYAf0m2l2Jbf7+1TYOV9Xyq8t0iKZSnsCl0BeRySKSIyK5IvLYSR5PFJHlIvK9iGwSkaknefyIiDzcVoW3u9pqyP4MBl0Oft554nJXUQVvrNrFdSMSGBLv3SOXlPIUzYa+iPgCLwBTgFTgBhFJbXLYbGCuMWY4MB14scnjfwE+b325HWj7l3C0zKs3P//Doiz8fX14eNJAu0tRSrURV1r6o4BcY8wOY0wN8D5wZZNjDNDFeT0C2NfwgIhcBewAtrS+3A6UMR9CoiD5QrsrscWq7UUsyTzIvRf1I7ZLkN3lKKXaiCuhHw/kNbqd77yLlDkBAAARsUlEQVSvsSeAGSKSDywC7gcQkVDgUeC3ra60I9VUQM7n1rr5vn52V9Ph6h2Gpz7LIr5rMHeOS7a7HKVUG3Il9E929s40uX0D8IYxJgGYCrwlIj5YYf8XY8yR0/4CkZkiki4i6YWFha7U3b62LobaSq/t2pm3Lo/M/WU8OiWFIH8doqmUJ3GlGZsP9Gp0O4FG3TdOdwKTAYwxq0UkCIgGzgWuFZFngK6AQ0SqjTF/b/xkY8wrwCsAaWlpTb9QOl7GAgjvAYlj7K6kwx05WsezX2zlnMSuXH5WD7vLUUq1MVdCfy3QX0SSgb1YJ2pvbHLMHmAi8IaIDAKCgEJjzPkNB4jIE8CRpoHvdqpLYdsSGPkT8PG+Vu5LK3IpOnKU125N0yGaSnmgZrt3jDF1wH3AF0AW1iidLSLypIg07CjyEPBTEdkIvAfcZoyxv8XeEtkLob7GKydk5R2q5NX/7OSqYT0Z1qur3eUopdqBS2cpjTGLsE7QNr7v142uZwJjm3mNJ1pQX8fLmA9dEyF+hN2VdLg/Lc7GR+CRySl2l6KUaic6I7eximLYvtxq5XtZ18a63Yf4bNN+Zl7Ql55dg+0uRynVTjT0G8v6BEy913XtOByGJ/+dSVyXQO66sI/d5Sil2pGGfmMZCyB6AMQNsbuSDvXJxr1szC/lkUtTCAnwvnkJSnkTDf0GZfth1zde17VTWVPHnz7P4ayECK4e3nTOnVLK02joN8j8BDBet/n5Kyt3cKCsmtnTUvHx8Z4vO6W8lYZ+g4z50H0oxHjP/q8HSqv5x9c7mDa0B6OSu9ldjlKqA2joA5Tshvz/et0J3Ge+yKbeYXhsig7RVMpbaOgDbPnIuhx8tb11dKCNeYdZsH4vd4xLple3ELvLUUp1EA19sLp24tMgMsnuSjqEMYbffZZJdFgA917U1+5ylFIdSEO/aBsc2ORVXTuLNh8gfXcJD00aSHiQv93lKKU6kIZ+xgJAYPBVdlfSIapr6/nj51mkdA/n+rRezT9BKeVRvDv0jYGMedB7LHTpaXc1HWLOtzvJL6niV5el4qtDNJXyOt4d+ge3QNFWr9kspaC8mheXb+fiQXGM7RdtdzlKKRt4d+hnzAfxhdSmW/56pueWbKW6tp5fTtUhmkp5K+8NfWOs0O8zHkI9v9Wbua+MD9LzuGVMEn1iwuwuRyllE+8N/X3r4fBurxi1Y4zhqYWZRAT788DE/naXo5SykfeGfsYC8A2AlGl2V9LulmUVsGp7MbMuHkBEiA7RVMqbeWfoOxxW6Pe7BII9e1vAmjoHv1+YSd+YUG48N9HucpRSNvPO0M/7Dsr3ecWonTdX72JXcSWzL0vF39c7/3MrpY7zzhTImA9+wTBgst2VtKtDFTX87cttXDAghosGxtpdjlLKDXhf6NfXwZaPYeBkCPTsUSx/XbaVipp6Zk8bZHcpSik34X2hv2slVBZ5/KidbQfLeWfNHm4clciAuHC7y1FKuQnvC/2M+RAQbp3E9VBH6+p5eN4mQgJ8mXWJ92wKo5Rqnnftgl13FLL+DYMuA/8gu6tpN0/+O5ONeYd5ecYIuoUG2F2OUsqNeFdLf/tXUF3q0V0789fl886aPfzswj5MHtLd7nKUUm7Go0J/1fYijDGnPiBjPgRHWksveKDMfWX88qPNjOkTxS8mDbS7HKWUG/KY0P/PtkJufHUNLyzPPfkBNZWQvchaXM3X82alllbWctfb6+ga4s/fbhiOn47JV0qdhMckw7h+0Vw9PJ4/L9nKJxv2/vCAbUugtsIju3YcDsPP525gf2kVL940gpjwQLtLUkq5KZdCX0Qmi0iOiOSKyGMneTxRRJaLyPcisklEpjrvv0RE1onIZuflhLZ+A41q4OlrhjIquRu/+HATa3cdOvGAjPkQFmdtmOJhXlyRy5fZBcyelsqI3pF2l6OUcmPNhr6I+AIvAFOAVOAGEUltcthsYK4xZjgwHXjReX8RcLkxZihwK/BWWxV+MoF+vrxy8wgSIoOZ+WY6u4oqrAeqy6yW/uCrwce3PUvocP/ZVsj/Ld3KlcN6csuY3naXo5Ryc6609EcBucaYHcaYGuB9oOmuIwbo4rweAewDMMZ8b4zZ57x/CxAkIu3a99A1JIA5t40E4PY31lJSUQM5n0NdNQz2rLV29h6u4n/e+54BseH88UdDEdHtD5VSp+dK6McDeY1u5zvva+wJYIaI5AOLgPtP8jrXAN8bY442fUBEZopIuoikFxYWulT46SRFh/LKLWnsLaniZ2+vo37zPIjoBQkjW/3a7uJoXT33vL2OunrDSzPOISTAu6ZcKKVaxpXQP1nzsem4yBuAN4wxCcBU4C0ROfbaIjIY+BPws5P9AmPMK8aYNGNMWkxMjGuVN2NkUjeeve4scnbugdwvMYOvBh+POW/Nb/+dycb8Up697mzdCUsp5TJXUjAf6NXodgLO7ptG7gTmAhhjVgNBQDSAiCQAHwG3GGO2t7bgM3HlsHieG7oHX+p5t9JzWvnz1uXz7po93HVhX52ApZQ6I66E/lqgv4gki0gA1onaT5scsweYCCAig7BCv1BEugILgceNMd+2Xdmum1D3HwoDEvjf73z46Pt8O0poU1v2lfK/zglYD0/SdXWUUmem2dA3xtQB9wFfAFlYo3S2iMiTInKF87CHgJ+KyEbgPeA2Y02NvQ/oB/xKRDY4fzpuYffyg8iu/9Bt1A2M7hPFI/M2sWZHcYf9+rZWWlnL3W+vJzIkgP93o07AUkqdOTntsgU2SEtLM+np6W3zYmtegc9/AfesoTSsL1e/9C3FR2pYcM959O1k/eAOh+Gnb6azclsh788co+PxlVInEJF1xpi05o7z7KZixnyIHQyxKUSE+PPGbaPw8xHueGMthypq7K7ujLywXCdgKaVaz3NDvzTf2gu30T64iVEhvHJLGvtLq5n5ZjrVtfU2Fui6lVsLeW7ZVq7SCVhKqVby3NDf8pF12WTz8xG9I3nu+rNJ313CL+ZtwuFwr+6tpvJLKnngfWsC1h90ApZSqpU8d0ZPxnzoeQ506/ODhy47qyd7DlXyzOIckqJCeMhNlyE+WlfPve+sp67e8PLNI3QCllKq1TyzpV+8HfZ9f9oVNe++sC8/TuvF//sqlw/T8055nJ0aJmD9+fqzSY4OtbscpZQH8Mym45YF1uXgq055iIjw1NVDyD9cyeMLNhPfNZjz+kV3UIHN+zA979gErEsH6wQspVTb8MyWfsYCSBwDEQmnPczf14cXbxpBcnQod729jtyC8g4q8PS27Ctl9scZnNdXJ2AppdqW54X+wUwoyHR5s5SIYH/m3DaSAD8fbn9jLUVHfrAeXIdq2AErMiRAd8BSSrU5z0uULQtAfKxtEV3Uq1sIr906koKyo/zUxqGcDodh1twNHCit5sUZ5xAdpjtgKaXalmeFvjHWqJ3kCyDszFZ7GNarK3/98TA25B3mobkbbRnK+cLyXL7KLuBXl6VyTqJOwFJKtT3PCv39G+DQjhbvgztlaA8en5LCws37eXZJThsXd3pfN5qAdfNonYCllGofnjV6J2M++PhDymUtfomfnt+HXcWVvLRiO727hTB9VGIbFnhyDROwBsbpBCylVPvynNB3OCDjI+g3EUK6tfhlRIQnrxhMfkkVsz/OICEyhHH9228oZ3VtPfe8s576esNLM3QCllKqfXlO907+WijLb3HXTmN+vj68cONw+sWGcffb69h6sP2Gcv7235lsyi/l/3QCllKqA3hO6HcfAtf8EwZOaZOXCw/y55+3jSQowJfbX19LYXnbD+Wcm57He//dw93j+zJJJ2AppTqA54R+QCgMvRYCw9vsJeO7BjPn1pEcqqjhJ2+mU1XTdkM5M/aW8ivnBKyHLtEJWEqpjuE5od9OhiZE8Pz0YWzKP8ysDza0yVDO0spa7n5nHd1CdQKWUqpjadq4YNLg7syelsriLQf40+LsVr2Ww2F48IPvrQlYN+kELKVUx9KhIi66Y2wSu4sr+MfKHSRGhXDTuS0bS//35bkszynkd1cOZrhOwFJKdTANfReJCL++LJU9hyr59SdbSIgM4cIBMWf0Gl9vLeQvy7Zy9fB4ZugELKWUDbR75wz4+frw9xvPoX9sGPe+s57sA2UuPzfvUKMJWFfrBCyllD009M9QWKAfr98+ktBAX+54fS0FZdXNPqfpBKzgAN8OqFQppX5IQ78FekQE889bR3K4qpY7/5VOZU3daY//7b+3sHmvTsBSStlPQ7+FhsRH8P9uGM6WfaU88P4G6k8xlNOagJXHPToBSynlBjT0W2HioDh+fVkqSzMP8odFWT94PGOvtQPW2H5Rbrv5ulLKu+jonVa6bWwyu4or+ec3O+kdFcItY5IAOFxZw11vryMqNIC/TR+Or4+euFVK2U9Dvw386rJU8ksqeeLTLfRyDuWc9cEGDpZVM/dnY4jSCVhKKTfhUveOiEwWkRwRyRWRx07yeKKILBeR70Vkk4hMbfTY487n5YjIpW1ZvLvw9RGenz6cQT26cN+763l0/iaW5xTy68t1ApZSyr00G/oi4gu8AEwBUoEbRCS1yWGzgbnGmOHAdOBF53NTnbcHA5OBF52v53FCA/2Yc9tIugT78+G6fH40PJ4Z57b/BixKKXUmXGnpjwJyjTE7jDE1wPtA013HDdDFeT0C2Oe8fiXwvjHmqDFmJ5DrfD2PFNcliH/dMYqfXdCH3+sELKWUG3KlTz8eyGt0Ox84t8kxTwBLROR+IBS4uNFzv2vy3PgWVdpJDIgL5/Gpg+wuQymlTsqVlv7JmqtNB6XfALxhjEkApgJviYiPi89FRGaKSLqIpBcWFrpQklJKqZZwJfTzgV6NbidwvPumwZ3AXABjzGogCIh28bkYY14xxqQZY9JiYs5sETOllFKucyX01wL9RSRZRAKwTsx+2uSYPcBEABEZhBX6hc7jpotIoIgkA/2B/7ZV8Uoppc5Ms336xpg6EbkP+ALwBeYYY7aIyJNAujHmU+Ah4FURmYXVfXObMcYAW0RkLpAJ1AH3GmPabs9BpZRSZ0SsbHYfaWlpJj093e4ylFKqUxGRdcaYtOaO07V3lFLKi2joK6WUF9HQV0opL+J2ffoiUgjsbsVLRANFbVROZ6efxYn08zhOP4sTecLn0dsY0+yYd7cL/dYSkXRXTmZ4A/0sTqSfx3H6WZzImz4P7d5RSikvoqGvlFJexBND/xW7C3Aj+lmcSD+P4/SzOJHXfB4e16evlFLq1Dyxpa+UUuoUPCb0m9vS0ZuISC/n9pVZIrJFRB6wuya7iYivczvPz+yuxW4i0lVE5olItvPfyBi7a7KTiMxy/n+SISLviUiQ3TW1J48IfRe3dPQmdcBDxphBwGjgXi//PAAeALLsLsJNPA8sNsakAGfjxZ+LiMQD/wOkGWOGYC0qOd3eqtqXR4Q+rm3p6DWMMfuNMeud18ux/qf26B3LTkdEEoBpwGt212I3EekCXAD8E8AYU2OMOWxvVbbzA4JFxA8I4SR7fngSTwn9k23p6LUh15iIJAHDgTX2VmKrvwKPAA67C3EDfbD2unjd2d31moiE2l2UXYwxe4E/Y+0Jsh8oNcYssbeq9uUpoe/StozeRkTCgPnAg8aYMrvrsYOIXAYUGGPW2V2Lm/ADzgFeMsYMByoArz0HJiKRWL0CyUBPIFREZthbVfvylNB3aVtGbyIi/liB/44xZoHd9dhoLHCFiOzC6vabICJv21uSrfKBfGNMw19+87C+BLzVxcBOY0yhMaYWWACcZ3NN7cpTQt+VLR29hogIVp9tljHmObvrsZMx5nFjTIIxJgnr38VXxhiPbsmdjjHmAJAnIgOdd03E2tnOW+0BRotIiPP/m4l4+IntZrdL7AxOtaWjzWXZaSxwM7BZRDY47/ulMWaRjTUp93E/8I6zgbQDuN3memxjjFkjIvOA9Vij3r7Hw2fn6oxcpZTyIp7SvaOUUsoFGvpKKeVFNPSVUsqLaOgrpZQX0dBXSikvoqGvlFJeRENfKaW8iIa+Ukp5kf8PIXCXllq4HLUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(val_acc, label='val')\n",
    "plt.plot(train_acc, label='train')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}