11_simple_nn_keras.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple Neural Net with Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Activation\n",
    "from keras.optimizers import SGD\n",
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000 train_samples\n",
      "10000 test_samples\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(1671) # for reproducibility\n",
    "\n",
    "# network and training\n",
    "NB_EPOCH = 250\n",
    "BATCH_SIZE = 128\n",
    "VERBOSE = 1\n",
    "NB_CLASSES = 10  # number of outputs\n",
    "OPTIMIZER = SGD()\n",
    "N_HIDDEN = 128\n",
    "VALIDATION_SPLIT = 0.2  # 20% of train reserved for validation\n",
    "\n",
    "# data: shuffled and split between train and test sets\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "\n",
    "RESHAPED = 784\n",
    "\n",
    "X_train = X_train.reshape(60000, RESHAPED).astype('float32')\n",
    "X_test = X_test.reshape(10000, RESHAPED).astype('float32')\n",
    "\n",
    "# normalize\n",
    "X_train /= 255\n",
    "X_test /= 255\n",
    "\n",
    "print(X_train.shape[0], 'train_samples')\n",
    "print(X_test.shape[0], 'test_samples')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# converting class vectors to binary class matrices\n",
    "Y_train = np_utils.to_categorical(y_train, NB_CLASSES)\n",
    "Y_test = np_utils.to_categorical(y_test, NB_CLASSES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 10)                7850      \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 7,850\n",
      "Trainable params: 7,850\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# building layers\n",
    "model = Sequential()\n",
    "model.add(Dense(NB_CLASSES, input_shape=(RESHAPED,)))\n",
    "model.add(Activation('softmax'))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compiling the model\n",
    "model.compile(loss='categorical_crossentropy', optimizer=OPTIMIZER, metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/250\n",
      "48000/48000 [==============================] - 1s 26us/step - loss: 1.3633 - acc: 0.6796 - val_loss: 0.8904 - val_acc: 0.8246\n",
      "Epoch 2/250\n",
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.7913 - acc: 0.8272 - val_loss: 0.6572 - val_acc: 0.8546\n",
      "Epoch 3/250\n",
      "48000/48000 [==============================] - 1s 31us/step - loss: 0.6436 - acc: 0.8497 - val_loss: 0.5625 - val_acc: 0.8681\n",
      "Epoch 4/250\n",
      "48000/48000 [==============================] - 1s 28us/step - loss: 0.5717 - acc: 0.8602 - val_loss: 0.5098 - val_acc: 0.8765\n",
      "Epoch 5/250\n",
      "48000/48000 [==============================] - 2s 35us/step - loss: 0.5276 - acc: 0.8678 - val_loss: 0.4758 - val_acc: 0.8826\n",
      "Epoch 6/250\n",
      "48000/48000 [==============================] - 1s 25us/step - loss: 0.4973 - acc: 0.8726 - val_loss: 0.4515 - val_acc: 0.8866\n",
      "Epoch 7/250\n",
      "48000/48000 [==============================] - 1s 29us/step - loss: 0.4748 - acc: 0.8775 - val_loss: 0.4333 - val_acc: 0.8882\n",
      "Epoch 8/250\n",
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.4574 - acc: 0.8803 - val_loss: 0.4189 - val_acc: 0.8920\n",
      "Epoch 9/250\n",
      "48000/48000 [==============================] - 1s 23us/step - loss: 0.4433 - acc: 0.8834 - val_loss: 0.4075 - val_acc: 0.8939\n",
      "Epoch 10/250\n",
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.4317 - acc: 0.8850 - val_loss: 0.3977 - val_acc: 0.8966\n",
      "Epoch 11/250\n",
      "48000/48000 [==============================] - 1s 23us/step - loss: 0.4218 - acc: 0.8873 - val_loss: 0.3896 - val_acc: 0.8984\n",
      "Epoch 12/250\n",
      "48000/48000 [==============================] - 1s 29us/step - loss: 0.4134 - acc: 0.8888 - val_loss: 0.3827 - val_acc: 0.8995\n",
      "Epoch 13/250\n",
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.4060 - acc: 0.8902 - val_loss: 0.3766 - val_acc: 0.9003\n",
      "Epoch 14/250\n",
      "48000/48000 [==============================] - 2s 32us/step - loss: 0.3995 - acc: 0.8918 - val_loss: 0.3712 - val_acc: 0.9013\n",
      "Epoch 15/250\n",
      "48000/48000 [==============================] - 2s 35us/step - loss: 0.3936 - acc: 0.8928 - val_loss: 0.3664 - val_acc: 0.9016\n",
      "Epoch 16/250\n",
      "48000/48000 [==============================] - 1s 30us/step - loss: 0.3884 - acc: 0.8945 - val_loss: 0.3621 - val_acc: 0.9031\n",
      "Epoch 17/250\n",
      "48000/48000 [==============================] - 1s 23us/step - loss: 0.3837 - acc: 0.8950 - val_loss: 0.3582 - val_acc: 0.9033\n",
      "Epoch 18/250\n",
      "48000/48000 [==============================] - 1s 23us/step - loss: 0.3794 - acc: 0.8962 - val_loss: 0.3546 - val_acc: 0.9039\n",
      "Epoch 19/250\n",
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.3755 - acc: 0.8970 - val_loss: 0.3514 - val_acc: 0.9048\n",
      "Epoch 20/250\n",
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.3718 - acc: 0.8979 - val_loss: 0.3485 - val_acc: 0.9053\n",
      "Epoch 21/250\n",
      "48000/48000 [==============================] - 1s 29us/step - loss: 0.3685 - acc: 0.8985 - val_loss: 0.3457 - val_acc: 0.9058\n",
      "Epoch 22/250\n",
      "48000/48000 [==============================] - 1s 23us/step - loss: 0.3653 - acc: 0.8995 - val_loss: 0.3431 - val_acc: 0.9058\n",
      "Epoch 23/250\n",
      "48000/48000 [==============================] - 1s 23us/step - loss: 0.3625 - acc: 0.8999 - val_loss: 0.3407 - val_acc: 0.9063\n",
      "Epoch 24/250\n",
      "48000/48000 [==============================] - 1s 23us/step - loss: 0.3598 - acc: 0.9008 - val_loss: 0.3385 - val_acc: 0.9070\n",
      "Epoch 25/250\n",
      "48000/48000 [==============================] - 1s 23us/step - loss: 0.3572 - acc: 0.9012 - val_loss: 0.3364 - val_acc: 0.9074\n",
      "Epoch 26/250\n",
      "48000/48000 [==============================] - 1s 30us/step - loss: 0.3548 - acc: 0.9019 - val_loss: 0.3345 - val_acc: 0.9084\n",
      "Epoch 27/250\n",
      "48000/48000 [==============================] - 1s 23us/step - loss: 0.3525 - acc: 0.9022 - val_loss: 0.3326 - val_acc: 0.9082\n",
      "Epoch 28/250\n",
      "48000/48000 [==============================] - 1s 23us/step - loss: 0.3504 - acc: 0.9032 - val_loss: 0.3311 - val_acc: 0.9090\n",
      "Epoch 29/250\n",
      "48000/48000 [==============================] - 1s 24us/step - loss: 0.3484 - acc: 0.9031 - val_loss: 0.3293 - val_acc: 0.9094\n",
      "Epoch 30/250\n",
      "48000/48000 [==============================] - 1s 23us/step - loss: 0.3465 - acc: 0.9041 - val_loss: 0.3277 - val_acc: 0.9097\n",
      "Epoch 31/250\n",
      "48000/48000 [==============================] - 1s 30us/step - loss: 0.3447 - acc: 0.9044 - val_loss: 0.3264 - val_acc: 0.9097\n",
      "Epoch 32/250\n",
      "48000/48000 [==============================] - 1s 23us/step - loss: 0.3430 - acc: 0.9047 - val_loss: 0.3249 - val_acc: 0.9097\n",
      "Epoch 33/250\n",
      "48000/48000 [==============================] - 1s 23us/step - loss: 0.3413 - acc: 0.9051 - val_loss: 0.3235 - val_acc: 0.9103\n",
      "Epoch 34/250\n",
      "48000/48000 [==============================] - 2s 33us/step - loss: 0.3397 - acc: 0.9056 - val_loss: 0.3222 - val_acc: 0.9104\n",
      "Epoch 35/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.3382 - acc: 0.9058 - val_loss: 0.3211 - val_acc: 0.9110\n",
      "Epoch 36/250\n",
      "48000/48000 [==============================] - 2s 35us/step - loss: 0.3368 - acc: 0.9062 - val_loss: 0.3198 - val_acc: 0.9110\n",
      "Epoch 37/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.3353 - acc: 0.9069 - val_loss: 0.3187 - val_acc: 0.9117\n",
      "Epoch 38/250\n",
      "48000/48000 [==============================] - 2s 33us/step - loss: 0.3340 - acc: 0.9075 - val_loss: 0.3177 - val_acc: 0.9120\n",
      "Epoch 39/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.3327 - acc: 0.9075 - val_loss: 0.3166 - val_acc: 0.9122\n",
      "Epoch 40/250\n",
      "48000/48000 [==============================] - 2s 36us/step - loss: 0.3314 - acc: 0.9078 - val_loss: 0.3159 - val_acc: 0.9118\n",
      "Epoch 41/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.3303 - acc: 0.9080 - val_loss: 0.3147 - val_acc: 0.9127\n",
      "Epoch 42/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.3291 - acc: 0.9084 - val_loss: 0.3138 - val_acc: 0.9132\n",
      "Epoch 43/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.3280 - acc: 0.9089 - val_loss: 0.3130 - val_acc: 0.9132\n",
      "Epoch 44/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.3270 - acc: 0.9091 - val_loss: 0.3121 - val_acc: 0.9132\n",
      "Epoch 45/250\n",
      "48000/48000 [==============================] - 2s 50us/step - loss: 0.3259 - acc: 0.9093 - val_loss: 0.3113 - val_acc: 0.9135\n",
      "Epoch 46/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.3249 - acc: 0.9095 - val_loss: 0.3105 - val_acc: 0.9137\n",
      "Epoch 47/250\n",
      "48000/48000 [==============================] - 2s 33us/step - loss: 0.3239 - acc: 0.9105 - val_loss: 0.3098 - val_acc: 0.9141\n",
      "Epoch 48/250\n",
      "48000/48000 [==============================] - 1s 27us/step - loss: 0.3230 - acc: 0.9105 - val_loss: 0.3090 - val_acc: 0.9146\n",
      "Epoch 49/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.3221 - acc: 0.9102 - val_loss: 0.3083 - val_acc: 0.9151\n",
      "Epoch 50/250\n",
      "48000/48000 [==============================] - 1s 21us/step - loss: 0.3212 - acc: 0.9109 - val_loss: 0.3075 - val_acc: 0.9150\n",
      "Epoch 51/250\n",
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.3204 - acc: 0.9109 - val_loss: 0.3070 - val_acc: 0.9150\n",
      "Epoch 52/250\n",
      "48000/48000 [==============================] - 1s 21us/step - loss: 0.3195 - acc: 0.9112 - val_loss: 0.3063 - val_acc: 0.9148\n",
      "Epoch 53/250\n",
      "48000/48000 [==============================] - 1s 24us/step - loss: 0.3187 - acc: 0.9114 - val_loss: 0.3057 - val_acc: 0.9153\n",
      "Epoch 54/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.3180 - acc: 0.9117 - val_loss: 0.3050 - val_acc: 0.9148\n",
      "Epoch 55/250\n",
      "48000/48000 [==============================] - 2s 35us/step - loss: 0.3171 - acc: 0.9121 - val_loss: 0.3044 - val_acc: 0.9149\n",
      "Epoch 56/250\n",
      "48000/48000 [==============================] - 2s 35us/step - loss: 0.3164 - acc: 0.9121 - val_loss: 0.3037 - val_acc: 0.9156\n",
      "Epoch 57/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.3157 - acc: 0.9128 - val_loss: 0.3034 - val_acc: 0.9152\n",
      "Epoch 58/250\n",
      "48000/48000 [==============================] - 1s 29us/step - loss: 0.3149 - acc: 0.9121 - val_loss: 0.3029 - val_acc: 0.9148\n",
      "Epoch 59/250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.3143 - acc: 0.9128 - val_loss: 0.3022 - val_acc: 0.9151\n",
      "Epoch 60/250\n",
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.3136 - acc: 0.9129 - val_loss: 0.3016 - val_acc: 0.9161\n",
      "Epoch 61/250\n",
      "48000/48000 [==============================] - 1s 26us/step - loss: 0.3130 - acc: 0.9133 - val_loss: 0.3011 - val_acc: 0.9158\n",
      "Epoch 62/250\n",
      "48000/48000 [==============================] - 1s 30us/step - loss: 0.3123 - acc: 0.9131 - val_loss: 0.3007 - val_acc: 0.9151\n",
      "Epoch 63/250\n",
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.3117 - acc: 0.9136 - val_loss: 0.3003 - val_acc: 0.9156\n",
      "Epoch 64/250\n",
      "48000/48000 [==============================] - 1s 27us/step - loss: 0.3110 - acc: 0.9137 - val_loss: 0.2997 - val_acc: 0.9158\n",
      "Epoch 65/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.3105 - acc: 0.9137 - val_loss: 0.2992 - val_acc: 0.9159\n",
      "Epoch 66/250\n",
      "48000/48000 [==============================] - 1s 29us/step - loss: 0.3098 - acc: 0.9138 - val_loss: 0.2988 - val_acc: 0.9161\n",
      "Epoch 67/250\n",
      "48000/48000 [==============================] - 1s 26us/step - loss: 0.3093 - acc: 0.9141 - val_loss: 0.2983 - val_acc: 0.9165\n",
      "Epoch 68/250\n",
      "48000/48000 [==============================] - 1s 28us/step - loss: 0.3087 - acc: 0.9139 - val_loss: 0.2979 - val_acc: 0.9166\n",
      "Epoch 69/250\n",
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.3082 - acc: 0.9144 - val_loss: 0.2976 - val_acc: 0.9164\n",
      "Epoch 70/250\n",
      "48000/48000 [==============================] - 1s 26us/step - loss: 0.3077 - acc: 0.9145 - val_loss: 0.2971 - val_acc: 0.9166\n",
      "Epoch 71/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.3071 - acc: 0.9146 - val_loss: 0.2967 - val_acc: 0.9172\n",
      "Epoch 72/250\n",
      "48000/48000 [==============================] - 2s 48us/step - loss: 0.3066 - acc: 0.9147 - val_loss: 0.2964 - val_acc: 0.9167\n",
      "Epoch 73/250\n",
      "48000/48000 [==============================] - 2s 37us/step - loss: 0.3061 - acc: 0.9151 - val_loss: 0.2960 - val_acc: 0.9169\n",
      "Epoch 74/250\n",
      "48000/48000 [==============================] - 2s 36us/step - loss: 0.3056 - acc: 0.9150 - val_loss: 0.2956 - val_acc: 0.9173\n",
      "Epoch 75/250\n",
      "48000/48000 [==============================] - 1s 27us/step - loss: 0.3051 - acc: 0.9151 - val_loss: 0.2952 - val_acc: 0.9177\n",
      "Epoch 76/250\n",
      "48000/48000 [==============================] - 1s 21us/step - loss: 0.3046 - acc: 0.9152 - val_loss: 0.2950 - val_acc: 0.9173\n",
      "Epoch 77/250\n",
      "48000/48000 [==============================] - 1s 21us/step - loss: 0.3042 - acc: 0.9154 - val_loss: 0.2945 - val_acc: 0.9172\n",
      "Epoch 78/250\n",
      "48000/48000 [==============================] - 1s 21us/step - loss: 0.3037 - acc: 0.9154 - val_loss: 0.2942 - val_acc: 0.9176\n",
      "Epoch 79/250\n",
      "48000/48000 [==============================] - 1s 30us/step - loss: 0.3032 - acc: 0.9157 - val_loss: 0.2939 - val_acc: 0.9179\n",
      "Epoch 80/250\n",
      "48000/48000 [==============================] - 1s 27us/step - loss: 0.3028 - acc: 0.9156 - val_loss: 0.2936 - val_acc: 0.9177\n",
      "Epoch 81/250\n",
      "48000/48000 [==============================] - 1s 21us/step - loss: 0.3024 - acc: 0.9157 - val_loss: 0.2933 - val_acc: 0.9179\n",
      "Epoch 82/250\n",
      "48000/48000 [==============================] - 1s 21us/step - loss: 0.3019 - acc: 0.9157 - val_loss: 0.2930 - val_acc: 0.9178\n",
      "Epoch 83/250\n",
      "48000/48000 [==============================] - 1s 27us/step - loss: 0.3015 - acc: 0.9160 - val_loss: 0.2926 - val_acc: 0.9182\n",
      "Epoch 84/250\n",
      "48000/48000 [==============================] - 1s 24us/step - loss: 0.3011 - acc: 0.9161 - val_loss: 0.2924 - val_acc: 0.9179\n",
      "Epoch 85/250\n",
      "48000/48000 [==============================] - 1s 28us/step - loss: 0.3007 - acc: 0.9165 - val_loss: 0.2920 - val_acc: 0.9184\n",
      "Epoch 86/250\n",
      "48000/48000 [==============================] - 1s 26us/step - loss: 0.3003 - acc: 0.9164 - val_loss: 0.2918 - val_acc: 0.9185\n",
      "Epoch 87/250\n",
      "48000/48000 [==============================] - 1s 29us/step - loss: 0.2999 - acc: 0.9165 - val_loss: 0.2914 - val_acc: 0.9185\n",
      "Epoch 88/250\n",
      "48000/48000 [==============================] - 1s 28us/step - loss: 0.2995 - acc: 0.9166 - val_loss: 0.2911 - val_acc: 0.9188\n",
      "Epoch 89/250\n",
      "48000/48000 [==============================] - 2s 35us/step - loss: 0.2991 - acc: 0.9167 - val_loss: 0.2909 - val_acc: 0.9191\n",
      "Epoch 90/250\n",
      "48000/48000 [==============================] - 1s 29us/step - loss: 0.2988 - acc: 0.9169 - val_loss: 0.2906 - val_acc: 0.9191\n",
      "Epoch 91/250\n",
      "48000/48000 [==============================] - 1s 27us/step - loss: 0.2984 - acc: 0.9168 - val_loss: 0.2903 - val_acc: 0.9192\n",
      "Epoch 92/250\n",
      "48000/48000 [==============================] - 1s 27us/step - loss: 0.2981 - acc: 0.9170 - val_loss: 0.2901 - val_acc: 0.9196\n",
      "Epoch 93/250\n",
      "48000/48000 [==============================] - 2s 37us/step - loss: 0.2977 - acc: 0.9171 - val_loss: 0.2898 - val_acc: 0.9195\n",
      "Epoch 94/250\n",
      "48000/48000 [==============================] - 1s 29us/step - loss: 0.2973 - acc: 0.9172 - val_loss: 0.2895 - val_acc: 0.9196\n",
      "Epoch 95/250\n",
      "48000/48000 [==============================] - 2s 33us/step - loss: 0.2970 - acc: 0.9174 - val_loss: 0.2894 - val_acc: 0.9196\n",
      "Epoch 96/250\n",
      "48000/48000 [==============================] - 1s 26us/step - loss: 0.2967 - acc: 0.9174 - val_loss: 0.2891 - val_acc: 0.9198\n",
      "Epoch 97/250\n",
      "48000/48000 [==============================] - 1s 30us/step - loss: 0.2963 - acc: 0.9176 - val_loss: 0.2889 - val_acc: 0.9197\n",
      "Epoch 98/250\n",
      "48000/48000 [==============================] - 1s 23us/step - loss: 0.2960 - acc: 0.9174 - val_loss: 0.2886 - val_acc: 0.9202\n",
      "Epoch 99/250\n",
      "48000/48000 [==============================] - 1s 25us/step - loss: 0.2957 - acc: 0.9176 - val_loss: 0.2884 - val_acc: 0.9202\n",
      "Epoch 100/250\n",
      "48000/48000 [==============================] - 1s 27us/step - loss: 0.2953 - acc: 0.9178 - val_loss: 0.2882 - val_acc: 0.9200\n",
      "Epoch 101/250\n",
      "48000/48000 [==============================] - 1s 30us/step - loss: 0.2950 - acc: 0.9179 - val_loss: 0.2879 - val_acc: 0.9201\n",
      "Epoch 102/250\n",
      "48000/48000 [==============================] - 2s 33us/step - loss: 0.2947 - acc: 0.9180 - val_loss: 0.2877 - val_acc: 0.9204\n",
      "Epoch 103/250\n",
      "48000/48000 [==============================] - 1s 29us/step - loss: 0.2944 - acc: 0.9180 - val_loss: 0.2875 - val_acc: 0.9202\n",
      "Epoch 104/250\n",
      "48000/48000 [==============================] - 1s 29us/step - loss: 0.2941 - acc: 0.9184 - val_loss: 0.2873 - val_acc: 0.9202\n",
      "Epoch 105/250\n",
      "48000/48000 [==============================] - 2s 34us/step - loss: 0.2938 - acc: 0.9183 - val_loss: 0.2871 - val_acc: 0.9206\n",
      "Epoch 106/250\n",
      "48000/48000 [==============================] - 2s 36us/step - loss: 0.2935 - acc: 0.9183 - val_loss: 0.2868 - val_acc: 0.9202\n",
      "Epoch 107/250\n",
      "48000/48000 [==============================] - 1s 28us/step - loss: 0.2932 - acc: 0.9186 - val_loss: 0.2867 - val_acc: 0.9206\n",
      "Epoch 108/250\n",
      "48000/48000 [==============================] - 1s 24us/step - loss: 0.2929 - acc: 0.9185 - val_loss: 0.2864 - val_acc: 0.9208\n",
      "Epoch 109/250\n",
      "48000/48000 [==============================] - 1s 21us/step - loss: 0.2927 - acc: 0.9185 - val_loss: 0.2863 - val_acc: 0.9206\n",
      "Epoch 110/250\n",
      "48000/48000 [==============================] - 2s 33us/step - loss: 0.2923 - acc: 0.9187 - val_loss: 0.2860 - val_acc: 0.9204\n",
      "Epoch 111/250\n",
      "48000/48000 [==============================] - 1s 27us/step - loss: 0.2921 - acc: 0.9184 - val_loss: 0.2858 - val_acc: 0.9210\n",
      "Epoch 112/250\n",
      "48000/48000 [==============================] - 1s 29us/step - loss: 0.2918 - acc: 0.9187 - val_loss: 0.2857 - val_acc: 0.9207\n",
      "Epoch 113/250\n",
      "48000/48000 [==============================] - 1s 24us/step - loss: 0.2915 - acc: 0.9189 - val_loss: 0.2854 - val_acc: 0.9210\n",
      "Epoch 114/250\n",
      "48000/48000 [==============================] - 1s 28us/step - loss: 0.2913 - acc: 0.9188 - val_loss: 0.2853 - val_acc: 0.9211\n",
      "Epoch 115/250\n",
      "48000/48000 [==============================] - 1s 26us/step - loss: 0.2910 - acc: 0.9189 - val_loss: 0.2852 - val_acc: 0.9205\n",
      "Epoch 116/250\n",
      "48000/48000 [==============================] - 2s 34us/step - loss: 0.2908 - acc: 0.9189 - val_loss: 0.2849 - val_acc: 0.9213\n",
      "Epoch 117/250\n",
      "48000/48000 [==============================] - 1s 31us/step - loss: 0.2905 - acc: 0.9193 - val_loss: 0.2847 - val_acc: 0.9213\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 118/250\n",
      "48000/48000 [==============================] - 1s 26us/step - loss: 0.2902 - acc: 0.9192 - val_loss: 0.2846 - val_acc: 0.9212\n",
      "Epoch 119/250\n",
      "48000/48000 [==============================] - 2s 38us/step - loss: 0.2900 - acc: 0.9191 - val_loss: 0.2844 - val_acc: 0.9212\n",
      "Epoch 120/250\n",
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.2898 - acc: 0.9192 - val_loss: 0.2842 - val_acc: 0.9212\n",
      "Epoch 121/250\n",
      "48000/48000 [==============================] - 1s 26us/step - loss: 0.2895 - acc: 0.9191 - val_loss: 0.2841 - val_acc: 0.9212\n",
      "Epoch 122/250\n",
      "48000/48000 [==============================] - 1s 24us/step - loss: 0.2892 - acc: 0.9192 - val_loss: 0.2840 - val_acc: 0.9212\n",
      "Epoch 123/250\n",
      "48000/48000 [==============================] - 2s 36us/step - loss: 0.2890 - acc: 0.9194 - val_loss: 0.2838 - val_acc: 0.9211\n",
      "Epoch 124/250\n",
      "48000/48000 [==============================] - 1s 29us/step - loss: 0.2888 - acc: 0.9197 - val_loss: 0.2837 - val_acc: 0.9210\n",
      "Epoch 125/250\n",
      "48000/48000 [==============================] - 2s 32us/step - loss: 0.2885 - acc: 0.9193 - val_loss: 0.2835 - val_acc: 0.9207\n",
      "Epoch 126/250\n",
      "48000/48000 [==============================] - 1s 30us/step - loss: 0.2883 - acc: 0.9197 - val_loss: 0.2834 - val_acc: 0.9217\n",
      "Epoch 127/250\n",
      "48000/48000 [==============================] - 2s 49us/step - loss: 0.2881 - acc: 0.9194 - val_loss: 0.2832 - val_acc: 0.9212\n",
      "Epoch 128/250\n",
      "48000/48000 [==============================] - 2s 37us/step - loss: 0.2879 - acc: 0.9194 - val_loss: 0.2830 - val_acc: 0.9210\n",
      "Epoch 129/250\n",
      "48000/48000 [==============================] - 2s 32us/step - loss: 0.2876 - acc: 0.9196 - val_loss: 0.2828 - val_acc: 0.9217\n",
      "Epoch 130/250\n",
      "48000/48000 [==============================] - 1s 24us/step - loss: 0.2874 - acc: 0.9197 - val_loss: 0.2826 - val_acc: 0.9216\n",
      "Epoch 131/250\n",
      "48000/48000 [==============================] - 1s 26us/step - loss: 0.2871 - acc: 0.9200 - val_loss: 0.2827 - val_acc: 0.9211\n",
      "Epoch 132/250\n",
      "48000/48000 [==============================] - 1s 20us/step - loss: 0.2870 - acc: 0.9197 - val_loss: 0.2824 - val_acc: 0.9213\n",
      "Epoch 133/250\n",
      "48000/48000 [==============================] - 1s 21us/step - loss: 0.2868 - acc: 0.9198 - val_loss: 0.2823 - val_acc: 0.9216\n",
      "Epoch 134/250\n",
      "48000/48000 [==============================] - 1s 21us/step - loss: 0.2866 - acc: 0.9199 - val_loss: 0.2822 - val_acc: 0.9214\n",
      "Epoch 135/250\n",
      "48000/48000 [==============================] - 1s 25us/step - loss: 0.2863 - acc: 0.9203 - val_loss: 0.2820 - val_acc: 0.9213\n",
      "Epoch 136/250\n",
      "48000/48000 [==============================] - 2s 35us/step - loss: 0.2861 - acc: 0.9196 - val_loss: 0.2818 - val_acc: 0.9215\n",
      "Epoch 137/250\n",
      "48000/48000 [==============================] - 2s 36us/step - loss: 0.2859 - acc: 0.9198 - val_loss: 0.2818 - val_acc: 0.9217\n",
      "Epoch 138/250\n",
      "48000/48000 [==============================] - 2s 36us/step - loss: 0.2857 - acc: 0.9203 - val_loss: 0.2815 - val_acc: 0.9218\n",
      "Epoch 139/250\n",
      "48000/48000 [==============================] - 2s 34us/step - loss: 0.2855 - acc: 0.9203 - val_loss: 0.2814 - val_acc: 0.9215\n",
      "Epoch 140/250\n",
      "48000/48000 [==============================] - 1s 31us/step - loss: 0.2853 - acc: 0.9201 - val_loss: 0.2812 - val_acc: 0.9216\n",
      "Epoch 141/250\n",
      "48000/48000 [==============================] - 1s 30us/step - loss: 0.2852 - acc: 0.9204 - val_loss: 0.2811 - val_acc: 0.9217\n",
      "Epoch 142/250\n",
      "48000/48000 [==============================] - 1s 27us/step - loss: 0.2849 - acc: 0.9201 - val_loss: 0.2810 - val_acc: 0.9217\n",
      "Epoch 143/250\n",
      "48000/48000 [==============================] - 1s 28us/step - loss: 0.2848 - acc: 0.9205 - val_loss: 0.2809 - val_acc: 0.9219\n",
      "Epoch 144/250\n",
      "48000/48000 [==============================] - 2s 38us/step - loss: 0.2846 - acc: 0.9208 - val_loss: 0.2808 - val_acc: 0.9217\n",
      "Epoch 145/250\n",
      "48000/48000 [==============================] - 1s 27us/step - loss: 0.2844 - acc: 0.9207 - val_loss: 0.2806 - val_acc: 0.9221\n",
      "Epoch 146/250\n",
      "48000/48000 [==============================] - 1s 31us/step - loss: 0.2841 - acc: 0.9206 - val_loss: 0.2806 - val_acc: 0.9220\n",
      "Epoch 147/250\n",
      "48000/48000 [==============================] - 1s 30us/step - loss: 0.2840 - acc: 0.9207 - val_loss: 0.2804 - val_acc: 0.9217\n",
      "Epoch 148/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.2838 - acc: 0.9209 - val_loss: 0.2803 - val_acc: 0.9218\n",
      "Epoch 149/250\n",
      "48000/48000 [==============================] - 2s 33us/step - loss: 0.2836 - acc: 0.9208 - val_loss: 0.2802 - val_acc: 0.9216\n",
      "Epoch 150/250\n",
      "48000/48000 [==============================] - 2s 32us/step - loss: 0.2835 - acc: 0.9210 - val_loss: 0.2800 - val_acc: 0.9225\n",
      "Epoch 151/250\n",
      "48000/48000 [==============================] - 2s 37us/step - loss: 0.2833 - acc: 0.9210 - val_loss: 0.2799 - val_acc: 0.9226\n",
      "Epoch 152/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.2831 - acc: 0.9211 - val_loss: 0.2798 - val_acc: 0.9222\n",
      "Epoch 153/250\n",
      "48000/48000 [==============================] - 1s 28us/step - loss: 0.2829 - acc: 0.9207 - val_loss: 0.2797 - val_acc: 0.9224\n",
      "Epoch 154/250\n",
      "48000/48000 [==============================] - 1s 28us/step - loss: 0.2827 - acc: 0.9209 - val_loss: 0.2796 - val_acc: 0.9222\n",
      "Epoch 155/250\n",
      "48000/48000 [==============================] - 2s 34us/step - loss: 0.2826 - acc: 0.9208 - val_loss: 0.2795 - val_acc: 0.9225\n",
      "Epoch 156/250\n",
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.2824 - acc: 0.9210 - val_loss: 0.2794 - val_acc: 0.9224\n",
      "Epoch 157/250\n",
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.2822 - acc: 0.9210 - val_loss: 0.2793 - val_acc: 0.9224\n",
      "Epoch 158/250\n",
      "48000/48000 [==============================] - 2s 32us/step - loss: 0.2821 - acc: 0.9214 - val_loss: 0.2792 - val_acc: 0.9226\n",
      "Epoch 159/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.2819 - acc: 0.9214 - val_loss: 0.2791 - val_acc: 0.9226\n",
      "Epoch 160/250\n",
      "48000/48000 [==============================] - 1s 30us/step - loss: 0.2817 - acc: 0.9213 - val_loss: 0.2790 - val_acc: 0.9225\n",
      "Epoch 161/250\n",
      "48000/48000 [==============================] - 1s 29us/step - loss: 0.2816 - acc: 0.9214 - val_loss: 0.2789 - val_acc: 0.9222\n",
      "Epoch 162/250\n",
      "48000/48000 [==============================] - 2s 32us/step - loss: 0.2814 - acc: 0.9215 - val_loss: 0.2788 - val_acc: 0.9227\n",
      "Epoch 163/250\n",
      "48000/48000 [==============================] - 2s 35us/step - loss: 0.2812 - acc: 0.9213 - val_loss: 0.2787 - val_acc: 0.9225\n",
      "Epoch 164/250\n",
      "48000/48000 [==============================] - 1s 28us/step - loss: 0.2811 - acc: 0.9216 - val_loss: 0.2786 - val_acc: 0.9225\n",
      "Epoch 165/250\n",
      "48000/48000 [==============================] - 2s 33us/step - loss: 0.2809 - acc: 0.9215 - val_loss: 0.2785 - val_acc: 0.9227\n",
      "Epoch 166/250\n",
      "48000/48000 [==============================] - 1s 26us/step - loss: 0.2807 - acc: 0.9216 - val_loss: 0.2784 - val_acc: 0.9225\n",
      "Epoch 167/250\n",
      "48000/48000 [==============================] - 1s 31us/step - loss: 0.2806 - acc: 0.9217 - val_loss: 0.2784 - val_acc: 0.9227\n",
      "Epoch 168/250\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.2804 - acc: 0.9219 - val_loss: 0.2782 - val_acc: 0.9228\n",
      "Epoch 169/250\n",
      "48000/48000 [==============================] - 2s 34us/step - loss: 0.2803 - acc: 0.9216 - val_loss: 0.2782 - val_acc: 0.9227\n",
      "Epoch 170/250\n",
      "48000/48000 [==============================] - 2s 33us/step - loss: 0.2801 - acc: 0.9216 - val_loss: 0.2781 - val_acc: 0.9227\n",
      "Epoch 171/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.2800 - acc: 0.9220 - val_loss: 0.2780 - val_acc: 0.9226\n",
      "Epoch 172/250\n",
      "48000/48000 [==============================] - 2s 36us/step - loss: 0.2798 - acc: 0.9218 - val_loss: 0.2778 - val_acc: 0.9231\n",
      "Epoch 173/250\n",
      "48000/48000 [==============================] - 2s 32us/step - loss: 0.2797 - acc: 0.9217 - val_loss: 0.2778 - val_acc: 0.9229\n",
      "Epoch 174/250\n",
      "48000/48000 [==============================] - 2s 33us/step - loss: 0.2796 - acc: 0.9217 - val_loss: 0.2777 - val_acc: 0.9227\n",
      "Epoch 175/250\n",
      "48000/48000 [==============================] - 1s 25us/step - loss: 0.2794 - acc: 0.9218 - val_loss: 0.2776 - val_acc: 0.9232\n",
      "Epoch 176/250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.2793 - acc: 0.9220 - val_loss: 0.2775 - val_acc: 0.9232\n",
      "Epoch 177/250\n",
      "48000/48000 [==============================] - 1s 31us/step - loss: 0.2791 - acc: 0.9219 - val_loss: 0.2774 - val_acc: 0.9234\n",
      "Epoch 178/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.2790 - acc: 0.9221 - val_loss: 0.2774 - val_acc: 0.9228\n",
      "Epoch 179/250\n",
      "48000/48000 [==============================] - 2s 35us/step - loss: 0.2788 - acc: 0.9221 - val_loss: 0.2773 - val_acc: 0.9232\n",
      "Epoch 180/250\n",
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.2787 - acc: 0.9221 - val_loss: 0.2771 - val_acc: 0.9235\n",
      "Epoch 181/250\n",
      "48000/48000 [==============================] - 2s 35us/step - loss: 0.2785 - acc: 0.9223 - val_loss: 0.2770 - val_acc: 0.9232\n",
      "Epoch 182/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.2784 - acc: 0.9220 - val_loss: 0.2769 - val_acc: 0.9231\n",
      "Epoch 183/250\n",
      "48000/48000 [==============================] - 2s 36us/step - loss: 0.2783 - acc: 0.9223 - val_loss: 0.2769 - val_acc: 0.9231\n",
      "Epoch 184/250\n",
      "48000/48000 [==============================] - 2s 35us/step - loss: 0.2781 - acc: 0.9223 - val_loss: 0.2768 - val_acc: 0.9230\n",
      "Epoch 185/250\n",
      "48000/48000 [==============================] - 2s 35us/step - loss: 0.2780 - acc: 0.9224 - val_loss: 0.2767 - val_acc: 0.9233\n",
      "Epoch 186/250\n",
      "48000/48000 [==============================] - 2s 35us/step - loss: 0.2779 - acc: 0.9223 - val_loss: 0.2766 - val_acc: 0.9236\n",
      "Epoch 187/250\n",
      "48000/48000 [==============================] - 2s 35us/step - loss: 0.2777 - acc: 0.9224 - val_loss: 0.2766 - val_acc: 0.9233\n",
      "Epoch 188/250\n",
      "48000/48000 [==============================] - 1s 29us/step - loss: 0.2776 - acc: 0.9226 - val_loss: 0.2765 - val_acc: 0.9236\n",
      "Epoch 189/250\n",
      "48000/48000 [==============================] - 1s 26us/step - loss: 0.2775 - acc: 0.9225 - val_loss: 0.2764 - val_acc: 0.9235\n",
      "Epoch 190/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.2773 - acc: 0.9225 - val_loss: 0.2764 - val_acc: 0.9235\n",
      "Epoch 191/250\n",
      "48000/48000 [==============================] - 1s 28us/step - loss: 0.2772 - acc: 0.9225 - val_loss: 0.2763 - val_acc: 0.9237\n",
      "Epoch 192/250\n",
      "48000/48000 [==============================] - 1s 30us/step - loss: 0.2770 - acc: 0.9226 - val_loss: 0.2762 - val_acc: 0.9238\n",
      "Epoch 193/250\n",
      "48000/48000 [==============================] - 2s 33us/step - loss: 0.2770 - acc: 0.9226 - val_loss: 0.2761 - val_acc: 0.9237\n",
      "Epoch 194/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.2768 - acc: 0.9226 - val_loss: 0.2761 - val_acc: 0.9236\n",
      "Epoch 195/250\n",
      "48000/48000 [==============================] - 1s 28us/step - loss: 0.2767 - acc: 0.9231 - val_loss: 0.2760 - val_acc: 0.9239\n",
      "Epoch 196/250\n",
      "48000/48000 [==============================] - 1s 27us/step - loss: 0.2766 - acc: 0.9226 - val_loss: 0.2758 - val_acc: 0.9241\n",
      "Epoch 197/250\n",
      "48000/48000 [==============================] - 1s 24us/step - loss: 0.2765 - acc: 0.9229 - val_loss: 0.2758 - val_acc: 0.9242\n",
      "Epoch 198/250\n",
      "48000/48000 [==============================] - 1s 28us/step - loss: 0.2763 - acc: 0.9231 - val_loss: 0.2758 - val_acc: 0.9236\n",
      "Epoch 199/250\n",
      "48000/48000 [==============================] - 1s 24us/step - loss: 0.2762 - acc: 0.9229 - val_loss: 0.2757 - val_acc: 0.9241\n",
      "Epoch 200/250\n",
      "48000/48000 [==============================] - 1s 26us/step - loss: 0.2761 - acc: 0.9230 - val_loss: 0.2756 - val_acc: 0.9241\n",
      "Epoch 201/250\n",
      "48000/48000 [==============================] - 2s 36us/step - loss: 0.2760 - acc: 0.9228 - val_loss: 0.2755 - val_acc: 0.9238\n",
      "Epoch 202/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.2758 - acc: 0.9231 - val_loss: 0.2754 - val_acc: 0.9241\n",
      "Epoch 203/250\n",
      "48000/48000 [==============================] - 2s 33us/step - loss: 0.2757 - acc: 0.9231 - val_loss: 0.2754 - val_acc: 0.9241\n",
      "Epoch 204/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.2756 - acc: 0.9231 - val_loss: 0.2754 - val_acc: 0.9241\n",
      "Epoch 205/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.2755 - acc: 0.9231 - val_loss: 0.2753 - val_acc: 0.9248\n",
      "Epoch 206/250\n",
      "48000/48000 [==============================] - 2s 36us/step - loss: 0.2754 - acc: 0.9231 - val_loss: 0.2752 - val_acc: 0.9245\n",
      "Epoch 207/250\n",
      "48000/48000 [==============================] - 2s 33us/step - loss: 0.2753 - acc: 0.9231 - val_loss: 0.2751 - val_acc: 0.9244\n",
      "Epoch 208/250\n",
      "48000/48000 [==============================] - 2s 36us/step - loss: 0.2751 - acc: 0.9232 - val_loss: 0.2751 - val_acc: 0.9243\n",
      "Epoch 209/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.2750 - acc: 0.9233 - val_loss: 0.2749 - val_acc: 0.9244\n",
      "Epoch 210/250\n",
      "48000/48000 [==============================] - 1s 25us/step - loss: 0.2749 - acc: 0.9231 - val_loss: 0.2749 - val_acc: 0.9243\n",
      "Epoch 211/250\n",
      "48000/48000 [==============================] - 1s 23us/step - loss: 0.2748 - acc: 0.9234 - val_loss: 0.2749 - val_acc: 0.9243\n",
      "Epoch 212/250\n",
      "48000/48000 [==============================] - 1s 24us/step - loss: 0.2747 - acc: 0.9235 - val_loss: 0.2748 - val_acc: 0.9240\n",
      "Epoch 213/250\n",
      "48000/48000 [==============================] - 1s 31us/step - loss: 0.2745 - acc: 0.9233 - val_loss: 0.2748 - val_acc: 0.9235\n",
      "Epoch 214/250\n",
      "48000/48000 [==============================] - 2s 35us/step - loss: 0.2745 - acc: 0.9235 - val_loss: 0.2748 - val_acc: 0.9244\n",
      "Epoch 215/250\n",
      "48000/48000 [==============================] - 1s 29us/step - loss: 0.2743 - acc: 0.9232 - val_loss: 0.2746 - val_acc: 0.9244\n",
      "Epoch 216/250\n",
      "48000/48000 [==============================] - 1s 27us/step - loss: 0.2742 - acc: 0.9235 - val_loss: 0.2746 - val_acc: 0.9248\n",
      "Epoch 217/250\n",
      "48000/48000 [==============================] - 1s 27us/step - loss: 0.2741 - acc: 0.9236 - val_loss: 0.2745 - val_acc: 0.9250\n",
      "Epoch 218/250\n",
      "48000/48000 [==============================] - 2s 35us/step - loss: 0.2740 - acc: 0.9235 - val_loss: 0.2745 - val_acc: 0.9245\n",
      "Epoch 219/250\n",
      "48000/48000 [==============================] - 1s 27us/step - loss: 0.2739 - acc: 0.9236 - val_loss: 0.2745 - val_acc: 0.9243\n",
      "Epoch 220/250\n",
      "48000/48000 [==============================] - 1s 31us/step - loss: 0.2738 - acc: 0.9236 - val_loss: 0.2743 - val_acc: 0.9244\n",
      "Epoch 221/250\n",
      "48000/48000 [==============================] - 2s 33us/step - loss: 0.2737 - acc: 0.9237 - val_loss: 0.2743 - val_acc: 0.9243\n",
      "Epoch 222/250\n",
      "48000/48000 [==============================] - 2s 38us/step - loss: 0.2736 - acc: 0.9236 - val_loss: 0.2742 - val_acc: 0.9243\n",
      "Epoch 223/250\n",
      "48000/48000 [==============================] - 1s 25us/step - loss: 0.2735 - acc: 0.9239 - val_loss: 0.2743 - val_acc: 0.9244\n",
      "Epoch 224/250\n",
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.2734 - acc: 0.9236 - val_loss: 0.2741 - val_acc: 0.9245\n",
      "Epoch 225/250\n",
      "48000/48000 [==============================] - 1s 21us/step - loss: 0.2733 - acc: 0.9238 - val_loss: 0.2741 - val_acc: 0.9248\n",
      "Epoch 226/250\n",
      "48000/48000 [==============================] - 1s 21us/step - loss: 0.2732 - acc: 0.9238 - val_loss: 0.2740 - val_acc: 0.9243\n",
      "Epoch 227/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.2731 - acc: 0.9238 - val_loss: 0.2739 - val_acc: 0.9247\n",
      "Epoch 228/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.2730 - acc: 0.9238 - val_loss: 0.2738 - val_acc: 0.9243\n",
      "Epoch 229/250\n",
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.2729 - acc: 0.9237 - val_loss: 0.2738 - val_acc: 0.9247\n",
      "Epoch 230/250\n",
      "48000/48000 [==============================] - 1s 20us/step - loss: 0.2728 - acc: 0.9239 - val_loss: 0.2738 - val_acc: 0.9245\n",
      "Epoch 231/250\n",
      "48000/48000 [==============================] - 1s 24us/step - loss: 0.2727 - acc: 0.9238 - val_loss: 0.2737 - val_acc: 0.9248\n",
      "Epoch 232/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.2726 - acc: 0.9240 - val_loss: 0.2737 - val_acc: 0.9246\n",
      "Epoch 233/250\n",
      "48000/48000 [==============================] - 1s 31us/step - loss: 0.2725 - acc: 0.9239 - val_loss: 0.2736 - val_acc: 0.9246\n",
      "Epoch 234/250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48000/48000 [==============================] - 2s 32us/step - loss: 0.2724 - acc: 0.9240 - val_loss: 0.2735 - val_acc: 0.9248\n",
      "Epoch 235/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.2723 - acc: 0.9239 - val_loss: 0.2736 - val_acc: 0.9244\n",
      "Epoch 236/250\n",
      "48000/48000 [==============================] - 2s 35us/step - loss: 0.2722 - acc: 0.9241 - val_loss: 0.2735 - val_acc: 0.9246\n",
      "Epoch 237/250\n",
      "48000/48000 [==============================] - 2s 37us/step - loss: 0.2721 - acc: 0.9243 - val_loss: 0.2734 - val_acc: 0.9248\n",
      "Epoch 238/250\n",
      "48000/48000 [==============================] - 1s 31us/step - loss: 0.2720 - acc: 0.9244 - val_loss: 0.2734 - val_acc: 0.9244\n",
      "Epoch 239/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.2719 - acc: 0.9242 - val_loss: 0.2734 - val_acc: 0.9248\n",
      "Epoch 240/250\n",
      "48000/48000 [==============================] - 1s 30us/step - loss: 0.2718 - acc: 0.9242 - val_loss: 0.2733 - val_acc: 0.9245\n",
      "Epoch 241/250\n",
      "48000/48000 [==============================] - 1s 25us/step - loss: 0.2717 - acc: 0.9242 - val_loss: 0.2732 - val_acc: 0.9246\n",
      "Epoch 242/250\n",
      "48000/48000 [==============================] - 1s 21us/step - loss: 0.2716 - acc: 0.9246 - val_loss: 0.2732 - val_acc: 0.9249\n",
      "Epoch 243/250\n",
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.2715 - acc: 0.9243 - val_loss: 0.2731 - val_acc: 0.9248\n",
      "Epoch 244/250\n",
      "48000/48000 [==============================] - 2s 38us/step - loss: 0.2714 - acc: 0.9242 - val_loss: 0.2731 - val_acc: 0.9247\n",
      "Epoch 245/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.2713 - acc: 0.9241 - val_loss: 0.2730 - val_acc: 0.9245\n",
      "Epoch 246/250\n",
      "48000/48000 [==============================] - 2s 34us/step - loss: 0.2712 - acc: 0.9245 - val_loss: 0.2729 - val_acc: 0.9250\n",
      "Epoch 247/250\n",
      "48000/48000 [==============================] - 2s 36us/step - loss: 0.2712 - acc: 0.9244 - val_loss: 0.2729 - val_acc: 0.9248\n",
      "Epoch 248/250\n",
      "48000/48000 [==============================] - 1s 23us/step - loss: 0.2711 - acc: 0.9244 - val_loss: 0.2728 - val_acc: 0.9248\n",
      "Epoch 249/250\n",
      "48000/48000 [==============================] - 1s 22us/step - loss: 0.2710 - acc: 0.9243 - val_loss: 0.2727 - val_acc: 0.9247\n",
      "Epoch 250/250\n",
      "48000/48000 [==============================] - 2s 32us/step - loss: 0.2709 - acc: 0.9245 - val_loss: 0.2728 - val_acc: 0.9247\n"
     ]
    }
   ],
   "source": [
    "# training the model\n",
    "history = model.fit(X_train, Y_train,\n",
    "                    batch_size=BATCH_SIZE,\n",
    "                    epochs=NB_EPOCH,\n",
    "                    verbose=VERBOSE,\n",
    "                    validation_split=VALIDATION_SPLIT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 0s 27us/step\n",
      "Test Score: 0.274649174964\n",
      "Test Accuracy: 0.9227\n"
     ]
    }
   ],
   "source": [
    "# Evaluating the model\n",
    "score = model.evaluate(X_test, Y_test, verbose=VERBOSE)\n",
    "print('Test Score:', score[0])\n",
    "print('Test Accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training Accuracy: 92.45%\n",
    "# Validation Accuracy: 92.47%\n",
    "# Test Accuracy: 92.27%"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Improving the simple net with hidden layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_2 (Dense)              (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 10)                1290      \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 118,282\n",
      "Trainable params: 118,282\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# new network and training\n",
    "model = Sequential()\n",
    "model.add(Dense(N_HIDDEN, input_shape=(RESHAPED,)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(N_HIDDEN))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(NB_CLASSES))\n",
    "model.add(Activation('softmax'))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/250\n",
      "48000/48000 [==============================] - 3s 62us/step - loss: 1.4202 - acc: 0.6652 - val_loss: 0.7082 - val_acc: 0.8483\n",
      "Epoch 2/250\n",
      "48000/48000 [==============================] - 3s 60us/step - loss: 0.5696 - acc: 0.8603 - val_loss: 0.4358 - val_acc: 0.8888\n",
      "Epoch 3/250\n",
      "48000/48000 [==============================] - 3s 60us/step - loss: 0.4244 - acc: 0.8859 - val_loss: 0.3637 - val_acc: 0.9010\n",
      "Epoch 4/250\n",
      "48000/48000 [==============================] - 3s 62us/step - loss: 0.3703 - acc: 0.8976 - val_loss: 0.3282 - val_acc: 0.9072\n",
      "Epoch 5/250\n",
      "48000/48000 [==============================] - 4s 80us/step - loss: 0.3390 - acc: 0.9054 - val_loss: 0.3071 - val_acc: 0.9117\n",
      "Epoch 6/250\n",
      "48000/48000 [==============================] - 3s 69us/step - loss: 0.3168 - acc: 0.9106 - val_loss: 0.2889 - val_acc: 0.9177\n",
      "Epoch 7/250\n",
      "48000/48000 [==============================] - 3s 64us/step - loss: 0.2996 - acc: 0.9154 - val_loss: 0.2762 - val_acc: 0.9216\n",
      "Epoch 8/250\n",
      "48000/48000 [==============================] - 2s 48us/step - loss: 0.2850 - acc: 0.9198 - val_loss: 0.2637 - val_acc: 0.9241\n",
      "Epoch 9/250\n",
      "48000/48000 [==============================] - 3s 56us/step - loss: 0.2724 - acc: 0.9237 - val_loss: 0.2543 - val_acc: 0.9266\n",
      "Epoch 10/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.2609 - acc: 0.9269 - val_loss: 0.2458 - val_acc: 0.9302\n",
      "Epoch 11/250\n",
      "48000/48000 [==============================] - 2s 47us/step - loss: 0.2508 - acc: 0.9292 - val_loss: 0.2370 - val_acc: 0.9330\n",
      "Epoch 12/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.2415 - acc: 0.9316 - val_loss: 0.2300 - val_acc: 0.9353\n",
      "Epoch 13/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.2328 - acc: 0.9340 - val_loss: 0.2231 - val_acc: 0.9370\n",
      "Epoch 14/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.2249 - acc: 0.9365 - val_loss: 0.2171 - val_acc: 0.9389\n",
      "Epoch 15/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.2177 - acc: 0.9384 - val_loss: 0.2103 - val_acc: 0.9408\n",
      "Epoch 16/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.2105 - acc: 0.9409 - val_loss: 0.2049 - val_acc: 0.9423\n",
      "Epoch 17/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.2042 - acc: 0.9421 - val_loss: 0.2004 - val_acc: 0.9446\n",
      "Epoch 18/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.1981 - acc: 0.9437 - val_loss: 0.1956 - val_acc: 0.9453\n",
      "Epoch 19/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.1927 - acc: 0.9456 - val_loss: 0.1898 - val_acc: 0.9479\n",
      "Epoch 20/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.1871 - acc: 0.9476 - val_loss: 0.1862 - val_acc: 0.9487\n",
      "Epoch 21/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.1820 - acc: 0.9485 - val_loss: 0.1819 - val_acc: 0.9500\n",
      "Epoch 22/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.1773 - acc: 0.9500 - val_loss: 0.1780 - val_acc: 0.9518\n",
      "Epoch 23/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.1725 - acc: 0.9516 - val_loss: 0.1754 - val_acc: 0.9518\n",
      "Epoch 24/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.1686 - acc: 0.9527 - val_loss: 0.1715 - val_acc: 0.9534\n",
      "Epoch 25/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.1642 - acc: 0.9538 - val_loss: 0.1698 - val_acc: 0.9532\n",
      "Epoch 26/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.1605 - acc: 0.9543 - val_loss: 0.1653 - val_acc: 0.9550\n",
      "Epoch 27/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.1566 - acc: 0.9558 - val_loss: 0.1654 - val_acc: 0.9560\n",
      "Epoch 28/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.1530 - acc: 0.9573 - val_loss: 0.1592 - val_acc: 0.9567\n",
      "Epoch 29/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.1496 - acc: 0.9577 - val_loss: 0.1574 - val_acc: 0.9569\n",
      "Epoch 30/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.1464 - acc: 0.9590 - val_loss: 0.1563 - val_acc: 0.9571\n",
      "Epoch 31/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.1431 - acc: 0.9603 - val_loss: 0.1523 - val_acc: 0.9580\n",
      "Epoch 32/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.1402 - acc: 0.9602 - val_loss: 0.1506 - val_acc: 0.9597\n",
      "Epoch 33/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.1372 - acc: 0.9615 - val_loss: 0.1494 - val_acc: 0.9600\n",
      "Epoch 34/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.1345 - acc: 0.9624 - val_loss: 0.1464 - val_acc: 0.9597\n",
      "Epoch 35/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.1316 - acc: 0.9630 - val_loss: 0.1443 - val_acc: 0.9603\n",
      "Epoch 36/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.1290 - acc: 0.9633 - val_loss: 0.1425 - val_acc: 0.9605\n",
      "Epoch 37/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.1267 - acc: 0.9644 - val_loss: 0.1398 - val_acc: 0.9610\n",
      "Epoch 38/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.1242 - acc: 0.9649 - val_loss: 0.1390 - val_acc: 0.9609\n",
      "Epoch 39/250\n",
      "48000/48000 [==============================] - 2s 50us/step - loss: 0.1218 - acc: 0.9660 - val_loss: 0.1363 - val_acc: 0.9612\n",
      "Epoch 40/250\n",
      "48000/48000 [==============================] - 3s 55us/step - loss: 0.1196 - acc: 0.9660 - val_loss: 0.1357 - val_acc: 0.9617\n",
      "Epoch 41/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.1174 - acc: 0.9665 - val_loss: 0.1329 - val_acc: 0.9622\n",
      "Epoch 42/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.1151 - acc: 0.9676 - val_loss: 0.1333 - val_acc: 0.9630\n",
      "Epoch 43/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.1130 - acc: 0.9681 - val_loss: 0.1307 - val_acc: 0.9636\n",
      "Epoch 44/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.1111 - acc: 0.9686 - val_loss: 0.1293 - val_acc: 0.9633\n",
      "Epoch 45/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.1090 - acc: 0.9691 - val_loss: 0.1278 - val_acc: 0.9633\n",
      "Epoch 46/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.1072 - acc: 0.9697 - val_loss: 0.1270 - val_acc: 0.9631\n",
      "Epoch 47/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.1053 - acc: 0.9700 - val_loss: 0.1252 - val_acc: 0.9642\n",
      "Epoch 48/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.1037 - acc: 0.9707 - val_loss: 0.1247 - val_acc: 0.9643\n",
      "Epoch 49/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.1018 - acc: 0.9713 - val_loss: 0.1233 - val_acc: 0.9648\n",
      "Epoch 50/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.1002 - acc: 0.9716 - val_loss: 0.1216 - val_acc: 0.9650\n",
      "Epoch 51/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0986 - acc: 0.9722 - val_loss: 0.1209 - val_acc: 0.9655\n",
      "Epoch 52/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.0968 - acc: 0.9726 - val_loss: 0.1195 - val_acc: 0.9652\n",
      "Epoch 53/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0954 - acc: 0.9732 - val_loss: 0.1184 - val_acc: 0.9651\n",
      "Epoch 54/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0937 - acc: 0.9737 - val_loss: 0.1179 - val_acc: 0.9651\n",
      "Epoch 55/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0923 - acc: 0.9745 - val_loss: 0.1176 - val_acc: 0.9652\n",
      "Epoch 56/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0908 - acc: 0.9747 - val_loss: 0.1158 - val_acc: 0.9657\n",
      "Epoch 57/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.0895 - acc: 0.9749 - val_loss: 0.1146 - val_acc: 0.9667\n",
      "Epoch 58/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0880 - acc: 0.9754 - val_loss: 0.1148 - val_acc: 0.9667\n",
      "Epoch 59/250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.0866 - acc: 0.9757 - val_loss: 0.1138 - val_acc: 0.9662\n",
      "Epoch 60/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.0852 - acc: 0.9763 - val_loss: 0.1123 - val_acc: 0.9674\n",
      "Epoch 61/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0838 - acc: 0.9767 - val_loss: 0.1116 - val_acc: 0.9675\n",
      "Epoch 62/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0826 - acc: 0.9772 - val_loss: 0.1104 - val_acc: 0.9677\n",
      "Epoch 63/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0816 - acc: 0.9774 - val_loss: 0.1103 - val_acc: 0.9677\n",
      "Epoch 64/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0803 - acc: 0.9775 - val_loss: 0.1094 - val_acc: 0.9682\n",
      "Epoch 65/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0790 - acc: 0.9784 - val_loss: 0.1093 - val_acc: 0.9677\n",
      "Epoch 66/250\n",
      "48000/48000 [==============================] - 2s 48us/step - loss: 0.0778 - acc: 0.9787 - val_loss: 0.1077 - val_acc: 0.9683\n",
      "Epoch 67/250\n",
      "48000/48000 [==============================] - 2s 52us/step - loss: 0.0767 - acc: 0.9788 - val_loss: 0.1074 - val_acc: 0.9676\n",
      "Epoch 68/250\n",
      "48000/48000 [==============================] - 2s 47us/step - loss: 0.0757 - acc: 0.9795 - val_loss: 0.1064 - val_acc: 0.9684\n",
      "Epoch 69/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0745 - acc: 0.9794 - val_loss: 0.1075 - val_acc: 0.9691\n",
      "Epoch 70/250\n",
      "48000/48000 [==============================] - 2s 47us/step - loss: 0.0735 - acc: 0.9799 - val_loss: 0.1054 - val_acc: 0.9692\n",
      "Epoch 71/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.0722 - acc: 0.9801 - val_loss: 0.1053 - val_acc: 0.9688\n",
      "Epoch 72/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0714 - acc: 0.9805 - val_loss: 0.1043 - val_acc: 0.9692\n",
      "Epoch 73/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0702 - acc: 0.9810 - val_loss: 0.1039 - val_acc: 0.9692\n",
      "Epoch 74/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0692 - acc: 0.9809 - val_loss: 0.1044 - val_acc: 0.9702\n",
      "Epoch 75/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0682 - acc: 0.9812 - val_loss: 0.1033 - val_acc: 0.9692\n",
      "Epoch 76/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.0674 - acc: 0.9819 - val_loss: 0.1022 - val_acc: 0.9696\n",
      "Epoch 77/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0664 - acc: 0.9820 - val_loss: 0.1020 - val_acc: 0.9701\n",
      "Epoch 78/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0656 - acc: 0.9825 - val_loss: 0.1022 - val_acc: 0.9702\n",
      "Epoch 79/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0645 - acc: 0.9825 - val_loss: 0.1021 - val_acc: 0.9694\n",
      "Epoch 80/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0636 - acc: 0.9828 - val_loss: 0.1016 - val_acc: 0.9704\n",
      "Epoch 81/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0629 - acc: 0.9826 - val_loss: 0.1003 - val_acc: 0.9703\n",
      "Epoch 82/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0621 - acc: 0.9832 - val_loss: 0.1010 - val_acc: 0.9700\n",
      "Epoch 83/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.0612 - acc: 0.9837 - val_loss: 0.0989 - val_acc: 0.9706\n",
      "Epoch 84/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0604 - acc: 0.9840 - val_loss: 0.0989 - val_acc: 0.9707\n",
      "Epoch 85/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0595 - acc: 0.9841 - val_loss: 0.0991 - val_acc: 0.9704\n",
      "Epoch 86/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0587 - acc: 0.9843 - val_loss: 0.0992 - val_acc: 0.9706\n",
      "Epoch 87/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0580 - acc: 0.9846 - val_loss: 0.0981 - val_acc: 0.9716\n",
      "Epoch 88/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0572 - acc: 0.9849 - val_loss: 0.0980 - val_acc: 0.9707\n",
      "Epoch 89/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.0563 - acc: 0.9850 - val_loss: 0.0982 - val_acc: 0.9711\n",
      "Epoch 90/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0557 - acc: 0.9854 - val_loss: 0.0986 - val_acc: 0.9715\n",
      "Epoch 91/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0550 - acc: 0.9858 - val_loss: 0.0975 - val_acc: 0.9713\n",
      "Epoch 92/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.0543 - acc: 0.9859 - val_loss: 0.0967 - val_acc: 0.9722\n",
      "Epoch 93/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.0536 - acc: 0.9861 - val_loss: 0.0969 - val_acc: 0.9718\n",
      "Epoch 94/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0528 - acc: 0.9864 - val_loss: 0.0959 - val_acc: 0.9719\n",
      "Epoch 95/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0522 - acc: 0.9864 - val_loss: 0.0956 - val_acc: 0.9726\n",
      "Epoch 96/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0514 - acc: 0.9867 - val_loss: 0.0952 - val_acc: 0.9715\n",
      "Epoch 97/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0508 - acc: 0.9872 - val_loss: 0.0949 - val_acc: 0.9726\n",
      "Epoch 98/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0501 - acc: 0.9873 - val_loss: 0.0949 - val_acc: 0.9724\n",
      "Epoch 99/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0496 - acc: 0.9870 - val_loss: 0.0956 - val_acc: 0.9728\n",
      "Epoch 100/250\n",
      "48000/48000 [==============================] - 2s 47us/step - loss: 0.0490 - acc: 0.9874 - val_loss: 0.0950 - val_acc: 0.9726\n",
      "Epoch 101/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0482 - acc: 0.9878 - val_loss: 0.0939 - val_acc: 0.9729\n",
      "Epoch 102/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0476 - acc: 0.9878 - val_loss: 0.0949 - val_acc: 0.9724\n",
      "Epoch 103/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0471 - acc: 0.9879 - val_loss: 0.0936 - val_acc: 0.9732\n",
      "Epoch 104/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0464 - acc: 0.9883 - val_loss: 0.0927 - val_acc: 0.9737\n",
      "Epoch 105/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0458 - acc: 0.9885 - val_loss: 0.0929 - val_acc: 0.9725\n",
      "Epoch 106/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0453 - acc: 0.9889 - val_loss: 0.0933 - val_acc: 0.9723\n",
      "Epoch 107/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.0447 - acc: 0.9890 - val_loss: 0.0925 - val_acc: 0.9733\n",
      "Epoch 108/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0442 - acc: 0.9894 - val_loss: 0.0925 - val_acc: 0.9727\n",
      "Epoch 109/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0436 - acc: 0.9891 - val_loss: 0.0924 - val_acc: 0.9732\n",
      "Epoch 110/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0430 - acc: 0.9893 - val_loss: 0.0933 - val_acc: 0.9728\n",
      "Epoch 111/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0425 - acc: 0.9898 - val_loss: 0.0923 - val_acc: 0.9736\n",
      "Epoch 112/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0419 - acc: 0.9899 - val_loss: 0.0921 - val_acc: 0.9733\n",
      "Epoch 113/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0415 - acc: 0.9901 - val_loss: 0.0921 - val_acc: 0.9733\n",
      "Epoch 114/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0409 - acc: 0.9902 - val_loss: 0.0920 - val_acc: 0.9729\n",
      "Epoch 115/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0403 - acc: 0.9904 - val_loss: 0.0915 - val_acc: 0.9731\n",
      "Epoch 116/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0400 - acc: 0.9905 - val_loss: 0.0920 - val_acc: 0.9729\n",
      "Epoch 117/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0394 - acc: 0.9906 - val_loss: 0.0922 - val_acc: 0.9737\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 118/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.0390 - acc: 0.9906 - val_loss: 0.0912 - val_acc: 0.9735\n",
      "Epoch 119/250\n",
      "48000/48000 [==============================] - 2s 38us/step - loss: 0.0384 - acc: 0.9908 - val_loss: 0.0913 - val_acc: 0.9735\n",
      "Epoch 120/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0380 - acc: 0.9910 - val_loss: 0.0909 - val_acc: 0.9740\n",
      "Epoch 121/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.0376 - acc: 0.9910 - val_loss: 0.0908 - val_acc: 0.9737\n",
      "Epoch 122/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.0371 - acc: 0.9912 - val_loss: 0.0907 - val_acc: 0.9733\n",
      "Epoch 123/250\n",
      "48000/48000 [==============================] - 2s 49us/step - loss: 0.0366 - acc: 0.9916 - val_loss: 0.0914 - val_acc: 0.9731\n",
      "Epoch 124/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0363 - acc: 0.9914 - val_loss: 0.0905 - val_acc: 0.9737\n",
      "Epoch 125/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.0357 - acc: 0.9915 - val_loss: 0.0911 - val_acc: 0.9737\n",
      "Epoch 126/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0353 - acc: 0.9916 - val_loss: 0.0906 - val_acc: 0.9737\n",
      "Epoch 127/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0349 - acc: 0.9921 - val_loss: 0.0907 - val_acc: 0.9739\n",
      "Epoch 128/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0345 - acc: 0.9922 - val_loss: 0.0904 - val_acc: 0.9735\n",
      "Epoch 129/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0341 - acc: 0.9920 - val_loss: 0.0900 - val_acc: 0.9737\n",
      "Epoch 130/250\n",
      "48000/48000 [==============================] - 2s 48us/step - loss: 0.0336 - acc: 0.9923 - val_loss: 0.0899 - val_acc: 0.9743\n",
      "Epoch 131/250\n",
      "48000/48000 [==============================] - 3s 62us/step - loss: 0.0332 - acc: 0.9925 - val_loss: 0.0903 - val_acc: 0.9732\n",
      "Epoch 132/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0329 - acc: 0.9925 - val_loss: 0.0900 - val_acc: 0.9740\n",
      "Epoch 133/250\n",
      "48000/48000 [==============================] - 2s 48us/step - loss: 0.0325 - acc: 0.9927 - val_loss: 0.0906 - val_acc: 0.9735\n",
      "Epoch 134/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0320 - acc: 0.9926 - val_loss: 0.0904 - val_acc: 0.9733\n",
      "Epoch 135/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0317 - acc: 0.9929 - val_loss: 0.0903 - val_acc: 0.9738\n",
      "Epoch 136/250\n",
      "48000/48000 [==============================] - 2s 49us/step - loss: 0.0313 - acc: 0.9930 - val_loss: 0.0895 - val_acc: 0.9742\n",
      "Epoch 137/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0309 - acc: 0.9932 - val_loss: 0.0902 - val_acc: 0.9738\n",
      "Epoch 138/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0306 - acc: 0.9936 - val_loss: 0.0904 - val_acc: 0.9735\n",
      "Epoch 139/250\n",
      "48000/48000 [==============================] - 2s 47us/step - loss: 0.0301 - acc: 0.9934 - val_loss: 0.0899 - val_acc: 0.9738\n",
      "Epoch 140/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0298 - acc: 0.9936 - val_loss: 0.0901 - val_acc: 0.9738\n",
      "Epoch 141/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.0294 - acc: 0.9939 - val_loss: 0.0901 - val_acc: 0.9736\n",
      "Epoch 142/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0292 - acc: 0.9937 - val_loss: 0.0898 - val_acc: 0.9742\n",
      "Epoch 143/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0289 - acc: 0.9939 - val_loss: 0.0895 - val_acc: 0.9738\n",
      "Epoch 144/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0285 - acc: 0.9943 - val_loss: 0.0903 - val_acc: 0.9734\n",
      "Epoch 145/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0280 - acc: 0.9941 - val_loss: 0.0901 - val_acc: 0.9740\n",
      "Epoch 146/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0278 - acc: 0.9943 - val_loss: 0.0901 - val_acc: 0.9741\n",
      "Epoch 147/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0274 - acc: 0.9945 - val_loss: 0.0894 - val_acc: 0.9741\n",
      "Epoch 148/250\n",
      "48000/48000 [==============================] - ETA: 0s - loss: 0.0271 - acc: 0.994 - 2s 40us/step - loss: 0.0271 - acc: 0.9945 - val_loss: 0.0901 - val_acc: 0.9739\n",
      "Epoch 149/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0267 - acc: 0.9945 - val_loss: 0.0901 - val_acc: 0.9737\n",
      "Epoch 150/250\n",
      "48000/48000 [==============================] - 2s 47us/step - loss: 0.0265 - acc: 0.9947 - val_loss: 0.0903 - val_acc: 0.9741\n",
      "Epoch 151/250\n",
      "48000/48000 [==============================] - 2s 47us/step - loss: 0.0261 - acc: 0.9947 - val_loss: 0.0900 - val_acc: 0.9745\n",
      "Epoch 152/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0259 - acc: 0.9949 - val_loss: 0.0906 - val_acc: 0.9741\n",
      "Epoch 153/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0256 - acc: 0.9947 - val_loss: 0.0891 - val_acc: 0.9746\n",
      "Epoch 154/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.0253 - acc: 0.9951 - val_loss: 0.0888 - val_acc: 0.9747\n",
      "Epoch 155/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0250 - acc: 0.9954 - val_loss: 0.0889 - val_acc: 0.9742\n",
      "Epoch 156/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.0246 - acc: 0.9955 - val_loss: 0.0901 - val_acc: 0.9736\n",
      "Epoch 157/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.0244 - acc: 0.9955 - val_loss: 0.0896 - val_acc: 0.9743\n",
      "Epoch 158/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0241 - acc: 0.9958 - val_loss: 0.0905 - val_acc: 0.9738\n",
      "Epoch 159/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.0239 - acc: 0.9958 - val_loss: 0.0893 - val_acc: 0.9745\n",
      "Epoch 160/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.0236 - acc: 0.9957 - val_loss: 0.0896 - val_acc: 0.9743\n",
      "Epoch 161/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0233 - acc: 0.9959 - val_loss: 0.0895 - val_acc: 0.9743\n",
      "Epoch 162/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0230 - acc: 0.9958 - val_loss: 0.0894 - val_acc: 0.9745\n",
      "Epoch 163/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0228 - acc: 0.9961 - val_loss: 0.0898 - val_acc: 0.9746\n",
      "Epoch 164/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0225 - acc: 0.9962 - val_loss: 0.0900 - val_acc: 0.9745\n",
      "Epoch 165/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0222 - acc: 0.9963 - val_loss: 0.0900 - val_acc: 0.9743\n",
      "Epoch 166/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0220 - acc: 0.9962 - val_loss: 0.0896 - val_acc: 0.9745\n",
      "Epoch 167/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0217 - acc: 0.9964 - val_loss: 0.0891 - val_acc: 0.9748\n",
      "Epoch 168/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0215 - acc: 0.9963 - val_loss: 0.0890 - val_acc: 0.9751\n",
      "Epoch 169/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.0212 - acc: 0.9967 - val_loss: 0.0904 - val_acc: 0.9746\n",
      "Epoch 170/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0211 - acc: 0.9965 - val_loss: 0.0890 - val_acc: 0.9749\n",
      "Epoch 171/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0208 - acc: 0.9966 - val_loss: 0.0898 - val_acc: 0.9746\n",
      "Epoch 172/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.0205 - acc: 0.9967 - val_loss: 0.0895 - val_acc: 0.9746\n",
      "Epoch 173/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0203 - acc: 0.9967 - val_loss: 0.0894 - val_acc: 0.9747\n",
      "Epoch 174/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.0201 - acc: 0.9968 - val_loss: 0.0892 - val_acc: 0.9747\n",
      "Epoch 175/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0198 - acc: 0.9971 - val_loss: 0.0901 - val_acc: 0.9746\n",
      "Epoch 176/250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0196 - acc: 0.9970 - val_loss: 0.0893 - val_acc: 0.9752\n",
      "Epoch 177/250\n",
      "48000/48000 [==============================] - 2s 50us/step - loss: 0.0194 - acc: 0.9971 - val_loss: 0.0900 - val_acc: 0.9751\n",
      "Epoch 178/250\n",
      "48000/48000 [==============================] - 3s 57us/step - loss: 0.0191 - acc: 0.9972 - val_loss: 0.0900 - val_acc: 0.9747\n",
      "Epoch 179/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0190 - acc: 0.9971 - val_loss: 0.0907 - val_acc: 0.9743\n",
      "Epoch 180/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0187 - acc: 0.9973 - val_loss: 0.0901 - val_acc: 0.9755\n",
      "Epoch 181/250\n",
      "48000/48000 [==============================] - 2s 47us/step - loss: 0.0186 - acc: 0.9972 - val_loss: 0.0900 - val_acc: 0.9743\n",
      "Epoch 182/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0183 - acc: 0.9973 - val_loss: 0.0894 - val_acc: 0.9750\n",
      "Epoch 183/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.0181 - acc: 0.9975 - val_loss: 0.0899 - val_acc: 0.9745\n",
      "Epoch 184/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0179 - acc: 0.9973 - val_loss: 0.0901 - val_acc: 0.9746\n",
      "Epoch 185/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0178 - acc: 0.9975 - val_loss: 0.0904 - val_acc: 0.9749\n",
      "Epoch 186/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.0175 - acc: 0.9977 - val_loss: 0.0907 - val_acc: 0.9745\n",
      "Epoch 187/250\n",
      "48000/48000 [==============================] - 2s 50us/step - loss: 0.0173 - acc: 0.9975 - val_loss: 0.0900 - val_acc: 0.9748\n",
      "Epoch 188/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.0171 - acc: 0.9976 - val_loss: 0.0900 - val_acc: 0.9750\n",
      "Epoch 189/250\n",
      "48000/48000 [==============================] - 2s 49us/step - loss: 0.0170 - acc: 0.9975 - val_loss: 0.0903 - val_acc: 0.9750\n",
      "Epoch 190/250\n",
      "48000/48000 [==============================] - 2s 48us/step - loss: 0.0167 - acc: 0.9977 - val_loss: 0.0900 - val_acc: 0.9748\n",
      "Epoch 191/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0165 - acc: 0.9978 - val_loss: 0.0900 - val_acc: 0.9748\n",
      "Epoch 192/250\n",
      "48000/48000 [==============================] - 3s 56us/step - loss: 0.0164 - acc: 0.9977 - val_loss: 0.0899 - val_acc: 0.9748\n",
      "Epoch 193/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0162 - acc: 0.9978 - val_loss: 0.0897 - val_acc: 0.9752\n",
      "Epoch 194/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0161 - acc: 0.9978 - val_loss: 0.0907 - val_acc: 0.9743\n",
      "Epoch 195/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0158 - acc: 0.9980 - val_loss: 0.0904 - val_acc: 0.9747\n",
      "Epoch 196/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0158 - acc: 0.9979 - val_loss: 0.0902 - val_acc: 0.9751\n",
      "Epoch 197/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0155 - acc: 0.9980 - val_loss: 0.0901 - val_acc: 0.9751\n",
      "Epoch 198/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0154 - acc: 0.9979 - val_loss: 0.0909 - val_acc: 0.9745\n",
      "Epoch 199/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0152 - acc: 0.9980 - val_loss: 0.0901 - val_acc: 0.9751\n",
      "Epoch 200/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0150 - acc: 0.9981 - val_loss: 0.0903 - val_acc: 0.9750\n",
      "Epoch 201/250\n",
      "48000/48000 [==============================] - 2s 49us/step - loss: 0.0148 - acc: 0.9981 - val_loss: 0.0905 - val_acc: 0.9746\n",
      "Epoch 202/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0148 - acc: 0.9981 - val_loss: 0.0905 - val_acc: 0.9753\n",
      "Epoch 203/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0146 - acc: 0.9982 - val_loss: 0.0912 - val_acc: 0.9744\n",
      "Epoch 204/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0144 - acc: 0.9984 - val_loss: 0.0911 - val_acc: 0.9749\n",
      "Epoch 205/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.0143 - acc: 0.9983 - val_loss: 0.0906 - val_acc: 0.9753\n",
      "Epoch 206/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0142 - acc: 0.9982 - val_loss: 0.0909 - val_acc: 0.9751\n",
      "Epoch 207/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.0140 - acc: 0.9983 - val_loss: 0.0911 - val_acc: 0.9752\n",
      "Epoch 208/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0138 - acc: 0.9984 - val_loss: 0.0907 - val_acc: 0.9750\n",
      "Epoch 209/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0137 - acc: 0.9983 - val_loss: 0.0907 - val_acc: 0.9747\n",
      "Epoch 210/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.0135 - acc: 0.9983 - val_loss: 0.0915 - val_acc: 0.9745\n",
      "Epoch 211/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0134 - acc: 0.9985 - val_loss: 0.0913 - val_acc: 0.9746\n",
      "Epoch 212/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0133 - acc: 0.9984 - val_loss: 0.0909 - val_acc: 0.9748\n",
      "Epoch 213/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0131 - acc: 0.9984 - val_loss: 0.0911 - val_acc: 0.9753\n",
      "Epoch 214/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0130 - acc: 0.9985 - val_loss: 0.0914 - val_acc: 0.9753\n",
      "Epoch 215/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0129 - acc: 0.9985 - val_loss: 0.0915 - val_acc: 0.9748\n",
      "Epoch 216/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0127 - acc: 0.9986 - val_loss: 0.0913 - val_acc: 0.9750\n",
      "Epoch 217/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0126 - acc: 0.9986 - val_loss: 0.0918 - val_acc: 0.9750\n",
      "Epoch 218/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0125 - acc: 0.9986 - val_loss: 0.0912 - val_acc: 0.9747\n",
      "Epoch 219/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.0124 - acc: 0.9985 - val_loss: 0.0916 - val_acc: 0.9749\n",
      "Epoch 220/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0122 - acc: 0.9987 - val_loss: 0.0920 - val_acc: 0.9749\n",
      "Epoch 221/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0121 - acc: 0.9987 - val_loss: 0.0917 - val_acc: 0.9752\n",
      "Epoch 222/250\n",
      "48000/48000 [==============================] - 2s 39us/step - loss: 0.0120 - acc: 0.9988 - val_loss: 0.0917 - val_acc: 0.9747\n",
      "Epoch 223/250\n",
      "48000/48000 [==============================] - 2s 44us/step - loss: 0.0119 - acc: 0.9988 - val_loss: 0.0921 - val_acc: 0.9748\n",
      "Epoch 224/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0117 - acc: 0.9989 - val_loss: 0.0918 - val_acc: 0.9752\n",
      "Epoch 225/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0116 - acc: 0.9988 - val_loss: 0.0911 - val_acc: 0.9751\n",
      "Epoch 226/250\n",
      "48000/48000 [==============================] - 2s 47us/step - loss: 0.0115 - acc: 0.9989 - val_loss: 0.0922 - val_acc: 0.9748\n",
      "Epoch 227/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0114 - acc: 0.9989 - val_loss: 0.0918 - val_acc: 0.9750\n",
      "Epoch 228/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0113 - acc: 0.9989 - val_loss: 0.0924 - val_acc: 0.9750\n",
      "Epoch 229/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0112 - acc: 0.9989 - val_loss: 0.0924 - val_acc: 0.9748\n",
      "Epoch 230/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0111 - acc: 0.9989 - val_loss: 0.0930 - val_acc: 0.9751\n",
      "Epoch 231/250\n",
      "48000/48000 [==============================] - 2s 42us/step - loss: 0.0110 - acc: 0.9990 - val_loss: 0.0923 - val_acc: 0.9756\n",
      "Epoch 232/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.0109 - acc: 0.9989 - val_loss: 0.0922 - val_acc: 0.9752\n",
      "Epoch 233/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0108 - acc: 0.9990 - val_loss: 0.0924 - val_acc: 0.9749\n",
      "Epoch 234/250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0107 - acc: 0.9990 - val_loss: 0.0926 - val_acc: 0.9752\n",
      "Epoch 235/250\n",
      "48000/48000 [==============================] - 2s 48us/step - loss: 0.0105 - acc: 0.9989 - val_loss: 0.0929 - val_acc: 0.9751\n",
      "Epoch 236/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0104 - acc: 0.9991 - val_loss: 0.0926 - val_acc: 0.9752\n",
      "Epoch 237/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0104 - acc: 0.9990 - val_loss: 0.0927 - val_acc: 0.9751\n",
      "Epoch 238/250\n",
      "48000/48000 [==============================] - 2s 47us/step - loss: 0.0103 - acc: 0.9992 - val_loss: 0.0926 - val_acc: 0.9749\n",
      "Epoch 239/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0102 - acc: 0.9991 - val_loss: 0.0930 - val_acc: 0.9751\n",
      "Epoch 240/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0101 - acc: 0.9991 - val_loss: 0.0926 - val_acc: 0.9756\n",
      "Epoch 241/250\n",
      "48000/48000 [==============================] - 2s 46us/step - loss: 0.0100 - acc: 0.9992 - val_loss: 0.0932 - val_acc: 0.9752\n",
      "Epoch 242/250\n",
      "48000/48000 [==============================] - 2s 40us/step - loss: 0.0099 - acc: 0.9992 - val_loss: 0.0931 - val_acc: 0.9752\n",
      "Epoch 243/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0098 - acc: 0.9992 - val_loss: 0.0933 - val_acc: 0.9757\n",
      "Epoch 244/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0097 - acc: 0.9992 - val_loss: 0.0930 - val_acc: 0.9756\n",
      "Epoch 245/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0096 - acc: 0.9992 - val_loss: 0.0929 - val_acc: 0.9748\n",
      "Epoch 246/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0095 - acc: 0.9992 - val_loss: 0.0930 - val_acc: 0.9745\n",
      "Epoch 247/250\n",
      "48000/48000 [==============================] - 2s 45us/step - loss: 0.0094 - acc: 0.9992 - val_loss: 0.0933 - val_acc: 0.9752\n",
      "Epoch 248/250\n",
      "48000/48000 [==============================] - 2s 43us/step - loss: 0.0093 - acc: 0.9993 - val_loss: 0.0934 - val_acc: 0.9751\n",
      "Epoch 249/250\n",
      "48000/48000 [==============================] - 2s 47us/step - loss: 0.0093 - acc: 0.9993 - val_loss: 0.0935 - val_acc: 0.9752\n",
      "Epoch 250/250\n",
      "48000/48000 [==============================] - 2s 41us/step - loss: 0.0092 - acc: 0.9994 - val_loss: 0.0930 - val_acc: 0.9751\n"
     ]
    }
   ],
   "source": [
    "# compiling the model\n",
    "model.compile(loss='categorical_crossentropy', optimizer=OPTIMIZER, metrics=['accuracy'])\n",
    "\n",
    "# training the model\n",
    "history = model.fit(X_train, Y_train,\n",
    "                   batch_size=BATCH_SIZE,\n",
    "                   epochs=NB_EPOCH,\n",
    "                   verbose=VERBOSE,\n",
    "                   validation_split=VALIDATION_SPLIT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 0s 45us/step\n",
      "Test score: 0.0883470876337\n",
      "Test accuracy: 0.9767\n"
     ]
    }
   ],
   "source": [
    "# Evaluating the model\n",
    "score = model.evaluate(X_test, Y_test, verbose=VERBOSE)\n",
    "print('Test score:',score[0])\n",
    "print('Test accuracy:',score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training Accuracy: 99.94%   -- Overfitting\n",
    "# Validation Accuracy: 97.51%\n",
    "# Test Accuracy: 97.67%"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Further improving the simple net with Dropout (Regularization)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers.core import Dropout\n",
    "\n",
    "DROPOUT = 0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_5 (Dense)              (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "activation_5 (Activation)    (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "activation_6 (Activation)    (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 10)                1290      \n",
      "_________________________________________________________________\n",
      "activation_7 (Activation)    (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 118,282\n",
      "Trainable params: 118,282\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# building the model\n",
    "model = Sequential()\n",
    "model.add(Dense(N_HIDDEN, input_shape=(RESHAPED,)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(DROPOUT))\n",
    "model.add(Dense(N_HIDDEN))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(DROPOUT))\n",
    "model.add(Dense(NB_CLASSES))\n",
    "model.add(Activation('softmax'))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/250\n",
      "48000/48000 [==============================] - 4s 73us/step - loss: 1.7249 - acc: 0.4526 - val_loss: 0.9304 - val_acc: 0.7893\n",
      "Epoch 2/250\n",
      "48000/48000 [==============================] - 4s 76us/step - loss: 0.9429 - acc: 0.7073 - val_loss: 0.5476 - val_acc: 0.8652\n",
      "Epoch 3/250\n",
      "48000/48000 [==============================] - 4s 80us/step - loss: 0.7090 - acc: 0.7811 - val_loss: 0.4289 - val_acc: 0.8884\n",
      "Epoch 4/250\n",
      "48000/48000 [==============================] - 3s 63us/step - loss: 0.6008 - acc: 0.8182 - val_loss: 0.3728 - val_acc: 0.8984\n",
      "Epoch 5/250\n",
      "48000/48000 [==============================] - 4s 86us/step - loss: 0.5356 - acc: 0.8381 - val_loss: 0.3378 - val_acc: 0.9066\n",
      "Epoch 6/250\n",
      "48000/48000 [==============================] - 3s 67us/step - loss: 0.4920 - acc: 0.8532 - val_loss: 0.3143 - val_acc: 0.9105\n",
      "Epoch 7/250\n",
      "48000/48000 [==============================] - 3s 71us/step - loss: 0.4608 - acc: 0.8621 - val_loss: 0.2955 - val_acc: 0.9154\n",
      "Epoch 8/250\n",
      "48000/48000 [==============================] - 4s 93us/step - loss: 0.4356 - acc: 0.8709 - val_loss: 0.2813 - val_acc: 0.9183\n",
      "Epoch 9/250\n",
      "48000/48000 [==============================] - 4s 83us/step - loss: 0.4120 - acc: 0.8798 - val_loss: 0.2677 - val_acc: 0.9220\n",
      "Epoch 10/250\n",
      "48000/48000 [==============================] - 4s 91us/step - loss: 0.3944 - acc: 0.8820 - val_loss: 0.2563 - val_acc: 0.9242\n",
      "Epoch 11/250\n",
      "48000/48000 [==============================] - 3s 69us/step - loss: 0.3781 - acc: 0.8893 - val_loss: 0.2481 - val_acc: 0.9267\n",
      "Epoch 12/250\n",
      "48000/48000 [==============================] - 4s 74us/step - loss: 0.3616 - acc: 0.8939 - val_loss: 0.2381 - val_acc: 0.9295\n",
      "Epoch 13/250\n",
      "48000/48000 [==============================] - 3s 69us/step - loss: 0.3555 - acc: 0.8951 - val_loss: 0.2309 - val_acc: 0.9312\n",
      "Epoch 14/250\n",
      "48000/48000 [==============================] - 3s 63us/step - loss: 0.3419 - acc: 0.8987 - val_loss: 0.2250 - val_acc: 0.9323\n",
      "Epoch 15/250\n",
      "48000/48000 [==============================] - 3s 71us/step - loss: 0.3334 - acc: 0.9015 - val_loss: 0.2167 - val_acc: 0.9340\n",
      "Epoch 16/250\n",
      "48000/48000 [==============================] - 3s 55us/step - loss: 0.3209 - acc: 0.9074 - val_loss: 0.2112 - val_acc: 0.9368\n",
      "Epoch 17/250\n",
      "48000/48000 [==============================] - 4s 93us/step - loss: 0.3135 - acc: 0.9079 - val_loss: 0.2053 - val_acc: 0.9385\n",
      "Epoch 18/250\n",
      "48000/48000 [==============================] - 4s 75us/step - loss: 0.3026 - acc: 0.9114 - val_loss: 0.1996 - val_acc: 0.9406\n",
      "Epoch 19/250\n",
      "48000/48000 [==============================] - 4s 84us/step - loss: 0.3002 - acc: 0.9125 - val_loss: 0.1955 - val_acc: 0.9417\n",
      "Epoch 20/250\n",
      "48000/48000 [==============================] - 4s 74us/step - loss: 0.2905 - acc: 0.9146 - val_loss: 0.1907 - val_acc: 0.9431\n",
      "Epoch 21/250\n",
      "48000/48000 [==============================] - 4s 78us/step - loss: 0.2809 - acc: 0.9165 - val_loss: 0.1865 - val_acc: 0.9435\n",
      "Epoch 22/250\n",
      "48000/48000 [==============================] - 3s 56us/step - loss: 0.2758 - acc: 0.9195 - val_loss: 0.1829 - val_acc: 0.9452\n",
      "Epoch 23/250\n",
      "48000/48000 [==============================] - 3s 65us/step - loss: 0.2703 - acc: 0.9199 - val_loss: 0.1790 - val_acc: 0.9458\n",
      "Epoch 24/250\n",
      "48000/48000 [==============================] - 3s 56us/step - loss: 0.2655 - acc: 0.9230 - val_loss: 0.1743 - val_acc: 0.9482\n",
      "Epoch 25/250\n",
      "48000/48000 [==============================] - 4s 74us/step - loss: 0.2607 - acc: 0.9226 - val_loss: 0.1718 - val_acc: 0.9496\n",
      "Epoch 26/250\n",
      "48000/48000 [==============================] - 4s 85us/step - loss: 0.2549 - acc: 0.9253 - val_loss: 0.1690 - val_acc: 0.9496\n",
      "Epoch 27/250\n",
      "48000/48000 [==============================] - 4s 82us/step - loss: 0.2501 - acc: 0.9266 - val_loss: 0.1654 - val_acc: 0.9512\n",
      "Epoch 28/250\n",
      "48000/48000 [==============================] - 3s 56us/step - loss: 0.2473 - acc: 0.9282 - val_loss: 0.1625 - val_acc: 0.9527\n",
      "Epoch 29/250\n",
      "48000/48000 [==============================] - 3s 65us/step - loss: 0.2418 - acc: 0.9288 - val_loss: 0.1596 - val_acc: 0.9529\n",
      "Epoch 30/250\n",
      "48000/48000 [==============================] - 3s 57us/step - loss: 0.2400 - acc: 0.9295 - val_loss: 0.1578 - val_acc: 0.9539\n",
      "Epoch 31/250\n",
      "48000/48000 [==============================] - 3s 64us/step - loss: 0.2331 - acc: 0.9315 - val_loss: 0.1548 - val_acc: 0.9549\n",
      "Epoch 32/250\n",
      "48000/48000 [==============================] - 3s 57us/step - loss: 0.2270 - acc: 0.9324 - val_loss: 0.1520 - val_acc: 0.9556\n",
      "Epoch 33/250\n",
      "48000/48000 [==============================] - 3s 65us/step - loss: 0.2297 - acc: 0.9323 - val_loss: 0.1502 - val_acc: 0.9565\n",
      "Epoch 34/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.2248 - acc: 0.9334 - val_loss: 0.1484 - val_acc: 0.9568\n",
      "Epoch 35/250\n",
      "48000/48000 [==============================] - 3s 63us/step - loss: 0.2198 - acc: 0.9358 - val_loss: 0.1455 - val_acc: 0.9577\n",
      "Epoch 36/250\n",
      "48000/48000 [==============================] - 3s 54us/step - loss: 0.2176 - acc: 0.9363 - val_loss: 0.1442 - val_acc: 0.9583\n",
      "Epoch 37/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.2159 - acc: 0.9360 - val_loss: 0.1423 - val_acc: 0.9583\n",
      "Epoch 38/250\n",
      "48000/48000 [==============================] - 3s 56us/step - loss: 0.2126 - acc: 0.9369 - val_loss: 0.1405 - val_acc: 0.9589\n",
      "Epoch 39/250\n",
      "48000/48000 [==============================] - 3s 54us/step - loss: 0.2055 - acc: 0.9397 - val_loss: 0.1384 - val_acc: 0.9597\n",
      "Epoch 40/250\n",
      "48000/48000 [==============================] - 3s 60us/step - loss: 0.2032 - acc: 0.9405 - val_loss: 0.1364 - val_acc: 0.9613\n",
      "Epoch 41/250\n",
      "48000/48000 [==============================] - 3s 54us/step - loss: 0.2018 - acc: 0.9401 - val_loss: 0.1356 - val_acc: 0.9602\n",
      "Epoch 42/250\n",
      "48000/48000 [==============================] - 3s 61us/step - loss: 0.1991 - acc: 0.9419 - val_loss: 0.1339 - val_acc: 0.9613\n",
      "Epoch 43/250\n",
      "48000/48000 [==============================] - 3s 54us/step - loss: 0.1969 - acc: 0.9431 - val_loss: 0.1330 - val_acc: 0.9618\n",
      "Epoch 44/250\n",
      "48000/48000 [==============================] - 3s 62us/step - loss: 0.1984 - acc: 0.9422 - val_loss: 0.1307 - val_acc: 0.9623\n",
      "Epoch 45/250\n",
      "48000/48000 [==============================] - 3s 55us/step - loss: 0.1931 - acc: 0.9427 - val_loss: 0.1302 - val_acc: 0.9629\n",
      "Epoch 46/250\n",
      "48000/48000 [==============================] - 3s 60us/step - loss: 0.1900 - acc: 0.9453 - val_loss: 0.1283 - val_acc: 0.9645\n",
      "Epoch 47/250\n",
      "48000/48000 [==============================] - 3s 54us/step - loss: 0.1893 - acc: 0.9447 - val_loss: 0.1279 - val_acc: 0.9637\n",
      "Epoch 48/250\n",
      "48000/48000 [==============================] - 3s 60us/step - loss: 0.1872 - acc: 0.9441 - val_loss: 0.1262 - val_acc: 0.9640\n",
      "Epoch 49/250\n",
      "48000/48000 [==============================] - 3s 54us/step - loss: 0.1878 - acc: 0.9441 - val_loss: 0.1244 - val_acc: 0.9647\n",
      "Epoch 50/250\n",
      "48000/48000 [==============================] - 3s 56us/step - loss: 0.1822 - acc: 0.9457 - val_loss: 0.1237 - val_acc: 0.9646\n",
      "Epoch 51/250\n",
      "48000/48000 [==============================] - 3s 57us/step - loss: 0.1789 - acc: 0.9473 - val_loss: 0.1226 - val_acc: 0.9649\n",
      "Epoch 52/250\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.1786 - acc: 0.9475 - val_loss: 0.1215 - val_acc: 0.9654\n",
      "Epoch 53/250\n",
      "48000/48000 [==============================] - 3s 61us/step - loss: 0.1791 - acc: 0.9475 - val_loss: 0.1204 - val_acc: 0.9657\n",
      "Epoch 54/250\n",
      "48000/48000 [==============================] - 3s 54us/step - loss: 0.1725 - acc: 0.9488 - val_loss: 0.1195 - val_acc: 0.9657\n",
      "Epoch 55/250\n",
      "48000/48000 [==============================] - 3s 61us/step - loss: 0.1702 - acc: 0.9501 - val_loss: 0.1183 - val_acc: 0.9658\n",
      "Epoch 56/250\n",
      "48000/48000 [==============================] - 3s 54us/step - loss: 0.1694 - acc: 0.9490 - val_loss: 0.1172 - val_acc: 0.9668\n",
      "Epoch 57/250\n",
      "48000/48000 [==============================] - 3s 61us/step - loss: 0.1702 - acc: 0.9491 - val_loss: 0.1164 - val_acc: 0.9665\n",
      "Epoch 58/250\n",
      "48000/48000 [==============================] - 3s 54us/step - loss: 0.1683 - acc: 0.9500 - val_loss: 0.1155 - val_acc: 0.9669\n",
      "Epoch 59/250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.1657 - acc: 0.9510 - val_loss: 0.1145 - val_acc: 0.9668\n",
      "Epoch 60/250\n",
      "48000/48000 [==============================] - 2s 52us/step - loss: 0.1656 - acc: 0.9509 - val_loss: 0.1134 - val_acc: 0.9680\n",
      "Epoch 61/250\n",
      "48000/48000 [==============================] - 3s 60us/step - loss: 0.1610 - acc: 0.9530 - val_loss: 0.1121 - val_acc: 0.9678\n",
      "Epoch 62/250\n",
      "48000/48000 [==============================] - 3s 56us/step - loss: 0.1610 - acc: 0.9524 - val_loss: 0.1119 - val_acc: 0.9681\n",
      "Epoch 63/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.1592 - acc: 0.9524 - val_loss: 0.1110 - val_acc: 0.9681\n",
      "Epoch 64/250\n",
      "48000/48000 [==============================] - 3s 55us/step - loss: 0.1598 - acc: 0.9532 - val_loss: 0.1105 - val_acc: 0.9685\n",
      "Epoch 65/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.1580 - acc: 0.9536 - val_loss: 0.1093 - val_acc: 0.9689\n",
      "Epoch 66/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.1568 - acc: 0.9531 - val_loss: 0.1091 - val_acc: 0.9689\n",
      "Epoch 67/250\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.1537 - acc: 0.9544 - val_loss: 0.1086 - val_acc: 0.9683\n",
      "Epoch 68/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.1526 - acc: 0.9554 - val_loss: 0.1072 - val_acc: 0.9693\n",
      "Epoch 69/250\n",
      "48000/48000 [==============================] - 2s 52us/step - loss: 0.1524 - acc: 0.9552 - val_loss: 0.1074 - val_acc: 0.9692\n",
      "Epoch 70/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.1511 - acc: 0.9550 - val_loss: 0.1065 - val_acc: 0.9692\n",
      "Epoch 71/250\n",
      "48000/48000 [==============================] - 2s 52us/step - loss: 0.1513 - acc: 0.9556 - val_loss: 0.1055 - val_acc: 0.9697\n",
      "Epoch 72/250\n",
      "48000/48000 [==============================] - 3s 55us/step - loss: 0.1498 - acc: 0.9564 - val_loss: 0.1050 - val_acc: 0.9701\n",
      "Epoch 73/250\n",
      "48000/48000 [==============================] - 3s 55us/step - loss: 0.1469 - acc: 0.9569 - val_loss: 0.1049 - val_acc: 0.9712\n",
      "Epoch 74/250\n",
      "48000/48000 [==============================] - 2s 52us/step - loss: 0.1453 - acc: 0.9574 - val_loss: 0.1043 - val_acc: 0.9702\n",
      "Epoch 75/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.1446 - acc: 0.9570 - val_loss: 0.1041 - val_acc: 0.9704\n",
      "Epoch 76/250\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.1410 - acc: 0.9580 - val_loss: 0.1030 - val_acc: 0.9708\n",
      "Epoch 77/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.1411 - acc: 0.9577 - val_loss: 0.1029 - val_acc: 0.9710\n",
      "Epoch 78/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.1411 - acc: 0.9580 - val_loss: 0.1025 - val_acc: 0.9713\n",
      "Epoch 79/250\n",
      "48000/48000 [==============================] - 3s 59us/step - loss: 0.1392 - acc: 0.9585 - val_loss: 0.1015 - val_acc: 0.9714\n",
      "Epoch 80/250\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.1403 - acc: 0.9589 - val_loss: 0.1011 - val_acc: 0.9718\n",
      "Epoch 81/250\n",
      "48000/48000 [==============================] - 3s 57us/step - loss: 0.1370 - acc: 0.9595 - val_loss: 0.1007 - val_acc: 0.9717\n",
      "Epoch 82/250\n",
      "48000/48000 [==============================] - 3s 52us/step - loss: 0.1360 - acc: 0.9589 - val_loss: 0.0998 - val_acc: 0.9718\n",
      "Epoch 83/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.1326 - acc: 0.9607 - val_loss: 0.0996 - val_acc: 0.9719\n",
      "Epoch 84/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.1339 - acc: 0.9602 - val_loss: 0.0994 - val_acc: 0.9717\n",
      "Epoch 85/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.1338 - acc: 0.9595 - val_loss: 0.0989 - val_acc: 0.9718\n",
      "Epoch 86/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.1322 - acc: 0.9613 - val_loss: 0.0987 - val_acc: 0.9716\n",
      "Epoch 87/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.1308 - acc: 0.9615 - val_loss: 0.0981 - val_acc: 0.9719\n",
      "Epoch 88/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.1307 - acc: 0.9609 - val_loss: 0.0977 - val_acc: 0.9720\n",
      "Epoch 89/250\n",
      "48000/48000 [==============================] - 2s 52us/step - loss: 0.1312 - acc: 0.9612 - val_loss: 0.0972 - val_acc: 0.9723\n",
      "Epoch 90/250\n",
      "48000/48000 [==============================] - 3s 56us/step - loss: 0.1270 - acc: 0.9614 - val_loss: 0.0971 - val_acc: 0.9722\n",
      "Epoch 91/250\n",
      "48000/48000 [==============================] - 3s 57us/step - loss: 0.1292 - acc: 0.9619 - val_loss: 0.0968 - val_acc: 0.9721\n",
      "Epoch 92/250\n",
      "48000/48000 [==============================] - 2s 52us/step - loss: 0.1296 - acc: 0.9614 - val_loss: 0.0962 - val_acc: 0.9722\n",
      "Epoch 93/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.1260 - acc: 0.9617 - val_loss: 0.0963 - val_acc: 0.9724\n",
      "Epoch 94/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.1244 - acc: 0.9623 - val_loss: 0.0958 - val_acc: 0.9724\n",
      "Epoch 95/250\n",
      "48000/48000 [==============================] - 3s 59us/step - loss: 0.1263 - acc: 0.9620 - val_loss: 0.0952 - val_acc: 0.9723\n",
      "Epoch 96/250\n",
      "48000/48000 [==============================] - 3s 52us/step - loss: 0.1248 - acc: 0.9635 - val_loss: 0.0951 - val_acc: 0.9725\n",
      "Epoch 97/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.1212 - acc: 0.9630 - val_loss: 0.0942 - val_acc: 0.9727\n",
      "Epoch 98/250\n",
      "48000/48000 [==============================] - 2s 52us/step - loss: 0.1244 - acc: 0.9622 - val_loss: 0.0941 - val_acc: 0.9728\n",
      "Epoch 99/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.1206 - acc: 0.9642 - val_loss: 0.0943 - val_acc: 0.9728\n",
      "Epoch 100/250\n",
      "48000/48000 [==============================] - 2s 52us/step - loss: 0.1219 - acc: 0.9634 - val_loss: 0.0945 - val_acc: 0.9722\n",
      "Epoch 101/250\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.1213 - acc: 0.9640 - val_loss: 0.0938 - val_acc: 0.9726\n",
      "Epoch 102/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.1202 - acc: 0.9638 - val_loss: 0.0927 - val_acc: 0.9726\n",
      "Epoch 103/250\n",
      "48000/48000 [==============================] - 2s 52us/step - loss: 0.1211 - acc: 0.9637 - val_loss: 0.0929 - val_acc: 0.9729\n",
      "Epoch 104/250\n",
      "48000/48000 [==============================] - 3s 63us/step - loss: 0.1171 - acc: 0.9642 - val_loss: 0.0928 - val_acc: 0.9726\n",
      "Epoch 105/250\n",
      "48000/48000 [==============================] - 3s 55us/step - loss: 0.1177 - acc: 0.9645 - val_loss: 0.0928 - val_acc: 0.9727\n",
      "Epoch 106/250\n",
      "48000/48000 [==============================] - 3s 59us/step - loss: 0.1179 - acc: 0.9647 - val_loss: 0.0922 - val_acc: 0.9732\n",
      "Epoch 107/250\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.1163 - acc: 0.9647 - val_loss: 0.0916 - val_acc: 0.9733\n",
      "Epoch 108/250\n",
      "48000/48000 [==============================] - 3s 60us/step - loss: 0.1146 - acc: 0.9659 - val_loss: 0.0917 - val_acc: 0.9731\n",
      "Epoch 109/250\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.1146 - acc: 0.9655 - val_loss: 0.0910 - val_acc: 0.9732\n",
      "Epoch 110/250\n",
      "48000/48000 [==============================] - 3s 59us/step - loss: 0.1149 - acc: 0.9651 - val_loss: 0.0907 - val_acc: 0.9738\n",
      "Epoch 111/250\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.1143 - acc: 0.9656 - val_loss: 0.0910 - val_acc: 0.9738\n",
      "Epoch 112/250\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.1127 - acc: 0.9663 - val_loss: 0.0900 - val_acc: 0.9737\n",
      "Epoch 113/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.1115 - acc: 0.9656 - val_loss: 0.0901 - val_acc: 0.9737\n",
      "Epoch 114/250\n",
      "48000/48000 [==============================] - 2s 52us/step - loss: 0.1098 - acc: 0.9675 - val_loss: 0.0898 - val_acc: 0.9736\n",
      "Epoch 115/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.1108 - acc: 0.9664 - val_loss: 0.0896 - val_acc: 0.9733\n",
      "Epoch 116/250\n",
      "48000/48000 [==============================] - 3s 54us/step - loss: 0.1098 - acc: 0.9669 - val_loss: 0.0890 - val_acc: 0.9735\n",
      "Epoch 117/250\n",
      "48000/48000 [==============================] - 3s 60us/step - loss: 0.1083 - acc: 0.9676 - val_loss: 0.0888 - val_acc: 0.9733\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 118/250\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.1062 - acc: 0.9676 - val_loss: 0.0886 - val_acc: 0.9735\n",
      "Epoch 119/250\n",
      "48000/48000 [==============================] - 3s 61us/step - loss: 0.1078 - acc: 0.9668 - val_loss: 0.0885 - val_acc: 0.9739\n",
      "Epoch 120/250\n",
      "48000/48000 [==============================] - 3s 57us/step - loss: 0.1063 - acc: 0.9677 - val_loss: 0.0891 - val_acc: 0.9741\n",
      "Epoch 121/250\n",
      "48000/48000 [==============================] - 3s 61us/step - loss: 0.1092 - acc: 0.9663 - val_loss: 0.0887 - val_acc: 0.9740\n",
      "Epoch 122/250\n",
      "48000/48000 [==============================] - 3s 56us/step - loss: 0.1068 - acc: 0.9680 - val_loss: 0.0882 - val_acc: 0.9738\n",
      "Epoch 123/250\n",
      "48000/48000 [==============================] - 3s 59us/step - loss: 0.1060 - acc: 0.9675 - val_loss: 0.0882 - val_acc: 0.9743\n",
      "Epoch 124/250\n",
      "48000/48000 [==============================] - 3s 57us/step - loss: 0.1064 - acc: 0.9682 - val_loss: 0.0874 - val_acc: 0.9738\n",
      "Epoch 125/250\n",
      "48000/48000 [==============================] - 3s 59us/step - loss: 0.1018 - acc: 0.9692 - val_loss: 0.0872 - val_acc: 0.9742\n",
      "Epoch 126/250\n",
      "48000/48000 [==============================] - 3s 60us/step - loss: 0.1027 - acc: 0.9689 - val_loss: 0.0876 - val_acc: 0.9740\n",
      "Epoch 127/250\n",
      "48000/48000 [==============================] - 3s 57us/step - loss: 0.1060 - acc: 0.9683 - val_loss: 0.0872 - val_acc: 0.9744\n",
      "Epoch 128/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.1035 - acc: 0.9686 - val_loss: 0.0870 - val_acc: 0.9747\n",
      "Epoch 129/250\n",
      "48000/48000 [==============================] - 3s 56us/step - loss: 0.1035 - acc: 0.9685 - val_loss: 0.0871 - val_acc: 0.9747\n",
      "Epoch 130/250\n",
      "48000/48000 [==============================] - 3s 61us/step - loss: 0.1045 - acc: 0.9685 - val_loss: 0.0867 - val_acc: 0.9746\n",
      "Epoch 131/250\n",
      "48000/48000 [==============================] - 3s 55us/step - loss: 0.1008 - acc: 0.9687 - val_loss: 0.0872 - val_acc: 0.9742\n",
      "Epoch 132/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.1027 - acc: 0.9677 - val_loss: 0.0866 - val_acc: 0.9746\n",
      "Epoch 133/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.1021 - acc: 0.9690 - val_loss: 0.0863 - val_acc: 0.9747\n",
      "Epoch 134/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0986 - acc: 0.9701 - val_loss: 0.0867 - val_acc: 0.9748\n",
      "Epoch 135/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.0998 - acc: 0.9695 - val_loss: 0.0857 - val_acc: 0.9748\n",
      "Epoch 136/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0987 - acc: 0.9698 - val_loss: 0.0860 - val_acc: 0.9746\n",
      "Epoch 137/250\n",
      "48000/48000 [==============================] - 2s 52us/step - loss: 0.0980 - acc: 0.9703 - val_loss: 0.0860 - val_acc: 0.9745\n",
      "Epoch 138/250\n",
      "48000/48000 [==============================] - 3s 57us/step - loss: 0.0969 - acc: 0.9708 - val_loss: 0.0857 - val_acc: 0.9752\n",
      "Epoch 139/250\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.0997 - acc: 0.9700 - val_loss: 0.0856 - val_acc: 0.9742\n",
      "Epoch 140/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0959 - acc: 0.9711 - val_loss: 0.0852 - val_acc: 0.9748\n",
      "Epoch 141/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0967 - acc: 0.9711 - val_loss: 0.0851 - val_acc: 0.9752\n",
      "Epoch 142/250\n",
      "48000/48000 [==============================] - 3s 55us/step - loss: 0.0974 - acc: 0.9702 - val_loss: 0.0847 - val_acc: 0.9755\n",
      "Epoch 143/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0959 - acc: 0.9715 - val_loss: 0.0847 - val_acc: 0.9751\n",
      "Epoch 144/250\n",
      "48000/48000 [==============================] - 2s 52us/step - loss: 0.0981 - acc: 0.9696 - val_loss: 0.0846 - val_acc: 0.9747\n",
      "Epoch 145/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0936 - acc: 0.9719 - val_loss: 0.0847 - val_acc: 0.9753\n",
      "Epoch 146/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.0925 - acc: 0.9723 - val_loss: 0.0841 - val_acc: 0.9752\n",
      "Epoch 147/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0960 - acc: 0.9711 - val_loss: 0.0846 - val_acc: 0.9756\n",
      "Epoch 148/250\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.0923 - acc: 0.9725 - val_loss: 0.0845 - val_acc: 0.9747\n",
      "Epoch 149/250\n",
      "48000/48000 [==============================] - 3s 56us/step - loss: 0.0915 - acc: 0.9716 - val_loss: 0.0844 - val_acc: 0.9755\n",
      "Epoch 150/250\n",
      "48000/48000 [==============================] - 3s 54us/step - loss: 0.0903 - acc: 0.9718 - val_loss: 0.0841 - val_acc: 0.9755\n",
      "Epoch 151/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.0935 - acc: 0.9720 - val_loss: 0.0842 - val_acc: 0.9751\n",
      "Epoch 152/250\n",
      "48000/48000 [==============================] - 3s 60us/step - loss: 0.0901 - acc: 0.9730 - val_loss: 0.0836 - val_acc: 0.9757\n",
      "Epoch 153/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0912 - acc: 0.9726 - val_loss: 0.0841 - val_acc: 0.9754\n",
      "Epoch 154/250\n",
      "48000/48000 [==============================] - 3s 59us/step - loss: 0.0914 - acc: 0.9721 - val_loss: 0.0840 - val_acc: 0.9747\n",
      "Epoch 155/250\n",
      "48000/48000 [==============================] - 2s 52us/step - loss: 0.0890 - acc: 0.9728 - val_loss: 0.0838 - val_acc: 0.9757\n",
      "Epoch 156/250\n",
      "48000/48000 [==============================] - 3s 59us/step - loss: 0.0920 - acc: 0.9716 - val_loss: 0.0837 - val_acc: 0.9755\n",
      "Epoch 157/250\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.0891 - acc: 0.9726 - val_loss: 0.0837 - val_acc: 0.9757\n",
      "Epoch 158/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0880 - acc: 0.9730 - val_loss: 0.0830 - val_acc: 0.9761\n",
      "Epoch 159/250\n",
      "48000/48000 [==============================] - 2s 52us/step - loss: 0.0865 - acc: 0.9735 - val_loss: 0.0833 - val_acc: 0.9759\n",
      "Epoch 160/250\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.0859 - acc: 0.9736 - val_loss: 0.0830 - val_acc: 0.9758\n",
      "Epoch 161/250\n",
      "48000/48000 [==============================] - 3s 56us/step - loss: 0.0852 - acc: 0.9736 - val_loss: 0.0831 - val_acc: 0.9758\n",
      "Epoch 162/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.0878 - acc: 0.9721 - val_loss: 0.0827 - val_acc: 0.9759\n",
      "Epoch 163/250\n",
      "48000/48000 [==============================] - 3s 59us/step - loss: 0.0881 - acc: 0.9728 - val_loss: 0.0824 - val_acc: 0.9758\n",
      "Epoch 164/250\n",
      "48000/48000 [==============================] - 3s 54us/step - loss: 0.0865 - acc: 0.9738 - val_loss: 0.0823 - val_acc: 0.9761\n",
      "Epoch 165/250\n",
      "48000/48000 [==============================] - 3s 60us/step - loss: 0.0868 - acc: 0.9733 - val_loss: 0.0824 - val_acc: 0.9755\n",
      "Epoch 166/250\n",
      "48000/48000 [==============================] - 3s 54us/step - loss: 0.0861 - acc: 0.9732 - val_loss: 0.0819 - val_acc: 0.9756\n",
      "Epoch 167/250\n",
      "48000/48000 [==============================] - 3s 59us/step - loss: 0.0860 - acc: 0.9736 - val_loss: 0.0817 - val_acc: 0.9765\n",
      "Epoch 168/250\n",
      "48000/48000 [==============================] - 3s 52us/step - loss: 0.0850 - acc: 0.9734 - val_loss: 0.0816 - val_acc: 0.9759\n",
      "Epoch 169/250\n",
      "48000/48000 [==============================] - 3s 57us/step - loss: 0.0833 - acc: 0.9744 - val_loss: 0.0816 - val_acc: 0.9759\n",
      "Epoch 170/250\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.0825 - acc: 0.9756 - val_loss: 0.0818 - val_acc: 0.9758\n",
      "Epoch 171/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.0835 - acc: 0.9739 - val_loss: 0.0820 - val_acc: 0.9758\n",
      "Epoch 172/250\n",
      "48000/48000 [==============================] - 3s 59us/step - loss: 0.0847 - acc: 0.9744 - val_loss: 0.0816 - val_acc: 0.9758\n",
      "Epoch 173/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.0847 - acc: 0.9739 - val_loss: 0.0810 - val_acc: 0.9765\n",
      "Epoch 174/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0837 - acc: 0.9742 - val_loss: 0.0811 - val_acc: 0.9762\n",
      "Epoch 175/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.0835 - acc: 0.9741 - val_loss: 0.0812 - val_acc: 0.9760\n",
      "Epoch 176/250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48000/48000 [==============================] - 3s 60us/step - loss: 0.0812 - acc: 0.9745 - val_loss: 0.0809 - val_acc: 0.9762\n",
      "Epoch 177/250\n",
      "48000/48000 [==============================] - 2s 52us/step - loss: 0.0815 - acc: 0.9746 - val_loss: 0.0809 - val_acc: 0.9758\n",
      "Epoch 178/250\n",
      "48000/48000 [==============================] - 3s 56us/step - loss: 0.0824 - acc: 0.9751 - val_loss: 0.0810 - val_acc: 0.9758\n",
      "Epoch 179/250\n",
      "48000/48000 [==============================] - 3s 52us/step - loss: 0.0825 - acc: 0.9745 - val_loss: 0.0808 - val_acc: 0.9767\n",
      "Epoch 180/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.0803 - acc: 0.9752 - val_loss: 0.0809 - val_acc: 0.9760\n",
      "Epoch 181/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0831 - acc: 0.9735 - val_loss: 0.0812 - val_acc: 0.9760\n",
      "Epoch 182/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.0820 - acc: 0.9749 - val_loss: 0.0810 - val_acc: 0.9761\n",
      "Epoch 183/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0799 - acc: 0.9750 - val_loss: 0.0803 - val_acc: 0.9762\n",
      "Epoch 184/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.0812 - acc: 0.9749 - val_loss: 0.0799 - val_acc: 0.9764\n",
      "Epoch 185/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0793 - acc: 0.9750 - val_loss: 0.0800 - val_acc: 0.9762\n",
      "Epoch 186/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.0791 - acc: 0.9759 - val_loss: 0.0803 - val_acc: 0.9762\n",
      "Epoch 187/250\n",
      "48000/48000 [==============================] - 3s 69us/step - loss: 0.0792 - acc: 0.9757 - val_loss: 0.0806 - val_acc: 0.9765\n",
      "Epoch 188/250\n",
      "48000/48000 [==============================] - 4s 75us/step - loss: 0.0771 - acc: 0.9766 - val_loss: 0.0798 - val_acc: 0.9771\n",
      "Epoch 189/250\n",
      "48000/48000 [==============================] - 4s 75us/step - loss: 0.0797 - acc: 0.9750 - val_loss: 0.0807 - val_acc: 0.9767\n",
      "Epoch 190/250\n",
      "48000/48000 [==============================] - 4s 83us/step - loss: 0.0783 - acc: 0.9755 - val_loss: 0.0799 - val_acc: 0.9761\n",
      "Epoch 191/250\n",
      "48000/48000 [==============================] - 3s 71us/step - loss: 0.0797 - acc: 0.9747 - val_loss: 0.0795 - val_acc: 0.9765\n",
      "Epoch 192/250\n",
      "48000/48000 [==============================] - 3s 66us/step - loss: 0.0789 - acc: 0.9750 - val_loss: 0.0799 - val_acc: 0.9773\n",
      "Epoch 193/250\n",
      "48000/48000 [==============================] - 3s 62us/step - loss: 0.0754 - acc: 0.9765 - val_loss: 0.0801 - val_acc: 0.9768\n",
      "Epoch 194/250\n",
      "48000/48000 [==============================] - 3s 56us/step - loss: 0.0782 - acc: 0.9754 - val_loss: 0.0796 - val_acc: 0.9771\n",
      "Epoch 195/250\n",
      "48000/48000 [==============================] - 3s 61us/step - loss: 0.0757 - acc: 0.9755 - val_loss: 0.0802 - val_acc: 0.9768\n",
      "Epoch 196/250\n",
      "48000/48000 [==============================] - 3s 57us/step - loss: 0.0740 - acc: 0.9776 - val_loss: 0.0795 - val_acc: 0.9767\n",
      "Epoch 197/250\n",
      "48000/48000 [==============================] - 3s 63us/step - loss: 0.0773 - acc: 0.9764 - val_loss: 0.0795 - val_acc: 0.9767\n",
      "Epoch 198/250\n",
      "48000/48000 [==============================] - 3s 54us/step - loss: 0.0754 - acc: 0.9766 - val_loss: 0.0798 - val_acc: 0.9767\n",
      "Epoch 199/250\n",
      "48000/48000 [==============================] - 3s 67us/step - loss: 0.0754 - acc: 0.9768 - val_loss: 0.0799 - val_acc: 0.9772\n",
      "Epoch 200/250\n",
      "48000/48000 [==============================] - 3s 55us/step - loss: 0.0748 - acc: 0.9760 - val_loss: 0.0797 - val_acc: 0.9772\n",
      "Epoch 201/250\n",
      "48000/48000 [==============================] - 3s 69us/step - loss: 0.0757 - acc: 0.9765 - val_loss: 0.0795 - val_acc: 0.9772\n",
      "Epoch 202/250\n",
      "48000/48000 [==============================] - 3s 56us/step - loss: 0.0719 - acc: 0.9776 - val_loss: 0.0791 - val_acc: 0.9771\n",
      "Epoch 203/250\n",
      "48000/48000 [==============================] - 3s 70us/step - loss: 0.0716 - acc: 0.9776 - val_loss: 0.0794 - val_acc: 0.9771\n",
      "Epoch 204/250\n",
      "48000/48000 [==============================] - 3s 63us/step - loss: 0.0746 - acc: 0.9770 - val_loss: 0.0796 - val_acc: 0.9771\n",
      "Epoch 205/250\n",
      "48000/48000 [==============================] - 3s 63us/step - loss: 0.0721 - acc: 0.9771 - val_loss: 0.0796 - val_acc: 0.9775\n",
      "Epoch 206/250\n",
      "48000/48000 [==============================] - 3s 55us/step - loss: 0.0720 - acc: 0.9780 - val_loss: 0.0794 - val_acc: 0.9772\n",
      "Epoch 207/250\n",
      "48000/48000 [==============================] - 3s 62us/step - loss: 0.0743 - acc: 0.9766 - val_loss: 0.0791 - val_acc: 0.9777\n",
      "Epoch 208/250\n",
      "48000/48000 [==============================] - 3s 55us/step - loss: 0.0737 - acc: 0.9770 - val_loss: 0.0789 - val_acc: 0.9776\n",
      "Epoch 209/250\n",
      "48000/48000 [==============================] - 3s 62us/step - loss: 0.0738 - acc: 0.9766 - val_loss: 0.0790 - val_acc: 0.9768\n",
      "Epoch 210/250\n",
      "48000/48000 [==============================] - 3s 54us/step - loss: 0.0730 - acc: 0.9773 - val_loss: 0.0789 - val_acc: 0.9770\n",
      "Epoch 211/250\n",
      "48000/48000 [==============================] - 3s 55us/step - loss: 0.0740 - acc: 0.9764 - val_loss: 0.0787 - val_acc: 0.9768\n",
      "Epoch 212/250\n",
      "48000/48000 [==============================] - 3s 57us/step - loss: 0.0736 - acc: 0.9778 - val_loss: 0.0787 - val_acc: 0.9772\n",
      "Epoch 213/250\n",
      "48000/48000 [==============================] - 3s 67us/step - loss: 0.0713 - acc: 0.9778 - val_loss: 0.0793 - val_acc: 0.9771\n",
      "Epoch 214/250\n",
      "48000/48000 [==============================] - 4s 86us/step - loss: 0.0703 - acc: 0.9787 - val_loss: 0.0787 - val_acc: 0.9768\n",
      "Epoch 215/250\n",
      "48000/48000 [==============================] - 3s 64us/step - loss: 0.0702 - acc: 0.9779 - val_loss: 0.0794 - val_acc: 0.9770\n",
      "Epoch 216/250\n",
      "48000/48000 [==============================] - 3s 56us/step - loss: 0.0716 - acc: 0.9784 - val_loss: 0.0788 - val_acc: 0.9773\n",
      "Epoch 217/250\n",
      "48000/48000 [==============================] - 3s 61us/step - loss: 0.0730 - acc: 0.9774 - val_loss: 0.0781 - val_acc: 0.9777\n",
      "Epoch 218/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0686 - acc: 0.9781 - val_loss: 0.0785 - val_acc: 0.9773\n",
      "Epoch 219/250\n",
      "48000/48000 [==============================] - 3s 59us/step - loss: 0.0710 - acc: 0.9774 - val_loss: 0.0783 - val_acc: 0.9777\n",
      "Epoch 220/250\n",
      "48000/48000 [==============================] - 3s 59us/step - loss: 0.0708 - acc: 0.9778 - val_loss: 0.0787 - val_acc: 0.9771\n",
      "Epoch 221/250\n",
      "48000/48000 [==============================] - 3s 57us/step - loss: 0.0684 - acc: 0.9786 - val_loss: 0.0786 - val_acc: 0.9778\n",
      "Epoch 222/250\n",
      "48000/48000 [==============================] - 4s 78us/step - loss: 0.0714 - acc: 0.9779 - val_loss: 0.0783 - val_acc: 0.9777\n",
      "Epoch 223/250\n",
      "48000/48000 [==============================] - 3s 70us/step - loss: 0.0691 - acc: 0.9785 - val_loss: 0.0786 - val_acc: 0.9776\n",
      "Epoch 224/250\n",
      "48000/48000 [==============================] - 3s 68us/step - loss: 0.0692 - acc: 0.9788 - val_loss: 0.0783 - val_acc: 0.9772\n",
      "Epoch 225/250\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.0695 - acc: 0.9786 - val_loss: 0.0789 - val_acc: 0.9777\n",
      "Epoch 226/250\n",
      "48000/48000 [==============================] - 3s 64us/step - loss: 0.0686 - acc: 0.9782 - val_loss: 0.0783 - val_acc: 0.9777\n",
      "Epoch 227/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0703 - acc: 0.9775 - val_loss: 0.0780 - val_acc: 0.9777\n",
      "Epoch 228/250\n",
      "48000/48000 [==============================] - 3s 64us/step - loss: 0.0673 - acc: 0.9789 - val_loss: 0.0786 - val_acc: 0.9776\n",
      "Epoch 229/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0683 - acc: 0.9785 - val_loss: 0.0783 - val_acc: 0.9777\n",
      "Epoch 230/250\n",
      "48000/48000 [==============================] - 3s 63us/step - loss: 0.0654 - acc: 0.9795 - val_loss: 0.0782 - val_acc: 0.9776\n",
      "Epoch 231/250\n",
      "48000/48000 [==============================] - 3s 59us/step - loss: 0.0668 - acc: 0.9791 - val_loss: 0.0785 - val_acc: 0.9771\n",
      "Epoch 232/250\n",
      "48000/48000 [==============================] - 3s 61us/step - loss: 0.0662 - acc: 0.9786 - val_loss: 0.0783 - val_acc: 0.9774\n",
      "Epoch 233/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0667 - acc: 0.9796 - val_loss: 0.0787 - val_acc: 0.9772\n",
      "Epoch 234/250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48000/48000 [==============================] - 3s 60us/step - loss: 0.0670 - acc: 0.9789 - val_loss: 0.0782 - val_acc: 0.9777\n",
      "Epoch 235/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.0684 - acc: 0.9788 - val_loss: 0.0778 - val_acc: 0.9782\n",
      "Epoch 236/250\n",
      "48000/48000 [==============================] - 3s 57us/step - loss: 0.0656 - acc: 0.9799 - val_loss: 0.0780 - val_acc: 0.9779\n",
      "Epoch 237/250\n",
      "48000/48000 [==============================] - 3s 54us/step - loss: 0.0645 - acc: 0.9794 - val_loss: 0.0780 - val_acc: 0.9778\n",
      "Epoch 238/250\n",
      "48000/48000 [==============================] - 3s 62us/step - loss: 0.0661 - acc: 0.9787 - val_loss: 0.0782 - val_acc: 0.9769\n",
      "Epoch 239/250\n",
      "48000/48000 [==============================] - 3s 54us/step - loss: 0.0661 - acc: 0.9797 - val_loss: 0.0775 - val_acc: 0.9778\n",
      "Epoch 240/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0648 - acc: 0.9801 - val_loss: 0.0783 - val_acc: 0.9774\n",
      "Epoch 241/250\n",
      "48000/48000 [==============================] - 3s 60us/step - loss: 0.0678 - acc: 0.9787 - val_loss: 0.0780 - val_acc: 0.9777\n",
      "Epoch 242/250\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.0661 - acc: 0.9790 - val_loss: 0.0775 - val_acc: 0.9777\n",
      "Epoch 243/250\n",
      "48000/48000 [==============================] - 3s 61us/step - loss: 0.0653 - acc: 0.9795 - val_loss: 0.0776 - val_acc: 0.9775\n",
      "Epoch 244/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.0644 - acc: 0.9791 - val_loss: 0.0788 - val_acc: 0.9774\n",
      "Epoch 245/250\n",
      "48000/48000 [==============================] - 3s 57us/step - loss: 0.0633 - acc: 0.9803 - val_loss: 0.0783 - val_acc: 0.9779\n",
      "Epoch 246/250\n",
      "48000/48000 [==============================] - 2s 51us/step - loss: 0.0629 - acc: 0.9803 - val_loss: 0.0777 - val_acc: 0.9782\n",
      "Epoch 247/250\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.0616 - acc: 0.9810 - val_loss: 0.0781 - val_acc: 0.9778\n",
      "Epoch 248/250\n",
      "48000/48000 [==============================] - 3s 54us/step - loss: 0.0640 - acc: 0.9799 - val_loss: 0.0776 - val_acc: 0.9777\n",
      "Epoch 249/250\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.0636 - acc: 0.9808 - val_loss: 0.0778 - val_acc: 0.9776\n",
      "Epoch 250/250\n",
      "48000/48000 [==============================] - 3s 57us/step - loss: 0.0631 - acc: 0.9798 - val_loss: 0.0778 - val_acc: 0.9772\n"
     ]
    }
   ],
   "source": [
    "# compile and train the model\n",
    "model.compile(loss='categorical_crossentropy', optimizer=OPTIMIZER, metrics=['accuracy'])\n",
    "\n",
    "# train the model\n",
    "history = model.fit(X_train, Y_train,\n",
    "                   batch_size=BATCH_SIZE,\n",
    "                   epochs=NB_EPOCH,\n",
    "                   verbose=VERBOSE,\n",
    "                   validation_split=VALIDATION_SPLIT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 1s 55us/step\n",
      "Test score: 0.0706871715334\n",
      "Test accuracy: 0.9786\n"
     ]
    }
   ],
   "source": [
    "# Evaluating the model\n",
    "score=model.evaluate(X_test, Y_test, verbose=VERBOSE)\n",
    "print('Test score:',score[0])\n",
    "print('Test accuracy:',score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training Accuracy: 97.98%\n",
    "# Validation Accuracy: 97.72%\n",
    "# Test Accuracy: 97.86%"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}