mnist_mlp .ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### 1. Load MNIST Database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The MNIST database has a training set of 60000 examples.\n",
      "The MNIST database has a test set of 10000 examples.\n"
     ]
    }
   ],
   "source": [
    "from keras.datasets import mnist\n",
    "\n",
    "# use Keras to import pre-shuffled MNIST database\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "\n",
    "print(\"The MNIST database has a training set of %d examples.\" % len(X_train))\n",
    "print(\"The MNIST database has a test set of %d examples.\" % len(X_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### 2. Rescale the Images by Dividing Every Pixel in Every Image by 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# rescale [0,255] --> [0,1]\n",
    "X_train = X_train.astype('float32')/255\n",
    "X_test = X_test.astype('float32')/255 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### 3. Encode Categorical Integer Labels Using a One-Hot Scheme"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Integer-valued labels:\n",
      "[5 0 4 1 9 2 1 3 1 4]\n",
      "One-hot labels:\n",
      "[[ 0.  0.  0.  0.  0.  1.  0.  0.  0.  0.]\n",
      " [ 1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.  0.  0.  0.]\n",
      " [ 0.  1.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  1.]\n",
      " [ 0.  0.  1.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  1.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  1.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.  0.  0.  0.]]\n"
     ]
    }
   ],
   "source": [
    "from keras.utils import np_utils\n",
    "\n",
    "# print first ten training labels\n",
    "print('Integer-valued labels:')\n",
    "print(y_train[:10])\n",
    "\n",
    "# one-hot encode the labels\n",
    "y_train = np_utils.to_categorical(y_train, 10)\n",
    "y_test = np_utils.to_categorical(y_test, 10)\n",
    "\n",
    "\n",
    "print('One-hot labels:')\n",
    "print(y_train[:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### 4. Define the Model Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_1 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 512)               401920    \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 512)               262656    \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                5130      \n",
      "=================================================================\n",
      "Total params: 669,706\n",
      "Trainable params: 669,706\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "\n",
    "# define the model\n",
    "model = Sequential()\n",
    "model.add(Flatten(input_shape=X_train.shape[1:]))\n",
    "model.add(Dense(512, activation='relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(512, activation='relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(10, activation='softmax'))\n",
    "\n",
    "# summarize the model\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### 5. Compile the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# compile the model\n",
    "model.compile(loss='categorical_crossentropy', optimizer='rmsprop', \n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### 6. Train the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/10\n",
      "47744/48000 [============================>.] - ETA: 0s - loss: 0.9607 - acc: 0.7053Epoch 00000: val_loss improved from inf to 0.42680, saving model to mnist.model.best.hdf5\n",
      "48000/48000 [==============================] - 16s - loss: 0.9581 - acc: 0.7061 - val_loss: 0.4268 - val_acc: 0.8784\n",
      "Epoch 2/10\n",
      "47872/48000 [============================>.] - ETA: 0s - loss: 0.4036 - acc: 0.8792Epoch 00001: val_loss improved from 0.42680 to 0.30999, saving model to mnist.model.best.hdf5\n",
      "48000/48000 [==============================] - 15s - loss: 0.4037 - acc: 0.8792 - val_loss: 0.3100 - val_acc: 0.9072\n",
      "Epoch 3/10\n",
      "47744/48000 [============================>.] - ETA: 0s - loss: 0.3165 - acc: 0.9050Epoch 00002: val_loss improved from 0.30999 to 0.26536, saving model to mnist.model.best.hdf5\n",
      "48000/48000 [==============================] - 16s - loss: 0.3164 - acc: 0.9050 - val_loss: 0.2654 - val_acc: 0.9212\n",
      "Epoch 4/10\n",
      "47744/48000 [============================>.] - ETA: 0s - loss: 0.2602 - acc: 0.9213Epoch 00003: val_loss improved from 0.26536 to 0.20558, saving model to mnist.model.best.hdf5\n",
      "48000/48000 [==============================] - 16s - loss: 0.2599 - acc: 0.9214 - val_loss: 0.2056 - val_acc: 0.9391\n",
      "Epoch 5/10\n",
      "47872/48000 [============================>.] - ETA: 0s - loss: 0.2157 - acc: 0.9354Epoch 00004: val_loss improved from 0.20558 to 0.17608, saving model to mnist.model.best.hdf5\n",
      "48000/48000 [==============================] - 16s - loss: 0.2156 - acc: 0.9354 - val_loss: 0.1761 - val_acc: 0.9483\n",
      "Epoch 6/10\n",
      "47744/48000 [============================>.] - ETA: 0s - loss: 0.1827 - acc: 0.9454Epoch 00005: val_loss improved from 0.17608 to 0.15463, saving model to mnist.model.best.hdf5\n",
      "48000/48000 [==============================] - 17s - loss: 0.1829 - acc: 0.9454 - val_loss: 0.1546 - val_acc: 0.9543\n",
      "Epoch 7/10\n",
      "47744/48000 [============================>.] - ETA: 0s - loss: 0.1607 - acc: 0.9516Epoch 00006: val_loss improved from 0.15463 to 0.15396, saving model to mnist.model.best.hdf5\n",
      "48000/48000 [==============================] - 17s - loss: 0.1607 - acc: 0.9516 - val_loss: 0.1540 - val_acc: 0.9551\n",
      "Epoch 8/10\n",
      "47872/48000 [============================>.] - ETA: 0s - loss: 0.1418 - acc: 0.9570Epoch 00007: val_loss improved from 0.15396 to 0.12646, saving model to mnist.model.best.hdf5\n",
      "48000/48000 [==============================] - 16s - loss: 0.1417 - acc: 0.9570 - val_loss: 0.1265 - val_acc: 0.9626\n",
      "Epoch 9/10\n",
      "47872/48000 [============================>.] - ETA: 0s - loss: 0.1265 - acc: 0.9615Epoch 00008: val_loss improved from 0.12646 to 0.12206, saving model to mnist.model.best.hdf5\n",
      "48000/48000 [==============================] - 16s - loss: 0.1267 - acc: 0.9615 - val_loss: 0.1221 - val_acc: 0.9637\n",
      "Epoch 10/10\n",
      "47744/48000 [============================>.] - ETA: 0s - loss: 0.1137 - acc: 0.9656Epoch 00009: val_loss improved from 0.12206 to 0.10992, saving model to mnist.model.best.hdf5\n",
      "48000/48000 [==============================] - 17s - loss: 0.1135 - acc: 0.9656 - val_loss: 0.1099 - val_acc: 0.9670\n"
     ]
    }
   ],
   "source": [
    "from keras.callbacks import ModelCheckpoint   \n",
    "\n",
    "# train the model and save the best weights\n",
    "checkpointer = ModelCheckpoint(filepath='mnist.model.best.hdf5', \n",
    "                               verbose=1, save_best_only=True)\n",
    "hist = model.fit(X_train, y_train, batch_size=128, epochs=10,\n",
    "          validation_split=0.2, callbacks=[checkpointer],\n",
    "          verbose=1, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### 7. Load the Model with the Best Classification Accuracy on the Validation Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# load the weights that yielded the best validation accuracy\n",
    "model.load_weights('mnist.model.best.hdf5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### 8. Calculate the Classification Accuracy on the Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 96.6700%\n"
     ]
    }
   ],
   "source": [
    "# evaluate test accuracy\n",
    "score = model.evaluate(X_test, y_test, verbose=0)\n",
    "accuracy = 100*score[1]\n",
    "\n",
    "# print test accuracy\n",
    "print('Test accuracy: %.4f%%' % accuracy)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:tenf]",
   "language": "python",
   "name": "conda-env-tenf-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}