MNIST Competiton_9980/mnist_competition_9980_Final.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "from keras.models import Sequential, model_from_json\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten, Convolution2D, MaxPooling2D\n",
    "from keras.optimizers import Adam\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline \n",
    "tf.logging.set_verbosity(tf.logging.ERROR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tf.set_random_seed(2017)\n",
    "np.random.seed(2017)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Data Agumentation\n",
    "- image rotation(10, 20, 30)\n",
    "- image width shift(0.1, 0.2, 0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n",
    "# test data\n",
    "test_images = mnist.test.images.reshape(10000, 28, 28, 1)\n",
    "test_labels = mnist.test.labels[:]\n",
    "\n",
    "augmentation_size = 440000\n",
    "images = np.concatenate((mnist.train.images.reshape(55000, 28, 28, 1), mnist.validation.images.reshape(5000, 28, 28, 1)), axis=0)\n",
    "labels = np.concatenate((mnist.train.labels, mnist.validation.labels), axis=0)\n",
    "\n",
    "datagen_list = [\n",
    "                ImageDataGenerator(rotation_range=10),\n",
    "                ImageDataGenerator(rotation_range=20),\n",
    "                ImageDataGenerator(rotation_range=30),\n",
    "                ImageDataGenerator(width_shift_range=0.1),\n",
    "                ImageDataGenerator(width_shift_range=0.2),\n",
    "                ImageDataGenerator(width_shift_range=0.3),\n",
    "               ]\n",
    "\n",
    "for datagen in datagen_list:\n",
    "    datagen.fit(images)\n",
    "    for image, label in datagen.flow(images, labels, batch_size=augmentation_size, shuffle=True, seed=2017):\n",
    "        images = np.concatenate((images, image), axis=0)\n",
    "        labels = np.concatenate((labels, label), axis=0)\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Data Set : (1800000, 28, 28, 1)\n",
      "Test Data Set  : (10000, 28, 28, 1)\n"
     ]
    }
   ],
   "source": [
    "print('Train Data Set :', images.shape)\n",
    "print('Test Data Set  :', test_images.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Training\n",
    "\n",
    "\n",
    "## Architecture\n",
    "1. Convolution * 2 + MaxPool + Dropout\n",
    "2. Convolution * 2 + MaxPool + Dropout\n",
    "3. Convolution + MaxPool + Dropout\n",
    "4. Dense + Dropout\n",
    "5. Dense + Dropout\n",
    "6. Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Point\n",
    "1. Using Small Vggnet Ensemble\n",
    "2. Three different Convolutional Layer filter sizes(Model1 = 3, Model2 = 5, Model3 = 7)\n",
    "3. Elu Activation Function\n",
    "4. Adam Optimizer(learning rate = 0.0001)\n",
    "5. Data Augmentation(image rotation, image width shift)\n",
    "6. Batch Data Shuffle in training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1800000 samples, validate on 10000 samples\n",
      "Epoch 1/20\n",
      "1800000/1800000 [==============================] - 425s - loss: 0.2764 - acc: 0.9133 - val_loss: 0.0293 - val_acc: 0.9899\n",
      "Epoch 2/20\n",
      "1800000/1800000 [==============================] - 424s - loss: 0.1080 - acc: 0.9665 - val_loss: 0.0230 - val_acc: 0.9926\n",
      "Epoch 3/20\n",
      "1800000/1800000 [==============================] - 434s - loss: 0.0770 - acc: 0.9760 - val_loss: 0.0168 - val_acc: 0.9949\n",
      "Epoch 4/20\n",
      "1800000/1800000 [==============================] - 433s - loss: 0.0622 - acc: 0.9805 - val_loss: 0.0160 - val_acc: 0.9954\n",
      "Epoch 5/20\n",
      "1800000/1800000 [==============================] - 424s - loss: 0.0524 - acc: 0.9835 - val_loss: 0.0148 - val_acc: 0.9960\n",
      "Epoch 6/20\n",
      "1800000/1800000 [==============================] - 424s - loss: 0.0463 - acc: 0.9853 - val_loss: 0.0131 - val_acc: 0.9962\n",
      "Epoch 7/20\n",
      "1800000/1800000 [==============================] - 424s - loss: 0.0416 - acc: 0.9869 - val_loss: 0.0145 - val_acc: 0.9962\n",
      "Epoch 8/20\n",
      "1800000/1800000 [==============================] - 424s - loss: 0.0382 - acc: 0.9879 - val_loss: 0.0147 - val_acc: 0.9960\n",
      "Epoch 9/20\n",
      "1800000/1800000 [==============================] - 423s - loss: 0.0352 - acc: 0.9887 - val_loss: 0.0130 - val_acc: 0.9960\n",
      "Epoch 10/20\n",
      "1800000/1800000 [==============================] - 423s - loss: 0.0332 - acc: 0.9895 - val_loss: 0.0136 - val_acc: 0.9966\n",
      "Epoch 11/20\n",
      "1800000/1800000 [==============================] - 424s - loss: 0.0314 - acc: 0.9899 - val_loss: 0.0123 - val_acc: 0.9964\n",
      "Epoch 12/20\n",
      "1800000/1800000 [==============================] - 423s - loss: 0.0296 - acc: 0.9906 - val_loss: 0.0123 - val_acc: 0.9967\n",
      "Epoch 13/20\n",
      "1800000/1800000 [==============================] - 423s - loss: 0.0283 - acc: 0.9910 - val_loss: 0.0143 - val_acc: 0.9964\n",
      "Epoch 14/20\n",
      "1800000/1800000 [==============================] - 423s - loss: 0.0274 - acc: 0.9913 - val_loss: 0.0133 - val_acc: 0.9966\n",
      "Epoch 15/20\n",
      "1800000/1800000 [==============================] - 423s - loss: 0.0264 - acc: 0.9916 - val_loss: 0.0147 - val_acc: 0.9960\n",
      "Epoch 16/20\n",
      "1800000/1800000 [==============================] - 423s - loss: 0.0255 - acc: 0.9918 - val_loss: 0.0142 - val_acc: 0.9970\n",
      "Epoch 17/20\n",
      "1800000/1800000 [==============================] - 423s - loss: 0.0248 - acc: 0.9921 - val_loss: 0.0141 - val_acc: 0.9966\n",
      "Epoch 18/20\n",
      "1800000/1800000 [==============================] - 423s - loss: 0.0238 - acc: 0.9925 - val_loss: 0.0145 - val_acc: 0.9967\n",
      "Epoch 19/20\n",
      "1800000/1800000 [==============================] - 423s - loss: 0.0236 - acc: 0.9925 - val_loss: 0.0152 - val_acc: 0.9967\n",
      "Epoch 20/20\n",
      "1800000/1800000 [==============================] - 423s - loss: 0.0231 - acc: 0.9927 - val_loss: 0.0161 - val_acc: 0.9969\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x22e9aff05f8>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model1 = Sequential([Convolution2D(filters=64, kernel_size=(3, 3), padding='same', activation='elu', input_shape=(28, 28, 1)), \n",
    "                     Convolution2D(filters=128, kernel_size=(3, 3), padding='same', activation='elu'),\n",
    "                     MaxPooling2D(pool_size=(2, 2)),\n",
    "                     Dropout(0.5),\n",
    "                     Convolution2D(filters=128, kernel_size=(3, 3), padding='same', activation='elu'),\n",
    "                     Convolution2D(filters=128, kernel_size=(3, 3), padding='same', activation='elu'),\n",
    "                     MaxPooling2D(pool_size=(2, 2)),\n",
    "                     Dropout(0.5),\n",
    "                     Convolution2D(filters=128, kernel_size=(3, 3), padding='same', activation='elu'),\n",
    "                     MaxPooling2D(pool_size=(2, 2)),\n",
    "                     Dropout(0.5),\n",
    "                     Flatten(),\n",
    "                     Dense(1024, activation='elu'),\n",
    "                     Dropout(0.5),\n",
    "                     Dense(1024, activation='elu'),\n",
    "                     Dropout(0.5),\n",
    "                     Dense(10, activation='softmax'),\n",
    "                     ])\n",
    "model1.compile(optimizer=Adam(lr=0.0001), loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "model1.fit(images, labels, batch_size=256, epochs=20, shuffle=True, verbose=1, validation_data=(test_images, test_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved model to disk\n"
     ]
    }
   ],
   "source": [
    "model_json = model1.to_json()\n",
    "with open(\"model1.json\", \"w\") as json_file:\n",
    "    json_file.write(model_json)\n",
    "model1.save_weights(\"model1.h5\")\n",
    "print(\"Saved model to disk\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1800000 samples, validate on 10000 samples\n",
      "Epoch 1/20\n",
      "1800000/1800000 [==============================] - 607s - loss: 0.2144 - acc: 0.9333 - val_loss: 0.0209 - val_acc: 0.9936\n",
      "Epoch 2/20\n",
      "1800000/1800000 [==============================] - 606s - loss: 0.0838 - acc: 0.9742 - val_loss: 0.0170 - val_acc: 0.9947\n",
      "Epoch 3/20\n",
      "1800000/1800000 [==============================] - 610s - loss: 0.0570 - acc: 0.9823 - val_loss: 0.0137 - val_acc: 0.9958\n",
      "Epoch 4/20\n",
      "1800000/1800000 [==============================] - 613s - loss: 0.0442 - acc: 0.9862 - val_loss: 0.0146 - val_acc: 0.9960\n",
      "Epoch 5/20\n",
      "1800000/1800000 [==============================] - 612s - loss: 0.0360 - acc: 0.9887 - val_loss: 0.0128 - val_acc: 0.9964\n",
      "Epoch 6/20\n",
      "1800000/1800000 [==============================] - 613s - loss: 0.0313 - acc: 0.9903 - val_loss: 0.0139 - val_acc: 0.9960\n",
      "Epoch 7/20\n",
      "1800000/1800000 [==============================] - 612s - loss: 0.0276 - acc: 0.9912 - val_loss: 0.0108 - val_acc: 0.9964\n",
      "Epoch 8/20\n",
      "1800000/1800000 [==============================] - 611s - loss: 0.0251 - acc: 0.9921 - val_loss: 0.0132 - val_acc: 0.9962\n",
      "Epoch 9/20\n",
      "1800000/1800000 [==============================] - 610s - loss: 0.0230 - acc: 0.9927 - val_loss: 0.0120 - val_acc: 0.9974\n",
      "Epoch 10/20\n",
      "1800000/1800000 [==============================] - 610s - loss: 0.0214 - acc: 0.9932 - val_loss: 0.0110 - val_acc: 0.9972\n",
      "Epoch 11/20\n",
      "1800000/1800000 [==============================] - 610s - loss: 0.0201 - acc: 0.9937 - val_loss: 0.0125 - val_acc: 0.9966\n",
      "Epoch 12/20\n",
      "1800000/1800000 [==============================] - 610s - loss: 0.0188 - acc: 0.9941 - val_loss: 0.0123 - val_acc: 0.9972\n",
      "Epoch 13/20\n",
      "1800000/1800000 [==============================] - 610s - loss: 0.0183 - acc: 0.9943 - val_loss: 0.0100 - val_acc: 0.9972\n",
      "Epoch 14/20\n",
      "1800000/1800000 [==============================] - 610s - loss: 0.0174 - acc: 0.9946 - val_loss: 0.0138 - val_acc: 0.9968\n",
      "Epoch 15/20\n",
      "1800000/1800000 [==============================] - 610s - loss: 0.0166 - acc: 0.9948 - val_loss: 0.0136 - val_acc: 0.9968\n",
      "Epoch 16/20\n",
      "1800000/1800000 [==============================] - 610s - loss: 0.0160 - acc: 0.9950 - val_loss: 0.0116 - val_acc: 0.9974\n",
      "Epoch 17/20\n",
      "1800000/1800000 [==============================] - 610s - loss: 0.0155 - acc: 0.9952 - val_loss: 0.0152 - val_acc: 0.9959\n",
      "Epoch 18/20\n",
      "1800000/1800000 [==============================] - 610s - loss: 0.0151 - acc: 0.9953 - val_loss: 0.0139 - val_acc: 0.9968\n",
      "Epoch 19/20\n",
      "1800000/1800000 [==============================] - 610s - loss: 0.0148 - acc: 0.9955 - val_loss: 0.0140 - val_acc: 0.9971\n",
      "Epoch 20/20\n",
      "1800000/1800000 [==============================] - 610s - loss: 0.0145 - acc: 0.9956 - val_loss: 0.0128 - val_acc: 0.9972\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x22e9abd20f0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model2 = Sequential([Convolution2D(filters=64, kernel_size=(5, 5), padding='same', activation='elu', input_shape=(28, 28, 1)), \n",
    "                     Convolution2D(filters=128, kernel_size=(5, 5), padding='same', activation='elu'),\n",
    "                     MaxPooling2D(pool_size=(2, 2)),\n",
    "                     Dropout(0.5),\n",
    "                     Convolution2D(filters=128, kernel_size=(5, 5), padding='same', activation='elu'),\n",
    "                     Convolution2D(filters=128, kernel_size=(5, 5), padding='same', activation='elu'),\n",
    "                     MaxPooling2D(pool_size=(2, 2)),\n",
    "                     Dropout(0.5),\n",
    "                     Convolution2D(filters=128, kernel_size=(5, 5), padding='same', activation='elu'),\n",
    "                     MaxPooling2D(pool_size=(2, 2)),\n",
    "                     Dropout(0.5),\n",
    "                     Flatten(),\n",
    "                     Dense(1024, activation='elu'),\n",
    "                     Dropout(0.5),\n",
    "                     Dense(1024, activation='elu'),\n",
    "                     Dropout(0.5),\n",
    "                     Dense(10, activation='softmax'),\n",
    "                     ])\n",
    "model2.compile(optimizer=Adam(lr=0.0001), loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "model2.fit(images, labels, batch_size=256, epochs=20, shuffle=True, verbose=1, validation_data=(test_images, test_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved model to disk\n"
     ]
    }
   ],
   "source": [
    "model_json = model2.to_json()\n",
    "with open(\"model2.json\", \"w\") as json_file:\n",
    "    json_file.write(model_json)\n",
    "model2.save_weights(\"model2.h5\")\n",
    "print(\"Saved model to disk\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1800000 samples, validate on 10000 samples\n",
      "Epoch 1/20\n",
      "1800000/1800000 [==============================] - 842s - loss: 0.1917 - acc: 0.9408 - val_loss: 0.0209 - val_acc: 0.9936\n",
      "Epoch 2/20\n",
      "1800000/1800000 [==============================] - 840s - loss: 0.0708 - acc: 0.9785 - val_loss: 0.0146 - val_acc: 0.9956\n",
      "Epoch 3/20\n",
      "1800000/1800000 [==============================] - 840s - loss: 0.0462 - acc: 0.9858 - val_loss: 0.0117 - val_acc: 0.9967\n",
      "Epoch 4/20\n",
      "1800000/1800000 [==============================] - 839s - loss: 0.0341 - acc: 0.9894 - val_loss: 0.0139 - val_acc: 0.9967\n",
      "Epoch 5/20\n",
      "1800000/1800000 [==============================] - 839s - loss: 0.0276 - acc: 0.9914 - val_loss: 0.0139 - val_acc: 0.9966\n",
      "Epoch 6/20\n",
      "1800000/1800000 [==============================] - 838s - loss: 0.0234 - acc: 0.9927 - val_loss: 0.0145 - val_acc: 0.9964\n",
      "Epoch 7/20\n",
      "1800000/1800000 [==============================] - 840s - loss: 0.0205 - acc: 0.9936 - val_loss: 0.0127 - val_acc: 0.9972\n",
      "Epoch 8/20\n",
      "1800000/1800000 [==============================] - 843s - loss: 0.0186 - acc: 0.9942 - val_loss: 0.0147 - val_acc: 0.9967\n",
      "Epoch 9/20\n",
      "1800000/1800000 [==============================] - 842s - loss: 0.0171 - acc: 0.9948 - val_loss: 0.0154 - val_acc: 0.9967\n",
      "Epoch 10/20\n",
      "1800000/1800000 [==============================] - 846s - loss: 0.0156 - acc: 0.9952 - val_loss: 0.0161 - val_acc: 0.9965\n",
      "Epoch 11/20\n",
      "1800000/1800000 [==============================] - 852s - loss: 0.0148 - acc: 0.9955 - val_loss: 0.0147 - val_acc: 0.9967\n",
      "Epoch 12/20\n",
      "1800000/1800000 [==============================] - 846s - loss: 0.0140 - acc: 0.9958 - val_loss: 0.0142 - val_acc: 0.9971\n",
      "Epoch 13/20\n",
      "1800000/1800000 [==============================] - 842s - loss: 0.0132 - acc: 0.9960 - val_loss: 0.0148 - val_acc: 0.9972\n",
      "Epoch 14/20\n",
      "1800000/1800000 [==============================] - 841s - loss: 0.0127 - acc: 0.9962 - val_loss: 0.0150 - val_acc: 0.9972\n",
      "Epoch 15/20\n",
      "1800000/1800000 [==============================] - 836s - loss: 0.0124 - acc: 0.9964 - val_loss: 0.0145 - val_acc: 0.9971\n",
      "Epoch 16/20\n",
      "1800000/1800000 [==============================] - 836s - loss: 0.0120 - acc: 0.9964 - val_loss: 0.0186 - val_acc: 0.9967\n",
      "Epoch 17/20\n",
      "1800000/1800000 [==============================] - 851s - loss: 0.0117 - acc: 0.9965 - val_loss: 0.0162 - val_acc: 0.9974\n",
      "Epoch 18/20\n",
      "1800000/1800000 [==============================] - 850s - loss: 0.0114 - acc: 0.9967 - val_loss: 0.0170 - val_acc: 0.9974\n",
      "Epoch 19/20\n",
      "1800000/1800000 [==============================] - 850s - loss: 0.0115 - acc: 0.9967 - val_loss: 0.0146 - val_acc: 0.9968\n",
      "Epoch 20/20\n",
      "1800000/1800000 [==============================] - 849s - loss: 0.0113 - acc: 0.9968 - val_loss: 0.0129 - val_acc: 0.9972\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x22e9b7c4898>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model3 = Sequential([Convolution2D(filters=64, kernel_size=(7, 7), padding='same', activation='elu', input_shape=(28, 28, 1)), \n",
    "                     Convolution2D(filters=128, kernel_size=(7, 7), padding='same', activation='elu'),\n",
    "                     MaxPooling2D(pool_size=(2, 2)),\n",
    "                     Dropout(0.5),\n",
    "                     Convolution2D(filters=128, kernel_size=(7, 7), padding='same', activation='elu'),\n",
    "                     Convolution2D(filters=128, kernel_size=(7, 7), padding='same', activation='elu'),\n",
    "                     MaxPooling2D(pool_size=(2, 2)),\n",
    "                     Dropout(0.5),\n",
    "                     Convolution2D(filters=128, kernel_size=(7, 7), padding='same', activation='elu'),\n",
    "                     MaxPooling2D(pool_size=(2, 2)),\n",
    "                     Dropout(0.5),\n",
    "                     Flatten(),\n",
    "                     Dense(1024, activation='elu'),\n",
    "                     Dropout(0.5),\n",
    "                     Dense(1024, activation='elu'),\n",
    "                     Dropout(0.5),\n",
    "                     Dense(10, activation='softmax'),\n",
    "                     ])\n",
    "model3.compile(optimizer=Adam(lr=0.0001), loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "model3.fit(images, labels, batch_size=256, epochs=20, shuffle=True, verbose=1, validation_data=(test_images, test_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved model to disk\n"
     ]
    }
   ],
   "source": [
    "model_json = model3.to_json()\n",
    "with open(\"model3.json\", \"w\") as json_file:\n",
    "    json_file.write(model_json)\n",
    "model3.save_weights(\"model3.h5\")\n",
    "print(\"Saved model to disk\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from keras.models import model_from_json\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "from keras.optimizers import Adam\n",
    "import numpy as np\n",
    "\n",
    "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n",
    "# test data\n",
    "test_images = mnist.test.images.reshape(10000, 28, 28, 1)\n",
    "test_labels = mnist.test.labels[:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load json and create model\n",
    "def model_open(name, test_images, test_labels):\n",
    "    json_file = open(name + '.json', 'r')\n",
    "    loaded_model_json = json_file.read()\n",
    "    json_file.close()\n",
    "    loaded_model = model_from_json(loaded_model_json)\n",
    "    # load weights into new model\n",
    "    loaded_model.load_weights(name + '.h5')\n",
    "    print(\"Loaded model from disk\")\n",
    "    loaded_model.compile(optimizer=Adam(lr=0.0001), loss='categorical_crossentropy', metrics=['acc'])\n",
    "    prob = loaded_model.predict_proba(test_images)\n",
    "    acc = np.mean(np.equal(np.argmax(prob, axis=1), np.argmax(test_labels, axis=1)))\n",
    "    print('\\nmodel : %s, test accuracy : %.4f\\n' % (name, acc))\n",
    "    return prob, acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded model from disk\n",
      " 9888/10000 [============================>.] - ETA: 0s\n",
      "model : model1, test accuracy : 0.9969\n",
      "\n",
      "Loaded model from disk\n",
      " 9952/10000 [============================>.] - ETA: 0s\n",
      "model : model2, test accuracy : 0.9972\n",
      "\n",
      "Loaded model from disk\n",
      "10000/10000 [==============================] - 2s     \n",
      "\n",
      "model : model3, test accuracy : 0.9972\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model_1_prob, model_1_acc = model_open('model1', test_images, test_labels)\n",
    "model_2_prob, model_2_acc = model_open('model2', test_images, test_labels)\n",
    "model_3_prob, model_3_acc = model_open('model3', test_images, test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Final Result(Ensemble)\n",
    "- This ensembe method gives higher weight to the model with higher performance. If the performances of two models were equal, the higher weight was given to the one with a bigger filter size\n",
    "- best accuracy with a single model is $0.9972$\n",
    "- Final Ensemble Accuracy is $0.9980$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def model_ensemble(prob1, acc1, prob2, acc2, prob3, acc3):\n",
    "    prob_list = [prob1, prob2, prob3]\n",
    "    acc_list = [acc1, acc2, acc3]\n",
    "    idx_acc_list = {idx: acc for idx, acc in enumerate(acc_list)}\n",
    "    sorted_acc_list = [idx for idx, _ in sorted(idx_acc_list.items(), key=lambda value: (value[1], value[0]), reverse=True)]\n",
    "    final_prob = 0\n",
    "    for i in sorted_acc_list:\n",
    "        final_prob += prob_list[i] * (i+1)\n",
    "    final_score = np.mean(np.equal(np.argmax(final_prob, axis=1), np.argmax(test_labels, axis=1))) # Test\n",
    "    print('Final test accuracy : %.4f' % final_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final test accuracy : 0.9980\n"
     ]
    }
   ],
   "source": [
    "model_ensemble(model_1_prob, model_1_acc, model_2_prob, model_2_acc, model_3_prob, model_3_acc)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}