classifier_from_little_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.user-defined convnet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 2000 images belonging to 2 classes.\n",
      "Found 800 images belonging to 2 classes.\n",
      "Epoch 1/50\n",
      "125/125 [==============================] - 31s 250ms/step - loss: 0.7089 - acc: 0.5555 - val_loss: 0.6774 - val_acc: 0.5363\n",
      "Epoch 2/50\n",
      "125/125 [==============================] - 15s 120ms/step - loss: 0.6760 - acc: 0.5885 - val_loss: 0.6230 - val_acc: 0.6737\n",
      "Epoch 3/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.6432 - acc: 0.6375 - val_loss: 0.6037 - val_acc: 0.6937\n",
      "Epoch 4/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.6044 - acc: 0.6825 - val_loss: 0.6176 - val_acc: 0.6813\n",
      "Epoch 5/50\n",
      "125/125 [==============================] - 13s 106ms/step - loss: 0.5940 - acc: 0.6900 - val_loss: 0.5458 - val_acc: 0.7188\n",
      "Epoch 6/50\n",
      "125/125 [==============================] - 13s 108ms/step - loss: 0.5588 - acc: 0.7195 - val_loss: 0.5967 - val_acc: 0.6625\n",
      "Epoch 7/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.5507 - acc: 0.7325 - val_loss: 0.5463 - val_acc: 0.7125\n",
      "Epoch 8/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.5491 - acc: 0.7455 - val_loss: 0.5621 - val_acc: 0.7175\n",
      "Epoch 9/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.5424 - acc: 0.7375 - val_loss: 0.5189 - val_acc: 0.7275\n",
      "Epoch 10/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.5333 - acc: 0.7395 - val_loss: 0.5235 - val_acc: 0.7350\n",
      "Epoch 11/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.5265 - acc: 0.7380 - val_loss: 0.5015 - val_acc: 0.7388\n",
      "Epoch 12/50\n",
      "125/125 [==============================] - 14s 108ms/step - loss: 0.5048 - acc: 0.7625 - val_loss: 0.5838 - val_acc: 0.6987\n",
      "Epoch 13/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.4851 - acc: 0.7775 - val_loss: 0.5258 - val_acc: 0.7425\n",
      "Epoch 14/50\n",
      "125/125 [==============================] - 13s 106ms/step - loss: 0.4708 - acc: 0.7850 - val_loss: 0.4836 - val_acc: 0.7887\n",
      "Epoch 15/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.4733 - acc: 0.7875 - val_loss: 0.5301 - val_acc: 0.7375\n",
      "Epoch 16/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.4932 - acc: 0.7875 - val_loss: 0.4883 - val_acc: 0.7600\n",
      "Epoch 17/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.4690 - acc: 0.7845 - val_loss: 0.5443 - val_acc: 0.7362\n",
      "Epoch 18/50\n",
      "125/125 [==============================] - 14s 109ms/step - loss: 0.4617 - acc: 0.7850 - val_loss: 0.4980 - val_acc: 0.7650\n",
      "Epoch 19/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.4545 - acc: 0.7810 - val_loss: 0.4719 - val_acc: 0.7712\n",
      "Epoch 20/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.4617 - acc: 0.7990 - val_loss: 0.4873 - val_acc: 0.7700\n",
      "Epoch 21/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.4579 - acc: 0.7995 - val_loss: 0.4937 - val_acc: 0.7600\n",
      "Epoch 22/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.4381 - acc: 0.7935 - val_loss: 0.4474 - val_acc: 0.8025\n",
      "Epoch 23/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.4497 - acc: 0.8065 - val_loss: 0.4936 - val_acc: 0.7575\n",
      "Epoch 24/50\n",
      "125/125 [==============================] - 14s 109ms/step - loss: 0.4408 - acc: 0.8045 - val_loss: 0.5472 - val_acc: 0.7500\n",
      "Epoch 25/50\n",
      "125/125 [==============================] - 14s 110ms/step - loss: 0.4589 - acc: 0.7945 - val_loss: 0.5158 - val_acc: 0.7475\n",
      "Epoch 26/50\n",
      "125/125 [==============================] - 14s 108ms/step - loss: 0.4306 - acc: 0.7990 - val_loss: 0.4792 - val_acc: 0.7850\n",
      "Epoch 27/50\n",
      "125/125 [==============================] - 14s 111ms/step - loss: 0.4561 - acc: 0.7945 - val_loss: 0.5487 - val_acc: 0.7562\n",
      "Epoch 28/50\n",
      "125/125 [==============================] - 14s 113ms/step - loss: 0.4555 - acc: 0.8040 - val_loss: 0.5425 - val_acc: 0.7762\n",
      "Epoch 29/50\n",
      "125/125 [==============================] - 14s 111ms/step - loss: 0.4262 - acc: 0.8100 - val_loss: 0.5094 - val_acc: 0.7887\n",
      "Epoch 30/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.4251 - acc: 0.8115 - val_loss: 0.6068 - val_acc: 0.7837\n",
      "Epoch 31/50\n",
      "125/125 [==============================] - 13s 108ms/step - loss: 0.4418 - acc: 0.8025 - val_loss: 0.5382 - val_acc: 0.7762\n",
      "Epoch 32/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.4269 - acc: 0.8135 - val_loss: 0.4648 - val_acc: 0.7788\n",
      "Epoch 33/50\n",
      "125/125 [==============================] - 13s 108ms/step - loss: 0.4468 - acc: 0.8160 - val_loss: 0.5176 - val_acc: 0.7675\n",
      "Epoch 34/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.4268 - acc: 0.8275 - val_loss: 0.5225 - val_acc: 0.7775\n",
      "Epoch 35/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.4306 - acc: 0.8180 - val_loss: 0.5270 - val_acc: 0.7775\n",
      "Epoch 36/50\n",
      "125/125 [==============================] - 14s 108ms/step - loss: 0.4342 - acc: 0.8150 - val_loss: 0.5792 - val_acc: 0.7700\n",
      "Epoch 37/50\n",
      "125/125 [==============================] - 13s 108ms/step - loss: 0.4372 - acc: 0.8150 - val_loss: 0.7990 - val_acc: 0.7462\n",
      "Epoch 38/50\n",
      "125/125 [==============================] - 14s 109ms/step - loss: 0.4347 - acc: 0.8130 - val_loss: 0.5207 - val_acc: 0.7588\n",
      "Epoch 39/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.4093 - acc: 0.8285 - val_loss: 0.4617 - val_acc: 0.8100\n",
      "Epoch 40/50\n",
      "125/125 [==============================] - 14s 108ms/step - loss: 0.4195 - acc: 0.8200 - val_loss: 0.5509 - val_acc: 0.7788\n",
      "Epoch 41/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.4147 - acc: 0.8180 - val_loss: 0.5095 - val_acc: 0.8013\n",
      "Epoch 42/50\n",
      "125/125 [==============================] - 14s 109ms/step - loss: 0.4331 - acc: 0.8200 - val_loss: 0.6727 - val_acc: 0.7412\n",
      "Epoch 43/50\n",
      "125/125 [==============================] - 13s 108ms/step - loss: 0.4280 - acc: 0.8070 - val_loss: 0.5197 - val_acc: 0.7612\n",
      "Epoch 44/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.4506 - acc: 0.8065 - val_loss: 0.4797 - val_acc: 0.8013\n",
      "Epoch 45/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.4148 - acc: 0.8260 - val_loss: 0.5536 - val_acc: 0.7662\n",
      "Epoch 46/50\n",
      "125/125 [==============================] - 14s 112ms/step - loss: 0.4188 - acc: 0.8290 - val_loss: 0.5770 - val_acc: 0.7562\n",
      "Epoch 47/50\n",
      "125/125 [==============================] - 14s 109ms/step - loss: 0.4935 - acc: 0.8010 - val_loss: 0.4898 - val_acc: 0.7775\n",
      "Epoch 48/50\n",
      "125/125 [==============================] - 14s 108ms/step - loss: 0.4240 - acc: 0.8125 - val_loss: 0.9731 - val_acc: 0.6787\n",
      "Epoch 49/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.4237 - acc: 0.8185 - val_loss: 0.5027 - val_acc: 0.7825\n",
      "Epoch 50/50\n",
      "125/125 [==============================] - 13s 107ms/step - loss: 0.4214 - acc: 0.8250 - val_loss: 0.6178 - val_acc: 0.7987\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras.layers import Activation, Dropout, Flatten, Dense\n",
    "from keras import backend as K\n",
    "\n",
    "# dimensions of our images.\n",
    "img_width, img_height = 150, 150\n",
    "\n",
    "train_data_dir = 'data/train'\n",
    "validation_data_dir = 'data/validation'\n",
    "nb_train_samples = 2000\n",
    "nb_validation_samples = 800\n",
    "epochs = 50\n",
    "batch_size = 16\n",
    "\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    input_shape = (3, img_width, img_height)\n",
    "else:\n",
    "    input_shape = (img_width, img_height, 3)\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Conv2D(32, (3, 3), input_shape=input_shape))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "model.add(Conv2D(32, (3, 3)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "model.add(Conv2D(64, (3, 3)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(64))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(1))\n",
    "model.add(Activation('sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer='rmsprop',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "# this is the augmentation configuration we will use for training\n",
    "train_datagen = ImageDataGenerator(\n",
    "    rescale=1. / 255,\n",
    "    shear_range=0.2,\n",
    "    zoom_range=0.2,\n",
    "    horizontal_flip=True)\n",
    "\n",
    "# this is the augmentation configuration we will use for testing:\n",
    "# only rescaling\n",
    "test_datagen = ImageDataGenerator(rescale=1. / 255)\n",
    "\n",
    "train_generator = train_datagen.flow_from_directory(\n",
    "    train_data_dir,\n",
    "    target_size=(img_width, img_height),\n",
    "    batch_size=batch_size,\n",
    "    class_mode='binary')\n",
    "\n",
    "validation_generator = test_datagen.flow_from_directory(\n",
    "    validation_data_dir,\n",
    "    target_size=(img_width, img_height),\n",
    "    batch_size=batch_size,\n",
    "    class_mode='binary')\n",
    "\n",
    "model.fit_generator(\n",
    "    train_generator,\n",
    "    steps_per_epoch=nb_train_samples // batch_size,\n",
    "    epochs=epochs,\n",
    "    validation_data=validation_generator,\n",
    "    validation_steps=nb_validation_samples // batch_size)\n",
    "\n",
    "model.save_weights('first_try.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.Feature extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 2000 images belonging to 2 classes.\n",
      "Found 800 images belonging to 2 classes.\n",
      "Train on 2000 samples, validate on 800 samples\n",
      "Epoch 1/50\n",
      "2000/2000 [==============================] - 1s 357us/step - loss: 0.7105 - acc: 0.7555 - val_loss: 0.4812 - val_acc: 0.7812\n",
      "Epoch 2/50\n",
      "2000/2000 [==============================] - 1s 306us/step - loss: 0.3632 - acc: 0.8455 - val_loss: 0.2504 - val_acc: 0.8975\n",
      "Epoch 3/50\n",
      "2000/2000 [==============================] - 1s 305us/step - loss: 0.3067 - acc: 0.8800 - val_loss: 0.2682 - val_acc: 0.9125\n",
      "Epoch 4/50\n",
      "2000/2000 [==============================] - 1s 309us/step - loss: 0.2638 - acc: 0.8945 - val_loss: 0.3771 - val_acc: 0.8562\n",
      "Epoch 5/50\n",
      "2000/2000 [==============================] - 1s 307us/step - loss: 0.2289 - acc: 0.9070 - val_loss: 0.3375 - val_acc: 0.8800\n",
      "Epoch 6/50\n",
      "2000/2000 [==============================] - 1s 307us/step - loss: 0.2177 - acc: 0.9230 - val_loss: 0.2642 - val_acc: 0.8988\n",
      "Epoch 7/50\n",
      "2000/2000 [==============================] - 1s 315us/step - loss: 0.1976 - acc: 0.9265 - val_loss: 0.3649 - val_acc: 0.8600\n",
      "Epoch 8/50\n",
      "2000/2000 [==============================] - 1s 313us/step - loss: 0.1914 - acc: 0.9265 - val_loss: 0.3553 - val_acc: 0.8712\n",
      "Epoch 9/50\n",
      "2000/2000 [==============================] - 1s 317us/step - loss: 0.1517 - acc: 0.9395 - val_loss: 0.3221 - val_acc: 0.9012\n",
      "Epoch 10/50\n",
      "2000/2000 [==============================] - 1s 333us/step - loss: 0.1405 - acc: 0.9555 - val_loss: 0.3799 - val_acc: 0.8962\n",
      "Epoch 11/50\n",
      "2000/2000 [==============================] - 1s 351us/step - loss: 0.1187 - acc: 0.9565 - val_loss: 0.4627 - val_acc: 0.8750\n",
      "Epoch 12/50\n",
      "2000/2000 [==============================] - 1s 340us/step - loss: 0.1060 - acc: 0.9600 - val_loss: 0.5297 - val_acc: 0.8862\n",
      "Epoch 13/50\n",
      "2000/2000 [==============================] - 1s 324us/step - loss: 0.1007 - acc: 0.9630 - val_loss: 0.5451 - val_acc: 0.8525\n",
      "Epoch 14/50\n",
      "2000/2000 [==============================] - 1s 349us/step - loss: 0.0963 - acc: 0.9670 - val_loss: 0.5844 - val_acc: 0.8812\n",
      "Epoch 15/50\n",
      "2000/2000 [==============================] - 1s 315us/step - loss: 0.0807 - acc: 0.9720 - val_loss: 0.4015 - val_acc: 0.8950\n",
      "Epoch 16/50\n",
      "2000/2000 [==============================] - 1s 311us/step - loss: 0.0740 - acc: 0.9720 - val_loss: 0.5324 - val_acc: 0.8875\n",
      "Epoch 17/50\n",
      "2000/2000 [==============================] - 1s 345us/step - loss: 0.0648 - acc: 0.9770 - val_loss: 0.5512 - val_acc: 0.8988\n",
      "Epoch 18/50\n",
      "2000/2000 [==============================] - 1s 326us/step - loss: 0.0636 - acc: 0.9755 - val_loss: 0.5232 - val_acc: 0.9012\n",
      "Epoch 19/50\n",
      "2000/2000 [==============================] - 1s 315us/step - loss: 0.0616 - acc: 0.9790 - val_loss: 0.6277 - val_acc: 0.8750\n",
      "Epoch 20/50\n",
      "2000/2000 [==============================] - 1s 333us/step - loss: 0.0378 - acc: 0.9860 - val_loss: 0.6000 - val_acc: 0.8988\n",
      "Epoch 21/50\n",
      "2000/2000 [==============================] - 1s 308us/step - loss: 0.0585 - acc: 0.9790 - val_loss: 0.6029 - val_acc: 0.8988\n",
      "Epoch 22/50\n",
      "2000/2000 [==============================] - 1s 330us/step - loss: 0.0423 - acc: 0.9870 - val_loss: 0.6599 - val_acc: 0.8900\n",
      "Epoch 23/50\n",
      "2000/2000 [==============================] - 1s 324us/step - loss: 0.0292 - acc: 0.9880 - val_loss: 0.7533 - val_acc: 0.8838\n",
      "Epoch 24/50\n",
      "2000/2000 [==============================] - 1s 349us/step - loss: 0.0419 - acc: 0.9835 - val_loss: 0.6340 - val_acc: 0.8875\n",
      "Epoch 25/50\n",
      "2000/2000 [==============================] - 1s 318us/step - loss: 0.0355 - acc: 0.9885 - val_loss: 0.6432 - val_acc: 0.8950\n",
      "Epoch 26/50\n",
      "2000/2000 [==============================] - 1s 305us/step - loss: 0.0320 - acc: 0.9885 - val_loss: 0.6678 - val_acc: 0.8950\n",
      "Epoch 27/50\n",
      "2000/2000 [==============================] - 1s 336us/step - loss: 0.0324 - acc: 0.9905 - val_loss: 0.6395 - val_acc: 0.8975\n",
      "Epoch 28/50\n",
      "2000/2000 [==============================] - 1s 339us/step - loss: 0.0265 - acc: 0.9905 - val_loss: 0.9504 - val_acc: 0.8688\n",
      "Epoch 29/50\n",
      "2000/2000 [==============================] - 1s 333us/step - loss: 0.0321 - acc: 0.9925 - val_loss: 0.7385 - val_acc: 0.8875\n",
      "Epoch 30/50\n",
      "2000/2000 [==============================] - 1s 320us/step - loss: 0.0191 - acc: 0.9925 - val_loss: 0.9142 - val_acc: 0.8775\n",
      "Epoch 31/50\n",
      "2000/2000 [==============================] - 1s 336us/step - loss: 0.0335 - acc: 0.9900 - val_loss: 1.0616 - val_acc: 0.8650\n",
      "Epoch 32/50\n",
      "2000/2000 [==============================] - 1s 324us/step - loss: 0.0139 - acc: 0.9940 - val_loss: 1.4463 - val_acc: 0.8225\n",
      "Epoch 33/50\n",
      "2000/2000 [==============================] - 1s 312us/step - loss: 0.0305 - acc: 0.9900 - val_loss: 0.7823 - val_acc: 0.8912\n",
      "Epoch 34/50\n",
      "2000/2000 [==============================] - 1s 326us/step - loss: 0.0233 - acc: 0.9930 - val_loss: 1.0125 - val_acc: 0.8762\n",
      "Epoch 35/50\n",
      "2000/2000 [==============================] - 1s 325us/step - loss: 0.0233 - acc: 0.9920 - val_loss: 0.7708 - val_acc: 0.8888\n",
      "Epoch 36/50\n",
      "2000/2000 [==============================] - 1s 345us/step - loss: 0.0214 - acc: 0.9945 - val_loss: 0.7433 - val_acc: 0.8925\n",
      "Epoch 37/50\n",
      "2000/2000 [==============================] - 1s 328us/step - loss: 0.0143 - acc: 0.9960 - val_loss: 0.8040 - val_acc: 0.8938\n",
      "Epoch 38/50\n",
      "2000/2000 [==============================] - 1s 317us/step - loss: 0.0244 - acc: 0.9915 - val_loss: 0.8735 - val_acc: 0.8800\n",
      "Epoch 39/50\n",
      "2000/2000 [==============================] - 1s 320us/step - loss: 0.0132 - acc: 0.9950 - val_loss: 0.8662 - val_acc: 0.8938\n",
      "Epoch 40/50\n",
      "2000/2000 [==============================] - 1s 318us/step - loss: 0.0224 - acc: 0.9965 - val_loss: 0.8074 - val_acc: 0.8925\n",
      "Epoch 41/50\n",
      "2000/2000 [==============================] - 1s 344us/step - loss: 0.0273 - acc: 0.9940 - val_loss: 0.8362 - val_acc: 0.8888\n",
      "Epoch 42/50\n",
      "2000/2000 [==============================] - 1s 319us/step - loss: 0.0131 - acc: 0.9950 - val_loss: 0.8852 - val_acc: 0.8950\n",
      "Epoch 43/50\n",
      "2000/2000 [==============================] - 1s 306us/step - loss: 0.0122 - acc: 0.9965 - val_loss: 1.0102 - val_acc: 0.8738\n",
      "Epoch 44/50\n",
      "2000/2000 [==============================] - 1s 360us/step - loss: 0.0239 - acc: 0.9945 - val_loss: 0.8518 - val_acc: 0.8900\n",
      "Epoch 45/50\n",
      "2000/2000 [==============================] - 1s 309us/step - loss: 0.0097 - acc: 0.9960 - val_loss: 0.9587 - val_acc: 0.8875\n",
      "Epoch 46/50\n",
      "2000/2000 [==============================] - 1s 328us/step - loss: 0.0261 - acc: 0.9915 - val_loss: 0.9202 - val_acc: 0.8825\n",
      "Epoch 47/50\n",
      "2000/2000 [==============================] - 1s 319us/step - loss: 0.0104 - acc: 0.9970 - val_loss: 0.8639 - val_acc: 0.8925\n",
      "Epoch 48/50\n",
      "2000/2000 [==============================] - 1s 315us/step - loss: 0.0248 - acc: 0.9955 - val_loss: 0.8292 - val_acc: 0.8850\n",
      "Epoch 49/50\n",
      "2000/2000 [==============================] - 1s 307us/step - loss: 0.0059 - acc: 0.9980 - val_loss: 0.9302 - val_acc: 0.8938\n",
      "Epoch 50/50\n",
      "2000/2000 [==============================] - 1s 308us/step - loss: 0.0229 - acc: 0.9935 - val_loss: 0.8796 - val_acc: 0.8925\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dropout, Flatten, Dense\n",
    "from keras import applications\n",
    "\n",
    "# dimensions of our images.\n",
    "img_width, img_height = 150, 150\n",
    "\n",
    "top_model_weights_path = 'bottleneck_fc_model.h5'\n",
    "train_data_dir = 'data/train'\n",
    "validation_data_dir = 'data/validation'\n",
    "nb_train_samples = 2000\n",
    "nb_validation_samples = 800\n",
    "epochs = 50\n",
    "batch_size = 16\n",
    "\n",
    "def save_bottlebeck_features():\n",
    "    datagen = ImageDataGenerator(rescale=1. / 255)\n",
    "\n",
    "    # build the VGG16 network\n",
    "    model = applications.VGG16(include_top=False, weights='imagenet')\n",
    "\n",
    "    generator = datagen.flow_from_directory(\n",
    "        train_data_dir,\n",
    "        target_size=(img_width, img_height),\n",
    "        batch_size=batch_size,\n",
    "        class_mode=None,\n",
    "        shuffle=False)\n",
    "    bottleneck_features_train = model.predict_generator(\n",
    "        generator, nb_train_samples // batch_size)\n",
    "    np.save(open('bottleneck_features_train.npy', 'w'),\n",
    "            bottleneck_features_train)\n",
    "\n",
    "    generator = datagen.flow_from_directory(\n",
    "        validation_data_dir,\n",
    "        target_size=(img_width, img_height),\n",
    "        batch_size=batch_size,\n",
    "        class_mode=None,\n",
    "        shuffle=False)\n",
    "    bottleneck_features_validation = model.predict_generator(\n",
    "        generator, nb_validation_samples // batch_size)\n",
    "    np.save(open('bottleneck_features_validation.npy', 'w'),\n",
    "            bottleneck_features_validation)\n",
    "\n",
    "def train_top_model():\n",
    "    train_data = np.load(open('bottleneck_features_train.npy'))\n",
    "    train_labels = np.array(\n",
    "        [0] * (nb_train_samples / 2) + [1] * (nb_train_samples / 2))\n",
    "\n",
    "    validation_data = np.load(open('bottleneck_features_validation.npy'))\n",
    "    validation_labels = np.array(\n",
    "        [0] * (nb_validation_samples / 2) + [1] * (nb_validation_samples / 2))\n",
    "\n",
    "    model = Sequential()\n",
    "    model.add(Flatten(input_shape=train_data.shape[1:]))\n",
    "    model.add(Dense(256, activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "    model.compile(optimizer='rmsprop',\n",
    "                  loss='binary_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "    model.fit(train_data, train_labels,\n",
    "              epochs=epochs,\n",
    "              batch_size=batch_size,\n",
    "              validation_data=(validation_data, validation_labels))\n",
    "    model.save_weights(top_model_weights_path)\n",
    "\n",
    "save_bottlebeck_features()\n",
    "train_top_model()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.fine-tune the convnet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model loaded.\n",
      "Found 2000 images belonging to 2 classes.\n",
      "Found 800 images belonging to 2 classes.\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 150, 150, 3)       0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 150, 150, 64)      1792      \n",
      "_________________________________________________________________\n",
      "block1_conv2 (Conv2D)        (None, 150, 150, 64)      36928     \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 75, 75, 64)        0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, 75, 75, 128)       73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 75, 75, 128)       147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 37, 37, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 37, 37, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 37, 37, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 37, 37, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 18, 18, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 18, 18, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 18, 18, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 18, 18, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 9, 9, 512)         0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 9, 9, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 9, 9, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 9, 9, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 4, 4, 512)         0         \n",
      "_________________________________________________________________\n",
      "sequential_3 (Sequential)    (None, 1)                 2097665   \n",
      "=================================================================\n",
      "Total params: 16,812,353\n",
      "Trainable params: 9,177,089\n",
      "Non-trainable params: 7,635,264\n",
      "_________________________________________________________________\n",
      "Epoch 1/50\n",
      " - 15s - loss: 0.3715 - acc: 0.8725 - val_loss: 0.3659 - val_acc: 0.8938\n",
      "Epoch 2/50\n",
      " - 14s - loss: 0.2603 - acc: 0.9045 - val_loss: 0.2937 - val_acc: 0.8988\n",
      "Epoch 3/50\n",
      " - 14s - loss: 0.1627 - acc: 0.9365 - val_loss: 0.2748 - val_acc: 0.9237\n",
      "Epoch 4/50\n",
      " - 14s - loss: 0.1659 - acc: 0.9390 - val_loss: 0.2317 - val_acc: 0.9087\n",
      "Epoch 5/50\n",
      " - 14s - loss: 0.1429 - acc: 0.9455 - val_loss: 0.3563 - val_acc: 0.9175\n",
      "Epoch 6/50\n",
      " - 14s - loss: 0.1435 - acc: 0.9415 - val_loss: 0.3152 - val_acc: 0.9200\n",
      "Epoch 7/50\n",
      " - 14s - loss: 0.0798 - acc: 0.9670 - val_loss: 0.3890 - val_acc: 0.9187\n",
      "Epoch 8/50\n",
      " - 14s - loss: 0.0957 - acc: 0.9660 - val_loss: 0.2808 - val_acc: 0.9175\n",
      "Epoch 9/50\n",
      " - 14s - loss: 0.0867 - acc: 0.9725 - val_loss: 0.3106 - val_acc: 0.9200\n",
      "Epoch 10/50\n",
      " - 14s - loss: 0.0726 - acc: 0.9755 - val_loss: 0.4084 - val_acc: 0.9025\n",
      "Epoch 11/50\n",
      " - 14s - loss: 0.0528 - acc: 0.9825 - val_loss: 0.3168 - val_acc: 0.9275\n",
      "Epoch 12/50\n",
      " - 14s - loss: 0.0622 - acc: 0.9795 - val_loss: 0.3072 - val_acc: 0.9350\n",
      "Epoch 13/50\n",
      " - 14s - loss: 0.0526 - acc: 0.9805 - val_loss: 0.3163 - val_acc: 0.9250\n",
      "Epoch 14/50\n",
      " - 14s - loss: 0.0630 - acc: 0.9835 - val_loss: 0.2458 - val_acc: 0.9313\n",
      "Epoch 15/50\n",
      " - 14s - loss: 0.0376 - acc: 0.9855 - val_loss: 0.4641 - val_acc: 0.9187\n",
      "Epoch 16/50\n",
      " - 14s - loss: 0.0573 - acc: 0.9815 - val_loss: 0.3046 - val_acc: 0.9400\n",
      "Epoch 17/50\n",
      " - 14s - loss: 0.0508 - acc: 0.9850 - val_loss: 0.3575 - val_acc: 0.9250\n",
      "Epoch 18/50\n",
      " - 14s - loss: 0.0350 - acc: 0.9860 - val_loss: 0.2890 - val_acc: 0.9237\n",
      "Epoch 19/50\n",
      " - 14s - loss: 0.0420 - acc: 0.9835 - val_loss: 0.3344 - val_acc: 0.9300\n",
      "Epoch 20/50\n",
      " - 14s - loss: 0.0393 - acc: 0.9865 - val_loss: 0.3465 - val_acc: 0.9075\n",
      "Epoch 21/50\n",
      " - 14s - loss: 0.0488 - acc: 0.9845 - val_loss: 0.2701 - val_acc: 0.9275\n",
      "Epoch 22/50\n",
      " - 14s - loss: 0.0280 - acc: 0.9920 - val_loss: 0.3501 - val_acc: 0.9175\n",
      "Epoch 23/50\n",
      " - 14s - loss: 0.0199 - acc: 0.9945 - val_loss: 0.3821 - val_acc: 0.9350\n",
      "Epoch 24/50\n",
      " - 14s - loss: 0.0309 - acc: 0.9885 - val_loss: 0.2827 - val_acc: 0.9337\n",
      "Epoch 25/50\n",
      " - 14s - loss: 0.0276 - acc: 0.9915 - val_loss: 0.3428 - val_acc: 0.9213\n",
      "Epoch 26/50\n",
      " - 14s - loss: 0.0400 - acc: 0.9910 - val_loss: 0.2730 - val_acc: 0.9350\n",
      "Epoch 27/50\n",
      " - 14s - loss: 0.0294 - acc: 0.9895 - val_loss: 0.3399 - val_acc: 0.9250\n",
      "Epoch 28/50\n",
      " - 14s - loss: 0.0107 - acc: 0.9975 - val_loss: 0.4212 - val_acc: 0.9237\n",
      "Epoch 29/50\n",
      " - 14s - loss: 0.0143 - acc: 0.9955 - val_loss: 0.4036 - val_acc: 0.9350\n",
      "Epoch 30/50\n",
      " - 14s - loss: 0.0211 - acc: 0.9930 - val_loss: 0.3973 - val_acc: 0.9287\n",
      "Epoch 31/50\n",
      " - 14s - loss: 0.0434 - acc: 0.9875 - val_loss: 0.3429 - val_acc: 0.9363\n",
      "Epoch 32/50\n",
      " - 14s - loss: 0.0135 - acc: 0.9950 - val_loss: 0.4766 - val_acc: 0.9300\n",
      "Epoch 33/50\n",
      " - 14s - loss: 0.0280 - acc: 0.9885 - val_loss: 0.3023 - val_acc: 0.9313\n",
      "Epoch 34/50\n",
      " - 14s - loss: 0.0138 - acc: 0.9955 - val_loss: 0.3226 - val_acc: 0.9413\n",
      "Epoch 35/50\n",
      " - 14s - loss: 0.0152 - acc: 0.9960 - val_loss: 0.4136 - val_acc: 0.9375\n",
      "Epoch 36/50\n",
      " - 14s - loss: 0.0197 - acc: 0.9930 - val_loss: 0.3554 - val_acc: 0.9313\n",
      "Epoch 37/50\n",
      " - 14s - loss: 0.0176 - acc: 0.9945 - val_loss: 0.4826 - val_acc: 0.9287\n",
      "Epoch 38/50\n",
      " - 14s - loss: 0.0157 - acc: 0.9940 - val_loss: 0.5301 - val_acc: 0.9287\n",
      "Epoch 39/50\n",
      " - 14s - loss: 0.0194 - acc: 0.9915 - val_loss: 0.4547 - val_acc: 0.9387\n",
      "Epoch 40/50\n",
      " - 14s - loss: 0.0234 - acc: 0.9940 - val_loss: 0.4648 - val_acc: 0.9300\n",
      "Epoch 41/50\n",
      " - 14s - loss: 0.0086 - acc: 0.9965 - val_loss: 0.5046 - val_acc: 0.9313\n",
      "Epoch 42/50\n",
      " - 14s - loss: 0.0065 - acc: 0.9980 - val_loss: 0.4849 - val_acc: 0.9325\n",
      "Epoch 43/50\n",
      " - 14s - loss: 0.0625 - acc: 0.9890 - val_loss: 0.4474 - val_acc: 0.9087\n",
      "Epoch 44/50\n",
      " - 14s - loss: 0.0144 - acc: 0.9960 - val_loss: 0.4337 - val_acc: 0.9275\n",
      "Epoch 45/50\n",
      " - 14s - loss: 0.0179 - acc: 0.9920 - val_loss: 0.3557 - val_acc: 0.9250\n",
      "Epoch 46/50\n",
      " - 14s - loss: 0.0159 - acc: 0.9950 - val_loss: 0.4520 - val_acc: 0.9200\n",
      "Epoch 47/50\n",
      " - 14s - loss: 0.0153 - acc: 0.9950 - val_loss: 0.3654 - val_acc: 0.9337\n",
      "Epoch 48/50\n",
      " - 14s - loss: 0.0106 - acc: 0.9965 - val_loss: 0.3865 - val_acc: 0.9350\n",
      "Epoch 49/50\n",
      " - 14s - loss: 0.0029 - acc: 0.9990 - val_loss: 0.4377 - val_acc: 0.9363\n",
      "Epoch 50/50\n",
      " - 14s - loss: 0.0064 - acc: 0.9975 - val_loss: 0.5257 - val_acc: 0.9250\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fca5c135f50>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras import applications\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras import optimizers\n",
    "from keras.models import Sequential\n",
    "from keras.models import Model\n",
    "from keras.layers import Dropout, Flatten, Dense\n",
    "\n",
    "# path to the model weights files.\n",
    "top_model_weights_path = 'bottleneck_fc_model.h5'\n",
    "# dimensions of our images.\n",
    "img_width, img_height = 150, 150\n",
    "\n",
    "train_data_dir = 'data/train'\n",
    "validation_data_dir = 'data/validation'\n",
    "nb_train_samples = 2000\n",
    "nb_validation_samples = 800\n",
    "epochs = 50\n",
    "batch_size = 16\n",
    "\n",
    "# build the VGG16 network\n",
    "base_model = applications.VGG16(weights='imagenet', include_top=False, input_shape=(150,150,3))\n",
    "print('Model loaded.')\n",
    "\n",
    "# build a classifier model to put on top of the convolutional model\n",
    "top_model = Sequential()\n",
    "top_model.add(Flatten(input_shape=base_model.output_shape[1:]))\n",
    "top_model.add(Dense(256, activation='relu'))\n",
    "top_model.add(Dropout(0.5))\n",
    "top_model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "# note that it is necessary to start with a fully-trained\n",
    "# classifier, including the top classifier,\n",
    "# in order to successfully do fine-tuning\n",
    "top_model.load_weights(top_model_weights_path)\n",
    "\n",
    "# add the model on top of the convolutional base\n",
    "# model.add(top_model)\n",
    "model = Model(inputs=base_model.input, outputs=top_model(base_model.output))\n",
    "\n",
    "# set the first 25 layers (up to the last conv block)\n",
    "# to non-trainable (weights will not be updated)\n",
    "for layer in model.layers[:15]:\n",
    "    layer.trainable = False\n",
    "\n",
    "# compile the model with a SGD/momentum optimizer\n",
    "# and a very slow learning rate.\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "# prepare data augmentation configuration\n",
    "train_datagen = ImageDataGenerator(\n",
    "    rescale=1. / 255,\n",
    "    shear_range=0.2,\n",
    "    zoom_range=0.2,\n",
    "    horizontal_flip=True)\n",
    "\n",
    "test_datagen = ImageDataGenerator(rescale=1. / 255)\n",
    "\n",
    "train_generator = train_datagen.flow_from_directory(\n",
    "    train_data_dir,\n",
    "    target_size=(img_height, img_width),\n",
    "    batch_size=batch_size,\n",
    "    class_mode='binary')\n",
    "\n",
    "validation_generator = test_datagen.flow_from_directory(\n",
    "    validation_data_dir,\n",
    "    target_size=(img_height, img_width),\n",
    "    batch_size=batch_size,\n",
    "    class_mode='binary')\n",
    "\n",
    "model.summary()\n",
    "\n",
    "# fine-tune the model\n",
    "model.fit_generator(\n",
    "    train_generator,\n",
    "    steps_per_epoch=nb_train_samples // batch_size,\n",
    "    epochs=epochs,\n",
    "    validation_data=validation_generator,\n",
    "    validation_steps=nb_validation_samples // batch_size,\n",
    "    verbose=2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}