features_extractor.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Keras features extractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing import image\n",
    "from keras.models import Model\n",
    "import numpy as np\n",
    "\n",
    "from glob import glob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## VGG16 Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### See the architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 224, 224, 3)       0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "block1_conv2 (Conv2D)        (None, 224, 224, 64)      36928     \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, 112, 112, 128)     73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 138,357,544\n",
      "Trainable params: 138,357,544\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.applications.vgg16 import VGG16, preprocess_input\n",
    "vgg16_model = VGG16(weights='imagenet', include_top=True)\n",
    "\n",
    "vgg16_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### load the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vgg16_model_extractfeatures = Model(inputs=vgg16_model.input, outputs=vgg16_model.get_layer('fc2').output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def path_to_tensor(img_path):\n",
    "    # loads RGB image as PIL.Image.Image type\n",
    "    img = image.load_img(img_path, target_size=(224, 224))\n",
    "    # convert PIL.Image.Image type to 3D tensor with shape (224, 224, 3)\n",
    "    x = image.img_to_array(img)\n",
    "    # convert 3D tensor to 4D tensor with shape (1, 224, 224, 3) and return 4D tensor\n",
    "    return np.expand_dims(x, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def extract_features(img_path, model):\n",
    "    x = preprocess_input(path_to_tensor(img_path))\n",
    "    fc2_features = model.predict(x)\n",
    "    return fc2_features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Extract features of a single image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "img_path = 'COCO/images/COCO_train2014_000000357413.jpg'\n",
    "feats = extract_features(img_path, vgg16_model_extractfeatures)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 4096)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feats.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resnet Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### See architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_4 (InputLayer)             (None, 224, 224, 3)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv1 (Conv2D)                   (None, 112, 112, 64)  9472        input_4[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "bn_conv1 (BatchNormalization)    (None, 112, 112, 64)  256         conv1[0][0]                      \n",
      "____________________________________________________________________________________________________\n",
      "activation_50 (Activation)       (None, 112, 112, 64)  0           bn_conv1[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2D)   (None, 55, 55, 64)    0           activation_50[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res2a_branch2a (Conv2D)          (None, 55, 55, 64)    4160        max_pooling2d_2[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "bn2a_branch2a (BatchNormalizatio (None, 55, 55, 64)    256         res2a_branch2a[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_51 (Activation)       (None, 55, 55, 64)    0           bn2a_branch2a[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res2a_branch2b (Conv2D)          (None, 55, 55, 64)    36928       activation_51[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn2a_branch2b (BatchNormalizatio (None, 55, 55, 64)    256         res2a_branch2b[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_52 (Activation)       (None, 55, 55, 64)    0           bn2a_branch2b[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res2a_branch2c (Conv2D)          (None, 55, 55, 256)   16640       activation_52[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res2a_branch1 (Conv2D)           (None, 55, 55, 256)   16640       max_pooling2d_2[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "bn2a_branch2c (BatchNormalizatio (None, 55, 55, 256)   1024        res2a_branch2c[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "bn2a_branch1 (BatchNormalization (None, 55, 55, 256)   1024        res2a_branch1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "add_17 (Add)                     (None, 55, 55, 256)   0           bn2a_branch2c[0][0]              \n",
      "                                                                   bn2a_branch1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "activation_53 (Activation)       (None, 55, 55, 256)   0           add_17[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "res2b_branch2a (Conv2D)          (None, 55, 55, 64)    16448       activation_53[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn2b_branch2a (BatchNormalizatio (None, 55, 55, 64)    256         res2b_branch2a[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_54 (Activation)       (None, 55, 55, 64)    0           bn2b_branch2a[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res2b_branch2b (Conv2D)          (None, 55, 55, 64)    36928       activation_54[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn2b_branch2b (BatchNormalizatio (None, 55, 55, 64)    256         res2b_branch2b[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_55 (Activation)       (None, 55, 55, 64)    0           bn2b_branch2b[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res2b_branch2c (Conv2D)          (None, 55, 55, 256)   16640       activation_55[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn2b_branch2c (BatchNormalizatio (None, 55, 55, 256)   1024        res2b_branch2c[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "add_18 (Add)                     (None, 55, 55, 256)   0           bn2b_branch2c[0][0]              \n",
      "                                                                   activation_53[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_56 (Activation)       (None, 55, 55, 256)   0           add_18[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "res2c_branch2a (Conv2D)          (None, 55, 55, 64)    16448       activation_56[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn2c_branch2a (BatchNormalizatio (None, 55, 55, 64)    256         res2c_branch2a[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_57 (Activation)       (None, 55, 55, 64)    0           bn2c_branch2a[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res2c_branch2b (Conv2D)          (None, 55, 55, 64)    36928       activation_57[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn2c_branch2b (BatchNormalizatio (None, 55, 55, 64)    256         res2c_branch2b[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_58 (Activation)       (None, 55, 55, 64)    0           bn2c_branch2b[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res2c_branch2c (Conv2D)          (None, 55, 55, 256)   16640       activation_58[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn2c_branch2c (BatchNormalizatio (None, 55, 55, 256)   1024        res2c_branch2c[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "add_19 (Add)                     (None, 55, 55, 256)   0           bn2c_branch2c[0][0]              \n",
      "                                                                   activation_56[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_59 (Activation)       (None, 55, 55, 256)   0           add_19[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "res3a_branch2a (Conv2D)          (None, 28, 28, 128)   32896       activation_59[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn3a_branch2a (BatchNormalizatio (None, 28, 28, 128)   512         res3a_branch2a[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_60 (Activation)       (None, 28, 28, 128)   0           bn3a_branch2a[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res3a_branch2b (Conv2D)          (None, 28, 28, 128)   147584      activation_60[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn3a_branch2b (BatchNormalizatio (None, 28, 28, 128)   512         res3a_branch2b[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_61 (Activation)       (None, 28, 28, 128)   0           bn3a_branch2b[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res3a_branch2c (Conv2D)          (None, 28, 28, 512)   66048       activation_61[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res3a_branch1 (Conv2D)           (None, 28, 28, 512)   131584      activation_59[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn3a_branch2c (BatchNormalizatio (None, 28, 28, 512)   2048        res3a_branch2c[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "bn3a_branch1 (BatchNormalization (None, 28, 28, 512)   2048        res3a_branch1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "add_20 (Add)                     (None, 28, 28, 512)   0           bn3a_branch2c[0][0]              \n",
      "                                                                   bn3a_branch1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "activation_62 (Activation)       (None, 28, 28, 512)   0           add_20[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "res3b_branch2a (Conv2D)          (None, 28, 28, 128)   65664       activation_62[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn3b_branch2a (BatchNormalizatio (None, 28, 28, 128)   512         res3b_branch2a[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_63 (Activation)       (None, 28, 28, 128)   0           bn3b_branch2a[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res3b_branch2b (Conv2D)          (None, 28, 28, 128)   147584      activation_63[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn3b_branch2b (BatchNormalizatio (None, 28, 28, 128)   512         res3b_branch2b[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_64 (Activation)       (None, 28, 28, 128)   0           bn3b_branch2b[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res3b_branch2c (Conv2D)          (None, 28, 28, 512)   66048       activation_64[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn3b_branch2c (BatchNormalizatio (None, 28, 28, 512)   2048        res3b_branch2c[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "add_21 (Add)                     (None, 28, 28, 512)   0           bn3b_branch2c[0][0]              \n",
      "                                                                   activation_62[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_65 (Activation)       (None, 28, 28, 512)   0           add_21[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "res3c_branch2a (Conv2D)          (None, 28, 28, 128)   65664       activation_65[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn3c_branch2a (BatchNormalizatio (None, 28, 28, 128)   512         res3c_branch2a[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_66 (Activation)       (None, 28, 28, 128)   0           bn3c_branch2a[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res3c_branch2b (Conv2D)          (None, 28, 28, 128)   147584      activation_66[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn3c_branch2b (BatchNormalizatio (None, 28, 28, 128)   512         res3c_branch2b[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_67 (Activation)       (None, 28, 28, 128)   0           bn3c_branch2b[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res3c_branch2c (Conv2D)          (None, 28, 28, 512)   66048       activation_67[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn3c_branch2c (BatchNormalizatio (None, 28, 28, 512)   2048        res3c_branch2c[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "add_22 (Add)                     (None, 28, 28, 512)   0           bn3c_branch2c[0][0]              \n",
      "                                                                   activation_65[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_68 (Activation)       (None, 28, 28, 512)   0           add_22[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "res3d_branch2a (Conv2D)          (None, 28, 28, 128)   65664       activation_68[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn3d_branch2a (BatchNormalizatio (None, 28, 28, 128)   512         res3d_branch2a[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_69 (Activation)       (None, 28, 28, 128)   0           bn3d_branch2a[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res3d_branch2b (Conv2D)          (None, 28, 28, 128)   147584      activation_69[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn3d_branch2b (BatchNormalizatio (None, 28, 28, 128)   512         res3d_branch2b[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_70 (Activation)       (None, 28, 28, 128)   0           bn3d_branch2b[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res3d_branch2c (Conv2D)          (None, 28, 28, 512)   66048       activation_70[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn3d_branch2c (BatchNormalizatio (None, 28, 28, 512)   2048        res3d_branch2c[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "add_23 (Add)                     (None, 28, 28, 512)   0           bn3d_branch2c[0][0]              \n",
      "                                                                   activation_68[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_71 (Activation)       (None, 28, 28, 512)   0           add_23[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "res4a_branch2a (Conv2D)          (None, 14, 14, 256)   131328      activation_71[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn4a_branch2a (BatchNormalizatio (None, 14, 14, 256)   1024        res4a_branch2a[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_72 (Activation)       (None, 14, 14, 256)   0           bn4a_branch2a[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res4a_branch2b (Conv2D)          (None, 14, 14, 256)   590080      activation_72[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn4a_branch2b (BatchNormalizatio (None, 14, 14, 256)   1024        res4a_branch2b[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_73 (Activation)       (None, 14, 14, 256)   0           bn4a_branch2b[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res4a_branch2c (Conv2D)          (None, 14, 14, 1024)  263168      activation_73[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res4a_branch1 (Conv2D)           (None, 14, 14, 1024)  525312      activation_71[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn4a_branch2c (BatchNormalizatio (None, 14, 14, 1024)  4096        res4a_branch2c[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "bn4a_branch1 (BatchNormalization (None, 14, 14, 1024)  4096        res4a_branch1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "add_24 (Add)                     (None, 14, 14, 1024)  0           bn4a_branch2c[0][0]              \n",
      "                                                                   bn4a_branch1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "activation_74 (Activation)       (None, 14, 14, 1024)  0           add_24[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "res4b_branch2a (Conv2D)          (None, 14, 14, 256)   262400      activation_74[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn4b_branch2a (BatchNormalizatio (None, 14, 14, 256)   1024        res4b_branch2a[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_75 (Activation)       (None, 14, 14, 256)   0           bn4b_branch2a[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res4b_branch2b (Conv2D)          (None, 14, 14, 256)   590080      activation_75[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn4b_branch2b (BatchNormalizatio (None, 14, 14, 256)   1024        res4b_branch2b[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_76 (Activation)       (None, 14, 14, 256)   0           bn4b_branch2b[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res4b_branch2c (Conv2D)          (None, 14, 14, 1024)  263168      activation_76[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn4b_branch2c (BatchNormalizatio (None, 14, 14, 1024)  4096        res4b_branch2c[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "add_25 (Add)                     (None, 14, 14, 1024)  0           bn4b_branch2c[0][0]              \n",
      "                                                                   activation_74[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_77 (Activation)       (None, 14, 14, 1024)  0           add_25[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "res4c_branch2a (Conv2D)          (None, 14, 14, 256)   262400      activation_77[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn4c_branch2a (BatchNormalizatio (None, 14, 14, 256)   1024        res4c_branch2a[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_78 (Activation)       (None, 14, 14, 256)   0           bn4c_branch2a[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res4c_branch2b (Conv2D)          (None, 14, 14, 256)   590080      activation_78[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn4c_branch2b (BatchNormalizatio (None, 14, 14, 256)   1024        res4c_branch2b[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_79 (Activation)       (None, 14, 14, 256)   0           bn4c_branch2b[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res4c_branch2c (Conv2D)          (None, 14, 14, 1024)  263168      activation_79[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn4c_branch2c (BatchNormalizatio (None, 14, 14, 1024)  4096        res4c_branch2c[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "add_26 (Add)                     (None, 14, 14, 1024)  0           bn4c_branch2c[0][0]              \n",
      "                                                                   activation_77[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_80 (Activation)       (None, 14, 14, 1024)  0           add_26[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "res4d_branch2a (Conv2D)          (None, 14, 14, 256)   262400      activation_80[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn4d_branch2a (BatchNormalizatio (None, 14, 14, 256)   1024        res4d_branch2a[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_81 (Activation)       (None, 14, 14, 256)   0           bn4d_branch2a[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res4d_branch2b (Conv2D)          (None, 14, 14, 256)   590080      activation_81[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn4d_branch2b (BatchNormalizatio (None, 14, 14, 256)   1024        res4d_branch2b[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_82 (Activation)       (None, 14, 14, 256)   0           bn4d_branch2b[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res4d_branch2c (Conv2D)          (None, 14, 14, 1024)  263168      activation_82[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn4d_branch2c (BatchNormalizatio (None, 14, 14, 1024)  4096        res4d_branch2c[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "add_27 (Add)                     (None, 14, 14, 1024)  0           bn4d_branch2c[0][0]              \n",
      "                                                                   activation_80[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_83 (Activation)       (None, 14, 14, 1024)  0           add_27[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "res4e_branch2a (Conv2D)          (None, 14, 14, 256)   262400      activation_83[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn4e_branch2a (BatchNormalizatio (None, 14, 14, 256)   1024        res4e_branch2a[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_84 (Activation)       (None, 14, 14, 256)   0           bn4e_branch2a[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res4e_branch2b (Conv2D)          (None, 14, 14, 256)   590080      activation_84[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn4e_branch2b (BatchNormalizatio (None, 14, 14, 256)   1024        res4e_branch2b[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_85 (Activation)       (None, 14, 14, 256)   0           bn4e_branch2b[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res4e_branch2c (Conv2D)          (None, 14, 14, 1024)  263168      activation_85[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn4e_branch2c (BatchNormalizatio (None, 14, 14, 1024)  4096        res4e_branch2c[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "add_28 (Add)                     (None, 14, 14, 1024)  0           bn4e_branch2c[0][0]              \n",
      "                                                                   activation_83[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_86 (Activation)       (None, 14, 14, 1024)  0           add_28[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "res4f_branch2a (Conv2D)          (None, 14, 14, 256)   262400      activation_86[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn4f_branch2a (BatchNormalizatio (None, 14, 14, 256)   1024        res4f_branch2a[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_87 (Activation)       (None, 14, 14, 256)   0           bn4f_branch2a[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res4f_branch2b (Conv2D)          (None, 14, 14, 256)   590080      activation_87[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn4f_branch2b (BatchNormalizatio (None, 14, 14, 256)   1024        res4f_branch2b[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_88 (Activation)       (None, 14, 14, 256)   0           bn4f_branch2b[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res4f_branch2c (Conv2D)          (None, 14, 14, 1024)  263168      activation_88[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn4f_branch2c (BatchNormalizatio (None, 14, 14, 1024)  4096        res4f_branch2c[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "add_29 (Add)                     (None, 14, 14, 1024)  0           bn4f_branch2c[0][0]              \n",
      "                                                                   activation_86[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_89 (Activation)       (None, 14, 14, 1024)  0           add_29[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "res5a_branch2a (Conv2D)          (None, 7, 7, 512)     524800      activation_89[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn5a_branch2a (BatchNormalizatio (None, 7, 7, 512)     2048        res5a_branch2a[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_90 (Activation)       (None, 7, 7, 512)     0           bn5a_branch2a[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res5a_branch2b (Conv2D)          (None, 7, 7, 512)     2359808     activation_90[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn5a_branch2b (BatchNormalizatio (None, 7, 7, 512)     2048        res5a_branch2b[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_91 (Activation)       (None, 7, 7, 512)     0           bn5a_branch2b[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res5a_branch2c (Conv2D)          (None, 7, 7, 2048)    1050624     activation_91[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res5a_branch1 (Conv2D)           (None, 7, 7, 2048)    2099200     activation_89[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn5a_branch2c (BatchNormalizatio (None, 7, 7, 2048)    8192        res5a_branch2c[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "bn5a_branch1 (BatchNormalization (None, 7, 7, 2048)    8192        res5a_branch1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "add_30 (Add)                     (None, 7, 7, 2048)    0           bn5a_branch2c[0][0]              \n",
      "                                                                   bn5a_branch1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "activation_92 (Activation)       (None, 7, 7, 2048)    0           add_30[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "res5b_branch2a (Conv2D)          (None, 7, 7, 512)     1049088     activation_92[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn5b_branch2a (BatchNormalizatio (None, 7, 7, 512)     2048        res5b_branch2a[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_93 (Activation)       (None, 7, 7, 512)     0           bn5b_branch2a[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res5b_branch2b (Conv2D)          (None, 7, 7, 512)     2359808     activation_93[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn5b_branch2b (BatchNormalizatio (None, 7, 7, 512)     2048        res5b_branch2b[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_94 (Activation)       (None, 7, 7, 512)     0           bn5b_branch2b[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res5b_branch2c (Conv2D)          (None, 7, 7, 2048)    1050624     activation_94[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn5b_branch2c (BatchNormalizatio (None, 7, 7, 2048)    8192        res5b_branch2c[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "add_31 (Add)                     (None, 7, 7, 2048)    0           bn5b_branch2c[0][0]              \n",
      "                                                                   activation_92[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_95 (Activation)       (None, 7, 7, 2048)    0           add_31[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "res5c_branch2a (Conv2D)          (None, 7, 7, 512)     1049088     activation_95[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn5c_branch2a (BatchNormalizatio (None, 7, 7, 512)     2048        res5c_branch2a[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_96 (Activation)       (None, 7, 7, 512)     0           bn5c_branch2a[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res5c_branch2b (Conv2D)          (None, 7, 7, 512)     2359808     activation_96[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn5c_branch2b (BatchNormalizatio (None, 7, 7, 512)     2048        res5c_branch2b[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_97 (Activation)       (None, 7, 7, 512)     0           bn5c_branch2b[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "res5c_branch2c (Conv2D)          (None, 7, 7, 2048)    1050624     activation_97[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "bn5c_branch2c (BatchNormalizatio (None, 7, 7, 2048)    8192        res5c_branch2c[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "add_32 (Add)                     (None, 7, 7, 2048)    0           bn5c_branch2c[0][0]              \n",
      "                                                                   activation_95[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_98 (Activation)       (None, 7, 7, 2048)    0           add_32[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "avg_pool (AveragePooling2D)      (None, 1, 1, 2048)    0           activation_98[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "flatten_2 (Flatten)              (None, 2048)          0           avg_pool[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "fc1000 (Dense)                   (None, 1000)          2049000     flatten_2[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 25,636,712\n",
      "Trainable params: 25,583,592\n",
      "Non-trainable params: 53,120\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.applications.resnet50 import ResNet50, preprocess_input\n",
    "resnet_model = ResNet50(weights='imagenet', include_top=True)\n",
    "\n",
    "resnet_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "resnet_model_extractfeatures = Model(inputs=resnet_model.input, outputs=resnet_model.get_layer('flatten_1').output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "img_path = 'COCO/images/COCO_train2014_000000357413.jpg'\n",
    "feats = extract_features(img_path, resnet_model_extractfeatures)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 2048)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feats.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Extract features of all images of the training and validation sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import scipy.io\n",
    "import os\n",
    "\n",
    "def img_features_keras(path, output_name, model, dim):\n",
    "    img_paths = np.asarray(glob(path))\n",
    "    \n",
    "    features = np.zeros((len(img_paths), dim), dtype=np.float32)\n",
    "\n",
    "    for i, img_path in enumerate(img_paths):\n",
    "        features[i] = extract_features(img_path, model)\n",
    "        \n",
    "        if i % 1000 == 0:\n",
    "            print('.', end=' ', flush=True)\n",
    "    \n",
    "    scipy.io.savemat(os.path.join('preprocess_datas', output_name + '.mat'), mdict =  {'features': np.transpose(features)})\n",
    " \n",
    "    return None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### VGG16 extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . "
     ]
    }
   ],
   "source": [
    "img_features_keras('COCO/images/*.jpg', 'VGG16_feats', vgg16_model_extractfeatures, 4096)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Extract features of all images of the testing set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . "
     ]
    }
   ],
   "source": [
    "img_features_keras('COCO/images_test/*.jpg', 'VGG16_feats_test', vgg16_model_extractfeatures, 4096)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resnet extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . "
     ]
    }
   ],
   "source": [
    "img_features_keras('COCO/images/*.jpg', 'resnet_feats', resnet_model_extractfeatures, 2048)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . "
     ]
    }
   ],
   "source": [
    "img_features_keras('COCO/images_test/*.jpg', 'resnet_feats_test', resnet_model_extractfeatures, 2048)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}