7. Inception Feature Extraction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import sys, os, time, gc, random, pickle\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "import requests, shutil\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "from skimage import io\n",
    "from skimage.transform import resize\n",
    "from PIL import Image\n",
    "\n",
    "from keras import backend as K\n",
    "from keras.preprocessing.image import load_img, img_to_array\n",
    "from keras.models import load_model\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load the data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t\t (1225029, 3)\n",
      "Test:\t\t (117703, 2)\n",
      "Landmarks:\t 14951\n"
     ]
    }
   ],
   "source": [
    "# Load the data\n",
    "train = pd.read_csv('./data/all/train.csv')\n",
    "test = pd.read_csv('./data/all/test.csv')\n",
    "\n",
    "print('Train:\\t\\t', train.shape)\n",
    "print('Test:\\t\\t', test.shape)\n",
    "print('Landmarks:\\t', len(train['landmark_id'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>url</th>\n",
       "      <th>landmark_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cacf8152e2d2ae60</td>\n",
       "      <td>http://static.panoramio.com/photos/original/70...</td>\n",
       "      <td>4676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0a58358a2afd3e4e</td>\n",
       "      <td>http://lh6.ggpht.com/-igpT6wu0mIA/ROV8HnUuABI/...</td>\n",
       "      <td>6651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6b2bb500b6a38aa0</td>\n",
       "      <td>http://lh6.ggpht.com/-vKr5G5MEusk/SR6r6SJi6mI/...</td>\n",
       "      <td>11284</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b399f09dee9c3c67</td>\n",
       "      <td>https://lh3.googleusercontent.com/-LOW2cjAqubA...</td>\n",
       "      <td>8429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>19ace29d77a5be66</td>\n",
       "      <td>https://lh5.googleusercontent.com/-tnmSXwQcWL8...</td>\n",
       "      <td>6231</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id                                                url  \\\n",
       "0  cacf8152e2d2ae60  http://static.panoramio.com/photos/original/70...   \n",
       "1  0a58358a2afd3e4e  http://lh6.ggpht.com/-igpT6wu0mIA/ROV8HnUuABI/...   \n",
       "2  6b2bb500b6a38aa0  http://lh6.ggpht.com/-vKr5G5MEusk/SR6r6SJi6mI/...   \n",
       "3  b399f09dee9c3c67  https://lh3.googleusercontent.com/-LOW2cjAqubA...   \n",
       "4  19ace29d77a5be66  https://lh5.googleusercontent.com/-tnmSXwQcWL8...   \n",
       "\n",
       "   landmark_id  \n",
       "0         4676  \n",
       "1         6651  \n",
       "2        11284  \n",
       "3         8429  \n",
       "4         6231  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract Valid Image IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def valid(path):\n",
    "    \"\"\" function to determine whether or not the given image is valid \"\"\"\n",
    "    try:\n",
    "        img = Image.open(path)\n",
    "        if img.width < 256 or img.height < 256 or img.format != 'JPEG':\n",
    "            return False\n",
    "        _ = img.resize((256, 256))\n",
    "    except:\n",
    "        return False\n",
    "    \n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists('./result/train_imgs_id.npy'):\n",
    "    # Process the training set\n",
    "    train_imgs_id = []\n",
    "    train_landmark = []\n",
    "    t0 = time.time()\n",
    "\n",
    "    for i in range(len(train)):\n",
    "        # Helpful information\n",
    "        if i % 100000 == 0:\n",
    "            t = time.time() - t0\n",
    "            print('\\nProcess: {:9d}'.format(i), '   Used time: {} s'.format(np.round(t, 0)))\n",
    "            t0 = time.time()\n",
    "        if i % 1250 == 0:\n",
    "            print('=', end='')\n",
    "\n",
    "        path = './data/all/train_images/' + str(i) + '.jpg'\n",
    "        if valid(path):\n",
    "            train_imgs_id.append(i)\n",
    "            train_landmark.append(train.loc[i, 'landmark_id'])\n",
    "\n",
    "    np.save('./result/train_imgs_id.npy', train_imgs_id)\n",
    "    np.save('./result/train_landmark.npy', train_landmark)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists('./result/test_imgs_id.npy'):\n",
    "    # Process the test set\n",
    "    test_imgs_id = []\n",
    "    t0 = time.time()\n",
    "\n",
    "    for i in range(len(test)):\n",
    "        # Helpful information\n",
    "        if i % 50000 == 0:\n",
    "            t = time.time() - t0\n",
    "            print('\\nProcess: {:9d}'.format(i), '   Used time: {} s'.format(np.round(t, 0)))\n",
    "            t0 = time.time()\n",
    "        if i % 625 == 0:\n",
    "            print('=', end='')\n",
    "\n",
    "        path = './data/all/test_images/' + str(i) + '.jpg'\n",
    "        if valid(path):\n",
    "            test_imgs_id.append(i)\n",
    "\n",
    "    np.save('./result/test_imgs_id.npy', test_imgs_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load fine-tuned InceptionV3 Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3148: calling l2_normalize (from tensorflow.python.ops.nn_impl) with dim is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "dim is deprecated, use axis instead\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "inception_v3 (Model)         (None, 2048)              21802784  \n",
      "_________________________________________________________________\n",
      "l2_norm (Lambda)             (None, 2048)              0         \n",
      "=================================================================\n",
      "Total params: 21,802,784\n",
      "Trainable params: 6,073,536\n",
      "Non-trainable params: 15,729,248\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Load trained model\n",
    "model = load_model('./models/inception-base-0.1-model.h5')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract training and test image features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t 1192931 1192931 14943 (1192931, 2048)\n",
      "Test:\t 108383\n"
     ]
    }
   ],
   "source": [
    "# Process all the training images\n",
    "train_imgs_id = np.load('./result/train_imgs_id.npy')\n",
    "train_landmark = np.load('./result/train_landmark.npy')\n",
    "\n",
    "test_imgs_id = np.load('./result/test_imgs_id.npy')\n",
    "train_features = np.zeros(shape=(len(train_imgs_id), 2048), dtype=np.float32)\n",
    "\n",
    "print('Train:\\t', len(train_imgs_id), len(train_landmark), len(np.unique(train_landmark)), train_features.shape)\n",
    "print('Test:\\t', len(test_imgs_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Process:         0    Used time: 0.0 s\n",
      "================================================================================\n",
      "Process:     20000    Used time: 545.0 s\n",
      "================================================================================\n",
      "Process:     40000    Used time: 544.0 s\n",
      "================================================================================\n",
      "Process:     60000    Used time: 546.0 s\n",
      "================================================================================\n",
      "Process:     80000    Used time: 579.0 s\n",
      "================================================================================\n",
      "Process:    100000    Used time: 573.0 s\n",
      "================================================================================\n",
      "Process:    120000    Used time: 572.0 s\n",
      "================================================================================\n",
      "Process:    140000    Used time: 596.0 s\n",
      "================================================================================\n",
      "Process:    160000    Used time: 599.0 s\n",
      "================================================================================\n",
      "Process:    180000    Used time: 588.0 s\n",
      "================================================================================\n",
      "Process:    200000    Used time: 594.0 s\n",
      "================================================================================\n",
      "Process:    220000    Used time: 607.0 s\n",
      "================================================================================\n",
      "Process:    240000    Used time: 602.0 s\n",
      "================================================================================\n",
      "Process:    260000    Used time: 599.0 s\n",
      "================================================================================\n",
      "Process:    280000    Used time: 605.0 s\n",
      "================================================================================\n",
      "Process:    300000    Used time: 601.0 s\n",
      "================================================================================\n",
      "Process:    320000    Used time: 601.0 s\n",
      "================================================================================\n",
      "Process:    340000    Used time: 605.0 s\n",
      "================================================================================\n",
      "Process:    360000    Used time: 600.0 s\n",
      "================================================================================\n",
      "Process:    380000    Used time: 606.0 s\n",
      "================================================================================\n",
      "Process:    400000    Used time: 615.0 s\n",
      "================================================================================\n",
      "Process:    420000    Used time: 608.0 s\n",
      "================================================================================\n",
      "Process:    440000    Used time: 613.0 s\n",
      "================================================================================\n",
      "Process:    460000    Used time: 614.0 s\n",
      "================================================================================\n",
      "Process:    480000    Used time: 610.0 s\n",
      "================================================================================\n",
      "Process:    500000    Used time: 610.0 s\n",
      "================================================================================\n",
      "Process:    520000    Used time: 612.0 s\n",
      "================================================================================\n",
      "Process:    540000    Used time: 613.0 s\n",
      "================================================================================\n",
      "Process:    560000    Used time: 612.0 s\n",
      "================================================================================\n",
      "Process:    580000    Used time: 618.0 s\n",
      "================================================================================\n",
      "Process:    600000    Used time: 612.0 s\n",
      "================================================================================\n",
      "Process:    620000    Used time: 614.0 s\n",
      "================================================================================\n",
      "Process:    640000    Used time: 617.0 s\n",
      "================================================================================\n",
      "Process:    660000    Used time: 616.0 s\n",
      "================================================================================\n",
      "Process:    680000    Used time: 620.0 s\n",
      "================================================================================\n",
      "Process:    700000    Used time: 626.0 s\n",
      "================================================================================\n",
      "Process:    720000    Used time: 618.0 s\n",
      "================================================================================\n",
      "Process:    740000    Used time: 625.0 s\n",
      "================================================================================\n",
      "Process:    760000    Used time: 624.0 s\n",
      "================================================================================\n",
      "Process:    780000    Used time: 630.0 s\n",
      "================================================================================\n",
      "Process:    800000    Used time: 626.0 s\n",
      "================================================================================\n",
      "Process:    820000    Used time: 627.0 s\n",
      "================================================================================\n",
      "Process:    840000    Used time: 618.0 s\n",
      "================================================================================\n",
      "Process:    860000    Used time: 617.0 s\n",
      "================================================================================\n",
      "Process:    880000    Used time: 622.0 s\n",
      "================================================================================\n",
      "Process:    900000    Used time: 627.0 s\n",
      "================================================================================\n",
      "Process:    920000    Used time: 615.0 s\n",
      "================================================================================\n",
      "Process:    940000    Used time: 621.0 s\n",
      "================================================================================\n",
      "Process:    960000    Used time: 621.0 s\n",
      "================================================================================\n",
      "Process:    980000    Used time: 626.0 s\n",
      "================================================================================\n",
      "Process:   1000000    Used time: 618.0 s\n",
      "================================================================================\n",
      "Process:   1020000    Used time: 624.0 s\n",
      "================================================================================\n",
      "Process:   1040000    Used time: 630.0 s\n",
      "================================================================================\n",
      "Process:   1060000    Used time: 628.0 s\n",
      "================================================================================\n",
      "Process:   1080000    Used time: 621.0 s\n",
      "================================================================================\n",
      "Process:   1100000    Used time: 625.0 s\n",
      "================================================================================\n",
      "Process:   1120000    Used time: 618.0 s\n",
      "================================================================================\n",
      "Process:   1140000    Used time: 619.0 s\n",
      "================================================================================\n",
      "Process:   1160000    Used time: 625.0 s\n",
      "================================================================================\n",
      "Process:   1180000    Used time: 620.0 s\n",
      "===================================================="
     ]
    }
   ],
   "source": [
    "steps = 20000\n",
    "t0 = time.time()\n",
    "\n",
    "for i in range(0, len(train_imgs_id), steps):\n",
    "    # Helpful information\n",
    "    if i % steps == 0:\n",
    "        t = time.time() - t0\n",
    "        print('\\nProcess: {:9d}'.format(i), '   Used time: {} s'.format(np.round(t, 0)))\n",
    "        t0 = time.time()\n",
    "        \n",
    "    tmp_imgs = []\n",
    "    start = i\n",
    "    end = min(len(train_imgs_id), i + steps)\n",
    "    for idx in range(start, end):\n",
    "        if idx % 250 == 0:\n",
    "            print('=', end='')\n",
    "            \n",
    "        img_id = train_imgs_id[idx]\n",
    "        path = './data/all/train_images/' + str(img_id) + '.jpg'\n",
    "        img = load_img(path, target_size=(224, 224))\n",
    "        img = img_to_array(img)\n",
    "        tmp_imgs.append(img)\n",
    "        \n",
    "    tmp_imgs = np.array(tmp_imgs, dtype=np.float32) / 255.0\n",
    "    tmp_feature = model.predict(tmp_imgs)\n",
    "    train_features[start:end, ] = tmp_feature\n",
    "    _ = gc.collect()\n",
    "\n",
    "# Save to disk\n",
    "np.save('./data/all/train_features.npy', train_features)\n",
    "np.save('./data/all/train_id.npy', train_landmark)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Process:         0    Used time: 0.0 s\n",
      "================================================================================\n",
      "Process:     20000    Used time: 658.0 s\n",
      "================================================================================\n",
      "Process:     40000    Used time: 654.0 s\n",
      "================================================================================\n",
      "Process:     60000    Used time: 646.0 s\n",
      "================================================================================\n",
      "Process:     80000    Used time: 650.0 s\n",
      "================================================================================\n",
      "Process:    100000    Used time: 648.0 s\n",
      "=================================="
     ]
    }
   ],
   "source": [
    "# Process all the test images\n",
    "test_imgs_id = np.load('./result/test_imgs_id.npy')\n",
    "test_features = np.zeros(shape=(len(test_imgs_id), 2048), dtype=np.float32)\n",
    "\n",
    "steps = 20000\n",
    "t0 = time.time()\n",
    "\n",
    "for i in range(0, len(test_imgs_id), steps):\n",
    "    # Helpful information\n",
    "    if i % steps == 0:\n",
    "        t = time.time() - t0\n",
    "        print('\\nProcess: {:9d}'.format(i), '   Used time: {} s'.format(np.round(t, 0)))\n",
    "        t0 = time.time()\n",
    "        \n",
    "    tmp_imgs = []\n",
    "    start = i\n",
    "    end = min(len(test_imgs_id), i + steps)\n",
    "    for idx in range(start, end):\n",
    "        if idx % 250 == 0:\n",
    "            print('=', end='')\n",
    "            \n",
    "        img_id = test_imgs_id[idx]\n",
    "        path = './data/all/test_images/' + str(img_id) + '.jpg'\n",
    "        img = load_img(path, target_size=(224, 224))\n",
    "        img = img_to_array(img)\n",
    "        tmp_imgs.append(img)\n",
    "        \n",
    "    tmp_imgs = np.array(tmp_imgs, dtype=np.float32) / 255.0\n",
    "    tmp_feature = model.predict(tmp_imgs)\n",
    "    test_features[start:end, ] = tmp_feature\n",
    "    _ = gc.collect()\n",
    "\n",
    "# Save to disk\n",
    "np.save('./data/all/test_features.npy', test_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Environment (conda_tensorflow_p36)",
   "language": "python",
   "name": "conda_tensorflow_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}