5-1. ResNet50 Naive KNN Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import sys, os, time, gc\n",
    "import requests, shutil\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "import keras\n",
    "from keras.preprocessing.image import load_img, img_to_array\n",
    "from keras.applications.resnet50 import ResNet50\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Data Information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t\t (113783, 4)\n",
      "Validation:\t (22255, 4)\n",
      "Test:\t\t (22391, 4)\n",
      "\n",
      "Train Landmarks:\t 14943\n",
      "Validation Landmarks:\t 7674\n",
      "Test Landmarks:\t\t 14436\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv('./data/triplet/train.csv')\n",
    "val_df = pd.read_csv('./data/triplet/validation.csv')\n",
    "test_df = pd.read_csv('./data/triplet/test.csv')\n",
    "\n",
    "print('Train:\\t\\t', train_df.shape)\n",
    "print('Validation:\\t', val_df.shape)\n",
    "print('Test:\\t\\t', test_df.shape)\n",
    "\n",
    "print('\\nTrain Landmarks:\\t', len(train_df['landmark_id'].unique()))\n",
    "print('Validation Landmarks:\\t', len(val_df['landmark_id'].unique()))\n",
    "print('Test Landmarks:\\t\\t', len(test_df['landmark_id'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>id</th>\n",
       "      <th>url</th>\n",
       "      <th>landmark_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>465272</td>\n",
       "      <td>a2ccf8ed2e969f6a</td>\n",
       "      <td>https://lh4.googleusercontent.com/-TPHkS5gzvm4...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>64516</td>\n",
       "      <td>e205ca7c8dd7c027</td>\n",
       "      <td>https://lh3.googleusercontent.com/-V3RjsZtGpxE...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>928409</td>\n",
       "      <td>4e8ab93c1620e8a3</td>\n",
       "      <td>http://mw2.google.com/mw-panoramio/photos/medi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>88809</td>\n",
       "      <td>896bf928214d1ca4</td>\n",
       "      <td>http://lh5.ggpht.com/-Cy0l41uUaGA/R--yB8vy41I/...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1001133</td>\n",
       "      <td>375d2a153bdca926</td>\n",
       "      <td>http://lh6.ggpht.com/-UqzFpnqE9bU/S_0u1RovfdI/...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   image_id                id  \\\n",
       "0    465272  a2ccf8ed2e969f6a   \n",
       "1     64516  e205ca7c8dd7c027   \n",
       "2    928409  4e8ab93c1620e8a3   \n",
       "3     88809  896bf928214d1ca4   \n",
       "4   1001133  375d2a153bdca926   \n",
       "\n",
       "                                                 url  landmark_id  \n",
       "0  https://lh4.googleusercontent.com/-TPHkS5gzvm4...            0  \n",
       "1  https://lh3.googleusercontent.com/-V3RjsZtGpxE...            0  \n",
       "2  http://mw2.google.com/mw-panoramio/photos/medi...            0  \n",
       "3  http://lh5.ggpht.com/-Cy0l41uUaGA/R--yB8vy41I/...            0  \n",
       "4  http://lh6.ggpht.com/-UqzFpnqE9bU/S_0u1RovfdI/...            0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Extract Features using ResNet50 (No need to run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv1 (Conv2D)                  (None, 112, 112, 64) 9472        input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "bn_conv1 (BatchNormalization)   (None, 112, 112, 64) 256         conv1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 112, 112, 64) 0           bn_conv1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)  (None, 55, 55, 64)   0           activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "res2a_branch2a (Conv2D)         (None, 55, 55, 64)   4160        max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "bn2a_branch2a (BatchNormalizati (None, 55, 55, 64)   256         res2a_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 55, 55, 64)   0           bn2a_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res2a_branch2b (Conv2D)         (None, 55, 55, 64)   36928       activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2a_branch2b (BatchNormalizati (None, 55, 55, 64)   256         res2a_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 55, 55, 64)   0           bn2a_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res2a_branch2c (Conv2D)         (None, 55, 55, 256)  16640       activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "res2a_branch1 (Conv2D)          (None, 55, 55, 256)  16640       max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "bn2a_branch2c (BatchNormalizati (None, 55, 55, 256)  1024        res2a_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "bn2a_branch1 (BatchNormalizatio (None, 55, 55, 256)  1024        res2a_branch1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 55, 55, 256)  0           bn2a_branch2c[0][0]              \n",
      "                                                                 bn2a_branch1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 55, 55, 256)  0           add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res2b_branch2a (Conv2D)         (None, 55, 55, 64)   16448       activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2b_branch2a (BatchNormalizati (None, 55, 55, 64)   256         res2b_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 55, 55, 64)   0           bn2b_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res2b_branch2b (Conv2D)         (None, 55, 55, 64)   36928       activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2b_branch2b (BatchNormalizati (None, 55, 55, 64)   256         res2b_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, 55, 55, 64)   0           bn2b_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res2b_branch2c (Conv2D)         (None, 55, 55, 256)  16640       activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2b_branch2c (BatchNormalizati (None, 55, 55, 256)  1024        res2b_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 55, 55, 256)  0           bn2b_branch2c[0][0]              \n",
      "                                                                 activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, 55, 55, 256)  0           add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res2c_branch2a (Conv2D)         (None, 55, 55, 64)   16448       activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2c_branch2a (BatchNormalizati (None, 55, 55, 64)   256         res2c_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, 55, 55, 64)   0           bn2c_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res2c_branch2b (Conv2D)         (None, 55, 55, 64)   36928       activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2c_branch2b (BatchNormalizati (None, 55, 55, 64)   256         res2c_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_9 (Activation)       (None, 55, 55, 64)   0           bn2c_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res2c_branch2c (Conv2D)         (None, 55, 55, 256)  16640       activation_9[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2c_branch2c (BatchNormalizati (None, 55, 55, 256)  1024        res2c_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, 55, 55, 256)  0           bn2c_branch2c[0][0]              \n",
      "                                                                 activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_10 (Activation)      (None, 55, 55, 256)  0           add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res3a_branch2a (Conv2D)         (None, 28, 28, 128)  32896       activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3a_branch2a (BatchNormalizati (None, 28, 28, 128)  512         res3a_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_11 (Activation)      (None, 28, 28, 128)  0           bn3a_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3a_branch2b (Conv2D)         (None, 28, 28, 128)  147584      activation_11[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3a_branch2b (BatchNormalizati (None, 28, 28, 128)  512         res3a_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_12 (Activation)      (None, 28, 28, 128)  0           bn3a_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3a_branch2c (Conv2D)         (None, 28, 28, 512)  66048       activation_12[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3a_branch1 (Conv2D)          (None, 28, 28, 512)  131584      activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3a_branch2c (BatchNormalizati (None, 28, 28, 512)  2048        res3a_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "bn3a_branch1 (BatchNormalizatio (None, 28, 28, 512)  2048        res3a_branch1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 28, 28, 512)  0           bn3a_branch2c[0][0]              \n",
      "                                                                 bn3a_branch1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_13 (Activation)      (None, 28, 28, 512)  0           add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res3b_branch2a (Conv2D)         (None, 28, 28, 128)  65664       activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3b_branch2a (BatchNormalizati (None, 28, 28, 128)  512         res3b_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_14 (Activation)      (None, 28, 28, 128)  0           bn3b_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3b_branch2b (Conv2D)         (None, 28, 28, 128)  147584      activation_14[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3b_branch2b (BatchNormalizati (None, 28, 28, 128)  512         res3b_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_15 (Activation)      (None, 28, 28, 128)  0           bn3b_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3b_branch2c (Conv2D)         (None, 28, 28, 512)  66048       activation_15[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3b_branch2c (BatchNormalizati (None, 28, 28, 512)  2048        res3b_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 28, 28, 512)  0           bn3b_branch2c[0][0]              \n",
      "                                                                 activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_16 (Activation)      (None, 28, 28, 512)  0           add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res3c_branch2a (Conv2D)         (None, 28, 28, 128)  65664       activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3c_branch2a (BatchNormalizati (None, 28, 28, 128)  512         res3c_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_17 (Activation)      (None, 28, 28, 128)  0           bn3c_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3c_branch2b (Conv2D)         (None, 28, 28, 128)  147584      activation_17[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3c_branch2b (BatchNormalizati (None, 28, 28, 128)  512         res3c_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_18 (Activation)      (None, 28, 28, 128)  0           bn3c_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3c_branch2c (Conv2D)         (None, 28, 28, 512)  66048       activation_18[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3c_branch2c (BatchNormalizati (None, 28, 28, 512)  2048        res3c_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_6 (Add)                     (None, 28, 28, 512)  0           bn3c_branch2c[0][0]              \n",
      "                                                                 activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_19 (Activation)      (None, 28, 28, 512)  0           add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res3d_branch2a (Conv2D)         (None, 28, 28, 128)  65664       activation_19[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3d_branch2a (BatchNormalizati (None, 28, 28, 128)  512         res3d_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_20 (Activation)      (None, 28, 28, 128)  0           bn3d_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3d_branch2b (Conv2D)         (None, 28, 28, 128)  147584      activation_20[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3d_branch2b (BatchNormalizati (None, 28, 28, 128)  512         res3d_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_21 (Activation)      (None, 28, 28, 128)  0           bn3d_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3d_branch2c (Conv2D)         (None, 28, 28, 512)  66048       activation_21[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3d_branch2c (BatchNormalizati (None, 28, 28, 512)  2048        res3d_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_7 (Add)                     (None, 28, 28, 512)  0           bn3d_branch2c[0][0]              \n",
      "                                                                 activation_19[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_22 (Activation)      (None, 28, 28, 512)  0           add_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res4a_branch2a (Conv2D)         (None, 14, 14, 256)  131328      activation_22[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4a_branch2a (BatchNormalizati (None, 14, 14, 256)  1024        res4a_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_23 (Activation)      (None, 14, 14, 256)  0           bn4a_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4a_branch2b (Conv2D)         (None, 14, 14, 256)  590080      activation_23[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4a_branch2b (BatchNormalizati (None, 14, 14, 256)  1024        res4a_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_24 (Activation)      (None, 14, 14, 256)  0           bn4a_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4a_branch2c (Conv2D)         (None, 14, 14, 1024) 263168      activation_24[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4a_branch1 (Conv2D)          (None, 14, 14, 1024) 525312      activation_22[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4a_branch2c (BatchNormalizati (None, 14, 14, 1024) 4096        res4a_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "bn4a_branch1 (BatchNormalizatio (None, 14, 14, 1024) 4096        res4a_branch1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_8 (Add)                     (None, 14, 14, 1024) 0           bn4a_branch2c[0][0]              \n",
      "                                                                 bn4a_branch1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_25 (Activation)      (None, 14, 14, 1024) 0           add_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res4b_branch2a (Conv2D)         (None, 14, 14, 256)  262400      activation_25[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4b_branch2a (BatchNormalizati (None, 14, 14, 256)  1024        res4b_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_26 (Activation)      (None, 14, 14, 256)  0           bn4b_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4b_branch2b (Conv2D)         (None, 14, 14, 256)  590080      activation_26[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4b_branch2b (BatchNormalizati (None, 14, 14, 256)  1024        res4b_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_27 (Activation)      (None, 14, 14, 256)  0           bn4b_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4b_branch2c (Conv2D)         (None, 14, 14, 1024) 263168      activation_27[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4b_branch2c (BatchNormalizati (None, 14, 14, 1024) 4096        res4b_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_9 (Add)                     (None, 14, 14, 1024) 0           bn4b_branch2c[0][0]              \n",
      "                                                                 activation_25[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_28 (Activation)      (None, 14, 14, 1024) 0           add_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res4c_branch2a (Conv2D)         (None, 14, 14, 256)  262400      activation_28[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4c_branch2a (BatchNormalizati (None, 14, 14, 256)  1024        res4c_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_29 (Activation)      (None, 14, 14, 256)  0           bn4c_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4c_branch2b (Conv2D)         (None, 14, 14, 256)  590080      activation_29[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4c_branch2b (BatchNormalizati (None, 14, 14, 256)  1024        res4c_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_30 (Activation)      (None, 14, 14, 256)  0           bn4c_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4c_branch2c (Conv2D)         (None, 14, 14, 1024) 263168      activation_30[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4c_branch2c (BatchNormalizati (None, 14, 14, 1024) 4096        res4c_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_10 (Add)                    (None, 14, 14, 1024) 0           bn4c_branch2c[0][0]              \n",
      "                                                                 activation_28[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_31 (Activation)      (None, 14, 14, 1024) 0           add_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "res4d_branch2a (Conv2D)         (None, 14, 14, 256)  262400      activation_31[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4d_branch2a (BatchNormalizati (None, 14, 14, 256)  1024        res4d_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_32 (Activation)      (None, 14, 14, 256)  0           bn4d_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4d_branch2b (Conv2D)         (None, 14, 14, 256)  590080      activation_32[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4d_branch2b (BatchNormalizati (None, 14, 14, 256)  1024        res4d_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_33 (Activation)      (None, 14, 14, 256)  0           bn4d_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4d_branch2c (Conv2D)         (None, 14, 14, 1024) 263168      activation_33[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4d_branch2c (BatchNormalizati (None, 14, 14, 1024) 4096        res4d_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_11 (Add)                    (None, 14, 14, 1024) 0           bn4d_branch2c[0][0]              \n",
      "                                                                 activation_31[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_34 (Activation)      (None, 14, 14, 1024) 0           add_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "res4e_branch2a (Conv2D)         (None, 14, 14, 256)  262400      activation_34[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4e_branch2a (BatchNormalizati (None, 14, 14, 256)  1024        res4e_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_35 (Activation)      (None, 14, 14, 256)  0           bn4e_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4e_branch2b (Conv2D)         (None, 14, 14, 256)  590080      activation_35[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4e_branch2b (BatchNormalizati (None, 14, 14, 256)  1024        res4e_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_36 (Activation)      (None, 14, 14, 256)  0           bn4e_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4e_branch2c (Conv2D)         (None, 14, 14, 1024) 263168      activation_36[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4e_branch2c (BatchNormalizati (None, 14, 14, 1024) 4096        res4e_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_12 (Add)                    (None, 14, 14, 1024) 0           bn4e_branch2c[0][0]              \n",
      "                                                                 activation_34[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_37 (Activation)      (None, 14, 14, 1024) 0           add_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "res4f_branch2a (Conv2D)         (None, 14, 14, 256)  262400      activation_37[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4f_branch2a (BatchNormalizati (None, 14, 14, 256)  1024        res4f_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_38 (Activation)      (None, 14, 14, 256)  0           bn4f_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4f_branch2b (Conv2D)         (None, 14, 14, 256)  590080      activation_38[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4f_branch2b (BatchNormalizati (None, 14, 14, 256)  1024        res4f_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_39 (Activation)      (None, 14, 14, 256)  0           bn4f_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4f_branch2c (Conv2D)         (None, 14, 14, 1024) 263168      activation_39[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4f_branch2c (BatchNormalizati (None, 14, 14, 1024) 4096        res4f_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_13 (Add)                    (None, 14, 14, 1024) 0           bn4f_branch2c[0][0]              \n",
      "                                                                 activation_37[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_40 (Activation)      (None, 14, 14, 1024) 0           add_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "res5a_branch2a (Conv2D)         (None, 7, 7, 512)    524800      activation_40[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5a_branch2a (BatchNormalizati (None, 7, 7, 512)    2048        res5a_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_41 (Activation)      (None, 7, 7, 512)    0           bn5a_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5a_branch2b (Conv2D)         (None, 7, 7, 512)    2359808     activation_41[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5a_branch2b (BatchNormalizati (None, 7, 7, 512)    2048        res5a_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_42 (Activation)      (None, 7, 7, 512)    0           bn5a_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5a_branch2c (Conv2D)         (None, 7, 7, 2048)   1050624     activation_42[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5a_branch1 (Conv2D)          (None, 7, 7, 2048)   2099200     activation_40[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5a_branch2c (BatchNormalizati (None, 7, 7, 2048)   8192        res5a_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "bn5a_branch1 (BatchNormalizatio (None, 7, 7, 2048)   8192        res5a_branch1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_14 (Add)                    (None, 7, 7, 2048)   0           bn5a_branch2c[0][0]              \n",
      "                                                                 bn5a_branch1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_43 (Activation)      (None, 7, 7, 2048)   0           add_14[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "res5b_branch2a (Conv2D)         (None, 7, 7, 512)    1049088     activation_43[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5b_branch2a (BatchNormalizati (None, 7, 7, 512)    2048        res5b_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_44 (Activation)      (None, 7, 7, 512)    0           bn5b_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5b_branch2b (Conv2D)         (None, 7, 7, 512)    2359808     activation_44[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5b_branch2b (BatchNormalizati (None, 7, 7, 512)    2048        res5b_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_45 (Activation)      (None, 7, 7, 512)    0           bn5b_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5b_branch2c (Conv2D)         (None, 7, 7, 2048)   1050624     activation_45[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5b_branch2c (BatchNormalizati (None, 7, 7, 2048)   8192        res5b_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_15 (Add)                    (None, 7, 7, 2048)   0           bn5b_branch2c[0][0]              \n",
      "                                                                 activation_43[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_46 (Activation)      (None, 7, 7, 2048)   0           add_15[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "res5c_branch2a (Conv2D)         (None, 7, 7, 512)    1049088     activation_46[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5c_branch2a (BatchNormalizati (None, 7, 7, 512)    2048        res5c_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_47 (Activation)      (None, 7, 7, 512)    0           bn5c_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5c_branch2b (Conv2D)         (None, 7, 7, 512)    2359808     activation_47[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5c_branch2b (BatchNormalizati (None, 7, 7, 512)    2048        res5c_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_48 (Activation)      (None, 7, 7, 512)    0           bn5c_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5c_branch2c (Conv2D)         (None, 7, 7, 2048)   1050624     activation_48[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5c_branch2c (BatchNormalizati (None, 7, 7, 2048)   8192        res5c_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_16 (Add)                    (None, 7, 7, 2048)   0           bn5c_branch2c[0][0]              \n",
      "                                                                 activation_46[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_49 (Activation)      (None, 7, 7, 2048)   0           add_16[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "avg_pool (AveragePooling2D)     (None, 1, 1, 2048)   0           activation_49[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d_1 (Glo (None, 2048)         0           avg_pool[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 23,587,712\n",
      "Trainable params: 23,534,592\n",
      "Non-trainable params: 53,120\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "if not os.path.exists('./data/triplet/train_naive_resnet50_features.npy'):\n",
    "    # Load pre-trained ResNet50\n",
    "    img_size = (224, 224, 3)\n",
    "    resnet = ResNet50(include_top=False, weights='imagenet', input_shape=img_size, pooling='avg')\n",
    "    resnet.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Process:          0\n",
      "================================================================================\n",
      "Process:      20000\n",
      "================================================================================\n",
      "Process:      40000\n",
      "================================================================================\n",
      "Process:      60000\n",
      "================================================================================\n",
      "Process:      80000\n",
      "================================================================================\n",
      "Process:     100000\n",
      "========================================================"
     ]
    }
   ],
   "source": [
    "# Process training images\n",
    "if not os.path.exists('./data/triplet/train_naive_resnet50_features.npy'):\n",
    "    train_imgs = np.zeros(shape=(len(train_df), 2048), dtype=np.float32)\n",
    "    img_ids = train_df['image_id'].values\n",
    "    steps = 20000\n",
    "    for i in range(0, len(train_df), steps):\n",
    "        tmp_imgs = []\n",
    "        print('\\nProcess: {:10d}'.format(i))\n",
    "\n",
    "        start = i\n",
    "        end = min(len(train_df), i + steps)\n",
    "        for idx in range(start, end):\n",
    "            if idx % 250 == 0:\n",
    "                print('=', end='')\n",
    "\n",
    "            img_id = img_ids[idx]\n",
    "            path = './data/triplet/train/' + str(img_id) + '.jpg'\n",
    "            img = load_img(path, target_size=img_size[:2])\n",
    "            img = img_to_array(img)\n",
    "            tmp_imgs.append(img)\n",
    "\n",
    "        tmp_imgs = np.array(tmp_imgs, dtype=np.float32) / 255.0\n",
    "        tmp_prediction = resnet.predict(tmp_imgs)\n",
    "        train_imgs[start: end, ] = tmp_prediction\n",
    "        _ = gc.collect()\n",
    "        \n",
    "    # Save to disk\n",
    "    np.save('./data/triplet/train_naive_resnet50_features.npy', train_imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Process:          0\n",
      "================================================================================\n",
      "Process:       4000\n",
      "================================================================================\n",
      "Process:       8000\n",
      "================================================================================\n",
      "Process:      12000\n",
      "================================================================================\n",
      "Process:      16000\n",
      "================================================================================\n",
      "Process:      20000\n",
      "=============================================="
     ]
    }
   ],
   "source": [
    "# Process validation images\n",
    "if not os.path.exists('./data/triplet/validation_naive_resnet50_features.npy'):\n",
    "    val_imgs = np.zeros(shape=(len(val_df), 2048), dtype=np.float32)\n",
    "    img_ids = val_df['image_id'].values\n",
    "    steps = 4000\n",
    "    for i in range(0, len(val_df), steps):\n",
    "        tmp_imgs = []\n",
    "        print('\\nProcess: {:10d}'.format(i))\n",
    "\n",
    "        start = i\n",
    "        end = min(len(val_df), i + steps)\n",
    "        for idx in range(start, end):\n",
    "            if idx % 50 == 0:\n",
    "                print('=', end='')\n",
    "\n",
    "            img_id = img_ids[idx]\n",
    "            path = './data/triplet/validation/' + str(img_id) + '.jpg'\n",
    "            img = load_img(path, target_size=img_size[:2])\n",
    "            img = img_to_array(img)\n",
    "            tmp_imgs.append(img)\n",
    "\n",
    "        tmp_imgs = np.array(tmp_imgs, dtype=np.float32) / 255.0\n",
    "        tmp_prediction = resnet.predict(tmp_imgs)\n",
    "        val_imgs[start: end, ] = tmp_prediction\n",
    "        _ = gc.collect()\n",
    "\n",
    "    # Save to disk\n",
    "    np.save('./data/triplet/validation_naive_resnet50_features.npy', val_imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Process:          0\n",
      "================================================================================\n",
      "Process:       4000\n",
      "================================================================================\n",
      "Process:       8000\n",
      "================================================================================\n",
      "Process:      12000\n",
      "================================================================================\n",
      "Process:      16000\n",
      "================================================================================\n",
      "Process:      20000\n",
      "================================================"
     ]
    }
   ],
   "source": [
    "# Process test images\n",
    "if not os.path.exists('./data/triplet/test_naive_resnet50_features.npy'):\n",
    "    test_imgs = np.zeros(shape=(len(test_df), 2048), dtype=np.float32)\n",
    "    img_ids = test_df['image_id'].values\n",
    "    steps = 4000\n",
    "    for i in range(0, len(test_df), steps):\n",
    "        tmp_imgs = []\n",
    "        print('\\nProcess: {:10d}'.format(i))\n",
    "\n",
    "        start = i\n",
    "        end = min(len(test_df), i + steps)\n",
    "        for idx in range(start, end):\n",
    "            if idx % 50 == 0:\n",
    "                print('=', end='')\n",
    "\n",
    "            img_id = img_ids[idx]\n",
    "            path = './data/triplet/test/' + str(img_id) + '.jpg'\n",
    "            img = load_img(path, target_size=img_size[:2])\n",
    "            img = img_to_array(img)\n",
    "            tmp_imgs.append(img)\n",
    "\n",
    "        tmp_imgs = np.array(tmp_imgs, dtype=np.float32) / 255.0\n",
    "        tmp_prediction = resnet.predict(tmp_imgs)\n",
    "        test_imgs[start: end, ] = tmp_prediction\n",
    "        _ = gc.collect()\n",
    "\n",
    "    # Save to disk\n",
    "    np.save('./data/triplet/test_naive_resnet50_features.npy', test_imgs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Load Features and Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t\t (113783, 2048) (113783, 4)\n",
      "Validation:\t (22255, 2048) (22255, 4)\n",
      "Test:\t\t (22391, 2048) (22391, 4)\n"
     ]
    }
   ],
   "source": [
    "train_feature = np.load('./data/triplet/train_naive_resnet50_features.npy')\n",
    "val_feature = np.load('./data/triplet/validation_naive_resnet50_features.npy')\n",
    "test_feature = np.load('./data/triplet/test_naive_resnet50_features.npy')\n",
    "\n",
    "train_df = pd.read_csv('./data/triplet/train.csv')\n",
    "val_df = pd.read_csv('./data/triplet/validation.csv')\n",
    "test_df = pd.read_csv('./data/triplet/test.csv')\n",
    "\n",
    "print('Train:\\t\\t', train_feature.shape, train_df.shape)\n",
    "print('Validation:\\t', val_feature.shape, val_df.shape)\n",
    "print('Test:\\t\\t', test_feature.shape, test_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper function\n",
    "def accuracy(true_label, prediction, top=1):\n",
    "    \"\"\" function to calculate the prediction accuracy \"\"\"\n",
    "    prediction = prediction[:, :top]\n",
    "    count = 0\n",
    "    for i in range(len(true_label)):\n",
    "        if true_label[i] in prediction[i]:\n",
    "            count += 1\n",
    "            \n",
    "    return count / len(true_label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Implement KNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize features\n",
    "train_norm = np.linalg.norm(train_feature, axis=1, keepdims=True)\n",
    "val_norm = np.linalg.norm(val_feature, axis=1, keepdims=True)\n",
    "test_norm = np.linalg.norm(test_feature, axis=1, keepdims=True)\n",
    "\n",
    "train_norm_feature = train_feature / train_norm\n",
    "val_norm_feature = val_feature / val_norm\n",
    "test_norm_feature = test_feature / test_norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Merge train and validation features\n",
    "train_val_norm_feature = np.concatenate((train_norm_feature, val_norm_feature), axis=0)\n",
    "train_val_df = pd.concat((train_df, val_df), axis=0)\n",
    "train_val_df = train_val_df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Implement KNN model\n",
    "if not os.path.exists('./result/knn_naive_resnet50_distance.npy'):\n",
    "    knn = NearestNeighbors(n_neighbors=50, algorithm='auto', leaf_size=30, \n",
    "                           metric='minkowski', p=2, n_jobs=-1)\n",
    "    knn.fit(train_val_norm_feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists('./result/knn_naive_resnet50_distance.npy'):\n",
    "    # Search the first 50 neighbors\n",
    "    distance, neighbor_index = knn.kneighbors(test_norm_feature, return_distance=True)\n",
    "\n",
    "    # Save the results\n",
    "    np.save('./result/knn_naive_resnet50_distance.npy', distance)\n",
    "    np.save('./result/knn_naive_resnet50_neighbor.npy', neighbor_index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Search Neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_distance = np.load('./result/knn_naive_resnet50_distance.npy')\n",
    "knn_neighbor = np.load('./result/knn_naive_resnet50_neighbor.npy')\n",
    "\n",
    "# Get the first 50 neighbors\n",
    "predictions = []\n",
    "for neighbors in knn_neighbor:\n",
    "    predictions.append(train_val_df.loc[neighbors]['landmark_id'].values)\n",
    "\n",
    "predictions = np.array(predictions)\n",
    "np.save('./result/knn_naive_resnet50_test_prediction.npy', predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top  1 accuracy:\t 0.029163503193247287\n",
      "Top  5 accuracy:\t 0.06042606404358894\n",
      "Top 10 accuracy:\t 0.0807913893975258\n",
      "Top 20 accuracy:\t 0.10584609887901389\n"
     ]
    }
   ],
   "source": [
    "print('Top  1 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=1))\n",
    "print('Top  5 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=5))\n",
    "print('Top 10 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=10))\n",
    "print('Top 20 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_acc = []\n",
    "for i in range(1, 51):\n",
    "    tmp_acc = accuracy(test_df['landmark_id'].values, predictions, top=i)\n",
    "    knn_acc.append(tmp_acc)\n",
    "\n",
    "np.save('./result/knn_naive_resnet50_accuracy.npy', knn_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Environment (conda_tensorflow_p36)",
   "language": "python",
   "name": "conda_tensorflow_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}