4-2. InceptionV3 Triplet Network (margin 0.4).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import sys, os, time, gc, random, pickle\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "import requests, shutil\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "\n",
    "from keras import backend as K\n",
    "from keras.preprocessing.image import load_img, img_to_array\n",
    "from keras.models import Sequential, load_model, Model\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten, Input, Lambda\n",
    "from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau\n",
    "from keras.applications.inception_v3 import InceptionV3\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t\t (113783, 4)\n",
      "Validation:\t (22255, 4)\n",
      "Test:\t\t (22391, 4)\n",
      "\n",
      "Train Landmarks:\t 14943\n",
      "Validation Landmarks:\t 7674\n",
      "Test Landmarks:\t\t 14436\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv('./data/triplet/train.csv')\n",
    "val_df = pd.read_csv('./data/triplet/validation.csv')\n",
    "test_df = pd.read_csv('./data/triplet/test.csv')\n",
    "\n",
    "print('Train:\\t\\t', train_df.shape)\n",
    "print('Validation:\\t', val_df.shape)\n",
    "print('Test:\\t\\t', test_df.shape)\n",
    "\n",
    "print('\\nTrain Landmarks:\\t', len(train_df['landmark_id'].unique()))\n",
    "print('Validation Landmarks:\\t', len(val_df['landmark_id'].unique()))\n",
    "print('Test Landmarks:\\t\\t', len(test_df['landmark_id'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>id</th>\n",
       "      <th>url</th>\n",
       "      <th>landmark_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>465272</td>\n",
       "      <td>a2ccf8ed2e969f6a</td>\n",
       "      <td>https://lh4.googleusercontent.com/-TPHkS5gzvm4...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>64516</td>\n",
       "      <td>e205ca7c8dd7c027</td>\n",
       "      <td>https://lh3.googleusercontent.com/-V3RjsZtGpxE...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>928409</td>\n",
       "      <td>4e8ab93c1620e8a3</td>\n",
       "      <td>http://mw2.google.com/mw-panoramio/photos/medi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>88809</td>\n",
       "      <td>896bf928214d1ca4</td>\n",
       "      <td>http://lh5.ggpht.com/-Cy0l41uUaGA/R--yB8vy41I/...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1001133</td>\n",
       "      <td>375d2a153bdca926</td>\n",
       "      <td>http://lh6.ggpht.com/-UqzFpnqE9bU/S_0u1RovfdI/...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   image_id                id  \\\n",
       "0    465272  a2ccf8ed2e969f6a   \n",
       "1     64516  e205ca7c8dd7c027   \n",
       "2    928409  4e8ab93c1620e8a3   \n",
       "3     88809  896bf928214d1ca4   \n",
       "4   1001133  375d2a153bdca926   \n",
       "\n",
       "                                                 url  landmark_id  \n",
       "0  https://lh4.googleusercontent.com/-TPHkS5gzvm4...            0  \n",
       "1  https://lh3.googleusercontent.com/-V3RjsZtGpxE...            0  \n",
       "2  http://mw2.google.com/mw-panoramio/photos/medi...            0  \n",
       "3  http://lh5.ggpht.com/-Cy0l41uUaGA/R--yB8vy41I/...            0  \n",
       "4  http://lh6.ggpht.com/-UqzFpnqE9bU/S_0u1RovfdI/...            0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training set triplet generator\n",
    "def train_triplet_generator(df, batch_size=100, img_size=(224, 224), seed=42, \n",
    "                            prefix='./data/triplet/train/'):\n",
    "    \"\"\" training set triplet generator\n",
    "        it will generate 7400 triplet images in total\n",
    "    \"\"\"\n",
    "    # get images with only one training image landmark id and the rest landmark ids\n",
    "    np.random.seed(seed)\n",
    "    grouped = df[['landmark_id', 'image_id']].groupby('landmark_id').count().reset_index()\n",
    "    unique_neg_ids = list(grouped[grouped['image_id'] == 1]['landmark_id'].values)\n",
    "    rest_ids = list(grouped[grouped['image_id'] > 1]['landmark_id'].values)\n",
    "    size = 7400 * 2 - len(unique_neg_ids) \n",
    "    zeros = np.zeros((batch_size, 3, 1), dtype=K.floatx())\n",
    "    \n",
    "    while True:\n",
    "        # get positive and negative image landmark ids\n",
    "        np.random.shuffle(rest_ids)\n",
    "        candidate_ids = list(np.random.choice(rest_ids, size=size, replace=False))\n",
    "        pos_landmark_ids = candidate_ids[:7400]\n",
    "        neg_landmark_ids = candidate_ids[7400:] + unique_neg_ids\n",
    "        np.random.shuffle(neg_landmark_ids)\n",
    "        \n",
    "        # transform landmark id into image id\n",
    "        anc_img_ids = []\n",
    "        pos_img_ids = []\n",
    "        neg_img_ids = []\n",
    "        \n",
    "        for i in range(len(pos_landmark_ids)):\n",
    "            tmp_pos_ids = df[df['landmark_id'] == pos_landmark_ids[i]]['image_id'].values\n",
    "            anc_img_ids.append(tmp_pos_ids[0])\n",
    "            pos_img_ids.append(tmp_pos_ids[1])\n",
    "            \n",
    "            tmp_neg_ids = df[df['landmark_id'] == neg_landmark_ids[i]]['image_id'].values\n",
    "            neg_img_ids.append(tmp_neg_ids[0])\n",
    "        \n",
    "        # iterator to read batch images\n",
    "        for j in range(len(pos_img_ids) // batch_size):\n",
    "            batch_anc_img_ids = anc_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            batch_pos_img_ids = pos_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            batch_neg_img_ids = neg_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            \n",
    "            # get images\n",
    "            anc_imgs = []\n",
    "            pos_imgs = []\n",
    "            neg_imgs = []\n",
    "            \n",
    "            # iteratively read images\n",
    "            for k in range(batch_size):\n",
    "                anc_path = prefix + str(batch_anc_img_ids[k]) + '.jpg'\n",
    "                pos_path = prefix + str(batch_pos_img_ids[k]) + '.jpg'\n",
    "                neg_path = prefix + str(batch_neg_img_ids[k]) + '.jpg'\n",
    "                \n",
    "                tmp_anc_img = load_img(anc_path, target_size=img_size)\n",
    "                tmp_anc_img = img_to_array(tmp_anc_img)\n",
    "                anc_imgs.append(tmp_anc_img)\n",
    "                \n",
    "                tmp_pos_img = load_img(pos_path, target_size=img_size)\n",
    "                tmp_pos_img = img_to_array(tmp_pos_img)\n",
    "                pos_imgs.append(tmp_pos_img)\n",
    "                \n",
    "                tmp_neg_img = load_img(neg_path, target_size=img_size)\n",
    "                tmp_neg_img = img_to_array(tmp_neg_img)\n",
    "                neg_imgs.append(tmp_neg_img)\n",
    "        \n",
    "            # transform list to array\n",
    "            anc_imgs = np.array(anc_imgs, dtype=K.floatx()) / 255.0\n",
    "            pos_imgs = np.array(pos_imgs, dtype=K.floatx()) / 255.0\n",
    "            neg_imgs = np.array(neg_imgs, dtype=K.floatx()) / 255.0\n",
    "\n",
    "            yield [anc_imgs, pos_imgs, neg_imgs], zeros"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# validation set triplet generator\n",
    "def val_triplet_generator(df, batch_size=128, img_size=(224, 224), \n",
    "                          seed=42, prefix='./data/triplet/validation'):\n",
    "    \"\"\" validation set triplet collector \"\"\"\n",
    "    \n",
    "     # get images with only one image landmark id and the rest landmark ids\n",
    "    grouped = df[['landmark_id', 'image_id']].groupby('landmark_id').count().reset_index()\n",
    "    unique_neg_ids = list(grouped[grouped['image_id'] == 1]['landmark_id'].values)\n",
    "    rest_ids = list(grouped[grouped['image_id'] > 1]['landmark_id'].values)\n",
    "    size = 3072 * 2 - len(unique_neg_ids) \n",
    "    zeros = np.zeros((batch_size, 3, 1), dtype=K.floatx())\n",
    "    \n",
    "    while True:\n",
    "        # get positive and negative image landmark ids\n",
    "        np.random.seed(seed)\n",
    "        candidate_ids = list(np.random.choice(rest_ids, size=size, replace=False))\n",
    "        pos_landmark_ids = candidate_ids[:3072]\n",
    "        neg_landmark_ids = candidate_ids[3072:] + unique_neg_ids\n",
    "        np.random.shuffle(neg_landmark_ids)\n",
    "        \n",
    "        # transform landmark id into image id\n",
    "        anc_img_ids = []\n",
    "        pos_img_ids = []\n",
    "        neg_img_ids = []\n",
    "        \n",
    "        for i in range(len(pos_landmark_ids)):\n",
    "            tmp_pos_ids = df[df['landmark_id'] == pos_landmark_ids[i]]['image_id'].values\n",
    "            anc_img_ids.append(tmp_pos_ids[0])\n",
    "            pos_img_ids.append(tmp_pos_ids[1])\n",
    "            \n",
    "            tmp_neg_ids = df[df['landmark_id'] == neg_landmark_ids[i]]['image_id'].values\n",
    "            neg_img_ids.append(tmp_neg_ids[0])\n",
    "        \n",
    "        # iterator to read batch images\n",
    "        for j in range(len(pos_img_ids) // batch_size):\n",
    "            batch_anc_img_ids = anc_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            batch_pos_img_ids = pos_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            batch_neg_img_ids = neg_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            \n",
    "            # get images\n",
    "            anc_imgs = []\n",
    "            pos_imgs = []\n",
    "            neg_imgs = []\n",
    "            \n",
    "            # iteratively read images\n",
    "            for k in range(batch_size):\n",
    "                anc_path = prefix + str(batch_anc_img_ids[k]) + '.jpg'\n",
    "                pos_path = prefix + str(batch_pos_img_ids[k]) + '.jpg'\n",
    "                neg_path = prefix + str(batch_neg_img_ids[k]) + '.jpg'\n",
    "                \n",
    "                tmp_anc_img = load_img(anc_path, target_size=img_size)\n",
    "                tmp_anc_img = img_to_array(tmp_anc_img)\n",
    "                anc_imgs.append(tmp_anc_img)\n",
    "                \n",
    "                tmp_pos_img = load_img(pos_path, target_size=img_size)\n",
    "                tmp_pos_img = img_to_array(tmp_pos_img)\n",
    "                pos_imgs.append(tmp_pos_img)\n",
    "                \n",
    "                tmp_neg_img = load_img(neg_path, target_size=img_size)\n",
    "                tmp_neg_img = img_to_array(tmp_neg_img)\n",
    "                neg_imgs.append(tmp_neg_img)\n",
    "        \n",
    "            # transform list to array\n",
    "            anc_imgs = np.array(anc_imgs, dtype=K.floatx()) / 255.0\n",
    "            pos_imgs = np.array(pos_imgs, dtype=K.floatx()) / 255.0\n",
    "            neg_imgs = np.array(neg_imgs, dtype=K.floatx()) / 255.0\n",
    "            \n",
    "            yield [anc_imgs, pos_imgs, neg_imgs], zeros"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define Triplet Loss Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define base network for triplet network\n",
    "def base_net(input_shape=(224, 224, 3)):\n",
    "    \"\"\" define triplet network \"\"\"\n",
    "    # load pre-trained InceptionV3 model\n",
    "    inception = InceptionV3(include_top=False, weights='imagenet', input_shape=input_shape, pooling='avg')\n",
    "    \n",
    "    # frozen shallow layers\n",
    "    inception.trainable = True\n",
    "    \n",
    "    set_trainable = False\n",
    "    for layer in inception.layers:\n",
    "        if layer.name == 'mixed9':\n",
    "            set_trainable = True\n",
    "            \n",
    "        if set_trainable:\n",
    "            layer.trainable = True\n",
    "        else:\n",
    "            layer.trainable = False\n",
    "    \n",
    "    # define sequential model\n",
    "    model = Sequential(name='base_net')\n",
    "    model.add(inception)\n",
    "    model.add(Lambda(lambda x: K.l2_normalize(x, axis=1), name='l2_norm'))\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define triplet network\n",
    "def triplet_net(base_model, input_shape=(224, 224, 3)):\n",
    "    \"\"\" function to define triplet networks \"\"\"\n",
    "    # define input: anchor, positive, negative\n",
    "    anchor = Input(shape=input_shape, name='anchor_input')\n",
    "    positive = Input(shape=input_shape, name='positive_input')\n",
    "    negative = Input(shape=input_shape, name='negative_input')\n",
    "    \n",
    "    # extract vector represent using CNN based model\n",
    "    anc_vec = base_model(anchor)\n",
    "    pos_vec = base_model(positive)\n",
    "    neg_vec = base_model(negative)\n",
    "    \n",
    "    # stack outputs\n",
    "    stacks = Lambda(lambda x: K.stack(x, axis=1), name='output')([anc_vec, pos_vec, neg_vec])\n",
    "\n",
    "    # define inputs and outputs\n",
    "    inputs=[anchor, positive, negative]\n",
    "    outputs = stacks\n",
    "    \n",
    "    # define the triplet model\n",
    "    model = Model(inputs=inputs, outputs=outputs, name='triplet_net')\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define triplet loss\n",
    "def triplet_loss(y_true, y_pred):\n",
    "    \"\"\" function to compute triplet loss\n",
    "        margin is predefined coded, manually change if needed\n",
    "    \"\"\"\n",
    "    # define triplet margin\n",
    "    margin = K.constant(0.4)\n",
    "    zero = K.constant(0.0)\n",
    "    \n",
    "    # get the prediction vector\n",
    "    anchor, positive, negative = y_pred[:, 0], y_pred[:, 1], y_pred[:, 2]\n",
    "    \n",
    "    # compute distance\n",
    "    pos_distance = K.sum(K.square(anchor - positive), axis=1)\n",
    "    neg_distance = K.sum(K.square(anchor - negative), axis=1)\n",
    "    \n",
    "    # compute loss\n",
    "    partial_loss = pos_distance - neg_distance + margin\n",
    "    full_loss = K.sum(K.maximum(partial_loss, zero), axis=0)\n",
    "    \n",
    "    return full_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build Triplet Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For reproduciable purpose\n",
    "seed = 42\n",
    "K.clear_session()\n",
    "os.environ['PYTHONHASHSEED'] = '0'\n",
    "np.random.seed(seed)\n",
    "random.seed(seed)\n",
    "session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)\n",
    "tf.set_random_seed(seed)\n",
    "sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)\n",
    "K.set_session(sess)\n",
    "\n",
    "# Define Parameters\n",
    "img_size = (224, 224, 3)  # target image size\n",
    "\n",
    "# triplet image generator\n",
    "train_generator = train_triplet_generator(train_df, batch_size=100, img_size=img_size[:2], \n",
    "                                          seed=42, prefix='./data/triplet/train/')\n",
    "\n",
    "val_generator = val_triplet_generator(val_df, batch_size=64, img_size=img_size[:2], \n",
    "                                      seed=42, prefix='./data/triplet/validation/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3148: calling l2_normalize (from tensorflow.python.ops.nn_impl) with dim is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "dim is deprecated, use axis instead\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "inception_v3 (Model)         (None, 2048)              21802784  \n",
      "_________________________________________________________________\n",
      "l2_norm (Lambda)             (None, 2048)              0         \n",
      "=================================================================\n",
      "Total params: 21,802,784\n",
      "Trainable params: 6,073,536\n",
      "Non-trainable params: 15,729,248\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Define triplet network model\n",
    "base_model = base_net(input_shape=img_size)\n",
    "base_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "anchor_input (InputLayer)       (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "positive_input (InputLayer)     (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "negative_input (InputLayer)     (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "base_net (Sequential)           (None, 2048)         21802784    anchor_input[0][0]               \n",
      "                                                                 positive_input[0][0]             \n",
      "                                                                 negative_input[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "output (Lambda)                 (None, 3, 2048)      0           base_net[1][0]                   \n",
      "                                                                 base_net[2][0]                   \n",
      "                                                                 base_net[3][0]                   \n",
      "==================================================================================================\n",
      "Total params: 21,802,784\n",
      "Trainable params: 6,073,536\n",
      "Non-trainable params: 15,729,248\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "triplet_model = triplet_net(base_model=base_model, input_shape=img_size)\n",
    "triplet_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fit Triplet Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  0.001\n"
     ]
    }
   ],
   "source": [
    "# define learning scheduler\n",
    "def lr_schedule(epoch):\n",
    "    \"\"\" Learning rate schedule \"\"\"\n",
    "    lr = 1e-3\n",
    "    if epoch > 80:\n",
    "        lr *= 2e-1\n",
    "    elif epoch > 60:\n",
    "        lr *= 4e-1\n",
    "    elif epoch > 40:\n",
    "        lr *= 6e-1\n",
    "    elif epoch > 20:\n",
    "        lr *= 8e-1\n",
    "    print('Learning rate: ', lr)\n",
    "    return lr\n",
    "\n",
    "# define optimizer\n",
    "opt = keras.optimizers.Adam(lr=lr_schedule(0))\n",
    "\n",
    "# Create call backs\n",
    "lr_scheduler = LearningRateScheduler(lr_schedule)\n",
    "lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6)\n",
    "callbacks = [lr_reducer, lr_scheduler]\n",
    "\n",
    "# compile the model\n",
    "triplet_model.compile(optimizer=opt, loss=triplet_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "Learning rate:  0.001\n",
      " - 72s - loss: 9.4577 - val_loss: 5.4752\n",
      "Epoch 2/100\n",
      "Learning rate:  0.001\n",
      " - 51s - loss: 5.2701 - val_loss: 5.0196\n",
      "Epoch 3/100\n",
      "Learning rate:  0.001\n",
      " - 52s - loss: 4.1987 - val_loss: 4.6280\n",
      "Epoch 4/100\n",
      "Learning rate:  0.001\n",
      " - 51s - loss: 3.9098 - val_loss: 5.0792\n",
      "Epoch 5/100\n",
      "Learning rate:  0.001\n",
      " - 51s - loss: 3.3303 - val_loss: 4.9101\n",
      "Epoch 6/100\n",
      "Learning rate:  0.001\n",
      " - 51s - loss: 2.7693 - val_loss: 4.8455\n",
      "Epoch 7/100\n",
      "Learning rate:  0.001\n",
      " - 51s - loss: 2.6522 - val_loss: 4.3390\n",
      "Epoch 8/100\n",
      "Learning rate:  0.001\n",
      " - 51s - loss: 2.3822 - val_loss: 4.8301\n",
      "Epoch 9/100\n",
      "Learning rate:  0.001\n",
      " - 52s - loss: 2.3149 - val_loss: 4.3806\n",
      "Epoch 10/100\n",
      "Learning rate:  0.001\n",
      " - 51s - loss: 1.8642 - val_loss: 4.1901\n",
      "Epoch 11/100\n",
      "Learning rate:  0.001\n",
      " - 52s - loss: 2.0404 - val_loss: 4.4720\n",
      "Epoch 12/100\n",
      "Learning rate:  0.001\n",
      " - 52s - loss: 1.8926 - val_loss: 4.4903\n",
      "Epoch 13/100\n",
      "Learning rate:  0.001\n",
      " - 51s - loss: 1.6310 - val_loss: 4.2339\n",
      "Epoch 14/100\n",
      "Learning rate:  0.001\n",
      " - 51s - loss: 1.4849 - val_loss: 4.4353\n",
      "Epoch 15/100\n",
      "Learning rate:  0.001\n",
      " - 50s - loss: 1.2735 - val_loss: 4.3689\n",
      "Epoch 16/100\n",
      "Learning rate:  0.001\n",
      " - 54s - loss: 1.5059 - val_loss: 4.2429\n",
      "Epoch 17/100\n",
      "Learning rate:  0.001\n",
      " - 53s - loss: 1.3659 - val_loss: 4.1282\n",
      "Epoch 18/100\n",
      "Learning rate:  0.001\n",
      " - 51s - loss: 1.3062 - val_loss: 4.1500\n",
      "Epoch 19/100\n",
      "Learning rate:  0.001\n",
      " - 51s - loss: 1.2547 - val_loss: 4.2685\n",
      "Epoch 20/100\n",
      "Learning rate:  0.001\n",
      " - 51s - loss: 1.2075 - val_loss: 4.1305\n",
      "Epoch 21/100\n",
      "Learning rate:  0.001\n",
      " - 53s - loss: 1.0351 - val_loss: 4.2235\n",
      "Epoch 22/100\n",
      "Learning rate:  0.0008\n",
      " - 51s - loss: 0.9968 - val_loss: 4.1225\n",
      "Epoch 23/100\n",
      "Learning rate:  0.0008\n",
      " - 52s - loss: 1.0054 - val_loss: 4.2024\n",
      "Epoch 24/100\n",
      "Learning rate:  0.0008\n",
      " - 51s - loss: 1.0459 - val_loss: 4.2197\n",
      "Epoch 25/100\n",
      "Learning rate:  0.0008\n",
      " - 55s - loss: 0.8682 - val_loss: 4.1699\n",
      "Epoch 26/100\n",
      "Learning rate:  0.0008\n",
      " - 51s - loss: 0.7923 - val_loss: 4.1597\n",
      "Epoch 27/100\n",
      "Learning rate:  0.0008\n",
      " - 51s - loss: 0.8771 - val_loss: 4.2008\n",
      "Epoch 28/100\n",
      "Learning rate:  0.0008\n",
      " - 52s - loss: 0.7580 - val_loss: 4.3973\n",
      "Epoch 29/100\n",
      "Learning rate:  0.0008\n",
      " - 51s - loss: 0.8184 - val_loss: 4.0907\n",
      "Epoch 30/100\n",
      "Learning rate:  0.0008\n",
      " - 51s - loss: 0.7667 - val_loss: 4.3014\n",
      "Epoch 31/100\n",
      "Learning rate:  0.0008\n",
      " - 51s - loss: 0.8210 - val_loss: 4.4948\n",
      "Epoch 32/100\n",
      "Learning rate:  0.0008\n",
      " - 51s - loss: 0.8401 - val_loss: 4.2011\n",
      "Epoch 33/100\n",
      "Learning rate:  0.0008\n",
      " - 52s - loss: 0.8358 - val_loss: 4.1774\n",
      "Epoch 34/100\n",
      "Learning rate:  0.0008\n",
      " - 51s - loss: 0.7169 - val_loss: 4.0920\n",
      "Epoch 35/100\n",
      "Learning rate:  0.0008\n",
      " - 51s - loss: 0.7838 - val_loss: 4.0560\n",
      "Epoch 36/100\n",
      "Learning rate:  0.0008\n",
      " - 53s - loss: 0.6471 - val_loss: 4.3247\n",
      "Epoch 37/100\n",
      "Learning rate:  0.0008\n",
      " - 51s - loss: 0.7422 - val_loss: 4.1937\n",
      "Epoch 38/100\n",
      "Learning rate:  0.0008\n",
      " - 55s - loss: 0.6119 - val_loss: 4.3520\n",
      "Epoch 39/100\n",
      "Learning rate:  0.0008\n",
      " - 52s - loss: 0.7833 - val_loss: 4.5128\n",
      "Epoch 40/100\n",
      "Learning rate:  0.0008\n",
      " - 51s - loss: 0.6258 - val_loss: 4.4050\n",
      "Epoch 41/100\n",
      "Learning rate:  0.0008\n",
      " - 51s - loss: 0.5911 - val_loss: 4.2522\n",
      "Epoch 42/100\n",
      "Learning rate:  0.0006\n",
      " - 52s - loss: 0.6282 - val_loss: 4.3064\n",
      "Epoch 43/100\n",
      "Learning rate:  0.0006\n",
      " - 51s - loss: 0.5237 - val_loss: 4.3135\n",
      "Epoch 44/100\n",
      "Learning rate:  0.0006\n",
      " - 51s - loss: 0.5714 - val_loss: 4.2662\n",
      "Epoch 45/100\n",
      "Learning rate:  0.0006\n",
      " - 54s - loss: 0.6017 - val_loss: 4.2526\n",
      "Epoch 46/100\n",
      "Learning rate:  0.0006\n",
      " - 50s - loss: 0.6327 - val_loss: 4.3601\n",
      "Epoch 47/100\n",
      "Learning rate:  0.0006\n",
      " - 51s - loss: 0.5531 - val_loss: 4.1822\n",
      "Epoch 48/100\n",
      "Learning rate:  0.0006\n",
      " - 53s - loss: 0.5179 - val_loss: 4.2607\n",
      "Epoch 49/100\n",
      "Learning rate:  0.0006\n",
      " - 53s - loss: 0.5322 - val_loss: 4.2399\n",
      "Epoch 50/100\n",
      "Learning rate:  0.0006\n",
      " - 50s - loss: 0.5106 - val_loss: 4.2758\n",
      "Epoch 51/100\n",
      "Learning rate:  0.0006\n",
      " - 50s - loss: 0.4689 - val_loss: 4.2983\n",
      "Epoch 52/100\n",
      "Learning rate:  0.0006\n",
      " - 50s - loss: 0.4605 - val_loss: 4.3330\n",
      "Epoch 53/100\n",
      "Learning rate:  0.0006\n",
      " - 52s - loss: 0.4742 - val_loss: 4.1499\n",
      "Epoch 54/100\n",
      "Learning rate:  0.0006\n",
      " - 50s - loss: 0.4156 - val_loss: 4.2877\n",
      "Epoch 55/100\n",
      "Learning rate:  0.0006\n",
      " - 51s - loss: 0.4195 - val_loss: 4.3457\n",
      "Epoch 56/100\n",
      "Learning rate:  0.0006\n",
      " - 51s - loss: 0.4021 - val_loss: 4.3408\n",
      "Epoch 57/100\n",
      "Learning rate:  0.0006\n",
      " - 52s - loss: 0.4135 - val_loss: 4.2566\n",
      "Epoch 58/100\n",
      "Learning rate:  0.0006\n",
      " - 52s - loss: 0.4971 - val_loss: 4.1925\n",
      "Epoch 59/100\n",
      "Learning rate:  0.0006\n",
      " - 51s - loss: 0.4840 - val_loss: 4.3735\n",
      "Epoch 60/100\n",
      "Learning rate:  0.0006\n",
      " - 53s - loss: 0.3446 - val_loss: 4.2233\n",
      "Epoch 61/100\n",
      "Learning rate:  0.0006\n",
      " - 50s - loss: 0.4300 - val_loss: 4.1811\n",
      "Epoch 62/100\n",
      "Learning rate:  0.0004\n",
      " - 53s - loss: 0.4225 - val_loss: 4.1687\n",
      "Epoch 63/100\n",
      "Learning rate:  0.0004\n",
      " - 55s - loss: 0.3787 - val_loss: 4.2902\n",
      "Epoch 64/100\n",
      "Learning rate:  0.0004\n",
      " - 52s - loss: 0.3705 - val_loss: 4.3795\n",
      "Epoch 65/100\n",
      "Learning rate:  0.0004\n",
      " - 54s - loss: 0.3724 - val_loss: 4.3317\n",
      "Epoch 66/100\n",
      "Learning rate:  0.0004\n",
      " - 51s - loss: 0.3552 - val_loss: 4.2706\n",
      "Epoch 67/100\n",
      "Learning rate:  0.0004\n",
      " - 51s - loss: 0.3418 - val_loss: 4.2079\n",
      "Epoch 68/100\n",
      "Learning rate:  0.0004\n",
      " - 51s - loss: 0.3794 - val_loss: 4.2030\n",
      "Epoch 69/100\n",
      "Learning rate:  0.0004\n",
      " - 52s - loss: 0.3445 - val_loss: 4.2459\n",
      "Epoch 70/100\n",
      "Learning rate:  0.0004\n",
      " - 51s - loss: 0.3187 - val_loss: 4.3912\n",
      "Epoch 71/100\n",
      "Learning rate:  0.0004\n",
      " - 57s - loss: 0.3762 - val_loss: 4.3259\n",
      "Epoch 72/100\n",
      "Learning rate:  0.0004\n",
      " - 51s - loss: 0.3142 - val_loss: 4.3974\n",
      "Epoch 73/100\n",
      "Learning rate:  0.0004\n",
      " - 51s - loss: 0.3181 - val_loss: 4.3766\n",
      "Epoch 74/100\n",
      "Learning rate:  0.0004\n",
      " - 51s - loss: 0.2864 - val_loss: 4.2545\n",
      "Epoch 75/100\n",
      "Learning rate:  0.0004\n",
      " - 52s - loss: 0.3611 - val_loss: 4.2411\n",
      "Epoch 76/100\n",
      "Learning rate:  0.0004\n",
      " - 51s - loss: 0.3225 - val_loss: 4.1668\n",
      "Epoch 77/100\n",
      "Learning rate:  0.0004\n",
      " - 50s - loss: 0.3015 - val_loss: 4.1886\n",
      "Epoch 78/100\n",
      "Learning rate:  0.0004\n",
      " - 50s - loss: 0.3369 - val_loss: 4.3120\n",
      "Epoch 79/100\n",
      "Learning rate:  0.0004\n",
      " - 52s - loss: 0.3157 - val_loss: 4.2711\n",
      "Epoch 80/100\n",
      "Learning rate:  0.0004\n",
      " - 52s - loss: 0.2833 - val_loss: 4.3291\n",
      "Epoch 81/100\n",
      "Learning rate:  0.0004\n",
      " - 51s - loss: 0.2964 - val_loss: 4.2202\n",
      "Epoch 82/100\n",
      "Learning rate:  0.0002\n",
      " - 51s - loss: 0.2444 - val_loss: 4.2028\n",
      "Epoch 83/100\n",
      "Learning rate:  0.0002\n",
      " - 52s - loss: 0.2564 - val_loss: 4.2302\n",
      "Epoch 84/100\n",
      "Learning rate:  0.0002\n",
      " - 50s - loss: 0.2400 - val_loss: 4.2142\n",
      "Epoch 85/100\n",
      "Learning rate:  0.0002\n",
      " - 51s - loss: 0.2584 - val_loss: 4.1992\n",
      "Epoch 86/100\n",
      "Learning rate:  0.0002\n",
      " - 53s - loss: 0.2214 - val_loss: 4.1699\n",
      "Epoch 87/100\n",
      "Learning rate:  0.0002\n",
      " - 51s - loss: 0.2343 - val_loss: 4.1698\n",
      "Epoch 88/100\n",
      "Learning rate:  0.0002\n",
      " - 51s - loss: 0.2390 - val_loss: 4.1747\n",
      "Epoch 89/100\n",
      "Learning rate:  0.0002\n",
      " - 50s - loss: 0.2230 - val_loss: 4.2036\n",
      "Epoch 90/100\n",
      "Learning rate:  0.0002\n",
      " - 51s - loss: 0.2077 - val_loss: 4.2264\n",
      "Epoch 91/100\n",
      "Learning rate:  0.0002\n",
      " - 53s - loss: 0.2425 - val_loss: 4.1467\n",
      "Epoch 92/100\n",
      "Learning rate:  0.0002\n",
      " - 51s - loss: 0.1606 - val_loss: 4.1344\n",
      "Epoch 93/100\n",
      "Learning rate:  0.0002\n",
      " - 53s - loss: 0.2451 - val_loss: 4.1494\n",
      "Epoch 94/100\n",
      "Learning rate:  0.0002\n",
      " - 51s - loss: 0.2387 - val_loss: 4.1862\n",
      "Epoch 95/100\n",
      "Learning rate:  0.0002\n",
      " - 51s - loss: 0.2377 - val_loss: 4.2005\n",
      "Epoch 96/100\n",
      "Learning rate:  0.0002\n",
      " - 51s - loss: 0.1958 - val_loss: 4.2175\n",
      "Epoch 97/100\n",
      "Learning rate:  0.0002\n",
      " - 53s - loss: 0.2103 - val_loss: 4.2612\n",
      "Epoch 98/100\n",
      "Learning rate:  0.0002\n",
      " - 51s - loss: 0.1671 - val_loss: 4.3074\n",
      "Epoch 99/100\n",
      "Learning rate:  0.0002\n",
      " - 51s - loss: 0.1966 - val_loss: 4.2121\n",
      "Epoch 100/100\n",
      "Learning rate:  0.0002\n",
      " - 52s - loss: 0.2240 - val_loss: 4.2092\n"
     ]
    }
   ],
   "source": [
    "# fit the mode\n",
    "history = triplet_model.fit_generator(train_generator, steps_per_epoch=74, epochs=100, \n",
    "                                      validation_data=val_generator, validation_steps=48, \n",
    "                                      verbose=2, callbacks=callbacks)\n",
    "\n",
    "base_model.save('./models/inception-base-0.4-model.h5')\n",
    "pickle.dump(history.history, open('./models/inception-triplet-0.4-history.p', 'wb'))\n",
    "_ = gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAl8AAAG/CAYAAABmE7ICAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAIABJREFUeJzs3Xd8VFX+//HXSS+0BAgdQiegCEhVkN7ELpYFFRT7uqKufgXXvutvZde166qrWEBFsWADpWhWQQFBYQVCkd57SSA95/fHmYQkpMKUhLyfj0ceSe7cufdkbmbmPZ9z7rnGWouIiIiI+EdQoBsgIiIiUpUofImIiIj4kcKXiIiIiB8pfImIiIj4kcKXiIiIiB8pfImIiIj4kcKXSBVmjBlrjLHGmFaBbktFYIyJ9zwexX11CmDbdKxEThMhgW6AiEgF9Hfg8yKWr/V3Q0Tk9KPwJSJyog3W2oWBboSInJ7U7SgipTLGXGOMWW6MSTPG7DPGTDHGNCi0zihjzK/GmBRjzBFjzG/GmFvy3d7NGDPHGLPfGJNqjNlgjHm5hH3WN8ZkGWPuLOK2/zPGZBpj6np+H2qM+dEYc9iz/zXGmIe9+RgU2n9u9+TtxpinjTF7jDHHjDFfGmPiC60baoz5mzFmkzEmw/P9b8aY0ELrRRtjnjTGrDfGpBtjdhljPjbG1Cu0+zrGmHc9j/EOY8zzxpiIfNsJMcb81bOd3OM13xjT21ePh4iUjypfIlIiY8zNwKvAB8BEoCHw/4Aexpgu1toUzxv7VOB54D7cB7t2QC3PNqoB3wCLgbFAMhAPnFPcfq21u4wxc4FrPNvN71rga2vtXmNMC1wX4UfA40AG0BpocQp/dpAxpvDro7XWZhdaNhFYBlwPxOEel9nGmA7W2kzPOm8DV3pum4/7m//iad8oAGNMGDAHOAt4ElgI1ASGAjHA7nz7nAK8D1wG9AIeBQ4Cj3huvx+427OPZUANoCsQW/6HQUR8wlqrL33pq4p+4YKQBVoVc3sw7o3/u0LLe3vud6fn93uBAyXsp6tn/Y7lbN9oz/3a5lvWybPsSs/vIz2/1/DC4xHv2VZRXylFrLcKCMq3/FzP8nGe38/w/P5oof08mP/xAG7w/H5RGY7VY4WWfwmsLfT7J4H+39KXvvRV/Je6HUWkJG1xFZ138y+01s4HNgN9PYt+BmKMMVONMRcYY2oV2s464BDwqqcLs0kZ9/8pkIKrdOW6FjjM8QHxy4BMYJoxZqQxJq6M2y7J34Buhb76FLHeR9banNxfrLULgG24ihTAeZ7vUwvdL/f33MdvCLDLWlvUIP/Cvir0+29A03y//wycb4x5whjT21NVE5EKROFLREqS21W1s4jbduXebq39L3AF0AQXmPYaY+YaYzp6bj8M9Ad2AC8DW4wxK4wxl5e0c2vtMeBjYLRxgoE/ANOttWmedX7Hdc8F4brkdhljFhpj+ha33TLYbK1dUujr1yLW213Mskaen4t7/HYVur02sL2MbTtQ6Pd0IDzf7/8P1wV5EfADsN8Y86Yxpk4Zty8iPqbwJSIlyX2jr1/EbfXz3Y619iNrbV/cGKVLgQbA18aYIM/ty6y1l+MCRy9gPfChMeaMUtowBdfN1xsY6NnulPwrWGu/s9YOw40xGwRkAV/5IXAUHgyfuyw3SBX3+NUvdPs+jge2U2KtzbTWTrLWnol7rO4GLgde8sb2ReTUKXyJSEnW4Co5V+dfaIw5B2gGJBa+g7U2xVr7JW6QfgNcVSf/7VnWTePwEO41KKGUNnyH68q71vO1CVfROYG1Nt1a+y3wDyAaaF7Ktk/VyNxwCWCMORdoDPzkWfS95/vVhe432vM90fN9NlDfGHOhNxtnrd1lrX0dmIsbfyYiFYDOdhQRgGHGmF2Flh221s7xTNnwqjFmKm6sUiPgCdw4rskAxpjHcRWf73Bdi42BO4Fl1p2ReAFwMzAD2IgLRnfiznr8iRJYa3OMMe8CtwChwDPWWpt7uzHmVtzYqpnAVqAO7izEHcAKzzp9gXnADdbad8rweLQwxvQsYvlaa23+br/qwAxjzKtAXdzkrOuAdzxtX2GMeR941HP25I+4qt9DwPvW2t8825kK3AS8b4z5O7DIs+2hwLPW2tVlaDOev/UzYDnwC+4syM7AMFwYFpEKQOFLRABeKGLZSuAMa+1rxphjuCkkPsMNgJ8J/J+19qhn3UW4MPUMrltxD66a85Dn9nVAquf3BrjQ9TMw2Fq7rQztm4KbQiH35/yWA8NxwScO15U3HxhtrU31rGNwZ26Wtdo/0fNV2BW4KS1y/R1oBbyFC5TfAXfY49NMgDtLcQPujMYHcaFwEvBY7grW2kxjzBDcWK2bPd/3Aws4cYxXab73tPOPQBSwBVcJfKKc2xERHzH5PkCKiEgZeCZS3Qjc5OnWExEpM435EhEREfEjhS8RERERP1K3o4iIiIgfqfIlIiIi4kcV+mzHOnXq2Pj4eJ/u4+jRo0RHR/t0H1J+Oi4Vl45NxaTjUnHp2FRMvjguS5cu3WetrVvaehU6fMXHx7NkyRKf7iMxMZF+/fr5dB9SfjouFZeOTcWk41Jx6dhUTL44LsaYzWVZT92OIiIiIn6k8CUiIiLiRwpfIiIiIn6k8CUiIiLiRwpfIiIiIn5Uoc92FBERKc2RI0fYs2cPmZmZpa8cADVr1iQpKSnQzZBCyntcQkNDiYuLo0aNGqe8b4UvERGptI4cOcLu3btp1KgRkZGRGGMC3aQTJCcnU7169UA3Qwopz3Gx1pKamsr27dsBTjmAqdtRREQqrT179tCoUSOioqIqZPCS04MxhqioKBo1asSePXtOeXsKXyIiUmllZmYSGRkZ6GZIFREZGemV7m2FLxERqdRU8RJ/8db/msKXiIiIiB8pfImIiIj4kcKXiIjIaeTqq69m5MiR5bpPz549uffee33UIilMU02IiIj4UWnjhsaMGcNbb7110tt/9dVXsdaW6z4zZ84kNDT0pPdZVhMmTGDu3LksWbLE5/uqyKp0+Nq07yh7j+UEuhkiIlKF7Ny5M+/nL7/8kptuuqnAsuLO3szMzCxTQKpZs2a52xQbG1vu+8jJq9LdjrdOXcp7qzMC3QwREalC6tevn/dVq1atE5bVrFmT1atXY4xh+vTp9O3bl4iICN5++212797NVVddlTe32RlnnMG7775bYPuFux179uzJ3XffzX333UdsbCz169dn4sSJBapjhbsd69evz6RJk7jhhhuoXr06TZo04fnnny+wn1WrVnHuuecSERFB+/btmTNnDiEhIUybNu2kH5t9+/YxevRoYmJiiIqKYujQoaxZsybv9v379zNq1Cjq1q1LREQErVq14t///nfe7S+88AKtWrUiPDycunXrMnz48JNuiy9V6cpXeGgwmamBboWIiHjTY1+sZNWOI37dZ/uGNXjkwg5e3+6ECRP417/+xVlnnUV4eDipqan07NmTiRMnUqNGDWbNmsWYMWNo1qwZvXv3LnY7kydP5r777mPRokUsXryY6667ju7du3PppZcWe5+nnnqKv/71rzzwwAPMmDGD8ePH07t3b7p06UJWVhYXX3wxLVu2ZPHixRw5coS7776bnJxT600aPXo027dv54svvqB69ercf//9DB8+nKSkJMLDw5kwYQLr1q1j1qxZ1KlThw0bNnDo0CEAFixYwJ///GemTJlCr169OHDgAPPmzTul9vhKlQ5fESFBHMouX7+4iIiIv9xzzz1ccsklBZbdfffdeT//8Y9/ZM6cOUybNq3E8NWlSxcefPBBAFq3bs0rr7zCvHnzSgxfF1xwAbfeeisA9957L8899xzffvstXbp04auvvmLz5s0sWLCAuLg4ACZNmsTAgQNP+m/97bffmD17NosWLaJ79+4AvPvuuzRt2pTp06dzzTXXsHnzZrp27UrXrl0BiI+Pz7v/5s2bqVGjBhdeeCFRUVE0bdqUTp06nXR7fKlKh6/w0GAyNeRLROS04osKVKDkhoxcWVlZPPHEE3z00Uds376djIwM0tPTS+1e69ixY4HfGzZsWOplckq6z+rVq4mPj88LXgA9evQo9e8pSVJSEmFhYXTr1i1vWe3atUlISGDVqlWAC5tXX301CxcuZPDgwVx00UV5ofP888/nb3/7G/Hx8QwdOpShQ4dy6aWXEh0dfUrt8oUqPeYrPCRI4UtERCqswsHhiSee4KWXXmLixIl89913LFu2jPPPP5+MjJLHLxceqG+MKbWL8GTu4yu5Z4hefPHFbN68mfHjx7Nz506GDh3KbbfdBkCtWrVYvnw57777Lg0bNuTxxx+nffv2XrkWo7dV6fAVERpMhrodRUSkkpg/fz6XXnopo0aN4qyzzqJFixasXbvW7+1o164dmzdvZu/evXnLFi9efErbTEhIICMjg59//jlv2f79+0lKSqJ9+/Z5y+Li4hg7dixTpkzh5Zdf5vXXX88LhaGhoQwePJhJkyaxfPly9u7dy9dff31K7fKFqt3tqMqXiIhUIm3atOGrr77ip59+olatWjz99NPs2LGDZs2a+bUdI0aMoGnTpowZM4Ynn3yS5ORkJkyYgDGm1HnMUlNTWbZsWYFl1apV48wzz2To0KGMGzeOV155hWrVqjFhwgTq1avHFVdcAcADDzxAz549ad++Penp6cyYMYO2bdsSFBTEJ598wo4dO+jduzcxMTHMnj2btLQ0EhISfPY4nKwqXvkKIlOVLxERqSQee+wxOnbsyODBg+nXrx9xcXHlns3eG0JCQvjss884dOgQ3bp148Ybb+Thhx8GICIiosT7rlq1is6dOxf4Gjt2LABTp06lY8eOjBgxgl69epGTk8OsWbMICwsDXGXr/vvvp2PHjpx33nlkZ2fzySefABATE8P06dMZMGAACQkJvPDCC7zzzjsFxpBVFKa8s+D6U9euXa0vZ8H965erePenjax+YoTP9iEnJzExkX79+gW6GVIEHZuKqaoel6SkpApZ2cgvOTmZ6tWrB7oZPrdo0SJ69uzJihUr6NCh4p/0cLLHpaT/OWPMUmtt1yJvzKdKdztGhKrbUURE5GRMnz6dmJgYWrVqxfr167nrrrvo3r17pQhegValw1d4SDDZFrKycwgJrtI9sCIiIuVy+PBhJk6cyLZt26hduzYDBw7k6aefDnSzKoUqHr5c4ErPUvgSEREpjxtvvJEbb7wx0M2olKp04ogIDQYgLTM7wC0RERGRqqJKh6/8lS8RERERf6jS4Su38qXwJSIiIv5SpcNXbuVL3Y4iIiLiL1U7fIWq21FERET8q0qHr4gQDbgXERER/6rS4UuVLxERqcxef/11atWqVezvRXnyySdp1aqV1/ctZVe1w5cqXyIi4mcXXXQRAwcOLPK2pKQkjDHMnj37pLY9evRo1q5deyrNO0FWVhbGGGbMmOHzfRXlwQcfpFOnTj7fjz9V6fAVocqXiIj42bhx4/juu+/YtGnTCbe98cYbNGvWjEGDBp3UtiMjI4mLizvFFla8fZ1uqnT4yq18pavyJSIifjJixAjq1avHm2++WWB5ZmYmU6ZM4YYbbiAoyL0933vvvbRp04bIyEiaN2/OhAkTSE9PL3bbRXUF/v3vf6devXpUr16dsWPHcuzYsQK3L1q0iMGDB1OnTh1q1KhBnz59WLx4cd7t8fHxAFx66aUYY/K6LIva18svv0zLli0JCwujdevWTJ48Oe+23Ara66+/zuWXX050dDQtW7bk/fffL+MjV7QDBw5w7bXXEhMTQ1RUFEOGDCEpKSnv9oMHDzJ69Gjq1q1LREQELVu25MUXXyzQ5tatWxMeHk7dunUZNmwYOTm+LcpU7csLeSpfaap8iYicPmZNgF2/+Xef9c+E4U+WadWQkBDGjBnDW2+9xSOPPJIXtL744gv27dvH9ddfn7dujRo1eOutt2jYsCErV67klltuITIykkceeaRM+3rvvfd49NFHefHFF+nbty/Tpk3jqaeeKlCxSk5OZsyYMTz//PMAvPDCCwwfPpzff/+dmJgYfv75Zxo2bMibb77JsGHDCAkpOjpMnz6du+66i2effZZBgwYxc+ZMbr75Zho0aMDw4cPz1nvssceYNGkSkyZN4tVXX2Xs2LH06dOHxo0bl+lvKuzaa69l48aNfP7559SsWZOJEycybNgw1qxZQ0REBA888ACrV69m5syZxMXFsWHDBvbv3w+44Dl+/HjeeecdzjnnHA4ePMi33357Uu0oD1W+UOVLRET8a9y4cWzZsoW5c+fmLXvjjTcYMmQITZo0yVv28MMPc8455xAfH8+IESOYMGFCuSpFzz77LDfccAM33XQTbdq04eGHH6ZLly4F1hk0aBDXXHMNCQkJJCQk8NJLLxEUFMQ333wDQN26dQGoVasW9evXp06dOkXu66mnnmLs2LHcfvvttGnThrvuuourr76aSZMmFVhv7NixjBo1ilatWvHEE08AMH/+/DL/TfklJSUxc+ZMXn/9dfr06UPHjh2ZOnUqBw4cYNq0aQBs3ryZLl260K1bN5o1a0b//v0ZOXJk3m3Vq1fnwgsvpFmzZnTq1Il77rknLxD7SpWufGnMl4jIaaiMFahAat26NX379mXy5MkMGTKEHTt28M033+QFhlwffPABzz//POvXryclJYWsrKxyBYOkpCTuuOOOAst69erF9OnT837fvXs3Dz30EImJiezevZvs7GyOHTvGli1byvU3JSUlcfvttxdY1rt3bx5++OECyzp27Jj3c1hYGHXq1GHPnj3l2lf+fYaEhNCjR4+8ZTExMXTo0IFVq1YBcPvtt3PllVfy888/M3jwYC688ELOO+88AIYNG0bDhg1p3rw5Q4cOZciQIVx22WVUq1btpNpTVlW68hUWHIRBlS8REfG/cePGMWPGDA4cOMBbb71FbGwsF198cd7t8+fPZ/To0Zx//vl88cUX/Prrrzz++ONkZGR4tR3XXHMNv/76K88++yw//vgjy5Yto2HDhl7bjzGmwO+hoaEn3O6LMVa5+73gggvYvHkz99xzD7t372b48OHcdNNNgOvWXbZsGdOmTaNx48Y88cQTJCQksGvXLq+3J78qHb6MMYQGacyXiIj438iRI4mIiGDq1KlMnjyZ6667rkAwWbBgAc2aNeMvf/kL3bp1o3Xr1kWeIVmShIQEFi5cWGBZ4d/nz5/PnXfeyfnnn0+HDh2Ijo4uED6Cg4MJDg4mO7vkQkVCQgILFiw4Ydvt27cvV5vLIyEhgaysLBYtWpS37NChQ6xcubLAfuvWrct1113HO++8w2uvvcbkyZPJzMwE3Bi8gQMH8uSTT7J8+XIOHTrEzJkzfdZmqOLdjgChwap8iYiI/0VGRjJq1CgeffRRDh48yLhx4wrc3qZNG7Zs2cL7779P9+7dmTVrFh9++GG59jF+/HjGjRvH2WefTZ8+ffjwww9ZunRpgQH3bdq0YcqUKXTt2pXk5GTuu+8+wsPD8243xtC0aVPmzZvHueeeS3h4ODExMSfs67777mPUqFF07tyZQYMG8dVXXzFt2jS++OKLcj4yJ0pNTWXZsmUFlkVHR5OQkMCIESO46aabePXVV6lRowYTJ04kNjaWq666CnDzhHXr1o327duTmZnJp59+SuvWrQkNDeWzzz5j8+bNnHfeecTExDBv3jyOHTtGQkLCKbe5JFW68gUQGmQ05ktERALixhtv5ODBg5xzzjknvOFfeuml3H333dx555106tSJxMREHnvssXJtf/To0Tz44INMnDiRLl26sGbNGsaPH19gnbfeeotDhw7RuXNnRo0axS233FJg0D/A008/zZw5c2jSpAndunUrcl8jR47kmWee4amnnqJDhw689NJLvPrqqwXOdDxZa9eupXPnzgW+rr32WgDeeecdunTpwgUXXEDPnj3JyMjg66+/JiIiAnDjyiZOnMhZZ51F7969SUtL47PPPgPc+LBPPvmEgQMH0q5dO5555hnefPNNevXqdcptLomx1vp0B6eia9eudsmSJT7dR7fHZnJu2wY8e3Vnn+5HyicxMZF+/foFuhlSBB2biqmqHpekpCSfVylOVXJyMtWrVw90M6SQkz0uJf3PGWOWWmu7lrYNVb6CdbajiIiI+E+VD19hQUbXdhQRERG/qfLhKzRIlS8RERHxnyofvsLU7SgiIiJ+VOXDV4i6HUVEKjVfXwRZJJe3/teqfPhSt6OISOUVHR3N9u3bycjIoCKfvS+Vm7WWjIwMtm/fTnR09Clvr8pPshoWbEhLVeVLRKQyaty4Mfv27WPz5s1kZWUFujlFSktLy5tzSiqO8h6XkJAQatasWeyFxcujyocvVb5ERCqvoKAg4uLiCszYXtEkJibSubPmkqxoAnlcqny3Y1gwGvMlIiIiflPlw5cuLyQiIiL+pPAVBBlZORqoKSIiIn6h8BXsvqv6JSIiIv6g8BVkAEjPVPgSERER36vy4SvM8wikZWnQvYiIiPhelQ9fed2OqnyJiIiIH1T58BXm6XZU5UtERET8ocqHL1W+RERExJ8UvjyPQLoqXyIiIuIHCl+53Y6qfImIiIgfKHzlzfOlypeIiIj4XpUPX2GqfImIiIgfVfnwpTFfIiIi4k9+DV/GmLuNMSuNMSuMMe8bYyL8uf+ihOnyQiIiIuJHfgtfxphGwJ1AV2vtGUAwcLW/9l+c4wPuVfkSERER3/N3t2MIEGmMCQGigB1+3v8Jjnc7qvIlIiIivmestf7bmTHjgSeAVGC2tXZ0EevcDNwMUK9evbOnTZvm0zYdSU7hzgWGS1qFckmrMJ/uS8ouJSWFatWqBboZUgQdm4pJx6Xi0rGpmHxxXPr377/UWtu1tPVCvLrXEhhjYoCLgebAIWC6MeYaa+3U/OtZa18DXgPo2rWr7devn0/blZiYSGjwMRo0bkq/fu18ui8pu8TERHx97OXk6NhUTDouFZeOTcUUyOPiz27HQcBGa+1ea20m8Alwjh/3X6yIkGCN+RIRERG/8Gf42gL0NMZEGWMMMBBI8uP+ixUeGqQxXyIiIuIXfgtf1tpFwEfAL8Bvnn2/5q/9lyQ8JFgX1hYRERG/8NuYLwBr7SPAI/7cZ1mEhwaRpklWRURExA+q/Az3oMqXiIiI+I/CFxARGqTLC4mIiIhfKHwB4SFBqnyJiIiIXyh8ARGhwap8iYiIiF8ofOEqX2mqfImIiIgfKHzhGXCvypeIiIj4gcIXbsC9Kl8iIiLiDwpfqPIlIiIi/qPwhSpfIiIi4j8KXxyvfFlrA90UEREROc0pfOHOdsyxkJWj8CUiIiK+pfCFm+cLIC1T475ERETEtxS+cBfWBkjP0rgvERER8S2FLyAiRJUvERER8Q+FL1T5EhEREf9R+MKd7QiqfImIiIjvKXyhypeIiIj4j8IXbqoJgHRNtCoiIiI+pvBFvqkmdIkhERER8TGFL1T5EhEREf9R+OJ45UsX1xYRERFfU/hClS8RERHxH4Uvjk81ocqXiIiI+JrCFxDhmWoiTZUvERER8TGFL1T5EhEREf9R+AJCgw1BRpUvERER8T2FL8AYQ3hIsCpfIiIi4nMKXx4RoUGqfImIiIjPKXx5qPIlIiIi/qDw5REeGqQLa4uIiIjPKXx5RIQEk5apypeIiIj4lsKXhypfIiIi4g8KXx6qfImIiIg/KHx5qPIlIiIi/qDw5REeEqypJkRERMTnFL48XOVL3Y4iIiLiWwpfHuEhQaSr8iUiIiI+pvDlERGqSVZFRETE9xS+PFT5EhEREX9Q+PKICA0mTZUvERER8TGFL4/wkCAysy3ZOTbQTREREZHTmMKXR3hIMAAZmutLREREfEjhyyMi1D0UmuVeREREfEnhyyO38qVZ7kVERMSXFL48VPkSERERf1D48lDlS0RERPxB4ctDlS8RERHxB4UvD1W+RERExB8UvjzCPZUvXWJIREREfEnhyyPCU/lK0yWGRERExIcUvjxU+RIRERF/UPjyUOVLRERE/EHhy0OVLxEREfEHhS+P8JDcqSZU+RIRERHfUfjyiAjNnWpClS8RERHxHYUvj7BgT7ejKl8iIiLiQwpfHkFBhrCQINJU+RIREREfUvjKJzwkSJUvERER8SmFr3wiQoM15ktERER8SuErH1W+RERExNcUvvIJDwnShbVFRETEpxS+8okIDSYtU92OIiIi4jsKX/mo8iUiIiK+pvCVjypfIiIi4msKX/mo8iUiIiK+5tfwZYypZYz5yBiz2hiTZIzp5c/9lyY8RJUvERER8a0QP+/vOeBra+1IY0wYEOXn/ZcoIlSVLxEREfEtv4UvY0xN4DxgLIC1NgPI8Nf+yyI8RJOsioiIiG8Za61/dmRMJ+A1YBVwFrAUGG+tPVpovZuBmwHq1at39rRp03zarpSUFKpVqwbAlFXpLNyZxUsDo326Tyld/uMiFYuOTcWk41Jx6dhUTL44Lv37919qre1a2nr+7HYMAboAf7LWLjLGPAdMAB7Kv5K19jVcSKNr1662X79+Pm1UYmIiufv48VgSC3Zuwtf7lNLlPy5SsejYVEw6LhWXjk3FFMjj4s8B99uAbdbaRZ7fP8KFsQojIiSItMwc/FUNFBERkarHb+HLWrsL2GqMaetZNBDXBVlhhIcGA5CRrUH3IiIi4hv+PtvxT8C7njMdNwDX+3n/JQoPcVk0PSuH8JDgALdGRERETkd+DV/W2mVAqQPRAiW38pWWmU2NiNAAt0ZEREROR5rhPp+8ylemuh1FRETENxS+8onwVL4015eIiIj4isJXPrmVrzRVvkRERMRHFL7yUeVLREREfE3hKx+N+RIRERFfU/jKJ/9UEyIiIiK+oPCVT0S+qSZEREREfEHhKx9VvkRERMTXFL7yUeVLREREfE3hKx9VvkRERMTXFL7yCVflS0RERHxM4SufCFW+RERExMeqdvjKySE462jeryHBQQQHGU2yKiIiIj5TtcPXa+fRZu0rBRZFhATp8kIiIiLiM1U7fNXvSOyBpZCdmbcoPDRYlS8RERHxmaodvtoMIzTrKGxZmLdIlS8RERHxpaodvloOIMeEwJpZeYtc5UvhS0RERHyjaofwa1KLAAAgAElEQVSv8GocjOkIa2eBtW5RSBDpmmpCREREfKRqhy9gf+1ucGAD7FsHuMpXmipfIiIi4iMKX7W7uR/WzARU+RIRERHfqvLhKz2iLtQ/E9Z+DbjrO6ryJSIiIr5S5cMXAG2Gw9ZFcHQ/0WHBpKRlln4fERERkZOg8AXQdhjYHFg3m4a1Itl+KBXrGYAvIiIi4k0KXwANOkO1+rB2Fk1iIknLzGFfSkagWyUiIiKnIYUvgKAgaDMUfp9H05ohAGw9eCzAjRIREZHTkcJXrrbDISOFNmnLAdh6QOFLREREvE/hK1fzvhASSb2d3wGw7WBqgBskIiIip6NTCl/GmEhjzCBjTDNvNShgwqKgRT9Cf/+GOtGhpVe+cnLg8Da/NE1EREROH+UKX8aYt4wxt3t+DgMWA7OBNcaY4T5on3+1HQaHt3JujT2lj/la+BI8cwas/cY/bRMREZHTQnkrX0OBhZ6fLwKqA/WBRz1flVubYQAMDl7K1gOldDuunAFY+PhG2LvG920TERGR00J5w1cMsMfz8zDgY2vtHmAa0N6bDQuI6vWhYRfOTv2JHYdSyc4pZq6v5F2wfQmcfT2EhMP7f4DUQ/5tq4iIiFRK5Q1fu4AzjDHBuCrYXM/yasDpMS18h0tocDSJRnYnu46kFb2O51JEdL8JrpwCh7bAx+MgR9eEFBERkZKVN3xNBj4AVgDZwDzP8h7Aai+2K3DOuByAi4J+LH7Q/ZpZUKsZxLWHZr1gxFPw+1yY+6j/2ikiIiKVUrnCl7X2ceAG4DWgt7U2dxr4LGCSl9sWGDUbk9awJ5cEL2Dr/qMn3p5xFDYkQtvzwRi37Oyx0O0m+PF5WP6BP1srIiIilUy5p5qw1n5srX3GWrst37K3rbWfebdpgRNy1pW0DNpJ2rblJ964IRGy0tykrPkN+zvE94HP74Bv/gJHdvilrSIiIlK5lHeqiSuNMUPy/f6wMWabMeYbY0wD7zcvMELOvJQsgmm89csTb1wzE8JrQrNzCi4PDoUr34H2l8DCf8OzHeGzO2DfOv80WkRERCqF8la+Hs39wRjTBXgAeB4IBf7lvWYFWFQsy8O70vHQHDeZaq6cbFjzNbQe7MJWEffj8v/Anb+4rsjfpsOL3eDD6yA92W/NFxERkYqrvOGrGZA7qdWlwAxr7T+Ae4CB3mxYoCXVGUrtnP2w5cfjC7ctgWP7TuxyLCwm3g3Cv2sF9L4LVn0Gy6eVfB9r4fXBsOi1U267iIiIVFzlDV9puIlVwYWt3KkmDudbflo41HQQx2w42cs/PL5wzUwICoFWg8q2kWp1YdCjLoyt/7bkdXevhG2L4Ze3T7LFIiIiUhmUN3z9APzLGPMQ0BWY6VneBtjqzYYFWoM6tZmdc7arWmV5TupcMwvie0NkrfJtrEV/2PgDZJcwFdrvnhy7ewUc3HxyjRYREZEKr7zh6w4gAxgJ3GqtzT2lbzhwWl3ksHFMJJ9ln0tw+iFYPw/2r4d9a9wUE+XVcgBkJLtuy+L8Phei49zPa2adXKNFRESkwivvPF/brLUXWmvPstZOzrf8Lmvtnd5vXuA0iY3ih5wzSQut5QbO5wYiz/Ufy6X5eWCCiu96TE+BLQuh0x+gbjvXvSkiIiKnpXLP8wVgjBlgjLnDGPNHY0x/bzeqIqhXIwITHMqqmAGweias+AjqnQExzcq/scha0Ohs2PBd0bdv+gFyMqHlQDeYf/MCXStSRETkNFXeeb4aGWMWA3OA+4EJwFxjzCJjTENfNDBQgoMMjWpFkhjeD7JSYcevpZ/lWJKWA2D7Ukg9eOJtv8+F0Gho2tN1a+ZkHR8DJiIiIqeV8la+nsdd07GVtbaJtbYJ0Nqz7HlvNy7QmsRG8f2x5lCziVtwKuGrRX+wObDx+4LLrYV1c1zXZEg4NOoK0XXV9SgiInKaKm/4Ggz80Vq7MXeBtXYDcKfnttNK45gothxKh27joN6Z0KDzKWysK4RVh/WFuh4PbIBDm6GVZ5q0oCA3rmzdnONnWYqIiMhp42TGfNkyLqv0msRGcuBoBke7/Qlum++C0ckKDoXmfdyZkzbfw5XbvZh/7rB2IyD9iBv7JSIiIqeV8qaJecALxpgmuQuMMU2BZ4FSZhGtfJrERAGw9eAx72yw5QA4tMVVu3L9Pg9iW0Bs8+PLmveFkEh1PYqIiJyGyhu+7gSigQ3GmM3GmM3AeiAK+JO3GxdoTWI94etAqnc22MJzYmjuWY+Zae5Mx8Iz5odFuaC2ZlbBKpmIrxzdV/A6piIi4jPlnedrK9AFOB94yvM1HLgceNrrrQuwxjGRAGzzVuWrdkuo2fT4uK8tP0HmsaIvV9R2OBzeCrt+K/9+crLh0FY3q/7mH0tfX3wnJ8dd1/PYgUC3pHhrZsFTreG9Kyp2O0VEThMh5b2DtdbippqYk7vMGHMWLoCdVmpHhxEZGuy9ypcx0LI/rPwUsj3TSQSHuUsWFdZmGGDcG2ODjqVve8si+O8kOLjJdW3m5LuU0Y3fQuOzvfM3SPn89CLMeQi6XAcXvRDo1pxo53L4aJy7/ujG7+G1vnDVVGhwVqBbJiJy2jqFEeSnP2MMTWIjvTfmC1z4Sj/i5vxa/y007QVh0SeuV60uNOletnFfOdkw4zbY9T+ofyb0+iNc8CyM/hii6sDcR0ruvrQWti113aDiPTuXw7zHITQKlr0PR3aUfh9/Orwd3rsKImPg+llw/dfuf+mNIbDsvUC3TkTktKXwVYomMVFsPeDF8NW8L2Dg1ymwZ1XRXY652g6Hncvcm2RJVnwCB9bDiH/BlW/D4Meg6/XQehD0/T83ruz3ecXff+lb8PoAeO4s+PFFyDh6Mn+Z5JdxDD6+EaLrwNiv3BxvP70U6FYdl54C71/lvo/6AKrXd9XRW753oX/GbfDl3ZCVHuiWioicdhS+StEkNoptB1Ox3hr4HhULDTvDsnfd7yWGrxHu+9oSLrSdkwM/PAV1E6DdhSfefvb1UKsZzH206AHVBzbCN3+Bxt2hbhuY/Rd45gz47z91iaNcqz6Dd68oX2Vw9l9g31q49BVo1AXOuMyF3IowpionGz4eB7tXwhVvQv0zjt8WXQeu+RTOHQ9LJsMH17oucjm9HdkB6cn+3+/h7fDVvbAhsfjqfE4O/PYRvNoXXusPn9zsXp9WfOLGxOZk+7XJIt5QpvBljPm8pC/gOR+3M2Aax0SSkp7FoWOZpa9cVi0HuEpI9YYQl1D8enVaQ502sOg1V0kpStLnsHc1nHdv0fOQhYTBgIdg92/u+pT55WTDjNshKNi9CY/5AsbNgcbd4Lu/wbNnuhe9quzARvcYrZsN//ugbPdZPdMFl3P+BC36uWXn3gUZKfDzG75qadl98wCs/RrO/ye0LmJu5OAQGPy4q6Su+wa+HO/fs25TD7nHKT3Ff/usiKyFr/7sQsfR/b7ZR3oKzJoAT7eHf7aGD8dA0hf+GYKQsgfeuQh+/g+8czH8ZwAkfXn8Q6K1sPoreKW3+7CQlQ4RNWDTAvf69NH17rbJw3z3+Ij4SFkH3Jf2n70f2FjKOpVS3nQTB48REx3mnY227O+qVa0GukH4xTEGhk+CKZe6SsoFzxS83Vr4/imo3Ro6XFr8ds64HH58Dr79K7S/2F3GCGDhy7DlR7jk31CzsVvWpDuM/tCNV5p1v3vRO7oPet56an9zZZSdBZ/eCibYheCfXoLO15Y82W7yLvj8Dqjf0YXeXPXPgNZDYNG/3Zi8sCjvttVad93QqNiS11v4Cix6BXr+EbrdWPK63W50b5D/nQTV6sPAh0pe3xt2rYAProGDG113+cg3S36OeNPRfbBpPmSmwllXl2+/+9e7wPjbh+7KFKEREBIBoZFuzF+HS6Dn7W6y5bL65R34+XX383tXwHWfQ3i18v1NJVn/LXwx3p2gc/b17kPYyhmwagaE14CEC6HTaGh2jvePQepB97p2eDtc95k7UWj+s/DBaKjbzj3PVn7ixsbGtoTL34AOlx1/7mUcc0Mttix0lfvJQ+Caj92JI4GSnQXbl7j2R9YKXDukUihT+LLWXu/rhlRUeROtHkilY2MvPaGa9ICOV7nLFpWm5QA450748Xn3c0K+rsU1s1xF65JX3AtncYKCYNCjMPVyWPKmC1J7kmDeX13X5ll/OPE+Dc6Ca2e48PX1/XBsH/T/i//eCCuCBc/A1oVw2euuUvnpze4M1TZDil4/J8eNlco45t4sckNurt73wJvD4Nep0ONm77b127/Bgudg5BsuYBcl6Uv4egK0uwCG/LVs2+030QXKH56CavVObHdmqquU7PrNVfbSUzzfkyGiJrS/xI1dLEto+N+H8Pmd7n5dxsAvb7vnSs/bytbW8rDWdbVtX+pC3qb5bgxmrpws6HJtydvIyXaXAfv5P+7/IijEPbbV67vHJSvNfU/eBXMediddjPgXxJ9bevt2/g9m3ucqp91ugg+vc6F01Acn/l+VV+pB+OZBWDYVardyJ1s0O8fdNmwSbPwvrPjYHddl70KDTu4DQ4dLyxcei5OeDFNHum75UR8crw53usYFvx+edh82azR2ZwifNcpVY/MLi3InF9U/E+qdAe9fDa8Pdh8cG57CZeCKsmc1/PgCxLWDNsOhTquCt6fsdf+rS96EI9vcdEJXTYGGnbzbDinoyA5XGa3VFNoMDXRryq3cU01UNY1j3VxfXj3jMTgULnut7OsPeMi9QXz+J2jYBWo28lS9/uE+6Z15RenbaDnQXbz7+39Axyvh01vcG+KFzxUfqEIj4Iq34au74ft/wtG9MOLpkoNeSayF7EzXFVrRbf8FEp90VcOOV7h2z30Ufnqh+PC1dLKrJox42o2fK6xZL2jS072Qd73eO29kAMm7XVXOGJg+Fi5+GToVCtTblroTABp1gcv+U/ZjaIz7e47uhVn/587CJcZVqH5523XFph12U6aEV4ewase/b/8FVn/prtbQdph7LFsNdv9X+WVluDfbxa9B03PgiregWhwc2w+zH3Rvpk17nvzjk5kKWxe5au7eNa6bfu9ayPCMcQqNcts/cyTE93FBdua97rGq16Hobe5c7rroDm50VcF+D8DZY1zwKsqaWTDz/+Ct812YGPy457EsQtphmD7GVTEve92td9EL8Nnt7nl7+Rsn/xzcvhTeH+WOZ+97oO/9BY9HcIiryLcaCOc/5Y7vwpfhk5tgziPQ/SY4e2zpFdbiZKbC+3+AHb/Cle+4D5T5933mSPd/smeVC4ZlCZrNesG42S7QvTnCbbd1CWNpy2PzTy7YZaZCdrr7f4xt6aYCyj0bfeWnkJ3hTqbqcw/88C+YPNSdcV74eSin5shON9Rm5Qw3T2bulQ3PvMINo4iMCWjzykPhqxQ1IkKpGRnqvYlWT0ZImHvBfaWPG2w65nM3UeuOX+HC50/8VFgUY1z16z8D4I3BsP93uHJK8W8AuYJD3D6i67oXlWMH4PLXy//pO/Wge7Pau9p1n8S1K9/9/SnjmHucq9VzlQpwQannra6CsXP5ifNgHdri3pxa9IeuNxS/7d53u7MMV3wCZ13lnfbOf8a9+N+c6ELMjFsh8+jxbsWDm9w+q8XBHz4of5dncAiMnAzvXAKf3MzZkU0gcT0Eh0P7i9wcZvF9TgzxOTmucrjiY/diufJT14VbLc6FlGr1oXo9F+S2L3FdoYMfOx5KL3kZXuvn/m9u/cHdL7/tS93Aa5vtut7rtHJv2LVbue6sjYmw4b+wdbF74wS3z7ptodMoF5DrnenCXf4PBJe/7sYSTR8LN313YtVu53I3Rik02nWLJlxYepBuO9zz4ecpF77XfAXn3eeCTHj14+tZC5/dAQc3u7Nkc5+fnUe7MDrnIYiMdf+X5a1Cr/najZOKrgs3f1f6XG5hUe5DQpcx7pq0P70I8x5zf0OXa11FsjzdfOnJbk65TfPdh8+EC4pez5jiQ29x6raFG+fAuyPhvStdSIyJd8/h6vXd95pNyvfBL+lLV/mv2dh1aWLc2M+1X7tq58KX3IeMs8e651rdtu5+CRe5x3nGre5/dOj/K9/fIgUd2uKORdIXxwNXXHvo/4B77q363A2N2LTAvWa07B/oFpeJ8dpZfD7QtWtXu2TJEp/uIzExkX79+pW4zgUv/EDt6HDevqG7T9tSqmXvuW6tAQ+67o4jO+BPv5TvBeXDMa60f+aVcPl/yrf/n16Gbya6ucmumurOjCuLg5vc2YIHNrouJWNgzJfFB7DUg6z84kU6XHZ/+f627Cw4sh0Ob3MnKxR+sy6rr/7sxtpc9zm06JuvXYfgmQ7Q9vyCj521rkt3y0K4/SeIaVb8tnNy4JVz3X1u+/HULtYOLmQ839l98rvkJTdQevoY9wYx+HE3duaNIa7SceNc97icrGMH4J2LSUk+TLU+t7mu87JWQLKzjnfvJe+ClF3ue/IuwLpPrWcUMU/zrhXw+iBo3NV1gweHuP/7eY/D8vfdPHbV67sxV1lFTIZc70x3DJv3hSbdyv7JeOP3LmCdMdIFhdygkxu8wqq5E1TyX5O1rPaucVXEDYkQUcsFhR63uufTwldcN//gx90Zp4XNedh1L/e6w1Wuomvn3VTia9mSye7/usFZMOrDk39u7FrhQthv011XfPtL4Nw7i+/qS97lqn6rv3LdmdkZriLU1UcjWdKOuHGa62YXnGgaIKo2nPd/bt+lfXj8+Q1X/WzYxT1e+R5nwHWv71zmxnZG1Djx/tlZMO9RF7Sb9OTX2hfTeeBlEB1X8DmfnQX718GOZW57aYehRiMX+Go2cb0cNRsXDOi+lJMDhzZ7qsNr3HWHEy7073CTtMPu+sfr5niGNPzPLY/r4D7stb/kxPeO7Uvhk1vcY9n9FldoKMOHzLK8/5eXMWaptbZrqespfJX+4N82dSlrdifz7Z9LXs/nrHVdRys+Bqz79FvaoOnCDm+Dhf92Z0eeTIl2xcfu7L/cKkq99iWvv/VnV7bPyYKr33WfQN+6wL1wj/nixCfRujmuezV5p3vhGzm5+De4Q1vdG9GeJPfp6Mh2VwUBV2FpOcANnG57ftmqPYe2ujNC5z7q3tyGPnHiOl9PdN1j4//nXhjheCge/s+yjeVa/oEbP9bzdlcxiktw04GcTBD78h43MPtPS4+HvuxM10208lM3buboHhdcyjLWqAx88YJVomXvuypCzz+6gczzn3H/T73+6AJIRA33pnFku3vx3b/evdE2P6/sHxCK8t9/wHdPuMrv2WO8E7zy27bE/S2rv3JhoMNlLtS0HgxXv1f0G5617vnx6xQwQW5MXNvzoe35JK7YduJxsdadaPPDv6D1UHdWc1GTOpfX4e3uxI2lb7lJo2NbuoAQGnn8RIOUPa6iCa4K1e4C98bZpNup77801roPCym7jwf95e+7UF2rmRvKccblJz7nsjPdcf/+H955vFZ87CqZmZ6ek+BwqNXEtSE92Y2VzP3QEBrlXpOTdx1/HcsVXgNqNITqDVw4q9XUjSlrdPbJ/49npcPuFW54wPZfYM9K1x1f+ENM014w7EnvjGHLOOq6DpM9X0d2uO+HtrrX8MNbXPjK1bi7q5C2u8Bdnq/EbR9zr92LXwWMq0YHh7mxmMGh7nk7flmBuyh8FaOihK/n5q7j2XlrmXdPX1rU9eLZRicj7bDrEsnOgjt/PXH8jD/kjhvJSHHdoW2HFb3eyhlujEr1+jD6o+NVl33r4K0R7gVy7JeuXJ92xE2B8OsUqJvAuprn0nrbdPemeuGzbixIroxj7gSE+c8C1g0IjmnmXpBqNXUvUJsXwP+muwGwYdVcV0DTnq5SExl7/PuhLW4etbXfuBcicOOOrv206Mf24GZ4vpMLZ0P+6l4oX+ruyuBjZ5YtQGVnujO9Nv1wfFlolDtLqv3FLpSVpeJ3aAs838V1ARU+EzYn2w1eX/au60bL//idIr+HL4Av7oKlb7qf21/iuid9fWZbTranovmTGxv59QTvBa/89q1zHyKWT3NvsLf8t+QPRta6KsmaWW5ak93u+q+pEfWIrNfadVVGx7nvu1a4SneXMW7sXlmGKJRH2hEX/rcudFXX3JMMstJcAGs9FNqd754fgT5Zx1o3JnPuIy701O/oQvXh7W7w/761ruKSkwWdr4ELnvPO45W8i//NnkrHJjXdc/bQZvc6EhrpXrsadnLf67R2Y/mys1xgPLzdXd/3yHYXWI5sd2ElN7Dkjneq1cxVhut3dNss/Ddnp7tjk3nMHZeMFPeBdddvrhIJ7v+l/pnuNSiunfteu5WrPM173HV5d7kWBjxc+lCV/DJT3fNnQ6IbKpNbxcovrLqr7tVq6gmmTV3Vr2kvqNGg/I/3pvluf9mZ7ivH890EwQUFL0Gt8FWMihK+9ian03vSt1zSqRGTRpbhOou+dnS/exLlVl4C4cgON3B253L3RtjrDveism+dK1fvXuEG6zbp4T7FF/50tnctvH2Be3EY9Igb3H5ku+tq6TeRxPk/0a9TCzdGZNti1302fJLrTpv9sAtVHS5z3TO1mhTdxpwcTwib5sYFpB8pej0T7J7obYa6sTm1W5X8RjF9LPz+Ldyz0tPFMQduW1D+Lr20I+6x2rPKvRju+NW9idVt58JU7hloxfnsDneG4J2/Fv2/YK2bPqE8L5ZlEJDwlZXuqjfN+3qtglcmKXvdh52UXe4NwdvBq/C+goLLP5j90BZY8zV7lnxKXKR1Faejez3/78adpXzevYEPPxVFTo6rcH/7V/fYBYW47rU6bdwHwYadXaXFi4+X158zGUddV+X2pa66uP0XF9RKEhTqPuSFRri/tVEXVzlrdLarphX396YecidcLXrF3b/zte71JjrOva5Xi3MVvdxKVm5I3LfGXXM4O93tu0kPdx3j2ObuA3L1Bi5c+atLtQgKX8WoKOEL4OHPVvD+4i38977+NKwVWer6VULGMdfdtmqGewHLyTcTenRd1x0y/B/FV+dyA1jKbhd4Lnklr0si77hkZ0Li393p52HV3Blq9c902y0tnOSXleH2k3rAdUcc2+9OAoiMcd2T5XnD27bUXY6p5QD3SXrQo24gvTes+dqNNTm81X36HvzXotu2fz282M2NFxo+yTv7LqOAhK9A2rLIBb/hk3wXvLzghOOSW4nSnFNFy8rwTA3RxHtnHhfDL8+ZtMNFz/YfHOYqYid7hmyufevc2Z6/zztxPF1hEbVcBSu+jxsA37SXd+eo85JAhi+d7VhGN5/XgvcWbeE/P2zgkQvLeSbO6Sosyp3ttaS3q3rVaes+OdZuVbYwU7eNm2No3WzXLVLUuKzgUBj4sBu/kzjJnSHY+dryv5CEhHlK2sVUycqj8dnuxWT9t545kP506tvM1XYYNO/jzt758UXXtXTuXe4yVHEJxz+d/neSe1HtfY/39i1Fa9rDzR9V2YRGBGZYQmUREuYqXqeLiJq+3X6d1m5eNmsh7ZCr1B7d68aUZqUfH49Wo4F3xhWe5hS+yqhxTBQXd2rE+4u3cEf/VtSudooTHZ4ugoJc9eVk1W4JtcswiWaLfscnY6wI+k1wA58vfsn742jCol13aser3Blqcx5yX9FxLoQ2OMsNzO51h5uqQUTEX4xxPQaRMUXPZyhl4vcLaxtjgo0xvxpjvvT3vk/Vbf1akJ6Vw5sLNgW6KRJoLfq5Mx7zX5Ta2+p1gBu+hrt+cyGvRT83SH/OQ27sxbl3+W7fIiLiM4GofI0HkoAiJkep2FrFVWdYh/q8/dMmbu7bghoRvh0nIBWcvwYw12rqxn91vsaV/PeucWfuFJ57SEREKgW/Vr6MMY2BEcDr/tyvN93erxXJaVlMXbg50E2RqsgYz6ngKveLiFRWfj3b0RjzEfB3oDpwr7X2hOtLGGNuBm4GqFev3tnTpk3zaZtSUlKoVq18Z2E8tSSNzUeyeapvFOHBOn3bF07muIh/6NhUTDouFZeOTcXki+PSv3//inW2ozHmAmCPtXapMaZfcetZa18DXgM31YSvT889mVNNo5od4MpXf2JnRDxjz624p55XZlVuOoNKRMemYtJxqbh0bCqmQB4Xf3Y7ngtcZIzZBEwDBhhjpvpx/17TvXks3eJjeO37DWRl5wS6OSIiIlKJ+C18WWsnWmsbW2vjgauBb6211/hr/942ukczdhxOI2lncqCbIiIiIpWI36eaOF30aOEmEV286UCAWyIiIiKVSUDCl7U2sajB9pVJg5qRNI6J5OeNCl8iIiJSdqp8nYLu8bEs2XyAinx9TBEREalYFL5OQbfmsexLyWDjvqOBboqIiIhUEgpfp6BbfAwAP2vcl4iIiJSRwtcpaFm3GrHRYSzeeDDQTREREZFKQuHrFBhj6NosRpUvERERKTOFr1PUvXksWw4cY/eRtEA3RURERCoBha9T1C3eM9+XppwQERGRMlD4OkUdGtYgKiyYJep6FBERkTJQ+DpFIcFBdGkaw+JNGnQvIiIipVP48oJu8bGs3nWEw6mZgW6KiIiIVHAKX17QrXkM1sIvm1X9EhERkZIpfHlB5yYxhAQZXWRbRERESqXw5QWRYcGc0aimLrItIiIipVL48pLuzWP537bDpGVmB7opIiIiUoEpfHlJt/hYMrJz+N+2w4FuioiIiFRgCl9e0rWZLrItIiIipVP48pKY6DDa1Kumme5FRESkRApfXtQtPpZfNh8kO8cGuikiIiJSQSl8eVH35rEkp2eRtPNIoJsiIiIiFZTClxd1b+4usr1IXY8iIiJSDIUvL2pQM5KmsVEs2rA/0E0RERGRCkrhy8t6NI9l8aYD5Gjcl4iIiBRB4cvLerSozaFjmazbkxLopoiIiEgFpPDlZT3yxn2p61FEREROpPDlZY1jImlYM4JFGzToXkRERE6k8OVlxhh6tKjNoo37sVbjvkRERKQghS8f6NE8ln0pGazfezTQTREREZEKRkZeUT4AACAASURBVOHLB3q0qA1o3JeIiIicSOHLB+JrRxFXPVzjvkREROQECl8+YIyhe/NYFm88oHFfIiIiUoDCl4/0aFGbXUfS2HLgWKCbIiIiIhWIwpeP9Myd70tdjyIiIpKPwpePtIqrRu3oMBZq0L2IiIjko/DlI7njvlT5EhERkfwUvnyoR/NYth9KZdtBjfsSERERR+HLh3Ln+1q8UdUvERERcRS+fKhtverUjAxV16OIiIjkUfjyoaAgQ7f4WM10LyIiInkUvnysZ4tYNu0/xu4jaYFuioiIiFQACl8+1rt1HQD+/OFyDqdmBrg1IiIiEmgKXz7Wrn4N/jmyI4s27ufyf//Ilv0681FERKQqU/jygyu6NmHKuB7sTU7nkpcXsHSzBuCLiIhUVQpfftKzRW0+vf0cakaG8of/LOKzZdsD3SQREREJAIUvP2pRtxqf3HYOnZrUYvy0ZXy+fEegmyQiIiJ+pvDlZzHRYUwZ150ODWvwz29Wk5mdE+gmiYiIiB8pfAVAeEgwfx7Shq0HUvlo6bZAN0dERET8SOErQPq3jaNTk1q8MG8d6VnZgW6OiIiI+InCV4AYY7hncBt2HE7jw5+3Bro5IiIi4icKXwHUp3UdusXH8OJ3v5OWqeqXiIhIVaDwFUDGGO4e3IbdR9J5b9GWQDdHRERE/EDhK8DOaVmHXi1q83LielIzVP0SERE53Sl8VQD3DGnDvpR0pizcFOimiIiIiI8pfFUA3eJj6dO6Dq/8dwMp6VmBbo6IiIj4kMJXBXHP4DYcOJrBn977hdW7jgS6OSIiIuIjCl8VROemMUwY3o6fNx1k2LM/cPM7S1ix/XCgmyUiIiJeFhLoBshxt/ZtydXdmvDmgk1MXrCR2at2M6BdHA+OSKBF3WqBbp6IiIh4gSpfFUytqDDuHtyGBRMGcO+QNvy86QATPv4t0M0SERERL1Hlq4KqERHKHQNak5FtefHbdexPSad2tfBAN0tEREROkSpfFdzghHrkWPhuzd5AN0VERES8QOGrgjujUQ3q14hgzqpdgW6KiIiIeIHCVwVnjGFQ+zi+X7tP138UERE5DSh8VQKDEuqRmpnNT+v3B7opIiIicooUviqBXi1rEx0WzJyk3YFuioiIiJwiha9KIDwkmPPa1GXuqt3k5NhAN0dEREROgcJXJTG4fT32JKfzm2a9FxERqdT8Fr6MMU2MMd8ZY1YZY1YaY8b7a9+ng/5t4wgyMLeYrses7BwOH8v0c6tERESkvPxZ+coC/mytbQ/0BP5ojGnvx/1XajHRYXSNj2XOqhPDV3aO5Ya3l3DupG/537ZDAWidiIiIlJXfwpe1dqe19hfPz8lAEtDIX/s/HQxOqMfqXclsPXCswPJn567l+7V7CTIwZvJi1u1ODlALRUREpDTGWv8P4DbGxAPfA2dYa48Uuu1m4GaAevXqnT1t2jSftiUlJYVq1SrHRat3Hc1hwg+pjG4XxuD4UAB+3ZPFc7+kc17jEEY0D+WJ/9/efce3Vd/7H399JdmS94rtOPFKHGeYTMdZBEKgAcIqs2UGyubSPSn0Xjr5ld7ettwWyiVl75GyymwpI0D23iSO4zi2k9iO4z1lfX9/SAlJcBJCYklE7+fjoYekoyOdj/TNMW/O93u+Z2EHTgN3TPKQHvvlHdL3ZWqXSKO2CU9ql/CltglPfdEup5566lJrbcnh1gt6+DLGxAMfAHdZa1881LolJSV2yZIlfVrP+++/z/Tp0/t0G8fSjD9+QGaim6dumEx5XSvn3fsR+WlxvHDLFDxRTtZvb+LSB+aTGhfN87dMISPBE+qSv5AvW7tEErVNeFK7hC+1TXjqi3Yxxnyu8BXUQyPGmCjg78BThwte0rsZIzJZWFbPzqYObnlyKU6H4a9XFuOJcgIwIiuRR66dyM6mTq5+aJEG4YuIiISZYJ7taICHgPXW2j8Ga7vHm9OLMvD6LJc+MJ9Pdjbzv5eNIyc1dr91xuelMPvq8WyubeH6xxbj7fGFqFoRERE5UDCPfE0FZgGnGWNWBG5nB3H7x4WxOSmkxUVTvquNH8wYyilD03td7+TCdH538WiWbN3Ni8uqglyliIiIHIwrWBuy1n4EmGBt73jldBhunDaY8rpWvnnqkEOue+G4gTw2fyv3vLORr44dsLdrUkRERELny3s6XAS75ZQC7r54NA7HobOsMYafnDmM6sYOnl5YEaTqRERE5FAUvo5zU4f048SCNO57r5SWTm+oyxEREYl4Cl8R4MdnDmNXaxcPf7Ql1KWIiIhEPIWvCDAuN4UzijL529wydrd2hbocERGRiKbwFSF+dOYwWrq83P/B5lCXIiIiEtEUviLE0MwELhw7kMfmlbOjsSPU5YiIiEQsha8I8v3Th+Kzlj+/uynUpYiIiEQsha8IkpMay+UTc3lu8TaWlNeHuhwREZGIpPAVYb43Yyi5qbFc++hi1lQ1hrocERGRiKPwFWFS46J58oZJJLhdXPPwIkprWkJdkoiISERR+IpAA5NjePKGSRgDsx5ayLb6tlCXJCIiEjEUviLU4PR4nrh+Eq2dXq56aCE1TZ/vDMgtda2aKV9EROQoKHxFsBFZiTx63URqmzuZ9dAiGtu7D7l+eV0rZ94zl4v/Oo9dLZ1BqlJEROT4ovAV4YpzU/jb1SWU1rbw69fWHXLdX722DpfDUL6rlSsfXKjZ8kVERL4AhS9h6pB+3DxtMHOWVvLBxtpe1/n3+p28u6GG788YyoPXlFBW5w9gDW0KYCIiIkdC4UsA+M5XCilIj+OOF1d/ZkxXR3cPv3ptHUMy4vnG1HxOLkxn9qzxlNa0fK7uShEREfmUwpcA4Ily8t+XjKa6sZ3fv7Vhv9ce+mgLW3e18YvzTiDK6f8nM31YBv83q5gNO5q4+uFFNHUogImIiHweCl+y1/i8VK6Zks9j87eyODADfnVDO/e+W8pZI/tzUmG//dY/bXgm911RzNqqRmb84QPufXeTBuKLiIgchsKX7OfHZw4jOyWG2+asoqO7h7veWI/F8rNzRvS6/hkn9OfpGyczrH8C//PPjUy5+11+Mmcl67c3BblyERGRLwdXqAuQ8BLndnH3RaO56qGF3PTEUuZurOUHpw8lOyX2oO+ZOCiVJ66fxKadzTw6r5wXl1Xx/JJKJuancklJNmePyiLerX9qIiIioCNf0ouTCvtxaUkOczfWkpMaw03TBn+u9xVmJnDXhaOYf/tp/PSs4dS1dPKTOauY8Jt3+MFzK/i4tA6fz/Zx9SIiIuFNhyOkV3ecM4JdrV1cf9IgPFHOI3pvcmw0t5xSwM3TBrN8WwNzllbyj5XVvLi8iqwkD+eOzuLc0QMYnZ2EMaaPvoGIiEh4UviSXiXFRPHgNSVH9RnGGIpzUyjOTeHOc4v417qdvLKiikfnlfO3D7eQmxrLuaOzuKh4IEMyEo5R5SIiIuFN4UuCwhPl5LwxAzhvzAAa27p5e+0O/rGqmgfmljF7bhlP3ziZiYNSQ12miIhIn9OYLwm6pNgovj4hhyeun8T8n55GTmostz61lO2N7aEuTUREpM8pfElIZSR6mD1rPO1dPdzy5DI6vT2hLklERKRPKXxJyBVmJvCHr49l5bYG7nx5LdbqjEgRETl+acyXhIWZI/vz7dOG8Jd3SxmVnUR2qAsSERHpIzryJWHjezOGcuqwdH75j7Vs2v3Fux+rG9rp7vEdw8pERESOHR35krDhdBjuuWwc59/7EX9e3kZj7CdMG5rOuNzkvRf07k2X18eS8nre3VDDuxtqKKtr5fSiTB64ajwOh+YRExGR8KLwJWFlz/xitz7yEfd/sJl73yslwe1iSkEaUwrSMEBTh5em9m6aOrqpa+li0ZZ6Wjq9RDsdTBqcSnFeCnOWVvLA3DL+Y3pBqL+SiIjIfhS+JOwMyUjgjkkxjJs0lXmldczdVMvcjXX8c93OvevERjtJ9ESRFBPFuaOzOHV4BicN6Uec24W1lo7uHn7/9gbG5CRxYkG/EH4bERGR/Sl8SdhKionirFFZnDUqC2stO5o6iHY6SPBEEe06eDekMYbfXTyaDTua+c4zy3nt2yfTP8kTxMpFREQOTgPu5UvBGENWUgxp8e5DBq894twu/u+qYtq6evjm08vo8moAvoiIhAeFLzluDclI4L8vGc3Srbv57ZvrQ12OiIgIoPAlx7lzRw/guqmDeOTjch6bV47Pd/AJXOtaOvnh8yu5bPZ8djR2BLFKERGJJBrzJce9288ezsadzfz81bW8sHQbd5w1ghOHfDoI3+ezPLO4gt+9uYH27h6inA4uuO9jHv7GBIoGJB6TGqy11LV0UVHfSnldGxX1bfisxe1y4HY5cUc5cLscnFjQj5zU2GOyTRERCU8KX3Lci3I6ePy6iby6sprfv/0JVzy4kFOHpXP72SPo7vHxs5fWsGJbA5MHp/KbC0bR5fVx3aOL+dr/zeO+K4uZPixjv8/r6O7hpeVVzN+8i1OGpnPWqP7ERn92V2ps6+al5ZW8vKKaTTubae36dOJYE5h+7MArKSV4XDxw1fj9wqGIiBxfFL4kIjgchgvGDWTmyP48Nq+ce98rZeY9cwFIiY3mT5eO4YKxAzGBVPTyN6dy3aOLuf6xJfzq/BO4clIeOxo7eGJBOU8vrGB3WzcJHhevrqzmzlfWcPaoLC4Zn82E/FQWl9fz7OJtvLF6O51eHyMHJvK1khzy0mLJT4sjNy2W7JQYop0OvD5Lp9dHZ3cPNc2dfPfZ5Vz98CLuvng0l4zv/SJLqysbaWjv4uTC9GPy21href+TWsbnp5DoiTomnykiIgen8CURxRPl5OZTCvh6SQ4PzC3DWsut04eQFLt/6Oif5OH5W6bw7aeX8bOX1vDK8mqWVeymx1pOH5HJ9ScNYuKgVJZs3c2cJZW8tqqaF5ZWEhftpLWrhwS3i6+X5HDphBxGDkw6aD1RTkOU00G820VavJs5/3Eitz65jB+9sJKKXa18//ShewPh4vJ6/vJuKXM31gLw6/NPYNaU/KP+TR78cAt3vbGeCfkpPHH9JDxRzqP+TBEROTiFL4lIKXHR/PSs4YdcJ97t4m9Xl/Dr19bx4vIqrp6SzzdOzCc37dMxWRPyU5mQn8rPv1rEW2t28FFpHScW9OOcUVnERB95iEn0RPHItRP4z5fW8Od3S6mob+PC4mz++l4pC7fUkxYXzW0zh7N0az3/9cpaLHB1LwHM57M8Nr+cl5ZX8f8uHHXQAPjRpjp+++Z6Rg1MYnH5bn74wkr+ctk4XZZJRKQPKXyJHILL6eCX54/kF189Ye8RqN7ERru4qDibi4p77yo8ElFOB3dfPIrctFh+//YnvLyimowEN/91bhGXT8whNtpFl3cQ33x6GXe+shafz/KNqYP2vr+6oZ0fz1nJx6W7cLscXD57AY9cO4GS/NT9tlOxq41vPbOMwowEnr1pMk8u2Mpv39xAdnIMt5894qi/h4iI9E7hS+RzOFTw6qvtffPUIQzLTKCupZMLxg3crzsw2uXgviuK+dbTy/jFP9bhs3Dt1HxeXlHFna+spcdn+e1Fo5g2NJ1ZDy5k1kOLmH31+L3jxNq6vNz0xBKshdlXjyfO7eKmaYOp3N3OA3PLGJgS0+sRNREROXoKXyJhbEZR5kFfi3Y5uO/KYr799HJ+9do6/rGqmuUVDZTkpfDHr4/d2z363M1TmPXQQq5/dAl/uWIcZxRl8uMXVrFxZzOPXjuRvLQ4wB/4fn5eEdsb2/nFq2sZkBRzyO2LiMgXo0lWRb7EopwO/nLFOM4e1Z81VY3cNnM4z908Zb9xaekJbp67aQpFAxK59all3PLkUl5fvZ3bZg5n2tD9z5h0OR38+fJxjByYxLefWc4T88spq23BHjgnBv4pNz7cVMvdb27gifmHnsD2cHw+S2lNM88v3sZtc1Zx6QPzeXFZ5VF9pohIuNKRL5EvuSinvwuyqcNLUkzvU0UkxUbx5A2TuPGxJby9difnjRnATdMG97pubLSLh66ZwFUPLuS/XlkLQGaim8mD05g0KI0VZV08WLqQxeX1dHp9OAz4LPx7Qw1/+vpYUuKiD1tzc0c3K7c1sqxiN8srdrOsooHG9m5/rTFRpMZF84PnV/L4/K38/LwixuWmfMFfR0Qk/Ch8iRwHjDEHDV57xLtdPHLtBN5eu4MzivofchxbeoKbt753MlvqWplftosFZfV8XLqLV1ZUAzA0s4MrJ+VxcmE/Jg5K5cXlVfz6H+s4588f8pcrihmft39Y6uju4YONtby3oYblFQ1srGneO8FsYUY8Z43sT3FeCsW5KQzu5+8GfWl5FXe/tYEL/zqPi4oHctvM4WQmeo7iVxIRCQ8KXyIRxBPl5PyxAz/XusYYBqfHMzg9nisn5WGtZUtdKyuXLuLCmafst+6syXmMzU7m1qeXcukD87n97BFcPSWP+Zt38erKat5es4PmTi+JHhfjclM4e1QW43KTGZOTfNDQePH4bM4c2Z/73ivloQ+38NaaHZw6PIMTC9KYMjiNQf3ieg2Q7V09RDkNLqdGVYhIeFL4EpHPZU8Yq/D0HmpGZSfx2rdO5kdzVvLr19bxh39+QltgwtkzTujPV8cOYGpB2hGFoni3i9tmDueyCTnc+24pczfV8vqq7QD0T/QweXAqToeDnU0d7GzqYEdTB80dXqKchvy0OArS4ynI8N+Pzk6mIL33wCYiEkwKXyJyzCTFRjF71nieWLCVldsaOb0ok+nD0o961vy8tDh+/7Uxe4++zdu8i/llu/h48y5cDkNmoofB6XGcWJBGRqKHlk4vm2ta2FTTzDvrd+INDNzPT4tlxohMvjIikwn5KWFzdKyhrYt73tnE5toW0uPdpCd8ehuTnUx+oCtWRI4PCl8ickwZY/xzhE3pm8/e0xV61eS8z/We7h4fW3e1Mr+snn+v38nj87fy4EdbSIqJojg3meTYaBI8LhI9USTGuHAYQ21LJzVNndQ0d7CzqZMur4/zxw7gykl59E86duPOfD7LnGWV3P3mBhrbuzlhQCJlta3UNnfS1eMDwBPl4OFrJuhi6yLHEYUvETmuRTkdDMlIYEhGArMm59HS6eWjTbX8a10NG3Y0sbm2laaObprau9kzs0W000FGopuMBDeFGfG0dfVw73ul3P/+Zs4c2Z9rT8xnfF7KUXVhrt/exH++vIalW3dTkpfCry8YyYisRMB/sfOmdi+VDW18/7kVXPfYYh69diKTB6f1+lmd3h4+qe8hqWI3CR4X8e4o4j0uYqOculSUSBhS+BKRiBLvdjFzZBYzR2btt9xaS1tXD909PpJioj4TrCp2tfHEgnKeW7yN11dtpzAjnqSYKDq9Prq8Prp6/PfpCW5yUmPJTokhOyWGgckxeHss9a1d1LV2Ut/SxfbGDt5au4OkmCh+f8loLi7O3i8kGWNIio0iKTaJp26YzOV/W8C1jyzmsesmMnHQ/peJmr95Fz97eTVltR2waN5+r0U7HUwpSGPmyP7MGJFJeoL7GP+aIvJFKHyJiOAPPHHug/9JzE2L5WfnFPH904fy8vJq3li9HZ+1JHhcRLscRLucuByGmuYOVlU28Naa7XT3fHaS2JgoJ6lx0Vw+MYcfnTGM5NhDz4uWnuDm6RsncdnsBXzjkUU8ft1ESvJTqW/t4q7X1/P3ZZVkp8Rwy2g3k4pH09zppaXDS0tnN9sbO/j3+hpuf3E1d5jVTMhL5fSiTHJSY0n0uEiMiSLB4yLBE4XDgNdn8fksXp+lx2dJio0i0XPoKUwO1OOzrKpsYHtjByV5KWQc4fQgPp9l/Y4mCtLjj3qsoEi4UvgSETkCsdEurpiUyxWTcg+5Xo/PUtPcQdXudqKcDtLio0mLcxMTfeSBIiPBwzM3Tuay2Qu45uFF3DStgEfmbaGlw8ut0wv49mmFLJz3IdOHZ3zmvXeeW8SGHc28vXYHb63ZwV1vrD+ibSd6XGSn7DmSF8uAZM/ekwEyEjxkJLppbOvmw011fLiplo9L62jq8O59/+D0OCYP9k8PMmlwKhkJvYexhrYunl+yjScXVFBR30ZaXDSzpuQxa3IeafFf/IhdW5eXR+eVU17XytQh/ThpSL+j+jyRY0HhS0SkDzgdhqykGLKSYo7J52Um7glg8/nTOxuZkJ/CXReOYmhmwiHfZ4xhRFYiI7IS+d6Moexs6qC2uZPmDi/NHd177y3gchgcDoPLYTDG0NDWReXudip3t1O+q5UPN9XR3t1z0G1lJXmYObI/JxemMzAlhiXl9f653lZU8/TCisD3cFOUlUjRgESKspJIjYvmxWWVvLqymk6vj4n5qdw4bTDvb6jhnnc2cf/7m7moOJsbTh7E4H5xeH3W380b6OpNiY0m2vXZs1a9PT6eX1LJPe9spKa5kwSPi+eXVGIMjBqYxClD05lSkMaQjHjS492f6Wbe3drFgjL/WbVrq5von+ihID0ucMKH/z7+EEdKRQ5F/3JERL4k+id5eOGWE1ld1cD0oRlfaDB9ZqLnC18pYM+JADXN/gBX0+w/IzTa6eCkwn4UpMfvF2KKc1O4aVoB3h4fa6qbWFJez7rqJtZtb+LDTXV7pwCJjXZyyfhsZk3JY3h//0kHsybnUVrTzEMfbeHvyyp5ZlHF3ktZ7cvtcjAmJ5mSvBRK8v1XSVhQVs9/v72BstpWSvJSuP+qYsbmpLCmqpEPNtYyd2Mt971Xyl/eLQUgweNicHo8Bf3iSPC4WFy+m/U7mrDWX9sJAxJZW93Im2u277f9wox4SvJTmZCfwoT8VLJTYjDG4PNZdrd1UdvSSV1zF7vafV/o95bjl8KXiMiXSHqCm9OGZ4Zk25+eCBBF4WGOuO3L5XQwNieZsTnJe5d1dPdQWtNCVUM7UwrSeh1bNiQjgd9eNJofnjGMvy+tpLnDi9vlCIyxc+ByOthS28rSrfXMnlvGX9//NBkVpMcxe9Z4Ti/K3BsIx+T4r6rwna8U0tjWzYrKBspqWyirbaWsroX5ZbvY3dZFcW4KPzx9KFMK+jE6O4mowHxwnd4eKna1sbm2hY07W1hWsZvXVlbzzCL/Ub30BDcOA3UtXfTsk9IM8FLVQi4Zn82ZJ/T/Ql3PcnxR+BIRkaDzRDkZOTCJkQOTDrtuv3g3N59ScMh12rt6WFnZwNKtu8lIcHPhuIGHnEQ3KTaKU4amc8rQ9M9ds9vlpDAzgcLMBGaO9C/r8Vk27mxmSXk9yysacDoMGYnuwGS5HlLionj+vWUs2dXK955bQYLbxbljshiRlUhdSxe7WjrZ1dJFXUsnPmsZkOw/Q3ZgSgwDkmKIjXZSubudivq2vbeWTi8leSlMHdKPqUP6kXqQi9lba+nq8dHp9dHR3UNnt/9xSqz/4vVHMlWKz2fZVNOC2+U47KS/u1o6ee+TWqJdDuKincRGu4hzO4mNduIwZu/NGP9UMJmJn+32Pd4pfImIyJdeTLSTyYPTDjoXWl9xOj4dUzfrIBMLd22L5o/XncLCLfW8sHQbLy+v5plF2zAGUmKj6Rc4GQNgdVUj/1y7c+8ku/tuZ0Cyh9zUWNLionl99XaeXez/jBMGJFKSl0pbl5fa5k5qWzqpbe78zBG4fbldDgYkxzAg2UNWUgwDkjxkJcfQP8nDgCT/fU1Tx95xbwvK6qlv7cIYuHJSbq9n6vp8lueWbNs7afDnNTA5htOLMpkxIpOJg1I/M4bPWktTh5d4twvnEXS117V0srqqkTWVjdQ0d/LrC0Z+7vf2NYUvERGRPuZwGKYUpDGlII3fXOClpdNLamx0r0fnfD5LXUsnVQ3ttHX1kJMSS1ayZ2/3J/hPKFhd1chHm+r4qLSO5xZvIzHG5T8TNd5/UkNavJt4twu3y4E7yokn0F1b3+qfa66qoZ3qhnY+3FRLTXMntvecxoAkD9OHpTN5cBrrqpt4fH45r6/azk9mDufSkhwcDsO66ib+8+XVLKtoYOKgVO44ewTxbietnT20dnlp6+yhrbsHay0+a/H5wGctrZ1ePirdxbOLK3h0XjkJbhcnFfYjyulgR+CarTubOujo9pEaF82MERmcUdSfkwr77TcVSVNHN6u2NbJi225WVjaypqqR7Y0de18vSI+jy+vr9eSMUFD4EhERCaLYaBex0Qf/z6/DYchI9BxyjjSX08G43BTG5abw7a8UHnVN3T0+apo72d7QzvbGDrY3tpMUE8XkwWnkpsbu1y146YQcfv7KWm5/cTXPLqpgTE4yTy2sICkmij98bQwXFQ88om7Eb0wdRHtXDx+X1vHO+p3M3ViLy+mgf6KH0dnJ9E/0T22ytrqJN1fv4PkllcRGOzllaDpxbhcrtjWwubZlb3gc3C+OCfmpjM72d2ufMCCRhCOcr66vKXyJiIhEuCinwz/WLPnwU6OMyErkuZsn8+rKau56fT0rK7dy+cRcbpt5+EmDDyYm2smMokxmFB36ZJIur48FZbv457od/GvdTrw9lrE5yZw/ZgBjc5MZnZ1MUkx4Ba3eKHyJiIjIETHGcP7YgcwYkUlNcyeDDjMI/1iJdjmYNjSdaUPT+c0Fo4Kyzb6g8CUiIiJfSJzbxSBNNnvEgjryzBgz0xjziTGm1Bjz02BuW0RERCQcBC18GWOcwH3AWUARcLkxpihY2xcREREJB8E88jURKLXWlllru4BngfODuH0RERGRkDP2YBN7HOsNGXMJMNNae0Pg+SxgkrX2WwesdxNwE0BmZub4Z599tk/ramlpIT4+vk+3IUdO7RK+1DbhSe0SvtQ24akv2uXUU09daq0tOdx6YTdKzlo7G5gNUFJSYqdPn96n23v//ffp623IkVO7hC+1TXhSu4QvtU14CmW7BLPbsQrI2ed5dmCZiIiISMQIZvhaDBQaYwYZY6KBy4BXg7h9ERERkZALWrejtdZrjPkW8DbgBB621q4N1vZFLnM+oAAAB5RJREFUREREwkFQx3xZa98A3gjmNkVERETCSXhc3ltEREQkQih8iYiIiASRwpeIiIhIECl8iYiIiASRwpeIiIhIECl8iYiIiARR0K7t+EUYY2qBrX28mX5AXR9vQ46c2iV8qW3Ck9olfKltwlNftEuetTb9cCuFdfgKBmPMks9zEUwJLrVL+FLbhCe1S/hS24SnULaLuh1FREREgkjhS0RERCSIFL5gdqgLkF6pXcKX2iY8qV3Cl9omPIWsXSJ+zJeIiIhIMOnIl4iIiEgQKXyJiIiIBFHEhi9jzExjzCfGmFJjzE9DXU8kM8bkGGPeM8asM8asNcZ8N7A81RjzL2PMpsB9SqhrjUTGGKcxZrkx5rXA80HGmIWBfec5Y0x0qGuMRMaYZGPMHGPMBmPMemPMFO0zoWeM+X7g79gaY8wzxhiP9pnQMMY8bIypMcas2WdZr/uI8ftzoI1WGWOK+7K2iAxfxhgncB9wFlAEXG6MKQptVRHNC/zQWlsETAa+GWiPnwL/ttYWAv8OPJfg+y6wfp/nvwP+ZK0dAuwGrg9JVfK/wFvW2uHAGPxtpH0mhIwxA4HvACXW2pGAE7gM7TOh8igw84BlB9tHzgIKA7ebgPv7srCIDF/ARKDUWltmre0CngXOD3FNEctau91auyzwuBn/f0QG4m+TxwKrPQZcEJoKI5cxJhs4B3gw8NwApwFzAquoXULAGJMETAMeArDWdllrG9A+Ew5cQIwxxgXEAtvRPhMS1tq5QP0Biw+2j5wPPG79FgDJxpisvqotUsPXQGDbPs8rA8skxIwx+cA4YCGQaa3dHnhpB5AZorIi2T3ATwBf4Hka0GCt9Qaea98JjUFALfBIoEv4QWNMHNpnQspaWwX8D1CBP3Q1AkvRPhNODraPBDUXRGr4kjBkjIkH/g58z1rbtO9r1j8niuZFCSJjzLlAjbV2aahrkc9wAcXA/dbacUArB3Qxap8JvsD4ofPxh+MBQByf7faSMBHKfSRSw1cVkLPP8+zAMgkRY0wU/uD1lLX2xcDinXsO+wbua0JVX4SaCnzVGFOOv2v+NPzjjJIDXSqgfSdUKoFKa+3CwPM5+MOY9pnQmgFssdbWWmu7gRfx70faZ8LHwfaRoOaCSA1fi4HCwBko0fgHRL4a4poiVmAc0UPAemvtH/d56VXgmsDja4BXgl1bJLPW3m6tzbbW5uPfR9611l4JvAdcElhN7RIC1todwDZjzLDAoq8A69A+E2oVwGRjTGzg79qedtE+Ez4Oto+8ClwdOOtxMtC4T/fkMRexM9wbY87GP57FCTxsrb0rxCVFLGPMScCHwGo+HVt0B/5xX88DucBW4OvW2gMHT0oQGGOmAz+y1p5rjBmM/0hYKrAcuMpa2xnK+iKRMWYs/hMhooEy4Fr8/0OtfSaEjDG/BC7Ffxb3cuAG/GOHtM8EmTHmGWA60A/YCfwceJle9pFAWL4XfzdxG3CttXZJn9UWqeFLREREJBQitdtRREREJCQUvkRERESCSOFLREREJIgUvkRERESCSOFLREREJIgUvkREemGMscaYSw6/pojIkVH4EpGwY4x5NBB+DrwtCHVtIiJHy3X4VUREQuIdYNYBy7pCUYiIyLGkI18iEq46rbU7DrjVw94uwW8ZY143xrQZY7YaY67a983GmFHGmHeMMe3GmPrA0bSkA9a5xhiz2hjTaYzZaYx57IAaUo0xLxhjWo0xZb1s487AtjuNMTuMMY/3yS8hIscVhS8R+bL6Jf7rsY0FZgOPG2NKAIwxccDbQAswEbgQOBF4eM+bjTE3Aw8AjwCjgbOBNQds4078134bAzwHPGyMyQ28/2LgR8CtQCFwLrCoD76niBxndHkhEQk7xphHgauAjgNeus9ae5sxxgIPWmtv3Oc97wA7rLVXGWNuBP4HyLbWNgden47/AseF1tpSY0wl8KS19qcHqcECd1trbw88dwFNwE3W2ieNMT8AbgZGWmu7j9mXF5HjnsZ8iUi4mgvcdMCyhn0ezz/gtfnAOYHHI4BVe4JXwDz8F24vMsY04b/Y8b8PU8OqPQ+stV5jTC2QEVj0AvBdYIsx5m3gLeBVXTBZRA5H3Y4iEq7arLWlB9zqjsHnHsnh/gOPaFkCfzettduAYfiPfjUBfwCWBro8RUQOSuFLRL6sJvfyfH3g8XpglDEmYZ/XT8T/N2+9tbYGqAK+cjQFWGs7rLWvW2u/D0wATgCmHs1nisjxT92OIhKu3MaY/gcs67HW1gYeX2SMWQy8D1yCP0hNCrz2FP4B+Y8bY+4EUvAPrn/RWlsaWOcu4E/GmJ3A60As8BVr7R8+T3HGmG/g/xu6EP/A/kvxHynbdITfU0QijMKXiISrGcD2A5ZVAdmBx78ALgb+DNQC11prFwNYa9uMMWcC9+A/A7ED/1mL393zQdba+40xXcAPgd8B9cAbR1BfA3Ab/oH9UcA64CJr7ZYj+AwRiUA621FEvnQCZyJ+zVo7J9S1iIgcKY35EhEREQkihS8RERGRIFK3o4iIiEgQ6ciXiIiISBApfImIiIgEkcKXiIiISBApfImIiIgEkcKXiIiISBD9f6tipS07+nRkAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x504 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualize the training process\n",
    "train_loss = history.history['loss']\n",
    "val_loss = history.history['val_loss']\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(10, 7))\n",
    "ax.plot(train_loss, label='Training Loss')\n",
    "ax.plot(val_loss, label='Validation Loss')\n",
    "ax.set_title('Loss vs. Epochs', fontsize=16)\n",
    "ax.set_xlabel('Epochs', fontsize=14)\n",
    "ax.set_ylabel('Loss', fontsize=14)\n",
    "ax.legend(fontsize=14)\n",
    "ax.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract Features using Triplet Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t\t (113783, 4)\n",
      "Validation:\t (22255, 4)\n",
      "Test:\t\t (22391, 4)\n",
      "\n",
      "Train Landmarks:\t 14943\n",
      "Validation Landmarks:\t 7674\n",
      "Test Landmarks:\t\t 14436\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv('./data/triplet/train.csv')\n",
    "val_df = pd.read_csv('./data/triplet/validation.csv')\n",
    "test_df = pd.read_csv('./data/triplet/test.csv')\n",
    "\n",
    "print('Train:\\t\\t', train_df.shape)\n",
    "print('Validation:\\t', val_df.shape)\n",
    "print('Test:\\t\\t', test_df.shape)\n",
    "\n",
    "print('\\nTrain Landmarks:\\t', len(train_df['landmark_id'].unique()))\n",
    "print('Validation Landmarks:\\t', len(val_df['landmark_id'].unique()))\n",
    "print('Test Landmarks:\\t\\t', len(test_df['landmark_id'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "inception_v3 (Model)         (None, 2048)              21802784  \n",
      "_________________________________________________________________\n",
      "l2_norm (Lambda)             (None, 2048)              0         \n",
      "=================================================================\n",
      "Total params: 21,802,784\n",
      "Trainable params: 6,073,536\n",
      "Non-trainable params: 15,729,248\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Load trained model\n",
    "base_model = load_model('./models/inception-base-0.4-model.h5')\n",
    "base_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define train_imgs and test_imgs\n",
    "train_imgs = np.zeros(shape=(len(train_df), 2048), dtype=np.float32)\n",
    "val_imgs = np.zeros(shape=(len(val_df), 2048), dtype=np.float32)\n",
    "test_imgs = np.zeros(shape=(len(test_df), 2048), dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Process:          0\n",
      "================================================================================\n",
      "Process:      20000\n",
      "================================================================================\n",
      "Process:      40000\n",
      "================================================================================\n",
      "Process:      60000\n",
      "================================================================================\n",
      "Process:      80000\n",
      "================================================================================\n",
      "Process:     100000\n",
      "========================================================"
     ]
    }
   ],
   "source": [
    "# Process training images\n",
    "img_ids = train_df['image_id'].values\n",
    "steps = 20000\n",
    "for i in range(0, len(train_df), steps):\n",
    "    tmp_imgs = []\n",
    "    print('\\nProcess: {:10d}'.format(i))\n",
    "    \n",
    "    start = i\n",
    "    end = min(len(train_df), i + steps)\n",
    "    for idx in range(start, end):\n",
    "        if idx % 250 == 0:\n",
    "            print('=', end='')\n",
    "            \n",
    "        img_id = img_ids[idx]\n",
    "        path = './data/triplet/train/' + str(img_id) + '.jpg'\n",
    "        img = load_img(path, target_size=img_size[:2])\n",
    "        img = img_to_array(img)\n",
    "        tmp_imgs.append(img)\n",
    "        \n",
    "    tmp_imgs = np.array(tmp_imgs, dtype=np.float32) / 255.0\n",
    "    tmp_prediction = base_model.predict(tmp_imgs)\n",
    "    train_imgs[start: end, ] = tmp_prediction\n",
    "    _ = gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Process:          0\n",
      "================================================================================\n",
      "Process:       4000\n",
      "================================================================================\n",
      "Process:       8000\n",
      "================================================================================\n",
      "Process:      12000\n",
      "================================================================================\n",
      "Process:      16000\n",
      "================================================================================\n",
      "Process:      20000\n",
      "=============================================="
     ]
    }
   ],
   "source": [
    "# Process validation images\n",
    "img_ids = val_df['image_id'].values\n",
    "steps = 4000\n",
    "for i in range(0, len(val_df), steps):\n",
    "    tmp_imgs = []\n",
    "    print('\\nProcess: {:10d}'.format(i))\n",
    "    \n",
    "    start = i\n",
    "    end = min(len(val_df), i + steps)\n",
    "    for idx in range(start, end):\n",
    "        if idx % 50 == 0:\n",
    "            print('=', end='')\n",
    "            \n",
    "        img_id = img_ids[idx]\n",
    "        path = './data/triplet/validation/' + str(img_id) + '.jpg'\n",
    "        img = load_img(path, target_size=img_size[:2])\n",
    "        img = img_to_array(img)\n",
    "        tmp_imgs.append(img)\n",
    "        \n",
    "    tmp_imgs = np.array(tmp_imgs, dtype=np.float32) / 255.0\n",
    "    tmp_prediction = base_model.predict(tmp_imgs)\n",
    "    val_imgs[start: end, ] = tmp_prediction\n",
    "    _ = gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Process:          0\n",
      "================================================================================\n",
      "Process:       4000\n",
      "================================================================================\n",
      "Process:       8000\n",
      "================================================================================\n",
      "Process:      12000\n",
      "================================================================================\n",
      "Process:      16000\n",
      "================================================================================\n",
      "Process:      20000\n",
      "================================================"
     ]
    }
   ],
   "source": [
    "# Process test images\n",
    "img_ids = test_df['image_id'].values\n",
    "steps = 4000\n",
    "for i in range(0, len(test_df), steps):\n",
    "    tmp_imgs = []\n",
    "    print('\\nProcess: {:10d}'.format(i))\n",
    "    \n",
    "    start = i\n",
    "    end = min(len(test_df), i + steps)\n",
    "    for idx in range(start, end):\n",
    "        if idx % 50 == 0:\n",
    "            print('=', end='')\n",
    "            \n",
    "        img_id = img_ids[idx]\n",
    "        path = './data/triplet/test/' + str(img_id) + '.jpg'\n",
    "        img = load_img(path, target_size=img_size[:2])\n",
    "        img = img_to_array(img)\n",
    "        tmp_imgs.append(img)\n",
    "        \n",
    "    tmp_imgs = np.array(tmp_imgs, dtype=np.float32) / 255.0\n",
    "    tmp_prediction = base_model.predict(tmp_imgs)\n",
    "    test_imgs[start: end, ] = tmp_prediction\n",
    "    _ = gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t\t (113783, 2048)\n",
      "Validation:\t (22255, 2048)\n",
      "Test:\t\t (22391, 2048)\n"
     ]
    }
   ],
   "source": [
    "print('Train:\\t\\t', train_imgs.shape)\n",
    "print('Validation:\\t', val_imgs.shape)\n",
    "print('Test:\\t\\t', test_imgs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save to disk\n",
    "np.save('./data/triplet/train-triplet-inception-0.4-features.npy', train_imgs)\n",
    "np.save('./data/triplet/validation-triplet-inception-0.4-features.npy', val_imgs)\n",
    "np.save('./data/triplet/test-triplet-inception-0.4-features.npy', test_imgs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Features and Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t\t (113783, 2048) (113783, 4)\n",
      "Validation:\t (22255, 2048) (22255, 4)\n",
      "Test:\t\t (22391, 2048) (22391, 4)\n"
     ]
    }
   ],
   "source": [
    "# Already normalized\n",
    "train_feature = np.load('./data/triplet/train-triplet-inception-0.4-features.npy')\n",
    "val_feature = np.load('./data/triplet/validation-triplet-inception-0.4-features.npy')\n",
    "test_feature = np.load('./data/triplet/test-triplet-inception-0.4-features.npy')\n",
    "\n",
    "train_df = pd.read_csv('./data/triplet/train.csv')\n",
    "val_df = pd.read_csv('./data/triplet/validation.csv')\n",
    "test_df = pd.read_csv('./data/triplet/test.csv')\n",
    "\n",
    "print('Train:\\t\\t', train_feature.shape, train_df.shape)\n",
    "print('Validation:\\t', val_feature.shape, val_df.shape)\n",
    "print('Test:\\t\\t', test_feature.shape, test_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper function\n",
    "def accuracy(true_label, prediction, top=1):\n",
    "    \"\"\" function to calculate the prediction accuracy \"\"\"\n",
    "    prediction = prediction[:, :top]\n",
    "    count = 0\n",
    "    for i in range(len(true_label)):\n",
    "        if true_label[i] in prediction[i]:\n",
    "            count += 1\n",
    "            \n",
    "    return count / len(true_label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implement KNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Merge train and validation features\n",
    "train_val_feature = np.concatenate((train_feature, val_feature), axis=0)\n",
    "train_val_df = pd.concat((train_df, val_df), axis=0)\n",
    "train_val_df = train_val_df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NearestNeighbors(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "         metric_params=None, n_jobs=-1, n_neighbors=50, p=2, radius=1.0)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Implement KNN model\n",
    "knn = NearestNeighbors(n_neighbors=50, algorithm='auto', leaf_size=30, \n",
    "                       metric='minkowski', p=2, n_jobs=-1)\n",
    "knn.fit(train_val_feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Search the first 50 neighbors\n",
    "distance, neighbor_index = knn.kneighbors(test_feature, return_distance=True)\n",
    "\n",
    "# Save the results\n",
    "np.save('./result/knn-triplet-inception-0.4-distance.npy', distance)\n",
    "np.save('./result/knn-triplet-inception-0.4-neighbor.npy', neighbor_index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Search Neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_distance = np.load('./result/knn-triplet-inception-0.4-distance.npy')\n",
    "knn_neighbor = np.load('./result/knn-triplet-inception-0.4-neighbor.npy')\n",
    "\n",
    "# Get the first 50 neighbors\n",
    "predictions = []\n",
    "for neighbors in knn_neighbor:\n",
    "    predictions.append(train_val_df.loc[neighbors]['landmark_id'].values)\n",
    "\n",
    "predictions = np.array(predictions)\n",
    "np.save('./result/knn-triplet-inception-0.4-test-prediction.npy', predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top  1 accuracy:\t 0.3671564467866554\n",
      "Top  5 accuracy:\t 0.5336072529141173\n",
      "Top 10 accuracy:\t 0.603948014827386\n",
      "Top 20 accuracy:\t 0.6738868295297218\n"
     ]
    }
   ],
   "source": [
    "print('Top  1 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=1))\n",
    "print('Top  5 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=5))\n",
    "print('Top 10 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=10))\n",
    "print('Top 20 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_acc = []\n",
    "for i in range(1, 51):\n",
    "    tmp_acc = accuracy(test_df['landmark_id'].values, predictions, top=i)\n",
    "    knn_acc.append(tmp_acc)\n",
    "\n",
    "np.save('./result/knn-triplet-inception-0.4-accuracy.npy', knn_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Environment (conda_tensorflow_p36)",
   "language": "python",
   "name": "conda_tensorflow_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}