3-2. VGG16 Triplet Network (margin 0.4).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import sys, os, time, gc, random, pickle\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "import requests, shutil\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "\n",
    "from keras import backend as K\n",
    "from keras.preprocessing.image import load_img, img_to_array\n",
    "from keras.models import Sequential, load_model, Model\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten, Input, Lambda, MaxPooling2D\n",
    "from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau\n",
    "from keras.applications.vgg16 import VGG16\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t\t (113783, 4)\n",
      "Validation:\t (22255, 4)\n",
      "Test:\t\t (22391, 4)\n",
      "\n",
      "Train Landmarks:\t 14943\n",
      "Validation Landmarks:\t 7674\n",
      "Test Landmarks:\t\t 14436\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv('./data/triplet/train.csv')\n",
    "val_df = pd.read_csv('./data/triplet/validation.csv')\n",
    "test_df = pd.read_csv('./data/triplet/test.csv')\n",
    "\n",
    "print('Train:\\t\\t', train_df.shape)\n",
    "print('Validation:\\t', val_df.shape)\n",
    "print('Test:\\t\\t', test_df.shape)\n",
    "\n",
    "print('\\nTrain Landmarks:\\t', len(train_df['landmark_id'].unique()))\n",
    "print('Validation Landmarks:\\t', len(val_df['landmark_id'].unique()))\n",
    "print('Test Landmarks:\\t\\t', len(test_df['landmark_id'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>id</th>\n",
       "      <th>url</th>\n",
       "      <th>landmark_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>465272</td>\n",
       "      <td>a2ccf8ed2e969f6a</td>\n",
       "      <td>https://lh4.googleusercontent.com/-TPHkS5gzvm4...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>64516</td>\n",
       "      <td>e205ca7c8dd7c027</td>\n",
       "      <td>https://lh3.googleusercontent.com/-V3RjsZtGpxE...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>928409</td>\n",
       "      <td>4e8ab93c1620e8a3</td>\n",
       "      <td>http://mw2.google.com/mw-panoramio/photos/medi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>88809</td>\n",
       "      <td>896bf928214d1ca4</td>\n",
       "      <td>http://lh5.ggpht.com/-Cy0l41uUaGA/R--yB8vy41I/...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1001133</td>\n",
       "      <td>375d2a153bdca926</td>\n",
       "      <td>http://lh6.ggpht.com/-UqzFpnqE9bU/S_0u1RovfdI/...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   image_id                id  \\\n",
       "0    465272  a2ccf8ed2e969f6a   \n",
       "1     64516  e205ca7c8dd7c027   \n",
       "2    928409  4e8ab93c1620e8a3   \n",
       "3     88809  896bf928214d1ca4   \n",
       "4   1001133  375d2a153bdca926   \n",
       "\n",
       "                                                 url  landmark_id  \n",
       "0  https://lh4.googleusercontent.com/-TPHkS5gzvm4...            0  \n",
       "1  https://lh3.googleusercontent.com/-V3RjsZtGpxE...            0  \n",
       "2  http://mw2.google.com/mw-panoramio/photos/medi...            0  \n",
       "3  http://lh5.ggpht.com/-Cy0l41uUaGA/R--yB8vy41I/...            0  \n",
       "4  http://lh6.ggpht.com/-UqzFpnqE9bU/S_0u1RovfdI/...            0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training set triplet generator\n",
    "def train_triplet_generator(df, batch_size=100, img_size=(224, 224), seed=42, \n",
    "                            prefix='./data/triplet/train/'):\n",
    "    \"\"\" training set triplet generator\n",
    "        it will generate 7400 triplet images in total\n",
    "    \"\"\"\n",
    "    # get images with only one training image landmark id and the rest landmark ids\n",
    "    np.random.seed(seed)\n",
    "    grouped = df[['landmark_id', 'image_id']].groupby('landmark_id').count().reset_index()\n",
    "    unique_neg_ids = list(grouped[grouped['image_id'] == 1]['landmark_id'].values)\n",
    "    rest_ids = list(grouped[grouped['image_id'] > 1]['landmark_id'].values)\n",
    "    size = 7400 * 2 - len(unique_neg_ids) \n",
    "    zeros = np.zeros((batch_size, 3, 1), dtype=K.floatx())\n",
    "    \n",
    "    while True:\n",
    "        # get positive and negative image landmark ids\n",
    "        np.random.shuffle(rest_ids)\n",
    "        candidate_ids = list(np.random.choice(rest_ids, size=size, replace=False))\n",
    "        pos_landmark_ids = candidate_ids[:7400]\n",
    "        neg_landmark_ids = candidate_ids[7400:] + unique_neg_ids\n",
    "        np.random.shuffle(neg_landmark_ids)\n",
    "        \n",
    "        # transform landmark id into image id\n",
    "        anc_img_ids = []\n",
    "        pos_img_ids = []\n",
    "        neg_img_ids = []\n",
    "        \n",
    "        for i in range(len(pos_landmark_ids)):\n",
    "            tmp_pos_ids = df[df['landmark_id'] == pos_landmark_ids[i]]['image_id'].values\n",
    "            anc_img_ids.append(tmp_pos_ids[0])\n",
    "            pos_img_ids.append(tmp_pos_ids[1])\n",
    "            \n",
    "            tmp_neg_ids = df[df['landmark_id'] == neg_landmark_ids[i]]['image_id'].values\n",
    "            neg_img_ids.append(tmp_neg_ids[0])\n",
    "        \n",
    "        # iterator to read batch images\n",
    "        for j in range(len(pos_img_ids) // batch_size):\n",
    "            batch_anc_img_ids = anc_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            batch_pos_img_ids = pos_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            batch_neg_img_ids = neg_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            \n",
    "            # get images\n",
    "            anc_imgs = []\n",
    "            pos_imgs = []\n",
    "            neg_imgs = []\n",
    "            \n",
    "            # iteratively read images\n",
    "            for k in range(batch_size):\n",
    "                anc_path = prefix + str(batch_anc_img_ids[k]) + '.jpg'\n",
    "                pos_path = prefix + str(batch_pos_img_ids[k]) + '.jpg'\n",
    "                neg_path = prefix + str(batch_neg_img_ids[k]) + '.jpg'\n",
    "                \n",
    "                tmp_anc_img = load_img(anc_path, target_size=img_size)\n",
    "                tmp_anc_img = img_to_array(tmp_anc_img)\n",
    "                anc_imgs.append(tmp_anc_img)\n",
    "                \n",
    "                tmp_pos_img = load_img(pos_path, target_size=img_size)\n",
    "                tmp_pos_img = img_to_array(tmp_pos_img)\n",
    "                pos_imgs.append(tmp_pos_img)\n",
    "                \n",
    "                tmp_neg_img = load_img(neg_path, target_size=img_size)\n",
    "                tmp_neg_img = img_to_array(tmp_neg_img)\n",
    "                neg_imgs.append(tmp_neg_img)\n",
    "        \n",
    "            # transform list to array\n",
    "            anc_imgs = np.array(anc_imgs, dtype=K.floatx()) / 255.0\n",
    "            pos_imgs = np.array(pos_imgs, dtype=K.floatx()) / 255.0\n",
    "            neg_imgs = np.array(neg_imgs, dtype=K.floatx()) / 255.0\n",
    "\n",
    "            yield [anc_imgs, pos_imgs, neg_imgs], zeros"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# validation set triplet generator\n",
    "def val_triplet_generator(df, batch_size=128, img_size=(224, 224), \n",
    "                          seed=42, prefix='./data/triplet/validation'):\n",
    "    \"\"\" validation set triplet collector \"\"\"\n",
    "    \n",
    "     # get images with only one image landmark id and the rest landmark ids\n",
    "    grouped = df[['landmark_id', 'image_id']].groupby('landmark_id').count().reset_index()\n",
    "    unique_neg_ids = list(grouped[grouped['image_id'] == 1]['landmark_id'].values)\n",
    "    rest_ids = list(grouped[grouped['image_id'] > 1]['landmark_id'].values)\n",
    "    size = 3072 * 2 - len(unique_neg_ids) \n",
    "    zeros = np.zeros((batch_size, 3, 1), dtype=K.floatx())\n",
    "    \n",
    "    while True:\n",
    "        # get positive and negative image landmark ids\n",
    "        np.random.seed(seed)\n",
    "        candidate_ids = list(np.random.choice(rest_ids, size=size, replace=False))\n",
    "        pos_landmark_ids = candidate_ids[:3072]\n",
    "        neg_landmark_ids = candidate_ids[3072:] + unique_neg_ids\n",
    "        np.random.shuffle(neg_landmark_ids)\n",
    "        \n",
    "        # transform landmark id into image id\n",
    "        anc_img_ids = []\n",
    "        pos_img_ids = []\n",
    "        neg_img_ids = []\n",
    "        \n",
    "        for i in range(len(pos_landmark_ids)):\n",
    "            tmp_pos_ids = df[df['landmark_id'] == pos_landmark_ids[i]]['image_id'].values\n",
    "            anc_img_ids.append(tmp_pos_ids[0])\n",
    "            pos_img_ids.append(tmp_pos_ids[1])\n",
    "            \n",
    "            tmp_neg_ids = df[df['landmark_id'] == neg_landmark_ids[i]]['image_id'].values\n",
    "            neg_img_ids.append(tmp_neg_ids[0])\n",
    "        \n",
    "        # iterator to read batch images\n",
    "        for j in range(len(pos_img_ids) // batch_size):\n",
    "            batch_anc_img_ids = anc_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            batch_pos_img_ids = pos_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            batch_neg_img_ids = neg_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            \n",
    "            # get images\n",
    "            anc_imgs = []\n",
    "            pos_imgs = []\n",
    "            neg_imgs = []\n",
    "            \n",
    "            # iteratively read images\n",
    "            for k in range(batch_size):\n",
    "                anc_path = prefix + str(batch_anc_img_ids[k]) + '.jpg'\n",
    "                pos_path = prefix + str(batch_pos_img_ids[k]) + '.jpg'\n",
    "                neg_path = prefix + str(batch_neg_img_ids[k]) + '.jpg'\n",
    "                \n",
    "                tmp_anc_img = load_img(anc_path, target_size=img_size)\n",
    "                tmp_anc_img = img_to_array(tmp_anc_img)\n",
    "                anc_imgs.append(tmp_anc_img)\n",
    "                \n",
    "                tmp_pos_img = load_img(pos_path, target_size=img_size)\n",
    "                tmp_pos_img = img_to_array(tmp_pos_img)\n",
    "                pos_imgs.append(tmp_pos_img)\n",
    "                \n",
    "                tmp_neg_img = load_img(neg_path, target_size=img_size)\n",
    "                tmp_neg_img = img_to_array(tmp_neg_img)\n",
    "                neg_imgs.append(tmp_neg_img)\n",
    "        \n",
    "            # transform list to array\n",
    "            anc_imgs = np.array(anc_imgs, dtype=K.floatx()) / 255.0\n",
    "            pos_imgs = np.array(pos_imgs, dtype=K.floatx()) / 255.0\n",
    "            neg_imgs = np.array(neg_imgs, dtype=K.floatx()) / 255.0\n",
    "            \n",
    "            yield [anc_imgs, pos_imgs, neg_imgs], zeros"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define Triplet Loss Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define base network for triplet network\n",
    "def base_net(input_shape=(224, 224, 3)):\n",
    "    \"\"\" define triplet network \"\"\"\n",
    "    # load pre-trained VGG16 model\n",
    "    vgg16 = VGG16(include_top=False, weights='imagenet', input_shape=input_shape, pooling='avg')\n",
    "    \n",
    "    # frozen shallow layers\n",
    "    vgg16.trainable = True\n",
    "    \n",
    "    set_trainable = False\n",
    "    for layer in vgg16.layers:\n",
    "        if layer.name == 'block5_conv1':\n",
    "            set_trainable = True\n",
    "        if set_trainable:\n",
    "            layer.trainable = True\n",
    "        else:\n",
    "            layer.trainable = False\n",
    "    \n",
    "    # define sequential model\n",
    "    model = Sequential(name='base_net')\n",
    "    model.add(vgg16)\n",
    "    model.add(Lambda(lambda x: K.l2_normalize(x, axis=1), name='l2_norm'))\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define triplet network\n",
    "def triplet_net(base_model, input_shape=(224, 224, 3)):\n",
    "    \"\"\" function to define triplet networks \"\"\"\n",
    "    # define input: anchor, positive, negative\n",
    "    anchor = Input(shape=input_shape, name='anchor_input')\n",
    "    positive = Input(shape=input_shape, name='positive_input')\n",
    "    negative = Input(shape=input_shape, name='negative_input')\n",
    "    \n",
    "    # extract vector represent using CNN based model\n",
    "    anc_vec = base_model(anchor)\n",
    "    pos_vec = base_model(positive)\n",
    "    neg_vec = base_model(negative)\n",
    "    \n",
    "    # stack outputs\n",
    "    stacks = Lambda(lambda x: K.stack(x, axis=1), name='output')([anc_vec, pos_vec, neg_vec])\n",
    "\n",
    "    # define inputs and outputs\n",
    "    inputs=[anchor, positive, negative]\n",
    "    outputs = stacks\n",
    "    \n",
    "    # define the triplet model\n",
    "    model = Model(inputs=inputs, outputs=outputs, name='triplet_net')\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define triplet loss\n",
    "def triplet_loss(y_true, y_pred):\n",
    "    \"\"\" function to compute triplet loss\n",
    "        margin is predefined coded, manually change if needed\n",
    "    \"\"\"\n",
    "    # define triplet margin\n",
    "    margin = K.constant(0.4)\n",
    "    zero = K.constant(0.0)\n",
    "    \n",
    "    # get the prediction vector\n",
    "    anchor, positive, negative = y_pred[:, 0], y_pred[:, 1], y_pred[:, 2]\n",
    "    \n",
    "    # compute distance\n",
    "    pos_distance = K.sum(K.square(anchor - positive), axis=1)\n",
    "    neg_distance = K.sum(K.square(anchor - negative), axis=1)\n",
    "    \n",
    "    # compute loss\n",
    "    partial_loss = pos_distance - neg_distance + margin\n",
    "    full_loss = K.sum(K.maximum(partial_loss, zero), axis=0)\n",
    "    \n",
    "    return full_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build Triplet Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For reproduciable purpose\n",
    "seed = 42\n",
    "K.clear_session()\n",
    "os.environ['PYTHONHASHSEED'] = '0'\n",
    "np.random.seed(seed)\n",
    "random.seed(seed)\n",
    "session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)\n",
    "tf.set_random_seed(seed)\n",
    "sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)\n",
    "K.set_session(sess)\n",
    "\n",
    "# Define Parameters\n",
    "img_size = (224, 224, 3)  # target image size\n",
    "\n",
    "# triplet image generator\n",
    "train_generator = train_triplet_generator(train_df, batch_size=100, img_size=img_size[:2], \n",
    "                                          seed=42, prefix='./data/triplet/train/')\n",
    "\n",
    "val_generator = val_triplet_generator(val_df, batch_size=64, img_size=img_size[:2], \n",
    "                                      seed=42, prefix='./data/triplet/validation/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3148: calling l2_normalize (from tensorflow.python.ops.nn_impl) with dim is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "dim is deprecated, use axis instead\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "vgg16 (Model)                (None, 512)               14714688  \n",
      "_________________________________________________________________\n",
      "l2_norm (Lambda)             (None, 512)               0         \n",
      "=================================================================\n",
      "Total params: 14,714,688\n",
      "Trainable params: 7,079,424\n",
      "Non-trainable params: 7,635,264\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Define triplet network model\n",
    "base_model = base_net(input_shape=img_size)\n",
    "base_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "anchor_input (InputLayer)       (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "positive_input (InputLayer)     (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "negative_input (InputLayer)     (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "base_net (Sequential)           (None, 512)          14714688    anchor_input[0][0]               \n",
      "                                                                 positive_input[0][0]             \n",
      "                                                                 negative_input[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "output (Lambda)                 (None, 3, 512)       0           base_net[1][0]                   \n",
      "                                                                 base_net[2][0]                   \n",
      "                                                                 base_net[3][0]                   \n",
      "==================================================================================================\n",
      "Total params: 14,714,688\n",
      "Trainable params: 7,079,424\n",
      "Non-trainable params: 7,635,264\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "triplet_model = triplet_net(base_model=base_model, input_shape=img_size)\n",
    "triplet_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fit Triplet Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  0.001\n"
     ]
    }
   ],
   "source": [
    "# define learning scheduler\n",
    "def lr_schedule(epoch):\n",
    "    \"\"\" Learning rate schedule \"\"\"\n",
    "    lr = 1e-3\n",
    "    if epoch > 80:\n",
    "        lr *= 2e-1\n",
    "    elif epoch > 60:\n",
    "        lr *= 4e-1\n",
    "    elif epoch > 40:\n",
    "        lr *= 6e-1\n",
    "    elif epoch > 20:\n",
    "        lr *= 8e-1\n",
    "    print('Learning rate: ', lr)\n",
    "    return lr\n",
    "\n",
    "# define optimizer\n",
    "opt = keras.optimizers.Adam(lr=lr_schedule(0))\n",
    "\n",
    "# Create call backs\n",
    "lr_scheduler = LearningRateScheduler(lr_schedule)\n",
    "lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6)\n",
    "callbacks = [lr_reducer, lr_scheduler]\n",
    "\n",
    "# compile the model\n",
    "triplet_model.compile(optimizer=opt, loss=triplet_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "Learning rate:  0.001\n",
      " - 76s - loss: 16.8931 - val_loss: 9.4313\n",
      "Epoch 2/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 10.9009 - val_loss: 7.3265\n",
      "Epoch 3/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 9.5194 - val_loss: 7.0726\n",
      "Epoch 4/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 9.3693 - val_loss: 6.4936\n",
      "Epoch 5/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 8.1358 - val_loss: 6.4937\n",
      "Epoch 6/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 7.4033 - val_loss: 5.5559\n",
      "Epoch 7/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 6.4741 - val_loss: 5.7101\n",
      "Epoch 8/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 6.5257 - val_loss: 5.2532\n",
      "Epoch 9/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 6.0759 - val_loss: 5.1704\n",
      "Epoch 10/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 6.0503 - val_loss: 4.8307\n",
      "Epoch 11/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 5.9599 - val_loss: 4.9559\n",
      "Epoch 12/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 5.7197 - val_loss: 4.8210\n",
      "Epoch 13/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 5.0197 - val_loss: 5.0047\n",
      "Epoch 14/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 5.0374 - val_loss: 4.8337\n",
      "Epoch 15/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 4.9699 - val_loss: 4.6135\n",
      "Epoch 16/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 4.5826 - val_loss: 4.6355\n",
      "Epoch 17/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 4.5218 - val_loss: 4.6576\n",
      "Epoch 18/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 4.4567 - val_loss: 4.4846\n",
      "Epoch 19/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 4.0449 - val_loss: 4.2726\n",
      "Epoch 20/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 4.1356 - val_loss: 4.2681\n",
      "Epoch 21/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 4.0446 - val_loss: 3.9847\n",
      "Epoch 22/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 4.0503 - val_loss: 4.0450\n",
      "Epoch 23/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 3.3737 - val_loss: 3.6160\n",
      "Epoch 24/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 3.5857 - val_loss: 4.2215\n",
      "Epoch 25/100\n",
      "Learning rate:  0.0008\n",
      " - 60s - loss: 3.4327 - val_loss: 3.9904\n",
      "Epoch 26/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 3.1548 - val_loss: 4.0883\n",
      "Epoch 27/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 3.2128 - val_loss: 3.6694\n",
      "Epoch 28/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 3.1359 - val_loss: 3.8267\n",
      "Epoch 29/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 2.8764 - val_loss: 3.8566\n",
      "Epoch 30/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 2.8420 - val_loss: 3.6394\n",
      "Epoch 31/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 2.9368 - val_loss: 3.7545\n",
      "Epoch 32/100\n",
      "Learning rate:  0.0008\n",
      " - 61s - loss: 2.8514 - val_loss: 3.8099\n",
      "Epoch 33/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 3.0206 - val_loss: 3.9270\n",
      "Epoch 34/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 2.5993 - val_loss: 3.7256\n",
      "Epoch 35/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 2.5838 - val_loss: 3.8940\n",
      "Epoch 36/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 2.6944 - val_loss: 3.6305\n",
      "Epoch 37/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 2.5512 - val_loss: 3.5982\n",
      "Epoch 38/100\n",
      "Learning rate:  0.0008\n",
      " - 58s - loss: 2.5667 - val_loss: 3.8607\n",
      "Epoch 39/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 2.7227 - val_loss: 4.1632\n",
      "Epoch 40/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 2.3011 - val_loss: 3.7151\n",
      "Epoch 41/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 2.7018 - val_loss: 3.9308\n",
      "Epoch 42/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 2.0195 - val_loss: 3.8273\n",
      "Epoch 43/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 2.2367 - val_loss: 3.5598\n",
      "Epoch 44/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 2.0234 - val_loss: 3.4525\n",
      "Epoch 45/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 2.1041 - val_loss: 3.5457\n",
      "Epoch 46/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.8917 - val_loss: 3.5826\n",
      "Epoch 47/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.6874 - val_loss: 3.6383\n",
      "Epoch 48/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.7024 - val_loss: 3.5980\n",
      "Epoch 49/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.9806 - val_loss: 3.4995\n",
      "Epoch 50/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.7389 - val_loss: 3.3593\n",
      "Epoch 51/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.6371 - val_loss: 3.6373\n",
      "Epoch 52/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.8735 - val_loss: 3.3360\n",
      "Epoch 53/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.6325 - val_loss: 3.2803\n",
      "Epoch 54/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.5154 - val_loss: 3.2061\n",
      "Epoch 55/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.7239 - val_loss: 3.1555\n",
      "Epoch 56/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.6077 - val_loss: 3.3996\n",
      "Epoch 57/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.5463 - val_loss: 3.5696\n",
      "Epoch 58/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.6045 - val_loss: 3.4887\n",
      "Epoch 59/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.5949 - val_loss: 3.2556\n",
      "Epoch 60/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.3520 - val_loss: 3.2506\n",
      "Epoch 61/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.3592 - val_loss: 3.3398\n",
      "Epoch 62/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 1.3530 - val_loss: 3.0117\n",
      "Epoch 63/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 1.3686 - val_loss: 3.1292\n",
      "Epoch 64/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 1.2421 - val_loss: 3.2655\n",
      "Epoch 65/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 1.3319 - val_loss: 3.2062\n",
      "Epoch 66/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 1.2791 - val_loss: 3.2916\n",
      "Epoch 67/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 1.2546 - val_loss: 3.1328\n",
      "Epoch 68/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 1.2729 - val_loss: 3.1242\n",
      "Epoch 69/100\n",
      "Learning rate:  0.0004\n",
      " - 60s - loss: 1.2233 - val_loss: 3.2837\n",
      "Epoch 70/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 1.0497 - val_loss: 2.9798\n",
      "Epoch 71/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 1.0972 - val_loss: 3.1251\n",
      "Epoch 72/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 1.0412 - val_loss: 3.0709\n",
      "Epoch 73/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 1.0204 - val_loss: 3.0822\n",
      "Epoch 74/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 1.0111 - val_loss: 3.1298\n",
      "Epoch 75/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 1.0386 - val_loss: 2.9991\n",
      "Epoch 76/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 1.0417 - val_loss: 3.0480\n",
      "Epoch 77/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.9868 - val_loss: 2.9554\n",
      "Epoch 78/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 1.1818 - val_loss: 3.0222\n",
      "Epoch 79/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 1.0788 - val_loss: 2.9081\n",
      "Epoch 80/100\n",
      "Learning rate:  0.0004\n",
      " - 60s - loss: 0.9070 - val_loss: 2.9262\n",
      "Epoch 81/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.9139 - val_loss: 3.0147\n",
      "Epoch 82/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.8757 - val_loss: 2.9639\n",
      "Epoch 83/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.8388 - val_loss: 3.0153\n",
      "Epoch 84/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.7944 - val_loss: 2.8036\n",
      "Epoch 85/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.8086 - val_loss: 2.9671\n",
      "Epoch 86/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.6419 - val_loss: 2.9034\n",
      "Epoch 87/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.7672 - val_loss: 2.9772\n",
      "Epoch 88/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.7152 - val_loss: 3.0106\n",
      "Epoch 89/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.8051 - val_loss: 2.9147\n",
      "Epoch 90/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.6615 - val_loss: 2.9296\n",
      "Epoch 91/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.7298 - val_loss: 3.0289\n",
      "Epoch 92/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.7585 - val_loss: 2.9429\n",
      "Epoch 93/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.6102 - val_loss: 2.8812\n",
      "Epoch 94/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.6955 - val_loss: 2.8998\n",
      "Epoch 95/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.7239 - val_loss: 2.9584\n",
      "Epoch 96/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.7274 - val_loss: 3.0043\n",
      "Epoch 97/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.6717 - val_loss: 3.1150\n",
      "Epoch 98/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.7086 - val_loss: 3.0410\n",
      "Epoch 99/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.5554 - val_loss: 3.0235\n",
      "Epoch 100/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.6723 - val_loss: 3.0930\n"
     ]
    }
   ],
   "source": [
    "# fit the mode\n",
    "history = triplet_model.fit_generator(train_generator, steps_per_epoch=74, epochs=100, \n",
    "                                      validation_data=val_generator, validation_steps=48, \n",
    "                                      verbose=2, callbacks=callbacks)\n",
    "\n",
    "base_model.save('./models/vgg16-base-0.4-model.h5')\n",
    "pickle.dump(history.history, open('./models/vgg16-triplet-0.4-history.p', 'wb'))\n",
    "_ = gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmUAAAG/CAYAAAAZwqDzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAIABJREFUeJzs3Xd8FVX+//HXSe8hEJLQo9QA0gwaBCkiUgSRtS6gICruuq5t1QXXvrrKfl17WfkpoqBEsRdQAc0qKFVRkSpgEJBeQwhp5/fH3EQSQnKT3EZ4Px+P+0juzNw5n9xx3bfnnDljrLWIiIiIiH8F+bsAEREREVEoExEREQkICmUiIiIiAUChTERERCQAKJSJiIiIBACFMhEREZEAoFAmIscwxow1xlhjTCt/1xIIjDGpru/jeK8ufqxN10qkjgjxdwEiIieQh4EPKti+1teFiEjdo1AmIuK+Ddbahf4uQkTqJg1fikiNGWNGG2O+N8bkGWN2GWOmGWMalTtmpDHmO2NMjjHmgDHmR2PMdUft726MmWOM2W2MOWyM2WCMea6SNlOMMYXGmBsr2HeHMabAGNPQ9X6gMeZrY8x+V/trjDH3ePI7KNd+yTDn9caYx4wxO4wxucaYj4wxqeWODTXGPGiM+cUYk+/6+aAxJrTccdHGmEeMMeuNMUeMMduMMW8bY5LLNZ9ojHnN9R1vNcY8ZYyJOOo8IcaYf7rOU3K95htjennr+xCR6lFPmYjUiDFmPPAC8AYwEWgM/As40xjTzVqb4/o//OnAU8DtOP8h2A6o5zpHDPApsBgYCxwEUoGzjteutXabMWYuMNp13qNdAXxird1pjDkVZ6jxLeABIB9oDZxaiz87yBhT/t+b1lpbVG7bRGA5cBWQhPO9fGaM6WCtLXAd8wpwqWvffJy/+R+u+kYCGGPCgDlAZ+ARYCEQDwwEEoDtR7U5DZgB/AHoAdwH7AXude3/O3CLq43lQByQDtSv/tcgIl5hrdVLL730KvPCCUgWaHWc/cE4geCLctt7uT53o+v9bcCeStpJdx3fqZr1jXJ9ru1R27q4tl3qen+x632cB76PVNe5KnrlVHDcSiDoqO09Xduvdr3v6Hp/X7l27jr6+wDGud5f4Ma1ur/c9o+AteXev+Pvf7b00kuv4780fCkiNdEWpwfotaM3WmvnA9lAH9emJUCCMWa6MWaoMaZeufOsA/YBL7iGQpu52f67QA5Oz1iJK4D9/D4RfzlQAGQaYy42xiS5ee7KPAh0L/c6u4Lj3rLWFpe8sdYuADbj9GAB9Hb9nF7ucyXvS76/84Bt1tqKbi4o7+Ny738Emh/1fgkwxBjzkDGml6sXTkQCiEKZiNREyZDXbxXs21ay31r7P+ASoBlOkNppjJlrjOnk2r8f6AdsBZ4DNhljVhhjLqqscWttLvA2MMo4goE/AjOttXmuY37GGeYLwhna22aMWWiM6XO887oh21q7tNzruwqO236cbU1cvx/v+9tWbn8DYIubte0p9/4IEH7U+3/hDGVeAHwF7DbGvGyMSXTz/CLiZQplIlITJQEgpYJ9KUftx1r7lrW2D84cqBFAI+ATY0yQa/9ya+1FOEGkB7AeeNMY07GKGqbhDBf2Avq7zjvt6AOstV9YawfhzGE7FygEPvZBECk/Cb9kW0nAOt73l1Ju/y5+D3K1Yq0tsNZOstaehvNd3QJcBDzrifOLSO0plIlITazB6fm5/OiNxpizgBZAVvkPWGtzrLUf4dwc0AinF+jo/YXWWW7ibpx/N6VVUcMXOEOCV7hev+D0AB3DWnvEWvs58G8gGjilinPX1sUloRPAGNMTaAp849r0pevn5eU+N8r1M8v18zMgxRgzzJPFWWu3WWtfBObizG8TkQCguy9FpDKDjDHbym3bb62d41pa4gVjzHScuVBNgIdw5olNATDGPIDTQ/QFzhBlU+BGYLl17pAcCowH3gM24gSmG3HuwvyGSlhri40xrwHXAaHA49ZaW7LfGPMnnLlbs4BfgUScuyK3Aitcx/QB5gHjrLWvuvF9nGqMyahg+1pr7dHDh7HAe8aYF4CGOIvOrgNeddW+whgzA7jPdTfn1zi9hHcDM6y1P7rOMx24FphhjHkYWOQ690DgCWvtajdqxvW3vg98D3yLc1dmV2AQTkgWkQCgUCYilXm6gm0/AR2ttZONMbk4S128jzPxfhZwh7X2kOvYRTgh63Gc4ckdOL0/d7v2rwMOu943wgljS4AB1trNbtQ3DWeph5Lfj/Y9MBgnECXhDAnOB0ZZaw+7jjE4d5K6O2ow0fUq7xKcpTdKPAy0AqbiBM0vgBvs78thgHPX5AacOyzvwgmLk4D7Sw6w1hYYY87DmQs23vVzN7CAY+eQVeVLV51/AaKATTg9hw9V8zwi4iXmqP+wFBGRWnAtELsRuNY1PCgi4jbNKRMREREJAAplIiIiIgFAw5ciIiIiAUA9ZSIiIiIB4IS8+zIxMdGmpqZ6tY1Dhw4RHR3t1TakZnRtApOuS+DStQlMui6By9PXZtmyZbustQ2rOu6EDGWpqaksXbrUq21kZWXRt29fr7YhNaNrE5h0XQKXrk1g0nUJXJ6+NsaYbHeO0/CliIiISABQKBMREREJAAplIiIiIgFAoUxEREQkACiUiYiIiASAE/LuSxERkaocOHCAHTt2UFBQUPXBfhAfH8+qVav8XYZUoDrXJjQ0lKSkJOLi4mrdrkKZiIjUOQcOHGD79u00adKEyMhIjDH+LukYBw8eJDY21t9lSAXcvTbWWg4fPsyWLVsAah3MNHwpIiJ1zo4dO2jSpAlRUVEBGcikbjDGEBUVRZMmTdixY0etz6dQJiIidU5BQQGRkZH+LkNOEpGRkR4ZJlcoExGROkk9ZOIrnvpnTaFMREREJAAolImIiIgEAIUyERGRk8Dll1/OxRdfXK3PZGRkcNttt3mpIilPS2KIiIgEgKrmJY0ZM4apU6fW+PwvvPAC1tpqfWbWrFmEhobWuE13TZgwgblz57J06VKvtxXIFMoqsPdQPpsOFFFcbAkK0kRRERHxvt9++630948++ohrr722zLbj3U1aUFDgVnCKj4+vdk3169ev9mek5jR8WYG3v93MPV/nkZNf6O9SRETkJJGSklL6qlev3jHb4uPjWb16NcYYZs6cSZ8+fYiIiOCVV15h+/btXHbZZaVrs3Xs2JHXXnutzPnLD19mZGRwyy23cPvtt1O/fn1SUlKYOHFimd608sOXKSkpTJo0iXHjxhEbG0uzZs146qmnyrSzcuVKevbsSUREBO3bt2fOnDmEhISQmZlZ4+9m165djBo1ioSEBKKiohg4cCBr1qwp3b97925GjhxJw4YNiYiIoFWrVjz//POl+59++mlatWpFeHg4DRs2ZPDgwTWuxZvUU1aByLBgAPLyi4iL8H63rYiIeN/9H/7Eyq0HfNpm+8Zx3Dusg8fPO2HCBP7zn//QuXNnwsPDOXz4MBkZGUycOJG4uDhmz57NmDFjaNGiBb169TrueaZMmcLtt9/OokWLWLx4MVdeeSVnnHEGI0aMOO5nHn30Uf75z39y55138t5773HTTTfRq1cvunXrRmFhIcOHD6dly5YsXryYAwcOcMstt1BcXFyrv3fUqFFs2bKFDz/8kNjYWP7+978zePBgVq1aRXh4OBMmTGDdunXMnj2bxMRENmzYwL59+wBYsGABf/vb35g2bRo9evRgz549zJs3r1b1eItCWQWiXKEsN7/Iz5WIiIgc69Zbb+XCCy8ss+2WW24p/f0vf/kLc+bMITMzs9JQ1q1bN+666y4AWrduzX//+1/mzZtXaSgbOnQof/rTnwC47bbbePLJJ/n888/p1q0bH3/8MdnZ2SxYsICkpCQAJk2aRP/+/Wv8t/7444989tlnLFq0iDPOOAOA1157jebNmzNz5kxGjx5NdnY26enppKenA5Camlr6+ezsbOLi4hg2bBhRUVE0b96cLl261Lgeb1Ioq0BkqBPKDhcolImI1BXe6LHyl5LwUaKwsJCHHnqIt956iy1btpCfn8+RI0eqHKbr1KlTmfeNGzeu8nFBlX1m9erVpKamlgYygDPPPLPKv6cyq1atIiwsjO7du5dua9CgAWlpaaxcuRJwQujll1/OwoULGTBgABdccEFpGB0yZAgPPvggqampDBw4kIEDBzJixAiio6NrVZc3aE5ZBSIUykREJICVDxQPPfQQzz77LBMnTuSLL75g+fLlDBkyhPz8/ErPU/4GAWNMlUONNfmMt5TcsTp8+HCys7O56aab+O233xg4cCB//vOfAahXrx7ff/89r732Go0bN+aBBx6gffv2HnlWpacplFUgKszpQDys4UsRETkBzJ8/nxEjRjBy5Eg6d+7Mqaeeytq1a31eR7t27cjOzmbnzp2l2xYvXlyrc6alpZGfn8+SJUtKt+3evZtVq1bRvn370m1JSUmMHTuWadOm8dxzz/Hiiy+WhsXQ0FAGDBjApEmT+P7779m5cyeffPJJreryBg1fVqB0+FKhTERETgBt2rTh448/5ptvvqFevXo89thjbN26lRYtWvi0jvPPP5/mzZszZswYHnnkEQ4ePMiECRMwxlS5Dtvhw4dZvnx5mW0xMTGcdtppDBw4kKuvvpr//ve/xMTEMGHCBJKTk7nkkksAuPPOO8nIyKB9+/YcOXKE9957j7Zt2xIUFMQ777zD1q1b6dWrFwkJCXz22Wfk5eWRlpbmte+hptRTVoHIMOdr0fCliIicCO6//346derEgAED6Nu3L0lJSdVevd8TQkJCeP/999m3bx/du3fnmmuu4Z577gEgIiKi0s+uXLmSrl27lnmNHTsWgOnTp9OpUyfOP/98evToQXFxMbNnzyYsLAxwesL+/ve/06lTJ3r37k1RURHvvPMOAAkJCcycOZNzzjmHtLQ0nn76aV599dUyc9QChanu6r6BID093Xpz1d8t+w7T85HP+fdFnbi0ezOvtSM1k5WVRd++ff1dhpSj6xK4TsZrs2rVqoDsCTnawYMHiY2N9XcZXrdo0SIyMjJYsWIFHTqcGDdb1OTaVPbPnDFmmbU2vcKdR9HwZQV096WIiEjNzJw5k4SEBFq1asX69eu5+eabOeOMM06YQOZPCmUVUCgTERGpmf379zNx4kQ2b95MgwYN6N+/P4899pi/yzohKJRVICLUmVOmxWNFRESq55prruGaa67xdxknJE30r4AxhrBgyFNPmYiIiPiIQtlxhAdpSQwRERHxHYWy4wgLNhq+FBEREZ9RKDuOcA1fioiIiA/5LJQZY6YYY3YYY1aU2/5XY8xqY8xPxph/+6qeqoQFG919KSIiIj7jy56yqcCgozcYY/oBw4HO1toOwKM+rKdS4cGQm1/o7zJERETkJOGzUGat/RLYU27zn4FHrLVHXMcEzCPbnZ4y/zz1XkREpDZefPFF6tWrd9z3FXnkkUdo1aqVx9sW9/l7nbI2wNnGmIeAPOA2a+2Sig40xowHxgMkJyeTlZXl1cKCbCG79+73ejtSfTk5ObouAUjXJXCdjNcmPj6egwcP+ruMShUVFZWp8bLLLiM3N5cPP/zwmGPXrFlD9+7deffdd+nfv3+V587LywMoPf8FF1zAOeecU+l3cuTIEYqLi93+3goLC6lfvz6vv/46Q4cOLd3uTlue8MADD/Dpp5+yYMECj5+7/LVxR15eXq3/d+bvUBYC1AcygO7Am8aYU20FD+S01k4GJoPz7EtvP8dt8g+fsseGnnTPizsRnIzP8TsR6LoErpPx2qxatSrgnytZ/vmK1113HSNGjGD37t2kpqaWOTYzM5MWLVpwwQUXEBRU9SBXycO/S84fGxtLUlJSpZ8JDw8nKCjI7e+tsNCZ4hMZGVnmM+605QnVrbc6avLsy4iICLp27Vqrdv199+Vm4B3rWAwUA4l+rgmAsGA4nK/hSxER8Y3zzz+f5ORkXn755TLbCwoKmDZtGuPGjSsNZLfddhtt2rQhMjKSU045hQkTJnDkyJHjnruiIcWHH36Y5ORkYmNjGTt2LLm5uWX2L1q0iAEDBpCYmEhcXBxnn302ixcvLt1fEhxHjBiBMaZ06LOitp577jlatmxJWFgYrVu3ZsqUKaX7CgsLMcbw4osvctFFFxEdHU3Lli2ZMWOGm99cxfbs2cMVV1xBQkICUVFRnHfeeaxatap0/969exk1ahQNGzYkIiKCli1b8swzz5SpuXXr1oSHh9OwYUMGDRpEcbF3c4G/e8reA/oBXxhj2gBhwC7/luQID9KSGCIidcrsCbDtR9+2mXIaDH7ErUNDQkIYM2YMU6dO5d577y0NYB9++CG7du3iqquuKj02Li6OqVOn0rhxY3766Seuu+46IiMjuffee91q6/XXX+e+++7jmWeeoU+fPmRmZvLoo4+W6eE6ePAgY8aM4amnngLg6aefZvDgwfz8888kJCSwZMkSGjduzMsvv8ygQYMICak4UsycOZObb76ZJ554gnPPPZdZs2Yxfvx4GjVqxODBg0uPu//++5k0aRKTJk3ihRdeYOzYsZx99tk0bdrUrb+pvCuuuIKNGzfywQcfEB8fz8SJExk0aBBr1qwhIiKCO++8k9WrVzNr1iySkpLYsGEDu3fvBmDJkiXcdNNNvPrqq5x11lns3buXzz//vEZ1VIcvl8SYAXwDtDXGbDbGXA1MAU51LZORCYypaOjSH8JCDLn5hQRIOSIichK4+uqr2bRpE3Pnzi3d9tJLL3HeeefRrFmz0m333HMPZ511FqmpqZx//vlMmDChWj1LTzzxBOPGjePaa6+lTZs23HPPPXTr1q3MMeeeey6jR48mLS2NtLQ0nn32WYKCgvj0008BaNiwIQD16tUjJSWFxMSKB7oeffRRxo4dy/XXX0+bNm24+eabufzyy5k0aVKZ48aOHcvIkSNp1aoVDz30EADz5893+2862qpVq5g1axYvvvgiZ599Np06dWL69Ons2bOHzMxMALKzs+nWrRvdu3enRYsW9OvXj4svvhiAX3/9ldjYWIYNG0aLFi3o0qULt956q1tDx7Xhs54ya+0fj7NrtK9qqI7wICi2kF9UTHhIsL/LERGR2nKzx8qfWrduTZ8+fZgyZQrnnXceW7du5dNPPy0NEiXeeOMNnnrqKdavX09OTg6FhYXVCgyrVq3ihhtuKLOtR48ezJw5s/T99u3bufvuu8nKymL79u0UFRWRm5vLpk2bqvU3rVq1iuuvv77Mtl69enHPPfeU2dapU6fS38PCwkhMTGTHjpotyrBq1SpCQkI488wzS7clJCTQoUMHVq5cCcD111/PpZdeypIlSxgwYADDhg2jd+/egBNIGzduzCmnnMLAgQM577zz+MMf/kBMTEyN6nGXv+eUBaywYANAnuaViYiID1199dW899577Nmzh6lTp1K/fn2GDx9eun/+/PmMGjWKIUOG8OGHH/Ldd9/xwAMPkJ+f79E6Ro8ezXfffccTTzzB119/zfLly2ncuLHH2jHGlHkfGhp6zH5vzOEqaXfo0KFkZ2dz6623sn37dgYPHsy1114LOMPDy5cvJzMzk6ZNm/LQQw+RlpbGtm3bPF7P0RTKjiPc1TmWW6AFZEVExHcuvvhiIiIimD59OlOmTOHKK68sE1gWLFhAixYt+Mc//kH37t1p3bo1v/zyS7XaSEtLY+HChWW2lX8/f/58brzxRoYMGUKHDh2Ijo4uE0qCg4MJDg6mqKjy+ddpaWnHLFsxf/582rdvX62aqyMtLY3CwkIWLVpUum3fvn389NNPZdpt2LAhV155Ja+++iqTJ09mypQpFBQUAM4cv/79+/PII4/w/fffs2/fPmbNmuW1msH/E/0DVklP2WE9lFxERHwoMjKSkSNHct9997F3716uvvrqMvvbtGnDpk2bmDFjBmeccQazZ8/mzTffrFYbN910E1dffTWnn346Z599Nm+++SbLli0rM9G/TZs2TJs2jfT0dA4ePMjtt99OeHh46X5jDM2bN2fevHn07NmT8PBwEhISjmnr9ttvZ+TIkXTt2pVzzz2Xjz/+mMzMzArXY6uuw4cPs3z58jLboqOjSUtL4/zzz+faa6/lhRdeIC4ujokTJ1K/fn0uu+wyAO666y66d+9O+/btKSgo4N1336V169aEhoby9ttvs2PHDnr37k1CQgLz5s0jNzeXtLS0WtdcGfWUHUeYq6dMz78UERFfu+aaa9i7dy9nnXXWMUFgxIgR3HLLLdx444106dKFrKws7r///mqdf9SoUdx1111MnDiRbt26sWbNGm666aYyx0ydOpV9+/bRtWtXRo4cyXXXXVfmZgOAxx57jDlz5tCsWTO6d+9eYVsXX3wxjz/+OI8++igdOnTg2Wef5YUXXihz52VNrV27lq5du5Z5XXHFFQC8+uqrdOvWjaFDh5KRkUF+fj6ffPJJ6RpuYWFhTJw4kc6dO9OrVy/y8vJ4//33AefmhXfeeYf+/fvTrl07Hn/8cV5++WV69OhR65orY07EuwvT09Pt0qVLvdrGM2/N49Glebz1px6kp9b3altSPSfjQpgnAl2XwHUyXptVq1Z5vVejtmqyQKn4Rk2uTWX/zBljlllr06s6h3rKjiNcPWUiIiLiQwplx1E6fKk5ZSIiIuIDCmXHEV4y0V89ZSIiIuIDCmXHoZ4yERER8SWFsuMIC1JPmYjIiczbD48WKeGpf9YUyo4j3LWCW656ykRETjjR0dFs2bKF/Px8PcNYvMZaS35+Plu2bCE6OrrW59PisccRYiDIQJ56ykRETjhNmzZl165dZGdnU1gYmE9mycvLK10zSwJLda5NSEgI8fHxx30ge3UolB2HMYbI0GDNKRMROQEFBQWRlJRUZoX6QJOVlUXXrl39XYZUwF/XRsOXlYgMCyFXPWUiIiLiAwpllYgMCyJPPWUiIiLiAwpllYgMDdZEfxEREfEJhbJKRIaFaEkMERER8QmFskpEhgYplImIiIhPKJRVQndfioiIiK8olFUiSsOXIiIi4iMKZZWIUE+ZiIiI+IhCWSUiwzSnTERERHxDoawSUWEh6ikTERERn1Aoq0REaDCHC4r0MFsRERHxOoWySkSGBgOQV1Ds50pERESkrlMoq0RUmBPKNK9MREREvE2hrBIlPWUKZSIiIuJtCmWViCjpKcsv9HMlIiIiUtcplFUiqqSnLF9zykRERMS7FMoqEak5ZSIiIuIjCmWViHD1lOVq+FJERES8TKGsEiV3X+app0xERES8TKGsErr7UkRERHxFoawSJXPKcvWoJREREfEyn4UyY8wUY8wOY8yKCvb9zRhjjTGJvqrHHaUT/RXKRERExMt82VM2FRhUfqMxphlwHrDJh7W45ffHLCmUiYiIiHf5LJRZa78E9lSw63HgDiDgnvodGhxESJDR8KWIiIh4XYg/GzfGDAe2WGu/N8ZUdex4YDxAcnIyWVlZXq0tJyeHrKwsQoMs6zZmk5W1zavtiftKro0EFl2XwKVrE5h0XQKXv66N30KZMSYKuBNn6LJK1trJwGSA9PR027dvX+8VB2RlZdG3b19iF8wlMTmJvn07ebU9cV/JtZHAousSuHRtApOuS+Dy17Xx592XLYFTgO+NMb8ATYFvjTEpfqzpGJFhwRq+FBEREa/zW0+ZtfZHIKnkvSuYpVtrd/mrpopEhgbr7ksRERHxOl8uiTED+AZoa4zZbIy52ldt10ZkWLAWjxURERGv81lPmbX2j1XsT/VRKdWinjIRERHxBa3oX4Uo9ZSJiIiIDyiUVSEiVKFMREREvE+hrAoavhQRERFfUCirgoYvRURExBcUyqoQEaaeMhEREfE+hbIqRIYGc6SwmKLigHs0p4iIiNQhCmVViAoLBiBPQ5giIiLiRQplVYgMdUKZ5pWJiIiINymUVSGiJJRpXpmIiIh4kUJZFaLCnIceqKdMREREvEmhrAqRYc5XpJ4yERER8SaFsiqUDF/mKpSJiIiIFymUVaFk+FJ3X4qIiIg3KZRVQXdfioiIiC8olFUhUsOXIiIi4gMKZVWIDFNPmYiIiHifQlkVSkJZnnrKRERExIsUyqqg4UsRERHxBYWyKgQHGcJCgjR8KSIiIl6lUOaGyNBgLYkhIiIiXqVQ5obI0GBy8wv9XYaIiIjUYQplbogKC+ZwQbG/yxAREZE6TKHMDRGhwXr2pYiIiHiVQpkbIsOCOVyg4UsRERHxHoUyN0SFqadMREREvEuhzA0RoZpTJiIiIt6lUOaGyNBgDuvuSxEREfEihTI3OHdfavhSREREvEehzA26+1JERES8TaHMDZHqKRMREREvUyhzQ1RoMAVFloIiTfYXERER71Aoc0NkWDCAestERETEaxTK3BAR6oSyPM0rExERES/xWSgzxkwxxuwwxqw4atv/GWNWG2N+MMa8a4yp56t6qiNKPWUiIiLiZb7sKZsKDCq3bQ7Q0VrbCVgLTPRhPW6LdPWU5aqnTERERLzEZ6HMWvslsKfcts+stSWrsi4EmvqqnuqIUE+ZiIiIeJmx1vquMWNSgY+stR0r2Pch8Ia1dvpxPjseGA+QnJx8emZmphcrhZycHGJiYgBYs6eIhxfncUf3CNo3CPZqu1K1o6+NBA5dl8ClaxOYdF0Cl6evTb9+/ZZZa9OrOi7EYy3WgjHmH0Ah8NrxjrHWTgYmA6Snp9u+fft6taasrCxK2qi/eR8sXkCbtI70bZ/s1XalakdfGwkcui6BS9cmMOm6BC5/XRu/hzJjzFhgKNDf+rLbrhpK5pRp+FJERES8xa+hzBgzCLgD6GOtzfVnLZXROmUiIiLibb5cEmMG8A3Q1hiz2RhzNfAMEAvMMcYsN8b811f1VEdpT5nuvhQREREv8VlPmbX2jxVsfslX7deGespERETE27SivxsiQtRTJiIiIt6lUOaGoCBDRGiQespERETEaxTK3BQZGqyeMhEREfEahTI3RYWFqKdMREREvEahzE0RoUHqKRMRERGvUShzU2RYsHrKRERExGsUytwUFRqinjIRERHxGoUyN0WEBZOrnjIRERHxEoUyN0WGBpGnnjIRERHxEoUyN+nuSxGilTTvAAAgAElEQVQREfEmhTI3RYQGk6ueMhEREfEShTI3RYYGk6eeMhEREfEShTI3RbmWxLDW+rsUERERqYMUytwUGRZMUbElv6jY36WIiIhIHaRQ5qaI0GAA8vIVykRERMTzFMrcFBXmhDLdgSkiIiLeoFDmpkhXT1lufqGfKxEREZG6SKHMTTHhIQDsO1zg50pERESkLlIoc1PLpBgAft6e4+dKREREpC5SKHNT8/pRRIQGsWb7QX+XIiIiInWQQpmbgoMMbZJjWbNNoUxEREQ8T6GsGtomx7JaoUxERES8QKGsGtqmxLIr5wi7c474uxQRERGpYxTKqqFdShyAhjBFRETE4xTKqqFNinMHpoYwRURExNMUyqqhYUw49aPD1FMmIiIiHqdQVg3GGNomx2pZDBEREfE4hbJqapsSy9rtBykutv4uRUREROoQhbJqapcSS25+EZv3HvZ3KSIiIlKHKJRVU9uUWABWbzvg50pERESkLlEoq6Y2yU4o02R/ERER8SSFsmqKDg+hWf1IVmuyv4iIiHiQQlkNtE2OU0+ZiIiIeJTPQpkxZooxZocxZsVR2+obY+YYY9a5fib4qp7aaJcSy8ZdhzhSWOTvUkRERKSO8GVP2VRgULltE4B51trWwDzX+4DXNiWWomLL+h2H/F2KiIiI1BE+C2XW2i+BPeU2Dwdecf3+CnChr+qpjXauOzDXbNcdmCIiIuIZ/p5Tlmyt/c31+zYg2Z/FuCs1MZqw4CA9A1NEREQ8xljru5XpjTGpwEfW2o6u9/ustfWO2r/XWlvhvDJjzHhgPEBycvLpmZmZXq01JyeHmJiY4+6/e8FhEsINt6ZHeLUOOVZV10b8Q9clcOnaBCZdl8Dl6WvTr1+/Zdba9KqOC/FYizWz3RjTyFr7mzGmEbDjeAdaaycDkwHS09Nt3759vVpYVlYWlbVx+rbvWLRxT6XHiHdUdW3EP3RdApeuTWDSdQlc/ro2/h6+/AAY4/p9DPC+H2uplrYpcfy2P4/9uQX+LkVERETqAF8uiTED+AZoa4zZbIy5GngEGGCMWQec63p/QiiZ7L92h+aViYiISO35bPjSWvvH4+zq76saPOn3Z2AepHtqfT9XIyIiIic6fw9fnrAaxUcQGxHCGj2YXERERDxAoayGjDG0S4nV45ZERETEIxTKaqFNciyrtx3El8uKiIiISN2kUFYL7VJiOZhXyG/78/xdioiIiJzgFMpqIa1RHAD/9+ka9uXm+7kaEREROZEplFXk8D5iD6yr8rDTWyRwXZ9T+eD7rfT/z/9497vNGsoUERGRGlEoq8h30zj929sgb3+lhxljmDg4jQ9v6EWz+lHc8sb3jH5pERt25vioUBEREakrFMoqEtfE+bl/s1uHt28cx9t/Pot/XtiRHzbvZ9ATX7Fww24vFigiIiJ1jUJZReKbOj/3b3H7I8FBhisyWjDvb30ICTZ8smKbl4oTERGRukihrCIloeyAez1lR0uKjaB9ozhWbKl86FNERETkaLUKZcaYSGPMucaYFp4qKCDEJFNsgt0eviyvY5N4Vv52gKJiTfoXERER91QrlBljphpjrnf9HgYsBj4D1hhjBnuhPv8ICiY/rEG1hi+P1rFJPLn5RWzcdcjDhYmIiEhdVd2esoHAQtfvFwCxQApwn+tVZ+RFJNaip8xZv0xDmCIiIuKu6oayBGCH6/dBwNvW2h1AJtDek4X525HwxBrNKQNo1TCG8JAghTIRERFxW3VD2TagozEmGKfXbK5rewxQ4MnC/O1IeKIzfFlcXO3PhgQH0a5RHCu2KpSJiIiIe6obyqYAbwArgCJgnmv7mcBqD9bld3kRiVBcAId21ujzpzWJ46ctByjWZH8RERFxQ7VCmbX2AWAcMBnoZa0teeBjITDJw7X51ZHwhs4vNRzC7Ng4noNHCtm0J9eDVYmIiEhdFVLdD1hr365g2yueKSdwHAlPdH7ZvxmanF7tz3dsEg/Aiq37SU2M9mRpIiIiUgdVd0mMS40x5x31/h5jzGZjzKfGmEaeL89/8iJcPWU1XBajdXIMocGGFVsOeLAqERERqauqO6fsvpJfjDHdgDuBp4BQ4D+eK8v/CkNiIDSqxstihIcE0zYllp802V9ERETcUN1Q1gJY4/p9BPCetfbfwK1Af08W5nfGOA8mr+GcMnDmla3Ysh9rNdlfREREKlfdUJaHs2AsOCGsZEmM/Udtrzvim9a4pwygQ5N49uYWsGXfYQ8WJSIiInVRdUPZV8B/jDF3A+nALNf2NsCvniwsIMQ3qfGcMoCOjUtW9te8MhEREalcdUPZDUA+cDHwJ2vtVtf2wcCnniwsIMQ3g5ztUJhf9bEVSGsUR3CQ0bwyERERqVK1lsSw1m4GhlWw/WaPVRRI4poAFg5uhYTUan88IjSY1kkxetySiIiIVKna65QBGGPOwXnWpQVWWmu/8GhVgSK+qfNz/5YahTKADo3j+XJdzZ4KICIiIiePaoUyY0wT4F3gdKBk6LKxMWYpMOKo4cy6oTSU1eIOzCZxvP3tZrYfyCM5LsJDhYmIiEhdU905ZU/hPPOylbW2mbW2GdDate0pTxfnd3FNnJ+1WBbjtJKV/TWEKSIiIpWobigbAPzFWruxZIO1dgNwo2tf3RIWBZH1a9VTltYoDmN0B6aIiIhUrrqhDJx5ZO5sqxtquSxGdHgIpyZGs0J3YIqIiEglqhvK5gFPG2OalWwwxjQHngA+92RhASO+GRyoeSgD5+HkP2n4UkRERCpR3VB2IxANbDDGZBtjsoH1QBTwV08XFxDimsD+2q2Le1qTeLbuz2N3zhEPFSUiIiJ1TXXXKfvV9SDyc4F2rs2rgJ+Bx4BLPVteAIhvAnn74chBCK/Zk6Q6NHZN9t96gD5tGnqyOhEREakjqr1OmXWerj3H9QLAGNMZuMiDdQWOeNdI7f4tkNSu8mOPo33p45b2K5SJiIhIhWoy0d/jjDG3GGN+MsasMMbMMMYEzoJeHlgWIz4ylHYpsWSt2eGhokRERKSu8Xsocy1IeyOQbq3tCAQDl/u3qqN4YAFZgCGnNWJp9l62H8jzQFEiIiJS1/g9lLmEAJHGmBCcmwYC58kAsY3ABNVqWQxwQpm18MmKbR4qTEREROoS40wRq+IgYz6o4pA44GxrbXCNijDmJuAh4DDwmbV2VAXHjAfGAyQnJ5+emZlZk6bclpOTQ0xMDAAZ34xjb0Jn1rS7qVbn/Mf8XGJCDRPPjPREiSeto6+NBA5dl8ClaxOYdF0Cl6evTb9+/ZZZa9OrOs7dif673di/sYpjKmSMSQCGA6cA+4CZxpjR1trpRx9nrZ0MTAZIT0+3ffv2rUlzbsvKyqK0jZ9b0ii0kEa1bPOSwrU8OW8d7btlkKTnYNZYmWsjAUPXJXDp2gQmXZfA5a9r41Yos9Ze5cUazgU2Wmt3Ahhj3gHOAqZX+ilfim8Cv/1Q69Ocf1ojnpi7jk9/2sYVPVJrX5eIiIjUGYEwp2wTkGGMiTLGGKA/ztpngSO+qbOqvxtDvZVpnRxLq6QYPv7xNw8VJiIiInWF30OZtXYR8BbwLfAjTk2T/VpUeXFNoTAPcqsaxa3akNMasXjjHnYe1Or+IiIi8ju/hzIAa+291tp21tqO1torrLWBlVhKl8Wo3eOWAIaclkKxhU9/0l2YIiIi8ruACGUBL961gGwtl8UAaJscy6kNo5m9QkOYIiIi8juFMneUPGrpQO1DmTGGIR0b8c363XpAuYiIiJRSKHNHVAMIifDI8CU488qcIcztHjmfiIiInPgUytxhDMQ19sjwJUBao1hOSdQQpoiIiPxOocxd8U1r/fzLEsYYBndM4ev1u9lzKN8j5xQREZETm0KZu+KaemROWYkhpzWiqNgyZ6XuwhQRERGFMvfFN4WDv0FRoUdO16FxHM3rR2lemYiIiAAKZe6LbwK2GPZv8sjpjDH0bpPIog27KSgq9sg5RURE5MSlUOauZmdCUCi8cx0cyfHIKXu2TORQfhE/bN7nkfOJiIjIiUuhzF1JaXDJy7BlGcy4HAoO1/qUGac2wBj4+ufaP75JRERETmwKZdWRNgwufB5+mQ9vjoHC2t05mRAdRvtGcSxYv8tDBYqIiMiJSqGsujpfBkMfh3WfwjvX1nrif89WiXybvY/D+UUeKlBERERORAplNZF+FQz8F6x8Dz64AYprPlG/R8sG5BcVsyx7rwcLFBERkRONQllN9fgL9L0Tvp8BP7xR49OckVqfkCCjIUwREZGTnEJZbfS5AxLbwOLJNT5FdHgIXZvX4+ufFcpEREROZgpltWEMdL8Gtn7r3JVZQz1aJvLjlv3sP1zgweJERETkRKJQVludL4fQaFj8Yo1P0bNlA4otLNqgpTFEREROVgpltRUR79yRueJtOFSzUNW1eQIRoUF8vV6hTERE5GSlUOYJ3a+FoiPw3bQafTwsJIjuqfX5WpP9RURETloKZZ6Q3B5a9ISlL0FxzdYb69kqkbXbc9hxMM/DxYmIiMiJQKHMU7pfA/s2wbo5Nfp4z5aJAHyjIUwREZGTkkKZp6QNg5gUWFKzCf/tG8cRFxGi52CKiIicpBTKPCU4FE4fCz/PhT0bqv/xIEOPlg20iKyIiMhJSqHMk04fC0HBsOSlGn28Z6tENu89zKbduZ6tS0RERAKeQpknxTWCdkPhu+mQX/1gdVbLBgCld2Fu25/HJyt+45HZq3n2i589WqqIiIgElhB/F1DnnHGt86Dyh5s6K/6XMs5DzM8cf9yPtmwYQ1JsOE/OW8fjc9ey/cAR55MGrIWuzepxVqtEL/8BIiIi4g8KZZ7WoicMeRQO/lZ2+4q3YcVblYYyYwyXn9GcD7/fSqem8XRpVo8uzerRMimGQY9/yaRPVvPeX3piyoQ9ERERqQsUyjzNGKe3rCILnoQjByE89rgfv3VAG24d0OaY7bcMaMPtb/3A7BXbGHJaI09VKyIiIgFCc8p85ZTeUFwImxbW6ON/6NaUNskx/N+naygoKvZwcSIiIuJvCmW+0uxMCA6Djf+r0ceDgwx3DGzHxl2HeGPJrx4uTkRERPxNocxXQiOdYLahZqEMoH9aEt1TE3hy3jpy8ws9WJyIiIj4m0KZL53SG7b9CLl7avRxYwwTBrdj58EjTJm/0cPFiYiIiD8plPnSKb0BC7/Mr/EpTm9RnwHtk/nv/zaw51C+52oTERERvwqIUGaMqWeMecsYs9oYs8oY08PfNXlF424QGg0bv6zVae4Y2Jbc/EKenLuW/EJN+hcREakLAmVJjCeBT6y1FxtjwoAofxfkFSFh0KJHrUNZ6+RYLj69Ka98k80r32STGBNGclwEjeIjaJoQxeCOKXRPrU9QkNYzExEROVH4PZQZY+KB3sBYAGttPlB3x+VO6QNz7oaD2yA2pcaneWB4R9JT67N132G2H8hj2/48Nu89zIKfdzP1619oVj+Si7o15aJuTWlWv25mXBERkbrEWGv9W4AxXYDJwEqgM7AMuMlae6jcceOB8QDJycmnZ2ZmerWunJwcYmJiPH7emIPrSV92KyvTbmVHch+Pn/9IoWXZjiIWbClg5e5iLNA2IYirOoaTEh0Qo9W15q1rI7Wj6xK4dG0Ck65L4PL0tenXr98ya216VccFQihLBxYCPa21i4wxTwIHrLV3H+8z6enpdunSpV6tKysri759+3r+xMVF8O9TIW0YDH/G8+c/ypZ9h3n3281MWfALIUGGGeMzaNnwxP8XgNeujdSKrkvg0rUJTLougcvT18YY41YoC4Suk83AZmvtItf7t4BufqzHu4KCIbVXreeVuaNJvUhuOKc1meMzKLaWyycv5OcdOV5vV0RERKrP76HMWrsN+NUY09a1qT/OUGbddUof2JcNe3/xSXNtkmOZcW0G1uIKZgd90q6IiIi4z++hzOWvwGvGmB+ALsC//FyPd53S2/m58SufNdk6OZbM8RkY4wSzddsVzERERAJJQIQya+1ya226tbaTtfZCa+1ef9fkVQ3bQnRSxUOYO1bDdu90FLZKimHGtRkYY7h88kI27jpU9YdERETEJwIilJ10jHF6yzb+D0putMjPhc/ugufPgqlDIG+/V5pulRRD5vgMDuUX8uJXG7zShoiIiFSfQpm/nNIbcrbDrrWw/nN4LgO+fhrShsLhvfC19+7MbNkwhnPTkpm9YhsFRXoigIiISCBQKPOXknllb14J00ZAcCiM/RgufRU6jIBvnoWcHV5rfljnxuw5lM+Cn3d5rQ0RERFxn0KZvySkOq/dP8PZt8GfFjhLZQD0uwsK8+Cr/3it+b5tGxIbEcKH3//mtTZERETEfX5/zNJJyxgY9RaYIGjQsuy+xFbQdTQseQkyroeEFh5vPjwkmEEdUvhkxTbyCjoSERrs8TZERETEfeop86fE1scGshJ9/u4EtqyHvdb8sM6NOXikkKw1O73WhoiIiLhHoSxQxTeBM8fD95leWyLjrJYNaBAdxoffb/XK+UVERMR9CmWBrNetEB4Lnz/oldOHBAcx5LRGzFu9nZwjhV5pQ0RERNyjUBbIourDWTfCmo/h18VeaeKCLo3JKyhm3qrtXjm/iIiIuEehLNBl/BmiG8JHt8Dqj6HwiEdPf3rzBBrFR/DBcg1hioiI+JNCWaALj4Hz/wMHtkLmSPi/1vDun2HdXCgqqPXpg4IMwzo35st1O9mXm++BgkVERKQmFMpOBO2Hw21rYfTbkDbM6TF77SJ4qiscqP06Yxd0bkxBkeWTFds8UKyIiIjUhELZiSI4FFqdCxc+C7evg8umQ+5ueO9PUFy7RyV1aBzHKYnRfPiDhjBFRET8RaHsRBQS7vSYDXoYNmTBwudqdTpjnCHMb9bvZsfBPACstRw6UsiWfYfJzdedmSIiIt6mFf1PZN3GwLo5MO9+51majTrV+FQXdEphzRevc+NTu1hPE/bnFpDvelh5WHAQ6akJ9G7TkN6tG5LWKBZjjKf+ChEREUGh7MRmDAx7Cp4/C96+BsZnQVhU2WPyDji9aam9nCU2KpJ3gFZf3sgLYe+SXxjG7JTrWdnsMhKiw4mPDGXjrkP8b81OHpm9mkdmryYpNpxrzj6F8b2P8zQCERERqTaFshNddAMY8TxMGwFz7nbu1ATIz4Ul/w/mPwGH90BEPPS+A8641hn+LLFtBbx5Jez9BfreSdiWpQxf9wTDo1fA2c9BbDIAdw5JY9v+PL5ct5N3v93Cv2atplvzBNJTjxP0REREpFo0p6wuaHkO9LgBlrwIK9+HRZPhqS4w5x5o0s25KaBpd/jsH/DsGfDTu2AtfDsNXuwP+YdgzIfQ9+8w8k0Y8ij8Mh+e7wGrZ5U2kxIfwaXpzXhxTDpN6kVy57s/kl9Yu5sMRERExKFQVlf0vweST3N6vWbfDg1awVWzf19GY/TbMPodCI2GmWPhyc7wwQ3QPAP+NB9SezrnMcbpTRv/P4hrDJl/hE8mlrnDMzo8hAeGd2Dt9hz+31cb/PP3ioiI1DEKZXVFSDhc8jJ0vBiueBfGfgwtzip7TKv+8Kev4IKnITgM+k50glpMw2PPl9QOrpkHZ1zn3N359tVlnibQPy2ZwR1TeGreOrJ3H/LyHyciIlL3aU5ZXZLYGi5+qfJjgoKh25XOqyoh4TDk3xDfxBkKPbzHGQoNjwXg3mEd+GrdLu56bwWvjjtDd2SKiIjUgnrKpGo9b4ILn4eNX8HUoZCzE3DmmN0+sC1frdvFB99XsPBsfq4zX01ERESqpJ4ycU+XkRBZ35mPNmWgc5dnQS5XhG4ntP5Sjrz/CgUrwwjN2wU5253gln8QQiJgwD+deWrqSRMRETkuhTJxX9tBcOX78PqlMO1CwOlqHQnstTHs3Fif2MQmRDfqQlBMsjNXLftr58aD9fPggmcqnr8mIiIiCmVSTc3PhOu/gW0/QkwSRCdBdENeyfqFJ+aug4MQszWE7qkJZJzagB59xtGxZSZBc+91Frkd8bzzDE8REREpQ6FMqi+usfM6ys3ntmHkmc1ZtGEPCzfsZuGG3Xyxxpl7Vi+qJRc3fZYb9jxCvekXQcb1MOAB5yHrIiIiAiiUiQclxUYwrHNjhnV2AtuOg3l8s343X63bxUfrgpl24C4mhrzO2IXPceS3nwj/43SIiPNz1SIiIoFBoUy8Jik2guFdmjC8SxOstazfmcOXa7sw8dNTeDD7/8HLg2HUzGN63URERE5GWhJDfMIYQ6ukWMb1OoXEs6/mqvzbKdq9EV48F7b/5O/yRERE/E6hTHzu+r6t+Dn2TG6KfBhri2HKINiQ5e+yRERE/ErDl+JzkWHB3Hl+Gje8nsc5A1/hD6tuhlcvdJ4UEBTi3AAQFAqhkTDgfmh3vr9LFhER8Tr1lIlfnH9aIzJOrc8DX+1n3x8/hL4TnAVqO/4B2gyCU/s4Ae2tcbB5mWcaXT4DMkdB7h7PnE9ERMSD1FMmfmGM4b4LOjDkya947MttPDB8wjHH2JydmBf7w4zL4dp5UK95zRssLoLPH4QDm2HPRrjyPWedNancyg/AFkGHEf6uRESkzguYnjJjTLAx5jtjzEf+rkV8o11KHFdktGD6wmxWbj0AQGFRMZ+v3s71ry2j7cNLuZ4J5B85TNH0SyBvf80b+3muE8jO/BPs2QAvD4H9W45/vLU1b6uuWP8FzBwD71wH+371dzUiInVewIQy4CZglb+LEN+6dUBb4iNDueu9H/nXrFX0eORzxk1dysINe/hDtyZsoCljc/9K8c51rHzqIr5YuYVth4pZu/0gK7bs57tNe1m6Jpv9OVU8+HzZVOfpA+c9CFe8Awe3wcuDnF6zEoX5sOJt56Hr/2pyct8VuvcXeOsqqH+q8/6Lf/m1HBGRk0FADF8aY5oC5wMPAbf6uRzxofioUG4f2I473/2RHzbv55x2SVx8elP6tk0iLMT5b4aftnZm1ieW4Zse5tvX/8q/Cy/jjAWPkxG0koygVXQ22fwQdSZd/v5pxY3s3wJrP4GeNzs3EbQ4C8a8D9P+4PSYXfgsbPgfLH8NDu10hkmDQ2D232HMhyffg9TzcyFzNBQXw8g3YdnL8PUzcNYNkNzB39WJiNRZARHKgCeAO4BYfxcivnd592Y0qhdBpybxNIgJP2Z/h8bxdBg3gaLPchn99ZOMDpkHQFFQOAcaduOn3GZ0OfgV+36cTb3TBh/bwHfTwRZDtyt/39bkdBj7sfNg9WkjwARBm8GQPg5angNLX4JZt8HqjyBtmLf+9MBjLXzwV9i+wlnYt0FL6HUrfPsqzL0fRr3p7wpFROosY/08d8YYMxQYYq293hjTF7jNWju0guPGA+MBkpOTT8/MzPRqXTk5OcTExHi1DakmW0yL7DfJzy8gN6kbB+LaYINC+W3/EXos+yuR4WGs7vEkNij4qM8UkbFwPLlRTfmh8/3HnDIy9zca7F7MzoY9ORKRWLrdFBdx+rJbCC7KY0n3ZygODvPFX3gsW0TSjgVEH8omu8UlFAdHeLW5pr++T6v1U9hwymg2tbikdHuzTe/QcsMrfNflIfbX63jM5+rvXkr9rV+wq+lA9tU77eTrXQxw+vdZYNJ1CVyevjb9+vVbZq1Nr+q4QAhlDwNXAIVABBAHvGOtHX28z6Snp9ulS5d6ta6srCz69u3r1TakZiq6Ng/+3yTuOvQvGPq409tVYu1n8PolcOmr0H549RrakAWvDof+98DZf3PvM0WFzhBobErtgklRAfzwJsx/DHb/7GxL6QR/nAHxTY893lr4bhoseBIGTYLW51a/zQ1ZTq9hu6HO93V0/QWH4aluENcIrplXdt/Sl+HjW7EWDMXQMA3OHA+dLoOw6OrXIR6nf58FJl2XwOXpa2OMcSuU+X2iv7V2orW2qbU2Fbgc+LyyQCZSkaQzLmJRcTuK5j1Y9i7NZS87E/zbDqn+SU/t6wSUL/8DB347dn/+Ifj6aXjveufmgCdOgweT4LF2TpjLO1D9No/kwJKXnAD0/vXOArqXvurM7dqzESb3g83l/oMkZ6ez/toHf4UDW+GN0fDr4uq1u30lvDkGEtvChc8fGyhDI6HfnbBlGaz6wNlmrXMDwEc3Q8v+LOj5Cgx/zpm399Et8FgazLnHuYFCRESq5PdQJuIJF3RpyoOFowk+vBu+eszZeGCrM8G/6ygnKNTEef+E4gKYV27oc0MWPNcDPrvLWTqiqACaZUCvm6H3HZC9AKYOce7yrMz+zc4dn7PugBd6wyPN4eNbITbZCWLXfeX08LUZCNfMhbAo5+aEH2Y6n18zG57v4Sz5MfBfcON3Tm/Wa5fADjdvZt77i9NDFhoFI9+A8ON02XcZ6fSCzb3f6Tn74K/wv0nQdTT8cQaFoXHOd33dlzDuU2du3oIn4b0/OzcNiIhIpQJloj8A1tosIMvPZcgJKCU+grhTu/PJtn4MXPgcJv0qZ/iv/AT/6qp/KvT4C8x/HLpfAw1awZy7nYnv9VvC2FmQ2vPYzzU/E964El4aAKPfgcTWv+8rLoKV7zvn3PaDsy00yrn54OxbnTDTvMexvVVJ7eCaz+HNK+Gda5xewOwFkHwaXPkBJLd3jrviXXhpoBO0xn0KCS2O//cd3O484qowD8Z9UvmxQcFw7r3OYr7PZThhrvcdTg/a0bUaA80znFejzjD3PohuCIMe1lwzEZFKBFQoE6mN4V2acN9bf+C86G8wn90NW7+DU/v9vtZWTZ39N1j+ujNMmbcfDu2AnjdB34nOsF5FWp0LYz9yeqxeOs+5kzGlE/yQCfOf+P/t3Xd4lFXexvHvmZn03kkjBAKBQKihFymCiKCAqIi9oK6666Ku69rXXXV9t1jXvhZsqIiAggJSRAFBei8BQklCKEkI6cnMef84Q01CEtpMwu9zXXORmeeZmTNzfJLbUyF3O4S1NK1bCb0gql3dWvP8wkzomvkQrGoe7EkAACAASURBVPzYLPMx4DGwnTBrNaSZWYvtg8uPBzP/iKqvVZIPn1wNhTkm1EW2qf39Ww2Fpr1gz69Vx+9Vp/cfTfBb+qZp/eszofb3EEKIi5SEMtFoDG3XhCenRvBTxDgGbHrfPDjk72f/wl4BcOkzphsusq0ZbB/bufbnxXaGO2bDJ6PhoxHgEwIFmab16NqJZrzaiTNF68rmCVe+ZgKdVw2ryES1hXFfmbFtn15tvoeQZhAYa96zvNi0eB3YDOMmQXzXur23UjD2U9M13KTqLMxqz7/seTP54cdnzPi+TjfU9ZOenr0SctaZ8XOVZRDR2rQmBsaB5SxGZpQXmS7ltqNr7soVQojzQEKZaDQCvT24tE0UT2wfyC/+M1HaftIA/z25xWQcKqJPUjiqvt1oHa6H8FamtctWj+UxwlrAHXPMIHqLFa58FVoMOjfdeDUFsqOadjfhb9I4EwoBLB5mcVxlMbM6x/zPtOrVh2+oudWVxWImDxQfNOPQ/MLNGLn6sFdC/i44uM1MNtjzq9movqKanRw8/CAiGeK6Qpdb6rfgrcMOk++Ard/Dttlw7ccXvsvV4Ti7UCmEaLAklIlGZWSnWGasy+a3we/QLd7/WID6LSOX8ROXk19cwcDWkTx7VVviQnzr/sJKQVyts5mr5x8Jt39/Zs89W62GwIQNcGCTGQN29HY4E0a+Ae2uvjDlsHnCdZ+YWapf3mImRKTdUX23KpiZpmu/hOw1cGibue+oMMeUxXT3droB4rubm6cfHNhiPuf+zebfFR/CsrfN8bTbIWUkeJxmnTet4YdHTSBrMQg2fQs//wv6/emcfx3VOpJjQuuOBWZcYdtRkHw5eAdemPcXQrichDLRqFzSKoJgXw8+3RVIt56dAJixNpsJX64mNtiH23sn8uaC7Qx5aSEPDUnm1l7NsFoa+eDzgChzczWvALhhsgkeC14ws2TbXws97jWTFCpKYON0s95axs8mfIW1NC2UycPMZImwlubc6loJE3qa21HFuWYs4PL34Zu7TeDqfLMZ1+YTUvX5S/4Ly96Bnveb7t4p42Hec6Z1tL4te/W1eYb5XsqLoN1o2PmzCYdWT9OSmXqNCWkyUUKIRk1CmWhUPG0WrkiNZsrKTIrKKvls6W6em7mJLgkhvHdzGiF+nozqFMtT09bzt+82Mm11Js+PSqVdbJCri35x8I8wY9gOboNf34DVn5sQFt/dtHCVHTZj3wY+AR3GQVDsmb+Xb6jZr7PnfbBzodk6a/FrZoLEwCegy63Hx/RtnGaWN0m5Cgb/zYSfEa+aMXdf3wnj50N40rn4Bk5WVgizHoOVH0GTVBj9rplw4XBA5nLYMBU2ToUtM2HPUhj6DwlmQjRiMnBBNDojO8VSUmFn3HtLeW7mJoalNuHTO7sT4me6MuNDfXn/1q68Pq4TWfmljH5jMUt3HHJxqS8y4S3N7M0HN8LAJ81Cu8lD4Zbv4PerTJfh2QSyEykFzS8x4+vuXgiRKWYtuLf7mRapPctgyl1mDNqot4+P5/L0hbGfmVmxk8ad2WLAp5O1Gt7ua5ZX6f1Hs1PC0RmwFgvEd4Ohz8Mf15vWu6VvwbcPyJpvQjRiEspEo9OlaQhxIT6s2ZPP+L6JvH59Z7w9Tp7lqJRiePsYZk/oR3yoD3d9vIL0/YUuKvFFzDcU+j0M9/0Ko9+BxL7nd5B7k1SzVMk1H5mQ9dFwMwkiMAaun1R1iZPgpnDNh2ZSxDf3mIkA58LeFeZ9K8tMeQb/9eRlTU5ksZju1L4Pmxa1qb8zEx/qKn8PylGP84UQLiPdl6LRsVgU/7qmAwcLyxjePua054b6efLhbd0Y9cYibvtwGVN+15uIgBr+OIrGQSloO9KME1v8mtn1YfS7Zg246iT2g8ueM2PSXkw0y4fE9zCzW2O71H9/z8yVZv04nxC4dQYEx9etzIOeNBMV5v0dKktg9Hs1zwQuOmiW9VjzOWStonVkPxhwjmb9VsdeCSV5NU/cqI3W0i0rBBLKRCPVo3kNf2CrER/qy/9u6cp17yzhzonLmTS+Bz6eZ7B+mGhYPHzgkkfMrTbd7zGtadvnwe6lkO5c/05Zzc4FycPMTMmwFqd/nazV8PFI8AkyLWR1CWQn6vcnsPnA7MfNpIDWV5hlTqyeYLWZfUY3fQvbZoGj0kxSSBlJ1MapZuxefXa3cNhNqDuw2Yz5a9rr5OCqtdmHdd1XsGGKWYsu7Xa49K91nzGqtQmPs58wz61LXQjRiEkoEwLoEB/Mq2M7cfcnK/jDpFW8dWOXY7MyC0orWJGRx+Z9R+jUNJiuzUIb/4xNcTKlzCSAlKvM/ZI82PMb7F4MW2ebkDT7cbOAbfIwM4YtphN4nzCBJHuNWczXK8iMnQtuemZl6XW/aTGb+YjZ8/RU/lHQ43dmbb2otuCwk/fqDkJmPmLGzdVl54bt82HW47B/A1hspkURzHi8hF7mc62fAnk7weplxgP6hsHyD8z3MeIVaFnL+ndH9sF3D8KWGeATCvOfM2VrM6L+34kQjYSEMiGchrRtwtPDU3jm24089OVqgn09WbYzl037CtD6+Hlhfp5c2iaKy9pF0TspHC+btKpddHxCzBpwrYaY3R7yMmDLDyZgLHoFfvmPOS+spenijGwDi142S3nc+u3p9xiti653Quq1prXMUQF25007zBIi1hN+tVusbGrzIL3W/gm+ug3GzzOTGKpzYKvZ23XrDxCcYMbeJV9utizbtQgyFsGaSVBRbLp1+/0J2gw/Hj47jINp95ldJDreYLp9T11+RGvTAvfDo2ZM3ZC/mzXrPhpuxu2FtzKL/wpxEZJQJsQJbu2dyJ68Ev73y058PKx0TgjmgUEt6dYslFZNAli6I5dZG/Yxc102Xyzfg7+XjedHp3Jlh9OPXRONXEgz6HGPuZXkm10Hslaa8WM75ps9TwPj4JZvzbnngndgnbsJy71CzMzST0bDrL+YlqwT5e+GX16CFR+ZMXKDn4Vudx9fbPfoBvN9HzLjxyqKTm4FPCq+q5nhuvD/zB6vW743n9fDx9xs3ma8255foWlPuPL140uNXPsxvHOJmek6fl7V168oNYsBH9lnFnKO7w5Bcaf/4BWlkLMBsleblsriQ6YrOrFvnb430QBkrTYLRzfvf2brMVaWmTUR67L38AUgoUyIUzxxRRuu7xZPQpgfHtaTZwJe0T6aK9pHU1ZpZ8n2Q7w+L50JX6zGz9PKoDZusECrcD2fYEgaZG5gWoaOZJsWo5o2sL8QkgaZpTcWvQyJl5hFag9tN4v4rp0EKEi7Dfr/xWyFVROrDaynWdfPwxsGPQVtrjTdnqX5ZmHg4lyoLDVj1Ya+CN3uOnmmbVCsaZmbeKVpMbvu0+PHt86G7x8x3aU2b7PGHUBAjAmCIc3Me5QXQXmh+ffIPti/CbRzxqxPiOmK3fwdtBwCg56uef9WmXjg3rSG9Lmw+BWzBuFRsWnQepgZQhDRuuY6zNsF6XNg24+w8yfz312rIRem7LWQUCbEKZRSJEWefl9JL5uV/smRpDULZdy7v3Lvpyv56PZu9ZpgUFeZ+SVs2VfAwNYS+hokpcwkAXcw8AnYtdisd7bpW7MwrdXTdIf2+sO5WxsOIKaj2Vu1Ppr1hstegO//BAv/CR3Gwg9/Md3C4a3gpqnQrA/krDdj+vYshb3LYPNMs3m8p79p6fP0M995q6EQ3cGUJSjehMKlb5vu5bf6mHF3/R42rZtZK02LWtZqOLjVtAx2usmMcasuTOfuNDtPhCSaMtU3xB3eS3DeWihuX/NeslpD7g4T6pv2PL7Y8blWUWr2lvXwrf/kE4cDts8124Ml9DY7UNRnf+D6qCiFDd+YsL9/AwREm4klzfqaSThbZsDcZ80tINqMr/QNNYHcJ9TU0Y6f4OAW83rBCaab/Vz+d3+WlD5xsEwDkZaWppcvX35e32PBggX079//vL6HODPuVje5ReVc+/YS9h0u5fPxPUiNOze7A2itmbIyk6enb6CwrJLPxnenV4vTtGC4mLvVizjupLrJ320Cib0Sut5hFqZ1h224jtIapt4Laz4zrWLKamZl9rj33P2xL8413bVL3wZ72fHHfcPMBI3Q5rB1lgkqXkGQejW0H2u6P7fPNQEgd8fx58V2Mdt3JV9x+nX2yo6YMLzmc7NwMc6/vyGJ5jViO5vwmLPezGzNXGFaGsGEw9Hvnr61NXsNeAVCaGLN5xzea7qV9601wTJ3JxRkHi9LYj+z20Xr4TWvnQemG3rVx2ZyR/4uQJnX8Akx+8y2v850MZ/puoNlR8zn2bcOstea8h7YbGYVR6ZAr99DuzFV/5soyDJjInf/auq5JNdMzCnONV2VTbubltKWQyAsqcYwfa5/nymlVmita91AWUJZDeQPjPtyx7rJPlzCmDeXUFJh58u7e5IU6X/a83cdKuLrFXspszsYnhpDu9hA1Am/HPKKynl86jpmrttHt2ahZB0uwd/Lxne/74PN6p5rPrtjvQijSt0UZJs/uDW10LhaRQl8caMZVzb42drHjp2p/D2m5SWkmQljQXHH/0g7HGZyw6pPzDZclSXmcQ8/MyatxUATYHYthsWvmske4cnQ54+me7gkzwSCo8Fg968mkFUUmxDWYSxrD3nSPkqZ8JW5Cgr2mvdQFhM8YjubsFZ8COb+zcyevf7zqt3Lxbnw/Z9h3ZfmfnBTU4bm/U0ZC7LMVl1bZpqQA+AbbsJnaHMT4kISTWBfOREO7zYBtcP1prWxshRKDx+/5WyATdPBXg4JfaDr7ea8jF9g7ZfmfSqKIaip6TZP7Gtas/wja66L4lzYvcR8n7sWmUCmnbtX+EVCdHuzxEuzPua7P8/dyxLK6kFC2cXNXesm42ARY95agodV8dcr29I8wp/4UJ9jszNLK+zM3pjDF7/tZlH6ISwKrBZFhV3TPMKPqzrEclXHGHbnFvPwV2vIKy7nwcHJ3NWvObM27OPeT1fy95HtuLHHWc7cO0/ctV6E1M1ZKz1sxrUFRJnWn1NbkOyVpiv4l5dMK1d1vIOgnbPFLb4bKFW1Xo7sMy1ZEa1Nd+yJNk6HKeNNt+wNk4+vibd1Fkz/AxQfNBMx/CJMV2LGz6bcRymLKXvy5WbMVXjL6svpcMCOeWbSx5aZpmWqus/SfqxZWy6yddXjZYXmueunmKBWfsQ8HtHahCqrpwlhxYect4MmEIJZYiWuq1l6Ja6rCWMBTaov63nkqlAmY8qEOEeahfvx8R3duP7dX7nr4xUAWBTEBPsQH+LLpn0F5BdXEBfiw0ODWzEmLQ4fDyvfr9/HtNWZvDx3Ky/9uBWAlpH+vH9r12MbpV/ergndEkP5z5ytjOgQQ5CPe8wUEuKi4B0E7a+p+bjVBqljTOjaPtcEDJ9Q53gm579+EbXP8AtoUnMASbnSHPt8LPxvsOnK3DDFtORFpsC4L8zYOYBu482EiuzVZhkT3zCzg8XpJnAcZbGYcWFJl8KRHMhZZ7pwvYOcM36DnN3Kp2mp8vKH9team73StHplLDRdtqs/N8899t2EmYDZ+WbT6hbb+fTdpo2chDIhzqE20YH88ueBbMs5QsahIjIOFrPrUBEZh4rpkxTOdV3j6d0iHMsJi89e360p13drSlZ+Cd+tzaLSobm9d+JJ+3UqpXhqeAojXv+FV+du48nhKa74eEKI01HKhJnzJb4b3DEHPh1jljdRFujzIPR/tGqQsVid49S6nPn7BUSd/XhDqw3iuphbnwkys7UWEsqEOMf8vWx0ahpCp6YhtZ98gphgH+7qV/M2Pe1ig7guLZ6PFmcwrntTWkScftzauVBhd2B36CobugshXCSshQlmP//btMzF1doj5l4kkJ2We44YFkJU66EhyXh7WHluxqY6nZ99uIT3f9nJln1H6v1eP27Moe+L87n27SXYHacfe7p6Tz6TNpfTEMeoCtHg+IXD0BcaXiATtZKWMiEakIgAL34/MIkXvt/MT1sPcEmriGrP25RdwLsLdzB9jekOtVoUN/VIYMLgVrWORztYWMYz0zfw3dpsooO8Wbv3MF/8todx3avfq7Gs0s6EL1az82AFK3fn0SXBTWf0CSGEm5NQJkQDc2vvZny2bDdPT1vP9d2aEujjQZCPB4HeHpRW2Jn46y4Wbj2Ar6eVG3skMKZLHJN+283EJRl8uyaLPw9tzZgucSeNa4Pj66L9bcZGisvsPDS4FXdf0oIb31vKv2dvYXiHaAK9qwa6dxfuYOfBIiwKpq7KklAmhBBnSEKZEA2Ml83K365qx+8+WcEL32+ucjzc34uHh7Tixh4JBPuahRX/HpvK2K5NeXr6Bh75ei2fLt1FWrNQDpdUUFBSQUFpBfuPlLHjQBFdEkJ48erUY7saPDXCTDB4be42Hr/i5AkGe3KLeX1+Ope3a8KhgweYsS6bp0akVNmeSgghRO0klAnRAPVrFcH6v15GcbndBKvSCg4XV1Bud9C1WWi1A/PbxQYx+Z6efLMqk3/O2sKkZbtNC5uzla15uB+39U7khm5NT2pFaxcbxDVd4vhwcQbjuieQGO537Niz321EoXhyeAqTZi1i2b4yftl2kAGtT7NIpBBCiGpJKBOigVJK4edlw8/LRgx12+haKcXoznGM7ly/FdIfviyZGWuzeW7GRt67pSsA8zbnMGdjDn8e2pqYYB/aR1gJ8vFg6upMCWVCCHEGpI9BCFGryABv7huYxI+b9vPztgOUVth5evoGkiL9uaOP2WfPZlEMS41m9oYcisqqWQVcCCHEaUkoE0LUye29E4kP9eFv323kv/PT2ZNbwrNXtsXTdvzXyMiOMZRU2JmzMceFJRVCiIZJQpkQok68Paw8PqwNW3MKeW1eOiM6xNAr6eRtW7o2CyUmyJtpqzNdVEohhGi4JJQJIerssrZN6NUiDH8vG09c0abKcYtFMaJjDAu3HeRQYZkLSiiEEA2XhDIhRJ0ppfjfLV2ZPaEfUYHe1Z4zsmMsdodmxrrsKsdW7MrlureXsHj7wfNdVCGEaHAklAkh6sXH00pMcM2zPdtEB5IcFcDUVSd3Yc7ZmMO4d5eyLCOXW95fVuW4EEJc7CSUCSHOuas6xbBydz67DxUD8NnS3dz98XJaNwlg3kP96ZIQwh+/WM0bC9Jlv0whhHByeShTSsUrpeYrpTYqpTYopR5wdZmEEGfnyg4xAExbncnLP27lsW/W0a9VBJ/f1YPEcD8+ur0bV3aI4f9+2MKT09ZTaXcAUFJu5/t12dz32UraPzOLh79aQ3mlw5UfRQghLhh3WDy2EnhIa71SKRUArFBKzdFab3R1wYQQZyYuxJeuzUJ4bV465XYHY7rE8cLo1GPbL3nZrLx8XUdign1466ft7MktIcDbxrzN+ykutxPm50m3xDAmr9hLZl4Jb93UpdaN1IUQoqFzeSjTWmcD2c6fjyilNgGxgIQyIRqwa7rE81tGHvcPSOKhIa1Q6uQN0C0WxaOXtyYm2Jtnpm8gxNeTUZ1iuSI1mm6JodisFr5ZtZdHJq/lmrcW88Ft3Yg9zVi22hwsLGN95mH6J8tuA0II96TcaTyHUqoZsBBop7UuOOXYXcBdAFFRUV0mTZp0XstSWFiIv7//eX0PcWakbtzTqfWiteZAiSbSt/ZREsUVGi8rWC2qyrGNh+y8tqoUL6tiQhcvEgKr7utZl9d/fmkJews1V7XwYFRLz3q/RkMm14x7knpxX+e6bgYMGLBCa51W23luE8qUUv7AT8BzWusppzs3LS1NL1++/LyWZ8GCBfTv3/+8voc4M1I37ul81suWfUe49YNlFJRU8N8bOtertau80sFtHy5j6Y5ceieF89PWAzwwqCUTBreqVxkKSisI8LJVafFrCOSacU9SL+7rXNeNUqpOoczlA/0BlFIewNfAp7UFMiHExSe5SQDf3NubpmF+3PHRcj5eklGn52mteeybdSxKP8Q/rm7PB7d25Zoucbwydxsvzdla5/efuiqTTs/O4fGp63E43ON/ZIUQjY/LQ5ky/9v5P2CT1vo/ri6PEMI9NQny5qt7etK/VQRPTtvAM9M3HJu1WZNX56YzecVe/nhpS8Z0icNiUbx4dft6BbMvl+9hwperaRLozWdLd/P41HUSzIQQ54XLQxnQG7gJGKiUWu28DXN1oYQQ7sffy8Y7N6dxZ59EPlycwZ0Tl3OktKLac79esZeXftzK1Z3jeGBQy2OPnxrM/jVrS43h7rOlu3lk8lr6JIUz96FLuH9AEp8v28Nj30gwE0Kce+4w+/IXoOEN0hBCuITVonhieArNI/x5atp6rn5zMY8Na0N5pYOC0koKSio4WFjGuz/voFeLMF4YnVrtzM8Xr26PUvD6/HS+XrmXcd2aMrZbUyICvAD4aHEGT0/fwMDWkbxxQ2e8PazOWaTw2rx0AJ4flYqlmskJJ8otKue+T1eSV1zOPZe0YHj7aGxWd/j/YSGEu3F5KBNCiDMxrntTEsJ8+d0nK7j1g9+qHO/UNJg3b+yCp636AHQ0mA1OacLEJRn8e85WXp23jWGp0cQG+/DGgu0MSYni9XGdj72GUooHB7dCAa/OS0dreGF0zcFs58EibvtgGVmHS4kP8eGPX6zmP3O28rv+LRjdORYvW/1nkgohGi8JZUKIBqt3Ujg/PngJ6fsLCfTxINDbg0AfG/5etjq1RimlGJwSxeCUKLYfKOTjJbuYvGIvhWWVXJEazctjOx5b8PbE50wY3AqU4tW529iSc4Q/DEpiQHLkSS1yy3bmctfHy7Eoxefje9ApPpg5m3L47/x0/jJlHa/8uI2beibQJymctjGB1Za3sKySlbvy2JpzBE+bBW8PKz4eVnw9rYT6edIxPrhBzgYVQlRPQpkQokGLDPQmMtD7rF+nRYQ/z1zZlocvS2bFrjx6twirMdgdbTGLC/HhlR+3cfuHy0mJDuT3A5O4rG0Tvl2bxZ++WktcqA8f3NqVhDA/AC5r24QhKVH8kn6Q1+el889ZW/jnrC0EeNnolhhKzxZhxAT7sHJXHssyctmQVYD9NGPXbu3VjKdHpEgwE6KRkFAmhBAn8PeycUmriDqde21aPKM6xTJtdRZvzE/nd5+uJC7Eh715JXRPDOXtm7oQ7HvyQrVKKfq2jKBvywj2Hynl1x25LNl+iF93HGLu5v0AeNosdIwP5t7+LeiWGEq7mCAcWlNSYaek3E5JhZ0pKzP5cHEGgAQzIRoJCWVCCHEWPKwWxnSJY1SnWGauy+bdn3fQu0U4z45sW+uYscgAb67sEHNsA/d9h0vJPlxCSkxgrc9NjQ3CohTvL9qJUvDUcAlmQjR0EsqEEOIcsFoUIzrEMMIZsM5EkyBvmgTVrStWKcWTw9ug0XywKAOFuS/BTIiGS0KZEEI0UEopnhqeAsD7i3YCcFPPBMorHZRV2imvdFBe6WDTITsBu/LwtFrwtJlbaYWdgpIKDjtvBaWVNI/wo3+rCAl2QriIhDIhhGjAjgYzrU0wOxrOqvhtcZ1er1PTYP50WTK9WoRXObbvcCnfrMokfX8hbWMC6dQ0uE5drUKIupFQJoQQDZxSiqdHpNC3ZTgFpRV4Wq142Sx4eVjwsFpYtWo1Kantj7WcldvteNusBPp4EORcSsTf28bsDft4Ze42xr27lN5JYTw8JJnWTQKZvXEfk1fsZVH6QRwawvw8+XrlXgA8rRZSYgJpHxdETLAPUYFeRAV4ExnoRYivJ0VldvKKy8ktLie/uJwjpZU0C/MjNTaIED/PWj6ZEBcXCWVCCNEIKKUY1Caq2mOlu611mlE6tltTRnaK5dOlu3ljfjqj3liMj4eVkgo7scE+3DcgidGd40gM92Pf4VJW78lj1e58Vu3OZ8rKTArLKutV5qahvqTGBdEhLoiRnWKJDDj7pU2EaMgklAkhhDjG28PKHX0SGds1ng8XZ5CZX8Lw9tH0SAw7aeeCJkHeDA2KZmi76GOPFZZVsr+glJyCMvYfKSW3qJwAbw9CfD0I9vUkxNcDfy8b6QcKWbv3MOv2HmbNnnxmrM3mzQXb+b8xHRicUn2wdDdaa2ZvzMFmUfRPjsRay3ZbQtSFhDIhhBBV+HnZuG9AUr2e4+9lwz/Cn+YR/qc9LzLQ+6Qxa+n7j/DApNWMn7icG3s05fFhKfh4uu84tdIKO49NWceUVZkAxIf6cFOPBK5Ni6+yLp0Q9SGhTAghhEslRQYw5d5e/Hv2Vt5ZuIOlO3J59fpOtIkOpKC0go1ZBazPPMyGrAKsFkVaQghpzUJoEeF/wWeKZuWXcM8nK1i79zATLm1Fyyh/PlycwfMzN/OfOVsZ1SmW8X2b1xpMhaiOhDIhhBAu52Wz8tiwNvRtGc6DX67hqtcXERPsTcah4mPnNAn0ptzuYPIKM8kg2NeDtIQQUqIDCQ/wIszPizB/T8L9PQn29cTHw4q3h/WkrsXSCjvZh0vJzi8hM7+EA4VlVFRq7A4Hdq2xO8x5raL8SUsIJT7U51jwW7Yzl3s/XUFphYN3b0471tU6LDWajVkFfLQ4gykrM5myMpPHhrXh5p4JsryIqBcJZUIIIdxG35YR/PBAX/7x/WYKSisY0yWOdrFBtI0JIiLAC601Ow8WsTwjj+W7clmekcfczfvRNW8RiqfNgo+HFaUgv7iixvOsFoXVotBaU2E3Lxju70WXhGBign34eMku4kN9mXRXF5IiA056bkpMIC+Oac9Dl7XikclreXr6BhZs2c//jelARIDXOfluROMnoUwIIYRbCfP34p/XdKj2mFKK5s5xa9d2jQfA7tDkFZdzqLCcQ4VlHCwyy2+UlNsprXBQUmGntMJOpcNBk0BvooN8iA72JibIh6hAbzxtFiyKY61adodm2/4jrNiVx4pdeazclcesDTkMbB3JS9d1JMjHo8ayRwZ488GtXZm4ZBfPzdzE0JcX8s9r2jOwdcOYwCBc3p+crwAADflJREFUS0KZEEKIBs1qUYT7exHu7wUE1Hp+XV6vdZNAWjcJ5IbuCQAUlVXi51W3P5lKKW7p1YyeLcL4w+eruP3D5fRsHobVoo7ttFBW6aC0uIT22atoEeFPUqQ/LSL9aBbmh7eH+05yEOeXhDIhhBCiFnUNZCdqFRXAtPt789KcbSxKP4inzYKXzYKflw1Pq4WsymJW7s5j+pqsY8+xKIgN8aF5uD/NI/xoHuFPiwg/WjcJJFQW2230JJQJIYQQ54mXzcqjl7eu9tiCBQvo378/JeV2dhwsJH1/IdsPFLHjQCE7DhSxbGcuJRX2Y+dHBXrRJjqQNtGBtIryx+GAI6UVFJZVcqS0kuJyO8PbR9O9edhZl7us0s78zfs5VFTO1Z3jpPXuApFQJoQQQriQj6eVtjFmMsOJHA7NvoJS0vcXsnlfAZuyj7Apu4Bfth2k0nHyzAYvmwWrRfHJ0l3c0TuRhy9LrjZI5RSU8s7CHeQWldMhLoiOTc3sVU+bBa01a/ce5uuVe5m+JuvYpIi3ftrOU8PbcmmbyBpnk2qtOVxSQWZ+CZl5JWTll5BbXMFVHWNoIcuD1JmEMiGEEMINWSyKmGAfYoJ96HfCNllllXZ2HyrGw2ohwNtGgLcHnjYLRWWVvPD9Jt77ZSc/bT3AS9d1pF2sCXqHCst466ftTFyyC7tDE+LnyTfOxW+P7l9aVFbJtv2FeNosDEmJ4uoucXhaLTwzfQPjJy6nf3IET49oS2K4HwC7DxXz09b9/LT1AEt35HKkmm223lqwnbsvac59A5JqbG3bk1vM1pwjZOWXkJlfSla+CXVHSiudy5RoKh0OHA4I8vGgc0IwaQmhdEkIIS7k+JIl+cXlx1oay+2OBtnCJ6FMCCGEaEC8bFZaRlWd0ODnZePvI1O5tE0Uj0xey8j/LuL3A1tS6XDw/i87KamwM6pTHA8Makl8qA/Zh0tZvSef1XvyWbU7jxA/T54flcoV7aNPmmE684G+TFyyi5fnbOWylxYypG0UG7MK2HGwCDB7mI7oGEPzcD9inSEyNsQHh9Y8P2MTr81LZ9rqLP56VVsGJEcCsP9IKd+tyWbamizW7Mk/9l4eVkV0kA8xwd4khPniYTUtgDaLwmJR5BSUMnVVFp/8uhswXboxwT7sOlRMblH5Sd/HOwt38PyoVHonhXMqrTULth7gjfnp/H1kKslNzn6CyLkgoUwIIYRoRPonRzJ7Qj+enLaBl37cCsAV7aOZcGkrkiKPdyUebYUblhpd00sB4GG1cEefREZ0iOYf32/mx405dE4I4aaeCfRPjqRZmG+N3Zovj+3EtWnxPDFtPbd98BuDU6IorbCzKP0gDg1tYwJ5bFhr0pqFEhfsQ7i/10l7rFbH7tBs3lfAil15LM/II6eglCEpUWZiRLg/LSL9ycov4Ymp67nhvaVc3TmOx69oQ6ifJw6H2bP09fnbWJ9ZQEyQNzkFpRLKhBBCCHF+BPt68tr1nbipRwKBPjZaNwk869eMDPDmP9d2rPfzeiWF8/0DfXnv5528Nm8bUYHe3D8giSs7xlRZhLcurBZ1bAzezT2bVXtOYrgf3z/Ql9fnpfPWT9uZv2U/N/dMYOa6bLbmFJIQ5suLV6cyqlMcnjZLvctwvkgoE0IIIRqpbomhri4CYLpc7xuQxN39mmO1qAuy/ZS3h5WHL0tmRIcYHp2ylpd/3EbLSH9eGduRK1KjsVndJ4wdJaFMCCGEEBeEK4JQcpMAvr6nFzsPFZEY5ldr96grSSgTQgghRKNmsagGsTSH+7XdCSGEEEJchCSUCSGEEEK4AQllQgghhBBuQEKZEEIIIYQbkFAmhBBCCOEGJJQJIYQQQrgBtwhlSqmhSqktSql0pdSjri6PEEIIIcSF5vJQppSyAv8FLgdSgOuVUimuLZUQQgghxIXl8lAGdAPStdY7tNblwCTgKheXSQghhBDiglJaa9cWQKkxwFCt9Z3O+zcB3bXW959y3l3AXQBRUVFdJk2adF7LVVhYiL+/+6/+ezGSunFPUi/uS+rGPUm9uK9zXTcDBgxYobVOq+28BrPNktb6HeAdgLS0NN2/f//z+n4LFizgfL+HODNSN+5J6sV9Sd24J6kX9+WqunGH7stMIP6E+3HOx4QQQgghLhruEMp+A1oqpRKVUp7AWGC6i8skhBBCCHFBubz7UmtdqZS6H5gFWIH3tdYbXFwsIYQQQogLyuWhDEBrPROY6epyCCGEEEK4istnX54JpdQBYNd5fptw4OB5fg9xZqRu3JPUi/uSunFPUi/u61zXTYLWOqK2kxpkKLsQlFLL6zJ9VVx4UjfuSerFfUnduCepF/flqrpxh4H+QgghhBAXPQllQgghhBBuQEJZzd5xdQFEjaRu3JPUi/uSunFPUi/uyyV1I2PKhBBCCCHcgLSUCSGEEEK4AQllQgghhBBuQEJZNZRSQ5VSW5RS6UqpR11dnouVUipeKTVfKbVRKbVBKfWA8/FQpdQcpdQ2578hri7rxUgpZVVKrVJKfee8n6iUWuq8br5wbpsmLjClVLBSarJSarNSapNSqqdcM+5BKTXB+btsvVLqc6WUt1w3F55S6n2l1H6l1PoTHqv2GlHGq876WauU6nw+yyah7BRKKSvwX+ByIAW4XimV4tpSXbQqgYe01ilAD+A+Z108CszVWrcE5jrviwvvAWDTCfdfBF7SWicBecAdLimVeAX4QWvdGuiAqSO5ZlxMKRUL/AFI01q3w2wrOBa5blzhQ2DoKY/VdI1cDrR03u4C3jyfBZNQVlU3IF1rvUNrXQ5MAq5ycZkuSlrrbK31SufPRzB/XGIx9fGR87SPgJGuKeHFSykVB1wBvOe8r4CBwGTnKVIvLqCUCgL6Af8D0FqXa63zkWvGXdgAH6WUDfAFspHr5oLTWi8Eck95uKZr5CpgojZ+BYKVUtHnq2wSyqqKBfaccH+v8zHhQkqpZkAnYCkQpbXOdh7aB0S5qFgXs5eBRwCH834YkK+1rnTel+vGNRKBA8AHzq7l95RSfsg143Ja60zgX8BuTBg7DKxArht3UdM1ckEzgYQy4faUUv7A18AftdYFJx7TZk0XWdflAlJKDQf2a61XuLosogob0Bl4U2vdCSjilK5KuWZcwzlG6SpMcI4B/KjahSbcgCuvEQllVWUC8Sfcj3M+JlxAKeWBCWSfaq2nOB/OOdp87Px3v6vKd5HqDVyplMrAdO8PxIxjCnZ2y4BcN66yF9irtV7qvD8ZE9LkmnG9S4GdWusDWusKYArmWpLrxj3UdI1c0Ewgoayq34CWzhkxnpiBmNNdXKaLknOc0v+ATVrr/5xwaDpwi/PnW4BpF7psFzOt9V+01nFa62aY62Oe1voGYD4wxnma1IsLaK33AXuUUsnOhwYBG5Frxh3sBnoopXydv9uO1o1cN+6hpmtkOnCzcxZmD+DwCd2c55ys6F8NpdQwzJgZK/C+1vo5FxfpoqSU6gP8DKzj+NilxzDjyr4EmgK7gGu11qcO2hQXgFKqP/Cw1nq4Uqo5puUsFFgF3Ki1LnNl+S5GSqmOmAkYnsAO4DbM/4DLNeNiSqm/AtdhZpavAu7EjE+S6+YCUkp9DvQHwoEc4GlgKtVcI84A/Tqmq7kYuE1rvfy8lU1CmRBCCCGE60n3pRBCCCGEG5BQJoQQQgjhBiSUCSGEEEK4AQllQgghhBBuQEKZEEIIIYQbkFAmhBD1oJTSSqkxtZ8phBD1I6FMCNFgKKU+dIaiU2+/urpsQghxtmy1nyKEEG7lR+CmUx4rd0VBhBDiXJKWMiFEQ1Omtd53yi0XjnUt3q+UmqGUKlZK7VJK3Xjik5VSqUqpH5VSJUqpXGfrW9Ap59yilFqnlCpTSuUopT46pQyhSqmvlFJFSqkd1bzHU873LlNK7VNKTTwv34QQolGRUCaEaGz+itmvriPwDjBRKZUGoJTyA2YBhUA3YBTQC3j/6JOVUncDbwMfAO2BYcD6U97jKczeeB2AL4D3lVJNnc+/GngYuBdoCQwHlp2HzymEaGRkmyUhRIOhlPoQuBEoPeXQf7XWf1ZKaeA9rfX4E57zI7BPa32jUmo88C8gTmt9xHm8P2ZT6JZa63Sl1F7gE631ozWUQQP/0Fr/xXnfBhQAd2mtP1FKPQjcDbTTWlecsw8vhGj0ZEyZEKKhWQjcdcpj+Sf8vOSUY0uAK5w/twHWHg1kTosxG96nKKUKMBtEz62lDGuP/qC1rlRKHQAinQ99BTwA7FRKzQJ+AKbLJtNCiNpI96UQoqEp1lqnn3I7eA5etz7dBqe2gGmcv0+11nuAZExrWQHwb2CFs+tUCCFqJKFMCNHY9Kjm/ibnz5uAVKVUwAnHe2F+F27SWu8HMoFBZ1MArXWp1nqG1noC0BVoC/Q+m9cUQjR+0n0phGhovJRSTU55zK61PuD8ebRS6jdgATAGE7C6O499ipkIMFEp9RQQghnUP0Vrne485zngJaVUDjAD8AUGaa3/XZfCKaVuxfxuXYqZUHAdpmVtWz0/pxDiIiOhTAjR0FwKZJ/yWCYQ5/z5GeBq4FXgAHCb1vo3AK11sVLqMuBlzIzIUswsygeOvpDW+k2lVDnwEPAikAvMrEf58oE/YyYUeAAbgdFa6531eA0hxEVIZl8KIRoN58zIa7TWk11dFiGEqC8ZUyaEEEII4QYklAkhhBBCuAHpvhRCCCGEcAPSUiaEEEII4QYklAkhhBBCuAEJZUIIIYQQbkBCmRBCCCGEG5BQJoQQQgjhBv4fL8vvtJ+tYmoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x504 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualize the training process\n",
    "train_loss = history.history['loss']\n",
    "val_loss = history.history['val_loss']\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(10, 7))\n",
    "ax.plot(train_loss, label='Training Loss')\n",
    "ax.plot(val_loss, label='Validation Loss')\n",
    "ax.set_title('Loss vs. Epochs', fontsize=16)\n",
    "ax.set_xlabel('Epochs', fontsize=14)\n",
    "ax.set_ylabel('Loss', fontsize=14)\n",
    "ax.legend(fontsize=14)\n",
    "ax.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract Features using Triplet Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t\t (113783, 4)\n",
      "Validation:\t (22255, 4)\n",
      "Test:\t\t (22391, 4)\n",
      "\n",
      "Train Landmarks:\t 14943\n",
      "Validation Landmarks:\t 7674\n",
      "Test Landmarks:\t\t 14436\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv('./data/triplet/train.csv')\n",
    "val_df = pd.read_csv('./data/triplet/validation.csv')\n",
    "test_df = pd.read_csv('./data/triplet/test.csv')\n",
    "\n",
    "print('Train:\\t\\t', train_df.shape)\n",
    "print('Validation:\\t', val_df.shape)\n",
    "print('Test:\\t\\t', test_df.shape)\n",
    "\n",
    "print('\\nTrain Landmarks:\\t', len(train_df['landmark_id'].unique()))\n",
    "print('Validation Landmarks:\\t', len(val_df['landmark_id'].unique()))\n",
    "print('Test Landmarks:\\t\\t', len(test_df['landmark_id'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "vgg16 (Model)                (None, 512)               14714688  \n",
      "_________________________________________________________________\n",
      "l2_norm (Lambda)             (None, 512)               0         \n",
      "=================================================================\n",
      "Total params: 14,714,688\n",
      "Trainable params: 7,079,424\n",
      "Non-trainable params: 7,635,264\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Load trained model\n",
    "base_model = load_model('./models/vgg16-base-0.4-model.h5')\n",
    "base_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define train_imgs and test_imgs\n",
    "train_imgs = np.zeros(shape=(len(train_df), 512), dtype=np.float32)\n",
    "val_imgs = np.zeros(shape=(len(val_df), 512), dtype=np.float32)\n",
    "test_imgs = np.zeros(shape=(len(test_df), 512), dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Process:          0\n",
      "================================================================================\n",
      "Process:      20000\n",
      "================================================================================\n",
      "Process:      40000\n",
      "================================================================================\n",
      "Process:      60000\n",
      "================================================================================\n",
      "Process:      80000\n",
      "================================================================================\n",
      "Process:     100000\n",
      "========================================================"
     ]
    }
   ],
   "source": [
    "# Process training images\n",
    "img_ids = train_df['image_id'].values\n",
    "steps = 20000\n",
    "for i in range(0, len(train_df), steps):\n",
    "    tmp_imgs = []\n",
    "    print('\\nProcess: {:10d}'.format(i))\n",
    "    \n",
    "    start = i\n",
    "    end = min(len(train_df), i + steps)\n",
    "    for idx in range(start, end):\n",
    "        if idx % 250 == 0:\n",
    "            print('=', end='')\n",
    "            \n",
    "        img_id = img_ids[idx]\n",
    "        path = './data/triplet/train/' + str(img_id) + '.jpg'\n",
    "        img = load_img(path, target_size=img_size[:2])\n",
    "        img = img_to_array(img)\n",
    "        tmp_imgs.append(img)\n",
    "        \n",
    "    tmp_imgs = np.array(tmp_imgs, dtype=np.float32) / 255.0\n",
    "    tmp_prediction = base_model.predict(tmp_imgs)\n",
    "    train_imgs[start: end, ] = tmp_prediction\n",
    "    _ = gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Process:          0\n",
      "================================================================================\n",
      "Process:       4000\n",
      "================================================================================\n",
      "Process:       8000\n",
      "================================================================================\n",
      "Process:      12000\n",
      "================================================================================\n",
      "Process:      16000\n",
      "================================================================================\n",
      "Process:      20000\n",
      "=============================================="
     ]
    }
   ],
   "source": [
    "# Process validation images\n",
    "img_ids = val_df['image_id'].values\n",
    "steps = 4000\n",
    "for i in range(0, len(val_df), steps):\n",
    "    tmp_imgs = []\n",
    "    print('\\nProcess: {:10d}'.format(i))\n",
    "    \n",
    "    start = i\n",
    "    end = min(len(val_df), i + steps)\n",
    "    for idx in range(start, end):\n",
    "        if idx % 50 == 0:\n",
    "            print('=', end='')\n",
    "            \n",
    "        img_id = img_ids[idx]\n",
    "        path = './data/triplet/validation/' + str(img_id) + '.jpg'\n",
    "        img = load_img(path, target_size=img_size[:2])\n",
    "        img = img_to_array(img)\n",
    "        tmp_imgs.append(img)\n",
    "        \n",
    "    tmp_imgs = np.array(tmp_imgs, dtype=np.float32) / 255.0\n",
    "    tmp_prediction = base_model.predict(tmp_imgs)\n",
    "    val_imgs[start: end, ] = tmp_prediction\n",
    "    _ = gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Process:          0\n",
      "================================================================================\n",
      "Process:       4000\n",
      "================================================================================\n",
      "Process:       8000\n",
      "================================================================================\n",
      "Process:      12000\n",
      "================================================================================\n",
      "Process:      16000\n",
      "================================================================================\n",
      "Process:      20000\n",
      "================================================"
     ]
    }
   ],
   "source": [
    "# Process test images\n",
    "img_ids = test_df['image_id'].values\n",
    "steps = 4000\n",
    "for i in range(0, len(test_df), steps):\n",
    "    tmp_imgs = []\n",
    "    print('\\nProcess: {:10d}'.format(i))\n",
    "    \n",
    "    start = i\n",
    "    end = min(len(test_df), i + steps)\n",
    "    for idx in range(start, end):\n",
    "        if idx % 50 == 0:\n",
    "            print('=', end='')\n",
    "            \n",
    "        img_id = img_ids[idx]\n",
    "        path = './data/triplet/test/' + str(img_id) + '.jpg'\n",
    "        img = load_img(path, target_size=img_size[:2])\n",
    "        img = img_to_array(img)\n",
    "        tmp_imgs.append(img)\n",
    "        \n",
    "    tmp_imgs = np.array(tmp_imgs, dtype=np.float32) / 255.0\n",
    "    tmp_prediction = base_model.predict(tmp_imgs)\n",
    "    test_imgs[start: end, ] = tmp_prediction\n",
    "    _ = gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t\t (113783, 512)\n",
      "Validation:\t (22255, 512)\n",
      "Test:\t\t (22391, 512)\n"
     ]
    }
   ],
   "source": [
    "print('Train:\\t\\t', train_imgs.shape)\n",
    "print('Validation:\\t', val_imgs.shape)\n",
    "print('Test:\\t\\t', test_imgs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save to disk\n",
    "np.save('./data/triplet/train-triplet-vgg16-0.4-features.npy', train_imgs)\n",
    "np.save('./data/triplet/validation-triplet-vgg16-0.4-features.npy', val_imgs)\n",
    "np.save('./data/triplet/test-triplet-vgg16-0.4-features.npy', test_imgs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Features and Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t\t (113783, 512) (113783, 4)\n",
      "Validation:\t (22255, 512) (22255, 4)\n",
      "Test:\t\t (22391, 512) (22391, 4)\n"
     ]
    }
   ],
   "source": [
    "# Already normalized\n",
    "train_feature = np.load('./data/triplet/train-triplet-vgg16-0.4-features.npy')\n",
    "val_feature = np.load('./data/triplet/validation-triplet-vgg16-0.4-features.npy')\n",
    "test_feature = np.load('./data/triplet/test-triplet-vgg16-0.4-features.npy')\n",
    "\n",
    "train_df = pd.read_csv('./data/triplet/train.csv')\n",
    "val_df = pd.read_csv('./data/triplet/validation.csv')\n",
    "test_df = pd.read_csv('./data/triplet/test.csv')\n",
    "\n",
    "print('Train:\\t\\t', train_feature.shape, train_df.shape)\n",
    "print('Validation:\\t', val_feature.shape, val_df.shape)\n",
    "print('Test:\\t\\t', test_feature.shape, test_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper function\n",
    "def accuracy(true_label, prediction, top=1):\n",
    "    \"\"\" function to calculate the prediction accuracy \"\"\"\n",
    "    prediction = prediction[:, :top]\n",
    "    count = 0\n",
    "    for i in range(len(true_label)):\n",
    "        if true_label[i] in prediction[i]:\n",
    "            count += 1\n",
    "            \n",
    "    return count / len(true_label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implement KNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Merge train and validation features\n",
    "train_val_feature = np.concatenate((train_feature, val_feature), axis=0)\n",
    "train_val_df = pd.concat((train_df, val_df), axis=0)\n",
    "train_val_df = train_val_df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NearestNeighbors(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "         metric_params=None, n_jobs=-1, n_neighbors=50, p=2, radius=1.0)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Implement KNN model\n",
    "knn = NearestNeighbors(n_neighbors=50, algorithm='auto', leaf_size=30, \n",
    "                       metric='minkowski', p=2, n_jobs=-1)\n",
    "knn.fit(train_val_feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Search the first 50 neighbors\n",
    "distance, neighbor_index = knn.kneighbors(test_feature, return_distance=True)\n",
    "\n",
    "# Save the results\n",
    "np.save('./result/knn-triplet-vgg16-0.4-distance.npy', distance)\n",
    "np.save('./result/knn-triplet-vgg16-0.4-neighbor.npy', neighbor_index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Search Neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_distance = np.load('./result/knn-triplet-vgg16-0.4-distance.npy')\n",
    "knn_neighbor = np.load('./result/knn-triplet-vgg16-0.4-neighbor.npy')\n",
    "\n",
    "# Get the first 50 neighbors\n",
    "predictions = []\n",
    "for neighbors in knn_neighbor:\n",
    "    predictions.append(train_val_df.loc[neighbors]['landmark_id'].values)\n",
    "\n",
    "predictions = np.array(predictions)\n",
    "np.save('./result/knn-triplet-vgg16-0.4-test-prediction.npy', predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top  1 accuracy:\t 0.32285293198159976\n",
      "Top  5 accuracy:\t 0.4972980215265062\n",
      "Top 10 accuracy:\t 0.5715242731454603\n",
      "Top 20 accuracy:\t 0.644232057523112\n"
     ]
    }
   ],
   "source": [
    "print('Top  1 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=1))\n",
    "print('Top  5 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=5))\n",
    "print('Top 10 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=10))\n",
    "print('Top 20 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_acc = []\n",
    "for i in range(1, 51):\n",
    "    tmp_acc = accuracy(test_df['landmark_id'].values, predictions, top=i)\n",
    "    knn_acc.append(tmp_acc)\n",
    "\n",
    "np.save('./result/knn-triplet-vgg16-0.4-accuracy.npy', knn_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Environment (conda_tensorflow_p36)",
   "language": "python",
   "name": "conda_tensorflow_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}