MobileNet - 5-fold CV.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "#Import all required libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "\n",
    "from skimage.transform import rescale\n",
    "import skimage.color as color\n",
    "import cv2\n",
    "from PIL import Image\n",
    "from skimage.external import tifffile\n",
    "from skimage.transform import resize\n",
    "\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "from keras import backend\n",
    "from keras import backend as K\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "from keras.layers import Conv2D, MaxPool2D, MaxPooling2D,GlobalAveragePooling2D,Input\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential,Model, load_model\n",
    "from keras.optimizers import SGD, Adam\n",
    "from keras.applications import ResNet50, VGG16, MobileNet, Xception, DenseNet121\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.callbacks import TensorBoard\n",
    "from keras.backend import clear_session\n",
    "\n",
    "## Image Augmentation\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.preprocessing.image import img_to_array\n",
    "from keras.preprocessing.image import load_img\n",
    "import argparse\n",
    "\n",
    "from sklearn.metrics import fbeta_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "import cv2\n",
    "import os\n",
    "from os import listdir\n",
    "from tqdm import tqdm\n",
    "import time\n",
    "import csv\n",
    "import pickle\n",
    "\n",
    "from sklearn.model_selection import KFold\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, Callback, ReduceLROnPlateau\n",
    "from tensorflow.keras import Sequential\n",
    "from tensorflow.keras.backend import clear_session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define model parameter \n",
    "\n",
    "INPUT_SHAPE = (128, 128, 3) # Image Dimensions\n",
    "BATCH_SIZE = 128\n",
    "DROPOUT_RATE = 0.5\n",
    "EPOCHS = 2\n",
    "LR = 0.0001 # Learning Rate\n",
    "REG_STRENGTH = 0.01 # Regularization Strength\n",
    "NFOLDS = 2 # No of folds for cross validation\n",
    "WORKERS = 4 # Multithreading no of threads\n",
    "MAXQ = 10 # Max Queue size for multithreading\n",
    "THRES = [0.2] * 17 # Threshold for truth value of label, applied on sigmoid output.\n",
    "\n",
    "TRAIN_PATH = 'data/train-jpg'\n",
    "TEST_PATH = 'data/test-jpg'\n",
    "\n",
    "TRAIN_CSV_PATH = 'data/train_v2.csv'\n",
    "TEST_CSV_PATH = 'data/sample_submission_v2.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    image_name                                       tags\n",
      "0  train_0.jpg                               haze primary\n",
      "1  train_1.jpg            agriculture clear primary water\n",
      "2  train_2.jpg                              clear primary\n",
      "3  train_3.jpg                              clear primary\n",
      "4  train_4.jpg  agriculture clear habitation primary road\n",
      "\n",
      "   image_name                                        tags\n",
      "0  test_0.jpg  [primary, clear, agriculture, road, water]\n",
      "1  test_1.jpg  [primary, clear, agriculture, road, water]\n",
      "2  test_2.jpg  [primary, clear, agriculture, road, water]\n",
      "3  test_3.jpg  [primary, clear, agriculture, road, water]\n",
      "4  test_4.jpg  [primary, clear, agriculture, road, water]\n"
     ]
    }
   ],
   "source": [
    "# Construct dataframes holding training and test data information\n",
    "\n",
    "df_train = pd.read_csv(TRAIN_CSV_PATH)\n",
    "df_test = pd.read_csv(TEST_CSV_PATH)\n",
    "\n",
    "df_train['image_name'] = df_train['image_name'].astype(str) + '.jpg'\n",
    "df_test['image_name'] = df_test['image_name'].astype(str) + '.jpg'\n",
    "\n",
    "df_test['tags'] = df_test['tags'].apply(lambda x: x.split(' '))\n",
    "\n",
    "print(df_train.head())\n",
    "print()\n",
    "print(df_test.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_name</th>\n",
       "      <th>tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>test_0.jpg</td>\n",
       "      <td>[primary, clear, agriculture, road, water]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>test_1.jpg</td>\n",
       "      <td>[primary, clear, agriculture, road, water]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>test_2.jpg</td>\n",
       "      <td>[primary, clear, agriculture, road, water]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>test_3.jpg</td>\n",
       "      <td>[primary, clear, agriculture, road, water]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>test_4.jpg</td>\n",
       "      <td>[primary, clear, agriculture, road, water]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   image_name                                        tags\n",
       "0  test_0.jpg  [primary, clear, agriculture, road, water]\n",
       "1  test_1.jpg  [primary, clear, agriculture, road, water]\n",
       "2  test_2.jpg  [primary, clear, agriculture, road, water]\n",
       "3  test_3.jpg  [primary, clear, agriculture, road, water]\n",
       "4  test_4.jpg  [primary, clear, agriculture, road, water]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(61191, 2)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# These files are numpy Arrays with shape Nx1\n",
    "X_train_files = np.array(df_train['image_name'].tolist())\n",
    "X_train_files.reshape((X_train_files.shape[0], 1))\n",
    "y_train = np.array(df_train['tags'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['agriculture', 'artisinal_mine', 'bare_ground', 'blooming', 'blow_down', 'clear', 'cloudy', 'conventional_mine', 'cultivation', 'habitation', 'haze', 'partly_cloudy', 'primary', 'road', 'selective_logging', 'slash_burn', 'water']\n"
     ]
    }
   ],
   "source": [
    "# There are 17 Labels and each image is tagged with multiple labels\n",
    "# Lets print out the unique label list\n",
    "\n",
    "labels = []\n",
    "\n",
    "for tag in df_train['tags'].values:\n",
    "    labels_in_tag = tag.split(' ')\n",
    "    for label in labels_in_tag:\n",
    "        if label not in labels:\n",
    "            labels.append(label)\n",
    "        \n",
    "labels.sort()\n",
    "print(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "conventional_mine is tagged least no of times: 99\n",
      "primary is tagged max no of times: 37512\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtYAAAIqCAYAAAAAQA2kAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3X287eWc//HXpzu5me4406Sihn6ahFAKzSAjJyFMTCFNPyNGZjKYETN+EWYyMxjMyORXKXfJbSHSEBWlTko36OFIftWk4iS5Vz6/P65rddbZ7ZtT51rr+u7T6/l47Mfe61pr7fXZd2u/1/d7XZ8rMhNJkiRJa2ad3gVIkiRJawODtSRJktSAwVqSJElqwGAtSZIkNWCwliRJkhowWEuSJEkNGKwlSZKkBgzWkiRJUgMGa0mSJKkBg7UkSZLUwHq9C7iz7nOf++Q222zTuwxJkiStxS644IIfZeaS1bntog3W22yzDcuWLetdhiRJktZiEfGD1b2tU0EkSZKkBgzWkiRJUgMGa0mSJKkBg7UkSZLUgMFakiRJasBgLUmSJDVgsJYkSZIaMFhLkiRJDRisJUmSpAYM1pIkSVIDBmtJkiSpAYO1JEmS1IDBWpIkSWrAYC1JkiQ1YLCWJEmSGjBYS5IkSQ0YrCVJkqQGDNaSJElSA+v1LkCSJEnz2+awz079Ma88cu+pP+Zi5xFrSZIkqQGDtSRJktSAwVqSJElqwGAtSZIkNWCwliRJkhowWEuSJEkNGKwlSZKkBgzWkiRJUgMGa0mSJKkBg7UkSZLUgMFakiRJasBgLUmSJDVgsJYkSZIaMFhLkiRJDRisJUmSpAYM1pIkSVIDBmtJkiSpAYO1JEmS1IDBWpIkSWrAYC1JkiQ1YLCWJEmSGjBYS5IkSQ0YrCVJkqQGFgzWEbFhRJwXEd+MiMsi4g11/H0R8f2IuKi+7VTHIyLeGRHLI+LiiHjE2Oc6MCK+W98OHBt/ZERcUu/zzoiISXyxkiRJ0qSstxq3+TWwR2b+LCLWB86OiM/V6/4uMz824/Z7AdvVt12Bo4BdI2Iz4HBgZyCBCyLilMy8sd7mRcDXgVOBpcDnkCRJkhaJBY9YZ/GzenH9+pbz3GUf4IR6v3OBTSJiC+DJwOmZuaKG6dOBpfW6jTLz3MxM4ATgGWvwNUmSJElTt1pzrCNi3Yi4CLieEo6/Xq96c53u8faIuFsd2xK4auzuV9ex+cavnmVckiRJWjRWK1hn5q2ZuROwFfCoiNgReA2wPbALsBnw6olVWUXEwRGxLCKW3XDDDZN+OEmSJGm13aGuIJn5E+AMYGlmXlune/waOA54VL3ZNcDWY3fbqo7NN77VLOOzPf7RmblzZu68ZMmSO1K6JEmSNFGr0xVkSURsUj++O/Ak4Dt1bjS1g8czgEvrXU4BXlC7g+wG3JSZ1wKnAXtGxKYRsSmwJ3Bave6nEbFb/VwvAE5u+2VKkiRJk7U6XUG2AI6PiHUpQfykzPxMRHwpIpYAAVwEvKTe/lTgKcBy4BfAQQCZuSIi3gicX293RGauqB+/FHgfcHdKNxA7gkiSJGlRWTBYZ+bFwMNnGd9jjtsncMgc1x0LHDvL+DJgx4VqkSRJkobKnRclSZKkBgzWkiRJUgMGa0mSJKkBg7UkSZLUgMFakiRJasBgLUmSJDVgsJYkSZIaMFhLkiRJDRisJUmSpAYM1pIkSVIDBmtJkiSpAYO1JEmS1IDBWpIkSWrAYC1JkiQ1YLCWJEmSGjBYS5IkSQ0YrCVJkqQGDNaSJElSAwZrSZIkqQGDtSRJktSAwVqSJElqwGAtSZIkNWCwliRJkhowWEuSJEkNGKwlSZKkBgzWkiRJUgMGa0mSJKkBg7UkSZLUgMFakiRJasBgLUmSJDVgsJYkSZIaMFhLkiRJDRisJUmSpAYM1pIkSVIDBmtJkiSpAYO1JEmS1IDBWpIkSWrAYC1JkiQ1YLCWJEmSGjBYS5IkSQ0YrCVJkqQGDNaSJElSAwZrSZIkqQGDtSRJktSAwVqSJElqwGAtSZIkNbBgsI6IDSPivIj4ZkRcFhFvqOPbRsTXI2J5RHwkIjao43erl5fX67cZ+1yvqeOXR8STx8aX1rHlEXFY+y9TkiRJmqzVOWL9a2CPzHwYsBOwNCJ2A94CvD0zHwjcCLyw3v6FwI11/O31dkTEDsB+wIOBpcC7I2LdiFgX+E9gL2AHYP96W0mSJGnRWDBYZ/GzenH9+pbAHsDH6vjxwDPqx/vUy9TrnxgRUcdPzMxfZ+b3geXAo+rb8sy8IjN/A5xYbytJkiQtGqs1x7oeWb4IuB44Hfge8JPMvKXe5Gpgy/rxlsBVAPX6m4B7j4/PuM9c45IkSdKisVrBOjNvzcydgK0oR5i3n2hVc4iIgyNiWUQsu+GGG3qUIEmSJM3qDnUFycyfAGcAjwY2iYj16lVbAdfUj68Btgao128M/Hh8fMZ95hqf7fGPzsydM3PnJUuW3JHSJUmSpIlana4gSyJik/rx3YEnAd+mBOx9680OBE6uH59SL1Ov/1JmZh3fr3YN2RbYDjgPOB/YrnYZ2YCywPGUFl+cJEmSNC3rLXwTtgCOr9071gFOyszPRMS3gBMj4k3AhcAx9fbHAO+PiOXACkpQJjMvi4iTgG8BtwCHZOatABHxMuA0YF3g2My8rNlXKEmSJE3BgsE6My8GHj7L+BWU+dYzx38FPHuOz/Vm4M2zjJ8KnLoa9UqSJEmD5M6LkiRJUgMGa0mSJKkBg7UkSZLUgMFakiRJasBgLUmSJDVgsJYkSZIaMFhLkiRJDRisJUmSpAYM1pIkSVIDBmtJkiSpAYO1JEmS1IDBWpIkSWrAYC1JkiQ1sF7vArT22eawz079Ma88cu+pP6YkSdI4j1hLkiRJDRisJUmSpAYM1pIkSVIDBmtJkiSpAYO1JEmS1IDBWpIkSWrAYC1JkiQ1YLCWJEmSGjBYS5IkSQ0YrCVJkqQGDNaSJElSAwZrSZIkqQGDtSRJktSAwVqSJElqwGAtSZIkNWCwliRJkhowWEuSJEkNGKwlSZKkBgzWkiRJUgMGa0mSJKkBg7UkSZLUgMFakiRJasBgLUmSJDVgsJYkSZIaMFhLkiRJDRisJUmSpAYM1pIkSVIDBmtJkiSpAYO1JEmS1IDBWpIkSWrAYC1JkiQ1YLCWJEmSGjBYS5IkSQ0sGKwjYuuIOCMivhURl0XEoXX89RFxTURcVN+eMnaf10TE8oi4PCKePDa+tI4tj4jDxsa3jYiv1/GPRMQGrb9QSZIkaZJW54j1LcArM3MHYDfgkIjYoV739szcqb6dClCv2w94MLAUeHdErBsR6wL/CewF7ADsP/Z53lI/1wOBG4EXNvr6JEmSpKlYMFhn5rWZ+Y368c3At4Et57nLPsCJmfnrzPw+sBx4VH1bnplXZOZvgBOBfSIigD2Aj9X7Hw88485+QZIkSVIPd2iOdURsAzwc+HodellEXBwRx0bEpnVsS+CqsbtdXcfmGr838JPMvGXG+GyPf3BELIuIZTfccMMdKV2SJEmaqNUO1hFxL+DjwMsz86fAUcADgJ2Aa4G3TqTCMZl5dGbunJk7L1myZNIPJ0mSJK229VbnRhGxPiVUfzAzPwGQmdeNXf9e4DP14jXA1mN336qOMcf4j4FNImK9etR6/PaSJEnSorA6XUECOAb4dma+bWx8i7GbPRO4tH58CrBfRNwtIrYFtgPOA84HtqsdQDagLHA8JTMTOAPYt97/QODkNfuyJEmSpOlanSPWjwUOAC6JiIvq2GspXT12AhK4EngxQGZeFhEnAd+idBQ5JDNvBYiIlwGnAesCx2bmZfXzvRo4MSLeBFxICfKSJEnSorFgsM7Ms4GY5apT57nPm4E3zzJ+6mz3y8wrKF1DJEmSpEXJnRclSZKkBgzWkiRJUgMGa0mSJKkBg7UkSZLUgMFakiRJasBgLUmSJDVgsJYkSZIaMFhLkiRJDRisJUmSpAYM1pIkSVIDBmtJkiSpAYO1JEmS1IDBWpIkSWrAYC1JkiQ1YLCWJEmSGjBYS5IkSQ0YrCVJkqQGDNaSJElSAwZrSZIkqQGDtSRJktSAwVqSJElqwGAtSZIkNWCwliRJkhowWEuSJEkNGKwlSZKkBgzWkiRJUgMGa0mSJKkBg7UkSZLUgMFakiRJasBgLUmSJDVgsJYkSZIaMFhLkiRJDRisJUmSpAYM1pIkSVIDBmtJkiSpAYO1JEmS1IDBWpIkSWrAYC1JkiQ1YLCWJEmSGjBYS5IkSQ0YrCVJkqQGDNaSJElSAwZrSZIkqQGDtSRJktSAwVqSJElqwGAtSZIkNbBgsI6IrSPijIj4VkRcFhGH1vHNIuL0iPhufb9pHY+IeGdELI+IiyPiEWOf68B6++9GxIFj44+MiEvqfd4ZETGJL1aSJEmalNU5Yn0L8MrM3AHYDTgkInYADgO+mJnbAV+slwH2ArarbwcDR0EJ4sDhwK7Ao4DDR2G83uZFY/dbuuZfmiRJkjQ9CwbrzLw2M79RP74Z+DawJbAPcHy92fHAM+rH+wAnZHEusElEbAE8GTg9M1dk5o3A6cDSet1GmXluZiZwwtjnkiRJkhaFOzTHOiK2AR4OfB3YPDOvrVf9ENi8frwlcNXY3a6uY/ONXz3LuCRJkrRorHawjoh7AR8HXp6ZPx2/rh5pzsa1zVbDwRGxLCKW3XDDDZN+OEmSJGm1rVawjoj1KaH6g5n5iTp8XZ3GQX1/fR2/Bth67O5b1bH5xreaZfx2MvPozNw5M3desmTJ6pQuSZIkTcXqdAUJ4Bjg25n5trGrTgFGnT0OBE4eG39B7Q6yG3BTnTJyGrBnRGxaFy3uCZxWr/tpROxWH+sFY59LkiRJWhTWW43bPBY4ALgkIi6qY68FjgROiogXAj8AnlOvOxV4CrAc+AVwEEBmroiINwLn19sdkZkr6scvBd4H3B34XH2TJEmSFo0Fg3Vmng3M1Vf6ibPcPoFD5vhcxwLHzjK+DNhxoVokSZKkoXLnRUmSJKkBg7UkSZLUgMFakiRJasBgLUmSJDVgsJYkSZIaMFhLkiRJDRisJUmSpAYM1pIkSVIDBmtJkiSpAYO1JEmS1IDBWpIkSWrAYC1JkiQ1YLCWJEmSGjBYS5IkSQ0YrCVJkqQGDNaSJElSAwZrSZIkqQGDtSRJktSAwVqSJElqwGAtSZIkNWCwliRJkhowWEuSJEkNGKwlSZKkBgzWkiRJUgMGa0mSJKkBg7UkSZLUgMFakiRJasBgLUmSJDVgsJYkSZIaMFhLkiRJDRisJUmSpAYM1pIkSVIDBmtJkiSpAYO1JEmS1IDBWpIkSWrAYC1JkiQ1YLCWJEmSGjBYS5IkSQ0YrCVJkqQGDNaSJElSAwZrSZIkqQGDtSRJktSAwVqSJElqwGAtSZIkNWCwliRJkhowWEuSJEkNLBisI+LYiLg+Ii4dG3t9RFwTERfVt6eMXfeaiFgeEZdHxJPHxpfWseURcdjY+LYR8fU6/pGI2KDlFyhJkiRNw+ocsX4fsHSW8bdn5k717VSAiNgB2A94cL3PuyNi3YhYF/hPYC9gB2D/eluAt9TP9UDgRuCFa/IFSZIkST0sGKwz80xgxWp+vn2AEzPz15n5fWA58Kj6tjwzr8jM3wAnAvtERAB7AB+r9z8eeMYd/BokSZKk7tZkjvXLIuLiOlVk0zq2JXDV2G2urmNzjd8b+Elm3jJjXJIkSVpU7mywPgp4ALATcC3w1mYVzSMiDo6IZRGx7IYbbpjGQ0qSJEmr5U4F68y8LjNvzczfAe+lTPUAuAbYeuymW9WxucZ/DGwSEevNGJ/rcY/OzJ0zc+clS5bcmdIlSZKkibhTwToithi7+Exg1DHkFGC/iLhbRGwLbAecB5wPbFc7gGxAWeB4SmYmcAawb73/gcDJd6YmSZIkqaf1FrpBRHwYeDxwn4i4GjgceHxE7AQkcCXwYoDMvCwiTgK+BdwCHJKZt9bP8zLgNGBd4NjMvKw+xKuBEyPiTcCFwDHNvjpJkiRpShYM1pm5/yzDc4bfzHwz8OZZxk8FTp1l/ApWTiWRJEmSFiV3XpQkSZIaMFhLkiRJDRisJUmSpAYM1pIkSVIDBmtJkiSpAYO1JEmS1IDBWpIkSWrAYC1JkiQ1YLCWJEmSGjBYS5IkSQ0YrCVJkqQGDNaSJElSAwZrSZIkqQGDtSRJktSAwVqSJElqwGAtSZIkNWCwliRJkhowWEuSJEkNGKwlSZKkBgzWkiRJUgMGa0mSJKkBg7UkSZLUgMFakiRJasBgLUmSJDWwXu8CJEmStLhsc9hnp/6YVx6599Qf847yiLUkSZLUgMFakiRJasBgLUmSJDVgsJYkSZIaMFhLkiRJDRisJUmSpAYM1pIkSVIDBmtJkiSpAYO1JEmS1IDBWpIkSWrAYC1JkiQ1YLCWJEmSGjBYS5IkSQ0YrCVJkqQGDNaSJElSAwZrSZIkqQGDtSRJktSAwVqSJElqwGAtSZIkNWCwliRJkhowWEuSJEkNLBisI+LYiLg+Ii4dG9ssIk6PiO/W95vW8YiId0bE8oi4OCIeMXafA+vtvxsRB46NPzIiLqn3eWdEROsvUpIkSZq01Tli/T5g6Yyxw4AvZuZ2wBfrZYC9gO3q28HAUVCCOHA4sCvwKODwURivt3nR2P1mPpYkSZI0eAsG68w8E1gxY3gf4Pj68fHAM8bGT8jiXGCTiNgCeDJwemauyMwbgdOBpfW6jTLz3MxM4ISxzyVJkiQtGnd2jvXmmXlt/fiHwOb14y2Bq8Zud3Udm2/86lnGJUmSpEVljRcv1iPN2aCWBUXEwRGxLCKW3XDDDdN4SEmSJGm13NlgfV2dxkF9f30dvwbYeux2W9Wx+ca3mmV8Vpl5dGbunJk7L1my5E6WLkmSJLV3Z4P1KcCos8eBwMlj4y+o3UF2A26qU0ZOA/aMiE3rosU9gdPqdT+NiN1qN5AXjH0uSZIkadFYb6EbRMSHgccD94mIqyndPY4EToqIFwI/AJ5Tb34q8BRgOfAL4CCAzFwREW8Ezq+3OyIzRwsiX0rpPHJ34HP1TZIkSVpUFgzWmbn/HFc9cZbbJnDIHJ/nWODYWcaXATsuVIckSZI0ZO68KEmSJDVgsJYkSZIaMFhLkiRJDRisJUmSpAYM1pIkSVIDBmtJkiSpAYO1JEmS1IDBWpIkSWrAYC1JkiQ1YLCWJEmSGjBYS5IkSQ0YrCVJkqQGDNaSJElSAwZrSZIkqQGDtSRJktSAwVqSJElqYL3eBUiS1NI2h3126o955ZF7T/0xJQ2PR6wlSZKkBgzWkiRJUgMGa0mSJKkBg7UkSZLUgMFakiRJasBgLUmSJDVgsJYkSZIasI+1JGmN2DdakgqDtSRJ0hhfLOrOciqIJEmS1IDBWpIkSWrAYC1JkiQ1YLCWJEmSGjBYS5IkSQ0YrCVJkqQGDNaSJElSAwZrSZIkqQGDtSRJktSAwVqSJElqwGAtSZIkNWCwliRJkhowWEuSJEkNGKwlSZKkBgzWkiRJUgMGa0mSJKkBg7UkSZLUgMFakiRJasBgLUmSJDVgsJYkSZIaWKNgHRFXRsQlEXFRRCyrY5tFxOkR8d36ftM6HhHxzohYHhEXR8Qjxj7PgfX2342IA9fsS5IkSZKmr8UR6ydk5k6ZuXO9fBjwxczcDvhivQywF7BdfTsYOApKEAcOB3YFHgUcPgrjkiRJ0mIxiakg+wDH14+PB54xNn5CFucCm0TEFsCTgdMzc0Vm3gicDiydQF2SJEnSxKxpsE7gCxFxQUQcXMc2z8xr68c/BDavH28JXDV236vr2FzjkiRJ0qKx3href/fMvCYifh84PSK+M35lZmZE5Bo+xm1qeD8Y4H73u1+rTytJkiStsTU6Yp2Z19T31wOfpMyRvq5O8aC+v77e/Bpg67G7b1XH5hqf7fGOzsydM3PnJUuWrEnpkiRJUlN3OlhHxD0j4vdGHwN7ApcCpwCjzh4HAifXj08BXlC7g+wG3FSnjJwG7BkRm9ZFi3vWMUmSJGnRWJOpIJsDn4yI0ef5UGZ+PiLOB06KiBcCPwCeU29/KvAUYDnwC+AggMxcERFvBM6vtzsiM1esQV2SJEnS1N3pYJ2ZVwAPm2X8x8ATZxlP4JA5PtexwLF3thZJkiSptzVdvHiXs81hn536Y1555N5Tf0xJkiTdMW5pLkmSJDVgsJYkSZIaMFhLkiRJDRisJUmSpAYM1pIkSVIDBmtJkiSpAYO1JEmS1IDBWpIkSWrAYC1JkiQ14M6L0pS5e6ckSWsnj1hLkiRJDRisJUmSpAYM1pIkSVIDBmtJkiSpAYO1JEmS1IDBWpIkSWrAYC1JkiQ1YLCWJEmSGjBYS5IkSQ0YrCVJkqQG3NJc0uC47bskaTHyiLUkSZLUgMFakiRJasBgLUmSJDVgsJYkSZIaMFhLkiRJDdgVRJKkuxg770iT4RFrSZIkqQGDtSRJktSAwVqSJElqwGAtSZIkNWCwliRJkhowWEuSJEkNGKwlSZKkBgzWkiRJUgMGa0mSJKkBg7UkSZLUgMFakiRJasBgLUmSJDVgsJYkSZIaWK93AZKkO2abwz479ce88si9p/6YkrTYeMRakiRJasBgLUmSJDVgsJYkSZIaMFhLkiRJDRisJUmSpAYG0xUkIpYC7wDWBf5vZh7ZuSStJeygMD+/P9Lk+Xcm3TUMIlhHxLrAfwJPAq4Gzo+IUzLzW30rkyRJk+YLD60tBhGsgUcByzPzCoCIOBHYBzBYL8AnI2ny/DuTJK2OyMzeNRAR+wJLM/Mv6+UDgF0z82UzbncwcHC9+CDg8qkWuubuA/yodxFjrGd+Q6sHhleT9cxvaPXA8GqynvlZz8KGVpP1zG9o9ayO+2fmktW54VCOWK+WzDwaOLp3HXdWRCzLzJ171zFiPfMbWj0wvJqsZ35DqweGV5P1zM96Fja0mqxnfkOrp7WhdAW5Bth67PJWdUySJElaFIYSrM8HtouIbSNiA2A/4JTONUmSJEmrbRBTQTLzloh4GXAapd3esZl5WeeyJmFo01isZ35DqweGV5P1zG9o9cDwarKe+VnPwoZWk/XMb2j1NDWIxYuSJEnSYjeUqSCSJEnSomawliRJkhowWEuSJEkNGKzVTUSsGxEf7F2HFr+IuHtEPKh3HZpfRKwTEc/pXYek6burPE+7eHGCIuJ/AX8H3J+xDiyZuUfHmjYH/gm4b2buFRE7AI/OzGM61XM2sEdm/qbH489Uf2ZHAZtn5o4R8VDg6Zn5pk71PBZ4PSt/hwLIzPzDHvXUml4xy/BNwAWZeVGHep4G/BuwQWZuGxE7AUdk5tOnXcuMurbk9n/7Z065hmfNd31mfmJatYwMaXOIiPg0MOc/wV6/QxFxM7ev6yZgGfDKzLxiyvW8lQF264qIdYHNWfVv7P91quUTwDHA5zLzdz1qmFHPoH5mQ32engSD9QRFxDeB9wAXALeOxjPzgo41fQ44DviHzHxYRKwHXJiZD+lUzwnAH1H6lv98NJ6Zb+tUz1coL4b+KzMfXscuzcwdO9XzHeBvuf3v0I971FNr+hCwM/DpOvRU4GJgG+CjmfkvU67nAmAP4MtjP7NLev1O18d/C/DnwLdY+XPLaf8TiYjj6oe/DzwG+FK9/ATga5n51GnWU2s6krKd8UdY9W9+RYdaHlc/fBbwB8AH6uX9gesy82+nXVOt643A1cCHKC+m9wMeAHwD+KvMfPyU6/lL4CBKgD0O+HBm3jTNGmap6a+Bw4HrgFGQzcx8aKd6/pTyPdoN+ChwXGZe3qOWWs+gfmZDfJ6eFIP1BEXEBZn5yN51jIuI8zNzl4i4cOyX+6LM3KlTPYfPNp6Zb5h2LTDI78/XM3PXHo89l4g4E3hKZv6sXr4X8FlgKeWo9Q5TrufczNxtxs/s4l7/YOvjXw48NDN/3auGcRHxBeDAzLy2Xt4CeF9mPrlDLd+fZbj3WZjbHUXveWQ9Ir6ZmQ+bMXZRZu4023VTrOtBlLC2P/BV4L2ZeUanWpYDu/Y8yDCbiNiY8v35B+Aq4L3ABzLzt53qGcTPbIjP05MyiA1i1mKfjoiXAp8EbvsH2+PIzJifR8S9qacZI2I3yinGLnoF6Hn8KCIewMrvz77AtR3rOSMi/hX4BKv+Dn2jX0n8/ngtwG8pU2d+GRE9guRlEfFcYN2I2A74G+BrHeoYdwWwPqt+n3raehSqq+uA+/UoJDO37fG4C7hnRPzhaIpFRGwL3LNjPb+oc9E/Vi/vC/yqftzlaFiddrF9ffsR8E3gFRHx4szcr0NJV9Hxf9ds6v/W5wMHABcCHwR2Bw4EHt+hniH9zIb4PD0RHrGeoIEemXkE8C5gR+BSYAmwb2Ze3KmeJcDfAw8GNhyN95qHHhF/SNkV6jHAjcD3gedn5pWd6pntyEJ2nqf/OuCZwMl16GmUqTxvBY7OzOdNuZ57UI4O7VmHTgPelJm/mvteE6/p48DDgC+y6guiv+lUz38A2wEfrkN/DizPzL/uUMs9gFcA98vMg+s/2Qdl5memXctYTUspf/dXUKZe3B94cWae1qmePwTeATyaEqTPpUwJuwZ4ZGaePeV63k75O/8icExmnjd23eWZOfUFaRFxDPAgytmy8b+xXtMIP1nreT/lbNC1Y9dN/exH/Zk9lTL9q/vPbIjP05NisL4LqvOqH0T5B3J5r1NUtZYvUOZavgp4CeWV/Q2Z+epeNdW67gmsk5k396xjqCJiF8qLD4CvZuayjrX8MWW+8K1jY4/oeVQ/Ig6cbTwzj592LSN1IeMf14tnZuYnO9XxEcqagRfUBcL3oPz8uky3GqvrbpQjewDfGco0niGIiIOAkzLz57Nct3GPubsDnEb4hF7TYmaKiAD+EXjbEH5m9cj5WzLzVdN6zJ4M1hMUEesDfwX8SR36MmVRXLcgCxARj6EsNBtfSX1Cp1ouyMxHjs+1Gs1z7lTP3YA/4/bfnyM61bMxZYHO6HfoK5SV1L0XDg1pNf4vgPOBZ2fm9XXsG5n5iB711Md/IiUs/rJXDUM1Ono3Y65lt3nDY3XtCOzAqmfOej0vLgFexO2fh/5TQD4CAAAgAElEQVT3lOuY92+o14vXIQW1GGDnHRjewsDRHOvedUyDc6wn6yjKPMt318sH1LG/7FVQRLyfsrr8Isa6FQBd/oFQ5ucCXBsRewP/A2zWqRYo0xtuohxRG8IRq2MpU3ZGvX8PoKzwnvfJfJJmrMa/lXLmI4Fei1AuB/4V+EpEvDAzv1Zr6ukFwFERsQI4CzgTODszb+xRTP3n/xbK/Piob5mZG3Uo5zcRcXdWrmN4AJ3/1urRz8dTgvWpwF7A2fR7XjyZ8nvz34x1A+rgrfX9hpROQN+k/O48lNL679E9isrMW6O0Ih2Cp9X3s3beoayP6eEbEbFLZp7f6fFnujAiTqF0TBnvBtTr+zMxHrGeoDlWdnc9MhMR3wZ2yIH84CPiqZR/IFtT5n5vBLwhM0/pVE+31nqzma0jSc8uJfXxB7Uaf3R0us7V/Qjlxcj/7nnEeiQi7ktZePYqSu/4Lgcz6s/saZn57R6PP6OWPSlzLXcAvgA8Fjio52n0iLiEMif+wixtSDendHJ4Uqd6uv6NzxSlR/PhmXlJvbwj8PrM3LdjTUcBWzKQoBYRp1OmN3XvvFMf/zvAA4EfUL4/oxfTvdoRHjfLcE77LMw0eMR6sm6NiAdk5vfgtgUpPY8+QDn6+Qf07XRxm7EFSzdRXuH39rWIeMjoH8gA/DIidh8tVqpHaXpPLxjaavwAyMzvRsSfUIJ11xZOEfF8ynzmh1BW4/8H5QVkL9cNIVQDZOYXovS03Y3yszs0M3/UuaxfZubvIuKWiNgIuJ7yYr+Xz0TEUzLz1I41jHvQ+HNiZl4aEX/UsyDKUfQfU3ojjyT9jhBvNZTOO1WXQD+XzDyodw3T4hHrCarzLI9j1ZXmvY/MnAHsBJzHqiupe+0wNrSdDr9FeZX/fcr3p/er/J2A44GNay0rgL/IzG/2qKfWNKjV+LOJiPv1mvNdH/9HwPcoG0Sd0aurzFg976C8oP4Uq/7Meuy8+MXMfOJCY1Ou6d3AaykbsbwS+BlwUa8wEGXnxXtSfla/pe/UHSLiw5SjnqMNdJ4H3Csz9+9RzxANqfPOuIj4fVZdN9BrLcxxzNIqcm08Ym2wnrC6GG7U1uby3ivNY+VOY6vIzK9MuxaAGN5Oh/efbTwzfzDtWsbVo2hk5k971lFrGdpq/A2BF3L7lo1dn7Aj4sGURae7U/7hXp6ZB3Sqpftp2PpzugdwBmU+82ge/EbA5zNz+znuOlURsQ2wUXZqQTpE9Wc3vhD/TOConq3ShhjUhtJ5p9bydMoc+ftSzsDcH/h2Zj64Uz1/NnZxQ0rL1v/JTi1IJ8mpIBMQEXtk5pdmWS38wIjoOlm/V4Cexz0y87zSHeg2t0y7iIjYqIbWQbTXi4jnZ+YHIuIVM8aBvkeHewXoebwf+A7l1OcRlKNpXac91BdC96P8M9uGcsbhd/PdZ5IGchr2xcDLKf/oL2BlsP4pZapMVzWIjHffmXqwjojtM/M7c3Xj6NWFowbot9e3oRjve35bUOtUC3DbGaChLMZ7I2W61X9n5sMj4gmUzWu6yMyPj1+uZ0Gm2o99WgzWk/E4ysrgp81yXZc5YBFxdmbuXk8xjr/K73qKkeHsdPghSjP9C2ot40k/gWlv6jPa9e33Zrmu185r/56ZL4+IT89WQ6/pRMADM/PZEbFPZh4fER+i73xmKP8wRm//kZlX9yxmCEf3MvMdwDsi4q8z813TetzVERFHArtQdsoD+JuIeHRmvnbKpbwCOJiV3TjGJavOJ56aKJudzfb7022zs6EFtRn/WzegdAT7ecf/rb/NzB9HxDoRsU5mnhER/96pltlsR+mkstYxWE9AZo5OlR+Rmavsvhhlq9ypy8zd6/vZglpPh1B2PNs+Iq6h7nQ47SIy86n1/SC2W87M/6of/ndmfnX8uo5tpt5f3/9bp8efy6hl409qt4If0vkJO1f2ZL9XzzrGDOboXma+a0g9o6unADtl5u8AIuJ4ypbUUw3WmXlwfT+EhdzjxncN3BB4Nn3bos6ma1Ab/98a5dTiPpQjxr38pD7/nAl8MCKuZ6x7yrTNclDvh0DXjeAmxTnWExSzbFIRdUOUXjXVGjalrHgf33ig2y51wKB2OqwLKLdh1e9PrxZOs/0Odd38ZGgi4i+Bj1M6cLwPuBfwurEXJz1q2pHyQmQzytmPG4ADM/PSXjWNi4h1KH21H7Pgjds/9qw9ozu3brsYeHxmrqiXNwO+3HHR8mx96m8CLsm6CVJvvf+XjQW1UR/9HwKvmXkku6cY2wSpw2PfE/gV5fvzPMp0tA/mQNqkrs08Yj0BEbE9ZSHVxjOeIDdi7AhNDxHxRuAvKJ1KRnM+p36Kcebc4bHxUlCnOcQRMWrVdhmrfn+mGqwj4tGUzQaWzPhebQSsO81aZorSe/yNlPnD69F/OtH7Wblb5mjL8M071TJyNPCKUQegiHh8HZt6kJ1Dz6N7+7KyZ/RBUXtGd6pldHTx3ygbWJxB+X3+E+CwXjVRFuM+mrLQE8oLkQuAbSPiiMx8/1x3nIQZc77XoRzB7pofhnb2dcb/+tH3qNvizqxbmdf1Hp/uVcfIELsBTYrBejIeRJmvuwmrzrO+mbJNbU/PAR6Qmb/pXMfoSXHmfObRWC+7ZeYOHR9/ZAPKkdf1WHWe9U8pwaSnf6fs/HhJDuOU19B2ywS453hbzcz8cj2C1MUcR/d6nYb9VQ6oZ3RmZkT8HeW0/S51+NWZ+cNeNVH+7v8oM68DqC8+TgB2pZzan2qwZtU537cAV7JyN9huapjdnfI7fVZmfqpjOeP/60ffo336lAIR8WLgDZRw/ztW/u1PdV78WDeg+9Sz5ePdgLacZi3TYrCegMw8GTi5Ln45p3c9M1xKCfxdTyeOOkvUuYyHZuZP6uVNmX3hzrScExE7ZOa3OtYw6t7ylYh4X+9Wf7O4Crh0IKEaysYMS3sXMcMVEfE6Vgag51POEnUxsKN750fEJsB7KS+Gfgb0fp78BuX3qMuOr7PYehSqq+vr2IqI+O1cd5qUAc75HvUefyAr+0a/JCKelJmH9KhnIJ13xr0K2DH7b7403g1ofMrpILoBTYJzrCdoiP11I2JnyhG+SxnGBjG3m4PWeV7a44BTKEf0hrBBzBLg77n971CX7gC1pl0oU0G+wgA2iImIo4F35XB2yxy9QHwD5WgalC4lr8/MGzvWNN5O7su5ctfTadfxAcrvzlmUo2nde0bH8LZ/fjelXeNH69CfAVdTev5/ZtpBNyI2Bg5n1XaER2Rmtx1Y68/sj0Yv8Ou6gcsys8uOkBGxFfAuYLS4/CzKQaMuHYEi4vPAszLzFz0ef6YhdgOaFI9YT9bg+utS5qC+BbiEjn11x6wTEZuOAkddNNTz9/IY4ACG8/35IPARytSilwAHUhbC9fRmylHGDSlTVrqIiEsopzbXAw6KiCsYwIshyoPfCAxm44NZ2skdGhGP6dBODsrf2B9TQsgDKHObz6zt+HoZ1PbPlG5Jf8bKkHYC8PEaInscPT6WcjBmNP3jAMquwrMtspyW5ZQXH6MzelvXsV6Oo7RtfXa9/Pw69qRO9bwG+FpEfJ1VD4BM9XlpbO75NbMtyu3VGGCSPGI9QaMjrxFxcWY+NCLWp8wD69aCJyLOz8xdFr7ldETECygtrUZHZp4NvHnai3PG6jknMx/d47FnM1p5P/odqmNdf4bRcWfMGXXMukvmSI8pNDFHj++RjmeGLmbVdnLrUhYP9joiuy4l6D+B8oLxlzmQnRd1exFxUWbutNDYlGoZ/Y1tTPkdOq9e3hU4LzMfP+2aal2D+R7Vxz6P0td7lYNEmXn8nHeaTB2z7fo6kj3P4E+KR6wna3D9dYGzIuKfKdMdxl/F9trR64SIWMbKriTP6jy/+cIoG4x8mlW/P71eVY9+h66NiL0pvYd79489NSL2zMwv9CxigHPPYWWP72cBf8DKbhf7A9fNeo/p2QRYUT/euFcREfFFygZI51BOl+8ylBZyQzFLz18oC3SXAa/MzGnP1/9lROyemWfX+h4L/HLKNYwMrY/+yI8j4vmsnPO9P9Cztd36mTlr961pGuDc84nziPUEjfXXfSjllNAQ+uueMctw9pyzOyRzvLru9qq6trY7i3Ka812UldSvz8xu7ZPqP/17Ar9hZfDv2W5vcCJiWWbuvNDYFOvZHziS0r7ttnZymfmRDrW8HXgk5YXrVyldLs7JzF5BbXBqW9SrKVMLAtiPMm3mG8BfTfuobETsRJlGuHGtZwXwF5n5zWnWcUdM++xjPYP2LkqbxAS+BvxNZv6/adUwo55/onQmmXmQaMVc95lwPf9ntvHMPGLatUyawXqCImLdzLy1dx13REQcOO1TRYtJRLwmM/95io/32Jxl58WZYxqWiPg2sPfoyGKUHVdP7bGwqvZp3orSAmw0hei8zu3kiIjfo/TUfxXwB5l5t571DElEfDMzHzZj7KLM3Gm266ZY10YAmfnTHo9/R/RcBD8EUbahnymz0zb0EfHKsYsbUtYNfXttnApisJ6giPh/wOcpi8++NKD2ZHMKd/Wb17S/P7M93hB+RkPpMDFUEbGUsiHMFZQjfPcHDu41fSYiLsnMh/R47Jki4mWUxYuPpBxRO4uy9uRLPesakog4B3g78LE6tC9lw6HdpjlvN+bYyGukVyeg1TGt58mIeBfzr6sYzCLmIYmIuwGn9ZoTP0nOsZ6s7Smvyg4Bjq2LLk4czVMbqJmbtWhVU/n+xLB3Xpytw8RjM/M1HcsalMz8fERsR3kOAPhOZt52Orb22z19iiV9IyJ2yczzp/iYc9kQeBtwQWbe0ruYgXoe8A7g3fXyOcDzI+LuwMumWMdQN/IakmW9CxgXEXtk5pdm68ABg+rCcQ/KmbS1jkesp6T2tX0H8LzM7BqM5jOEo6FDNsWjII+jbGP8EuA9Y1fdDHw6M7876RrmMrQOE4tRhzMfg+rTrMUl5tjIa8in8e+qU0Ei4g2ZefgA1wuN2qNCOTi0hNILfa3bJMYj1hNWA9KfA0spr2y7bwO7AI9Yz28q358c9s6LMJAOE4vYtP/OhtanWfMY2mYjwENHoRpKn/aIGHpoPWCaDxYRpwPPnvHi48TMnOrfXg3V6wCfy8yTpvnYC3jq2Me3ANetrWesDNYTFBFXAhcCJwF/l5k/71vRanFR3Pw+uvBN1lxE/Htmvhz4j4i43WmlXv2Qq3+mtCVcpcNEx3oWo2mfKrx5Ncc0DEPbbGRoG3mNNh55C6WFbbDyLMxogeWlUy5pySwvPrq0183M30XE31Oyx1BsQdkZ82Yoi5cjYofM/HrnuppzKsgERcRGQ1k9vZgXoUzD0BagRMQjM/OCesZjtnq+Ms16ZoqILRhQh4nFpsNUkCspLRtvpASQTSh99a8DXpSZF0yrFi1sgJuNDGojr1rTcuBpmdl7N2OgbOYFPHPUXq+23/tkr6mVdS3MjyjNE247qNex3d6FwCNGTRzqUfVla+PUU49YT0BE/H1m/gvw5jmONvZYJfx7C9/kLm1QC1BGQSczvxIRG1AWwSVweWb+pmtxxS6s7AqSlF6pWn1XTvnxTgc+lpmnAUTEnpQts4+jLJDbdcr1aH6D2mwkh7eRF5SpBIMI1dU/AGdHxFcoL17/GDi4Yz1/TnlufumM8S7t9igHcm/LQ/Wo+lqZQT1iPQER8bTM/HREHDjb9faJ1uqquy2+B/ge5cl6W+DFmfm5jjXN7AqyP3B+Zr62V01DExFnA1+hzI396uj0Z8d6btduLyIuzsyH9jwSqtkNbbORIRnrdvE4yu6mn2IYu+QSEfcBdqsXz83MH41d9+DMvGyKtdydEqp3p/wOnQW8JzttxBQRnwC+DBxVh14KPCEzn9GjnkkyWE9QRDw7Mz+60NiUa9oQeCHwYErbKwCGvLp7miJiCfBqYAdW/f502ZmydnN4amYur5cfAHw2M7ef/54TrcmuIAuoG8L8cX3bjfKP/6zM/NtO9XwB+CJwYh36c8p83aWUF0Vr3elYrZ3m6HYx0q3rxUI6TP86CfgpKw+APBfYODO7NFCo883fSTnrkZTno5dn5vU96pmktfIw/IC8htsvdpttbJreD3yH0iXgCEq/1CGdTuvtg5Q5aXtTWt0dCNzQsZ6bR6G6uoJhLDqzK8g8MvP7EfEryrbvvwGeAEx918UxzwUOpxzdS8oi5edS2l4NvVPRXcbQ1noMUWYe1LuGO2nanYB2zMwdxi6fERHdpu/UAL1fr8efJoP1BETEXsBTgC0j4p1jV21EaTPT0wMz89kRsU9mHh8RH6KcIlJx78w8JiIOHWt5N/VNNcZOdy6LiFMpq7uTsmio9yYfdgVZQER8j7Jw6EPAMcBfj47w91BPSf/1HFcvj4h3ZeZc12t6BrXWY8gi4l+ANwG/pOxw/FDgbzPzA10Lm9u0pwd8IyJ2y8xzASJiVzr8fo3WnM31onFtfLFosJ6M/wEuAJ5e34/cDHQ5FTzmt/X9TyJiR0pngC4tgQZq9P25ts5v/h9gsw51PG3s4+so8wmhHD2/+/TLWSkzPxwRX2ZlV5BX2xXkdt5Jmdu4P/Bwygu0MzPze33LmtNjF76JJm1119/4QgiAPTPz7yPimZTFwM8CzgSGGqyn7ZHA1yJiNC//fsDlo41apjh1b3RG/C7zotFgPQGZ+U3gmxHxgQE2QD+6Nq5/HXAKcC/g//QtaVDeFBEbA6+kLB7aiA4vhoZ4ujMiZs4PHG1Wcd+IuG9mfmPaNQ1VZr4DeEdE3As4CHg9Zfvewe66qkXFF0Ir88vewEcz86aIQe9vNu1uTkun/Hizqo0c1gUekpmv6l3PNLh4cQJmbN15Oy7y0uoa0mLTOvXjthLGr6IcAemywHOIIuKtlCPW9wLOoUy3Oiszr+ha2BymvbBKa8af123diZ5BmQryKMq6j89kZpfWkbXrxTGUHQ+7Tfsaqog4JzMf3buOaTBYT0BtlTSnnltUR8TdKP1rt2HsjEVmHtGrpiGpXUFexO2/P11WmkfERymLTZ/L2GLTzDy0Rz21ptnaOB2Vmb/qVdPQRMS+lCB9Xe9aVkdEXJiZQ9+iWpXBuqg7QN6UmbdGxD2AjXpNS4uIP6WcndqN0qDguMy8vEctQxQRRwFbUr434xvWdGuPOCkG67uYiPg8cBNl7veto/HMfGu3ogYkIr5GCYozvz8f71TPhZn58LGew+tTAttuC955cjUNqo3TUEXE01m5ic5XMrP7JjoRcY/M/MUs43+Rme/rUJLuBF8IFXWd0MzWqCf0qwjqVML9KRvGXAW8F/hAZv523juu5eZokzjY9ohrwjnWExQRN7PylPkGwPrAzzNzo35VsVVmDmLu1UDdIzNf3buIMUNcbDqoNk5DFBH/TDk9PXrx8TcR8ehem+hExGOA/0uZmnK/iHgYZaOhlwIYqodprhdCwDumXszARMThwOMpwfpUYC/gbKBbsI6IewPPBw4ALqT8/e9Oadv6+F51DcEQ1w1NisF6gjLztm3Eo6yq2IeVuzL18rWIeEhmXtK5jqH6TEQ8JTNP7V1INVps+o+sXGz6ur4lDaON08Dtzaqb6BxP+Ufba3fKt1N6158CZYF1RPzJ/HdRL74QWi37Ag+jbE51UERsTseOIBHxSeBBlL0inpaZ19arPlK3g79Lm9F6eOQmYFlmnjzteibJYD0lWebcfKq+yu7Z83d34C8i4vuU3eBGC89cUFkcCrw2In5NOVo8+v70OsvwflbOiR+14tq8RyFji3LXZ2UbpwTuT5kHrlUNahOdzLxqRteEW+e6rbrzhdDCfpmZv4uIWyJiI+B6YOuO9bwzM8+Y7YrM3HnaxQzQhsD2rNwg78+A7wMPi4gnZObLu1XWmMF6gsY2+QBYB9gZ6L3Aa6/Ojz9o42cZBuJkVs6J/3XnWp7a+fEXk6FtonNVPQqadZ7+objj6qD5QmhByyJiE8oc5guAn1E68EzV+P/5Gf/zgbVzcd6d9FDgsZl5K9y2mPEsysG+teoMusF6ssY3+biF0sT+6X1KKTLzB/W04h/XobNq3+27tIjYPjO/M0uvZgA69mgezJz4nt1sFpsBbqLzEsq83C2Ba4AvAId0rEfz84XQAkbTYoD31EX5G2XmxR1Kedo81yVgsC42pUxtuqleviewWe3o0vugUVMG68laBzg0M38CUOfKvhXotgo2Ig6ltJMb/bF/ICKOzsx39appIF4BHEz5+cyUQK8ezc6JX0SGuIlO3ZzhgMx83rQfW3eaL4TmMNfBj9F10/4bGy3Ki4htM/P7M+rZdpq1DNy/ABfVAw6js3j/FBH3BP67Z2Gt2W5vgmZridS7TVJEXAw8OjN/Xi/fEzjHOdbDMjafeT1gO+AKnBM/eEPdRCcizs/MXRa+pYYgIpZk5g296xiiGX9jI7f9rXX8G7tdb/GIuCAzH9mjniGKiPtSOqZ8m3L0+urMPLNvVe15xHqy1omITTPzRritmX3v73mw6ly9W+uYgIh4NvD5zLw5Iv4ReATwxsy8cMqlOJ95EcrMJ8Dcm+h0LO3siPgP4COsujmD29AP01cj4krKz+vjo7OeWuVv7DmU5+qfRsTrqM/V064nIran7Iy78Yw51hsx1l/7ri4i/pIypWkr4CJKh7Rz6Hc2eGJ6h7y13VuBc+rueQDPBt7csR6A44Cv19ZAULaEPaZjPUPzusz8aETsDvwp8K/Ae4CpbpPrfOZF73jKJjqjFlPPpfTX7bWJzk71/fgOqz2nOGkemfm/IuJRwH7AP9Q+8SdmZrd2cgP0j5l5Un2u3gP4N8qL12lvaf4gyoGQTVh1vvXNlGmXKg6lrDk5NzOfUF+Q/FPnmibCqSATFhE7sPKf15cys/tGGnWO2u714lkdjsYO1thOh/8MXJKZH+o9fUeLT0R8a8YmOrOOSQuJiPsAbwOel5nr9q5nKIb0XF3XMbw6M9fKoNjCaDpaRFwE7JqZv46IyzLzwb1ra80j1hNWg/QQwvRG9ZTZZpTuJFeOXbdZZq6Y6753MddExH8BTwLeEhF3oyxCle6IwW2iExF7U05Zj2//fMTc91AvtS/zMylHrB8AfJKyk6dWGsxzde1s8QzW0iOwjVxd2yN+Cjg9Im4E1sozsx6xvouIiM9k5lPrxjCzLar6w06lDUpE3ANYSjkC8t2I2AJ4SGZ+oXNpWgRmbKLzIGCVTXR6HbGOiPcA9wCeQNnRb1/gvMx8YY96NL/6PP0p4KTMnHpv5sVgaM/VEfF2yt+96xgWEBGPo2ya9fn/3979h+pZl3Ecf3+0MCsnrZU1VlZDlGFzBZZKghkVgmbNfpBIf5QVFZIEQRmB/2QgC7IF1R+xTDLcyGyWPzKzBa2CwqWhmf1RZEU/VrmVbf7o6o/7fuZpO7/aTud7n3O/X/Dw7L6fc+CCh7Pner7f73VdVfVo63gWmom1xCEr+odwRV/zkeTE2V5vdXY+yT1VtX7K8zOBW6vq7Dl/WYsuScoP5yVlpm4lrbqUqB2PgoxMkjur6jVz3Ruh6+kKUH5Kt8I4tVNKAa7oa04DLjr9V//8SN/yajfw/IbxaBpJPt2Pdt6e5JDEuqqaDhjTzCbdSiQT65FI8jS6reBV/aCaSeK4gm4IwahV1fn9sw39tRx9sz/feDXdl0fojoRoWK7rnzc1jUKHxToGgYn1mLwXuBxYTffBOkms9wCfbRXU0Liir2VqE/A+4Gy63rGt+2prGlU1+dKzoaqumfpaPzV3x+JHpfmYqY6haVBqwjPWI5PkMseXH2rKiv5dwDn894r+bVV1SqPQpCOWZCtdX91JH+SLgeOrqlVfbc1ihil+tv0cMOsYNOGK9chU1eYkZwEvYsr7X1VfbhbUMLiir+Xs1IM6ktzVDx3RgCR5O92Xnhcn2T7lpeMAC6iHzToGASbWo5PkOrq+qLt4crR50U2FG62quqYf+XxFVS36WFzp/2xwfbU1rZ3AH4BVdJN7J/YC9zSJSPNlHYMAj4KMTpL7gXW2cpqe261aTobaV1uzS/IS4PdVta+/PhY4oap+3TQwzah/jyZ1DEVfxzB5DzUerliPz8+B59GtiuhQdya5CLjRLx9aBs5vHYAOy1bgrCnXTwDbgNPbhKN5uJZuZ+Ez/fXFdDvB1jGMjCvWI9M3sd9AV628f3Lf/qidJHuBZwCPA/t4cjLliqaBSRqNJLuqasNB935WVae1ikmzS3LfwTtA093T8ueK9fhc2TqAIauq4/rpiycxpRepJC2iPyd5Q1VtB0hyIfCXxjFpdtYxCHDFepT6scsnVdV3kjwdOLqq9raOawiSXAp8EFhDV+B5BrDTPtaSFkuStcBX6IZ3FfAQ8I6q+lXTwDSjvn5pUscA8ELgAbrdz6qq9a1i0+IysR6ZJO8G3gOsrKq1SU4CPm/i2OmLvU4HflRVG5KcAlxVVRsbhyZpZPpeyFTVP1rHotn1C1YzqqrfLFYsasujIOPzAeAVwI8BqurBJM9tG9Kg7KuqfUlIckxV/SLJya2DkjQeSU4ArgJWV9V5SdYBZ1bVFxuHphmYOGviqNYBaNHtr6pHJxdJnkK31ajOQ30v0puAO5J8A/A/TEmL6UvA7XQDqwB+STfAStLAuWI9PjuSXAEcm+S1wPuBmxvHNBhV9ab+n1f2HVSOB25rGJKk8VlVVVuTfBSgqh5P8sRcvySpPRPr8fkI8C7gXrox3rfgdKhpVdWO1jFIGqV/Jnk2/W5ikjOAh9uGJGk+LF4cmSQbgW9V1f45f1iStOiSvBzYDJxKN9TrOcCbq8qx5tLAmViPTJItwLnA94EbgNuq6vG2UUmSpurrX06mG1L1QFU91jgkSfNgYj1CSZ4KnAe8DXgVcEdVXdo2Kkkat35HcUZVdeNixSLp8HjGeoSq6rEkt9Kd3zsWeCNgYi1JbV0wy2sFmFhLA+eK9cgkmaxUn4KT9gAAAAISSURBVAN8D9gKfNvjIJIkSUfGxHpkknyV7mz1rRYwStLwOCBGWrpMrCVJGpD+qN4W4GNVdVpfyHh3Vb20cWiS5uDkxZFJsjHJg0keTrInyd4ke1rHJUk6YFVVbQX+Dd2AGMABMdISYPHi+FwNXFBV97cORJI0LQfESEuUifX4/NGkWpIG7UPAdmBtkh/QD4hpG5Kk+TCxHp+fJLkBuAk4ULxof1RJGoy1dLMGXgBcBLwSP6+lJcE/1PFZATwCvG7KPfujStJwfLyqtiV5FvBqYBPwOboEW9KA2RVEkqQBSXJ3Vb0sySeBe6vq+sm91rFJmp1dQUYmyZokX0/yp/7xtSRrWsclSTrgd0m+QDfM65Ykx+DntbQk+Ic6PlvoimJW94+b+3uSpGF4K3A78Pqq+juwEvhw25AkzYdHQUYmya6q2jDXPUmSJP1vXLEen91JLklydP+4BNjdOihJkqSlzhXrkUlyIrAZOJOuG8hO4LKq+m3TwCRJkpY4E+uRSXItcHlV/a2/Xglsqqp3to1MkiRpafMoyPisnyTVAFX1V8AWTpIkSUfIxHp8juqHDgAHVqwdFCRJknSETKjG51PAD5Ns66/fAnyiYTySJEnLgmesRyjJOuDc/vK7VXVfy3gkSZKWAxNrSZIkaQF4xlqSJElaACbWkiRJ0gIwsZYkSZIWgIm1JEmStABMrCVJkqQF8B9rESfNTNg0LwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Distribution of the labels :\n",
    "\n",
    "plt.figure(figsize=(12, 8))\n",
    "\n",
    "labels_count = {}\n",
    "\n",
    "for tag in df_train['tags'].values:\n",
    "    labels_in_tag = tag.split(' ')\n",
    "    for label in labels_in_tag:\n",
    "        if label in labels_count:\n",
    "            labels_count[label] += 1\n",
    "        else:\n",
    "            labels_count[label] = 0\n",
    "            \n",
    "min_label = min(labels_count, key=labels_count.get)\n",
    "max_label = max(labels_count, key=labels_count.get)\n",
    "\n",
    "print(min_label+\" is tagged least no of times: \"+str(labels_count[min_label]))\n",
    "print(max_label+\" is tagged max no of times: \"+str(labels_count[max_label]))\n",
    "            \n",
    "plt.bar(range(len(labels_count)), list(labels_count.values()), align='center')\n",
    "plt.xticks(range(len(labels_count)), list(labels_count.keys()), rotation=90)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras_applications/mobilenet.py:207: UserWarning: `input_shape` is undefined or non-square, or `rows` is not in [128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "  warnings.warn('`input_shape` is undefined or non-square, '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 128, 128, 3)       0         \n",
      "_________________________________________________________________\n",
      "conv1_pad (ZeroPadding2D)    (None, 129, 129, 3)       0         \n",
      "_________________________________________________________________\n",
      "conv1 (Conv2D)               (None, 64, 64, 32)        864       \n",
      "_________________________________________________________________\n",
      "conv1_bn (BatchNormalization (None, 64, 64, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv1_relu (ReLU)            (None, 64, 64, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv_dw_1 (DepthwiseConv2D)  (None, 64, 64, 32)        288       \n",
      "_________________________________________________________________\n",
      "conv_dw_1_bn (BatchNormaliza (None, 64, 64, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv_dw_1_relu (ReLU)        (None, 64, 64, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_1 (Conv2D)           (None, 64, 64, 64)        2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_1_bn (BatchNormaliza (None, 64, 64, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv_pw_1_relu (ReLU)        (None, 64, 64, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_pad_2 (ZeroPadding2D)   (None, 65, 65, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_dw_2 (DepthwiseConv2D)  (None, 32, 32, 64)        576       \n",
      "_________________________________________________________________\n",
      "conv_dw_2_bn (BatchNormaliza (None, 32, 32, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv_dw_2_relu (ReLU)        (None, 32, 32, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_2 (Conv2D)           (None, 32, 32, 128)       8192      \n",
      "_________________________________________________________________\n",
      "conv_pw_2_bn (BatchNormaliza (None, 32, 32, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_pw_2_relu (ReLU)        (None, 32, 32, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_3 (DepthwiseConv2D)  (None, 32, 32, 128)       1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_3_bn (BatchNormaliza (None, 32, 32, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_dw_3_relu (ReLU)        (None, 32, 32, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_3 (Conv2D)           (None, 32, 32, 128)       16384     \n",
      "_________________________________________________________________\n",
      "conv_pw_3_bn (BatchNormaliza (None, 32, 32, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_pw_3_relu (ReLU)        (None, 32, 32, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pad_4 (ZeroPadding2D)   (None, 33, 33, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_4 (DepthwiseConv2D)  (None, 16, 16, 128)       1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_4_bn (BatchNormaliza (None, 16, 16, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_dw_4_relu (ReLU)        (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_4 (Conv2D)           (None, 16, 16, 256)       32768     \n",
      "_________________________________________________________________\n",
      "conv_pw_4_bn (BatchNormaliza (None, 16, 16, 256)       1024      \n",
      "_________________________________________________________________\n",
      "conv_pw_4_relu (ReLU)        (None, 16, 16, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_5 (DepthwiseConv2D)  (None, 16, 16, 256)       2304      \n",
      "_________________________________________________________________\n",
      "conv_dw_5_bn (BatchNormaliza (None, 16, 16, 256)       1024      \n",
      "_________________________________________________________________\n",
      "conv_dw_5_relu (ReLU)        (None, 16, 16, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_5 (Conv2D)           (None, 16, 16, 256)       65536     \n",
      "_________________________________________________________________\n",
      "conv_pw_5_bn (BatchNormaliza (None, 16, 16, 256)       1024      \n",
      "_________________________________________________________________\n",
      "conv_pw_5_relu (ReLU)        (None, 16, 16, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv_pad_6 (ZeroPadding2D)   (None, 17, 17, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_6 (DepthwiseConv2D)  (None, 8, 8, 256)         2304      \n",
      "_________________________________________________________________\n",
      "conv_dw_6_bn (BatchNormaliza (None, 8, 8, 256)         1024      \n",
      "_________________________________________________________________\n",
      "conv_dw_6_relu (ReLU)        (None, 8, 8, 256)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_6 (Conv2D)           (None, 8, 8, 512)         131072    \n",
      "_________________________________________________________________\n",
      "conv_pw_6_bn (BatchNormaliza (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_6_relu (ReLU)        (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_7 (DepthwiseConv2D)  (None, 8, 8, 512)         4608      \n",
      "_________________________________________________________________\n",
      "conv_dw_7_bn (BatchNormaliza (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_dw_7_relu (ReLU)        (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_7 (Conv2D)           (None, 8, 8, 512)         262144    \n",
      "_________________________________________________________________\n",
      "conv_pw_7_bn (BatchNormaliza (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_7_relu (ReLU)        (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_8 (DepthwiseConv2D)  (None, 8, 8, 512)         4608      \n",
      "_________________________________________________________________\n",
      "conv_dw_8_bn (BatchNormaliza (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_dw_8_relu (ReLU)        (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_8 (Conv2D)           (None, 8, 8, 512)         262144    \n",
      "_________________________________________________________________\n",
      "conv_pw_8_bn (BatchNormaliza (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_8_relu (ReLU)        (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_9 (DepthwiseConv2D)  (None, 8, 8, 512)         4608      \n",
      "_________________________________________________________________\n",
      "conv_dw_9_bn (BatchNormaliza (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_dw_9_relu (ReLU)        (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_9 (Conv2D)           (None, 8, 8, 512)         262144    \n",
      "_________________________________________________________________\n",
      "conv_pw_9_bn (BatchNormaliza (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_9_relu (ReLU)        (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_10 (DepthwiseConv2D) (None, 8, 8, 512)         4608      \n",
      "_________________________________________________________________\n",
      "conv_dw_10_bn (BatchNormaliz (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_dw_10_relu (ReLU)       (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_10 (Conv2D)          (None, 8, 8, 512)         262144    \n",
      "_________________________________________________________________\n",
      "conv_pw_10_bn (BatchNormaliz (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_10_relu (ReLU)       (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_11 (DepthwiseConv2D) (None, 8, 8, 512)         4608      \n",
      "_________________________________________________________________\n",
      "conv_dw_11_bn (BatchNormaliz (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_dw_11_relu (ReLU)       (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_11 (Conv2D)          (None, 8, 8, 512)         262144    \n",
      "_________________________________________________________________\n",
      "conv_pw_11_bn (BatchNormaliz (None, 8, 8, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_11_relu (ReLU)       (None, 8, 8, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_pad_12 (ZeroPadding2D)  (None, 9, 9, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_12 (DepthwiseConv2D) (None, 4, 4, 512)         4608      \n",
      "_________________________________________________________________\n",
      "conv_dw_12_bn (BatchNormaliz (None, 4, 4, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_dw_12_relu (ReLU)       (None, 4, 4, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_12 (Conv2D)          (None, 4, 4, 1024)        524288    \n",
      "_________________________________________________________________\n",
      "conv_pw_12_bn (BatchNormaliz (None, 4, 4, 1024)        4096      \n",
      "_________________________________________________________________\n",
      "conv_pw_12_relu (ReLU)       (None, 4, 4, 1024)        0         \n",
      "_________________________________________________________________\n",
      "conv_dw_13 (DepthwiseConv2D) (None, 4, 4, 1024)        9216      \n",
      "_________________________________________________________________\n",
      "conv_dw_13_bn (BatchNormaliz (None, 4, 4, 1024)        4096      \n",
      "_________________________________________________________________\n",
      "conv_dw_13_relu (ReLU)       (None, 4, 4, 1024)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_13 (Conv2D)          (None, 4, 4, 1024)        1048576   \n",
      "_________________________________________________________________\n",
      "conv_pw_13_bn (BatchNormaliz (None, 4, 4, 1024)        4096      \n",
      "_________________________________________________________________\n",
      "conv_pw_13_relu (ReLU)       (None, 4, 4, 1024)        0         \n",
      "_________________________________________________________________\n",
      "global_average_pooling2d_1 ( (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 128)               131200    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 17)                2193      \n",
      "=================================================================\n",
      "Total params: 3,362,257\n",
      "Trainable params: 3,340,369\n",
      "Non-trainable params: 21,888\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:9: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=Tensor(\"in..., outputs=Tensor(\"de...)`\n",
      "  if __name__ == '__main__':\n"
     ]
    }
   ],
   "source": [
    "#Define Mobile Net model :\n",
    "\n",
    "def create_model():\n",
    "    img = Input(shape = (128, 128, 3))\n",
    "    model_mob = MobileNet(include_top=False, weights='imagenet', input_tensor=img, input_shape=None, pooling='avg')\n",
    "\n",
    "    final_layer = model_mob.layers[-1].output\n",
    "    dense_layer_1 = Dense(128, activation = 'relu')(final_layer)\n",
    "    output_layer = Dense(17, activation = 'sigmoid')(dense_layer_1)\n",
    "\n",
    "    model = Model(input = model_mob.input, output = output_layer)\n",
    "\n",
    "    return model\n",
    "\n",
    "clear_session()\n",
    "\n",
    "model = create_model()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define evaluation metric - f2 score :\n",
    "\n",
    "def f2_score(y_true, y_pred):\n",
    "    y_true = tf.cast(y_true, \"int32\")\n",
    "    y_pred = tf.cast(tf.round(y_pred), \"int32\") # implicit 0.5 threshold via tf.round\n",
    "    y_correct = y_true * y_pred\n",
    "    sum_true = tf.reduce_sum(y_true, axis=1)\n",
    "    sum_pred = tf.reduce_sum(y_pred, axis=1)\n",
    "    sum_correct = tf.reduce_sum(y_correct, axis=1)\n",
    "    precision = sum_correct / sum_pred\n",
    "    recall = sum_correct / sum_true\n",
    "    f_score = 5 * precision * recall / (4 * precision + recall)\n",
    "    f_score = tf.where(tf.math.is_nan(f_score), tf.zeros_like(f_score), f_score)\n",
    "    return tf.reduce_mean(f_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 20239 validated image filenames belonging to 17 classes.\n",
      "Found 20240 validated image filenames belonging to 17 classes.\n",
      "Found 61191 validated image filenames belonging to 17 classes.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:9: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=Tensor(\"in..., outputs=Tensor(\"de...)`\n",
      "  if __name__ == '__main__':\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "159/159 [==============================] - 150s 942ms/step - loss: 0.1934 - f2_score: 0.7977 - val_loss: 0.2733 - val_f2_score: 0.7285\n",
      "Epoch 2/50\n",
      "159/159 [==============================] - 135s 850ms/step - loss: 0.1174 - f2_score: 0.8645 - val_loss: 0.1398 - val_f2_score: 0.8477\n",
      "Epoch 3/50\n",
      "159/159 [==============================] - 137s 860ms/step - loss: 0.1081 - f2_score: 0.8773 - val_loss: 0.0964 - val_f2_score: 0.8727\n",
      "Epoch 4/50\n",
      "159/159 [==============================] - 135s 850ms/step - loss: 0.1039 - f2_score: 0.8813 - val_loss: 0.1454 - val_f2_score: 0.8769\n",
      "Epoch 5/50\n",
      "159/159 [==============================] - 135s 848ms/step - loss: 0.1006 - f2_score: 0.8853 - val_loss: 0.1163 - val_f2_score: 0.8856\n",
      "Epoch 6/50\n",
      "159/159 [==============================] - 134s 845ms/step - loss: 0.0985 - f2_score: 0.8865 - val_loss: 0.0867 - val_f2_score: 0.8900\n",
      "Epoch 7/50\n",
      "159/159 [==============================] - 136s 856ms/step - loss: 0.0953 - f2_score: 0.8903 - val_loss: 0.1215 - val_f2_score: 0.8819\n",
      "Epoch 8/50\n",
      "159/159 [==============================] - 135s 849ms/step - loss: 0.0936 - f2_score: 0.8925 - val_loss: 0.0882 - val_f2_score: 0.8840\n",
      "Epoch 9/50\n",
      "159/159 [==============================] - 135s 846ms/step - loss: 0.0912 - f2_score: 0.8952 - val_loss: 0.0964 - val_f2_score: 0.8880\n",
      "Epoch 10/50\n",
      "159/159 [==============================] - 134s 841ms/step - loss: 0.0890 - f2_score: 0.8967 - val_loss: 0.0990 - val_f2_score: 0.8874\n",
      "Epoch 11/50\n",
      "159/159 [==============================] - 135s 852ms/step - loss: 0.0877 - f2_score: 0.8998 - val_loss: 0.0638 - val_f2_score: 0.8933\n",
      "Epoch 12/50\n",
      "159/159 [==============================] - 135s 847ms/step - loss: 0.0857 - f2_score: 0.9013 - val_loss: 0.0687 - val_f2_score: 0.8900\n",
      "Epoch 13/50\n",
      "159/159 [==============================] - 135s 849ms/step - loss: 0.0847 - f2_score: 0.9016 - val_loss: 0.0955 - val_f2_score: 0.8953\n",
      "Epoch 14/50\n",
      "159/159 [==============================] - 135s 848ms/step - loss: 0.0838 - f2_score: 0.9031 - val_loss: 0.1565 - val_f2_score: 0.8894\n",
      "Epoch 15/50\n",
      "159/159 [==============================] - 136s 856ms/step - loss: 0.0821 - f2_score: 0.9037 - val_loss: 0.0661 - val_f2_score: 0.8925\n",
      "Epoch 16/50\n",
      "159/159 [==============================] - 135s 848ms/step - loss: 0.0803 - f2_score: 0.9076 - val_loss: 0.0979 - val_f2_score: 0.8936\n",
      "Epoch 17/50\n",
      "159/159 [==============================] - 134s 846ms/step - loss: 0.0790 - f2_score: 0.9076 - val_loss: 0.1120 - val_f2_score: 0.8944\n",
      "Epoch 18/50\n",
      "159/159 [==============================] - 134s 844ms/step - loss: 0.0773 - f2_score: 0.9106 - val_loss: 0.1172 - val_f2_score: 0.8888\n",
      "Epoch 19/50\n",
      "159/159 [==============================] - 136s 852ms/step - loss: 0.0756 - f2_score: 0.9114 - val_loss: 0.0661 - val_f2_score: 0.8973\n",
      "Epoch 20/50\n",
      "159/159 [==============================] - 135s 848ms/step - loss: 0.0740 - f2_score: 0.9139 - val_loss: 0.1151 - val_f2_score: 0.9003\n",
      "Epoch 21/50\n",
      "159/159 [==============================] - 134s 845ms/step - loss: 0.0730 - f2_score: 0.9137 - val_loss: 0.0933 - val_f2_score: 0.8869\n",
      "Epoch 22/50\n",
      "159/159 [==============================] - 135s 847ms/step - loss: 0.0700 - f2_score: 0.9173 - val_loss: 0.0881 - val_f2_score: 0.8957\n",
      "Epoch 23/50\n",
      "159/159 [==============================] - 136s 854ms/step - loss: 0.0698 - f2_score: 0.9187 - val_loss: 0.0737 - val_f2_score: 0.8943\n",
      "Epoch 24/50\n",
      "159/159 [==============================] - 135s 846ms/step - loss: 0.0682 - f2_score: 0.9208 - val_loss: 0.1317 - val_f2_score: 0.8924\n",
      "Epoch 25/50\n",
      "159/159 [==============================] - 134s 843ms/step - loss: 0.0661 - f2_score: 0.9236 - val_loss: 0.1473 - val_f2_score: 0.8914\n",
      "Epoch 26/50\n",
      "159/159 [==============================] - 133s 837ms/step - loss: 0.0647 - f2_score: 0.9238 - val_loss: 0.0558 - val_f2_score: 0.8947\n",
      "Epoch 27/50\n",
      "159/159 [==============================] - 135s 847ms/step - loss: 0.0638 - f2_score: 0.9260 - val_loss: 0.1356 - val_f2_score: 0.8963\n",
      "Epoch 28/50\n",
      "159/159 [==============================] - 133s 839ms/step - loss: 0.0617 - f2_score: 0.9275 - val_loss: 0.0774 - val_f2_score: 0.8907\n",
      "Epoch 29/50\n",
      "159/159 [==============================] - 134s 844ms/step - loss: 0.0608 - f2_score: 0.9294 - val_loss: 0.0637 - val_f2_score: 0.8918\n",
      "Epoch 30/50\n",
      "159/159 [==============================] - 134s 841ms/step - loss: 0.0612 - f2_score: 0.9282 - val_loss: 0.1271 - val_f2_score: 0.8957\n",
      "Epoch 31/50\n",
      "159/159 [==============================] - 135s 848ms/step - loss: 0.0572 - f2_score: 0.9341 - val_loss: 0.1493 - val_f2_score: 0.8894\n",
      "Epoch 32/50\n",
      "159/159 [==============================] - 134s 846ms/step - loss: 0.0584 - f2_score: 0.9321 - val_loss: 0.0878 - val_f2_score: 0.8902\n",
      "Epoch 33/50\n",
      "159/159 [==============================] - 134s 841ms/step - loss: 0.0555 - f2_score: 0.9352 - val_loss: 0.0493 - val_f2_score: 0.8855\n",
      "Epoch 34/50\n",
      "159/159 [==============================] - 133s 839ms/step - loss: 0.0556 - f2_score: 0.9353 - val_loss: 0.1447 - val_f2_score: 0.8887\n",
      "Epoch 35/50\n",
      "159/159 [==============================] - 135s 850ms/step - loss: 0.0538 - f2_score: 0.9374 - val_loss: 0.0565 - val_f2_score: 0.8934\n",
      "Epoch 36/50\n",
      "159/159 [==============================] - 135s 852ms/step - loss: 0.0522 - f2_score: 0.9385 - val_loss: 0.1127 - val_f2_score: 0.8933\n",
      "Epoch 37/50\n",
      "159/159 [==============================] - 134s 842ms/step - loss: 0.0509 - f2_score: 0.9415 - val_loss: 0.2640 - val_f2_score: 0.8852\n",
      "Epoch 38/50\n",
      "159/159 [==============================] - 133s 840ms/step - loss: 0.0489 - f2_score: 0.9427 - val_loss: 0.1170 - val_f2_score: 0.8906\n",
      "Epoch 39/50\n",
      "159/159 [==============================] - 135s 848ms/step - loss: 0.0485 - f2_score: 0.9442 - val_loss: 0.1169 - val_f2_score: 0.8935\n",
      "Epoch 40/50\n",
      "159/159 [==============================] - 135s 847ms/step - loss: 0.0470 - f2_score: 0.9451 - val_loss: 0.1340 - val_f2_score: 0.8874\n",
      "Epoch 41/50\n",
      "159/159 [==============================] - 134s 842ms/step - loss: 0.0456 - f2_score: 0.9484 - val_loss: 0.0556 - val_f2_score: 0.8883\n",
      "Epoch 42/50\n",
      "159/159 [==============================] - 134s 842ms/step - loss: 0.0450 - f2_score: 0.9475 - val_loss: 0.1052 - val_f2_score: 0.8859\n",
      "Epoch 43/50\n",
      "159/159 [==============================] - 134s 846ms/step - loss: 0.0432 - f2_score: 0.9504 - val_loss: 0.0725 - val_f2_score: 0.8819\n",
      "Epoch 44/50\n",
      "159/159 [==============================] - 134s 845ms/step - loss: 0.0426 - f2_score: 0.9510 - val_loss: 0.2277 - val_f2_score: 0.8939\n",
      "Epoch 45/50\n",
      "159/159 [==============================] - 134s 843ms/step - loss: 0.0412 - f2_score: 0.9523 - val_loss: 0.0997 - val_f2_score: 0.8868\n",
      "Epoch 46/50\n",
      "159/159 [==============================] - 133s 839ms/step - loss: 0.0405 - f2_score: 0.9530 - val_loss: 0.0778 - val_f2_score: 0.8917\n",
      "Epoch 47/50\n",
      "159/159 [==============================] - 135s 849ms/step - loss: 0.0394 - f2_score: 0.9544 - val_loss: 0.1000 - val_f2_score: 0.8943\n",
      "Epoch 48/50\n",
      "159/159 [==============================] - 135s 846ms/step - loss: 0.0386 - f2_score: 0.9563 - val_loss: 0.2125 - val_f2_score: 0.8887\n",
      "Epoch 49/50\n",
      "159/159 [==============================] - 134s 841ms/step - loss: 0.0387 - f2_score: 0.9554 - val_loss: 0.0980 - val_f2_score: 0.8902\n",
      "Epoch 50/50\n",
      "159/159 [==============================] - 134s 840ms/step - loss: 0.0365 - f2_score: 0.9584 - val_loss: 0.1474 - val_f2_score: 0.8878\n",
      "Found 20240 validated image filenames belonging to 17 classes.\n",
      "Found 20239 validated image filenames belonging to 17 classes.\n",
      "Found 61191 validated image filenames belonging to 17 classes.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras_applications/mobilenet.py:207: UserWarning: `input_shape` is undefined or non-square, or `rows` is not in [128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "  warnings.warn('`input_shape` is undefined or non-square, '\n",
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:9: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=Tensor(\"in..., outputs=Tensor(\"de...)`\n",
      "  if __name__ == '__main__':\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      " 69/159 [============>.................] - ETA: 1:14 - loss: 0.2908 - f2_score: 0.7423"
     ]
    }
   ],
   "source": [
    "#Run the model  with k-fold cross validation:\n",
    "\n",
    "num_fold = 0\n",
    "EPOCHS = 50\n",
    "\n",
    "y_test = []\n",
    "\n",
    "folds = KFold(n_splits=NFOLDS, shuffle=True, random_state=1).split(X_train_files, y_train)\n",
    "\n",
    "for train_index, val_index in folds:\n",
    "    X_train_files_fold = X_train_files[train_index]\n",
    "    y_train_fold = y_train[train_index]\n",
    "    X_val_files_fold = X_train_files[val_index]\n",
    "    y_val_fold = np.array(y_train[val_index])\n",
    "    \n",
    "    train_df = pd.DataFrame(list(zip(X_train_files_fold, y_train_fold)), columns = ['image_name', 'tags'])\n",
    "    val_df = pd.DataFrame(list(zip(X_val_files_fold, y_val_fold)), columns = ['image_name', 'tags'])\n",
    "    \n",
    "    train_df['tags'] = train_df['tags'].apply(lambda x: x.split(' '))\n",
    "    val_df['tags'] = val_df['tags'].apply(lambda x: x.split(' '))\n",
    "\n",
    "    train_datagen = ImageDataGenerator(\n",
    "        rescale=1./255,\n",
    "        width_shift_range=0.2,\n",
    "        height_shift_range=0.2,\n",
    "        horizontal_flip=True,\n",
    "        vertical_flip=True\n",
    "    )\n",
    "    \n",
    "    train_generator = train_datagen.flow_from_dataframe(\n",
    "        train_df,\n",
    "        directory=TRAIN_PATH,\n",
    "        x_col='image_name',\n",
    "        y_col='tags',\n",
    "        target_size=(INPUT_SHAPE[0], INPUT_SHAPE[1]),\n",
    "        class_mode='categorical',\n",
    "        batch_size=BATCH_SIZE,\n",
    "        classes=labels,\n",
    "    )\n",
    "    \n",
    "    val_datagen = ImageDataGenerator(\n",
    "        rescale=1./255\n",
    "    )\n",
    "    \n",
    "    val_generator = val_datagen.flow_from_dataframe(\n",
    "        val_df,\n",
    "        directory=TRAIN_PATH,\n",
    "        x_col='image_name',\n",
    "        y_col='tags',\n",
    "        target_size=(INPUT_SHAPE[0], INPUT_SHAPE[1]),\n",
    "        class_mode='categorical',\n",
    "        batch_size=BATCH_SIZE,\n",
    "        classes=labels,\n",
    "    )\n",
    "    \n",
    "    test_datagen = ImageDataGenerator(\n",
    "        rescale=1./255\n",
    "    )\n",
    "    \n",
    "    test_generator = test_datagen.flow_from_dataframe(\n",
    "        df_test,\n",
    "        directory=TEST_PATH,\n",
    "        x_col='image_name',\n",
    "        y_col='tags',\n",
    "        target_size=(INPUT_SHAPE[0], INPUT_SHAPE[1]),\n",
    "        class_mode='categorical',\n",
    "        batch_size=BATCH_SIZE,\n",
    "        classes=labels,\n",
    "        shuffle=False,\n",
    "    )\n",
    "\n",
    "    model_path_of_fold = os.path.join('', 'weights_of_fold_' + str(num_fold) + '.h5')\n",
    "    \n",
    "    clear_session()\n",
    "    model = create_model()\n",
    "    \n",
    "    adam = Adam(learning_rate=LR)\n",
    "    model.compile(loss='binary_crossentropy', optimizer=adam, metrics=[f2_score])\n",
    "    \n",
    "    callbacks = [\n",
    "        ModelCheckpoint(model_path_of_fold, monitor='val_f2_score', save_best_only=True, mode='max'),\n",
    "        ReduceLROnPlateau(monitor='loss', factor=0.1, patience=3, mode='min', min_lr=0.000001)\n",
    "    ]\n",
    "    \n",
    "    model.fit_generator(train_generator, epochs=EPOCHS, validation_data=val_generator, callbacks=callbacks,\n",
    "                       workers=WORKERS, use_multiprocessing=False, max_queue_size=MAXQ)\n",
    "\n",
    "    model.load_weights(model_path_of_fold)\n",
    "\n",
    "    p_test = model.predict_generator(test_generator, workers=WORKERS, use_multiprocessing=False, max_queue_size=MAXQ)\n",
    "    y_test.append(p_test)\n",
    "    num_fold+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>agriculture</th>\n",
       "      <th>artisinal_mine</th>\n",
       "      <th>bare_ground</th>\n",
       "      <th>blooming</th>\n",
       "      <th>blow_down</th>\n",
       "      <th>clear</th>\n",
       "      <th>cloudy</th>\n",
       "      <th>conventional_mine</th>\n",
       "      <th>cultivation</th>\n",
       "      <th>habitation</th>\n",
       "      <th>haze</th>\n",
       "      <th>partly_cloudy</th>\n",
       "      <th>primary</th>\n",
       "      <th>road</th>\n",
       "      <th>selective_logging</th>\n",
       "      <th>slash_burn</th>\n",
       "      <th>water</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.000882</td>\n",
       "      <td>0.000205</td>\n",
       "      <td>0.000296</td>\n",
       "      <td>0.009190</td>\n",
       "      <td>0.000427</td>\n",
       "      <td>0.982471</td>\n",
       "      <td>0.000660</td>\n",
       "      <td>0.000112</td>\n",
       "      <td>0.001640</td>\n",
       "      <td>0.000319</td>\n",
       "      <td>0.003950</td>\n",
       "      <td>0.002227</td>\n",
       "      <td>0.999936</td>\n",
       "      <td>0.001157</td>\n",
       "      <td>0.001334</td>\n",
       "      <td>0.000341</td>\n",
       "      <td>0.001176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.005979</td>\n",
       "      <td>0.000796</td>\n",
       "      <td>0.000483</td>\n",
       "      <td>0.016046</td>\n",
       "      <td>0.004675</td>\n",
       "      <td>0.993903</td>\n",
       "      <td>0.002476</td>\n",
       "      <td>0.000078</td>\n",
       "      <td>0.008255</td>\n",
       "      <td>0.001928</td>\n",
       "      <td>0.003110</td>\n",
       "      <td>0.003847</td>\n",
       "      <td>0.999662</td>\n",
       "      <td>0.007284</td>\n",
       "      <td>0.015524</td>\n",
       "      <td>0.000339</td>\n",
       "      <td>0.004722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.006050</td>\n",
       "      <td>0.000126</td>\n",
       "      <td>0.000510</td>\n",
       "      <td>0.000370</td>\n",
       "      <td>0.000444</td>\n",
       "      <td>0.000291</td>\n",
       "      <td>0.001739</td>\n",
       "      <td>0.000738</td>\n",
       "      <td>0.005822</td>\n",
       "      <td>0.005035</td>\n",
       "      <td>0.000335</td>\n",
       "      <td>0.997362</td>\n",
       "      <td>0.998415</td>\n",
       "      <td>0.007003</td>\n",
       "      <td>0.002690</td>\n",
       "      <td>0.000373</td>\n",
       "      <td>0.036711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.243201</td>\n",
       "      <td>0.000901</td>\n",
       "      <td>0.007189</td>\n",
       "      <td>0.006202</td>\n",
       "      <td>0.001951</td>\n",
       "      <td>0.525588</td>\n",
       "      <td>0.000393</td>\n",
       "      <td>0.000440</td>\n",
       "      <td>0.141288</td>\n",
       "      <td>0.002951</td>\n",
       "      <td>0.002359</td>\n",
       "      <td>0.451764</td>\n",
       "      <td>0.999862</td>\n",
       "      <td>0.033785</td>\n",
       "      <td>0.015539</td>\n",
       "      <td>0.001778</td>\n",
       "      <td>0.021101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.010942</td>\n",
       "      <td>0.000208</td>\n",
       "      <td>0.000988</td>\n",
       "      <td>0.000047</td>\n",
       "      <td>0.000355</td>\n",
       "      <td>0.000193</td>\n",
       "      <td>0.166443</td>\n",
       "      <td>0.000179</td>\n",
       "      <td>0.002100</td>\n",
       "      <td>0.006678</td>\n",
       "      <td>0.001854</td>\n",
       "      <td>0.877271</td>\n",
       "      <td>0.837704</td>\n",
       "      <td>0.021272</td>\n",
       "      <td>0.000695</td>\n",
       "      <td>0.000441</td>\n",
       "      <td>0.021323</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   agriculture  artisinal_mine  bare_ground  blooming  blow_down     clear  \\\n",
       "0     0.000882        0.000205     0.000296  0.009190   0.000427  0.982471   \n",
       "1     0.005979        0.000796     0.000483  0.016046   0.004675  0.993903   \n",
       "2     0.006050        0.000126     0.000510  0.000370   0.000444  0.000291   \n",
       "3     0.243201        0.000901     0.007189  0.006202   0.001951  0.525588   \n",
       "4     0.010942        0.000208     0.000988  0.000047   0.000355  0.000193   \n",
       "\n",
       "     cloudy  conventional_mine  cultivation  habitation      haze  \\\n",
       "0  0.000660           0.000112     0.001640    0.000319  0.003950   \n",
       "1  0.002476           0.000078     0.008255    0.001928  0.003110   \n",
       "2  0.001739           0.000738     0.005822    0.005035  0.000335   \n",
       "3  0.000393           0.000440     0.141288    0.002951  0.002359   \n",
       "4  0.166443           0.000179     0.002100    0.006678  0.001854   \n",
       "\n",
       "   partly_cloudy   primary      road  selective_logging  slash_burn     water  \n",
       "0       0.002227  0.999936  0.001157           0.001334    0.000341  0.001176  \n",
       "1       0.003847  0.999662  0.007284           0.015524    0.000339  0.004722  \n",
       "2       0.997362  0.998415  0.007003           0.002690    0.000373  0.036711  \n",
       "3       0.451764  0.999862  0.033785           0.015539    0.001778  0.021101  \n",
       "4       0.877271  0.837704  0.021272           0.000695    0.000441  0.021323  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Average the k results obtained from k-fold cross validation\n",
    "\n",
    "result = np.array(y_test[0])\n",
    "for i in range(1, NFOLDS):\n",
    "    result += np.array(y_test[i])\n",
    "result /= NFOLDS\n",
    "result = pd.DataFrame(result, columns = labels)\n",
    "result.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pickling the result df\n",
    "import pickle\n",
    "pickle1_out = open('result_mb.pickle', 'wb')\n",
    "pickle.dump(result, pickle1_out, protocol=4)\n",
    "pickle1_out.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(61191, 17)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_name</th>\n",
       "      <th>tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>test_0.jpg</td>\n",
       "      <td>[primary, clear, agriculture, road, water]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>test_1.jpg</td>\n",
       "      <td>[primary, clear, agriculture, road, water]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>test_2.jpg</td>\n",
       "      <td>[primary, clear, agriculture, road, water]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>test_3.jpg</td>\n",
       "      <td>[primary, clear, agriculture, road, water]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>test_4.jpg</td>\n",
       "      <td>[primary, clear, agriculture, road, water]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   image_name                                        tags\n",
       "0  test_0.jpg  [primary, clear, agriculture, road, water]\n",
       "1  test_1.jpg  [primary, clear, agriculture, road, water]\n",
       "2  test_2.jpg  [primary, clear, agriculture, road, water]\n",
       "3  test_3.jpg  [primary, clear, agriculture, road, water]\n",
       "4  test_4.jpg  [primary, clear, agriculture, road, water]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:3: FutureWarning: \n",
      ".ix is deprecated. Please use\n",
      ".loc for label based indexing or\n",
      ".iloc for positional indexing\n",
      "\n",
      "See the documentation here:\n",
      "http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#ix-indexer-is-deprecated\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "#Create flags for predicted labels and subission file for kaggle submission\n",
    "\n",
    "preds = []\n",
    "for i in range(result.shape[0]):\n",
    "    a = result.ix[[i]]\n",
    "    a = a.apply(lambda x: x > THRES, axis=1)\n",
    "    a = a.transpose()\n",
    "    a = a.loc[a[i] == True]\n",
    "    ' '.join(list(a.index))\n",
    "    preds.append(' '.join(list(a.index)))\n",
    "    \n",
    "df_test['tags'] = preds\n",
    "df_test['image_name'] = df_test['image_name'].astype(str).str.slice(stop=-4)\n",
    "df_test.to_csv('submit.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}