text_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TASK:  \n",
    "Text classification (50+ classes).  \n",
    "Metric - accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-03T08:05:44.336808Z",
     "start_time": "2019-02-03T08:05:23.717600Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 102
    },
    "colab_type": "code",
    "id": "a5vjSZXscnGs",
    "outputId": "53217a41-8c3e-48b2-b6e7-86933d7f31ee"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n",
      "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import emoji\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from pymystem3 import Mystem\n",
    "from tqdm import tqdm\n",
    "from google.colab import drive\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.model_selection import GridSearchCV, RepeatedStratifiedKFold, train_test_split\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils import to_categorical\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.models import Model, load_model\n",
    "from keras.layers import (Input, Embedding, Dropout, Dense, concatenate,\n",
    "                          SpatialDropout1D, Convolution1D, GlobalMaxPool1D)\n",
    "\n",
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "nltk.download(\"stopwords\")\n",
    "drive.mount('/content/drive')\n",
    "sns.set_style('whitegrid', rc={'grid.linestyle': '--'});\n",
    "pd.options.display.max_columns=100\n",
    "pd.options.display.max_rows=100\n",
    "pd.options.display.max_colwidth = -1\n",
    "pd.options.mode.chained_assignment = None \n",
    "tqdm.pandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0RTDh111cnGy"
   },
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-03T08:05:44.381934Z",
     "start_time": "2019-02-03T08:05:44.338895Z"
    },
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "L4pGNOMDhZFz"
   },
   "outputs": [],
   "source": [
    "# Data directory\n",
    "dir_path = \"drive/My Drive/avito/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-03T08:05:44.427874Z",
     "start_time": "2019-02-03T08:05:44.384572Z"
    },
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "rQJybXiAcnGz"
   },
   "outputs": [],
   "source": [
    "# Function to display key information about dataset\n",
    "def preview_data(data):\n",
    "    print(\"data shape: {}\".format(data.shape))\n",
    "    print(\"size, MB: {:.2f}\".format(data.memory_usage(deep=True).sum() / 1e6))\n",
    "    display(pd.concat([data.isnull().sum().rename(\"NaN\"),\n",
    "                       data.dtypes.rename(\"dtype\")], axis=1))\n",
    "    display(data.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-03T08:05:44.587403Z",
     "start_time": "2019-02-03T08:05:44.430210Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 239
    },
    "colab_type": "code",
    "id": "Dl6gbgGQcnG2",
    "outputId": "5eeb487c-8ac9-4246-dae1-d5477894f8b3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data shape: (54, 1)\n",
      "size, MB: 0.01\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NaN</th>\n",
       "      <th>dtype</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <td>0</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      NaN   dtype\n",
       "name  0    object"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Бытовая электроника|Телефоны|iPhone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Бытовая электроника|Ноутбуки</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            name\n",
       "category_id                                     \n",
       "0            Бытовая электроника|Телефоны|iPhone\n",
       "1            Бытовая электроника|Ноутбуки       "
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Category data\n",
    "category = pd.read_csv(dir_path + \"category.csv\", index_col=\"category_id\")\n",
    "preview_data(category)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-03T08:05:50.408662Z",
     "start_time": "2019-02-03T08:05:44.589438Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 332
    },
    "colab_type": "code",
    "id": "l2Ok7XXHcnG6",
    "outputId": "e7a30f8b-1e6f-45a2-e699-790bb451d700"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data shape: (489517, 4)\n",
      "size, MB: 384.51\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NaN</th>\n",
       "      <th>dtype</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>title</th>\n",
       "      <td>0</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>description</th>\n",
       "      <td>0</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>price</th>\n",
       "      <td>0</td>\n",
       "      <td>float64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_id</th>\n",
       "      <td>0</td>\n",
       "      <td>int64</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             NaN    dtype\n",
       "title        0    object \n",
       "description  0    object \n",
       "price        0    float64\n",
       "category_id  0    int64  "
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>price</th>\n",
       "      <th>category_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>item_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Картина</td>\n",
       "      <td>Гобелен. Размеры 139х84см.</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Стулья из прессованной кожи</td>\n",
       "      <td>Продам недорого 4 стула из светлой прессованной кожи, стильные, ножки дугообразные.</td>\n",
       "      <td>1250.0</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               title  \\\n",
       "item_id                                \n",
       "0        Картина                       \n",
       "1        Стулья из прессованной кожи   \n",
       "\n",
       "                                                                                 description  \\\n",
       "item_id                                                                                        \n",
       "0        Гобелен. Размеры 139х84см.                                                            \n",
       "1        Продам недорого 4 стула из светлой прессованной кожи, стильные, ножки дугообразные.   \n",
       "\n",
       "          price  category_id  \n",
       "item_id                       \n",
       "0        1000.0  19           \n",
       "1        1250.0  22           "
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Train data\n",
    "train = pd.read_csv(dir_path + \"train.csv\", index_col=\"item_id\")\n",
    "preview_data(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-03T08:05:52.708461Z",
     "start_time": "2019-02-03T08:05:50.410619Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 335
    },
    "colab_type": "code",
    "id": "5WDRWyWu3R84",
    "outputId": "b17888a4-fa31-48b5-8cd0-94353f0e32f1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data shape: (243166, 3)\n",
      "size, MB: 188.52\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NaN</th>\n",
       "      <th>dtype</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>title</th>\n",
       "      <td>0</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>description</th>\n",
       "      <td>0</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>price</th>\n",
       "      <td>0</td>\n",
       "      <td>float64</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             NaN    dtype\n",
       "title        0    object \n",
       "description  0    object \n",
       "price        0    float64"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>item_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>489517</th>\n",
       "      <td>Стоик журнальный сталь</td>\n",
       "      <td>продам журнальный столик изготавливаю столы из полированной нержавеющей стали и стекла или камня  на всю продукцию даю гарантию  стоимость зависит от размеров и дизайна</td>\n",
       "      <td>10000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489518</th>\n",
       "      <td>iPhone 5 64Gb</td>\n",
       "      <td>Телефон в хорошем состоянии. Комплект, гарантия.  Салон сотовой связи. Огромный выбор аксессуаров и запчастей. Новые и б/у сотовые телефоны. Возможен обмен. Ремонт сотовых телефонов.  График работы с 8.30 до 18.00</td>\n",
       "      <td>12500.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          title  \\\n",
       "item_id                           \n",
       "489517   Стоик журнальный сталь   \n",
       "489518   iPhone 5 64Gb            \n",
       "\n",
       "                                                                                                                                                                                                                   description  \\\n",
       "item_id                                                                                                                                                                                                                          \n",
       "489517   продам журнальный столик изготавливаю столы из полированной нержавеющей стали и стекла или камня  на всю продукцию даю гарантию  стоимость зависит от размеров и дизайна                                                \n",
       "489518   Телефон в хорошем состоянии. Комплект, гарантия.  Салон сотовой связи. Огромный выбор аксессуаров и запчастей. Новые и б/у сотовые телефоны. Возможен обмен. Ремонт сотовых телефонов.  График работы с 8.30 до 18.00   \n",
       "\n",
       "           price  \n",
       "item_id           \n",
       "489517   10000.0  \n",
       "489518   12500.0  "
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Test data\n",
    "test = pd.read_csv(dir_path + \"test.csv\", index_col=\"item_id\")\n",
    "preview_data(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5P4KutLlcnG9"
   },
   "source": [
    "### Process data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KtZUaXYQcnG-"
   },
   "source": [
    "Expand hierarchy from category data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-03T08:05:52.773078Z",
     "start_time": "2019-02-03T08:05:52.710487Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 213
    },
    "colab_type": "code",
    "id": "uURzaXiucnG-",
    "outputId": "3b638f98-e812-43cb-a951-0329a1fc9eb9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name       0 \n",
      "level_0    0 \n",
      "level_1    0 \n",
      "level_2    4 \n",
      "level_3    47\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>level_0</th>\n",
       "      <th>level_1</th>\n",
       "      <th>level_2</th>\n",
       "      <th>level_3</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Бытовая электроника|Телефоны|iPhone</td>\n",
       "      <td>Бытовая электроника</td>\n",
       "      <td>Телефоны</td>\n",
       "      <td>iPhone</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            name              level_0  \\\n",
       "category_id                                                             \n",
       "0            Бытовая электроника|Телефоны|iPhone  Бытовая электроника   \n",
       "\n",
       "              level_1 level_2 level_3  \n",
       "category_id                            \n",
       "0            Телефоны  iPhone  NaN     "
      ]
     },
     "execution_count": 15,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hier_expanded = category.name.str.split(pat=\"|\", expand=True)\n",
    "hier_expanded.columns = [\"level_\" + str(x) for x in hier_expanded.columns]\n",
    "\n",
    "category = pd.concat([category, hier_expanded], axis=1).replace({None: np.nan})\n",
    "print(category.isnull().sum())\n",
    "category.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-29T20:23:11.340209Z",
     "start_time": "2019-01-29T20:23:11.316146Z"
    },
    "colab_type": "text",
    "id": "_7DY_xpDcnHB"
   },
   "source": [
    "→ There are omitted categories for level_2 and level_3 of hierarchy, they could be filled with values from previous level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-03T08:05:52.819120Z",
     "start_time": "2019-02-03T08:05:52.775140Z"
    },
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "4l1MF0ObXbNM"
   },
   "outputs": [],
   "source": [
    "category.fillna(method=\"ffill\", axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6Shn8wERcnHC"
   },
   "source": [
    "Prepare texts for analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-03T08:05:52.874207Z",
     "start_time": "2019-02-03T08:05:52.821804Z"
    },
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "T2hXM51bcnHD"
   },
   "outputs": [],
   "source": [
    "# Convenient class with text processing methods\n",
    "class TextProcessor:\n",
    "    def __init__(self, text):\n",
    "        self.text = text\n",
    "\n",
    "    def remove_multiple_spaces(self):\n",
    "        self.text = re.sub(pattern=\"\\s+\", repl=\" \", string=self.text)\n",
    "    \n",
    "    def make_lowercase(self):\n",
    "        self.text = self.text.lower()\n",
    "    \n",
    "    def capture_physical_property(self):\n",
    "        repl_dict = {(\"\\d+(?:[\\d,.]*\\d)?\\s*(х|x|\\*)\\s*\\d+(?:[\\d,.]*\\d)\"\n",
    "                      \"?(?:\\s*(х|x|\\*)\\s*\\d+(?:[\\d,.]*\\d)?)?\"): \" size \",\n",
    "                     \"\\d+(?:[\\d,.]*\\d)?\\s*(см|мм|м(етр)?|сантиметр|миллиметр)\": \" length \",\n",
    "                     \"\\d+(?:[\\d,.]*\\d)?\\s*(км|километр(ов)?)\": \" distance \",\n",
    "                     \"\\d+(?:[\\d,.]*\\d)?\\s*(м²|м2|квм|кв.\\s*м)\": \" square \",\n",
    "                     \"\\d+(?:[\\d,.]*\\d)?\\s*(р|р.|руб(лей)?)\": \" price \",\n",
    "                     \"\\d+\\s*(г(од)?)\": \" year \",\n",
    "                     \"\\d+(\\.|\\/)\\d+(\\.|\\/)\\d{4}\": \" date \",\n",
    "                     \"\\d+(?:[\\d,.]*\\d)?\\s*(л(итр)?|мл|миллилитр)\": \" volume \",\n",
    "                     \"\\d+(?:[\\d,.]*\\d)?\\s*(мг|г(рaм)?|кг|т(он)?|тн|килограм)\": \" weight \",\n",
    "                     \"\\d+(?:[\\d,.]*\\d)?\\s*(шт(ук)?|ед(иниц)?)\": \" quantity \",\n",
    "                     \"\\d+(?:[\\d,.]*\\d)?\\s*(вт|ват|w(att)?)\": \" power \",\n",
    "                     \"\\d+(?:[\\d,.]*\\d)?\\s*(в(ольт)?|v(olt)?)\": \" voltage \",\n",
    "                     \"\\d+(?:[\\d,.]*\\d)?\\s*%\": \" percent \",\n",
    "                    }\n",
    "        for pat,repl in repl_dict.items():\n",
    "            self.text = re.sub(pat, repl, self.text)\n",
    "\n",
    "    def capture_urls(self):\n",
    "        pat = \"http[s]?:[/]?[/]?(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+—\"\n",
    "        self.text = re.sub(pattern=pat, repl=\" url \", string=self.text)\n",
    "\n",
    "    def capture_numbers(self):\n",
    "        self.text = re.sub(pattern=r\"\\d+(?:[\\d,.]*\\d)?\", repl=\" num \", string=self.text)\n",
    "\n",
    "    def capture_punctuation(self):\n",
    "        repl_dict = {\"!\": \" exclm \",\n",
    "                     \"?\": \" qstn \"}\n",
    "        regex = re.compile(\"(%s)\" % \"|\".join(map(re.escape, repl_dict.keys())))\n",
    "        self.text = regex.sub(lambda x: repl_dict[x.string[x.start(): x.end()]], self.text)\n",
    "\n",
    "    def remove_punctuation(self):\n",
    "        repl_dict = str.maketrans({key: \" \" for key in \\\n",
    "                                   r\"!\\\"#$%&\\'’()*+,-./:;<=>?@[\\\\]^_`{|}~•…«»\"})\n",
    "        self.text = self.text.translate(repl_dict)\n",
    "\n",
    "    def get_lemmas(self):\n",
    "        tokens = mystem.lemmatize(self.text.lower())\n",
    "        tokens = [token for token in tokens if token not in russian_stopwords \\\n",
    "                                                        and token != \" \"]\n",
    "        self.text = \" \".join(tokens)\n",
    "\n",
    "    def convert_emoji(self):\n",
    "        self.text = emoji.demojize(self.text, delimiters=(\" \", \" \"))\n",
    "\n",
    "    def join_tokens(self):\n",
    "        self.text = \" \".join(self.text.split())\n",
    "        \n",
    "    def process_text(self):\n",
    "        self.remove_multiple_spaces()\n",
    "        self.make_lowercase()\n",
    "        self.capture_physical_property()\n",
    "        self.capture_urls()\n",
    "        self.capture_numbers()\n",
    "        self.capture_punctuation()        \n",
    "        self.remove_punctuation()        \n",
    "        self.get_lemmas()\n",
    "        self.convert_emoji()\n",
    "        self.join_tokens()\n",
    "        return self.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-03T08:44:05.139533Z",
     "start_time": "2019-02-03T08:05:52.876687Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1541
    },
    "colab_type": "code",
    "id": "2VrM9YHDcnHG",
    "outputId": "519ec942-6a7b-4802-d9db-62abbcedb2bd",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 489517/489517 [22:25<00:00, 363.77it/s]\n",
      "100%|██████████| 489517/489517 [03:17<00:00, 2483.38it/s]\n",
      "100%|██████████| 243166/243166 [10:55<00:00, 370.93it/s]\n",
      "100%|██████████| 243166/243166 [01:33<00:00, 2596.57it/s]\n"
     ]
    }
   ],
   "source": [
    "# Initialize Russian stemmer and stopwords\n",
    "mystem = Mystem()\n",
    "russian_stopwords = stopwords.words(\"russian\")\n",
    "\n",
    "# Process descriptions\n",
    "train[\"description_clean\"] = train[\"description\"].progress_apply(\n",
    "                                lambda x: TextProcessor(x).process_text())\n",
    "\n",
    "# Process titles\n",
    "train[\"title_clean\"] = train[\"title\"].progress_apply(\n",
    "                            lambda x: TextProcessor(x).process_text())\n",
    "\n",
    "# The same for test data\n",
    "test[\"description_clean\"] = test[\"description\"].progress_apply(\n",
    "                                lambda x: TextProcessor(x).process_text())\n",
    "\n",
    "test[\"title_clean\"] = test[\"title\"].progress_apply(\n",
    "                            lambda x: TextProcessor(x).process_text())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2019-02-02T00:02:50.234Z"
    },
    "colab_type": "text",
    "id": "hSht14y_cnHJ"
   },
   "source": [
    "### Save / Load clean data from this step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-03T08:46:20.207439Z",
     "start_time": "2019-02-03T08:46:08.202114Z"
    },
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "_ubIdvqLcnHK"
   },
   "outputs": [],
   "source": [
    "# train.to_pickle(dir_path + \"train_clean.pcl\")\n",
    "# test.to_pickle(dir_path + \"test_clean.pcl\")\n",
    "train = pd.read_pickle(dir_path + \"train_clean.pcl\")\n",
    "test = pd.read_pickle(dir_path + \"test_clean.pcl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RO0GaraccnHO"
   },
   "source": [
    "Plot distribution of classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-01T20:47:08.289846Z",
     "start_time": "2019-02-01T20:47:07.114236Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 281
    },
    "colab_type": "code",
    "id": "zNvKaCGscnHP",
    "outputId": "49ac2556-6b5b-4a10-b22b-2497b955bf17"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAs8AAAEICAYAAACgdxkmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3X1YVGX+P/A3ILMuhBAKJSVsQRte\nLT5hmhgmPoWY9qjmA1namq1YGZuiv0zNsqy1LV1LW0tdN9PNDXXN1L6t2ANkNqSQgS7kYgnpYIIG\nGqPcvz/4nvnOw5mZewbmcBjer+vyuuTMfeb+fM79wIdh5hAghBAgIiIiIiK3Als7ACIiIiKitoLF\nMxERERGRJBbPRERERESSWDwTEREREUli8UxEREREJInFMxERERGRpA6tHYAso9HY2iEQERERUTuR\nnJyserzNFM+A8yTq6uoQGhoq/Ty+bu8vfegxJi360GNMWvShx5i06EOPMWnRhx5j0qIPPcakRR96\njEmLPvQYkxZ96DEmLfpoyZhcvWjLt20QEREREUnyi+K5tLRUV+39pQ89xqRFH3qMSYs+9BiTFn3o\nMSYt+tBjTFr0oceYtOhDjzFp0YceY9KiDz3GpEUfWsQE+EnxTERERESkBRbPRERERESSghYtWrSo\ntYOQUVVVhZiYGKePh4WFefR8vm7vL33oMSYt+tBjTFr0oceYtOhDjzFp0YceY9KiDz3GpEUfeoxJ\niz70GJMWfegxJi36aKmYXNWdAUII4e5Jjx07hj/84Q948MEHMXnyZFRVVWHOnDm4fPkyoqKi8PLL\nL8NgMGDHjh3YsGEDAgMDMW7cOIwdOxZmsxk5OTmorKxEUFAQXnjhBXTr1g2lpaVQ6vYbb7wRixcv\ndhmD0Wh0ercNIiIiIqKW4qrudPu2jfr6eixZsgQDBgywHFuxYgUmTpyITZs2IS4uDlu3bkV9fT1W\nrVqF9evXY+PGjdiwYQNqamqwc+dOdOrUCe+++y5mzJiB5cuXAwCef/55zJ8/H5s3b8bPP/+M/fv3\ne51gUVGRrtr7Sx96jEmLPvQYkxZ96DEmLfrQY0xa9KHHmLToQ48xadGHHmPSog89xqRFH3qMSYs+\ntIgJkCieDQYD/vrXvyI6Otpy7MCBAxg6dCgAIC0tDQUFBTh8+DCSkpIQFhaGjh07ok+fPigsLERB\nQQGGDx8OAEhJSUFhYSEaGhpw8uRJ9OjRw+Y5vGU2m3XV3l/60GNMWvShx5i06EOPMWnRhx5j0qIP\nPcakRR96jEmLPvQYkxZ96DEmLfrQY0xa9KFFTIDEH0np0KEDOnSwbXbhwgUYDAYAQOfOnWEymVBd\nXY3IyEhLm8jISIfjgYGBCAgIQHV1NTp16mRpqzyHO9Y3rE5MTATwf7cYMRqN6Nq1K2JiYlBUVGS5\nGCEhIejevTsqKipQXV1tOb+hoQH19fUoLy+3HIuNjUVUVJRNP+Hh4QCAsrIy1NbWWo4nJyfDZDLh\nxIkTlmPx8fEICQlBcXGxJaYuXbogLi4OJSUlqK+vBwAEBwejR48eqKysRFVVleX8uro6m5wAuMzJ\n/pokJSW5zcloNCI8PBwJCQlSOdXU1NjkBMBlTu7GyT4npb2zcVLLSWE/Ts5ysm9rP072OSntnY2T\nWk4KmbmXlJTkEJOzuWefk/JrJHdzz5rM3FNysu5fdj0ZjUapuWfNk/VkHZfsejIajW7nnv2GKTP3\nlJysY5JdT0aj0e3cs84JgNTcU3Kyjkl2PRmNRum5p5CZe0pO1jF5sp4qKyul93KgaZ+S2csTEhJs\nYgLk9nKgaX7I7uVA034uu5dXVFTYxCW7nsrKyqT3cqDp+57sXl5ZWWkTk+x6Kikpkd7LY2NjbfoA\n3K8n6/ay66moqEh67nXt2tVyjsxervShxOSr9aRcW9m93DqmlqyNFm36wSZ+/O/XiyZeC6D1ayPA\ns71cpjZSJSStWLFCbNy4UQghxC233GI5/t///leMHz9e7NixQzz//POW46+88orYvHmzeOihh0RJ\nSYnleGpqqjh58qS48847Lcc+//xz8eSTT7rs/6uvvnL62Lfffiubhibt/aUPPcakRR96jEmLPvQY\nkxZ96DEmLfrQY0xa9KHHmLToQ48xadGHHmPSog9fxXTHk9tU//kqrtbM21XdKfWBQQBYuXIlrrzy\nSkyePBlDhw7FBx98gI4dO+LLL7/E3//+d0yaNAlbtmzBK6+8AgCYN28eRowYgT179mDUqFFITU2F\n2WzGkCFD8O9//xvDhw9HXl4eACA3NxfHjh3D3LlznfbPDwwS6cPo7O2qx/+1/E6NIyE94vwg8l/t\naX036wODalJSUrBnzx4AwN69e5GamoqePXuiuLgY586dQ11dHQoLC9G3b18MHDgQu3fvBgDs27cP\n/fv3R3BwMK6//np89dVXNs/hLeVXXi3ZfnT2dtV/rRmT1n3oMSYt+tBjTFr04U1MnvKXvP2hDy1i\n8oY/5K2XPvh9TJ8xadEH17dvz3FbPH/zzTfIzMxEbm4u/va3vyEzMxNZWVnYtm0bJk6ciJqaGtx1\n113o2LEjsrOzMW3aNDz00EOYOXMmwsLCkJGRgcbGRkyYMAHvvPMOsrOzAQDz58/HK6+8gvvvvx+x\nsbFISUnxOHiF/fvfWrq9N7SIydd96DEmLfrQY0xa9KHHdeHNOb5qb118ZK045FEh0pbzbu45vu5D\nj3nrtQ9P+UPeeoxJiz64vn17jtsPDP7ud7/Dxo0bHY6vW7fO4Vh6ejrS09Ntjin3draXkJCATZs2\neRIrEZEqhyL2fz/E4o+/SiTSE649Oao/aPNatVlui2ciIvIPLHSIiJrPL4pn5dZfvmrvDS1i8nUf\neoxJ9pzmFAltOW8t23tDi7z19vze9uEv4+frPtrq2mvuK5F6HAst+tDLeGv9g6he8m6utjre9vyi\neK6vr7fcd9oX7b2hRUy+7kOPMXl7jq+fX495+8O68PYcd7T+xLgex9sbeuzDH9aeN9prH3ocby3I\nxNTcfY3jLc+ru23ojdof0GjJ9t7QIiZf96HHmLw9x9fPr8e8/WFdeHuO3uhxvGVZf1Ayc8l+j+/Y\n4Ck9znO9jAX70Od4a0GPY6FFH75q39x9zS+KZyIiIiIiLbB4JiIiIiKS5BfveY6NjfVpe29oEZOv\n+9BjTN6e4+vn90Xezf1AkT+sC2/P0Ru9zKm2QJO15OGHu/Q4Fu21D3/4HuMNPY6FFn3oce0BflI8\nR0VF+bS9N7SIydd96DEmb89xp7kftGivc9BTehlvrel1LemRHvP2h5j8pY+2+j2mufQ4FjJcvY9Y\n5vurHtce4Cdv2zAajT5t7w0tYvJ1H3qMydtzfK29zkFP+ct4e0qva0mP9Ji3P8TkL33we0zb7sNT\nelx7gJ8Uz0REREREWvCLt20QERH5O/6FSDm8Tu1La4y3XxTP4eHhPm3vDS1i8nUfeozJ23N8ra3O\nQa03HX8Zb0/pdS3pkR7z1mNMWmiv11aP48e89cUviueEhASftveGFjH5ug89xuTtOb7mD3NQCzIx\nqX/A5AiAtvvKkV7Xkh7pMW89xqSF9npt9Th+zFtf/OI9z2VlZT5t7w0tYvJ1H3qMydtzfM0f5qAW\n9BiTFvS6lvRIj3nrMSYttNdrq8fxY9764hevPNfW1vq0vTe0iMnXfegxJm/P8TV/mINa0GNMWtDr\nWtIjPeatx5i00J6ureNvvPT12y49joUW9BgT4CfFM1Fr4IdSiIiI2h8Wz0REPqDFD1f8AY6ISHt+\nUTwnJyf7tL03tIjJ133oMSZvz9EbPc5BX2GB5x29jJ/W2up+7o/zXC/Xtjnt/YUex0ILeowJaKPF\nc3P/3KPJZPL5n3D0tA9vYvJ1H3qMydtz9EaPc5D0pb2OnxZ7iKc4Fq3bh7Pv+W35BxNP6WUstKbH\nmAA/uduGp06cOKG7PryJydd96DEmb8/RGz3OQdKX9jp+WuwhnuJYtO0+/IGvrtPo7O2Wf1NfzLf8\nXy/0Oj/a5CvP1L6oLmQ/+JUoERERtT0snv0cP7RERERE1HLa5ds24uPjddeHNzH5Og89Xid/wWtL\n7rTX8dNi7/QUx6Jt9+EP2ut10mve7fKV55CQEJ88r6cfamjuhyB8lYdWz69VH3rEa0vutNfx8zRv\nriXf4bXVj/Z6nfSad7ssnouLi3V7+xNP+CIPrT/V7C9j4Skt8m6v19ZftNfx8zRvriXf4bXVj/Z0\nndrC3VXaZfFMREREjvgZFiL3WDwTEVGbwgKPiFpTuyyeu3Tp0tohtAh/yMMfcvCGr/JmUeE/uDb0\nQ48x+QteWzm8TvrSLu+2ERcX19ohtAh/yMMfcvBGe82b5LXXOaLHvPUYk7/gtZXD66Qv7fKV55KS\nEnTv3r21w2g2f8jDH3LwRnvNm+S11zmix7z1GJO/4LV1ri18cK69apevPNfX17d2CC3CH/Lwhxy8\n0V7zJnntdY7oMW89xuQveG2pLWqXrzy3Vfwz1W0b349M5Eiv+xrXKxE541XxXFdXh7lz56K2thZm\nsxkzZ85EVFQUFi1aBAC48cYbsXjxYgDA2rVrsXv3bgQEBCArKwu33XYbzp8/j+zsbJw/fx4hISFY\nvnw5IiIiWiwpd4KDgzXri1zjWBCp49qg9oDznNoir4rn3NxcXHfddcjOzsapU6cwZcoUREVFYf78\n+ejRoweys7Oxf/9+XH/99di1axc2b96Mn3/+GRMnTsStt96KDRs2oF+/fnj44YexZcsW/PWvf8VT\nTz3V0rk51aNHD836Itc4FkTquDaoPeA8p7bIq+L5yiuvxNGjRwEA586dQ0REBE6ePGlZBGlpaSgo\nKIDJZEJqaioMBgMiIyNxzTXXoKysDAUFBVi6dKml7YwZM1ooHTmVlZWIiYnRtE9Sx7EgauLsw0EA\n3ypA/ovfA6gt8qp4HjVqFN5//30MHz4c586dwxtvvIFnn33W8njnzp1hMpkQERGByMhIy/HIyEiY\nTCZUV1dbjnfu3BmnT59uZhru8VOr+sGxICIiAKiqqmLxTG2OV8Xz9u3bERMTg7feegulpaWYOXMm\nwsLCLI8LIVTPUzvurK0ao9Eo1a6oqAhmsxkAEBIS4vI2ONbPGRsbi6ioKJf9KI8lJyfDZDLhxIkT\nUjGVlJRYPlUcHBzs8ldV1v137doVMTExKCoqcvn8RqMRSUlJqK+vR3l5uVRMZWVlqK2ttXydnJws\nFVOXLl0QFxdnk5OzcxITEwEApaWlUjFVVFSgurra8nVSUpLL9kpc4eHhSEhIcMhJrX18fDxCQkJQ\nXFwsFVNlZSWqqqosXys5uYtJmXv2Oam1l5l71uznnpKTu5iUuWefk1p767lnvZ6cqampsZl7Sk7u\nYgLk11NFRYXD3HP1nsm6ujqbuafkJBOT7HoqKyuTmnuKhoYGm7mnrCdXlLhk15Ny+y93c0+tD8B2\nPblrL7ueioqKpOae/Tn2e3lFRYXbmGTXk9FoVJ178fHxTs9R28srKyvdxqS2npxpaGhwmHuxsbFO\n26vt5SaTyW1Msns50LSWADisJ2fU9vL6+npkLtnv2Ph/P4y5aOK10ns50LTn2M89V39MxNle7mo9\nGY1G6b0caNqb7edeeHi4y3Os28qsJ6PRKL2XA//36n57q42U9s2tjdR4VTwXFhbi1ltvBdA0+X75\n5RdcunTJ8vipU6cQHR2N6OhoHD9+XPW4yWRCWFiY5ZgMS3H3vwvNGU/eQ6VWMCYnJzvtw7p9VFSU\nbXHgIi5P7mOpFlNTTscdG9udYzAYbM93EVNCQkKzYrLkJHGtZGOKi4vz6Gbw9nFZcmrBmGJiYjx6\nZcQ+JktOLRiTw9zzMCZLThIxOa6nY6rnREREuPwBzF1MlpzczA9AbT2pr43Q0NBmxWSznpzEpcw5\nh/XkpL3DGvUiLncxKdfHYT25uLZqMTXldKRFYlLmkcN6chGT2l7elM+hFolJedyT9aS2l7vaH+xj\nsuTkIm+DweDRPFHby13lY/+8Njk5iSs0NFT1XGfU9nKDweDyHOvntsnJSUzKjQYcY1KfH872ctnv\n+TY5OWmvXHfHmNTXkXpb+ZhscnLSXnm8VWojN/WaL2sj6/be1Eaufljw6j7PcXFxOHz4MADg5MmT\nCA0NRXx8PL766isAwN69e5GamopbbrkFeXl5aGhowKlTp3D69GkkJCRg4MCB2L17t01bIiIiIiK9\n8+qV5/Hjx2P+/PmYPHkyLl26hEWLFiEqKgrPPPMMGhsb0bNnT6SkpAAAxo0bh8mTJyMgIACLFi1C\nYGAgMjMz8dRTT2HixIno1KkTXn755RZNioiIiIjIF7wqnkNDQ/Haa685HN+0aZPDsczMTGRmZjqc\n//rrr3vTNRERERFRq2mXf56biIiIiMgbLJ6JiIiIiCSxeCYiIiIiksTimYiIiIhIEotnIiIiIiJJ\nLJ6JiIiIiCSxeCYiIiIiksTimYiIiIhIEotnIiIiIiJJLJ6JiIiIiCSxeCYiIiIiksTimYiIiIhI\nEotnIiIiIiJJLJ6JiIiIiCSxeCYiIiIiksTimYiIiIhIEotnIiIiIiJJLJ6JiIiIiCSxeCYiIiIi\nksTimYiIiIhIEotnIiIiIiJJLJ6JiIiIiCSxeCYiIiIiksTimYiIiIhIEotnIiIiIiJJLJ6JiIiI\niCSxeCYiIiIiksTimYiIiIhIEotnIiIiIiJJLJ6JiIiIiCSxeCYiIiIiksTimYiIiIhIUgdvT9yx\nYwfWrl2LDh064LHHHsONN96IOXPm4PLly4iKisLLL78Mg8GAHTt2YMOGDQgMDMS4ceMwduxYmM1m\n5OTkoLKyEkFBQXjhhRfQrVu3lsyLiIiIiKjFefXK89mzZ7Fq1Sps2rQJq1evxscff4wVK1Zg4sSJ\n2LRpE+Li4rB161bU19dj1apVWL9+PTZu3IgNGzagpqYGO3fuRKdOnfDuu+9ixowZWL58eUvnRURE\nRETU4rwqngsKCjBgwABcccUViI6OxpIlS3DgwAEMHToUAJCWloaCggIcPnwYSUlJCAsLQ8eOHdGn\nTx8UFhaioKAAw4cPBwCkpKSgsLCw5TIiIiIiIvIRr9628cMPP+DixYuYMWMGzp07h1mzZuHChQsw\nGAwAgM6dO8NkMqG6uhqRkZGW8yIjIx2OBwYGIiAgAA0NDZbziYiIiIj0yOv3PNfU1OAvf/kLKisr\n8cADD0AIYXnM+v/WPD1uz2g0SrUrKiqC2WwGAISEhKB79+5SzxkbG4uoqCiX/SiPJScnw2Qy4cSJ\nE1IxlZSUoL6+HgAQHByMHj16SMXUtWtXxMTEoKioyOXzG41GJCUlob6+HuXl5VIxlZWVoba21vJ1\ncnKyVExdunRBXFycTU7OzklMTAQAlJaWSsVUUVGB6upqy9dJSUku2ytxhYeHIyEhwSEntfbx8fEI\nCQlBcXGxVEyVlZWoqqqyfK3k5C4mZe7Z56TWXmbuWbOfe0pO7mJS5p59Tmrtreee9Xpypqamxmbu\nKTm5iwmQX08VFRUOcy84ONhp+7q6Opu5p+QkE5PseiorK5Oae4qGhgabuaesJ1eUuGTXU0lJidTc\nU+sDsF1P7trLrqeioiKpuWd/jv1eXlFR4TYm2fVkNBpV5158fLzTc9T28srKSrcxqa0nZxoaGhzm\nXmxsrNP2anu5yWRyG5PsXg40rSUADuvJGbW93F0fRqNRei8HmvYc+7nXpUsXp+2d7eWu1pPRaJTe\ny4Gmvdl+7oWHh7s8x7qtzHoyGo3SeznQlLfaXu7vtZHSvrm1kRqviufOnTujd+/e6NChA2JjYxEa\nGoqgoCBcvHgRHTt2xKlTpxAdHY3o6GibiXb69Gn06tUL0dHRMJlMSExMhNlshhBC6lVnS3G36QeX\n7VxdfKfPaX/MSR/W7aOiomyLAxdxuZqkMjE15XTc7TkGg8H2fBcxJSQkNCsmS04S10o2pri4OLeF\nhKu4LDm1YEwxMTEuCy53MVlyasGYHOaehzFZcpKIyXE9HVM9JyIiwuUPYO5isuTkZn4AautJfW2E\nhoY2Kyab9eQkLmXOOawnJ+0d1qgXcbmLSbk+DuvJxbVVi6kppyMtEpMyjxzWk4uY1PbypnwOtUhM\nyuOerCe1vdzV/mAfkyUnF3kbDAaP5onaXu4qH/vntcnJSVyhoaGq5zqjtpe7+z5v/dw2OTmJKSIi\nwklM6vPD2V4u+z3fJicn7ZXr7hiT+jpSbysfk01OTtorj7dKbeSmXvNlbWTd3pvayNUPC1695/nW\nW2/FF198gcbGRpw9exb19fVISUnBnj17AAB79+5FamoqevbsieLiYpw7dw51dXUoLCxE3759MXDg\nQOzevRsAsG/fPvTv39+bMIiIiIiINOXVK89XXXUVbr/9dowbNw4A8PTTTyMpKQlz587Fli1bEBMT\ng7vuugvBwcHIzs7GtGnTEBAQgJkzZyIsLAwZGRnIz8/HhAkTYDAY8OKLL7ZoUkREREREvuD1e57v\nv/9+3H///TbH1q1b59AuPT0d6enpNseUezsTEREREbUl/AuDRERERESSWDwTEREREUli8UxERERE\nJInFMxERERGRJBbPRERERESSWDwTEREREUli8UxEREREJInFMxERERGRJBbPRERERESSWDwTERER\nEUli8UxEREREJInFMxERERGRJBbPRERERESSWDwTEREREUli8UxEREREJInFMxERERGRJBbPRERE\nRESSWDwTEREREUli8UxEREREJInFMxERERGRJBbPRERERESSWDwTEREREUli8UxEREREJInFMxER\nERGRJBbPRERERESSWDwTEREREUli8UxEREREJInFMxERERGRJBbPRERERESSWDwTEREREUli8UxE\nREREJInFMxERERGRJBbPRERERESSmlU8X7x4EcOGDcP777+PqqoqZGZmYuLEiXj88cfR0NAAANix\nYwfuvfdejB07Fu+99x4AwGw2Izs7GxMmTMDkyZPx/fffNz8TIiIiIiIfa1bx/MYbbyA8PBwAsGLF\nCkycOBGbNm1CXFwctm7divr6eqxatQrr16/Hxo0bsWHDBtTU1GDnzp3o1KkT3n33XcyYMQPLly9v\nkWSIiIiIiHzJ6+K5vLwcZWVlGDx4MADgwIEDGDp0KAAgLS0NBQUFOHz4MJKSkhAWFoaOHTuiT58+\nKCwsREFBAYYPHw4ASElJQWFhYfMzISIiIiLyMa+L52XLliEnJ8fy9YULF2AwGAAAnTt3hslkQnV1\nNSIjIy1tIiMjHY4HBgYiICDA8jYPIiIiIiK96uDNSdu2bUOvXr3QrVs31ceFEC1y3J7RaJRqV1RU\nBLPZDAAICQlB9+7dpZ4zNjYWUVFRLvtRHktOTobJZMKJEyekYiopKUF9fT0AIDg4GD169JCKqWvX\nroiJiUFRUZHL5zcajUhKSkJ9fT3Ky8ulYiorK0Ntba3l6+TkZKmYunTpgri4OJucnJ2TmJgIACgt\nLZWKqaKiAtXV1Zavk5KSXLZX4goPD0dCQoJDTmrt4+PjERISguLiYqmYKisrUVVVZflaycldTMrc\ns89Jrb3M3LNmP/eUnNzFpMw9+5zU2lvPPev15ExNTY3N3FNychcTIL+eKioqHOZecHCw0/Z1dXU2\nc0/JSSYm2fVUVlYmNfcUDQ0NNnNPWU+uKHHJrqeSkhKpuafWB2C7nty1l11PRUVFUnPP/hz7vbyi\nosJtTLLryWg0qs69+Ph4p+eo7eWVlZVuY1JbT840NDQ4zL3Y2Fin7dX2cpPJ5DYm2b0caFpLABzW\nkzNqe7m7PoxGo/ReDjTtOfZzr0uXLk7bO9vLXa0no9EovZcDTXuz/dxT3t7qqg+FzHoyGo3SeznQ\nlLfaXu7vtZHSvrm1kRqviue8vDx8//33yMvLw48//giDwYCQkBBcvHgRHTt2xKlTpxAdHY3o6Gib\niXb69Gn06tUL0dHRMJlMSExMhNlshhDC8qq1K5bibtMPLtu5uvhOn9P+mJM+rNtHRUXZFgcu4nI1\nSWViasrpuNtzDAaD7fkuYkpISGhWTJacJK6VbExxcXFuCwlXcVlyasGYYmJiXBZc7mKy5NSCMTnM\nPQ9jsuQkEZPjejqmek5ERITLH8DcxWTJyc38ANTWk/raCA0NbVZMNuvJSVzKnHNYT07aO6xRL+Jy\nF5NyfRzWk4trqxZTU05HWiQmZR45rCcXMant5U35HGqRmJTHPVlPanu5q/3BPiZLTi7yNhgMHs0T\ntb3cVT72z2uTk5O4QkNDVc91Rm0vd/d93vq5bXJyElNERISTmNTnh7O9XPZ7vk1OTtor190xJvV1\npN5WPiabnJy0Vx5vldrITb3my9rIur03tZGrHxa8etvGq6++in/+85/4xz/+gbFjx+IPf/gDUlJS\nsGfPHgDA3r17kZqaip49e6K4uBjnzp1DXV0dCgsL0bdvXwwcOBC7d+8GAOzbtw/9+/f3JgwiIiIi\nIk159cqzmlmzZmHu3LnYsmULYmJicNdddyE4OBjZ2dmYNm0aAgICMHPmTISFhSEjIwP5+fmYMGEC\nDAYDXnzxxZYKg4iIiIjIZ5pdPM+aNcvy/3Xr1jk8np6ejvT0dJtjQUFBeOGFF5rbNRERERGRpvgX\nBomIiIiIJLF4JiIiIiKSxOKZiIiIiEgSi2ciIiIiIkksnomIiIiIJLF4JiIiIiKSxOKZiIiIiEgS\ni2ciIiIiIkksnomIiIiIJLF4JiIiIiKSxOKZiIiIiEgSi2ciIiIiIkksnomIiIiIJLF4JiIiIiKS\nxOKZiIiIiEgSi2ciIiIiIkksnomIiIiIJLF4JiIiIiKSxOKZiIiIiEgSi2ciIiIiIkksnomIiIiI\nJLF4JiIiIiKSxOKZiIiIiEgSi2ciIiIiIkksnomIiIiIJLF4JiIiIiKSxOKZiIiIiEgSi2ciIiIi\nIkksnomIiIiIJLF4JiIiIiKSxOKZiIiIiEgSi2ciIiIiIkkdvD3xpZdegtFoxKVLl/DII48gKSkJ\nc+bMweXLlxEVFYWXX34ZBoMBO3bswIYNGxAYGIhx48Zh7NixMJvNyMnJQWVlJYKCgvDCCy+gW7du\nLZkXEREREVGL86p4/uKLL/AX/qnSAAAbaElEQVSf//wHW7ZswdmzZ3H33XdjwIABmDhxIkaOHIlX\nXnkFW7duxV133YVVq1Zh69atCA4Oxn333Yfhw4dj37596NSpE5YvX47PPvsMy5cvx6uvvtrSuRER\nERERtSiv3rZx880347XXXgMAdOrUCRcuXMCBAwcwdOhQAEBaWhoKCgpw+PBhJCUlISwsDB07dkSf\nPn1QWFiIgoICDB8+HACQkpKCwsLCFkqHiIiIiMh3vCqeg4KCEBISAgDYunUrBg0ahAsXLsBgMAAA\nOnfuDJPJhOrqakRGRlrOi4yMdDgeGBiIgIAANDQ0NDcXIiIiIiKf8vo9zwDwP//zP9i6dSvefvtt\njBgxwnJcCKHa3tPj9oxGo1S7oqIimM1mAEBISAi6d+8u9ZyxsbGIiopy2Y/yWHJyMkwmE06cOCEV\nU0lJCerr6wEAwcHB6NGjh1RMXbt2RUxMDIqKilw+v9FoRFJSEurr61FeXi4VU1lZGWpray1fJycn\nS8XUpUsXxMXF2eTk7JzExEQAQGlpqVRMFRUVqK6utnydlJTksr0SV3h4OBISEhxyUmsfHx+PkJAQ\nFBcXS8VUWVmJqqoqy9dKTu5iUuaefU5q7WXmnjX7uafk5C4mZe7Z56TW3nruWa8nZ2pqamzmnpKT\nu5gA+fVUUVHhMPeCg4Odtq+rq7OZe0pOMjHJrqeysjKpuadoaGiwmXvKenJFiUt2PZWUlEjNPbU+\nANv15K697HoqKiqSmnv259jv5RUVFW5jkl1PRqNRde7Fx8c7PUdtL6+srHQbk9p6cqahocFh7sXG\nxjptr7aXm0wmtzHJ7uVA01oC4LCenFHby931YTQapfdyoGnPsZ97Xbp0cdre2V7uaj0ZjUbpvRxo\n2pvt5154eLjLc6zbyqwno9EovZcDTXmr7eX+Xhsp7ZtbG6nxunj+9NNPsXr1aqxduxZhYWEICQnB\nxYsX0bFjR5w6dQrR0dGIjo62mWinT59Gr169EB0dDZPJhMTERJjNZgghLK9au2Ip7jb94LKdq4vv\n9Dntjznpw7p9VFSUbXHgIi5Xk1Qmpqacjrs9x2Aw2J7vIqaEhIRmxWTJSeJaycYUFxfntpBwFZcl\npxaMKSYmxmXB5S4mS04tGJPD3PMwJktOEjE5rqdjqudERES4/AHMXUyWnNzMD0BtPamvjdDQ0GbF\nZLOenMSlzDmH9eSkvcMa9SIudzEp18dhPbm4tmoxNeV0pEViUuaRw3pyEZPaXt6Uz6EWiUl53JP1\npLaXu9of7GOy5OQib4PB4NE8UdvLXeVj/7w2OTmJKzQ0VPVcZ9T2cnff562f2yYnJzFFREQ4iUl9\nfjjby2W/59vk5KS9ct0dY1JfR+pt5WOyyclJe+XxVqmN3NRrvqyNrNt7Uxu5+mHBq7dtnD9/Hi+9\n9BLWrFljmbwpKSnYs2cPAGDv3r1ITU1Fz549UVxcjHPnzqGurg6FhYXo27cvBg4ciN27dwMA9u3b\nh/79+3sTBhERERGRprx65XnXrl04e/YsnnjiCcuxF198EU8//TS2bNmCmJgY3HXXXQgODkZ2djam\nTZuGgIAAzJw5E2FhYcjIyEB+fj4mTJgAg8GAF198scUSIiIiIiLyFa+K5/Hjx2P8+PEOx9etW+dw\nLD09Henp6TbHlHs7ExERERG1JfwLg0REREREklg8ExERERFJYvFMRERERCSJxTMRERERkSQWz0RE\nREREklg8ExERERFJYvFMRERERCSJxTMRERERkSQWz0REREREklg8ExERERFJYvFMRERERCSJxTMR\nERERkSQWz0REREREklg8ExERERFJYvFMRERERCSJxTMRERERkSQWz0REREREklg8ExERERFJYvFM\nRERERCSJxTMRERERkSQWz0REREREklg8ExERERFJYvFMRERERCSJxTMRERERkSQWz0REREREklg8\nExERERFJYvFMRERERCSJxTMRERERkSQWz0REREREklg8ExERERFJYvFMRERERCSJxTMRERERkSQW\nz0REREREkjq0ZudLly7F4cOHERAQgPnz56NHjx6tGQ4RERERkUutVjx/+eWXqKiowJYtW1BeXo75\n8+djy5YtrRUOEREREZFbrfa2jYKCAgwbNgwAEB8fj9raWvz888+tFQ4RERERkVutVjxXV1fjyiuv\ntHwdGRkJk8nUWuEQEREREbkVIIQQrdHxggULcNttt1lefZ4wYQKWLl2K6667TrW90WjUMjwiIiIi\naseSk5NVj7fae56jo6NRXV1t+fr06dOIiopy2t5ZAkREREREWmm1t20MHDgQe/bsAQAcOXIE0dHR\nuOKKK1orHCIiIiIit1rtlec+ffrgpptuwv3334+AgAAsXLiwtUIhIiIiIpLSau95JiIiIiJqa/gX\nBomIiIiIJLF4JiIiIiKS1CaL57q6OlRUVKCiogL19fVeP8+5c+ecPqb2bpYff/xR6nl/+uknj+Io\nKCjwqP2lS5dw8uRJXLp0SfocT2Pyhuw7gIQQ+Omnn3DmzBkfR0RERETUstpU8VxcXIz7778fY8eO\nxfz58zFv3jyMGTMGkyZNwtGjRz1+vqysLIdjH330EdLS0jBgwADMnTvX5q8ezpkzx6F9Xl4ebr/9\ndjz44IM4duwYxowZg8zMTAwZMgT79+93aL9t2zabf7m5uVi4cKHlazXPPfec5f/5+fkYPnw4nnji\nCYwYMQKffvqpQ/v9+/fjmWeeAdBUmKelpeGBBx7AkCFDkJeXp9pHnz59sGTJEumC9rPPPsPIkSMx\nadIkFBUV4d5778WgQYOQnp6OL7/8UvWc48ePY8aMGRgzZgyGDh2KRx55xJLLqVOnpPolIiIiak2t\ndrcNbyxduhTPP/884uPjbY4fOXIEzz77LN555x2Hc9SOKdQKtjfffBO5ubno1KkT3nvvPUybNg1r\n165FWFiY6iurb7zxBtatW4fKykrMmDEDr7/+OhITE1FdXY0ZM2bgtttus2m/atUqRERE2Bz/5Zdf\n8MMPPziN0/oHg1WrVuFvf/sbunXrBpPJhKysLKSmptq0X7FiBdasWePQ/uzZs3jkkUcwePBghz5u\nuukmpKenIzs7G127dsU999yD3r17o0MH9SmyatUqbNiwAbW1tcjMzMT69euRmJiIkydP4qmnnsKm\nTZsczlm4cCGef/55dOvWDd999x02btyIhQsX4pNPPsEf//hHbNy40aa92WzGP//5T+Tn51v++mR0\ndDRSU1Nx9913IygoyOk1U/OnP/0Jf/zjH22OnTlzBm+//TZqa2txxx134JZbbrE89uyzz1p+CFGc\nPXsW7733Hq666irceeedWLNmDQoLC3Hddddh+vTpiIyMlIolMzPTIV/F/v37LfOjpqYGK1euxLFj\nx/Db3/4WM2fOVO3j/Pnz+Oqrr5CWloZz585h9erVKC8vdxrXM888g7FjxyIpKUkq3vPnz2Pz5s24\n8sorcc899+Cdd97BkSNHEBcXh8zMTNXbTDY2NuLDDz/EZ599hjNnzkAIgWuuuQZpaWkO6wJo2fFW\nG2vAP8bb07EGON5ajrersQb8Y7w9HWuA4w1wvP1hvBVt6pVnIYRD4Qw0FX6XL19WPWf9+vU4evQo\nzp496/BP7W0PQUFBiIiIQGBgIMaPH4/f//73mDZtGn766ScEBAQ4tDcYDIiJiUHfvn0RHR2NxMRE\nAECXLl3wq1/9yqH9zp07kZKSgqNHj+Luu+9GVlYWrr76amRlZam+Eg7Apt/w8HB069YNABAVFaVa\n3F66dAmhoaEAgLCwMFx77bUAgIiICKdvrQgICMDNN9+M9evXY+LEifjXv/6FO+64A/feey+mT5/u\n0D44OBjR0dG44YYb0KlTJ0ve11xzjdNF0dDQYIn9N7/5jeWHgkGDBuHixYsO7efMmYOqqipMnToV\nL730EpYtW4bJkyejtLQU8+bNU+3jwoULTv8dOnTIof1TTz2FmJgYDBw4EKtWrcKqVassj5WVlanG\n1NDQAKPRiJkzZ+L8+fOYOXMmrr32WtXfTABAYmIiBg0ahKFDh2LIkCEYMmQIDh06hCFDhmDo0KEO\n7d966y3L/5csWYKrrroKixYtQnx8PObPn6/ax2OPPWb5o0OLFy9GWFgYsrKy8Jvf/AZz5851aH/o\n0CG8++67eOyxx5z+psDaU089hcbGRnz33XeYOHEiqqqqcMcddwCA05gWLVqEEydOYMKECejVqxd6\n9eqFYcOGYfv27Vi2bJlDe0/H29OxVvJo6+Pt6VgDHG9fjbenYw34x3h7OtYAxxvgeLe18XZJtCFL\nly4VjzzyiHjvvffExx9/LD7++GOxZcsWMXXqVLF8+XLVc44ePSoyMzPFL7/84vDY5MmTHY4tW7ZM\nTJ8+XVy4cMFy7JNPPhGjR48WqampDu0ff/xxsXbtWptjVVVV4vnnnxdPPvmk01zKy8vF9OnTxRtv\nvCHuu+8+p+2EEOKWW24Rjz32mJg1a5bIyMgQu3btEkII8dZbb4lHHnnEof327dvFiBEjxOLFi0VO\nTo6YMWOGWLNmjZg6darYsmWLah9q10IIIU6dOiW++eYbh+NZWVnilVdeEQsWLBDTpk0TCxYsEHv3\n7hUvvfSSePzxx1Wfa8GCBWL27Nli3bp1YurUqeLVV18VQggxb948kZOT49B+0qRJ6hfExWM33XST\nSEtLs/k3ZMgQkZaWJnr06OEy78uXL4snn3xSrFy50uExRWZmphBCiMbGRjFixAinz2Xtk08+EZMn\nTxa7d++2HBs3bpzT3JQ+hBDigQcekOrDeg7Zt5kwYYJDe6XNd999JxYtWiTGjBkjnn76afH3v//d\nMr+cxZSenu70MVexTpkyRTVehafj7elY28fUVsfb07G2bsfxbtnx9nSsrfsQou2Ot6djLQTHWwiO\nt6vH9DjerrSpt23MmzcPBw8eREFBAYqKigA0/RogKysLvXv3Vj3nt7/9LdasWaP6Cm1OTo7DsTlz\n5uDAgQM2rxqnpqaid+/e2LVrl0P7F198Ef/+979tjp05cwYxMTHIzs52msv111+PNWvWYNu2bZZX\nhp157bXXbL6Oi4sD0PTK8/Llyx3ajxkzBoMGDUJ+fj5OnjwJIQS6dOmCpUuX4qqrrlLt484771Q9\nHh0djejoaIfjy5YtQ25uLm688UZkZGRgx44d+PzzzxEXF4eZM2eqPtfixYvx8ccf47///S+mTJmC\nQYMGAQAeeOAB3HjjjQ7tAwICsHfvXqSlpSE4OBhA06vXe/bsgcFgUO1jzpw5OHPmDGbPnu3wWGZm\npsOxDh06YM+ePRgxYgQCAwPx8ssvY968eViwYAHq6uoc2isf1rzmmmvw9NNPW46XlpbCbDarxpSa\nmor+/ftj9erV2LFjB3JyclR/i6E4e/as5f3yBoMBpaWlSExMxPfff48LFy6onhMbG4ulS5di9OjR\n6N+/Pz788EPcfPPN+OSTT1T/7L3S/3XXXYeFCxfCbDbj4MGDKC4uxvHjxzFy5EiHvCsqKvDTTz+h\ntrYWhw4dQq9evVBeXu40byEEPvvsMyQlJSEvLw8dO3YEANXPAigx7dmzB0OGDJEab0/HGmga7927\nd+P222/X3XgLIRAcHOx2vD0da0D78d63b5/X4717926fj/fTTz/dIuPt6VgDtuMts77tx3vXrl3o\n169fq463p2tbiUltP2/p8fblfs7xllvbSkxtfbxd8qjUJtJQVVWVyMnJEUOGDBEDBgwQAwYMEMOG\nDRMLFiwQp06dcnpebm6uqKurczi+evVqh2OVlZVi7ty5Nr9pEKLp1ftRo0Y5tC8sLHR4Zf2jjz4S\nY8aMEUVFRW5z+u6778T06dPF8OHDhRBC1NbWOrTJycmx+VdQUCCEEGLWrFli7969qs9rNpvFO++8\nIx5++GExcuRIkZ6eLiZPnizefPNNcfr0aYf2jz32mNMY1WL68ssvxT333CMefvhhUVZWJh588EEx\nYMAAMWbMGPH111+rPk95ebl49NFHRUZGhpg9e7aoqqoSQgixYsUKcfjwYYf2yngPHTpUpKSk2Ix3\nZWWlah/btm0TP//8s8Pxv/zlL6rtlT4uXrxoc3z79u1i2LBhDu2//vprp+N98OBB1T4UjY2N4vjx\n4zbjrVwDa8o4z5kzx2G8c3NzHdorY/373/9eZGRk2Iy12vML8X/j3djY6PCY2jkHDx4U9957r5g6\ndarDeDvL29V4q53jarxNJpNqH9u2bbNZ22fOnBFCCPHGG2+otlf6sF7fZ86cETt27BDjx493aK82\n3rt27RKjR49W/Q2cNWWshw4d6rKd2vo+c+aMyMrKEgcOHHBobz3eytrOyMgQa9ascdi3FGrrW7lW\napTxVtb3lClTRPfu3Z2ub2WsR40aJWbPni1+/PFHcebMGfHaa6+J7777TrUP+/FOSUlxO972e3l+\nfr4QwvV42+/n+fn5Ho33hx9+KO68807p8VbWtjNq452fny9mzZrlcryt9/IJEyaIN99802HfUqiN\nt3Kt1Bw8eNBhP+/Xr59H452fny9Wrlwpjh8/rtqH9ffvlJQU0a9fP5GYmCjmz5+v+j1JCNvxNpvN\n4ocffhBms1l6vJVzcnNzpcbbbDaLdevWiYyMDLfjbY/FM7VJzn513JLn+LL9jz/+qElMWvTRUjHt\n3btXDB48WPTr10/MmTNHnD9/3mX7jz76SAwePFj0799fqr11H2rnqP3azlV7T/pwNd7e5i3b3ps8\nPI3Jm3P27dsnRowYIaZMmSJKS0vFHXfcITIyMsTgwYNFXl6eQ/u8vDxL+6NHj4rRo0eLjIwMkZaW\nJvbt26cak3UfyjmjRo1yeo5ae1cxqbVXftWs1t5dH2oxWeddWlpqk7enfTg7pyXydhdTbm6uyM3N\nFe+//77l37BhwyzHnbW3Pm/48OFO23tzTku0Hzp0qFRM77//vkd9WF8nd3krz+1JHspzK/+XGQvr\nftzFtGTJEsv/P//8czF48GBx3333ibS0NPHJJ5941H7//v3SfYwdO9bpOZ7G5EqbetsGtS+e3inF\nm3N83d5f+tAiJus73WzdutXtnW7WrFnj0Z1x7PuwP0ft16+e3n3H1TmA+r3QPe3DOm+Z6+RNH83N\nWyYu6zsVPfroo27vVPT666+7vLOR2l2E3N0Nyf4cT++e5Gl7b2KyzlvmOmmRhzd5q91pymw2O73T\nlDd3pvL0nJZo7yoHLWICmuaIL/PQ4i5h7torb/NszjnHjh3zKCZXWDyTbq1fvx4DBgxQfc+1sz8Q\n4+k5vm7vL31oEZNypxsAGDduHCIjIzFt2jSsXr1atbC1bj9+/Hh07tzZZXt353ja3ps+WiIPT6+T\nFjF5E5dyp6KYmBipOxV52l6LPvQYk17z3rlzJ15//XUcPXoUOTk5uOaaa/Dpp586vcuUp+216EOP\nMek1b0/vEuZpe2/P8ba9A49epybSkKd3SvHmHF+395c+tIjJ0zvdeNreX/rQY0zenOPpnYq8ubOR\nr/vQY0xa9OHtXaaE8OxOU96016IPPcakRR++vEuYp+216sMZFs+ka/X19eLy5csOx129ud/Tc3zd\n3l/60CKmL774wuFDbefPn3d6i0VP2/tLH3qMydNzLly4ID744AObY998841Yt26d6gejPG2vRR96\njEmLPryJyV5ubq544oknpNp6016LPvQYkxZ9yLQ/cOCAzT/lcx87duxQ/ZC3p+216sOZACGcvIGN\niIiIiIhstKm/MEhERERE1JpYPBMRERERSWLxTESkU6dOnUJBQUFrh2HjzTffRF5ensPxP//5z1i5\ncqX2ARERaYy3qiMi0qkDBw6gvLwcAwYMaO1QLKZPn97aIRARtSoWz0REGnv99dfx8ccfIzAwEHfe\neScSExPxpz/9CQaDARcvXsTChQvRqVMnvPrqqxBCICIiApMmTcKzzz6LiooK1NXV4Y477sDUqVPx\nyy+/YO7cuTh58iSuvvpqBAUFYeDAgRg7diy2bt2KzZs349e//jU6d+6M5557DldccQX69OmD++67\nD42Njfjmm28we/Zs9O/fHwDw8MMPIzMzU/UPXQBATk4OkpOTMXbsWPz5z3/Gvn370LVrV/z6179G\nfHy8lpeRiKhVsHgmItLQV199hby8PPzjH/9AY2MjZs2ahauvvhqLFi1CYmIidu7ciTVr1mDFihW4\n++67cenSJTz00ENYu3YtoqOj8dxzz+Hy5csYN24cUlJSUFxcjEuXLuG9996DyWRCRkYGBg4ciMrK\nSqxcuRIffPABrrjiCixbtgzr169HVlYW6uvrcdttt2HgwIHYtm0bcnNz0b9/f9TU1OD48eNSf2nr\n+PHj+Ne//oXdu3cjMDAQY8eOZfFMRO0Ci2ciIg0dPnwYycnJCAoKQlBQEFavXo1Dhw7hpZdewi+/\n/ILz588jPDzc4bwDBw7gxx9/xMGDBwEADQ0NOHHiBEpKStCvXz8ATX8pKzk5GQDw7bff4qabbsIV\nV1wBAOjXrx82b94MoOnPhPfp0wcAMHLkSLz66quoq6vDRx99hNGjRyMw0P3HYY4dO4abbroJBoMB\nANC3b99mXhkioraBxTMRkYYCAgJgf3v9OXPmYPHixRgwYAD27duHt99+2+E8g8GAmTNnIj093eZ4\nfn6+TbHrrPAVQtj8Odvg4GAAwK9+9SsMHz4cH330Efbs2YOFCxdK5WH/fI2NjVLnERG1dbzbBhGR\nhnr37o2CggKYzWZcunQJmZmZOHnyJG644QZcvnwZu3fvRkNDA4CmQvvSpUsAgOTkZHz44YcAmgrV\nF154ATU1Nbj++uvx9ddfAwDOnDkDo9EIAPjd736HI0eO4OeffwbQVGT37NlTNabx48fj3XffhRAC\n3bp1k8ojPj4e3377LRoaGmA2m/Hll196f1GIiNoQvvJMRKSh3r17Y8SIEZg0aRIAYNSoUUhJScGU\nKVMQExODadOmYc6cOVi/fj369u2L2bNnIzg4GI8++ij+85//YPz48bh8+TIGDx6MiIgI3HPPPcjL\ny8P48eNx7bXXom/fvggKCsLVV1+Nxx9/HA899BAMBgOuvvpqPPnkk6oxJSQk4PLly7jnnnuk87jh\nhhswbNgwjBs3DjExMejevXuLXB8iIr3jn+cmImrDTp06hcLCQowcORKNjY24++67sWjRIvTu3Vv6\nOX744QdMnz4d27dvt7ydg4iI1PGVZyKiNiwsLAy7du3CW2+9hYCAAAwaNMijwnn16tXYtWsXlixZ\nYimcV6xYYflgorXExET8v//3/1osdiKitoivPBMRERERSeIHBomIiIiIJLF4JiIiIiKSxOKZiIiI\niEgSi2ciIiIiIkksnomIiIiIJLF4JiIiIiKS9P8BKOm3aL55M+4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train.groupby(\"category_id\").size().plot(kind=\"bar\", figsize=(12,4));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "2mTVXI1mcnHU"
   },
   "source": [
    "### SVM + BOW"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FZoqxRCRcnHV"
   },
   "source": [
    "Fit `SVM with linear kernel` based on bag-of-words features + TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "NRGB-b2Imi_F"
   },
   "outputs": [],
   "source": [
    "# Concatenate title and description\n",
    "train[\"text_clean\"] = train[\"title_clean\"].str.cat(train[\"description_clean\"], sep=\" \")\n",
    "test[\"text_clean\"] = test[\"title_clean\"].str.cat(test[\"description_clean\"], sep=\" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-01T22:01:23.260919Z",
     "start_time": "2019-02-01T21:43:02.407412Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "X68yaCjMcnHX",
    "outputId": "e503c6bd-5a34-447f-9772-60b48a3c7669",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed: 16.2min remaining: 24.3min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean train accuracy: 0.91\n",
      "mean test accuracy: 0.88\n",
      "best params: {'features__bow__vectorizer__binary': False, 'features__bow__vectorizer__lowercase': False, 'features__bow__vectorizer__max_features': 20000, 'features__bow__vectorizer__ngram_range': (1, 1), 'features__bow__vectorizer__norm': None, 'features__bow__vectorizer__smooth_idf': False, 'features__bow__vectorizer__stop_words': None, 'features__bow__vectorizer__sublinear_tf': False, 'features__bow__vectorizer__use_idf': False, 'model__C': 0.02, 'model__penalty': 'l2'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed: 18.3min finished\n"
     ]
    }
   ],
   "source": [
    "# Initialize CV iterator\n",
    "cv = RepeatedStratifiedKFold(n_splits=5, n_repeats=1, random_state=13)\n",
    "\n",
    "# Pipeline\n",
    "pipe = Pipeline([(\"bow\", TfidfVectorizer(analyzer=\"word\")),\n",
    "                 (\"model\", LinearSVC(dual=False, class_weight=\"balanced\"))\n",
    "                ])\n",
    "\n",
    "# Parameters to tune\n",
    "params = {\"model__C\": [0.02],\n",
    "          \"model__penalty\": [\"l2\"],\n",
    "          \"bow__binary\": [False],\n",
    "          \"bow__stop_words\": [None],\n",
    "          \"bow__lowercase\": [False],\n",
    "          \"bow__max_features\": [20_000],\n",
    "          \"bow__ngram_range\": [(1,1)],\n",
    "          \"bow__use_idf\": [False],\n",
    "          \"bow__smooth_idf\": [False],\n",
    "          \"bow__norm\": [None],\n",
    "          \"bow__sublinear_tf\": [False],\n",
    "         }\n",
    "\n",
    "# Grid Search\n",
    "gs = GridSearchCV(estimator=pipe,\n",
    "                  param_grid=params,\n",
    "                  scoring=\"accuracy\",\n",
    "                  cv=cv,\n",
    "                  verbose=1,\n",
    "                  n_jobs=-1,\n",
    "                  return_train_score=True,\n",
    "                  refit=False,\n",
    "                 );\n",
    "\n",
    "gs.fit(train[[\"text_clean\"]], train[\"category_id\"]);\n",
    "\n",
    "# Accuracy\n",
    "acc_train = gs.cv_results_[\"mean_train_score\"][gs.best_index_]\n",
    "acc_test = gs.cv_results_[\"mean_test_score\"][gs.best_index_]\n",
    "\n",
    "print(f\"mean in-sample accuracy: {acc_train:.4f}\")\n",
    "print(f\"mean out-of-sample accuracy: {acc_test:.4f}\")\n",
    "print(\"best params: {}\".format(gs.best_params_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rwbeMXSVcnHa"
   },
   "source": [
    "→ Not bad. Let's try to improve result using more sophisticated algorithm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ARB09pQ2cnHb"
   },
   "source": [
    "### CNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "BXjMA4htcnHh"
   },
   "source": [
    "Split part of train data into validation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-02T00:24:49.720071Z",
     "start_time": "2019-02-02T00:24:49.363997Z"
    },
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "3_ws292BcnHi"
   },
   "outputs": [],
   "source": [
    "# Leave 15% of data for validation set\n",
    "X_train, X_val, y_train, y_val = \\\n",
    "            train_test_split(train, train[\"category_id\"],\n",
    "                             stratify=train[\"category_id\"], \n",
    "                             random_state=13,\n",
    "                             test_size=0.15\n",
    "                            )\n",
    "\n",
    "X_test = test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "F4lXT5hA09JJ"
   },
   "source": [
    "Prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-02T00:25:44.728680Z",
     "start_time": "2019-02-02T00:25:18.749908Z"
    },
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "JiaByLC-cnHn"
   },
   "outputs": [],
   "source": [
    "# Vocabulary size\n",
    "max_words = 100_000\n",
    "\n",
    "# Description\n",
    "tok_desc = Tokenizer(num_words=max_words)\n",
    "tok_desc.fit_on_texts(X_train[\"description_clean\"])\n",
    "desc_train = tok_desc.texts_to_sequences(X_train[\"description_clean\"])\n",
    "desc_val = tok_desc.texts_to_sequences(X_val[\"description_clean\"])\n",
    "desc_test = tok_desc.texts_to_sequences(X_test[\"description_clean\"])\n",
    "\n",
    "# Title\n",
    "tok_title = Tokenizer(num_words=max_words)\n",
    "tok_title.fit_on_texts(X_train[\"title_clean\"])\n",
    "title_train = tok_title.texts_to_sequences(X_train[\"title_clean\"])\n",
    "title_val = tok_title.texts_to_sequences(X_val[\"title_clean\"])\n",
    "title_test = tok_title.texts_to_sequences(X_test[\"title_clean\"])\n",
    "\n",
    "# Other features\n",
    "cols_other = [\"price\"]\n",
    "scaler = StandardScaler();\n",
    "other_train = scaler.fit_transform(X_train[cols_other])\n",
    "other_val = scaler.transform(X_val[cols_other])\n",
    "other_test = scaler.transform(X_test[cols_other])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 265
    },
    "colab_type": "code",
    "id": "44VT1JePmi_W",
    "outputId": "eee75be3-5b4d-46a0-baa9-ec496048b9a3"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmUAAAD4CAYAAACt6JGTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3X9UlOed///ngDPFMSBFBw+4gUaw\nyqlAIqYqSqxJbKw9acwPFSmk3WO6caPGtLRqWWPIp2sxWrqJiVsTTYyrJbqh3cZmPehJq55kRXYd\nzlexRzfFdtEIhSEVUUcL6nz/SGcK3jMywuB1yfV+nJNz5Oaee16v+5rcXNwz3LfN5/P5EEIIIYQQ\nSkWpDiCEEEIIIWRSJoQQQgihBZmUCSGEEEJoQCZlQgghhBAakEmZEEIIIYQGBqkO0Fdut1t1BCGE\nAjk5Oaoj9Jkcv4QwU6jj120/KYPwD84XL15kyJAh/ZwmPLpk0SUH6JNFcljpksWfYyBNZm5mcqnL\nONxKpnU2rS+Y1/lGxy95+1IIIYQQQgNGTcpOnDihOkKALll0yQH6ZJEcVrpk0SWHKib2N62zaX3B\nzM6hGDUpE0IIIYTQlUzKhBBCCCE0YNSkLCkpSXWEAF2y6JID9MkiOax0yaJLDlVM7G9aZ9P6gpmd\nQzFqUpacnKw6QoAuWXTJAfpkkRxWumTRJYcqJvY3rbNpfcHMzqH0OCm7dOkSS5cupbCwkDlz5rBv\n3z6ampooKiqioKCApUuX0tHRAcCuXbt4/PHHmTNnDu+++y4AnZ2dFBcXM3/+fAoLCzl9+jTw2Qf7\n8vPzyc/P54UXXgg83+bNm3niiSeYM2cOBw4ciGjZo0ePRnR7faFLFl1ygD5ZJIeVLll0yaGKif1N\n62xaXzCzcyg9Tsr27dvHuHHj2L59Oy+//DJr1qxh/fr1FBQUUFFRQWpqKpWVlXi9XjZs2MDbb7/N\ntm3b2Lp1K21tbbz//vvExcXxzjvvsHDhQsrLywFYvXo1JSUl7NixgwsXLnDgwAFOnz7N7t27qaio\n4PXXX6esrIyrV69GrGxnZ2fEttVXumTRJQfok0VyWOmSRZccqpjY37TOpvUFMzuH0uOkbNasWXzn\nO98BoKmpiREjRlBTU8MDDzwAwPTp06murubIkSNkZmYSGxtLTEwM48ePp7a2lurqambMmAFAbm4u\ntbW1dHR0cObMGbKysrpto6amhry8PBwOBwkJCYwcOZL6+vr+6i6EEEIIoY2wr+ifn5/Pn/70JzZu\n3Mjf//3f43A4ABg2bBgej4fW1lYSEhIC6yckJFiWR0VFYbPZaG1tJS4uLrCufxvx8fFBtzFmzJgb\nZut6ddyxY8cC3a97kpSURHJyMqUVn0DFJ90e+9qzd9Pa2hr4OjMzE6/Xy8mTJwPLUlJScLlc3Z5n\n6NChpKenU19fz7lz5wLLc3Jy8Hg8nDp1KrAsLS0Np9NJXV1dYFl0dDQAx48fx+v1AmC328nKyqKx\nsZGmpqawOh09ejTwW4bT6SQjI4OGhoawO3Xdf33tNHz4cFJTU3vdadCgz16Ofe3U13Hquk/62qmv\n4+TxeCL+2utNp677JFKvvd52amtrw1T+12d/eLj4PcuyX5c/0m/PF67+7Kwj0/qCmZ1Dsfl8Pl+4\nKx8/fpxly5bh8Xg4dOgQAA0NDSxfvpxvfvOb1NXVUVJSAsC//Mu/kJyczJ49e1i2bFng4H7fffex\nY8cOnnnmGX71q18BcPDgQX7xi18wevRoBg8ezLe+9S0Avv/97zN79mymTp0aMpPb7Q77NiW6HnSE\nEDfnZv6/15lOPeT4KMStcaP/73t8+/LYsWOB35wzMjK4evUqQ4YM4fLlywA0NzeTmJhIYmJit9+Q\nW1paAss9Hg/w2fvGPp8Pl8vV7bfdUNvwLx+IGhoaVEcA9MkB+mSRHFa6ZNElhyom9jets2l9wczO\nofQ4KTt8+DBvvfUWAK2trXi9XnJzc9mzZw8Ae/fuJS8vj+zsbOrq6mhvb+fixYvU1tYyYcIEpkyZ\nQlVVFfDZHw1MnDgRu93OqFGjOHz4cLdtTJo0if3799PR0UFzczMtLS2kp6f3V3eluk4+VdIlB+iT\nRXJY6ZJFlxyqmNjftM6m9QUzO4fS42fK8vPz+ad/+icKCgq4fPkyq1atYty4cSxfvpydO3eSnJzM\n7NmzsdvtFBcXs2DBAmw2G4sWLSI2NpZZs2Zx8OBB5s+fj8PhYM2aNQCUlJSwatUqrl27RnZ2Nrm5\nuQDMnTuXwsJCbDYbpaWlREUZdSk1IYQQQhiqx0lZTExM4DIWXW3ZssWybObMmcycObPbsujoaMrK\nyizrpqenU1FRYVleVFREUVFRT7GEEEIIIQYUOQ2lSGZmpuoIgD45QJ8sksNKlyy65FDFxP6mdTat\nL5jZORSZlCnivxSBarrkAH2ySA4rXbLokkMVE/ub1tm0vmBm51BkUqZI1+s2qaRLDtAni+Sw0iWL\nLjlUMbG/aZ1N6wtmdg5FJmVCCCGEEBqQSZkQQgghhAZkUqZISkqK6giAPjlAnyySw0qXLLrkUMXE\n/qZ1Nq0vmNk5FJmUKeK/76RquuQAfbJIDitdsuiSQxUT+5vW2bS+YGbnUGRSpkjXmzGrpEsO0CeL\n5LDSJYsuOVQxsb9pnU3rC2Z2DkUmZUIIIYQQGpBJmRBCCCGEBnq8zZLoH0OHDlUdAdAnB+iTRXJY\n6ZIlUjkuXrzI8uXLOXfuHJ2dnSxatAiXy0VpaSkAY8aM4cUXXwRg8+bNVFVVYbPZWLx4MdOmTeP8\n+fMUFxdz/vx5nE4n5eXlxMfHc/DgQX76058SHR3Nfffdx6JFiyKS10+XcbiVTOtsWl8ws3MoMilT\nJD09XXUEQJ8coE8WyWGlS5ZI5fiP//gP7rrrLoqLi2lubuZb3/oWLpeLkpISsrKyKC4u5sCBA4wa\nNYrdu3ezY8cOLly4QEFBAVOnTmXr1q18+ctf5qmnnmLnzp1s2rSJH/zgB/zzP/8zb775JiNGjKCw\nsJCHHnooovtOl3G4lUzrbFpfMLNzKPL2pSL19fWqIwD65AB9skgOK12yRCrH5z//edra2gBob28n\nPj6eM2fOkJWVBcD06dOprq6mpqaGvLw8HA4HCQkJjBw5kvr6eqqrq5kxY0a3dU+fPs3QoUNJSkoi\nKiqKadOmUV1dHZG8frqMw61kWmfT+oKZnUORM2WKnDt3TnUEQJ8coE8WyWGlS5ZI5fj617/OL3/5\nS2bMmEF7ezs/+9nP+H//7/8Fvj9s2DA8Hg/x8fEkJCQElickJODxeGhtbQ0sHzZsGC0tLXg8Hsu6\np0+f7jFL1788Gzt2LAAnTpwILEtKSiI5OZmjR4/S2dmJ2+3G6XSSkZFBQ0MDra2tgXUzMzPxer3d\nbluTkpKCy+Xq9jxDhw4lPT2d+vr6G+5Tt9tNWloaTqeTurq6wPLhw4eTmprK8ePHA/cttNvtZGVl\n0djYSFNT0011AkJ2Amhra+t1p5ycHDweD6dOnQosU93pRuN07ty5QK+B0inc195A6xRqnG5EJmVC\nCOO89957JCcn8+abb3LixAkWLVpEbGxs4Ps+ny/o44ItD7VuuHJycsJalpWVhdvt7va91NRUUlNT\nu63ncDjC3ma3t40qPrnhY3Jycni4+L2/fvUJ8P8B8OvyR7o9Jjk5meTk5LA7Xe/6Tm63m/j4+N51\n+iuXyxX0WljBHp+RkWFZFulOEHqcgm33du/U0zj5JzgDqZNfqE6hyNuXQgjj1NbWMnXqVOCz36b/\n8pe/cPbs2cD3m5ubSUxMJDExsdtvzl2XezyesNYVQohwyaRMkVC/Id1quuQAfbJIDitdskQqR2pq\nKkeOHAHgzJkzDBkyhLS0NA4fPgzA3r17ycvLY9KkSezfv5+Ojg6am5tpaWkhPT2dKVOmUFVV1W3d\nv/u7v+PChQt88sknXLlyhX379jFlypSI5PXTZRxuJdM6m9YXzOwcirx9qYjH49Hi1hK65AB9skgO\nK12yRCrHvHnzKCkpobCwkCtXrlBaWorL5WLVqlVcu3aN7OxscnNzAZg7dy6FhYXYbDZKS0uJioqi\nqKiIH/zgBxQUFBAXF8e6desAKC0tpbi4GIBZs2Zx11139TlrV7qMw61kWmfT+oKZnUORSZkip06d\n0uJFqEsO0CeL5LDSJUukcgwZMoRXXnnFsryiosKyrKioiKKiIsvj//Vf/9Wy7r333svOnTv7nC8U\nXcbhVjKts2l9wczOocjbl0IIIYQQGpBJmRBCCCGEBmRSpkhP1yq5VXTJAfpkkRxWumTRJYcqJvY3\nrbNpfcHMzqHIpEwRp9OpOgKgTw7QJ4vksNIliy45VDGxv2mdTesLZnYORSZlinS9QrFKuuQAfbJI\nDitdsuiSQxUT+5vW2bS+YGbnUGRSJoQQQgihAZmUCSGEEEJoIKzrlK1duxa3282VK1d4+umn+e1v\nf8vvfvc74uPjAViwYAFf+cpX2LVrF1u3biUqKoq5c+cyZ84cOjs7WbFiBY2NjURHR1NWVsadd97J\niRMnKC0tBWDMmDG8+OKLAGzevJmqqipsNhuLFy9m2rRp/dNcseHDh6uOAOiTA/TJIjmsdMmiSw5V\nTOxvWmfT+oKZnUPpcVJ26NAhfv/737Nz507Onj3Lo48+yqRJk/je977H9OnTA+t5vV42bNhAZWUl\ndrudJ554ghkzZrBv3z7i4uIoLy/no48+ory8nJdffpnVq1dTUlJCVlYWxcXFHDhwgFGjRrF79252\n7NjBhQsXKCgoYOrUqURHR/frTlDh+pueqqJLDtAni+Sw0iWLLjlUMbG/aZ1N6wtmdg6lx7cv7733\n3sCVr+Pi4rh06RJXr161rHfkyBEyMzOJjY0lJiaG8ePHU1tbS3V1NTNmzAAgNzeX2tpaOjo6OHPm\nTOAu7dOnT6e6upqamhry8vJwOBwkJCQwcuRI6uvrI9lXG8ePH1cdAdAnB+iTRXJY6ZJFlxyqmNjf\ntM6m9QUzO4fS45my6OjowJ+rVlZWct999xEdHc327dvZsmULw4YN4/nnn6e1tZWEhITA4xISEvB4\nPN2WR0VFYbPZaG1tJS4uLrDusGHD8Hg8xMfHB93GmDFjbpjR7XYH/j127FgATpw4EViWlJREcnJy\n0Mc2NDTQ2toa+DozMxOv18vJkycDy1JSUnC5XN2eZ+jQoaSnp1NfX8+5c+cCy3NycvB4PJw6dSqw\nLC0tDafTGfQvTI4fP47X6wXAbreTlZVFY2MjTU1NYXU6evQonZ2dwGd/VpyRkXFTnbxeb6BXXzsN\nHz6c1NTUXnfy62unvo5T133S1059HSf/PeEi+drrTaeu+yRSr73edmpra8NU/vEyiWmdTesLZnYO\nJex7X37wwQdUVlby1ltvcezYMeLj48nIyOCNN97gtdde45577um2vs/nC7qdYMtvZt1ggt1hPty7\nzqemplpOnTocjrC3mZ6eblnmcrmC3ser6+P9P5AyMjIs6yUnJwedRAZ7fv/Zxq5uplOw7fa2k19v\nO/n3SV879XWcgm1DxTi53e5Avki+9vxuplOwbagYJ7fbHfgsqxBCDDRh/fXlhx9+yMaNG9m0aROx\nsbFMnjw5cEC///77+fjjj0lMTOz2G3JLSwuJiYkkJibi8XgA6OzsxOfz4XK5uv2229zcHFi36zb8\nywciu92uOgKgTw7QJ4vksNIliy45VDGxv2mdTesLZnYOpcdJ2fnz51m7di2vv/564DfUJUuWcPr0\naQBqamoYPXo02dnZ1NXV0d7ezsWLF6mtrWXChAlMmTKFqqoqAPbt28fEiROx2+2MGjWKw4cPA7B3\n717y8vKYNGkS+/fvp6Ojg+bmZlpaWoL+5jwQBDvLoIIuOUCfLJLDSpcsuuRQxcT+pnU2rS+Y2TmU\nHt++3L17N2fPnuW5554LLHvsscd47rnnGDx4ME6nk7KyMmJiYiguLmbBggXYbDYWLVpEbGwss2bN\n4uDBg8yfPx+Hw8GaNWsAKCkpYdWqVVy7do3s7Gxyc3MBmDt3LoWFhdhsNkpLS4mKGpiXUmtsbAz5\nNpGJOUCfLJLDSpcsuuRQxcT+pnU2rS+Y2TmUHidl8+bNY968eZbljz76qGXZzJkzmTlzZrdl/muT\nXS89PZ2KigrL8qKiIoqKinqKddtramrS4kWoSw7QJ4vksNIliy45VDGxv2mdTesLZnYOZWCehhJC\nCCGEuM3IpEwIIYQQQgMyKVPEf/0n1XTJAfpkkRxWumTRJYcqJvY3rbNpfcHMzqHIpEwIIYQQQgMy\nKVMk2JXsVdAlB+iTRXJY6ZJFlxyqmNjftM6m9QUzO4cikzIhhBBCCA3IpEwIIYQQQgMyKVMkKSlJ\ndQRAnxygTxbJYaVLFl1yqGJif9M6m9YXzOwcikzKFNHlQnm65AB9skgOK12y6JJDFRP7m9bZtL5g\nZudQZFKmyNGjR1VHAPTJAfpkkRxWumTRJYcqt2v/h4vfs/wXrtu1c2+Z1hfM7ByKTMoU6ezsVB0B\n0CcH6JNFcljpkkWXHKqY2N+0zqb1BTM7hyKTMiGEEEIIDcikTBGn06k6AqBPDtAni+Sw0iWLLjlU\nMbG/aZ1N6wtmdg5FJmWKZGRkqI4A6JMD9MkiOax0yaJLDlVM7G9aZ9P6gpmdQ5FJmSINDQ2qIwD6\n5AB9skgOK12y6JJDFRP7m9bZtL5gZudQZFKmSGtrq+oIgD45QJ8sksNKlyy65FDFxP6mdTatL5jZ\nORSZlAkhhBBCaEAmZUIIIYQQGpBJmSKZmZmqIwD65AB9skgOK12y6JJDFRP7m9bZtL5gZudQZFKm\niNfrVR0B0CcH6JNFcljpkkWXHKqY2N+0zqb1BTM7hyKTMkVOnjypOgKgTw7QJ4vksNIliy45VDGx\nv2mdTesLZnYORSZlQgghhBAaGKQ6gBBCqLBr1y42b97MoEGDePbZZxkzZgzLli3j6tWruFwu1q1b\nh8PhYNeuXWzdupWoqCjmzp3LnDlz6OzsZMWKFTQ2NhIdHU1ZWRl33nknJ06coLS0FIAxY8bw4osv\nqi0phLityJkyRVJSUlRHAPTJAfpkkRxWumSJVI6zZ8+yYcMGKioq2LhxI7/5zW9Yv349BQUFVFRU\nkJqaSmVlJV6vlw0bNvD222+zbds2tm7dSltbG++//z5xcXG88847LFy4kPLycgBWr15NSUkJO3bs\n4MKFCxw4cCAief10GYdbybTOpvUFMzuHIpMyRVwul+oIgD45QJ8sksNKlyyRylFdXc3kyZO54447\nSExM5Ec/+hE1NTU88MADAEyfPp3q6mqOHDlCZmYmsbGxxMTEMH78eGpra6murmbGjBkA5ObmUltb\nS0dHB2fOnCErK6vbNiJJl3G4lUzrbFpfMLNzKGG9fbl27VrcbjdXrlzh6aefJjMzs99O82/evJmq\nqipsNhuLFy9m2rRp/VZeJbfbTU5OjuoY2uQAfbJIDitdskQqxyeffMLly5dZuHAh7e3tLFmyhEuX\nLuFwOAAYNmwYHo+H1tZWEhISAo9LSEiwLI+KisJms9Ha2kpcXFxgXf82wunkN3bsWABOnDgRWJaU\nlERycjJHjx6ls7MT+OwGzhkZGTQ0NHS7GnpmZiZer7fbB6dTUlJwuVzdnmfo0KGkp6dTX1/PuXPn\nbpgtLS0Np9NJXV3dDfPb7XaysrJobGykqanJ0imYcDoBpKWl9bpTTk4OHo+HU6dOddve9Z2GDx9O\namoqx48fD/w1YE+dVI2TdLq9O91Ij5OyQ4cO8fvf/56dO3dy9uxZHn30USZPnkxBQQFf+9rX+OlP\nf0plZSWzZ89mw4YNVFZWYrfbeeKJJ5gxYwb79u0jLi6O8vJyPvroI8rLy3n55ZcDp/mzsrIoLi7m\nwIEDjBo1it27dwdO/RcUFDB16lSio6N7iimEEDelra2N1157jcbGRp588kl8Pl/ge13/3dXNLA+1\n7vWCTTKDLcvKyrJMSlNTU0lNTe22nsPhCHub6enpf/ui4pMbPiYnJ6fHdQCSk5NJTk62rBeM/6xi\nV9d3crvdxMfH967TX7lcrqBnY4I9PtjNsUN1CjVO17uZcQq23du9U0/j5J/gDKROfqE6hdLj25f3\n3nsvr7zyCgBxcXFcunSp307z19TUkJeXh8PhICEhgZEjR1JfXx92GSGECMewYcO45557GDRoECkp\nKQwZMoQhQ4Zw+fJlAJqbm0lMTCQxMbHbb84tLS2B5f6zYJ2dnfh8PlwuF21tbYF1/dsQQohw9Xim\nLDo6GqfTCUBlZSX33XcfH330Ub+c5o+Pjw+6jTFjxtwwY7in/4NRdQrWbrcDKD8F23X/qT6t/LnP\nfQ4I7y2N/hynrvtE9alyj8ejxen/rvtE9en/rhOf3po6dSorVqzgO9/5DufOncPr9TJ16lT27NnD\nI488wt69e8nLyyM7O5uVK1fS3t5OdHQ0tbW1lJSUcOHCBaqqqsjLy2Pfvn1MnDgRu93OqFGjOHz4\nMBMmTGDv3r0UFRX1OWtXQ4cOjej2bgemdTatL5jZOZSwL4nxwQcfUFlZyVtvvcVXv/rVwPL+PM0f\n6dP/wdzK05UD8RSsdJJOXelw+j8cI0aM4KGHHmLu3LkArFy5kszMTJYvX87OnTtJTk5m9uzZ2O12\niouLWbBgATabjUWLFhEbG8usWbM4ePAg8+fPx+FwsGbNGgBKSkpYtWoV165dIzs7m9zc3IjmDrZ/\nBjrTOpvWF8zsHEpYk7IPP/yQjRs3snnzZmJjY3E6nVy+fJmYmJgbnua/++67A6f5x44d2+Np/sTE\nRP74xz9alg9E9fX1WrwQdckB+mSRHFa6ZIlkjvz8fPLz87st27Jli2W9mTNnMnPmzG7L/H+0dL30\n9HQqKioiki8YXcbhVjKts2l9wczOofT4mbLz58+zdu1aXn/9deLj44HPPhu2Z88egG6n+evq6mhv\nb+fixYvU1tYyYcIEpkyZQlVVFUDQ0/xdtzFp0iT2799PR0cHzc3NtLS0DNiButFfPN1KuuQAfbJI\nDitdsuiSQxUT+5vW2bS+YGbnUHo8U7Z7927Onj3Lc889F1i2Zs0aVq5c2S+n+efOnUthYSE2m43S\n0lKiouRSakIIIYQY+HqclM2bN4958+ZZlvfXaf6ioqKIfzhWCCGEEEJ3chpKER0uxAn65AB9skgO\nK12y6JJDFRP7m9bZtL5gZudQZFKmSDhX+r4VdMkB+mSRHFa6ZNElhyom9jets2l9wczOocikTJGu\n111SSZccoE8WyWGlSxZdcqhiYn/TOpvWF8zsHErY1ykTQgghbqWHi9/77B9/vb3Tr8sfUZhGiP4n\nZ8qEEEIIITQgkzJFerpT/K2iSw7QJ4vksNIliy45VDG9vwlMHGMTO4cikzJF/PdYVE2XHKBPFslh\npUsWXXKoYnp/E5g4xiZ2DkUmZYp0vUG0SrrkAH2ySA4rXbLokkMV0/ubwMQxNrFzKDIpE0IIIYTQ\ngEzKhBBCCCE0IJMyRYYPH646AqBPDtAni+Sw0iWLLjlUMb2/CUwcYxM7hyKTMkVSU1NVRwD0yQH6\nZJEcVrpk0SWHKqb3N4GJY2xi51BkUqbI8ePHVUcA9MkB+mSRHFa6ZNElhyqm9zeBiWNsYudQZFKm\niNfrVR0B0CcH6JNFcljpkkWXHKqY3t8EJo6xiZ1DkUmZEEIIIYQGZFKmiN1uVx0B0CcH6JNFcljp\nkkWXHKqY3t8EJo6xiZ1DkUmZIllZWaojAPrkAH2ySA4rXbLokkMV0/ubwMQxNrFzKDIpU6SxsVF1\nBECfHKBPFslhpUsWXXKoYnp/E5g4xiZ2DkUmZYo0NTWpjgDokwP0ySI5rHTJoksOVUzvbwITx9jE\nzqHIpEwIIYQQQgMyKRNCCCGE0IBMyhQZO3as6giAPjlAnyySw0qXLLrkUMX0/iYwcYxN7ByKTMqE\nEEIIITQgkzJFTpw4oToCoE8O0CeL5LDSJYsuOVQxvb8JTBxjEzuHIpMyIYQQQggNhDUp+/jjj3nw\nwQfZvn07ACtWrODhhx+mqKiIoqIi9u/fD8CuXbt4/PHHmTNnDu+++y4AnZ2dFBcXM3/+fAoLCzl9\n+jTw2cw4Pz+f/Px8XnjhhcBzbd68mSeeeII5c+Zw4MCBSHYVQgghhNDWoJ5W8Hq9/OhHP2Ly5Mnd\nln/ve99j+vTp3dbbsGEDlZWV2O12nnjiCWbMmMG+ffuIi4ujvLycjz76iPLycl5++WVWr15NSUkJ\nWVlZFBcXc+DAAUaNGsXu3bvZsWMHFy5coKCggKlTpxIdHR355oolJSWpjgDokwP0ySI5rHTJoksO\nVUzvbwITx9jEzqH0eKbM4XCwadMmEhMTb7jekSNHyMzMJDY2lpiYGMaPH09tbS3V1dXMmDEDgNzc\nXGpra+no6ODMmTOBWytMnz6d6upqampqyMvLw+FwkJCQwMiRI6mvr49ATf0kJyerjgDokwP0ySI5\nrHTJoksOVUzvbwITx9jEzqH0OCkbNGgQMTExluXbt2/nySef5Lvf/S5//vOfaW1tJSEhIfD9hIQE\nPB5Pt+VRUVHYbDZaW1uJi4sLrDts2DDLul23MRAdPXpUdQRAnxygTxbJYaVLFl1yqGJ6fxOYOMYm\ndg6lx7cvg3nkkUeIj48nIyODN954g9dee4177rmn2zo+ny/oY4Mtv5l1g3G73YF/+6930vWvOZKS\nkkLOxBsaGmhtbQ18nZmZidfr5eTJk4FlKSkpuFyubs8zdOhQ0tPTqa+v59y5c4HlOTk5eDweTp06\nFViWlpaG0+mkrq7O8vzHjx/H6/UCYLfbycrKorGxsdttJ27U6ejRo3R2dgLgdDrJyMi4qU6dnZ2B\nXn3tNHz4cFJTU3vdya+vnfo6Tl33SV879XWcPB5PxF97venUdZ9E6rXX205tbW2Yyr+/xcBl4hib\n2DmUXk3Kun6+7P7776e0tJSHHnqo28G4paWFu+++m8TERDweD2PHjqWzsxOfz4fL5ep2YG1ubiYx\nMZHExET++Mc/Wpb3JCcnJ6xlwaSmppKamtptmcPhCHub6enplmUulwuXy3XDx/t/IGVkZFjWS05O\nDjqJDPb8/reAu7qZTsG229u0pW+qAAAerElEQVROfr3t5N8nfe3U13EKtg0V4+R2uwP5Ivna87uZ\nTsG2oWKc3G438fHxQfMJIcTtrleXxFiyZEngryhramoYPXo02dnZ1NXV0d7ezsWLF6mtrWXChAlM\nmTKFqqoqAPbt28fEiROx2+2MGjWKw4cPA7B3717y8vKYNGkS+/fvp6Ojg+bmZlpaWoIepAcCp9Op\nOgKgTw7QJ4vksNIliy45VDG9vwlMHGMTO4fS45myY8eO8dJLL3HmzBkGDRrEnj17KCws5LnnnmPw\n4ME4nU7KysqIiYmhuLiYBQsWYLPZWLRoEbGxscyaNYuDBw8yf/58HA4Ha9asAaCkpIRVq1Zx7do1\nsrOzyc3NBWDu3LkUFhZis9koLS0lKmpgXkot2FkKFXTJAfpkkRxWumTRJYcqpvc3gYljbGLnUHqc\nlI0bN45t27ZZlj/00EOWZTNnzmTmzJndlkVHR1NWVmZZNz09nYqKCsty/7XPBrqGhgbL2zwm5wB9\nskgOK12y6JJDFdP7m8DEMTaxcygD8zTUbaDr5+9U0iUH6JNFcljpkkWXHKqY3t8EJo6xiZ1DkUmZ\nEMJIly9f5sEHH+SXv/wlTU1NFBUVUVBQwNKlS+no6AAic5cSIYQIl0zKhBBG+tnPfsbQoUMBWL9+\nPQUFBVRUVJCamkplZWXgLiVvv/0227ZtY+vWrbS1tfH+++8TFxfHO++8w8KFCykvLwcI3KXEf0cS\nuU2cEOJmyaRMkczMTNURAH1ygD5ZJIeVLlkilePkyZPU19fzla98Bfjsr8gfeOAB4G93GInEXUoi\nTZdxEP3HxDE2sXMovbpOmeg7r9eLw+FQHUObHKBPFslhpUuWSOV46aWXeP755/nVr34FwKVLlwLb\n7ekOIzdzl5JwhHvx61txsd5g2W508euu+Xu6AHEw4XQK9lz9cUFv1ReK9nfqmmmgdBqI49SXTjci\nkzJFTp48GfYFbk3IAfpkkRxWumSJRI5f/epX3H333dx5551Bv3+zdxi5mbuUBBPuhXWzsrJwu93d\nvhfRi/VWfHLDx+Tk5PS4Dtz4AsTX6/ECxNc93/XPlZOTw8PF7/31q0+A3wHw6/JHuq13Ky5+7TeQ\nLujt198XivZPcAZSJ78bXaQ8GJmUCSGMsn//fk6fPs3+/fv505/+hMPhwOl0cvnyZWJiYrrdYaSv\ndykRQoibIZ8pE0IY5eWXX+YXv/gF//7v/86cOXN45plnyM3NZc+ePcDf7jASibuUCCHEzZAzZYqk\npKSojgDokwP0ySI5rHTJ0l85lixZwvLly9m5cyfJycnMnj0bu90ekbuURJIu4yD6j4ljbGLnUGRS\npsjNvMfcn3TJAfpkkRxWumSJdI4lS5YE/r1lyxbL9yNxl5JI0mUcRP8xcYxN7ByKvH2pSNe/3FBJ\nlxygTxbJYaVLFl1yqGJ6fxOYOMYmdg5FJmVCCCGEEBqQSZkQQgghhAZkUqaI//YuqumSA/TJIjms\ndMmiSw5VTO9vAhPH2MTOocgH/RUJdpE5FXTJAfpkkRxWumTRJYcq4fT/28VU/+b6i6kKfZn4Gjex\ncyhypkyR+vp61REAfXKAPlkkh5UuWXTJoYrp/U1g4hib2DkUmZQpcqP7zd1KuuQAfbJIDitdsuiS\nQxXT+5vAxDE2sXMoMikTQgghhNCATMqEEEIIITQgkzJFgt1hXgVdcoA+WSSHlS5ZdMmhiun9TWDi\nGJvYORSZlCni8XhURwD0yQH6ZJEcVrpk0SWHKqb3N4GJY2xi51BkUqbIqVOnVEcA9MkB+mSRHFa6\nZNElhyqm9zeBiWNsYudQZFImhBBCCKEBmZQJIYQQQmhAJmWKpKWlqY4A6JMD9MkiOax0yaJLDlVM\n728CE8fYxM6hyKRMEafTqToCoE8O0CeL5LDSJYsuOVQxvb8JTBxjEzuHEtak7OOPP+bBBx9k+/bt\nADQ1NVFUVERBQQFLly6lo6MDgF27dvH4448zZ84c3n33XQA6OzspLi5m/vz5FBYWcvr0aQBOnDhB\nfn4++fn5vPDCC4Hn2rx5M0888QRz5szhwIEDES2rk7q6OtURAH1ygD5ZJIeVLll0yaGK6f1NYOIY\nm9g5lB4nZV6vlx/96EdMnjw5sGz9+vUUFBRQUVFBamoqlZWVeL1eNmzYwNtvv822bdvYunUrbW1t\nvP/++8TFxfHOO++wcOFCysvLAVi9ejUlJSXs2LGDCxcucODAAU6fPs3u3bupqKjg9ddfp6ysjKtX\nr/ZfeyGEEEIITfQ4KXM4HGzatInExMTAspqaGh544AEApk+fTnV1NUeOHCEzM5PY2FhiYmIYP348\ntbW1VFdXM2PGDAByc3Opra2lo6ODM2fOkJWV1W0bNTU15OXl4XA4SEhIYOTIkXKjUiGEEEIYYVCP\nKwwaxKBB3Ve7dOkSDocDgGHDhuHxeGhtbSUhISGwTkJCgmV5VFQUNpuN1tZW4uLiAuv6txEfHx90\nG2PGjLlhRrfbHfj32LFjgc/eHvVLSkoiOTk56GMbGhpobW0NfJ2ZmYnX6+XkyZOBZSkpKbhcrm7P\nM3ToUNLT06mvr+92M9WcnBw8Hk+3666kpaXhdDq7naL93Oc+B8Dx48fxer0A2O12srKyaGxspKmp\nKaxOR48epbOzE/jsffmMjIyb6tR1//W10/Dhw0lNTe11p8GDBwP0uVNfx6nrPulrp76Ok8fjifhr\nrzeduu6TSL32etupra0NUw0fPlx1BNHPTBxjEzuH0uOkrCc+n6/Py292G9cLdouGcG/bkJqaSmpq\nardlDocj7G2mp6dblrlcrsCEp6fHZ2RkWJYlJycHnUQGe7z/bGNX0kk6hfP4272Tia7ft2LgMXGM\nTewcSq/++tLpdHL58mUAmpubSUxMJDExsdtvyC0tLYHl/lsodHZ24vP5cLlc3X7bDbUN//KB6Pjx\n46ojAPrkAH2ySA4rXbLokkMV0/ubwMQxNrFzKL2alOXm5rJnzx4A9u7dS15eHtnZ2dTV1dHe3s7F\nixepra1lwoQJTJkyhaqqKgD27dvHxIkTsdvtjBo1isOHD3fbxqRJk9i/fz8dHR00NzfT0tIS9Dfn\ngcD/tpFquuQAfbJIDitdsuiSQxXT+5vAxDE2sXMoPb59eezYMV566SXOnDnDoEGD2LNnDz/5yU9Y\nsWIFO3fuJDk5mdmzZ2O32ykuLmbBggXYbDYWLVpEbGwss2bN4uDBg8yfPx+Hw8GaNWsAKCkpYdWq\nVVy7do3s7Gxyc3MBmDt3LoWFhdhsNkpLS4mKkkupCSGEEGLg63FSNm7cOLZt22ZZvmXLFsuymTNn\nMnPmzG7LoqOjKSsrs6ybnp5ORUWFZXlRURFFRUU9xbrt2e121REAfXKAPlkkh5UuWXTJoYrp/U1g\n4hib2DkUOQ2lSLAPSaugSw7QJ4vksNIliy45VDG9vwlMHGMTO4cikzJFGhsbVUcA9MkB+mSRHFa6\nZNElhyqm9zeBiWNsYudQZFKmSNdrQamkSw7QJ4vksNIliy45VDG9vwlMHGMTO4cikzIhhBBCCA3I\npEwIIYQQQgMyKVOk6+1rVNIlB+iTRXJY6ZJFlxyqmN7fBCaOsYmdQ5FJmRBCCCGEBmRSpkjXm1ar\npEsO0CeL5LDSJYsuOVQxvX8kPVz8Xrf/dGHiGJvYORSZlAkhhBBCaEAmZUIIIYQQGpBJmSJJSUmq\nIwD65AB9skgOK12y6JJDFdP7m8DEMTaxcygyKVMkOTlZdQRAnxygTxbJYaVLFl1yqGJ6fxOYOMYm\ndg5FJmWKHD16VHUEQJ8coE8WyWGlSxZdcqhien8TmDjGJnYOZZDqAKbq7OxUHQHQJwfok0VyWOmS\nJZI51q5di9vt5sqVKzz99NNkZmaybNkyrl69isvlYt26dTgcDnbt2sXWrVuJiopi7ty5zJkzh87O\nTlasWEFjYyPR0dGUlZVx5513cuLECUpLSwEYM2YML774YsTygj7jIPqPiWNsYudQ5EyZEMI4hw4d\n4ve//z07d+5k8+bN/PjHP2b9+vUUFBRQUVFBamoqlZWVeL1eNmzYwNtvv822bdvYunUrbW1tvP/+\n+8TFxfHOO++wcOFCysvLAVi9ejUlJSXs2LGDCxcucODAAcVNhRC3E5mUKeJ0OlVHAPTJAfpkkRxW\numSJVI57772XV155BYC4uDguXbpETU0NDzzwAADTp0+nurqaI0eOkJmZSWxsLDExMYwfP57a2lqq\nq6uZMWMGALm5udTW1tLR0cGZM2fIysrqto1I0mUcRP8xcYxN7ByKvH2pSEZGhuoIgD45QJ8sksNK\nlyyRyhEdHR34QVBZWcl9993HRx99hMPhAGDYsGF4PB5aW1tJSEgIPC4hIcGyPCoqCpvNRmtrK3Fx\ncYF1/dvoidvtDvzbf7uZrhfTTEpKIjk5maNHj9LZ2Ynb7cbpdJKRkUFDQwOtra2BdTMzM8N6nqFD\nh5Kenk59fT3nzp274WPS0tJwOp3U1dXdcLt2u52srCwaGxtpamqydArG3wkI2SnYc6WkpOByubp1\nCtU1JycHj8fDqVOnAsvS0tIsj2loaCA1NZXjx4/j9XrD6nSjcbpRp8zMTLxeLydPngws83fyer2B\n/KHGKVSn68dp+PDhWnQK97U30DqF+9rrSiZlivgPAKrpkgP0ySI5rHTJEukcH3zwAZWVlbz11lt8\n9atfDSz3+XxB17+Z5aHWvV5OTk5Yy7Kysiz9U1NTw94fwbaZnp7+ty8qPrnhY3JycnpcBz77S7pw\n/5rOf1axq26drnu+658r3EwulwuXy3XDLP7nDDbxD9Up1DgF2/b14+RwOII+3j/p6KrbOP1VqE7B\ntqm6U0+vvYaGBmBgdfIL57XXlbx9qUio3wRvNV1ygD5ZJIeVLlkimePDDz9k48aNbNq0idjYWJxO\nJ5cvXwagubmZxMREEhMTuz1nS0tLYLn/LFhnZyc+nw+Xy0VbW1tgXf82IkmXcRD9x8QxNrFzKMZP\nyq6//5lO90ATQvSP8+fPs3btWl5//XXi4+OBzz4btmfPHgD27t1LXl4e2dnZ1NXV0d7ezsWLF6mt\nrWXChAlMmTKFqqoqAPbt28fEiROx2+2MGjWKw4cPd9uGEEKES96+FEIYZ/fu3Zw9e5bnnnsusGzN\nmjWsXLmSnTt3kpyczOzZs7Hb7RQXF7NgwQJsNhuLFi0iNjaWWbNmcfDgQebPn4/D4WDNmjUAlJSU\nsGrVKq5du0Z2dja5ubmqKgohbkMyKVPkRh/IvZV0yQH6ZJEcVrpkiVSOefPmMW/ePMvyLVu2WJbN\nnDmTmTNndlvmvzbZ9dLT06moqIhIxmB0GQfRf0wcYxM7h2L825eq+P9qRDVdcoA+WSSHlS5ZdMmh\niun9TWDiGJvYORSZlCnS9U9sVdIlB+iTRXJY6ZJFlxyqmN7fBCaOsYmdQ5FJmRBCCCGEBnr1mbKa\nmhqWLl3K6NGjAfjiF7/IU089pf1944QQQgghdNXrM2Vf/vKX2bZtG9u2beP555+X+8bdpJSUFNUR\nAH1ygD5ZJIeVLll0yaGK6f1NYOIYm9g5lIi9fXk73DdOJzdzhd/+pEsO0CeL5LDSJYsuOVQxvb8J\nTBxjEzuH0utLYtTX17Nw4ULOnTvH4sWLuXTpkpL7xkH4944LV1tbW8TvdRXs3nE5OTm39f27In1P\nMv9zDaROA3GcdOhkKrfbHfT2LmLgMHGMTewcSq8mZV/4whdYvHgxX/va1zh9+jRPPvkkV69eDXz/\nVt43DsK/d1y44uPj++VeV10f7/+BpPr+XcG2q+o+a/59osM9ya7fhopxcrvdgXyRfO353UynYNtQ\nMU5utztwBX4hhBhoevX25YgRI5g1axY2m42UlBSGDx/OuXPntL9vnBBCCCGErno1Kdu1axdvvvkm\nAB6Ph08//ZTHHntM7ht3E4YOHao6AqBPDtAni+Sw0iWLLjlUMb2/CUwcYxM7h9Krty/vv/9+vv/9\n7/Ob3/yGzs5OSktLycjIYPny5XLfuDAFe5tGBV1ygD5ZJIeVLll0yaGK6f118HDxe92+/nX5IxHd\nvoljbGLnUHo1KbvjjjvYuHGjZbnu943TSX19vRYvRF1ygD5ZJIeVLll0yaGK6f1NYOIYm9g5FLmi\nvyJd/8JMJV1ygD5ZJIeVLll0yaGK6f1NYOIYm9g5FJmUCSGEEEJoQCZlQgghhBAakEmZIrpcKE+X\nHKBPFslhpUsWXXKoYnp/E5g4xiZ2DkUmZYqEe7eC/qZLDtAni+Sw0iWLLjlUMb2/CUwcYxM7hyKT\nMkW63jZGJV1ygD5ZJIeVLll0yaGK6f1NYOIYm9g5FJmUCSGEEEJoQCZlQgghhBAakEmZImlpaaoj\nAPrkAH2ySA4rXbLokkMV0/ubwMQxNrFzKDIpU8TpdKqOAOiTA/TJIjmsdMmiSw5VTO9vAhPH2MTO\nocikTJG6ujrVEQB9coA+WSSHlS5ZdMmhiun9TWDiGJvYORSZlAkhhBBCaKBXNyQf6B4ufq/b178u\nf0RREiGEELeb63+GgPwcEeGRM2WKDB8+XHUEQJ8coE8WyWGlSxZdcqhien8TmDjGJnYORSZliqSm\npqqOAOiTA/TJIjmsdMmiSw5VTO9vAhPH2MTOocikTJHjx4+rjgDokwP0ySI5rHTJoksOVUzvbwIT\nx9jEzqHIpEwRr9erOgKgTw7QJ4vksNIliy45VDG9vwlMHGMTO4cikzIhhBBCCA3IpEwRu92uOgKg\nTw7QJ4vksNIliy45VDG9vwlMHGMTO4cikzJFsrKyVEcA9MkB+mSRHFa6ZNElhyqm9zeBiWNsYudQ\n5DplYeiPa840NjaSnJzcp21Egi45QJ8sksNKlyy65FDF9P4mMHGMTewcipwpU6SpqUl1BECfHKBP\nFslhpUsWXXKoYnr/ge7h4vd4uvx/eLj4vaAnAwYqeV3/jUzKhBBCCCE0IJMyIYQQQggNyKRMkbFj\nx6qOAOiTA/TJIjmsdMmiSw5VTO8vBiZ5Xf+NfNC/l+SGs0IIIW41+dkzsGl5puzHP/4x8+bNIz8/\nn6NHj6qOEzb/hzPD+ZDmiRMnblGqG9MlB+iTRXJY6ZJFlxw30p/Hr9uhvxA3S17Xf6PdmbL//u//\npqGhgZ07d3Ly5ElKSkrYuXOn6li9Ir/RCGGWgXT8EgPL9T+P5GeRnrSblFVXV/Pggw8CkJaWxrlz\n57hw4QJ33HGH4mSR0e1/jIpPwn6c/A8khP4G+vFLmCecyZxM+CLH5vP5fKpDdPX8888zbdq0wIGt\noKCA1atXc9dddwVd3+1238p4QghN5OTkqI5gIccvIUQ4Qh2/tDtTdr2e5ow6HpiFEALk+CWEuDna\nfdA/MTGR1tbWwNctLS24XC6FiYQQIjxy/BJC9IV2k7IpU6awZ88eAH73u9+RmJgon8cQQtwW5Pgl\nhOgL7d6+HD9+PF/60pfIz8/HZrPxwgsvqI4khBBhkeOXEKIvtPugvxBCCCGEibR7+1IIIYQQwkQy\nKRNCCCGE0IB2nynrLz/+8Y85cuQINpuNkpISsrKy+v05165di9vt5sqVKzz99NP89re/5Xe/+x3x\n8fEALFiwgK985Svs2rWLrVu3EhUVxdy5c5kzZ07EMtTU1LB06VJGjx4NwBe/+EWeeuopli1bxtWr\nV3G5XKxbtw6Hw9GvOQDeffdddu3aFfj62LFjjBs3Dq/Xi9PpBGD58uWMGzeOzZs3U1VVhc1mY/Hi\nxUybNq3Pz//xxx/zzDPP8O1vf5vCwkKamprC3g+dnZ2sWLGCxsZGoqOjKSsr484774xolh/+8Idc\nuXKFQYMGsW7dOlwuF1/60pcYP3584HFvv/02165di1iW63OsWLEi7Ndof++TZ599lrNnzwLQ1tbG\n3XffzdNPP83DDz/MuHHjAPj85z/P+vXrOX/+PMXFxZw/fx6n00l5eXmgw0Cg4vilUrDj1vPPP684\nVf8I97g0kIR73DGSzwA1NTW+f/iHf/D5fD5ffX29b+7cuf3+nNXV1b6nnnrK5/P5fH/+859906ZN\n8y1fvtz329/+ttt6Fy9e9H31q1/1tbe3+y5duuT7+te/7jt79mzEchw6dMi3ZMmSbstWrFjh2717\nt8/n8/nKy8t9P//5z/s9x/Vqamp8paWlvsLCQt///u//dvveqVOnfI8++qjvL3/5i+/TTz/1PfTQ\nQ74rV6706fkuXrzoKyws9K1cudK3bds2n893c/vhl7/8pa+0tNTn8/l8H374oW/p0qURzbJs2TLf\nf/7nf/p8Pp9v+/btvpdeesnn8/l8X/7yly2Pj1SWYDlu5jXa3/ukqxUrVviOHDniO336tO/RRx+1\nfP/VV1/1bdq0yefz+Xw7duzwrV27ttdZdKPi+KVasOPWQBTucWkgCfe4Yyoj3r4MdeuT/nTvvffy\nyiuvABAXF8elS5e4evWqZb0jR46QmZlJbGwsMTExjB8/ntra2n7NVlNTwwMPPADA9OnTqa6uvuU5\nNmzYwDPPPBMyX15eHg6Hg4SEBEaOHEl9fX2fns/hcLBp0yYSExO7PU+4+6G6upoZM2YAkJub26d9\nEyzLCy+8wEMPPQR8dvanra0t5OMjlSVYjmBU7RO/P/zhD5w/f/6GZ4e6ZvGP5UCh4vglbo1wj0sD\nSbjHHVMZMSlrbW3l85//fODrhIQEPB5Pvz5ndHR04C25yspK7rvvPqKjo9m+fTtPPvkk3/3ud/nz\nn/9Ma2srCQkJ/Zqtvr6ehQsXMn/+fP7rv/6LS5cuBU6HDxs2DI/Hc0ty+B09epSkpKTARTXXr1/P\nN7/5TVatWsXly5f7JcugQYOIiYnptuxm9kPX5VFRUdhsNjo6OiKWxel0Eh0dzdWrV6moqODhhx8G\noKOjg+LiYvLz89myZQtAxLIEywGE/Rrt733i92//9m8UFhYGvm5tbeXZZ58lPz8/8HZ41yzDhg2j\npaWlVzl0pOL4pYPrj1sDUbjHpYEk3OOOqYz5TFlXvlt4FZAPPviAyspK3nrrLY4dO0Z8fDwZGRm8\n8cYbvPbaa9xzzz39mu0LX/gCixcv5mtf+xqnT5/mySef7HbGLtTz9ec+qqys5NFHHwXgySefZMyY\nMaSkpPDCCy/w85///JZm6ek5buX+uXr1KsuWLWPSpElMnjwZgGXLlvGNb3wDm81GYWEhEyZM6Ncs\njzzySK9fo/2xTzo6OnC73ZSWlgIQHx/P0qVL+cY3vsH58+eZM2cOkyZN6vccOhno/SD4cWvv3r0D\n7rNVPTFhrCH4cWfVqlWqYylhxJkyVbc++fDDD9m4cSObNm0iNjaWyZMnk5GRAcD999/Pxx9/HDRb\nJE/rjhgxglmzZmGz2UhJSWH48OGcO3eOy5cvA9Dc3ExiYmK/5+iqpqYm8IN+xowZpKSkAKH3iT9j\npDmdzrD3Q2JiYuA31s7OTnw+X8R/QPzwhz8kNTWVxYsXB5bNnz+fIUOG4HQ6mTRpUmD/9FeWm3mN\n3op98j//8z/d3ra84447ePzxx7Hb7SQkJDBu3Dj+8Ic/dMvSX68XVUy8dVOw41Zzc7PqWLdEsOPS\nQBfsuGMqIyZlKm59cv78edauXcvrr78e+IuSJUuWcPr0aeCzicno0aPJzs6mrq6O9vZ2Ll68SG1t\nbdCzIb21a9cu3nzzTQA8Hg+ffvopjz32WGB/7N27l7y8vH7P4dfc3MyQIUNwOBz4fD6+/e1v097e\nDvxtn0yaNIn9+/fT0dFBc3MzLS0tpKenRzxLbm5u2PthypQpVFVVAbBv3z4mTpwY0Sy7du3Cbrfz\n7LPPBpb94Q9/oLi4GJ/Px5UrV6itrWX06NH9muVmXqP9vU8A6urqGDt2bODrQ4cOUVZWBoDX6+XE\niRPcdddd3bL4x3KgMPHWTcGOWyNGjFCc6tYIdlwa6IIdd0xlzBX9f/KTn3D48OHArU+6Huj7w86d\nO3n11Ve56667Assee+wxtm/fzuDBg3E6nZSVlTFs2DCqqqp48803A29RfeMb34hYjgsXLvD973+f\n9vZ2Ojs7Wbx4MRkZGSxfvpy//OUvJCcnU1ZWht1u79ccfseOHePll19m8+bNAOzevZvNmzczePBg\nRowYwerVqxk8eDDbtm3j17/+NTabjeeeey7wdl5fnvell17izJkzDBo0iBEjRvCTn/yEFStWhLUf\nrl69ysqVK/m///s/HA4Ha9asISkpKWJZPv30Uz73uc8FftimpaVRWlrKunXrOHToEFFRUdx///38\n4z/+Y8SyBMtRWFjIG2+8EdZrtL/3yauvvsqrr75KTk4Os2bNAuDKlSusXLmSP/7xj1y9epX58+fz\n+OOPc/HiRX7wgx/Q1tZGXFwc69atIzY2tldZdHSrj1+qBTtuReKyOLq5mePSQHEzxx0TGTMpE0II\nIYTQmRFvXwohhBBC6E4mZUIIIYQQGpBJmRBCCCGEBmRSJoQQQgihAZmUCSGEEEJoQCZlQgghhBAa\nkEmZEEIIIYQG/n9tA5JeSpYXbgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot distributions of text lengths\n",
    "fig, ax = plt.subplots(1, 2, figsize=(10,4));\n",
    "ax[0].hist([len(x) for x in desc_train], bins=50);\n",
    "ax[1].hist([len(x) for x in title_train], bins=50);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "J0XgiTg30x8s"
   },
   "source": [
    "→ Most texts have < 250 words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-02T00:26:40.393495Z",
     "start_time": "2019-02-02T00:26:37.550323Z"
    },
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "-EUMSqiLcnHw"
   },
   "outputs": [],
   "source": [
    "desc_len = 250\n",
    "title_len = 15\n",
    "\n",
    "# Sequences of indices to matrix\n",
    "desc_train = pad_sequences(desc_train, truncating=\"post\", maxlen=desc_len)\n",
    "desc_val = pad_sequences(desc_val, truncating=\"post\", maxlen=desc_len)\n",
    "desc_test = pad_sequences(desc_test, truncating=\"post\", maxlen=desc_len)\n",
    "\n",
    "title_train = pad_sequences(title_train, truncating=\"post\", maxlen=title_len)\n",
    "title_val = pad_sequences(title_val, truncating=\"post\", maxlen=title_len)\n",
    "title_test = pad_sequences(title_test, truncating=\"post\", maxlen=title_len)\n",
    "\n",
    "# Target vector with labels to matrix\n",
    "num_classes = y_train.unique().shape[0]\n",
    "y_train = to_categorical(y_train, num_classes)\n",
    "y_val = to_categorical(y_val, num_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "W5Cks8n-1Srq"
   },
   "source": [
    "CNN architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 646
    },
    "colab_type": "code",
    "id": "ClUW8IXzOZ-B",
    "outputId": "21d016fa-add2-4928-a668-cb0191bb66a8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "desc_input (InputLayer)         (None, 250)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "title_input (InputLayer)        (None, 15)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 250, 300)     30000000    desc_input[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, 15, 300)      30000000    title_input[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d_1 (SpatialDro (None, 250, 300)     0           embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d_2 (SpatialDro (None, 15, 300)      0           embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)               (None, 248, 256)     230656      spatial_dropout1d_1[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)               (None, 13, 256)      230656      spatial_dropout1d_2[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_1 (GlobalM (None, 256)          0           conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_2 (GlobalM (None, 256)          0           conv1d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "other_input (InputLayer)        (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 513)          0           global_max_pooling1d_1[0][0]     \n",
      "                                                                 global_max_pooling1d_2[0][0]     \n",
      "                                                                 other_input[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 513)          0           concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "output (Dense)                  (None, 54)           27756       dropout_1[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 60,489,068\n",
      "Trainable params: 60,489,068\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Build Keras CNN model\n",
    "def CNN():\n",
    "    # Description input\n",
    "    desc_input = Input(shape=(desc_len,), dtype='int32', name='desc_input')\n",
    "    desc_x = Embedding(max_words, 300, input_length=desc_len)(desc_input)\n",
    "    desc_x = SpatialDropout1D(0.5)(desc_x)\n",
    "    desc_x = Convolution1D(256, 3, activation=\"relu\")(desc_x)\n",
    "    desc_x = GlobalMaxPool1D()(desc_x)\n",
    "    \n",
    "    # Title input\n",
    "    title_input = Input(shape=(title_len,), dtype='int32', name='title_input')\n",
    "    title_x = Embedding(max_words, 300, input_length=title_len)(title_input)\n",
    "    title_x = SpatialDropout1D(0.5)(title_x)\n",
    "    title_x = Convolution1D(256, 3, activation=\"relu\")(title_x)\n",
    "    title_x = GlobalMaxPool1D()(title_x)\n",
    "    \n",
    "    # Other input\n",
    "    other_input = Input(shape=(len(cols_other),), name='other_input')\n",
    "    x = concatenate([desc_x, title_x, other_input])\n",
    "    x = Dropout(0.3)(x)\n",
    "    out = Dense(num_classes, activation='linear', name=\"output\")(x)\n",
    "    model = Model(inputs=[desc_input, title_input, other_input], outputs=[out])\n",
    "    return model\n",
    "\n",
    "model = CNN()\n",
    "\n",
    "# Compile model\n",
    "model.summary()\n",
    "model.compile(loss='categorical_hinge', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-mHYmsgG1fuF"
   },
   "source": [
    "Train CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 221
    },
    "colab_type": "code",
    "id": "K3pSwiSb-hHk",
    "outputId": "13fc7141-8700-47ef-c5ea-1d426de1d7d9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 416089 samples, validate on 73428 samples\n",
      "Epoch 1/20\n",
      "416089/416089 [==============================] - 149s 358us/step - loss: 0.6041 - acc: 0.6869 - val_loss: 0.3146 - val_acc: 0.8632\n",
      "Epoch 2/20\n",
      "416089/416089 [==============================] - 144s 345us/step - loss: 0.2899 - acc: 0.8786 - val_loss: 0.2695 - val_acc: 0.8843\n",
      "Epoch 3/20\n",
      "416089/416089 [==============================] - 144s 345us/step - loss: 0.2399 - acc: 0.9022 - val_loss: 0.2597 - val_acc: 0.8888\n",
      "Epoch 4/20\n",
      "416089/416089 [==============================] - 143s 345us/step - loss: 0.2091 - acc: 0.9159 - val_loss: 0.2594 - val_acc: 0.8912\n",
      "Epoch 5/20\n",
      "416089/416089 [==============================] - 144s 347us/step - loss: 0.1862 - acc: 0.9262 - val_loss: 0.2625 - val_acc: 0.8908\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f8eb14cfbe0>"
      ]
     },
     "execution_count": 10,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit the model with early stopping\n",
    "model.fit(x=[desc_train, title_train, other_train],\n",
    "          y=y_train,\n",
    "          validation_data=([desc_val, title_val, other_val], y_val),\n",
    "          batch_size=2048,\n",
    "          epochs=20,\n",
    "          callbacks=[EarlyStopping(monitor='val_loss',\n",
    "                                   min_delta=0.0001, \n",
    "                                   restore_best_weights=True)]\n",
    "         )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "jyXztnZH1mDc"
   },
   "source": [
    "→ Now we have slightly better results than with SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "yDVjR-eD3OfN"
   },
   "outputs": [],
   "source": [
    "# Save model\n",
    "model.save(dir_path + \"cnn_keras.h5\")\n",
    "# model = load_model(dir_path + \"cnn_keras.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-Z1yDiJzufoz"
   },
   "source": [
    "Make predictions for validation data and calculate accuracy for various hierarchical levels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "pLJF5uZ-uNMm"
   },
   "outputs": [],
   "source": [
    "# Make predictions\n",
    "X_val[\"prediction_id\"] = model.predict(x=[desc_val, title_val, other_val]).argmax(axis=-1)\n",
    "\n",
    "# Provide category labels\n",
    "X_val = pd.merge(X_val, category, left_on=\"category_id\", right_index=True)\n",
    "X_val = pd.merge(X_val, category, left_on=\"prediction_id\", right_index=True, suffixes=(\"_true\", \"_pred\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "o6FaDWyTsbFU",
    "outputId": "b5a7c38c-ac4f-43c2-8dc9-5ba1306e5eb4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name: 0.8912\n",
      "level_0: 0.9640\n",
      "level_1: 0.9461\n",
      "level_2: 0.8972\n"
     ]
    }
   ],
   "source": [
    "# Calculate accuracy for each level of validation set\n",
    "for lvl in (\"name\", \"level_0\", \"level_1\", \"level_2\"):    \n",
    "    acc_lvl = (X_val[lvl + \"_true\"] == X_val[lvl + \"_pred\"]).sum() / X_val.shape[0]\n",
    "    print(f\"{lvl}: {acc_lvl:.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lDTzCsSCyBv_"
   },
   "source": [
    "Create submission file for test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "VpiNUIh2yAdp"
   },
   "outputs": [],
   "source": [
    "# Predict\n",
    "X_test[\"category_id\"] = model.predict(x=[desc_test, title_test, other_test]).argmax(axis=-1)\n",
    "subm = X_train[[\"category_id\"]]\n",
    "\n",
    "# Save to csv\n",
    "subm.to_csv(dir_path + \"submission.csv\", index=True)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "avito_category.ipynb",
   "provenance": [],
   "toc_visible": true,
   "version": "0.3.2"
  },
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}