train-ft-images.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['sample_submission.csv', 'data', 'periods_test.csv', 'periods_train.csv', 'test_active.csv', 'test.csv', 'train_active.csv', 'train.csv', '.ipynb_checkpoints', 'train.ipynb', 'submit_nn_p3_0.2260.csv', '.git', 'ru.bin', 'ru.vec', 'dask-worker-space', 'data.h5', 'tr_desc_pad.pk', 'tr_desc.npy', 'cc.ru.300.bin', 'desc_vectors.npz', 'tr_desc.npz', 'train-ft-images.ipynb', 'test_ru.txt', 'best-use_pretrainedTrue-use_imagesTrue-finetune_pretrainedFalse-val0.22229', 'best-use_pretrainedTrue-use_imagesFalse-finetune_pretrainedFalse-val0.22126.hdf5']\n"
     ]
    }
   ],
   "source": [
    "#import dask.dataframe as pd\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython.display import Image\n",
    "from IPython.core.display import HTML \n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "from tqdm import tqdm\n",
    "import random\n",
    "import re\n",
    "\n",
    "PATH = '.'\n",
    "print(os.listdir(f'{PATH}'))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_pretrained = True\n",
    "use_images     = False\n",
    "finetune_pretrained = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(f'{PATH}/train.csv')\n",
    "df_test  = pd.read_csv(f'{PATH}/test.csv')\n",
    "\n",
    "df_train_active = pd.read_csv(f'{PATH}/train_active.csv', usecols=['item_id', 'user_id'])\n",
    "df_test_active  = pd.read_csv(f'{PATH}/test_active.csv',  usecols=['item_id', 'user_id'])\n",
    "\n",
    "df_train_periods = pd.read_csv(f'{PATH}/periods_train.csv', parse_dates=['date_from', 'date_to'])\n",
    "df_test_periods  = pd.read_csv(f'{PATH}/periods_test.csv',  parse_dates=['date_from', 'date_to'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_samples = pd.concat([\n",
    "    df_train[['item_id', 'user_id']],\n",
    "    df_train_active,\n",
    "    df_test[['item_id', 'user_id']],\n",
    "    df_test_active\n",
    "]).reset_index(drop=True)\n",
    "all_samples.drop_duplicates(['item_id'], inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>activation_date</th>\n",
       "      <th>date_from</th>\n",
       "      <th>date_to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8f5caef7afb0</td>\n",
       "      <td>2017-02-14</td>\n",
       "      <td>2017-03-15</td>\n",
       "      <td>2017-03-16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>66218ff526d1</td>\n",
       "      <td>2017-02-16</td>\n",
       "      <td>2017-03-15</td>\n",
       "      <td>2017-03-18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b237d9539b21</td>\n",
       "      <td>2017-03-01</td>\n",
       "      <td>2017-03-15</td>\n",
       "      <td>2017-03-28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>80bf58082ad3</td>\n",
       "      <td>2017-03-19</td>\n",
       "      <td>2017-03-19</td>\n",
       "      <td>2017-03-28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>67a9944a7373</td>\n",
       "      <td>2017-03-14</td>\n",
       "      <td>2017-03-15</td>\n",
       "      <td>2017-03-28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        item_id activation_date  date_from    date_to\n",
       "0  8f5caef7afb0      2017-02-14 2017-03-15 2017-03-16\n",
       "1  66218ff526d1      2017-02-16 2017-03-15 2017-03-18\n",
       "2  b237d9539b21      2017-03-01 2017-03-15 2017-03-28\n",
       "3  80bf58082ad3      2017-03-19 2017-03-19 2017-03-28\n",
       "4  67a9944a7373      2017-03-14 2017-03-15 2017-03-28"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_periods = pd.concat([\n",
    "    df_train_periods,\n",
    "    df_test_periods\n",
    "])\n",
    "\n",
    "all_periods.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_periods['days_up'] = all_periods['date_to'].dt.dayofyear - all_periods['date_from'].dt.dayofyear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>days_up_sum</th>\n",
       "      <th>times_put_up</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00000077ff21</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000002c54018</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000005570503</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0000060018e6</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000006497719</td>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        item_id  days_up_sum  times_put_up\n",
       "0  00000077ff21           13             1\n",
       "1  000002c54018            6             1\n",
       "2  000005570503            1             1\n",
       "3  0000060018e6            6             1\n",
       "4  000006497719           19             2"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gp = all_periods.groupby(['item_id'])[['days_up']]\n",
    "\n",
    "gp_df = pd.DataFrame()\n",
    "gp_df['days_up_sum']  = gp.sum()['days_up']\n",
    "gp_df['times_put_up'] = gp.count()['days_up']\n",
    "gp_df.reset_index(inplace=True)\n",
    "gp_df.rename(index=str, columns={'index': 'item_id'})\n",
    "\n",
    "gp_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>activation_date</th>\n",
       "      <th>date_from</th>\n",
       "      <th>date_to</th>\n",
       "      <th>days_up</th>\n",
       "      <th>days_up_sum</th>\n",
       "      <th>times_put_up</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8f5caef7afb0</td>\n",
       "      <td>2017-02-14</td>\n",
       "      <td>2017-03-15</td>\n",
       "      <td>2017-03-16</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>66218ff526d1</td>\n",
       "      <td>2017-02-16</td>\n",
       "      <td>2017-03-15</td>\n",
       "      <td>2017-03-18</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b237d9539b21</td>\n",
       "      <td>2017-03-01</td>\n",
       "      <td>2017-03-15</td>\n",
       "      <td>2017-03-28</td>\n",
       "      <td>13</td>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>80bf58082ad3</td>\n",
       "      <td>2017-03-19</td>\n",
       "      <td>2017-03-19</td>\n",
       "      <td>2017-03-28</td>\n",
       "      <td>9</td>\n",
       "      <td>17</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>67a9944a7373</td>\n",
       "      <td>2017-03-14</td>\n",
       "      <td>2017-03-15</td>\n",
       "      <td>2017-03-28</td>\n",
       "      <td>13</td>\n",
       "      <td>18</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        item_id activation_date  date_from    date_to  days_up  days_up_sum  \\\n",
       "0  8f5caef7afb0      2017-02-14 2017-03-15 2017-03-16        1           17   \n",
       "1  66218ff526d1      2017-02-16 2017-03-15 2017-03-18        3           18   \n",
       "2  b237d9539b21      2017-03-01 2017-03-15 2017-03-28       13           19   \n",
       "3  80bf58082ad3      2017-03-19 2017-03-19 2017-03-28        9           17   \n",
       "4  67a9944a7373      2017-03-14 2017-03-15 2017-03-28       13           18   \n",
       "\n",
       "   times_put_up  \n",
       "0             4  \n",
       "1             3  \n",
       "2             2  \n",
       "3             4  \n",
       "4             3  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_periods.drop_duplicates(['item_id'], inplace=True)\n",
    "all_periods = all_periods.merge(gp_df, on='item_id', how='left')\n",
    "all_periods.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>activation_date</th>\n",
       "      <th>date_from</th>\n",
       "      <th>date_to</th>\n",
       "      <th>days_up</th>\n",
       "      <th>days_up_sum</th>\n",
       "      <th>times_put_up</th>\n",
       "      <th>user_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8f5caef7afb0</td>\n",
       "      <td>2017-02-14</td>\n",
       "      <td>2017-03-15</td>\n",
       "      <td>2017-03-16</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>4</td>\n",
       "      <td>e292cce69842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>66218ff526d1</td>\n",
       "      <td>2017-02-16</td>\n",
       "      <td>2017-03-15</td>\n",
       "      <td>2017-03-18</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>3</td>\n",
       "      <td>a326c04a24ec</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b237d9539b21</td>\n",
       "      <td>2017-03-01</td>\n",
       "      <td>2017-03-15</td>\n",
       "      <td>2017-03-28</td>\n",
       "      <td>13</td>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "      <td>06d275498a56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>80bf58082ad3</td>\n",
       "      <td>2017-03-19</td>\n",
       "      <td>2017-03-19</td>\n",
       "      <td>2017-03-28</td>\n",
       "      <td>9</td>\n",
       "      <td>17</td>\n",
       "      <td>4</td>\n",
       "      <td>831c8c4a622c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>67a9944a7373</td>\n",
       "      <td>2017-03-14</td>\n",
       "      <td>2017-03-15</td>\n",
       "      <td>2017-03-28</td>\n",
       "      <td>13</td>\n",
       "      <td>18</td>\n",
       "      <td>3</td>\n",
       "      <td>248102e50d79</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        item_id activation_date  date_from    date_to  days_up  days_up_sum  \\\n",
       "0  8f5caef7afb0      2017-02-14 2017-03-15 2017-03-16        1           17   \n",
       "1  66218ff526d1      2017-02-16 2017-03-15 2017-03-18        3           18   \n",
       "2  b237d9539b21      2017-03-01 2017-03-15 2017-03-28       13           19   \n",
       "3  80bf58082ad3      2017-03-19 2017-03-19 2017-03-28        9           17   \n",
       "4  67a9944a7373      2017-03-14 2017-03-15 2017-03-28       13           18   \n",
       "\n",
       "   times_put_up       user_id  \n",
       "0             4  e292cce69842  \n",
       "1             3  a326c04a24ec  \n",
       "2             2  06d275498a56  \n",
       "3             4  831c8c4a622c  \n",
       "4             3  248102e50d79  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_periods = all_periods.merge(all_samples, on='item_id', how='left')\n",
    "all_periods.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>avg_days_up_user</th>\n",
       "      <th>avg_times_up_user</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00000077ff21</td>\n",
       "      <td>12.5</td>\n",
       "      <td>2.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000006497719</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00000b4d72f6</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00000d642d7e</td>\n",
       "      <td>13.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0000126b80a4</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.75</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id  avg_days_up_user  avg_times_up_user\n",
       "0  00000077ff21              12.5               2.00\n",
       "1  000006497719              19.0               2.00\n",
       "2  00000b4d72f6               3.0               1.00\n",
       "3  00000d642d7e              13.0               1.00\n",
       "4  0000126b80a4              12.0               1.75"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gp = all_periods.groupby(['user_id'])[['days_up_sum', 'times_put_up']].mean().reset_index() \\\n",
    "    .rename(index=str, columns={\n",
    "        'days_up_sum' : 'avg_days_up_user',\n",
    "        'times_put_up': 'avg_times_up_user'\n",
    "    })\n",
    "gp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>avg_days_up_user</th>\n",
       "      <th>avg_times_up_user</th>\n",
       "      <th>n_user_items</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00000077ff21</td>\n",
       "      <td>12.5</td>\n",
       "      <td>2.00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000006497719</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2.00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00000b4d72f6</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00000d642d7e</td>\n",
       "      <td>13.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0000126b80a4</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.75</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id  avg_days_up_user  avg_times_up_user  n_user_items\n",
       "0  00000077ff21              12.5               2.00             2\n",
       "1  000006497719              19.0               2.00             1\n",
       "2  00000b4d72f6               3.0               1.00             1\n",
       "3  00000d642d7e              13.0               1.00             2\n",
       "4  0000126b80a4              12.0               1.75             8"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_user_items = all_samples.groupby(['user_id'])[['item_id']].count().reset_index() \\\n",
    "    .rename(index=str, columns={\n",
    "        'item_id': 'n_user_items'\n",
    "    })\n",
    "gp = gp.merge(n_user_items, on='user_id', how='outer')\n",
    "\n",
    "gp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_y_train = df_train['deal_probability']\n",
    "df_x_train = df_train.drop(['deal_probability'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_x_train = df_x_train.merge(gp, on='user_id', how='left')\n",
    "df_test = df_test.merge(gp, on='user_id', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>region</th>\n",
       "      <th>city</th>\n",
       "      <th>parent_category_name</th>\n",
       "      <th>category_name</th>\n",
       "      <th>param_1</th>\n",
       "      <th>param_2</th>\n",
       "      <th>param_3</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>price</th>\n",
       "      <th>item_seq_number</th>\n",
       "      <th>activation_date</th>\n",
       "      <th>user_type</th>\n",
       "      <th>image</th>\n",
       "      <th>image_top_1</th>\n",
       "      <th>avg_days_up_user</th>\n",
       "      <th>avg_times_up_user</th>\n",
       "      <th>n_user_items</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b912c3c6a6ad</td>\n",
       "      <td>e00f8ff2eaf9</td>\n",
       "      <td>Свердловская область</td>\n",
       "      <td>Екатеринбург</td>\n",
       "      <td>Личные вещи</td>\n",
       "      <td>Товары для детей и игрушки</td>\n",
       "      <td>Постельные принадлежности</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Кокоби(кокон для сна)</td>\n",
       "      <td>Кокон для сна малыша,пользовались меньше месяц...</td>\n",
       "      <td>400.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2017-03-28</td>\n",
       "      <td>Private</td>\n",
       "      <td>d10c7e016e03247a3bf2d13348fe959fe6f436c1caf64c...</td>\n",
       "      <td>1008.0</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2dac0150717d</td>\n",
       "      <td>39aeb48f0017</td>\n",
       "      <td>Самарская область</td>\n",
       "      <td>Самара</td>\n",
       "      <td>Для дома и дачи</td>\n",
       "      <td>Мебель и интерьер</td>\n",
       "      <td>Другое</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Стойка для Одежды</td>\n",
       "      <td>Стойка для одежды, под вешалки. С бутика.</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>19</td>\n",
       "      <td>2017-03-26</td>\n",
       "      <td>Private</td>\n",
       "      <td>79c9392cc51a9c81c6eb91eceb8e552171db39d7142700...</td>\n",
       "      <td>692.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ba83aefab5dc</td>\n",
       "      <td>91e2f88dd6e3</td>\n",
       "      <td>Ростовская область</td>\n",
       "      <td>Ростов-на-Дону</td>\n",
       "      <td>Бытовая электроника</td>\n",
       "      <td>Аудио и видео</td>\n",
       "      <td>Видео, DVD и Blu-ray плееры</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Philips bluray</td>\n",
       "      <td>В хорошем состоянии, домашний кинотеатр с blu ...</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>9</td>\n",
       "      <td>2017-03-20</td>\n",
       "      <td>Private</td>\n",
       "      <td>b7f250ee3f39e1fedd77c141f273703f4a9be59db4b48a...</td>\n",
       "      <td>3032.0</td>\n",
       "      <td>4.428571</td>\n",
       "      <td>1.142857</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>02996f1dd2ea</td>\n",
       "      <td>bf5cccea572d</td>\n",
       "      <td>Татарстан</td>\n",
       "      <td>Набережные Челны</td>\n",
       "      <td>Личные вещи</td>\n",
       "      <td>Товары для детей и игрушки</td>\n",
       "      <td>Автомобильные кресла</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Автокресло</td>\n",
       "      <td>Продам кресло от0-25кг</td>\n",
       "      <td>2200.0</td>\n",
       "      <td>286</td>\n",
       "      <td>2017-03-25</td>\n",
       "      <td>Company</td>\n",
       "      <td>e6ef97e0725637ea84e3d203e82dadb43ed3cc0a1c8413...</td>\n",
       "      <td>796.0</td>\n",
       "      <td>16.714286</td>\n",
       "      <td>2.642857</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7c90be56d2ab</td>\n",
       "      <td>ef50846afc0b</td>\n",
       "      <td>Волгоградская область</td>\n",
       "      <td>Волгоград</td>\n",
       "      <td>Транспорт</td>\n",
       "      <td>Автомобили</td>\n",
       "      <td>С пробегом</td>\n",
       "      <td>ВАЗ (LADA)</td>\n",
       "      <td>2110</td>\n",
       "      <td>ВАЗ 2110, 2003</td>\n",
       "      <td>Все вопросы по телефону.</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2017-03-16</td>\n",
       "      <td>Private</td>\n",
       "      <td>54a687a3a0fc1d68aed99bdaaf551c5c70b761b16fd0a2...</td>\n",
       "      <td>2264.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>51e0962387f7</td>\n",
       "      <td>bbfad0b1ad0a</td>\n",
       "      <td>Татарстан</td>\n",
       "      <td>Чистополь</td>\n",
       "      <td>Личные вещи</td>\n",
       "      <td>Товары для детей и игрушки</td>\n",
       "      <td>Автомобильные кресла</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Авто люлька</td>\n",
       "      <td>В хорошем состоянии</td>\n",
       "      <td>1300.0</td>\n",
       "      <td>9</td>\n",
       "      <td>2017-03-28</td>\n",
       "      <td>Private</td>\n",
       "      <td>eb6ad1231c59d3dc7e4020e724ffe8e4d302023ddcbb99...</td>\n",
       "      <td>796.0</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>c4f260a2b48a</td>\n",
       "      <td>08f469d2e6f7</td>\n",
       "      <td>Нижегородская область</td>\n",
       "      <td>Нижний Новгород</td>\n",
       "      <td>Для дома и дачи</td>\n",
       "      <td>Ремонт и строительство</td>\n",
       "      <td>Сантехника и сауна</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Водонагреватель 100 литров нержавейка плоский</td>\n",
       "      <td>Электро водонагреватель накопительный на 100 л...</td>\n",
       "      <td>11000.0</td>\n",
       "      <td>125</td>\n",
       "      <td>2017-03-23</td>\n",
       "      <td>Private</td>\n",
       "      <td>0330f6ac561f5db1fa8226dd5e7e127b5671d44d075a98...</td>\n",
       "      <td>2823.0</td>\n",
       "      <td>7.333333</td>\n",
       "      <td>1.222222</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>6b71309d6a8a</td>\n",
       "      <td>fef86baa002c</td>\n",
       "      <td>Пермский край</td>\n",
       "      <td>Пермь</td>\n",
       "      <td>Личные вещи</td>\n",
       "      <td>Одежда, обувь, аксессуары</td>\n",
       "      <td>Женская одежда</td>\n",
       "      <td>Джинсы</td>\n",
       "      <td>26</td>\n",
       "      <td>Бойфренды colins</td>\n",
       "      <td>Бойфренды в хорошем состоянии.</td>\n",
       "      <td>500.0</td>\n",
       "      <td>61</td>\n",
       "      <td>2017-03-25</td>\n",
       "      <td>Private</td>\n",
       "      <td>9bab29a519e81c14f4582024adfebd4f11a4ac71d323a6...</td>\n",
       "      <td>567.0</td>\n",
       "      <td>11.916667</td>\n",
       "      <td>1.666667</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>c5b969cb63a2</td>\n",
       "      <td>055825270190</td>\n",
       "      <td>Оренбургская область</td>\n",
       "      <td>Оренбург</td>\n",
       "      <td>Личные вещи</td>\n",
       "      <td>Одежда, обувь, аксессуары</td>\n",
       "      <td>Женская одежда</td>\n",
       "      <td>Платья и юбки</td>\n",
       "      <td>&gt; 50 (XXL)</td>\n",
       "      <td>Платье</td>\n",
       "      <td>54 раз мер очень удобное</td>\n",
       "      <td>500.0</td>\n",
       "      <td>85</td>\n",
       "      <td>2017-03-17</td>\n",
       "      <td>Private</td>\n",
       "      <td>75ce06d1f939a31dfb2af8ac55f08fa998fa336d13ee05...</td>\n",
       "      <td>415.0</td>\n",
       "      <td>13.517241</td>\n",
       "      <td>1.896552</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>b1570962e68c</td>\n",
       "      <td>f9e8f831d94c</td>\n",
       "      <td>Нижегородская область</td>\n",
       "      <td>Нижний Новгород</td>\n",
       "      <td>Личные вещи</td>\n",
       "      <td>Детская одежда и обувь</td>\n",
       "      <td>Для девочек</td>\n",
       "      <td>Обувь</td>\n",
       "      <td>25</td>\n",
       "      <td>Полу ботиночки замш натур.Бамбини</td>\n",
       "      <td>По стельке 15.5см мерить приокский район. Цвет...</td>\n",
       "      <td>400.0</td>\n",
       "      <td>136</td>\n",
       "      <td>2017-03-22</td>\n",
       "      <td>Company</td>\n",
       "      <td>54fb8521135fda77a860bfd2fac6bf46867ab7c06796e3...</td>\n",
       "      <td>46.0</td>\n",
       "      <td>15.632184</td>\n",
       "      <td>2.045977</td>\n",
       "      <td>93</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        item_id       user_id                 region              city  \\\n",
       "0  b912c3c6a6ad  e00f8ff2eaf9   Свердловская область      Екатеринбург   \n",
       "1  2dac0150717d  39aeb48f0017      Самарская область            Самара   \n",
       "2  ba83aefab5dc  91e2f88dd6e3     Ростовская область    Ростов-на-Дону   \n",
       "3  02996f1dd2ea  bf5cccea572d              Татарстан  Набережные Челны   \n",
       "4  7c90be56d2ab  ef50846afc0b  Волгоградская область         Волгоград   \n",
       "5  51e0962387f7  bbfad0b1ad0a              Татарстан         Чистополь   \n",
       "6  c4f260a2b48a  08f469d2e6f7  Нижегородская область   Нижний Новгород   \n",
       "7  6b71309d6a8a  fef86baa002c          Пермский край             Пермь   \n",
       "8  c5b969cb63a2  055825270190   Оренбургская область          Оренбург   \n",
       "9  b1570962e68c  f9e8f831d94c  Нижегородская область   Нижний Новгород   \n",
       "\n",
       "  parent_category_name               category_name  \\\n",
       "0          Личные вещи  Товары для детей и игрушки   \n",
       "1      Для дома и дачи           Мебель и интерьер   \n",
       "2  Бытовая электроника               Аудио и видео   \n",
       "3          Личные вещи  Товары для детей и игрушки   \n",
       "4            Транспорт                  Автомобили   \n",
       "5          Личные вещи  Товары для детей и игрушки   \n",
       "6      Для дома и дачи      Ремонт и строительство   \n",
       "7          Личные вещи   Одежда, обувь, аксессуары   \n",
       "8          Личные вещи   Одежда, обувь, аксессуары   \n",
       "9          Личные вещи      Детская одежда и обувь   \n",
       "\n",
       "                       param_1        param_2     param_3  \\\n",
       "0    Постельные принадлежности            NaN         NaN   \n",
       "1                       Другое            NaN         NaN   \n",
       "2  Видео, DVD и Blu-ray плееры            NaN         NaN   \n",
       "3         Автомобильные кресла            NaN         NaN   \n",
       "4                   С пробегом     ВАЗ (LADA)        2110   \n",
       "5         Автомобильные кресла            NaN         NaN   \n",
       "6           Сантехника и сауна            NaN         NaN   \n",
       "7               Женская одежда         Джинсы          26   \n",
       "8               Женская одежда  Платья и юбки  > 50 (XXL)   \n",
       "9                  Для девочек          Обувь          25   \n",
       "\n",
       "                                           title  \\\n",
       "0                          Кокоби(кокон для сна)   \n",
       "1                              Стойка для Одежды   \n",
       "2                                 Philips bluray   \n",
       "3                                     Автокресло   \n",
       "4                                 ВАЗ 2110, 2003   \n",
       "5                                    Авто люлька   \n",
       "6  Водонагреватель 100 литров нержавейка плоский   \n",
       "7                               Бойфренды colins   \n",
       "8                                         Платье   \n",
       "9              Полу ботиночки замш натур.Бамбини   \n",
       "\n",
       "                                         description    price  \\\n",
       "0  Кокон для сна малыша,пользовались меньше месяц...    400.0   \n",
       "1          Стойка для одежды, под вешалки. С бутика.   3000.0   \n",
       "2  В хорошем состоянии, домашний кинотеатр с blu ...   4000.0   \n",
       "3                             Продам кресло от0-25кг   2200.0   \n",
       "4                           Все вопросы по телефону.  40000.0   \n",
       "5                                В хорошем состоянии   1300.0   \n",
       "6  Электро водонагреватель накопительный на 100 л...  11000.0   \n",
       "7                     Бойфренды в хорошем состоянии.    500.0   \n",
       "8                           54 раз мер очень удобное    500.0   \n",
       "9  По стельке 15.5см мерить приокский район. Цвет...    400.0   \n",
       "\n",
       "   item_seq_number activation_date user_type  \\\n",
       "0                2      2017-03-28   Private   \n",
       "1               19      2017-03-26   Private   \n",
       "2                9      2017-03-20   Private   \n",
       "3              286      2017-03-25   Company   \n",
       "4                3      2017-03-16   Private   \n",
       "5                9      2017-03-28   Private   \n",
       "6              125      2017-03-23   Private   \n",
       "7               61      2017-03-25   Private   \n",
       "8               85      2017-03-17   Private   \n",
       "9              136      2017-03-22   Company   \n",
       "\n",
       "                                               image  image_top_1  \\\n",
       "0  d10c7e016e03247a3bf2d13348fe959fe6f436c1caf64c...       1008.0   \n",
       "1  79c9392cc51a9c81c6eb91eceb8e552171db39d7142700...        692.0   \n",
       "2  b7f250ee3f39e1fedd77c141f273703f4a9be59db4b48a...       3032.0   \n",
       "3  e6ef97e0725637ea84e3d203e82dadb43ed3cc0a1c8413...        796.0   \n",
       "4  54a687a3a0fc1d68aed99bdaaf551c5c70b761b16fd0a2...       2264.0   \n",
       "5  eb6ad1231c59d3dc7e4020e724ffe8e4d302023ddcbb99...        796.0   \n",
       "6  0330f6ac561f5db1fa8226dd5e7e127b5671d44d075a98...       2823.0   \n",
       "7  9bab29a519e81c14f4582024adfebd4f11a4ac71d323a6...        567.0   \n",
       "8  75ce06d1f939a31dfb2af8ac55f08fa998fa336d13ee05...        415.0   \n",
       "9  54fb8521135fda77a860bfd2fac6bf46867ab7c06796e3...         46.0   \n",
       "\n",
       "   avg_days_up_user  avg_times_up_user  n_user_items  \n",
       "0          8.000000           2.000000             2  \n",
       "1               NaN                NaN             1  \n",
       "2          4.428571           1.142857             9  \n",
       "3         16.714286           2.642857            32  \n",
       "4               NaN                NaN             1  \n",
       "5          4.000000           1.000000             2  \n",
       "6          7.333333           1.222222            11  \n",
       "7         11.916667           1.666667            23  \n",
       "8         13.517241           1.896552            38  \n",
       "9         15.632184           2.045977            93  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_x_train.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeoAAAHxCAYAAACifP9iAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3XmcnnV97//XZ7ZM9mEJECAL2EgFKqgR0Z56OKJHFBS14FJUpCju9hytS7Wt1h578PRUbX+tSxQrFYsgRUHcoAiKHFkSFARkk82QEALJJJlss31+f9zXxGGYyeyZb+Z+PR+Pedz3fa2f+c7yvr/f67qvKzITSZJUpoapLkCSJA3NoJYkqWAGtSRJBTOoJUkqmEEtSVLBDGpJkgpmUKvuRMQXI+KvJmhbiyOiIyIaq9fXRsRbx7itj0bEV3Yz/4yIuHKstQ6z73dGxLrqe9lvErb/tYj4XxOwnbdExM8moqYR7OuEiFi9J/Yl7U7TVBcgTaSIeBA4EOgGeoA7gX8DVmRmL0BmvmMU23prZv7nUMtk5sPAnPFVvWtbf9dv30uBB4DmzOyu5n8D+MZE7Ku/iGgGPgMcn5m3TvT2JY2PPWpNR6/IzLnAEuBc4MPAeRO9k4iYLm90DwRagTtGu2LU7JX/R6bRz0/T3F75ByaNRGZuyszLgdcBZ0bE0fDkYdiI2D8iroiI9ojYEBHXRURDRHwdWAx8txoO/lBELI2IjIizI+Jh4Mf9pvX/p/+0iLgpIjZFxGURsW+1r6cMpUbEgxHx4ur5JyLigmrWT6vH9mr/zx847BsRL4iIm6v93BwRL+g379qI+NuIuD4itkTElRGx/8A2ioinA3f329ePR7jtT0XE9cA24PBBtvusiLil2vdF1N4I9J9/SkT8smr3/xcRz+w37yMR8Ztq3Tsj4tUDtz+Yfj+LcyJiTUSsjYgP9Jv/iYi4JCIuiIjNwFsiYkZEfK5afk31fMaA7X40Ih6vflZnjKQWaSIZ1Jr2MvMmYDXwR4PM/kA1bwG1nuVHa6vkm4CHqfXO52Tm/+m3zn8FngG8dIhdvhn4U+BgakPw/zSGsl9YPbZV+/95/5lV+H+v2vZ+1Iauvzfg+PKfAGcBBwAtwJ8P3Elm3gMc1W9fLxrhtt8EnAPMBR4aUFsL8B3g68C+wLeAP+43/9nAV4G3V9v/EnB5v4D8DbWf1Xzgb4ALImLhkC31VP8NWAb8d+AjfW+EKqcClwBt1A4jfAw4HjgWOAY4DvjLfssfBOwPHAKcCayIiCNGUYs0bga16sUaaqExUBewEFiSmV2ZeV0OfwH8T2Tm1szcPsT8r2fm7Zm5Ffgr4LV9J5tNoJOBezPz65nZnZkXAncBr+i3zL9m5j1VnRdTC6OJ2vbXMvOOan7XgPWPB5qBz1Vteglwc7/5bwO+lJk3ZmZPZp4P7KzWIzO/lZlrMrM3My8C7qUWoCP1N9XP51fAvwJv6Dfv55n5nWrb24EzgE9m5mOZuZ7aG4M3DdjeX2Xmzsz8CbU3MK8dRS3SuBnUqheHABsGmf73wH3AlRFxf0R8ZATb+u0o5j9ELbSeMuw8TgczoCdbvT6k3+tH+z3fxshPehvJtnfXBgcDjwx4w9N/e0uAD1TD3u0R0Q4sqtYjIt7cb1i8HTia0bXfwPY/eDd1D/xeBy6/sXrDNdR8adIZ1Jr2IuK51ELmKR/rycwtmfmBzDycWo/x/RFxYt/sITY5XI97Ub/ni6n12h8HtgKz+tXVSG3IfSz7WEMt8PpbDDwyzHojMZJt766+tcAhERED1u/zW+BTmdnW72tWZl4YEUuALwPvAfbLzDbgdqD/toYzsP3X7Kbugd/rwOX3iYjZu5kvTTqDWtNWRMyLiFOAbwIXVEOhA5c5JSJ+rwqVzdQ+0tVTzV7HICdKjcAbI+LIiJgFfBK4JDN7gHuA1og4OWofifpLYMYQ21gP9O5m/98Hnh4RfxIRTRHxOuBI4Iox1DvR2/45tWPz76vWfw1PHrr+MvCOiHhe1Myu2mQuMJtamK4HiIizqPWoR+OvImJWRBxF7Rj9RbtZ9kLgLyNiQXWy3V8DFwxY5m8ioiUi/gg4hdoxd2mPMag1HX03IrZQ67l9jNrJUGcNsewy4D+BDmoB8/nMvLaa97+p/RNvj4innIi1G18HvkZt6LkVeB/UzkIH3gV8hVrvdCu1E9meIjO3AZ8Crq/2f/yA+U9QC40PAE8AHwJOyczHR1HnoMa77czsBF4DvAXYSO2s+0v7zV9J7Tj1P1fz76uWJTPvBP6B2s9iHfAHwPWj/BZ+Um3zauD/ZubuLhLzv4CVwG3Ar4Bbqml9Hq1qXEPt5LN3ZOZdo6xHGpcY/rwZSSpfDHKRGGk6sEctSVLBDGpJkgrm0LckSQWzRy1JUsEMakmSClbE3WP233//XLp06VSXIUnSHrNq1arHM3Ooix7tUkRQL126lJUrV051GZIk7TERMfBSvYNy6FuSpIIZ1JIkFcygliSpYAa1JEkFM6glSSqYQS1JUsEMakmSCmZQS5JUMINakqSCGdSSJBXMoJYkqWAGtSRJBTOoJUkqmEEtSVLBDGpJkgpmUEuSVDCDWpKkghnUkiQVzKCWJKlgdRvUK1atmOoSJEkaVt0GtSRJewODWpKkghnUkiQVzKCWJKlgBrUkSQUzqCVJKphBLUlSwQxqSZIKZlBLklQwg1qSpIIZ1JIkFcygliSpYAa1JEkFM6glSSqYQS1JUsEMakmSCmZQS5JUMINakqSCGdSSJBXMoJYkqWAGtSRJBTOoJUkqmEEtSVLBDGpJkgpmUEuSVDCDWpKkgg0b1BHx1Yh4LCJu7zft7yPiroi4LSK+HRFt/eb9RUTcFxF3R8RLJ6twSZLqwUh61F8DThow7Srg6Mx8JnAP8BcAEXEk8HrgqGqdz0dE44RVK0lSnRk2qDPzp8CGAdOuzMzu6uUNwKHV81OBb2bmzsx8ALgPOG4C65Ukqa5MxDHqPwV+UD0/BPhtv3mrq2mSJGkMxhXUEfExoBv4Rt+kQRbLIdY9JyJWRsTK9evXj6cMSZKmrTEHdUScCZwCnJGZfWG8GljUb7FDgTWDrZ+ZKzJzeWYuX7BgwVjLkCRpWhtTUEfEScCHgVdm5rZ+sy4HXh8RMyLiMGAZcNP4y5QkqT41DbdARFwInADsHxGrgY9TO8t7BnBVRADckJnvyMw7IuJi4E5qQ+LvzsyeySpekqTpbtigzsw3DDL5vN0s/yngU+MpSpIk1XhlMkmSCmZQS5JUMINakqSCGdSSJBXMoJYkqWAGtSRJBTOoJUkqmEEtSVLBDGpJkgpmUEuSVDCDWpKkghnUkiQVzKCWJKlgBrUkSQUzqCVJKphBLUlSwQxqSZIKZlBLklQwg1qSpIIZ1JIkFcygliSpYAa1JEkFM6glSSqYQS1JUsEMakmSCmZQS5JUMINakqSCGdSSJBXMoJYkqWAGtSRJBTOoJUkqmEEtSVLBDGpJkgpmUEuSVDCDWpKkghnUkiQVzKCWJKlgBrUkSQUzqCVJKphBLUlSwQxqSZIKZlBLklQwg1qSpIIZ1JIkFcygliSpYAa1JEkFM6glSSqYQS1JUsEMakmSCmZQS5JUMINakqSCGdSSJBXMoJYkqWAGtSRJBTOoJUkq2LBBHRFfjYjHIuL2ftP2jYirIuLe6nGfanpExD9FxH0RcVtEPHsyi5ckabobSY/6a8BJA6Z9BLg6M5cBV1evAV4GLKu+zgG+MDFlSpJUn4YN6sz8KbBhwORTgfOr5+cDr+o3/d+y5gagLSIWTlSxkiTVm7Eeoz4wM9cCVI8HVNMPAX7bb7nV1bSniIhzImJlRKxcv379GMuQJGl6m+iTyWKQaTnYgpm5IjOXZ+byBQsWTHAZkiRND2MN6nV9Q9rV42PV9NXAon7LHQqsGXt5kiTVt7EG9eXAmdXzM4HL+k1/c3X29/HApr4hckmSNHpNwy0QERcCJwD7R8Rq4OPAucDFEXE28DBwerX494GXA/cB24CzJqFmSZLqxrBBnZlvGGLWiYMsm8C7x1uUJEmq8cpkkiQVzKCWJKlgBrUkSQUzqCVJKphBLUlSwQxqSZIKZlBLklQwg1qSpIIZ1JIkFcygliSpYAa1JEkFM6glSSqYQS1JUsEMakmSCmZQS5JUMINakqSCGdSSJBXMoJYkqWAGtSRJBTOoJUkqmEEtSVLBDGpJkgpmUEuSVDCDWpKkghnUkiQVzKCWJKlgBrUkSQUzqCVJKphBLUlSwQxqSZIKZlBLklQwg1qSpIIZ1JIkFcygliSpYAa1JEkFM6glSSqYQS1JUsEMakmSCmZQS5JUMINakqSCGdSSJBXMoJYkqWAGtSRJBTOoJUkqmEEtSVLBDGpJkgpmUEuSVDCDWpKkghnUkiQVzKCWJKlgBrUkSQUzqCVJKphBLUlSwQxqSZIKZlBLklSwcQV1RPzPiLgjIm6PiAsjojUiDouIGyPi3oi4KCJaJqpYSZLqzZiDOiIOAd4HLM/Mo4FG4PXAp4HPZuYyYCNw9kQUKklSPRrv0HcTMDMimoBZwFrgRcAl1fzzgVeNcx+SJNWtMQd1Zj4C/F/gYWoBvQlYBbRnZne12GrgkMHWj4hzImJlRKxcv379WMuQJGlaG8/Q9z7AqcBhwMHAbOBlgyyag62fmSsyc3lmLl+wYMFYy5AkaVobz9D3i4EHMnN9ZnYBlwIvANqqoXCAQ4E146xRkqS6NZ6gfhg4PiJmRUQAJwJ3AtcAp1XLnAlcNr4SJUmqX+M5Rn0jtZPGbgF+VW1rBfBh4P0RcR+wH3DeBNQpSVJdahp+kaFl5seBjw+YfD9w3Hi2K0mSarwymSRJBTOoJUkqmEEtSVLBDGpJkgpmUEuSVDCDWpKkghnUkiQVzKCWJKlgBrUkSQUzqCVJKphBLUlSwQxqSZIKZlBLklQwg1qSpIIZ1JIkFcygliSpYAa1JEkFM6glSSqYQS1JUsEMakmSCmZQS5JUMINakqSCGdSSJBXMoJYkqWAGtSRJBTOoJUkqmEEtSVLBDGpJkgpmUEuSVDCDWpKkghnUkiQVzKCWJKlgBrUkSQUzqCVJKphBLUlSwQxqSZIKZlBLklQwg1qSpIIZ1JIkFcygliSpYAa1JEkFM6glSSqYQS1JUsEMakmSCmZQS5JUMINakqSCGdSSJBXMoJYkqWAGtSRJBTOoJUkqmEEtSVLBDGpJkgpmUEuSVDCDWpKkgo0rqCOiLSIuiYi7IuLXEfH8iNg3Iq6KiHurx30mqlhJkurNeHvU/wj8MDN/HzgG+DXwEeDqzFwGXF29liRJYzDmoI6IecALgfMAMrMzM9uBU4Hzq8XOB1413iIlSapX4+lRHw6sB/41In4REV+JiNnAgZm5FqB6PGAC6pQkqS6NJ6ibgGcDX8jMZwFbGcUwd0ScExErI2Ll+vXrx1GGJEnT13iCejWwOjNvrF5fQi2410XEQoDq8bHBVs7MFZm5PDOXL1iwYBxlSJI0fY05qDPzUeC3EXFENelE4E7gcuDMatqZwGXjqlCSpDrWNM713wt8IyJagPuBs6iF/8URcTbwMHD6OPchSVLdGldQZ+YvgeWDzDpxPNuVJEk1XplMkqSCGdSSJBXMoJYkqWAGtSRJBTOoJUkqmEEtSVLBDGpJkgpmUEuSVDCDWpKkghnUkiQVzKCWJKlgBrUkSQUzqCVJKphBLUlSwQxqSZIKZlBLklQwg1qSpIIZ1JIkFcygliSpYAa1JEkFM6glSSqYQS1JUsEMakmSCmZQS5JUMINakqSCGdSSJBXMoJYkqWAGtSRJBTOoJUkqmEEtSVLBDGpJkgpmUEuSVDCDWpKkghnUkiQVzKCWJKlgBrUkSQUzqCVJKphBLUlSwQxqSZIKZlBLklQwg1qSpIIZ1JIkFcygliSpYAa1JEkFM6glSSqYQS1JUsEMakmSCmZQS5JUMINakqSCGdSSJBXMoJYkqWAGtSRJBTOoJUkqmEEtSVLBDGpJkgo27qCOiMaI+EVEXFG9PiwiboyIeyPioohoGX+ZkiTVp4noUf8Z8Ot+rz8NfDYzlwEbgbMnYB+SJNWlcQV1RBwKnAx8pXodwIuAS6pFzgdeNZ59SJJUz8bbo/4c8CGgt3q9H9Cemd3V69XAIePchyRJdWvMQR0RpwCPZeaq/pMHWTSHWP+ciFgZESvXr18/1jIkSZrWxtOj/kPglRHxIPBNakPenwPaIqKpWuZQYM1gK2fmisxcnpnLFyxYMI4yJEmavsYc1Jn5F5l5aGYuBV4P/DgzzwCuAU6rFjsTuGzcVUqSVKcm43PUHwbeHxH3UTtmfd4k7EOSpLrQNPwiw8vMa4Frq+f3A8dNxHYlSap3XplMkqSCGdSSJBXMoJYkqWAGtSRJBTOoJUkqmEEtSVLBDGpJkgpmUEuSVDCDWpKkghnUkiQVzKCWJKlgBrUkSQUzqCVJKphBLUlSwQxqSZIKZlBLklQwg1qSpIIZ1JIkFcygliSpYAa1JEkFM6glSSqYQS1JUsEMakmSCmZQS5JUsLoN6h/95kd8/JqPT3UZkiTtVl0G9ZdXfZlLf30p/377v091KZIk7VbdBfVVv7mKd37vnQRBR2fHVJcjSdJuNU11AXvS7Y/dzmnfOo2jDjiKuS1zuXXdrVNdkiRJu1VXPeo/vexPmd08myvecAVtrW1s7dxKb/ZOdVmSJA2proL6oU0PceoRp7Jo/iJmNM4gSbZ3bZ/qsiRJGlJdBfWWnVuY0zIHgBlNMwA8Ti1JKlrdBHVPbw/bu7cb1JKkvUrdBPXWrq0AzJ0xF4AZjQa1JKl8dRPUfYHc16NubWp90nRJkkpUN0G9ZecW4HdBbY9akrQ3qJugHtij9hi1JGlvUHdBPbeldozaoW9J0t6g7oLaoW9J0t6kfoPaoW9J0l6gboJ6S+eTTyZrbmimIRoMaklS0eomqHcdo64+Rx0RzGmZY1BLkopWd0E9u3n2rmkGtSSpdHUV1DMaZ9Dc2Lxr2pyWOXR0GdSSpHLVTVD3vyFHH3vUkqTS1U1Qd3R1GNSSpL1O/QR1Z8euE8n6GNSSpNLVVVAP1qPuuwa4JEklqpugHvQYdbM9aklS2eomqIfqURvUkqSS1VVQ992Qo09fUGfmFFUlSdLu1VVQD9aj7skedvbsnKKqJEnavboJ6i2dg3+OGrwxhySpXHUR1N293ezo3mFQS5L2OnUR1Fs7twIMeowaDGpJUrnqIqgH3ou6j0EtSSrdmIM6IhZFxDUR8euIuCMi/qyavm9EXBUR91aP+0xcuWNjUEuS9lbj6VF3Ax/IzGcAxwPvjogjgY8AV2fmMuDq6vWU2tJZu/qYQS1J2tuMOagzc21m3lI93wL8GjgEOBU4v1rsfOBV4y1yvOxRS5L2VhNyjDoilgLPAm4EDszMtVALc+CAidjHePQF8WA35eg/X5Kk0ow7qCNiDvAfwP/IzM2jWO+ciFgZESvXr18/3jJ2a6gedV9wG9SSpFKNK6gjoplaSH8jMy+tJq+LiIXV/IXAY4Otm5krMnN5Zi5fsGDBeMoYVt8dsgYG9azmWYBBLUkq13jO+g7gPODXmfmZfrMuB86snp8JXDb28ibGUD3qhmhgdvNsg1qSVKymcaz7h8CbgF9FxC+raR8FzgUujoizgYeB08dX4vgNFdR90wxqSVKpxhzUmfkzIIaYfeJYtzsZOjo7aG1qpanhqd+uQS1JKlldXJlssBty9DGoJUklq4ugHuwWl30MaklSyeomqAfekKOPQS1JKlndBLU9aknS3qgugtpj1JKkvVVdBPVwPeq+m3ZIklQag9oetSSpYHUT1Ls7mayzp5POns49XJUkScOri6DesnP3x6gBtnZu3ZMlSZI0ItM+qLt6utjZs3PYoHb4W5JUomkf1Fu7aj1lg1qStDea9kHdF8B9954eyKCWJJWsboLaHrUkaW807YN6y87aZ6QNaknS3mjaB7U9aknS3sygNqglSQWrm6De3QVP+i8nSVJJpn1Q913H2x61JGlvNO2Derih76aGJlqbWg1qSVKR6iaoZ7fMHnIZb8whSSpVXQT1zKaZNDU0DbnMnJY5dHQZ1JKk8kz7oN7dDTn62KOWJJVq2gd1R9fQ96LuY1BLkko1/YO606CWJO296iKoh7ohRx+DWpJUqmkf1B6jliTtzaZ9UI9o6LvZoJYklcmgxh61JKlc9RHUzcMH9baubXT3du+hqiRJGpm6COrhTiY7cM6BAKzrWLcnSpIkacSmdVB39XSxs2fnsEPfS+YvAeChTQ/tibIkSRqxaR3Um3ZuAoa+xWWfxfMXA/DwpocnvSZJkkZjWgd1X/Aumr9ot8v1zTeoJUmlmdZB/WD7gwAsbVu62+XmzZhHW2ubQS1JKk5dBPVhbYcNu+yS+Us8Ri1JKs60D+q+3vJwFs9fbI9aklScaR/US9uWEhHDLmtQS5JKVBdBPRKL5y+mfUc7m3duntyiJEkahWkb1JlZC+r5S0e0fN9nqe1VS9qbrVi1ghWrVkx1GZpA0zaoN+7YyJbOLaPqUYNBLUkqy7QN6gc2PgAM/9GsPga1JKlE0zaod300a5/hP5oFcNCcg2hqaDKoJUlFmfZBPdIedWNDI4vmLfKz1JKkokzroJ4/Y/6IPkPdx49oSZJKM32DetPIP5rVx6CWJJVm+gb1KD5D3Wfx/MU8svkRunu7J6coSZJGaVoG9a7PUI8yqJfMX0JP9rBmy5rJKUySpFGalkG9YfsGOjo7xtSjBj+iJakMXrxEME2D+oH20X2Guo9BLUkj45uIPWdaBvVobm/Zn0EtSSrNtA7qJW1LRrXe7JbZ7DdzPx5q97PUkqQyTNugbmttG9VnqPssnr+YhzeX36N22EmS6sO0DerRHp/u42epJU2WyX6DvaN7B09se4Le7J20fWjPa5rqAibDg+0Psmy/ZWNad/H8xVzz4DUTXNHE6urp4vbHbqcxGunNXhpiz7zf6vsHc85zztkj+5M0cnc/fjcvveClPLTpIVoaW/jyLV/mhCUn8Hcn/h0zmmZMdXkah2kX1H2foX7J4S8Z0/pL5i9h887NtO9oH9PQ+WTp7u3m+oev56+v/WtWrVnF1q6tAPzgvh/w3uPey5nHnsmcljlTXKWkiXL9w9fzw/t+yHMWPmfYZW9cfSMn//vJNEQDpx95Ohu2b6AhGvjMDZ/hrifu4j9e+x+0NrVOSF3burbx9Vu/TvuOdvaZuc+EbFO7N2lBHREnAf8INAJfycxzJ2tf/T2x/Qm2dm0d19A3wLk/O5cP/+GHR/SLuLueZmdPJ492PMraLWt5tONRGqKBfWfuyz4z92Fm00wiAoCe3h52dO9gZ8/O2mN37XHjjo1c+ZsrueKeK3hi+xM0NzRz7EHH8txDnsuOrh386rFf8Z4fvIcPXvVBTjz8RE5ZdgonLD2B3uylo7ODjs4Ounu7+e493yUzOeOZZ7Bo3iIOmnMQjQ2Nw35v7Tvaue6h6/jxAz/m0rsupSEa+MLKL7BP6z6c8Qdn8PJlL2fh3IWjaWIANu3YxPm3nk9rU6s9dKmf7V3b+diPP8bnbvgcSfLtu77NdQ9fx9uf83ZefPiLmTtjLlD7v9PV08V+s/bj7MvPZuGchfzwjT/kxw/8GKj9P/ryqi/z9ivezisufAXfed13mN0ye9d+xjJCtmrNKs649AzufuJuWptaOe0Zp/G2Z79t1/+xvdHeMFI4KUEdEY3AvwAvAVYDN0fE5Zl552Tsr7+++1CP9PaWA5389JM59YhT+fT1n+Zfbv4Xznn2OazevJpt3dtYtu8yNu3YxKadta/2He1s2rGJRzsepSd7+PT1n2b/Wfszp2UO67euZ23HWh7f9vi4v6e21jZOXlara23H2ie9M37eoc/jRYe9iK2dW/nevd/jinuu2O22/vnmfwagMRrZb9Z+zG2Zy5yWObQ2tdKbvfRkD509nWzcvpEN2zewvXs7AK1NrSyZv4TGhkbWdazjrsfv2nWIYPnByzlhyQksaVvCofMO5cDZBz7pDciG7Rt4fNvjrNu6jtvW3cbNa27mvg33EQQHzz2Ymx+5maMOOIr5M+Yzb8Y8WhpbaN/RzhX3XMGOnh2csuwUlrQtYcn8JbS1tjGzeSYzm2bS3Ng87raF2mhFECN647I7faM59264lw3bN9Tar2s7D216iKVtS3n/898/IfVq77Ktaxvfvfu7/OC+H7ByzUq2dm3lMz//DMccdAwnPe0kTvq9k5jZPJN7n7iXux6/i09d9ynufuJu3rX8XRw05yBuWnMTt6y9hddc/BoaooFjDzqW5QuXc+2D13J/+/1093bz7IXP5vt/8n0OnHPgrqAGeNtz3saMphmcddlZvOTrL+GsY8/imQc+k6MPOHpU30NnTyf/8P/+gb++9q85cPaBXPDqC/jkTz/JBb+6gNVbVvPZl36WYw86dqKbTpXIzInfaMTzgU9k5kur138BkJn/e7Dlly9fnitXrpyQfd/0yE188KoP8sWTv8gzFjxjyOVWrFrxlHdQ/d9Z3bbuNs792blcdMdFu07MmNE4Y1dIzGyeydP3ezptrW2s2byGhoYGtnVuo6Org/kz5rNg9gLat7czr3UebTPamN9aCyGo/eFu7dpKV0/Xrn1HBM0NzTQ3NNPU0ERzY+15c2MzC+csHFGIZCaPdjzKA+0P0NzQTGtTKy2NLTQ2NBIRBMHWzq1s2LGBjds3srVz665efFdvFw00EBE0RiOzWmYxu3k2Jx52IscfejzHH3o85996/pP29ciWR7ht3W2s27qOW9beQmdP57A1Lpq3iP1n7c/i+Yvp6e3hgfYHWNuxlg3bNwy77kBNDU27fhZ932tLYwuN0UhP9tDd201Pbw8zm2cyu3k2s5pnsbNnJx2dHWzZuYWOzg7ad7TT1dtFYzSycO5CDp13KAfPPZh9WvehrbWNuS1zaYgGkqTvb6Xv+c6enbXtdHXwUPtD/PLRX7Jp56Yh6z2s7TCOPuBojlxwJM/Y/xksmL2A5oZmWhpbiAg6ezrp7OmkN3tpaWxhRuMMZjTNeNIjQG/2PuUryV3Pu3u7nzQq03+kZtcuK/biAAALeklEQVTr7p00RAO3rL2FxoZGXvZ7L2PB7AUcMPsA9mndh5bGFpoba7X1/R62NLZM+PkQmUlXbxcdnR1s3L6R9h3tbNxRe7zsrsvo6u3i1b//ag6acxAHzTmI+a3zmdsyl1nNs8bdi8vMQdtyqK/u3m427dy0q06o/Q72/b32Pd/RvYNHOx7l0Y5HWbV2FZf++lI6OjtYMGsBs5pnMbtlNkfsdwQ3rL6BtR1rn1LXonmL+OqpX+XFh7941/+ks449i5889BOue+g6rnv4OlatXUVbaxtH7HcE73ve+3jx4S/e9QZ+sB7ixXdczNuvePuuugHmtMyhbUYbRx94NE/f9+kcc9AxHHPgMSxpW8LMptrf1D1P3MN5vziP8289n8e3Pc5rj3otXzj5C+w7c1++tPJL/Ozhn3H5PZezeedmXrDoBbxr+bt4+bKXM791/h47d2Y0erOXjds3snrzam565CZuWH0DV/7mSnb27KSttY2GaGDx/MW8YNELeMGiF3DUgqOY0zKHWc2zJqxj0F9ErMrM5cMuN0lBfRpwUma+tXr9JuB5mfmewZafyKAeqZGeebm1s3YsuLWpddw9rumsb6i9fUc7m3du3jU9CGa3zGZOyxzmtswd9KSWzGRb17ZdIdLV28XMppnMbpnNjMYZbN65mSe2P8GG7Rtq83u66OythVpXT9eu1z29PfRkD729vTQ0NNAYjQSxa9nOnk6aG5p/F3xV+LU2tdLd270rJDbt2MT2ru1s69622zcfTQ1Nu9afN2Mei+cv5tB5h7JwzsJdf9yNDY08svkRHtz0IE0NTdzx2B3c88Q9dPV2DbndkjVEw67g7vtHPPB/SPK71/3nDZze3ds95nZoiNrPd6j9jaSGPWH+jPn88TP+mDc+8428cMkLOe8X5z2pruMPPZ4rf3MlNz5yIwfMPoB3P/fdLNtvGS2NLcDuh2WHmjfU9N7s5aH2h7h13a3c/tjtXPWbq9i0cxNNDU3c9fhdu857GaipoYlTjziVc55zDi85/CW73iD17ef0I0/n/FvP5/M3f557N9wL1P7u586Yy+zm2bs6CX3r9T0falp/A39eI/1dG2zdzp5OHt/2+JNuurTfzP1YOGchs5pn8bR9n0ZP9nDPE/dw27rbnnLmfHNDM4/++aPsO3PfQdtpLKY6qE8HXjogqI/LzPf2W+YcoO836Qjg7gkvZPf2B8Y/Ll1fbLOxsd3GxnYbG9ttbKai3ZZk5oLhFpqsk8lWA4v6vT4UeNItqTJzBTBlV+yIiJUjeSej37HNxsZ2GxvbbWxst7Epud0m6yDCzcCyiDgsIlqA1wOXT9K+JEmatialR52Z3RHxHuBH1D6e9dXMvGMy9iVJ0nQ2aZ+jzszvA9+frO1PAC+UPXq22djYbmNju42N7TY2xbbbpJxMJkmSJkZ5H3STJEm7TOugjoiTIuLuiLgvIj4yyPwZEXFRNf/GiFi656sszwja7f0RcWdE3BYRV0fE6G78PU0N1279ljstIjIiijzDdE8bSbtFxGur37k7IuLf93SNJRrB3+niiLgmIn5R/a2+fCrqLElEfDUiHouI24eYHxHxT1Wb3hYRz97TNQ4qM6flF7WT2H4DHA60ALcCRw5Y5l3AF6vnrwcumuq6p/prhO3234BZ1fN32m4ja7dqubnAT4EbgOVTXfdUf43w920Z8Atgn+r1AVNd91R/jbDdVgDvrJ4fCTw41XVP9RfwQuDZwO1DzH858AMggOOBG6e65syc1j3q44D7MvP+zOwEvgmcOmCZU4G+62JeApwYe/PV5SfGsO2Wmddk5rbq5Q3UPidf70by+wbwt8D/AXbsyeIKNpJ2exvwL5m5ESAzH9vDNZZoJO2WwLzq+XwGXMuiHmXmT4HdXa/4VODfsuYGoC0iRn/XoQk2nYP6EOC3/V6vrqYNukxmdgObgP32SHXlGkm79Xc2tXeg9W7YdouIZwGLMnP3d06pLyP5fXs68PSIuD4ibqjuzFfvRtJunwDeGBGrqX0C571oOKP9/7dHTLv7UfczWM944CnuI1mm3oy4TSLijcBy4L9OakV7h922W0Q0AJ8F3rKnCtpLjOT3rYna8PcJ1EZvrouIozOzfeCKdWQk7fYG4GuZ+Q/VjZK+XrVb7yDrqqbITJjOPephL2Paf5mIaKI2PDT62zhNLyNpNyLixcDHgFdm5s49VFvJhmu3ucDRwLUR8SC141+Xe0LZiP9OL8vMrsx8gNp9AZbtofpKNZJ2Oxu4GCAzfw60UruetYY2ov9/e9p0DuqRXMb0cuDM6vlpwI+zOqOgjg3bbtUQ7peohbTHC2t2226ZuSkz98/MpZm5lNqx/Vdm5p69bVx5RvJ3+h1qJzASEftTGwq/f49WWZ6RtNvDwIkAEfEMakG9fo9Wufe5HHhzdfb38cCmzHzqvUj3sGk79J1DXMY0Ij4JrMzMy4HzqA0H3UetJ/36qau4DCNst78H5gDfqs69ezgzXzllRRdghO2mAUbYbj8C/ntE3An0AB/MzCemruqpN8J2+wDw5Yj4n9SGb99S7x2RiLiQ2iGU/atj9x8HmgEy84vUjuW/HLgP2AacNTWVPplXJpMkqWDTeehbkqS9nkEtSVLBDGpJkgpmUEuSVDCDWpKkghnUkiQVzKCW9rCI+ERE/PkY1ntLRPzzZNRUbf+EiBjVdcgj4sHqIiQDp78jIt5cPf9aRJxWPf9KRBxZPf/oRNQtTXfT9oInkp4qIpqqG9BMquriEYNNf2u/lx8F/m6ya5H2dvaopT0gIj4WEXdHxH8CR1TTnhYRP4yIVRFxXUT8fjX9FRFxY0T8IiL+MyIOHOE+vhYRX6y2dU9EnFJNf0tEfCsivgtcWV0e8e8j4vaI+FVEvK7fZuZFxLcj4s5qWw3VNr4QESsj4o6I+JsBu/5gRNxUff1etfygowYRcW1ELI+Ic4GZEfHLiPhGRPxtRPxZv+U+FRHvG3EDS9OYPWppkkXEc6hdnvZZ1P7mbgFWASuAd2TmvRHxPODzwIuAnwHHZ2ZGxFuBD1G7HORILKV2N7OnAdf0BSfwfOCZmbkhIv4YOBY4htpNGm6OiJ9Wyx0HHAk8BPwQeA21e7V/rFq3Ebg6Ip6ZmbdV62zOzOOqoe7PAacMV2RmfiQi3pOZx1ZttBS4FPjH6s3B66tapLpnUEuT74+Ab2fmNoCIuJzaDRJewO+ulw4wo3o8FLgoajesbwEeGMW+Lq5uY3hvRNwP/H41/arM7Lsz3H8BLszMHmBdRPwEeC6wGbgpM++v6rywWvYS4LURcQ61/xkLqYV5X1Bf2O/xs6OodZfMfDAinqhu+HIg8It6v5631MeglvaMgRfVbwDa+3qUA/x/wGcy8/KIOAH4xDj20/d6a79pg91zd8j1I+Iw4M+B52bmxoj4GrU3GoOtM56bB3yF2v26DwK+Oo7tSNOKx6ilyfdT4NURMTMi5gKvoHZnngci4nSA6rjxMdXy84FHqudnPmVru3d6RDRExNOAw6ndu3mwel4XEY0RsQB4IXBTNe+46taJDcDrqA3Dz6MW9Juq4+UvG7C91/V7/Pkoau2KiOZ+r78NnEStd/+jUWxHmtbsUUuTLDNviYiLgF9SO/Z7XTXrDOALEfGX1G61903gVmo96G9FxCPU7lt92Ch2dzfwE2rDx+/IzB39htb7fJvaMetbqfWAP5SZj1Yns/0cOBf4A2qB/u3M7I2IXwB3ULsP9PUDtjcjIm6k9sb/DaOodQVwW0TckplnZGZnRFxDbaShZxTbkaY1b3MpTRPVkPQVmXnJVNcyFlUv/hbg9My8d6rrkUrh0LekKVddBOU+4GpDWnoye9TSXiYiPgacPmDytzLzU1NRj6TJZVBLklQwh74lSSqYQS1JUsEMakmSCmZQS5JUMINakqSC/f8sUh0XKol8mgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7ff828937c18>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize = (8, 8))\n",
    "plt.title('Distribuition for deal prob')\n",
    "sns.distplot(df_y_train ,color='green', kde=True,bins=200)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_y_train = df_y_train.round()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.12789\n",
       "1    0.00000\n",
       "2    0.43177\n",
       "3    0.80323\n",
       "4    0.20797\n",
       "5    0.80323\n",
       "6    0.00000\n",
       "7    0.80323\n",
       "8    0.00000\n",
       "9    0.00000\n",
       "Name: deal_probability, dtype: float64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_y_train.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>region</th>\n",
       "      <th>city</th>\n",
       "      <th>parent_category_name</th>\n",
       "      <th>category_name</th>\n",
       "      <th>param_1</th>\n",
       "      <th>param_2</th>\n",
       "      <th>param_3</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>price</th>\n",
       "      <th>item_seq_number</th>\n",
       "      <th>activation_date</th>\n",
       "      <th>user_type</th>\n",
       "      <th>image</th>\n",
       "      <th>image_top_1</th>\n",
       "      <th>avg_days_up_user</th>\n",
       "      <th>avg_times_up_user</th>\n",
       "      <th>n_user_items</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b912c3c6a6ad</td>\n",
       "      <td>e00f8ff2eaf9</td>\n",
       "      <td>Свердловская область</td>\n",
       "      <td>Екатеринбург</td>\n",
       "      <td>Личные вещи</td>\n",
       "      <td>Товары для детей и игрушки</td>\n",
       "      <td>Постельные принадлежности</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Кокоби(кокон для сна)</td>\n",
       "      <td>Кокон для сна малыша,пользовались меньше месяц...</td>\n",
       "      <td>400.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2017-03-28</td>\n",
       "      <td>Private</td>\n",
       "      <td>d10c7e016e03247a3bf2d13348fe959fe6f436c1caf64c...</td>\n",
       "      <td>1008.0</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2dac0150717d</td>\n",
       "      <td>39aeb48f0017</td>\n",
       "      <td>Самарская область</td>\n",
       "      <td>Самара</td>\n",
       "      <td>Для дома и дачи</td>\n",
       "      <td>Мебель и интерьер</td>\n",
       "      <td>Другое</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Стойка для Одежды</td>\n",
       "      <td>Стойка для одежды, под вешалки. С бутика.</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>19</td>\n",
       "      <td>2017-03-26</td>\n",
       "      <td>Private</td>\n",
       "      <td>79c9392cc51a9c81c6eb91eceb8e552171db39d7142700...</td>\n",
       "      <td>692.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ba83aefab5dc</td>\n",
       "      <td>91e2f88dd6e3</td>\n",
       "      <td>Ростовская область</td>\n",
       "      <td>Ростов-на-Дону</td>\n",
       "      <td>Бытовая электроника</td>\n",
       "      <td>Аудио и видео</td>\n",
       "      <td>Видео, DVD и Blu-ray плееры</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Philips bluray</td>\n",
       "      <td>В хорошем состоянии, домашний кинотеатр с blu ...</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>9</td>\n",
       "      <td>2017-03-20</td>\n",
       "      <td>Private</td>\n",
       "      <td>b7f250ee3f39e1fedd77c141f273703f4a9be59db4b48a...</td>\n",
       "      <td>3032.0</td>\n",
       "      <td>4.428571</td>\n",
       "      <td>1.142857</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>02996f1dd2ea</td>\n",
       "      <td>bf5cccea572d</td>\n",
       "      <td>Татарстан</td>\n",
       "      <td>Набережные Челны</td>\n",
       "      <td>Личные вещи</td>\n",
       "      <td>Товары для детей и игрушки</td>\n",
       "      <td>Автомобильные кресла</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Автокресло</td>\n",
       "      <td>Продам кресло от0-25кг</td>\n",
       "      <td>2200.0</td>\n",
       "      <td>286</td>\n",
       "      <td>2017-03-25</td>\n",
       "      <td>Company</td>\n",
       "      <td>e6ef97e0725637ea84e3d203e82dadb43ed3cc0a1c8413...</td>\n",
       "      <td>796.0</td>\n",
       "      <td>16.714286</td>\n",
       "      <td>2.642857</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7c90be56d2ab</td>\n",
       "      <td>ef50846afc0b</td>\n",
       "      <td>Волгоградская область</td>\n",
       "      <td>Волгоград</td>\n",
       "      <td>Транспорт</td>\n",
       "      <td>Автомобили</td>\n",
       "      <td>С пробегом</td>\n",
       "      <td>ВАЗ (LADA)</td>\n",
       "      <td>2110</td>\n",
       "      <td>ВАЗ 2110, 2003</td>\n",
       "      <td>Все вопросы по телефону.</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2017-03-16</td>\n",
       "      <td>Private</td>\n",
       "      <td>54a687a3a0fc1d68aed99bdaaf551c5c70b761b16fd0a2...</td>\n",
       "      <td>2264.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>51e0962387f7</td>\n",
       "      <td>bbfad0b1ad0a</td>\n",
       "      <td>Татарстан</td>\n",
       "      <td>Чистополь</td>\n",
       "      <td>Личные вещи</td>\n",
       "      <td>Товары для детей и игрушки</td>\n",
       "      <td>Автомобильные кресла</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Авто люлька</td>\n",
       "      <td>В хорошем состоянии</td>\n",
       "      <td>1300.0</td>\n",
       "      <td>9</td>\n",
       "      <td>2017-03-28</td>\n",
       "      <td>Private</td>\n",
       "      <td>eb6ad1231c59d3dc7e4020e724ffe8e4d302023ddcbb99...</td>\n",
       "      <td>796.0</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>c4f260a2b48a</td>\n",
       "      <td>08f469d2e6f7</td>\n",
       "      <td>Нижегородская область</td>\n",
       "      <td>Нижний Новгород</td>\n",
       "      <td>Для дома и дачи</td>\n",
       "      <td>Ремонт и строительство</td>\n",
       "      <td>Сантехника и сауна</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Водонагреватель 100 литров нержавейка плоский</td>\n",
       "      <td>Электро водонагреватель накопительный на 100 л...</td>\n",
       "      <td>11000.0</td>\n",
       "      <td>125</td>\n",
       "      <td>2017-03-23</td>\n",
       "      <td>Private</td>\n",
       "      <td>0330f6ac561f5db1fa8226dd5e7e127b5671d44d075a98...</td>\n",
       "      <td>2823.0</td>\n",
       "      <td>7.333333</td>\n",
       "      <td>1.222222</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>6b71309d6a8a</td>\n",
       "      <td>fef86baa002c</td>\n",
       "      <td>Пермский край</td>\n",
       "      <td>Пермь</td>\n",
       "      <td>Личные вещи</td>\n",
       "      <td>Одежда, обувь, аксессуары</td>\n",
       "      <td>Женская одежда</td>\n",
       "      <td>Джинсы</td>\n",
       "      <td>26</td>\n",
       "      <td>Бойфренды colins</td>\n",
       "      <td>Бойфренды в хорошем состоянии.</td>\n",
       "      <td>500.0</td>\n",
       "      <td>61</td>\n",
       "      <td>2017-03-25</td>\n",
       "      <td>Private</td>\n",
       "      <td>9bab29a519e81c14f4582024adfebd4f11a4ac71d323a6...</td>\n",
       "      <td>567.0</td>\n",
       "      <td>11.916667</td>\n",
       "      <td>1.666667</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>c5b969cb63a2</td>\n",
       "      <td>055825270190</td>\n",
       "      <td>Оренбургская область</td>\n",
       "      <td>Оренбург</td>\n",
       "      <td>Личные вещи</td>\n",
       "      <td>Одежда, обувь, аксессуары</td>\n",
       "      <td>Женская одежда</td>\n",
       "      <td>Платья и юбки</td>\n",
       "      <td>&gt; 50 (XXL)</td>\n",
       "      <td>Платье</td>\n",
       "      <td>54 раз мер очень удобное</td>\n",
       "      <td>500.0</td>\n",
       "      <td>85</td>\n",
       "      <td>2017-03-17</td>\n",
       "      <td>Private</td>\n",
       "      <td>75ce06d1f939a31dfb2af8ac55f08fa998fa336d13ee05...</td>\n",
       "      <td>415.0</td>\n",
       "      <td>13.517241</td>\n",
       "      <td>1.896552</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>b1570962e68c</td>\n",
       "      <td>f9e8f831d94c</td>\n",
       "      <td>Нижегородская область</td>\n",
       "      <td>Нижний Новгород</td>\n",
       "      <td>Личные вещи</td>\n",
       "      <td>Детская одежда и обувь</td>\n",
       "      <td>Для девочек</td>\n",
       "      <td>Обувь</td>\n",
       "      <td>25</td>\n",
       "      <td>Полу ботиночки замш натур.Бамбини</td>\n",
       "      <td>По стельке 15.5см мерить приокский район. Цвет...</td>\n",
       "      <td>400.0</td>\n",
       "      <td>136</td>\n",
       "      <td>2017-03-22</td>\n",
       "      <td>Company</td>\n",
       "      <td>54fb8521135fda77a860bfd2fac6bf46867ab7c06796e3...</td>\n",
       "      <td>46.0</td>\n",
       "      <td>15.632184</td>\n",
       "      <td>2.045977</td>\n",
       "      <td>93</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        item_id       user_id                 region              city  \\\n",
       "0  b912c3c6a6ad  e00f8ff2eaf9   Свердловская область      Екатеринбург   \n",
       "1  2dac0150717d  39aeb48f0017      Самарская область            Самара   \n",
       "2  ba83aefab5dc  91e2f88dd6e3     Ростовская область    Ростов-на-Дону   \n",
       "3  02996f1dd2ea  bf5cccea572d              Татарстан  Набережные Челны   \n",
       "4  7c90be56d2ab  ef50846afc0b  Волгоградская область         Волгоград   \n",
       "5  51e0962387f7  bbfad0b1ad0a              Татарстан         Чистополь   \n",
       "6  c4f260a2b48a  08f469d2e6f7  Нижегородская область   Нижний Новгород   \n",
       "7  6b71309d6a8a  fef86baa002c          Пермский край             Пермь   \n",
       "8  c5b969cb63a2  055825270190   Оренбургская область          Оренбург   \n",
       "9  b1570962e68c  f9e8f831d94c  Нижегородская область   Нижний Новгород   \n",
       "\n",
       "  parent_category_name               category_name  \\\n",
       "0          Личные вещи  Товары для детей и игрушки   \n",
       "1      Для дома и дачи           Мебель и интерьер   \n",
       "2  Бытовая электроника               Аудио и видео   \n",
       "3          Личные вещи  Товары для детей и игрушки   \n",
       "4            Транспорт                  Автомобили   \n",
       "5          Личные вещи  Товары для детей и игрушки   \n",
       "6      Для дома и дачи      Ремонт и строительство   \n",
       "7          Личные вещи   Одежда, обувь, аксессуары   \n",
       "8          Личные вещи   Одежда, обувь, аксессуары   \n",
       "9          Личные вещи      Детская одежда и обувь   \n",
       "\n",
       "                       param_1        param_2     param_3  \\\n",
       "0    Постельные принадлежности            NaN         NaN   \n",
       "1                       Другое            NaN         NaN   \n",
       "2  Видео, DVD и Blu-ray плееры            NaN         NaN   \n",
       "3         Автомобильные кресла            NaN         NaN   \n",
       "4                   С пробегом     ВАЗ (LADA)        2110   \n",
       "5         Автомобильные кресла            NaN         NaN   \n",
       "6           Сантехника и сауна            NaN         NaN   \n",
       "7               Женская одежда         Джинсы          26   \n",
       "8               Женская одежда  Платья и юбки  > 50 (XXL)   \n",
       "9                  Для девочек          Обувь          25   \n",
       "\n",
       "                                           title  \\\n",
       "0                          Кокоби(кокон для сна)   \n",
       "1                              Стойка для Одежды   \n",
       "2                                 Philips bluray   \n",
       "3                                     Автокресло   \n",
       "4                                 ВАЗ 2110, 2003   \n",
       "5                                    Авто люлька   \n",
       "6  Водонагреватель 100 литров нержавейка плоский   \n",
       "7                               Бойфренды colins   \n",
       "8                                         Платье   \n",
       "9              Полу ботиночки замш натур.Бамбини   \n",
       "\n",
       "                                         description    price  \\\n",
       "0  Кокон для сна малыша,пользовались меньше месяц...    400.0   \n",
       "1          Стойка для одежды, под вешалки. С бутика.   3000.0   \n",
       "2  В хорошем состоянии, домашний кинотеатр с blu ...   4000.0   \n",
       "3                             Продам кресло от0-25кг   2200.0   \n",
       "4                           Все вопросы по телефону.  40000.0   \n",
       "5                                В хорошем состоянии   1300.0   \n",
       "6  Электро водонагреватель накопительный на 100 л...  11000.0   \n",
       "7                     Бойфренды в хорошем состоянии.    500.0   \n",
       "8                           54 раз мер очень удобное    500.0   \n",
       "9  По стельке 15.5см мерить приокский район. Цвет...    400.0   \n",
       "\n",
       "   item_seq_number activation_date user_type  \\\n",
       "0                2      2017-03-28   Private   \n",
       "1               19      2017-03-26   Private   \n",
       "2                9      2017-03-20   Private   \n",
       "3              286      2017-03-25   Company   \n",
       "4                3      2017-03-16   Private   \n",
       "5                9      2017-03-28   Private   \n",
       "6              125      2017-03-23   Private   \n",
       "7               61      2017-03-25   Private   \n",
       "8               85      2017-03-17   Private   \n",
       "9              136      2017-03-22   Company   \n",
       "\n",
       "                                               image  image_top_1  \\\n",
       "0  d10c7e016e03247a3bf2d13348fe959fe6f436c1caf64c...       1008.0   \n",
       "1  79c9392cc51a9c81c6eb91eceb8e552171db39d7142700...        692.0   \n",
       "2  b7f250ee3f39e1fedd77c141f273703f4a9be59db4b48a...       3032.0   \n",
       "3  e6ef97e0725637ea84e3d203e82dadb43ed3cc0a1c8413...        796.0   \n",
       "4  54a687a3a0fc1d68aed99bdaaf551c5c70b761b16fd0a2...       2264.0   \n",
       "5  eb6ad1231c59d3dc7e4020e724ffe8e4d302023ddcbb99...        796.0   \n",
       "6  0330f6ac561f5db1fa8226dd5e7e127b5671d44d075a98...       2823.0   \n",
       "7  9bab29a519e81c14f4582024adfebd4f11a4ac71d323a6...        567.0   \n",
       "8  75ce06d1f939a31dfb2af8ac55f08fa998fa336d13ee05...        415.0   \n",
       "9  54fb8521135fda77a860bfd2fac6bf46867ab7c06796e3...         46.0   \n",
       "\n",
       "   avg_days_up_user  avg_times_up_user  n_user_items  \n",
       "0          8.000000           2.000000             2  \n",
       "1               NaN                NaN             1  \n",
       "2          4.428571           1.142857             9  \n",
       "3         16.714286           2.642857            32  \n",
       "4               NaN                NaN             1  \n",
       "5          4.000000           1.000000             2  \n",
       "6          7.333333           1.222222            11  \n",
       "7         11.916667           1.666667            23  \n",
       "8         13.517241           1.896552            38  \n",
       "9         15.632184           2.045977            93  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_x_train.head(10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1503424 entries, 0 to 1503423\n",
      "Data columns (total 20 columns):\n",
      "item_id                 1503424 non-null object\n",
      "user_id                 1503424 non-null object\n",
      "region                  1503424 non-null object\n",
      "city                    1503424 non-null object\n",
      "parent_category_name    1503424 non-null object\n",
      "category_name           1503424 non-null object\n",
      "param_1                 1441848 non-null object\n",
      "param_2                 848882 non-null object\n",
      "param_3                 640859 non-null object\n",
      "title                   1503424 non-null object\n",
      "description             1387148 non-null object\n",
      "price                   1418062 non-null float64\n",
      "item_seq_number         1503424 non-null int64\n",
      "activation_date         1503424 non-null object\n",
      "user_type               1503424 non-null object\n",
      "image                   1390836 non-null object\n",
      "image_top_1             1390836 non-null float64\n",
      "avg_days_up_user        1166549 non-null float64\n",
      "avg_times_up_user       1166549 non-null float64\n",
      "n_user_items            1503424 non-null int64\n",
      "dtypes: float64(4), int64(2), object(14)\n",
      "memory usage: 240.9+ MB\n"
     ]
    }
   ],
   "source": [
    "df_x_train.info()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 508438 entries, 0 to 508437\n",
      "Data columns (total 20 columns):\n",
      "item_id                 508438 non-null object\n",
      "user_id                 508438 non-null object\n",
      "region                  508438 non-null object\n",
      "city                    508438 non-null object\n",
      "parent_category_name    508438 non-null object\n",
      "category_name           508438 non-null object\n",
      "param_1                 485528 non-null object\n",
      "param_2                 275209 non-null object\n",
      "param_3                 202107 non-null object\n",
      "title                   508438 non-null object\n",
      "description             508438 non-null object\n",
      "price                   477853 non-null float64\n",
      "item_seq_number         508438 non-null int64\n",
      "activation_date         508438 non-null object\n",
      "user_type               508438 non-null object\n",
      "image                   465829 non-null object\n",
      "image_top_1             465829 non-null float64\n",
      "avg_days_up_user        384612 non-null float64\n",
      "avg_times_up_user       384612 non-null float64\n",
      "n_user_items            508438 non-null int64\n",
      "dtypes: float64(4), int64(2), object(14)\n",
      "memory usage: 81.5+ MB\n"
     ]
    }
   ],
   "source": [
    "df_test.info()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "item_id 1503424\n",
      "user_id 771769\n",
      "region 28\n",
      "city 1733\n",
      "parent_category_name 9\n",
      "category_name 47\n",
      "param_1 372\n",
      "param_2 272\n",
      "param_3 1220\n",
      "title 788377\n",
      "description 1317103\n",
      "price 17007\n",
      "item_seq_number 28232\n",
      "activation_date 21\n",
      "user_type 3\n",
      "image 1390837\n",
      "image_top_1 3063\n",
      "avg_days_up_user 16333\n",
      "avg_times_up_user 6882\n",
      "n_user_items 1002\n"
     ]
    }
   ],
   "source": [
    "# viewing # of unique value in each column \n",
    "for col in df_x_train.columns:\n",
    "    print(col, len(df_x_train[col].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "value = { \n",
    "    'image_top_1' : 3067,\n",
    "    'param_1' : \"_NA_\",\n",
    "    'param_2' : \"_NA_\",\n",
    "    'param_3' : \"_NA_\",\n",
    "    'description' : \"_NA_\",\n",
    "    'title' : \"_NA_\",\n",
    "    'avg_days_up_user' : -1,\n",
    "    'avg_times_up_user' : -1,\n",
    "    'n_user_items' : -1,\n",
    "}\n",
    "\n",
    "df_x_train = df_x_train.fillna(value=value)\n",
    "df_test = df_test.fillna(value=value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_x_train\n",
    "user_id_in_both = np.intersect1d(df_x_train['user_id'].values, df_test['user_id'].values) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/antor/miniconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils import to_categorical\n",
    "import argparse\n",
    "from fastText import load_model\n",
    "\n",
    "#create config init\n",
    "config = argparse.Namespace()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_categorical_idx(col, df_trn, df_test):\n",
    "    merged = pd.concat([df_trn[col], df_test[col]])\n",
    "    train_size = df_trn[col].shape[0]\n",
    "    idxs, uniques = pd.factorize(merged)\n",
    "    \n",
    "    return idxs[:train_size], idxs[train_size:], uniques"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_reg, te_reg, tknzr_reg    = to_categorical_idx('region', df_x_train, df_test)\n",
    "tr_pcn, te_pcn, tknzr_pcn    = to_categorical_idx('parent_category_name', df_x_train, df_test)\n",
    "tr_cn, te_cn, tknzr_cn       = to_categorical_idx('category_name', df_x_train, df_test)\n",
    "tr_ut, te_ut, tknzr_ut       = to_categorical_idx('user_type', df_x_train, df_test)\n",
    "tr_city, te_city, tknzr_city = to_categorical_idx('city', df_x_train, df_test)\n",
    "\n",
    "tr_p1, te_p1, tknzr_p1 = to_categorical_idx('param_1', df_x_train, df_test)\n",
    "tr_p2, te_p2, tknzr_p2 = to_categorical_idx('param_2', df_x_train, df_test)\n",
    "tr_p3, te_p3, tknzr_p3 = to_categorical_idx('param_3', df_x_train, df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_week = pd.to_datetime(df_x_train['activation_date']).dt.weekday.astype(np.int32).values\n",
    "te_week = pd.to_datetime(df_test['activation_date']).dt.weekday.astype(np.int32).values\n",
    "tr_week = np.expand_dims(tr_week, axis=-1)\n",
    "te_week = np.expand_dims(te_week, axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_imgt1 = df_x_train['image_top_1'].astype(np.int32).values\n",
    "te_imgt1 = df_test['image_top_1'].astype(np.int32).values\n",
    "tr_imgt1 = np.expand_dims(tr_imgt1, axis=-1)\n",
    "te_imgt1 = np.expand_dims(te_imgt1, axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsilon = 1e-8\n",
    "tr_price = np.log(df_x_train['price'] + epsilon)\n",
    "te_price = np.log(df_test['price'] + epsilon)\n",
    "tr_price[tr_price.isna()] = -1.\n",
    "te_price[te_price.isna()] = -1.\n",
    "\n",
    "tr_price = np.expand_dims(tr_price, axis=-1)\n",
    "te_price = np.expand_dims(te_price, axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_itemseq = np.log(df_x_train['item_seq_number'])\n",
    "te_itemseq = np.log(df_test['item_seq_number'])\n",
    "#tr_itemseq = df_x_train['item_seq_number']\n",
    "#te_itemseq = df_test['item_seq_number']\n",
    "# price_tr[price_tr.isna()] = -1.\n",
    "# price_te[price_te.isna()] = -1.\n",
    "\n",
    "tr_itemseq = np.expand_dims(tr_itemseq, axis=-1)\n",
    "te_itemseq = np.expand_dims(te_itemseq, axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_avg_days_up_user  = df_x_train['avg_days_up_user']\n",
    "tr_avg_times_up_user = df_x_train['avg_times_up_user']\n",
    "tr_n_user_items      = df_x_train['n_user_items']\n",
    "\n",
    "te_avg_days_up_user  = df_test['avg_days_up_user']\n",
    "te_avg_times_up_user = df_test['avg_times_up_user']\n",
    "te_n_user_items      = df_test['n_user_items']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from timeit import timeit\n",
    "from functools import lru_cache\n",
    "gc.collect()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "config.maxlen       = 256\n",
    "config.maxlen_title = 16\n",
    "config.emb_title = config.emb_desc = 300\n",
    "_emb_nwords = 50000 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "tknzr = Tokenizer(num_words=_emb_nwords if finetune_pretrained else None, lower=True)\n",
    "tknzr.fit_on_texts(pd.concat([\n",
    "    df_x_train['description'], \n",
    "    df_x_train['title'], \n",
    "    df_x_train['param_1'],\n",
    "    df_x_train['param_2'],\n",
    "    df_x_train['param_3'],\n",
    "    df_test['description'], \n",
    "    df_test['title'], \n",
    "    df_test['param_1'],\n",
    "    df_test['param_2'],\n",
    "    df_test['param_3'],\n",
    "]).values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50000 899487\n",
      "[('гладко', 49901), ('zuiko', 49902), ('проектами', 49903), ('коста', 49904), ('укреплены', 49905), ('обменяюсь', 49906), ('фару', 49907), ('чемоданов', 49908), ('6x8', 49909), ('53215', 49910), ('английских', 49911), ('240см', 49912), ('низкотемпературная', 49913), ('корректирует', 49914), ('3u', 49915), ('celeron®', 49916), ('офд', 49917), ('откосами', 49918), ('територии', 49919), ('domani', 49920), ('ninebot', 49921), ('джой', 49922), ('81а', 49923), ('бойфренд', 49924), ('→→', 49925), ('torx', 49926), ('мотопомпы', 49927), ('xc60', 49928), ('монголия', 49929), ('делюкс', 49930), ('кирпичными', 49931), ('days', 49932), ('обзоров', 49933), ('visconte', 49934), ('кармен', 49935), ('ттр', 49936), ('откатались', 49937), ('рисунке', 49938), ('эвкалипт', 49939), ('гигиеническая', 49940), ('885', 49941), ('чёткий', 49942), ('простоквашино', 49943), ('инвентарем', 49944), ('платят', 49945), ('карликовых', 49946), ('включающей', 49947), ('1ый', 49948), ('авиационный', 49949), ('1931', 49950), ('убер', 49951), ('бе', 49952), ('200w', 49953), ('апарт', 49954), ('кедровые', 49955), ('видик', 49956), ('активирует', 49957), ('12900', 49958), ('ne', 49959), ('камера\\xa0', 49960), ('фисташка', 49961), ('симками', 49962), ('трекинговые', 49963), ('многопозиционная', 49964), ('кефир', 49965), ('geobi', 49966), ('ветрины', 49967), ('циплята', 49968), ('mn', 49969), ('миостимуляция', 49970), ('очевидно', 49971), ('1p', 49972), ('2048мб', 49973), ('прятки', 49974), ('лиловая', 49975), ('booster', 49976), ('переводов', 49977), ('гребная', 49978), ('стыки', 49979), ('промерзания', 49980), ('m4a', 49981), ('зерновые', 49982), ('нестандартного', 49983), ('ростовые', 49984), ('651', 49985), ('bruno', 49986), ('121а', 49987), ('epiphone', 49988), ('щепы', 49989), ('детектива', 49990), ('integra', 49991), ('пеппы', 49992), ('bruder', 49993), ('250кв', 49994), ('rezzo', 49995), ('cla', 49996), ('abx', 49997), ('бесп', 49998), ('‐', 49999), ('кругликовской', 50000), ('blaze', 50001), ('крайнем', 50002), ('матрешки', 50003), ('павлово', 50004), ('оптике', 50005), ('крутого', 50006), ('карбонат', 50007), ('метионин', 50008), ('xхххххх', 50009), ('вытерты', 50010), ('ортеза', 50011), ('messi', 50012), ('2012года', 50013), ('алита', 50014), ('ports', 50015), ('клубных', 50016), ('пониженным', 50017), ('шифоновые', 50018), ('надстройка', 50019), ('отрезок', 50020), ('488', 50021), ('timer', 50022), ('kambrook', 50023), ('герда', 50024), ('mtouch', 50025), ('stalker', 50026), ('рока', 50027), ('чуни', 50028), ('сандалили', 50029), ('секущиеся', 50030), ('молотка', 50031), ('iphone4s', 50032), ('погребе', 50033), ('➜➜➜➜➜➜➜➜➜➜➜➜➜➜➜➜➜➜➜➜➜➜➜➜➜➜➜➜➜➜', 50034), ('игрушечные', 50035), ('запрета', 50036), ('скорпион', 50037), ('teac', 50038), ('геймпадов', 50039), ('мотков', 50040), ('врезных', 50041), ('временную', 50042), ('перекладинами', 50043), ('трицикл', 50044), ('пелёнок', 50045), ('тов', 50046), ('amerika', 50047), ('винилового', 50048), ('daniel', 50049), ('обаятельный', 50050), ('199р', 50051), ('дубае', 50052), ('60гр', 50053), ('i8262', 50054), ('s4mini', 50055), ('чёткие', 50056), ('аккорда', 50057), ('твист', 50058), ('брусника', 50059), ('видовой', 50060), ('включаеться', 50061), ('полировку', 50062), ('водостойкой', 50063), ('бифлекс', 50064), ('поилку', 50065), ('снилс', 50066), ('990\\xa0р', 50067), ('киндеров', 50068), ('\\xa040', 50069), ('miele', 50070), ('аналогичную', 50071), ('алкогольной', 50072), ('«золотая', 50073), ('уверенную', 50074), ('пятницы', 50075), ('пружине', 50076), ('«танцующего', 50077), ('сгибание', 50078), ('скелета', 50079), ('дистрибьютор', 50080), ('любопытных', 50081), ('гуччи', 50082), ('p5k', 50083), ('geneva', 50084), ('salvatore', 50085), ('натуру', 50086), ('куомы', 50087), ('скидываю', 50088), ('вложил', 50089), ('смычок', 50090), ('островные', 50091), ('клодин', 50092), ('амигуруми', 50093), ('ветровке', 50094), ('ремонтировались', 50095), ('0001', 50096), ('жидком', 50097), ('малоэтажном', 50098), ('книжных', 50099), ('отказались', 50100)]\n",
      "[[4802, 5064, 4999, 5000, 49999, 49998]]\n"
     ]
    }
   ],
   "source": [
    "emb_nwords = _emb_nwords if finetune_pretrained else len(tknzr.word_index)\n",
    "\n",
    "print(emb_nwords, len(tknzr.word_index))\n",
    "print([(k,v) for k,v in tknzr.word_index.items()][49900:50100])\n",
    "print(tknzr.texts_to_sequences([\"результат голубые складная оф кругликовской отказались ‐ бесп\"]))\n",
    "    \n",
    "if use_pretrained:\n",
    "    lang_model = load_model('cc.ru.300.bin')\n",
    "    embedding_matrix = np.zeros((emb_nwords+1, config.emb_desc), dtype=np.float32)\n",
    "    for word, i in list(tknzr.word_index.items())[:emb_nwords]:\n",
    "        embedding_vector = lang_model.get_word_vector(word)[:config.emb_desc]\n",
    "        if embedding_vector is not None:\n",
    "            # words not found in embedding index will be all-zeros.\n",
    "            embedding_matrix[i] = embedding_vector\n",
    "else:\n",
    "    embedding_matrix = None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-4.80882704e-01, -1.42119184e-01, -3.44307423e-01, -2.29391977e-02,\n",
       "        3.94547433e-01,  2.91411161e-01, -1.05855726e-01, -2.56554455e-01,\n",
       "       -2.93404073e-01, -5.48014678e-02, -3.74294311e-01, -1.40260443e-01,\n",
       "       -3.49020988e-01,  1.53446227e-01,  3.15531343e-02, -1.76761448e-01,\n",
       "        2.45778617e-02, -8.59476328e-02,  5.15500128e-01,  1.59831643e-02,\n",
       "       -2.32105345e-01,  2.04156250e-01, -1.15163393e-01,  1.41458377e-01,\n",
       "        1.22757323e-01, -2.37270236e-01,  2.53257435e-02, -8.47037286e-02,\n",
       "        2.45075807e-01,  2.92588472e-01, -2.42517248e-01,  1.08641326e-01,\n",
       "       -2.76228905e-01,  1.40370592e-01, -2.14673370e-01, -1.37534589e-01,\n",
       "        1.17154136e-01, -1.09972441e+00, -2.06661969e-01,  2.07523584e-01,\n",
       "        3.04954827e-01,  1.33845927e-02, -3.03797632e-01, -1.30874828e-01,\n",
       "        1.04621150e-01, -2.00393721e-01,  2.90637493e-01,  1.50292009e-01,\n",
       "       -2.84837484e-01,  2.51135558e-01,  3.86558652e-01,  6.36458695e-02,\n",
       "       -6.79564834e-01, -2.46899277e-02,  4.32357043e-02,  1.25492245e-01,\n",
       "       -5.17454982e-01,  2.84410149e-01, -1.21484661e+00,  3.14905554e-01,\n",
       "        8.30953792e-02,  1.62963837e-01, -1.80058286e-01, -6.79146051e-02,\n",
       "       -1.98700484e-02, -8.16161036e-02,  2.60349005e-01, -1.37096047e-01,\n",
       "        2.12698504e-01,  3.06295961e-01, -2.99450010e-01, -1.51876837e-01,\n",
       "       -5.03149092e-01, -1.51355207e-01,  4.04113978e-01, -2.70900950e-02,\n",
       "        1.15344808e-01,  5.04287034e-02, -4.56976518e-02, -1.70731276e-01,\n",
       "        5.94410114e-02, -1.22172581e-02,  1.09216630e-01,  1.51065394e-01,\n",
       "       -1.78426534e-01, -1.96715027e-01,  1.61974087e-01,  1.09501667e-01,\n",
       "       -1.35233849e-01, -4.07103688e-01,  2.20946416e-01, -1.68736458e-01,\n",
       "       -1.75750494e-01, -4.87068184e-02,  6.15603387e-01,  2.84168780e-01,\n",
       "        2.85909057e-01, -1.93571284e-01,  1.60793260e-01,  6.57097995e-01,\n",
       "        1.25936046e-01, -4.18570563e-02,  1.96060255e-01,  3.48662257e-01,\n",
       "       -3.03844154e-01, -6.41398737e-03, -3.33491713e-01,  3.45885664e-01,\n",
       "       -1.03600487e-01,  6.14721440e-02, -1.50552914e-01,  2.18793470e-02,\n",
       "       -2.14868695e-01, -9.38045979e-02, -4.65483069e-01, -7.35772355e-03,\n",
       "        2.75561690e-01,  1.00772426e-01,  5.65521680e-02, -1.95699766e-01,\n",
       "       -1.36684865e-01, -1.53542444e-01, -1.56725645e-01, -3.70550334e-01,\n",
       "       -4.10821028e-02,  1.29446954e-01, -1.99355125e-01,  9.36101899e-02,\n",
       "        1.94097459e-01, -1.02912404e-01,  1.87189296e-01,  2.47389879e-02,\n",
       "        3.55288714e-01,  1.78495646e-01, -1.79357544e-01, -5.59211254e-01,\n",
       "       -1.93152234e-01,  1.83898672e-01,  4.16407622e-02, -5.08280098e-02,\n",
       "       -4.03507560e-01, -1.13516487e-01, -4.82079804e-01, -4.37219292e-01,\n",
       "        1.96683258e-01, -2.90510077e-02,  2.56152868e-01, -9.07418206e-02,\n",
       "       -3.61095816e-02, -3.94969523e-01, -2.28065684e-01,  2.05522746e-01,\n",
       "       -1.37204215e-01, -3.16318750e-01, -2.49830827e-01, -1.15862815e-03,\n",
       "       -1.51522905e-01, -2.08981737e-01, -3.23431492e-01, -2.66959310e-01,\n",
       "        1.67513326e-01, -1.02680616e-01,  1.97482914e-01,  5.80347003e-03,\n",
       "        7.23480247e-03, -3.28858316e-01, -4.53501969e-04, -8.54864657e-01,\n",
       "       -2.30186611e-01,  6.06648445e-01,  2.47115076e-01,  1.23734213e-01,\n",
       "        1.27712876e-01, -3.04491132e-01, -5.08699834e-01, -2.74021383e-02,\n",
       "       -2.84293108e-02, -1.13415733e-01, -4.48718637e-01, -2.16708571e-01,\n",
       "       -3.39338571e-01, -1.15869544e-01,  3.91918510e-01, -2.75149882e-01,\n",
       "        8.50917697e-02, -1.09456584e-01,  3.45532328e-01,  2.76513100e-01,\n",
       "        5.07347845e-02,  2.25713700e-01, -3.76619875e-01, -3.79123807e-01,\n",
       "       -4.81199324e-02, -1.50218591e-01, -1.35209814e-01,  3.04707885e-01,\n",
       "       -2.43493423e-01, -2.85503507e-01,  1.91373572e-01, -3.28917876e-02,\n",
       "       -7.20361948e-01, -7.18348846e-03, -9.99226347e-02, -5.95013909e-02,\n",
       "        1.27429336e-01, -1.07993819e-01, -2.48690307e-01, -3.13828200e-01,\n",
       "        1.70907930e-01,  3.71027470e-01, -3.16249907e-01,  6.21878207e-02,\n",
       "        1.65690377e-01, -1.54322430e-01, -9.95288044e-02, -4.83549863e-01,\n",
       "       -4.49038297e-02, -3.82458419e-02, -4.18518364e-01, -5.21467067e-02,\n",
       "        8.04057121e-02, -1.95352405e-01,  4.74473163e-02, -2.11530566e-01,\n",
       "       -1.05102345e-01, -1.08444631e-01,  4.74796817e-02, -2.11374879e-01,\n",
       "        1.82441026e-01,  3.66060734e-01, -4.56872970e-01, -9.94112551e-01,\n",
       "        9.08549801e-02, -3.23930204e-01, -4.29938406e-01,  4.43126738e-01,\n",
       "        1.25188679e-01, -5.96499562e-01,  1.86027065e-01, -3.19248587e-02,\n",
       "       -8.65885988e-02, -4.66652274e-01, -4.99920873e-03,  2.75056243e-01,\n",
       "       -1.46447375e-01,  3.26858848e-01,  3.20962489e-01,  5.38668893e-02,\n",
       "       -1.93149820e-01, -7.57648423e-02, -3.77930671e-01,  2.18650669e-01,\n",
       "       -1.28315702e-01,  7.18608379e-01, -5.35113439e-02,  4.38364804e-01,\n",
       "        1.83942184e-01,  3.18120599e-01, -2.23343194e-01, -3.28848988e-01,\n",
       "       -1.61035329e-01, -1.73147395e-01, -1.98621616e-01,  1.85864985e-01,\n",
       "       -2.06833094e-01,  2.80693024e-01, -4.22798634e-01,  2.99967472e-02,\n",
       "        5.84998056e-02, -4.42556262e-01,  5.46675980e-01,  2.15664521e-01,\n",
       "        1.40996540e+00, -2.45899819e-02, -5.06662607e-01, -4.52166885e-01,\n",
       "        7.46845677e-02, -5.06224595e-02, -1.07427508e-01,  2.11628191e-02,\n",
       "        2.36059457e-01,  2.13688076e-01,  1.93670109e-01, -2.73271322e-01,\n",
       "        1.35101631e-01, -2.95318395e-01, -2.95054376e-01,  2.04721466e-01,\n",
       "       -2.89409429e-01, -1.88402951e-01,  1.46720201e-01,  5.35665546e-03,\n",
       "       -3.05544913e-01, -7.98139930e-01, -4.52574641e-02,  1.73469007e-01,\n",
       "        6.39799908e-02, -1.19313955e-01,  5.88333495e-02,  1.80552781e-01],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " embedding_matrix[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_desc_seq = tknzr.texts_to_sequences(df_x_train['description'].values)\n",
    "te_desc_seq = tknzr.texts_to_sequences(df_test['description'].values)\n",
    "\n",
    "tr_title_seq = tknzr.texts_to_sequences(df_x_train['title'].values)\n",
    "te_title_seq = tknzr.texts_to_sequences(df_test['title'].values)\n",
    "\n",
    "tr_desc_pad = pad_sequences(tr_desc_seq, maxlen=config.maxlen)\n",
    "te_desc_pad = pad_sequences(te_desc_seq, maxlen=config.maxlen)\n",
    "\n",
    "tr_title_pad = pad_sequences(tr_title_seq, maxlen=config.maxlen_title)\n",
    "te_title_pad = pad_sequences(te_title_seq, maxlen=config.maxlen_title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "## categorical\n",
    "config.len_reg   = len(tknzr_reg)\n",
    "config.len_pcn   = len(tknzr_pcn)\n",
    "config.len_cn    = len(tknzr_cn) \n",
    "config.len_ut    = len(tknzr_ut)\n",
    "config.len_city  = len(tknzr_city)\n",
    "config.len_week  = 7\n",
    "config.len_imgt1 = int(df_x_train['image_top_1'].max())+1\n",
    "config.len_p1    = len(tknzr_p1)\n",
    "config.len_p2    = len(tknzr_p2)\n",
    "config.len_p3    = len(tknzr_p3)\n",
    "\n",
    "## continuous\n",
    "config.len_price   = 1\n",
    "config.len_itemseq = 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "## categorical\n",
    "max_emb = 64\n",
    "config.emb_reg   = min(max_emb,(config.len_reg   + 1)//2)\n",
    "config.emb_pcn   = min(max_emb,(config.len_pcn   + 1)//2)\n",
    "config.emb_cn    = min(max_emb,(config.len_cn    + 1)//2)\n",
    "config.emb_ut    = min(max_emb,(config.len_ut    + 1)//2)\n",
    "config.emb_city  = min(max_emb,(config.len_city  + 1)//2)\n",
    "config.emb_week  = min(max_emb,(config.len_week  + 1)//2)\n",
    "config.emb_imgt1 = min(max_emb,(config.len_imgt1 + 1)//2)\n",
    "config.emb_p1    = min(max_emb,(config.len_p1    + 1)//2)\n",
    "config.emb_p2    = min(max_emb,(config.len_p2    + 1)//2)\n",
    "config.emb_p3    = min(max_emb,(config.len_p3    + 1)//2)\n",
    "\n",
    "#continuous\n",
    "config.emb_price   = 16\n",
    "config.emb_itemseq = 16\n",
    "\n",
    "#text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_idx = list(df_y_train.sample(frac=0.2, random_state=1991).index)\n",
    "train_idx = list(df_y_train[np.invert(df_y_train.index.isin(valid_idx))].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1503424,)\n",
      "(1503424,)\n",
      "(1503424,)\n",
      "(1503424,)\n",
      "(1503424,)\n",
      "(1503424, 1)\n",
      "(1503424, 1)\n",
      "(1503424,)\n",
      "(1503424,)\n",
      "(1503424,)\n",
      "(1503424, 1)\n",
      "(1503424, 1)\n",
      "[0.6931471805599453 2.9444389791664403 2.1972245773362196 ...\n",
      " 3.912023005428146 3.4965075614664802 2.8903717578961645]\n"
     ]
    }
   ],
   "source": [
    "[print(k.shape) for k in [tr_reg, tr_pcn, tr_cn, tr_ut, tr_city, tr_week, tr_imgt1, tr_p1, tr_p2, tr_p3, tr_price, tr_itemseq]]\n",
    "\n",
    "X      = np.array([tr_reg, tr_pcn, tr_cn, tr_ut.squeeze(), tr_city, tr_week.squeeze(), tr_imgt1.squeeze(), \n",
    "                   tr_p1, tr_p2, tr_p3, tr_price.squeeze(), tr_itemseq.squeeze(), \n",
    "                   tr_avg_days_up_user, tr_avg_times_up_user, tr_n_user_items,\n",
    "                   df_x_train['image'].values])\n",
    "\n",
    "X_test = np.array([te_reg, te_pcn, te_cn, te_ut.squeeze(), te_city, te_week.squeeze(), te_imgt1.squeeze(), \n",
    "                   te_p1, te_p2, te_p3, te_price.squeeze(), te_itemseq.squeeze(), \n",
    "                   te_avg_days_up_user, te_avg_times_up_user, te_n_user_items,\n",
    "                   df_test['image'].values])\n",
    "\n",
    "Y = df_y_train\n",
    "print(X[11])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  1,  2, ..., 10,  6, 11])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tr_reg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Input, Embedding, Dense, BatchNormalization, Activation, Dropout\n",
    "from keras.layers import GlobalMaxPool1D, GlobalMaxPool2D\n",
    "from keras.layers import concatenate, Flatten\n",
    "from keras.layers import LSTM, CuDNNGRU, CuDNNLSTM, GRU, Bidirectional\n",
    "from keras.models import Model\n",
    "from keras.utils import multi_gpu_model\n",
    "from keras.applications import *\n",
    "import jpeg4py as jpeg\n",
    "import cv2\n",
    "from keras import backend as K\n",
    "\n",
    "from keras.optimizers import RMSprop, Adam, SGD\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\n",
    "\n",
    "from keras_contrib.layers.normalization import InstanceNormalization\n",
    "\n",
    "### rmse loss for keras\n",
    "def root_mean_squared_error(y_true, y_pred):\n",
    "    return K.sqrt(K.mean(K.square(y_pred - y_true))) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "if use_images:\n",
    "    CROP_SIZE = 224\n",
    "    image_feature_extractor = 'ResNet50'\n",
    "    freeze_until = None# 'res5b_branch2a'\n",
    "    \n",
    "    classifier = globals()[image_feature_extractor]\n",
    "\n",
    "    kwargs = {\n",
    "        'include_top' : False,\n",
    "        'weights'     : 'imagenet',\n",
    "        'input_shape' : (CROP_SIZE, CROP_SIZE, 3), \n",
    "        'pooling'     : 'avg',\n",
    "     }\n",
    "\n",
    "    classifier_model = classifier(**kwargs)\n",
    "    \n",
    "    if freeze_until is not None:\n",
    "        trainable = False\n",
    "        n_trainable = 0\n",
    "\n",
    "        for i, layer in enumerate(classifier_model.layers):\n",
    "            if layer.name == freeze_until:\n",
    "                trainable = True\n",
    "            if trainable:\n",
    "                n_trainable += 1\n",
    "            layer.trainable = trainable\n",
    "\n",
    "    classifier_model.summary()\n",
    "\n",
    "    def preprocess_image(img):\n",
    "    \n",
    "        # find `preprocess_input` function specific to the classifier\n",
    "        classifier_to_module = { \n",
    "            'NASNetLarge'       : 'nasnet',\n",
    "            'NASNetMobile'      : 'nasnet',\n",
    "            'DenseNet121'       : 'densenet',\n",
    "            'DenseNet161'       : 'densenet',\n",
    "            'DenseNet201'       : 'densenet',\n",
    "            'InceptionResNetV2' : 'inception_resnet_v2',\n",
    "            'InceptionV3'       : 'inception_v3',\n",
    "            'MobileNet'         : 'mobilenet',\n",
    "            'ResNet50'          : 'resnet50',\n",
    "            'VGG16'             : 'vgg16',\n",
    "            'VGG19'             : 'vgg19',\n",
    "            'Xception'          : 'xception',\n",
    "\n",
    "            'VGG16Places365'        : 'vgg16_places365',\n",
    "            'VGG16PlacesHybrid1365' : 'vgg16_places_hybrid1365',\n",
    "\n",
    "            'SEDenseNetImageNet121' : 'se_densenet',\n",
    "            'SEDenseNetImageNet161' : 'se_densenet',\n",
    "            'SEDenseNetImageNet169' : 'se_densenet',\n",
    "            'SEDenseNetImageNet264' : 'se_densenet',\n",
    "            'SEInceptionResNetV2'   : 'se_inception_resnet_v2',\n",
    "            'SEMobileNet'           : 'se_mobilenets',\n",
    "            'SEResNet50'            : 'se_resnet',\n",
    "            'SEResNet101'           : 'se_resnet',\n",
    "            'SEResNet154'           : 'se_resnet',\n",
    "            'SEInceptionV3'         : 'se_inception_v3',\n",
    "            'SEResNext'             : 'se_resnet',\n",
    "            'SEResNextImageNet'     : 'se_resnet',\n",
    "\n",
    "            'ResNet152'             : 'resnet152',\n",
    "            'AResNet50'             : 'aresnet50',\n",
    "            'AXception'             : 'axception',\n",
    "            'AInceptionV3'          : 'ainceptionv3',\n",
    "        }\n",
    "\n",
    "        if image_feature_extractor in classifier_to_module:\n",
    "            classifier_module_name = classifier_to_module[image_feature_extractor]\n",
    "        else:\n",
    "            classifier_module_name = 'xception'\n",
    "\n",
    "        preprocess_input_function = getattr(globals()[classifier_module_name], 'preprocess_input')\n",
    "        return preprocess_input_function(img.astype(np.float32))\n",
    "    \n",
    " \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "config.batch_size = 64 if use_images else 1024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model():\n",
    "    do = 0\n",
    "    bn = True\n",
    "    \n",
    "    #K.clear_session()\n",
    "    inp_reg = Input(shape=(1, ), name='inp_region')\n",
    "    emb_reg = Embedding(config.len_reg, config.emb_reg, name='emb_region')(inp_reg)\n",
    "    \n",
    "    inp_pcn = Input(shape=(1, ), name='inp_parent_category_name')\n",
    "    emb_pcn = Embedding(config.len_pcn, config.emb_pcn, name='emb_parent_category_name')(inp_pcn)\n",
    "\n",
    "    inp_cn = Input(shape=(1, ), name='inp_category_name')\n",
    "    emb_cn = Embedding(config.len_cn, config.emb_cn, name=\"emb_category_name\" )(inp_cn)\n",
    "    \n",
    "    inp_ut = Input(shape=(1, ), name='inp_user_type')\n",
    "    emb_ut = Embedding(config.len_ut, config.emb_ut, name='emb_user_type' )(inp_ut)\n",
    "    \n",
    "    inp_city = Input(shape=(1, ), name='inp_city')\n",
    "    emb_city = Embedding(config.len_city, config.emb_city, name='emb_city' )(inp_city)\n",
    "\n",
    "    inp_week = Input(shape=(1, ), name='inp_week')\n",
    "    emb_week = Embedding(config.len_week, config.emb_week, name='emb_week' )(inp_week)\n",
    "\n",
    "    inp_imgt1 = Input(shape=(1, ), name='inp_imgt1')\n",
    "    emb_imgt1 = Embedding(config.len_imgt1, config.emb_imgt1, name='emb_imgt1')(inp_imgt1)\n",
    "    \n",
    "    inp_p1 = Input(shape=(1, ), name='inp_p1')\n",
    "    emb_p1 = Embedding(config.len_p1, config.emb_p1, name='emb_p1')(inp_p1)\n",
    "    \n",
    "    inp_p2 = Input(shape=(1, ), name='inp_p2')\n",
    "    emb_p2 = Embedding(config.len_p2, config.emb_p2, name='emb_p2')(inp_p2)\n",
    "    \n",
    "    inp_p3 = Input(shape=(1, ), name='inp_p3')\n",
    "    emb_p3 = Embedding(config.len_p3, config.emb_p3, name='emb_p3')(inp_p3)\n",
    "    \n",
    "\n",
    "    conc_cate = concatenate([emb_reg, emb_pcn,  emb_cn, emb_ut, emb_city, emb_week, emb_imgt1, \n",
    "                             emb_p1, emb_p2, emb_p3], \n",
    "                            axis=-1, name='concat_categorical_vars')\n",
    "    \n",
    "    conc_cate = Flatten()(conc_cate)\n",
    "    \n",
    "    inp_price = Input(shape=(1, ), name='inp_price')\n",
    "    emb_price = inp_price#InstanceNormalization()(inp_price)\n",
    "    #emb_price = Dense(config.emb_price, activation='tanh', name='emb_price')(inp_price)\n",
    "\n",
    "    inp_itemseq = Input(shape=(1, ), name='inp_itemseq')\n",
    "    emb_itemseq = inp_itemseq# InstanceNormalization()(inp_itemseq)\n",
    "    emb_itemseq = Dense(config.emb_itemseq, activation='tanh', name='emb_itemseq')(emb_itemseq)\n",
    "    \n",
    "    #tr_avg_days_up_user, tr_avg_times_up_user, tr_n_user_items,\n",
    "    inp_avg_days_up_user = Input(shape=(1, ), name='inp_avg_days_up_user')\n",
    "    emb_avg_days_up_user = inp_avg_days_up_user\n",
    "    \n",
    "    inp_avg_times_up_user = Input(shape=(1, ), name='avg_times_up_user')\n",
    "    emb_avg_times_up_user = inp_avg_times_up_user\n",
    "    \n",
    "    inp_n_user_items = Input(shape=(1, ), name='n_user_items')\n",
    "    emb_n_user_items = inp_n_user_items\n",
    "    \n",
    "    inp_filesize = Input(shape=(1, ), name='inp_filesize')\n",
    "    emb_filesize = inp_filesize#InstanceNormalization()(inp_price)\n",
    "\n",
    "    conc_cont = concatenate([\n",
    "        conc_cate, emb_price, emb_itemseq, \n",
    "        emb_avg_days_up_user, emb_avg_times_up_user, emb_n_user_items, \n",
    "        emb_filesize], axis=-1)\n",
    "    \n",
    "    x = conc_cont\n",
    "    \n",
    "    x = Dense(512)(x)\n",
    "    if bn: x = BatchNormalization()(x)\n",
    "    x = Activation(activation='relu')(x)\n",
    "    \n",
    "    x = Dense(512)(x)\n",
    "    if bn: x = BatchNormalization()(x)\n",
    "    x = Activation(activation='relu')(x)\n",
    "    if do > 0.: x = Dropout(do)(x)\n",
    "        \n",
    "    x = Dense(512)(x)\n",
    "    if bn: x = BatchNormalization()(x)\n",
    "    x = Activation(activation='relu')(x)\n",
    "    if do > 0.: x = Dropout(do)(x)\n",
    "\n",
    "    ### text\n",
    "\n",
    "    embedding_text = Embedding(emb_nwords+1, config.emb_desc, \n",
    "                               weights = [embedding_matrix], \n",
    "                               trainable=True if finetune_pretrained else False,\n",
    "                               name='emb_desc')\n",
    "\n",
    "    inp_desc = Input(shape=(config.maxlen, ), name='inp_desc')\n",
    "    emb_desc = embedding_text(inp_desc)\n",
    "    \n",
    "    desc_layer = Bidirectional(CuDNNGRU(config.emb_desc, return_sequences=True))(emb_desc)\n",
    "    desc_layer = Bidirectional(CuDNNGRU(config.emb_desc, return_sequences=False))(desc_layer)\n",
    "    \n",
    "    inp_title = Input(shape=(config.maxlen_title, ), name='inp_title')\n",
    "    emb_title = embedding_text(inp_title)\n",
    "    \n",
    "    title_layer = Bidirectional(CuDNNGRU(config.emb_desc, return_sequences=True))(emb_title)\n",
    "    title_layer = Bidirectional(CuDNNGRU(config.emb_desc, return_sequences=False))(title_layer)\n",
    "\n",
    "    conc_desc = concatenate([x, desc_layer, title_layer], axis=-1)\n",
    "    \n",
    "    if use_images:\n",
    "        inp_image = Input(shape=(CROP_SIZE, CROP_SIZE, 3), name='inp_image')\n",
    "        image_features = classifier_model(inp_image)\n",
    "        conc_desc = concatenate([conc_desc, image_features], axis=-1)\n",
    "        \n",
    "    conc_desc = Dense(2048)(conc_desc)\n",
    "    if bn: conc_desc = BatchNormalization()(conc_desc)\n",
    "    conc_desc = Activation(activation='relu')(conc_desc)\n",
    "\n",
    "    conc_desc = Dense(1024)(conc_desc)\n",
    "    if bn: conc_desc = BatchNormalization()(conc_desc)\n",
    "    conc_desc = Activation(activation='relu')(conc_desc)\n",
    "    \n",
    "    conc_desc = Dense(512)(conc_desc)\n",
    "    if bn: conc_desc = BatchNormalization()(conc_desc)\n",
    "    conc_desc = Activation(activation='relu')(conc_desc)\n",
    "    if do > 0.: conc_desc = Dropout(do)(conc_desc)\n",
    "\n",
    "    conc_desc = Dense(256)(conc_desc)\n",
    "    if bn: conc_desc = BatchNormalization()(conc_desc)\n",
    "    conc_desc = Activation(activation='relu')(conc_desc)\n",
    "    if do > 0.: conc_desc = Dropout(do)(conc_desc)\n",
    "\n",
    "    outp = Dense(1, activation='sigmoid', name='output')(conc_desc)\n",
    "\n",
    "    inputs = [inp_reg, inp_pcn, inp_cn, inp_ut, inp_city, inp_week, inp_imgt1, inp_p1, inp_p2, inp_p3,\n",
    "              inp_price, inp_itemseq, inp_desc, inp_title, \n",
    "              inp_avg_days_up_user, inp_avg_times_up_user, inp_n_user_items,\n",
    "              inp_filesize]\n",
    "    \n",
    "    if use_images:\n",
    "        inputs.append(inp_image)\n",
    "            \n",
    "    print(inputs)\n",
    "    model = Model(inputs = inputs, outputs = outp)\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen(idx, valid=False):\n",
    "    \n",
    "    if use_images:\n",
    "        load_img_fast_jpg  = lambda img_path: jpeg.JPEG(img_path).decode()\n",
    "        xi = np.empty((config.batch_size, CROP_SIZE, CROP_SIZE, 3), dtype=np.float32)\n",
    "\n",
    "    x = np.empty((config.batch_size, X.shape[0]-1 + 1), dtype=np.float32)\n",
    "    fname_idx = X.shape[0]-1\n",
    "    y = np.empty((config.batch_size, ), dtype=np.float32)\n",
    "    \n",
    "    print(x.shape, y.shape)\n",
    "    \n",
    "    xd = np.empty((config.batch_size, config.maxlen      ), dtype=np.float32)\n",
    "    xt = np.empty((config.batch_size, config.maxlen_title), dtype=np.float32)\n",
    "    \n",
    "    batch = 0\n",
    "    i = 0\n",
    "    while True:\n",
    "        \n",
    "        if i == len(idx):\n",
    "            i = 0\n",
    "            if not valid:\n",
    "                random.shuffle(idx)\n",
    "            \n",
    "        x[batch,:fname_idx] = X[:fname_idx,idx[i]]\n",
    "        y[batch,...] = Y[idx[i]]\n",
    "                \n",
    "        n_vect = tr_desc_pad[idx[i]].shape[0]\n",
    "        i_vect = config.maxlen - n_vect\n",
    "        xd[batch, i_vect:, ...] = tr_desc_pad[idx[i]]\n",
    "        xd[batch, :i_vect, ...] = 0\n",
    "\n",
    "        n_vect = min(tr_title_pad[idx[i]].shape[0], config.maxlen_title)\n",
    "        i_vect = config.maxlen_title - n_vect\n",
    "        \n",
    "        xt[batch, i_vect:, ...] = tr_title_pad[idx[i]][:n_vect]\n",
    "        xt[batch, :i_vect, ...] = 0\n",
    "                \n",
    "        path = f'{PATH}/data/competition_files/train_jpg/{X[fname_idx,idx[i]]}.jpg'\n",
    "\n",
    "        filesize = -1\n",
    "        try:\n",
    "            filesize = os.path.getsize(path) / 151868. # biggest file\n",
    "        except Exception:\n",
    "            pass\n",
    "        \n",
    "        x[batch,fname_idx] = 0 #filesize\n",
    "\n",
    "        if use_images:\n",
    "            xi[batch, ...] = 0.\n",
    "            try:\n",
    "                _img = load_img_fast_jpg(path)\n",
    "                _img = cv2.resize(_img, (CROP_SIZE, CROP_SIZE))\n",
    "                _img = preprocess_image(_img)\n",
    "                xi[batch, ...] = _img\n",
    "            except Exception:\n",
    "                pass\n",
    "            \n",
    "        batch += 1\n",
    "        i += 1\n",
    "        \n",
    "        if batch == config.batch_size:\n",
    "            _x = [x[:, 0], x[:, 1], x[:, 2], x[:, 3], \n",
    "                  x[:, 4], x[:, 5], x[:, 6], x[:, 7], \n",
    "                  x[:, 8], x[:, 9], x[:,10], x[:,11],\n",
    "                  xd, xt,  x[:,12], x[:,13], x[:,14],\n",
    "                  x[:,15] ]\n",
    "            if use_images:\n",
    "                _x.append(xi)\n",
    "                            \n",
    "            yield(_x, y)\n",
    "            \n",
    "            batch = 0\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<tf.Tensor 'inp_region:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'inp_parent_category_name:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'inp_category_name:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'inp_user_type:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'inp_city:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'inp_week:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'inp_imgt1:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'inp_p1:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'inp_p2:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'inp_p3:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'inp_price:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'inp_itemseq:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'inp_desc:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'inp_title:0' shape=(?, 16) dtype=float32>, <tf.Tensor 'inp_avg_days_up_user:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'avg_times_up_user:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'n_user_items:0' shape=(?, 1) dtype=float32>, <tf.Tensor 'inp_filesize:0' shape=(?, 1) dtype=float32>]\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "inp_region (InputLayer)         (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "inp_parent_category_name (Input (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "inp_category_name (InputLayer)  (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "inp_user_type (InputLayer)      (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "inp_city (InputLayer)           (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "inp_week (InputLayer)           (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "inp_imgt1 (InputLayer)          (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "inp_p1 (InputLayer)             (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "inp_p2 (InputLayer)             (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "inp_p3 (InputLayer)             (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "emb_region (Embedding)          (None, 1, 14)        392         inp_region[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "emb_parent_category_name (Embed (None, 1, 5)         45          inp_parent_category_name[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "emb_category_name (Embedding)   (None, 1, 24)        1128        inp_category_name[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "emb_user_type (Embedding)       (None, 1, 2)         6           inp_user_type[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "emb_city (Embedding)            (None, 1, 64)        112128      inp_city[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "emb_week (Embedding)            (None, 1, 4)         28          inp_week[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "emb_imgt1 (Embedding)           (None, 1, 64)        196352      inp_imgt1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "emb_p1 (Embedding)              (None, 1, 64)        23808       inp_p1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "emb_p2 (Embedding)              (None, 1, 64)        17792       inp_p2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "emb_p3 (Embedding)              (None, 1, 64)        81728       inp_p3[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "concat_categorical_vars (Concat (None, 1, 369)       0           emb_region[0][0]                 \n",
      "                                                                 emb_parent_category_name[0][0]   \n",
      "                                                                 emb_category_name[0][0]          \n",
      "                                                                 emb_user_type[0][0]              \n",
      "                                                                 emb_city[0][0]                   \n",
      "                                                                 emb_week[0][0]                   \n",
      "                                                                 emb_imgt1[0][0]                  \n",
      "                                                                 emb_p1[0][0]                     \n",
      "                                                                 emb_p2[0][0]                     \n",
      "                                                                 emb_p3[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "inp_itemseq (InputLayer)        (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 369)          0           concat_categorical_vars[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "inp_price (InputLayer)          (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "emb_itemseq (Dense)             (None, 16)           32          inp_itemseq[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "inp_avg_days_up_user (InputLaye (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "avg_times_up_user (InputLayer)  (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "n_user_items (InputLayer)       (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "inp_filesize (InputLayer)       (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 390)          0           flatten_1[0][0]                  \n",
      "                                                                 inp_price[0][0]                  \n",
      "                                                                 emb_itemseq[0][0]                \n",
      "                                                                 inp_avg_days_up_user[0][0]       \n",
      "                                                                 avg_times_up_user[0][0]          \n",
      "                                                                 n_user_items[0][0]               \n",
      "                                                                 inp_filesize[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 512)          200192      concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 512)          2048        dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 512)          0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 512)          262656      activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 512)          2048        dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 512)          0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "inp_desc (InputLayer)           (None, 256)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "inp_title (InputLayer)          (None, 16)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 512)          262656      activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "emb_desc (Embedding)            multiple             15000300    inp_desc[0][0]                   \n",
      "                                                                 inp_title[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 512)          2048        dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 256, 600)     1083600     emb_desc[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_3 (Bidirectional) (None, 16, 600)      1083600     emb_desc[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 512)          0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional) (None, 600)          1623600     bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_4 (Bidirectional) (None, 600)          1623600     bidirectional_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 1712)         0           activation_3[0][0]               \n",
      "                                                                 bidirectional_2[0][0]            \n",
      "                                                                 bidirectional_4[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 2048)         3508224     concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 2048)         8192        dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 2048)         0           batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 1024)         2098176     activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 1024)         4096        dense_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 1024)         0           batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 512)          524800      activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNor (None, 512)          2048        dense_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, 512)          0           batch_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 256)          131328      activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNor (None, 256)          1024        dense_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, 256)          0           batch_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "output (Dense)                  (None, 1)            257         activation_7[0][0]               \n",
      "==================================================================================================\n",
      "Total params: 27,857,932\n",
      "Trainable params: 27,847,180\n",
      "Non-trainable params: 10,752\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = get_model()\n",
    "model.summary()\n",
    "model = multi_gpu_model(model, gpus=2)\n",
    "# model.compile(optimizer=RMSprop(lr=0.0005, decay=0.00001), loss = root_mean_squared_error, metrics=['mse', root_mean_squared_error])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "### callbacks\n",
    "checkpoint = ModelCheckpoint(\n",
    "    f'best-use_pretrained{use_pretrained}-use_images{use_images}-finetune_pretrained{finetune_pretrained}.hdf5', \n",
    "    monitor='val_loss', verbose=1, save_best_only=True)\n",
    "early = EarlyStopping(patience=10, mode='min')\n",
    "reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=2, min_lr=1e-7, verbose=1, mode='min')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer=Adam(lr=1e-2, amsgrad=True) if use_images else RMSprop(lr=1e-2), \n",
    "              loss = root_mean_squared_error, metrics=[root_mean_squared_error])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1024, 16) (1024,)\n",
      "Epoch 1/100\n",
      "1173/1174 [============================>.] - ETA: 0s - loss: 0.2307 - root_mean_squared_error: 0.2307(1024, 16) (1024,)\n",
      "1174/1174 [==============================] - 650s 553ms/step - loss: 0.2307 - root_mean_squared_error: 0.2307 - val_loss: 0.2279 - val_root_mean_squared_error: 0.2279\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.22793, saving model to best-use_pretrainedTrue-use_imagesFalse-finetune_pretrainedTrue.hdf5\n",
      "Epoch 2/100\n",
      "1174/1174 [==============================] - 643s 548ms/step - loss: 0.2227 - root_mean_squared_error: 0.2227 - val_loss: 0.2263 - val_root_mean_squared_error: 0.2263\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.22793 to 0.22627, saving model to best-use_pretrainedTrue-use_imagesFalse-finetune_pretrainedTrue.hdf5\n",
      "Epoch 3/100\n",
      "1174/1174 [==============================] - 641s 546ms/step - loss: 0.2208 - root_mean_squared_error: 0.2208 - val_loss: 0.2233 - val_root_mean_squared_error: 0.2233\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.22627 to 0.22328, saving model to best-use_pretrainedTrue-use_imagesFalse-finetune_pretrainedTrue.hdf5\n",
      "Epoch 4/100\n",
      "1174/1174 [==============================] - 640s 545ms/step - loss: 0.2198 - root_mean_squared_error: 0.2198 - val_loss: 0.2245 - val_root_mean_squared_error: 0.2245\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.22328\n",
      "Epoch 5/100\n",
      "1174/1174 [==============================] - 640s 546ms/step - loss: 0.2192 - root_mean_squared_error: 0.2192 - val_loss: 0.2247 - val_root_mean_squared_error: 0.2247\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.22328\n",
      "\n",
      "Epoch 00005: ReduceLROnPlateau reducing learning rate to 0.004999999888241291.\n",
      "Epoch 6/100\n",
      "1174/1174 [==============================] - 639s 544ms/step - loss: 0.2174 - root_mean_squared_error: 0.2174 - val_loss: 0.2217 - val_root_mean_squared_error: 0.2217\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.22328 to 0.22170, saving model to best-use_pretrainedTrue-use_imagesFalse-finetune_pretrainedTrue.hdf5\n",
      "Epoch 7/100\n",
      "1174/1174 [==============================] - 641s 546ms/step - loss: 0.2166 - root_mean_squared_error: 0.2166 - val_loss: 0.2221 - val_root_mean_squared_error: 0.2221\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.22170\n",
      "Epoch 8/100\n",
      "1174/1174 [==============================] - 642s 547ms/step - loss: 0.2162 - root_mean_squared_error: 0.2162 - val_loss: 0.2218 - val_root_mean_squared_error: 0.2218\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.22170\n",
      "\n",
      "Epoch 00008: ReduceLROnPlateau reducing learning rate to 0.0024999999441206455.\n",
      "Epoch 9/100\n",
      "1174/1174 [==============================] - 640s 545ms/step - loss: 0.2148 - root_mean_squared_error: 0.2148 - val_loss: 0.2211 - val_root_mean_squared_error: 0.2211\n",
      "\n",
      "Epoch 00009: val_loss improved from 0.22170 to 0.22113, saving model to best-use_pretrainedTrue-use_imagesFalse-finetune_pretrainedTrue.hdf5\n",
      "Epoch 10/100\n",
      "1174/1174 [==============================] - 639s 544ms/step - loss: 0.2142 - root_mean_squared_error: 0.2142 - val_loss: 0.2210 - val_root_mean_squared_error: 0.2210\n",
      "\n",
      "Epoch 00010: val_loss improved from 0.22113 to 0.22101, saving model to best-use_pretrainedTrue-use_imagesFalse-finetune_pretrainedTrue.hdf5\n",
      "Epoch 11/100\n",
      "1174/1174 [==============================] - 640s 545ms/step - loss: 0.2138 - root_mean_squared_error: 0.2138 - val_loss: 0.2220 - val_root_mean_squared_error: 0.2220\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 0.22101\n",
      "Epoch 12/100\n",
      "1174/1174 [==============================] - 640s 545ms/step - loss: 0.2135 - root_mean_squared_error: 0.2135 - val_loss: 0.2216 - val_root_mean_squared_error: 0.2216\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 0.22101\n",
      "\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 0.0012499999720603228.\n",
      "Epoch 13/100\n",
      "1174/1174 [==============================] - 641s 546ms/step - loss: 0.2122 - root_mean_squared_error: 0.2122 - val_loss: 0.2215 - val_root_mean_squared_error: 0.2215\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 0.22101\n",
      "Epoch 14/100\n",
      "1174/1174 [==============================] - 639s 544ms/step - loss: 0.2116 - root_mean_squared_error: 0.2116 - val_loss: 0.2225 - val_root_mean_squared_error: 0.2225\n",
      "\n",
      "Epoch 00014: val_loss did not improve from 0.22101\n",
      "\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 0.0006249999860301614.\n",
      "Epoch 15/100\n",
      "1174/1174 [==============================] - 640s 545ms/step - loss: 0.2102 - root_mean_squared_error: 0.2102 - val_loss: 0.2234 - val_root_mean_squared_error: 0.2234\n",
      "\n",
      "Epoch 00015: val_loss did not improve from 0.22101\n",
      "Epoch 16/100\n",
      "1174/1174 [==============================] - 641s 546ms/step - loss: 0.2094 - root_mean_squared_error: 0.2094 - val_loss: 0.2234 - val_root_mean_squared_error: 0.2234\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 0.22101\n",
      "\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 0.0003124999930150807.\n",
      "Epoch 17/100\n",
      "1174/1174 [==============================] - 641s 546ms/step - loss: 0.2080 - root_mean_squared_error: 0.2080 - val_loss: 0.2242 - val_root_mean_squared_error: 0.2242\n",
      "\n",
      "Epoch 00017: val_loss did not improve from 0.22101\n",
      "Epoch 18/100\n",
      "1174/1174 [==============================] - 642s 547ms/step - loss: 0.2069 - root_mean_squared_error: 0.2069 - val_loss: 0.2245 - val_root_mean_squared_error: 0.2245\n",
      "\n",
      "Epoch 00018: val_loss did not improve from 0.22101\n",
      "\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 0.00015624999650754035.\n",
      "Epoch 19/100\n",
      "1174/1174 [==============================] - 639s 544ms/step - loss: 0.2057 - root_mean_squared_error: 0.2057 - val_loss: 0.2253 - val_root_mean_squared_error: 0.2253\n",
      "\n",
      "Epoch 00019: val_loss did not improve from 0.22101\n",
      "Epoch 20/100\n",
      "1174/1174 [==============================] - 639s 544ms/step - loss: 0.2047 - root_mean_squared_error: 0.2047 - val_loss: 0.2259 - val_root_mean_squared_error: 0.2259\n",
      "\n",
      "Epoch 00020: val_loss did not improve from 0.22101\n",
      "\n",
      "Epoch 00020: ReduceLROnPlateau reducing learning rate to 7.812499825377017e-05.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7feb991e7f28>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit_generator(\n",
    "    generator        = gen(train_idx),\n",
    "    steps_per_epoch  = len(train_idx) // config.batch_size, \n",
    "    validation_data  = gen(valid_idx, valid=True), \n",
    "    validation_steps = len(valid_idx) // config.batch_size, \n",
    "    epochs = 100, \n",
    "    callbacks=[checkpoint, early, reduce_lr], \n",
    "    verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Error when checking model input: the list of Numpy arrays that you are passing to your model is not the size the model expected. Expected to see 18 array(s), but instead got the following list of 1 arrays: [array([[4, 0, 12, ..., 24, 26, 16],\n       [0, 6, 2, ..., 0, 7, 1],\n       [6, 39, 2, ..., 5, 21, 4],\n       ...,\n       [1.5, -1.0, 1.0, ..., 1.25, -1.0, 1.3704819277108433],\n       [6, 1, 4, ..., 1...",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-54-fcfc7aeb927f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0msubm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'{PATH}/sample_submission.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0msubm\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'deal_probability'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpred\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0msubm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'submit_{}_{:.4f}.csv'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'nn_p3'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.2226\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, x, batch_size, verbose, steps)\u001b[0m\n\u001b[1;32m   1150\u001b[0m                              'argument.')\n\u001b[1;32m   1151\u001b[0m         \u001b[0;31m# Validate user data.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1152\u001b[0;31m         \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_standardize_user_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1153\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstateful\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1154\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_standardize_user_data\u001b[0;34m(self, x, y, sample_weight, class_weight, check_array_lengths, batch_size)\u001b[0m\n\u001b[1;32m    752\u001b[0m             \u001b[0mfeed_input_shapes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    753\u001b[0m             \u001b[0mcheck_batch_axis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0;31m# Don't enforce the batch size.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 754\u001b[0;31m             exception_prefix='input')\n\u001b[0m\u001b[1;32m    755\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    756\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0my\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.6/site-packages/keras/engine/training_utils.py\u001b[0m in \u001b[0;36mstandardize_input_data\u001b[0;34m(data, names, shapes, check_batch_axis, exception_prefix)\u001b[0m\n\u001b[1;32m     98\u001b[0m                 \u001b[0;34m'Expected to see '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnames\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m' array(s), '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     99\u001b[0m                 \u001b[0;34m'but instead got the following list of '\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 100\u001b[0;31m                 str(len(data)) + ' arrays: ' + str(data)[:200] + '...')\n\u001b[0m\u001b[1;32m    101\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnames\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m             raise ValueError(\n",
      "\u001b[0;31mValueError\u001b[0m: Error when checking model input: the list of Numpy arrays that you are passing to your model is not the size the model expected. Expected to see 18 array(s), but instead got the following list of 1 arrays: [array([[4, 0, 12, ..., 24, 26, 16],\n       [0, 6, 2, ..., 0, 7, 1],\n       [6, 39, 2, ..., 5, 21, 4],\n       ...,\n       [1.5, -1.0, 1.0, ..., 1.25, -1.0, 1.3704819277108433],\n       [6, 1, 4, ..., 1..."
     ]
    }
   ],
   "source": [
    "pred = model.predict(X_test)\n",
    "\n",
    "subm = pd.read_csv(f'{PATH}/sample_submission.csv')\n",
    "subm['deal_probability'] = pred\n",
    "subm.to_csv('submit_{}_{:.4f}.csv'.format('nn_p3', 0.2226), index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}