avito_LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import gc\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "import os\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from keras.layers import Dense,Input,LSTM,Bidirectional,Activation,Conv1D,CuDNNLSTM,CuDNNGRU,GRU\n",
    "from keras.layers import Dropout,Embedding,GlobalMaxPooling1D, MaxPooling1D, Add, Flatten\n",
    "from keras import backend as K\n",
    "from keras.preprocessing import text, sequence\n",
    "from keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, SpatialDropout1D\n",
    "from keras import initializers, regularizers, constraints, optimizers, layers, callbacks\n",
    "from keras.callbacks import EarlyStopping,ModelCheckpoint\n",
    "from keras.models import Model\n",
    "from keras.optimizers import Adam\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def root_mean_squared_error(y_true, y_pred):\n",
    "    return K.sqrt(K.mean(K.square(y_true - y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_dtypes = {    \n",
    "            'user_id': 'str',\n",
    "            'region': 'str',\n",
    "            'city': 'str',\n",
    "            'parent_category_name': 'str',\n",
    "            'category_name': 'str',\n",
    "            'param_1' : 'str',\n",
    "            'param_2':'str',\n",
    "            'param_3':'str',\n",
    "            'title':'str',\n",
    "            'description':'str',\n",
    "            'price':'float64',\n",
    "            'item_seq_number':'int64',\n",
    "            'activation_date':'str',\n",
    "            'user_type':'str',\n",
    "            'image':'str',\n",
    "            'image_top_1':'float64',\n",
    "            'deal_probability':'float64'\n",
    "        }\n",
    "\n",
    "test_dtypes = {    \n",
    "            'user_id': 'str',\n",
    "            'region': 'str',\n",
    "            'city': 'str',\n",
    "            'parent_category_name': 'str',\n",
    "            'category_name': 'str',\n",
    "            'param_1' : 'str',\n",
    "            'param_2':'str',\n",
    "            'param_3':'str',\n",
    "            'title':'str',\n",
    "            'description':'str',\n",
    "            'price':'float64',\n",
    "            'item_seq_number':'int64',\n",
    "            'activation_date':'str',\n",
    "            'user_type':'str',\n",
    "            'image':'str',\n",
    "            'image_top_1':'float64'\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "EMBEDDING_FILE = 'cc.ru.300.vec'\n",
    "train = pd.read_csv('train.csv', index_col = \"item_id\", dtype=train_dtypes, parse_dates = [\"activation_date\"])\n",
    "test = pd.read_csv('test.csv', index_col = \"item_id\", dtype=test_dtypes, parse_dates = [\"activation_date\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "max_features = 200000\n",
    "maxlen = 100\n",
    "embed_size = 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_idx = train.shape[0]\n",
    "y_train = train['deal_probability']\n",
    "train = train.drop('deal_probability',axis=1)\n",
    "df = pd.concat([train,test],axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Preprocess\n",
    "df['price'] = df['price'].fillna(0).astype('float32')\n",
    "df['category_name'] = df['category_name'].astype('category')\n",
    "df['parent_category_name'] = df['parent_category_name'].astype('category')\n",
    "df['region'] = df['region'].astype('category')\n",
    "df['city'] = df['city'].astype('category')\n",
    "\n",
    "df[\"Weekday\"] = df['activation_date'].dt.weekday"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "training_index = df.loc[df.activation_date<=pd.to_datetime('2017-04-07')].index\n",
    "validation_index = df.loc[df.activation_date>=pd.to_datetime('2017-04-08')].index\n",
    "df.drop([\"activation_date\",\"image\"],axis=1,inplace=True)\n",
    "\n",
    "df['text_feat'] = df.apply(lambda row: ' '.join([str(row['param_1']), str(row['param_2']), str(row['param_3'])]),axis=1) # Group Param Features\n",
    "df.drop([\"param_1\",\"param_2\",\"param_3\"],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cat_vars = [\"region\", \"city\", \"parent_category_name\", \"category_name\", \"user_type\"]\n",
    "\n",
    "lbl = LabelEncoder()\n",
    "for col in cat_vars:\n",
    "    df[col] = lbl.fit_transform(df[col].astype(str))\n",
    "df['price'] = np.log1p(df['price'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create new features\n",
    "textfeats = [\"description\",\"text_feat\", \"title\"]\n",
    "for cols in textfeats:\n",
    "    df[cols] = df[cols].astype(str) \n",
    "    df[cols] = df[cols].astype(str).fillna('blank') # FILL NA\n",
    "    df[cols] = df[cols].str.lower() # Lowercase all text, so that capitalized words dont get treated differently\n",
    "    df[cols + '_num_chars'] = df[cols].apply(len) # Count number of Characters\n",
    "    df[cols + '_num_words'] = df[cols].apply(lambda comment: len(comment.split())) # Count number of Words\n",
    "    df[cols + '_num_unique_words'] = df[cols].apply(lambda comment: len(set(w for w in comment.split())))\n",
    "    df[cols + '_words_vs_unique'] = df[cols+'_num_unique_words'] / df[cols+'_num_words'] * 100 # Count Unique Words\n",
    "    df[cols + '_num_letters'] = df[cols].apply(lambda comment: len(comment)) # Count number of Letters\n",
    "    df[cols + '_num_alphabets'] = df[cols].apply(lambda comment: (comment.count(r'[a-zA-Z]'))) # Count number of Alphabets\n",
    "    df[cols + '_num_alphanumeric'] = df[cols].apply(lambda comment: (comment.count(r'[A-Za-z0-9]'))) # Count number of AlphaNumeric\n",
    "    df[cols + '_num_digits'] = df[cols].apply(lambda comment: (comment.count('[0-9]'))) # Count number of Digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Fit transform\n",
    "df['title_description_textfeat']= (df['title']+\" \"+df['description']+\" \"+df['text_feat']).astype(str)\n",
    "print(\"Start Tokenization.....\")\n",
    "tokenizer = text.Tokenizer(num_words = max_features,lower=True)\n",
    "tokenizer.fit_on_texts(df['title_description_textfeat'])\n",
    "df['seq_title_description_textfeat']= tokenizer.texts_to_sequences(df['title_description_textfeat'].str.lower())\n",
    "del df['description'], df['title'], df['text_feat'],df['title_description_textfeat']\n",
    "gc.collect()\n",
    "print(\"End Tokenization.....\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embeddings_index = {}\n",
    "with open(EMBEDDING_FILE,encoding='utf8') as f:\n",
    "    for line in f:\n",
    "        values = line.rstrip().rsplit(' ')\n",
    "        word = values[0]\n",
    "        coefs = np.asarray(values[1:], dtype='float32')\n",
    "        embeddings_index[word] = coefs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_index = tokenizer.word_index\n",
    "#prepare embedding matrix\n",
    "num_words = min(max_features, len(word_index) + 1)\n",
    "embedding_matrix = np.zeros((num_words, embed_size))\n",
    "for word, i in word_index.items():\n",
    "    if i >= max_features:\n",
    "        continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector\n",
    "\n",
    "print('Embedding created')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seq_title_description_textfeat = Input(shape=(maxlen,), name=\"seq_title_description_textfeat\")\n",
    "region = Input(shape=[1], name=\"region\")\n",
    "city = Input(shape=[1], name=\"city\")\n",
    "category_name = Input(shape=[1], name=\"category_name\")\n",
    "parent_category_name = Input(shape=[1], name=\"parent_category_name\")\n",
    "price = Input(shape=[1], name=\"price\")\n",
    "\n",
    "emb_seq_title_description = Embedding(max_features, embed_size, weights = [embedding_matrix], trainable = False)(seq_title_description_textfeat)\n",
    "emb_region = Embedding(num_words, 10)(region)\n",
    "emb_city = Embedding(num_words, 10)(city)\n",
    "emb_category_name = Embedding(num_words, 10)(category_name)\n",
    "emb_parent_category_name = Embedding(num_words, 10)(parent_category_name)\n",
    "\n",
    "emb_seq_title_description = SpatialDropout1D(0.5)(emb_seq_title_description)\n",
    "rnn_layer1 = Bidirectional(LSTM(64, return_sequences=True,recurrent_dropout=0.1))(emb_seq_title_description)\n",
    "rnn_layer1 = Bidirectional(Conv1D(40, return_sequences=True,recurrent_dropout=0.1))(rnn_layer1)\n",
    "\n",
    "avg_pool = GlobalAveragePooling1D()(rnn_layer1)\n",
    "max_pool = GlobalMaxPooling1D()(rnn_layer1)\n",
    "rnn_layer1 = concatenate([avg_pool,max_pool])\n",
    "main_l = concatenate([\n",
    "          rnn_layer1\n",
    "        , Flatten() (emb_region)\n",
    "        , Flatten() (emb_city)\n",
    "        , Flatten() (emb_category_name)\n",
    "        , Flatten() (emb_parent_category_name)\n",
    "        , price\n",
    "    ])\n",
    "    \n",
    "main_l = Dropout(0.1)(Dense(512,activation='relu') (main_l))\n",
    "main_l = Dropout(0.1)(Dense(64,activation='relu') (main_l))\n",
    "\n",
    "#output\n",
    "output = Dense(1,activation=\"sigmoid\") (main_l)\n",
    "optimizer = Adam(clipnorm=0.8)\n",
    "#model\n",
    "model = Model([seq_title_description_textfeat, region, city, category_name, parent_category_name, price], output)\n",
    "model.compile(optimizer = optimizer,\n",
    "              loss= root_mean_squared_error,\n",
    "              metrics = [root_mean_squared_error])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filepath=\"weights_avito.best.hdf5\"\n",
    "checkpoint = ModelCheckpoint(filepath, monitor='val_root_mean_squared_error', verbose=1, save_best_only=True, mode='min')\n",
    "early = EarlyStopping(monitor=\"val_root_mean_squared_error\", mode=\"min\", patience=6)\n",
    "callbacks_list = [checkpoint, early]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_data_frame(dataset):\n",
    "    \n",
    "    df1 = pd.DataFrame()\n",
    "#     print(dataset.head())\n",
    "    df1['category_name'] = np.array(dataset[:,4])\n",
    "    df1['city'] = np.array(dataset[:,2])\n",
    "    df1['parent_category_name'] = np.array(dataset[:,3])\n",
    "    df1['price'] = np.array(dataset[:,5])\n",
    "    df1['region'] = np.array(dataset[:,1])\n",
    "    df1['seq_title_description_textfeat'] = np.array(dataset[:,22])\n",
    "    \n",
    "    return df1\n",
    "\n",
    "def get_keras_data(dataset):\n",
    "    X = {\n",
    "        'seq_title_description_textfeat': sequence.pad_sequences(dataset.seq_title_description_textfeat, maxlen=maxlen)\n",
    "        ,'region': np.array(dataset.region)\n",
    "        ,'city': np.array(dataset.city)\n",
    "        ,'category_name': np.array(dataset.category_name)\n",
    "        ,'parent_category_name': np.array(dataset.parent_category_name)\n",
    "        ,'price': np.array(dataset[[\"price\"]])\n",
    "\n",
    "    }\n",
    "    \n",
    "    print(\"Data ready for Vectorization\")\n",
    "    \n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test = df[:train_idx], df[train_idx:]\n",
    "X_tra, X_val, y_tra, y_val = train_test_split(X_train, y_train, train_size=0.95, random_state=233)\n",
    "X_train, X_val, X_test = np.array(X_tra), np.array(X_val), np.array(X_test)\n",
    "y_tra, y_val = np.array(y_tra), np.array(y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_final = get_data_frame(X_train)\n",
    "X_val_final = get_data_frame(X_val)\n",
    "X_test_final = get_data_frame(X_test)\n",
    "del X_train, X_val, X_test\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_f = get_keras_data(X_train_final)\n",
    "X_val_f = get_keras_data(X_val_final)\n",
    "X_test_f = get_keras_data(X_test_final)\n",
    "\n",
    "del X_train_final, X_val_final, X_test_final\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print('Starting model training')\n",
    "model.fit(X_train_f, y_tra, batch_size=64, epochs=3, validation_data=(X_val_f, y_val),callbacks = callbacks_list,verbose=1)\n",
    "\n",
    "model.save_weights(filepath)\n",
    "# model.load_weights(filepath)\n",
    "print('Ended model training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = model.predict(X_test_f,batch_size=1024,verbose=1)\n",
    "y_pred = y_pred.reshape(-1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.read_csv('sample_submission.csv')\n",
    "submission['deal_probability'] = y_pred\n",
    "submission.to_csv('submission_64.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}