LGB_CB_Python.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Introduction\n",
    "\n",
    "Here you'll learn to build models using Catboost, Lightgbm and NaiveBayes algorithm in Python. Given the text classification problem, you'll also learn to clean data, create bag of words matrix, tf-idf matrix. \n",
    "\n",
    "On top of what's done here, next you can create a simple voting ensemble from the predictions generated from these models here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load Libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import re\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.metrics import accuracy_score, make_scorer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load data\n",
    "train = pd.read_csv(\"train.csv\")\n",
    "test = pd.read_csv(\"test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Description</th>\n",
       "      <th>Browser_Used</th>\n",
       "      <th>Device_Used</th>\n",
       "      <th>Is_Response</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id10326</td>\n",
       "      <td>The room was kind of clean but had a VERY stro...</td>\n",
       "      <td>Edge</td>\n",
       "      <td>Mobile</td>\n",
       "      <td>not happy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id10327</td>\n",
       "      <td>I stayed at the Crown Plaza April -- - April -...</td>\n",
       "      <td>Internet Explorer</td>\n",
       "      <td>Mobile</td>\n",
       "      <td>not happy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id10328</td>\n",
       "      <td>I booked this hotel through Hotwire at the low...</td>\n",
       "      <td>Mozilla</td>\n",
       "      <td>Tablet</td>\n",
       "      <td>not happy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id10329</td>\n",
       "      <td>Stayed here with husband and sons on the way t...</td>\n",
       "      <td>InternetExplorer</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>happy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id10330</td>\n",
       "      <td>My girlfriends and I stayed here to celebrate ...</td>\n",
       "      <td>Edge</td>\n",
       "      <td>Tablet</td>\n",
       "      <td>not happy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   User_ID                                        Description  \\\n",
       "0  id10326  The room was kind of clean but had a VERY stro...   \n",
       "1  id10327  I stayed at the Crown Plaza April -- - April -...   \n",
       "2  id10328  I booked this hotel through Hotwire at the low...   \n",
       "3  id10329  Stayed here with husband and sons on the way t...   \n",
       "4  id10330  My girlfriends and I stayed here to celebrate ...   \n",
       "\n",
       "        Browser_Used Device_Used Is_Response  \n",
       "0               Edge      Mobile   not happy  \n",
       "1  Internet Explorer      Mobile   not happy  \n",
       "2            Mozilla      Tablet   not happy  \n",
       "3   InternetExplorer     Desktop       happy  \n",
       "4               Edge      Tablet   not happy  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# function to clean data\n",
    "\n",
    "stops = set(stopwords.words(\"english\"))\n",
    "def cleanData(text, lowercase = False, remove_stops = False, stemming = False):\n",
    "    txt = str(text)\n",
    "    txt = re.sub(r'[^A-Za-z0-9\\s]',r'',txt)\n",
    "    txt = re.sub(r'\\n',r' ',txt)\n",
    "    \n",
    "    if lowercase:\n",
    "        txt = \" \".join([w.lower() for w in txt.split()])\n",
    "        \n",
    "    if remove_stops:\n",
    "        txt = \" \".join([w for w in txt.split() if w not in stops])\n",
    "    \n",
    "    if stemming:\n",
    "        st = PorterStemmer()\n",
    "        txt = \" \".join([st.stem(w) for w in txt.split()])\n",
    "\n",
    "    return txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## join data\n",
    "test['Is_Response'] = np.nan\n",
    "alldata = pd.concat([train, test]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# clean description\n",
    "alldata['Description'] = alldata['Description'].map(lambda x: cleanData(x, lowercase=True, remove_stops=True, stemming=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# initialise the functions - we'll create separate models for each type.\n",
    "countvec = CountVectorizer(analyzer='word', ngram_range = (1,1), min_df=150, max_features=500)\n",
    "tfidfvec = TfidfVectorizer(analyzer='word', ngram_range = (1,1), min_df = 150, max_features=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create features\n",
    "bagofwords = countvec.fit_transform(alldata['Description'])\n",
    "tfidfdata = tfidfvec.fit_transform(alldata['Description'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# label encode categorical features in data given\n",
    "cols = ['Browser_Used','Device_Used']\n",
    "\n",
    "for x in cols:\n",
    "    lbl = LabelEncoder()\n",
    "    alldata[x] = lbl.fit_transform(alldata[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create dataframe for features\n",
    "bow_df = pd.DataFrame(bagofwords.todense())\n",
    "tfidf_df = pd.DataFrame(tfidfdata.todense())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# set column names\n",
    "bow_df.columns = ['col'+ str(x) for x in bow_df.columns]\n",
    "tfidf_df.columns = ['col' + str(x) for x in tfidf_df.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create separate data frame for bag of words and tf-idf\n",
    "\n",
    "bow_df_train = bow_df[:len(train)]\n",
    "bow_df_test = bow_df[len(train):]\n",
    "\n",
    "tfid_df_train = tfidf_df[:len(train)]\n",
    "tfid_df_test = tfidf_df[len(train):]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# split the merged data file into train and test respectively\n",
    "train_feats = alldata[~pd.isnull(alldata.Is_Response)]\n",
    "test_feats = alldata[pd.isnull(alldata.Is_Response)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/manish/anaconda2/envs/py35/lib/python3.5/site-packages/ipykernel/__main__.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  app.launch_new_instance()\n"
     ]
    }
   ],
   "source": [
    "### set target variable\n",
    "\n",
    "train_feats['Is_Response'] = [1 if x == 'happy' else 0 for x in train_feats['Is_Response']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# merge count (bag of word) features into train\n",
    "train_feats1 = pd.concat([train_feats[cols], bow_df_train], axis = 1)\n",
    "test_feats1 = pd.concat([test_feats[cols], bow_df_test], axis=1)\n",
    "\n",
    "test_feats1.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# merge into a new data frame with tf-idf features\n",
    "train_feats2 = pd.concat([train_feats[cols], tfid_df_train], axis=1)\n",
    "test_feats2 = pd.concat([test_feats[cols], tfid_df_test], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NaiveBayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# let's check cross validation score of the model\n",
    "# cv score acts a unbiased estimate of models accuracy on unseen data\n",
    "\n",
    "mod1 = GaussianNB()\n",
    "target = train_feats['Is_Response']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.76311844  0.7745      0.7515      0.765       0.75837919]\n"
     ]
    }
   ],
   "source": [
    "## Naive Bayes 1\n",
    "print(cross_val_score(mod1, train_feats1, target, cv=5, scoring=make_scorer(accuracy_score)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.79310345  0.811       0.8035      0.815       0.79789895]\n"
     ]
    }
   ],
   "source": [
    "## Naive Bayes 2 - tfidf is giving higher CV score\n",
    "print(cross_val_score(mod1, train_feats2, target, cv=5, scoring=make_scorer(accuracy_score)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GaussianNB(priors=None)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make our first set of predictions\n",
    "\n",
    "clf1 = GaussianNB()\n",
    "clf1.fit(train_feats1, target)\n",
    "\n",
    "clf2 = GaussianNB()\n",
    "clf2.fit(train_feats2, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "preds1 = clf1.predict(test_feats1)\n",
    "preds2 = clf2.predict(test_feats2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def to_labels(x):\n",
    "    if x == 1:\n",
    "        return \"happy\"\n",
    "    return \"not_happy\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sub1 = pd.DataFrame({'User_ID':test.User_ID, 'Is_Response':preds1})\n",
    "sub1['Is_Response'] = sub1['Is_Response'].map(lambda x: to_labels(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sub2 = pd.DataFrame({'User_ID':test.User_ID, 'Is_Response':preds2})\n",
    "sub2['Is_Response'] = sub2['Is_Response'].map(lambda x: to_labels(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sub1 = sub1[['User_ID', 'Is_Response']]\n",
    "sub2 = sub2[['User_ID', 'Is_Response']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## write submission files\n",
    "sub1.to_csv('submissions/sub1_cv.csv', index=False)\n",
    "sub2.to_csv('submissions/sub2_tf.csv', index=False)s"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LightGBM - 1\n",
    "\n",
    "We are prefering lightgbm over xgboost because of its speed. <br />\n",
    "In this model, we'll use count features for model training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import lightgbm as lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# set the data in format lgb accepts\n",
    "d_train = lgb.Dataset(train_feats1, label = target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## set parameters\n",
    "## you can tune the parameters can try to better score\n",
    "\n",
    "params = {'task': 'train',\n",
    "    'boosting_type': 'gbdt',\n",
    "    'objective': 'binary',\n",
    "    'metric': 'binary_error',\n",
    "    'learning_rate': 0.05, \n",
    "    'max_depth': 7, \n",
    "    'num_leaves': 21, \n",
    "    'feature_fraction': 0.3, \n",
    "    'bagging_fraction': 0.8, \n",
    "    'bagging_freq': 5}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[20]\tcv_agg's binary_error: 0.2132 + 0.00456488\n",
      "[40]\tcv_agg's binary_error: 0.195401 + 0.00625882\n",
      "[60]\tcv_agg's binary_error: 0.175601 + 0.00580722\n",
      "[80]\tcv_agg's binary_error: 0.1652 + 0.00589807\n",
      "[100]\tcv_agg's binary_error: 0.1568 + 0.00628195\n",
      "[120]\tcv_agg's binary_error: 0.1505 + 0.00328588\n",
      "[140]\tcv_agg's binary_error: 0.1487 + 0.00399728\n",
      "[160]\tcv_agg's binary_error: 0.147301 + 0.00497347\n",
      "[180]\tcv_agg's binary_error: 0.1445 + 0.00362296\n",
      "[200]\tcv_agg's binary_error: 0.1439 + 0.00429358\n",
      "[220]\tcv_agg's binary_error: 0.1417 + 0.00200147\n",
      "[240]\tcv_agg's binary_error: 0.1418 + 0.0040771\n",
      "[260]\tcv_agg's binary_error: 0.1401 + 0.00373791\n",
      "[280]\tcv_agg's binary_error: 0.1389 + 0.00517039\n",
      "[300]\tcv_agg's binary_error: 0.1376 + 0.00466764\n",
      "[320]\tcv_agg's binary_error: 0.136901 + 0.00507148\n",
      "[340]\tcv_agg's binary_error: 0.1357 + 0.00529898\n",
      "[360]\tcv_agg's binary_error: 0.1363 + 0.00505334\n",
      "[380]\tcv_agg's binary_error: 0.1353 + 0.0044035\n",
      "[400]\tcv_agg's binary_error: 0.1356 + 0.00428458\n",
      "[420]\tcv_agg's binary_error: 0.134501 + 0.00445613\n"
     ]
    }
   ],
   "source": [
    "lgb_cv = lgb.cv(params, d_train, num_boost_round=500, nfold= 5, shuffle=True, stratified=True, verbose_eval=20, early_stopping_rounds=40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## get nround value which hd lowest error\n",
    "nround = lgb_cv['binary_error-mean'].index(np.min(lgb_cv['binary_error-mean']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## train the model\n",
    "model = lgb.train(params, d_train, num_boost_round=nround)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## make predictions\n",
    "preds = model.predict(test_feats1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# make submission\n",
    "\n",
    "def to_labels(x):\n",
    "    if x > 0.66:  # cutoff - you can change it and see if accuracy improves or plot AUC curve. \n",
    "        return \"happy\"\n",
    "    return \"not_happy\"\n",
    "\n",
    "sub3 = pd.DataFrame({'User_ID':test.User_ID, 'Is_Response':preds})\n",
    "sub3['Is_Response'] = sub3['Is_Response'].map(lambda x: to_labels(x))\n",
    "sub3 = sub3[['User_ID','Is_Response']]\n",
    "sub3.to_csv('submissions/sub3_lgb.csv', index=False) # 0.85518"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LightGBM - 2\n",
    "\n",
    "In this model, we'll use tf-idf features for model training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# set data format\n",
    "d_train = lgb.Dataset(train_feats2, label = target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# same parameters as above\n",
    "params = {'task': 'train',\n",
    "    'boosting_type': 'gbdt',\n",
    "    'objective': 'binary',\n",
    "    'metric': 'binary_error',\n",
    "    'learning_rate': 0.05, \n",
    "    'max_depth': 5, \n",
    "    'num_leaves': 11,\n",
    "    'feature_fraction': 0.3, \n",
    "    'bagging_fraction': 0.8, \n",
    "    'bagging_freq': 5}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[20]\tcv_agg's binary_error: 0.226401 + 0.00518217\n",
      "[40]\tcv_agg's binary_error: 0.206602 + 0.00687761\n",
      "[60]\tcv_agg's binary_error: 0.183302 + 0.00791949\n",
      "[80]\tcv_agg's binary_error: 0.169801 + 0.00512283\n",
      "[100]\tcv_agg's binary_error: 0.164301 + 0.00632119\n",
      "[120]\tcv_agg's binary_error: 0.1578 + 0.00507081\n",
      "[140]\tcv_agg's binary_error: 0.1542 + 0.00524522\n",
      "[160]\tcv_agg's binary_error: 0.1516 + 0.00441669\n",
      "[180]\tcv_agg's binary_error: 0.148701 + 0.00512212\n",
      "[200]\tcv_agg's binary_error: 0.1461 + 0.00366096\n",
      "[220]\tcv_agg's binary_error: 0.1443 + 0.00362658\n",
      "[240]\tcv_agg's binary_error: 0.1437 + 0.00471092\n",
      "[260]\tcv_agg's binary_error: 0.143501 + 0.00450031\n",
      "[280]\tcv_agg's binary_error: 0.1405 + 0.00506581\n",
      "[300]\tcv_agg's binary_error: 0.1396 + 0.00578611\n",
      "[320]\tcv_agg's binary_error: 0.137801 + 0.00687571\n",
      "[340]\tcv_agg's binary_error: 0.138701 + 0.00679821\n",
      "[360]\tcv_agg's binary_error: 0.137201 + 0.00839438\n",
      "[380]\tcv_agg's binary_error: 0.137501 + 0.00738158\n",
      "[400]\tcv_agg's binary_error: 0.136401 + 0.00735946\n",
      "[420]\tcv_agg's binary_error: 0.136101 + 0.00702239\n",
      "[440]\tcv_agg's binary_error: 0.136901 + 0.00739423\n"
     ]
    }
   ],
   "source": [
    "## do cross validation to find nround i.e. at this round (iteration) we can expect lowest error\n",
    "lgb_cv = lgb.cv(params, d_train, num_boost_round=500, nfold= 5, shuffle=True, stratified=True, verbose_eval=20, early_stopping_rounds=40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# get nround value\n",
    "nround = lgb_cv['binary_error-mean'].index(np.min(lgb_cv['binary_error-mean']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# train model\n",
    "model = lgb.train(params, d_train, num_boost_round=nround)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# make prediction\n",
    "preds = model.predict(test_feats2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# make submission\n",
    "\n",
    "def to_labels(x):\n",
    "    if x > 0.66:\n",
    "        return \"happy\"\n",
    "    return \"not_happy\"\n",
    "\n",
    "sub4 = pd.DataFrame({'User_ID':test.User_ID, 'Is_Response':preds})\n",
    "sub4['Is_Response'] = sub4['Is_Response'].map(lambda x: to_labels(x))\n",
    "sub4 = sub4[['User_ID','Is_Response']]\n",
    "sub4.to_csv('submissions/sub4_lgb.csv', index=False) # 0.84925"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CatBoost\n",
    "\n",
    "Catboost is a new package recently launched by Yandex. It is said that it works well when the data has many categorical features. We'll use it on count data and see it our model improves."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## import library\n",
    "from catboost import CatBoostClassifier,cv, Pool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## catboost accepts categorical columns as a list of column numbers. In this data, all columns are categorical\n",
    "cat_cols = [x for x in range(502)] ## 502 == train_feats1.shape[1] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## set parameters\n",
    "## you can refer the parameters here: https://tech.yandex.com/catboost/doc/dg/concepts/python-reference_parameters-list-docpage/#python-reference_parameters-list\n",
    "param = {\n",
    "    'use_best_model':True,\n",
    "    'loss_function':'CrossEntropy',\n",
    "    'eval_metric':'Accuracy',\n",
    "    'iterations':1000,\n",
    "    'depth':6,\n",
    "    'learning_rate':0.03,\n",
    "    'rsm':0.3,\n",
    "    'random_seed':2017,\n",
    "    \n",
    "    \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## for doing cross validation, set data in Pool format\n",
    "my_dt =  Pool(train_feats1, \n",
    "           label=target,\n",
    "           cat_features=cat_cols,\n",
    "           column_description=None,\n",
    "           delimiter='\\t',\n",
    "           has_header=None,\n",
    "           weight=None, \n",
    "           baseline=None,\n",
    "           feature_names=None,\n",
    "           thread_count=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Borders generated\n",
      "0:\t0:\t0:\t0:\t0:\t1:\t1:\t1:\t1:\t1:\t2:\t2:\t2:\t2:\t2:\t3:\t3:\t3:\t3:\t3:\t4:\t4:\t4:\t4:\t4:\t5:\t5:\t5:\t5:\t5:\t6:\t6:\t6:\t6:\t6:\t7:\t7:\t7:\t7:\t7:\t8:\t8:\t8:\t8:\t8:\t9:\t9:\t9:\t9:\t9:\t10:\t10:\t10:\t10:\t10:\t11:\t11:\t11:\t11:\t11:\t12:\t12:\t12:\t12:\t12:\t13:\t13:\t13:\t13:\t13:\t14:\t14:\t14:\t14:\t14:\t15:\t15:\t15:\t15:\t15:\t16:\t16:\t16:\t16:\t16:\t17:\t17:\t17:\t17:\t17:\t18:\t18:\t18:\t18:\t18:\t19:\t19:\t19:\t19:\t19:\t20:\t20:\t20:\t20:\t20:\t21:\t21:\t21:\t21:\t21:\t22:\t22:\t22:\t22:\t22:\t23:\t23:\t23:\t23:\t23:\t24:\t24:\t24:\t24:\t24:\t25:\t25:\t25:\t25:\t25:\t26:\t26:\t26:\t26:\t26:\t27:\t27:\t27:\t27:\t27:\t28:\t28:\t28:\t28:\t28:\t29:\t29:\t29:\t29:\t29:\t30:\t30:\t30:\t30:\t30:\t31:\t31:\t31:\t31:\t31:\t32:\t32:\t32:\t32:\t32:\t33:\t33:\t33:\t33:\t33:\t34:\t34:\t34:\t34:\t34:\t35:\t35:\t35:\t35:\t35:\t36:\t36:\t36:\t36:\t36:\t37:\t37:\t37:\t37:\t37:\t38:\t38:\t38:\t38:\t38:\t39:\t39:\t39:\t39:\t39:\t40:\t40:\t40:\t40:\t40:\t41:\t41:\t41:\t41:\t41:\t42:\t42:\t42:\t42:\t42:\t43:\t43:\t43:\t43:\t43:\t44:\t44:\t44:\t44:\t44:\t45:\t45:\t45:\t45:\t45:\t46:\t46:\t46:\t46:\t46:\t47:\t47:\t47:\t47:\t47:\t48:\t48:\t48:\t48:\t48:\t49:\t49:\t49:\t49:\t49:\t50:\t50:\t50:\t50:\t50:\t51:\t51:\t51:\t51:\t51:\t52:\t52:\t52:\t52:\t52:\t53:\t53:\t53:\t53:\t53:\t54:\t54:\t54:\t54:\t54:\t55:\t55:\t55:\t55:\t55:\t56:\t56:\t56:\t56:\t56:\t57:\t57:\t57:\t57:\t57:\t58:\t58:\t58:\t58:\t58:\t59:\t59:\t59:\t59:\t59:\t60:\t60:\t60:\t60:\t60:\t61:\t61:\t61:\t61:\t61:\t62:\t62:\t62:\t62:\t62:\t63:\t63:\t63:\t63:\t63:\t64:\t64:\t64:\t64:\t64:\t65:\t65:\t65:\t65:\t65:\t66:\t66:\t66:\t66:\t66:\t67:\t67:\t67:\t67:\t67:\t68:\t68:\t68:\t68:\t68:\t69:\t69:\t69:\t69:\t69:\t70:\t70:\t70:\t70:\t70:\t71:\t71:\t71:\t71:\t71:\t72:\t72:\t72:\t72:\t72:\t73:\t73:\t73:\t73:\t73:\t74:\t74:\t74:\t74:\t74:\t75:\t75:\t75:\t75:\t75:\t76:\t76:\t76:\t76:\t76:\t77:\t77:\t77:\t77:\t77:\t78:\t78:\t78:\t78:\t78:\t79:\t79:\t79:\t79:\t79:\t80:\t80:\t80:\t80:\t80:\t81:\t81:\t81:\t81:\t81:\t82:\t82:\t82:\t82:\t82:\t83:\t83:\t83:\t83:\t83:\t84:\t84:\t84:\t84:\t84:\t85:\t85:\t85:\t85:\t85:\t86:\t86:\t86:\t86:\t86:\t87:\t87:\t87:\t87:\t87:\t88:\t88:\t88:\t88:\t88:\t89:\t89:\t89:\t89:\t89:\t90:\t90:\t90:\t90:\t90:\t91:\t91:\t91:\t91:\t91:\t92:\t92:\t92:\t92:\t92:\t93:\t93:\t93:\t93:\t93:\t94:\t94:\t94:\t94:\t94:\t95:\t95:\t95:\t95:\t95:\t96:\t96:\t96:\t96:\t96:\t97:\t97:\t97:\t97:\t97:\t98:\t98:\t98:\t98:\t98:\t99:\t99:\t99:\t99:\t99:\t100:\t100:\t100:\t100:\t100:\t101:\t101:\t101:\t101:\t101:\t102:\t102:\t102:\t102:\t102:\t103:\t103:\t103:\t103:\t103:\t104:\t104:\t104:\t104:\t104:\t105:\t105:\t105:\t105:\t105:\t106:\t106:\t106:\t106:\t106:\t107:\t107:\t107:\t107:\t107:\t108:\t108:\t108:\t108:\t108:\t109:\t109:\t109:\t109:\t109:\t110:\t110:\t110:\t110:\t110:\t111:\t111:\t111:\t111:\t111:\t112:\t112:\t112:\t112:\t112:\t113:\t113:\t113:\t113:\t113:\t114:\t114:\t114:\t114:\t114:\t115:\t115:\t115:\t115:\t115:\t116:\t116:\t116:\t116:\t116:\t117:\t117:\t117:\t117:\t117:\t118:\t118:\t118:\t118:\t118:\t119:\t119:\t119:\t119:\t119:\t120:\t120:\t120:\t120:\t120:\t121:\t121:\t121:\t121:\t121:\t122:\t122:\t122:\t122:\t122:\t123:\t123:\t123:\t123:\t123:\t124:\t124:\t124:\t124:\t124:\t125:\t125:\t125:\t125:\t125:\t126:\t126:\t126:\t126:\t126:\t127:\t127:\t127:\t127:\t127:\t128:\t128:\t128:\t128:\t128:\t129:\t129:\t129:\t129:\t129:\t130:\t130:\t130:\t130:\t130:\t131:\t131:\t131:\t131:\t131:\t132:\t132:\t132:\t132:\t132:\t133:\t133:\t133:\t133:\t133:\t134:\t134:\t134:\t134:\t134:\t135:\t135:\t135:\t135:\t135:\t136:\t136:\t136:\t136:\t136:\t137:\t137:\t137:\t137:\t137:\t138:\t138:\t138:\t138:\t138:\t139:\t139:\t139:\t139:\t139:\t140:\t140:\t140:\t140:\t140:\t141:\t141:\t141:\t141:\t141:\t142:\t142:\t142:\t142:\t142:\t143:\t143:\t143:\t143:\t143:\t144:\t144:\t144:\t144:\t144:\t145:\t145:\t145:\t145:\t145:\t146:\t146:\t146:\t146:\t146:\t147:\t147:\t147:\t147:\t147:\t148:\t148:\t148:\t148:\t148:\t149:\t149:\t149:\t149:\t149:\t150:\t150:\t150:\t150:\t150:\t151:\t151:\t151:\t151:\t151:\t152:\t152:\t152:\t152:\t152:\t153:\t153:\t153:\t153:\t153:\t154:\t154:\t154:\t154:\t154:\t155:\t155:\t155:\t155:\t155:\t156:\t156:\t156:\t156:\t156:\t157:\t157:\t157:\t157:\t157:\t158:\t158:\t158:\t158:\t158:\t159:\t159:\t159:\t159:\t159:\t160:\t160:\t160:\t160:\t160:\t161:\t161:\t161:\t161:\t161:\t162:\t162:\t162:\t162:\t162:\t163:\t163:\t163:\t163:\t163:\t164:\t164:\t164:\t164:\t164:\t165:\t165:\t165:\t165:\t165:\t166:\t166:\t166:\t166:\t166:\t167:\t167:\t167:\t167:\t167:\t168:\t168:\t168:\t168:\t168:\t169:\t169:\t169:\t169:\t169:\t170:\t170:\t170:\t170:\t170:\t171:\t171:\t171:\t171:\t171:\t172:\t172:\t172:\t172:\t172:\t173:\t173:\t173:\t173:\t173:\t174:\t174:\t174:\t174:\t174:\t175:\t175:\t175:\t175:\t175:\t176:\t176:\t176:\t176:\t176:\t177:\t177:\t177:\t177:\t177:\t178:\t178:\t178:\t178:\t178:\t179:\t179:\t179:\t179:\t179:\t180:\t180:\t180:\t180:\t180:\t181:\t181:\t181:\t181:\t181:\t182:\t182:\t182:\t182:\t182:\t183:\t183:\t183:\t183:\t183:\t184:\t184:\t184:\t184:\t184:\t185:\t185:\t185:\t185:\t185:\t186:\t186:\t186:\t186:\t186:\t187:\t187:\t187:\t187:\t187:\t188:\t188:\t188:\t188:\t188:\t189:\t189:\t189:\t189:\t189:\t190:\t190:\t190:\t190:\t190:\t191:\t191:\t191:\t191:\t191:\t192:\t192:\t192:\t192:\t192:\t193:\t193:\t193:\t193:\t193:\t194:\t194:\t194:\t194:\t194:\t195:\t195:\t195:\t195:\t195:\t196:\t196:\t196:\t196:\t196:\t197:\t197:\t197:\t197:\t197:\t198:\t198:\t198:\t198:\t198:\t199:\t199:\t199:\t199:\t199:\t200:\t200:\t200:\t200:\t200:\t201:\t201:\t201:\t201:\t201:\t202:\t202:\t202:\t202:\t202:\t203:\t203:\t203:\t203:\t203:\t204:\t204:\t204:\t204:\t204:\t205:\t205:\t205:\t205:\t205:\t206:\t206:\t206:\t206:\t206:\t207:\t207:\t207:\t207:\t207:\t208:\t208:\t208:\t208:\t208:\t209:\t209:\t209:\t209:\t209:\t210:\t210:\t210:\t210:\t210:\t211:\t211:\t211:\t211:\t211:\t212:\t212:\t212:\t212:\t212:\t213:\t213:\t213:\t213:\t213:\t214:\t214:\t214:\t214:\t214:\t215:\t215:\t215:\t215:\t215:\t216:\t216:\t216:\t216:\t216:\t217:\t217:\t217:\t217:\t217:\t218:\t218:\t218:\t218:\t218:\t219:\t219:\t219:\t219:\t219:\t220:\t220:\t220:\t220:\t220:\t221:\t221:\t221:\t221:\t221:\t222:\t222:\t222:\t222:\t222:\t223:\t223:\t223:\t223:\t223:\t224:\t224:\t224:\t224:\t224:\t225:\t225:\t225:\t225:\t225:\t226:\t226:\t226:\t226:\t226:\t227:\t227:\t227:\t227:\t227:\t228:\t228:\t228:\t228:\t228:\t229:\t229:\t229:\t229:\t229:\t230:\t230:\t230:\t230:\t230:\t231:\t231:\t231:\t231:\t231:\t232:\t232:\t232:\t232:\t232:\t233:\t233:\t233:\t233:\t233:\t234:\t234:\t234:\t234:\t234:\t235:\t235:\t235:\t235:\t235:\t236:\t236:\t236:\t236:\t236:\t237:\t237:\t237:\t237:\t237:\t238:\t238:\t238:\t238:\t238:\t239:\t239:\t239:\t239:\t239:\t240:\t240:\t240:\t240:\t240:\t241:\t241:\t241:\t241:\t241:\t242:\t242:\t242:\t242:\t242:\t243:\t243:\t243:\t243:\t243:\t244:\t244:\t244:\t244:\t244:\t245:\t245:\t245:\t245:\t245:\t246:\t246:\t246:\t246:\t246:\t247:\t247:\t247:\t247:\t247:\t248:\t248:\t248:\t248:\t248:\t249:\t249:\t249:\t249:\t249:\t250:\t250:\t250:\t250:\t250:\t251:\t251:\t251:\t251:\t251:\t252:\t252:\t252:\t252:\t252:\t253:\t253:\t253:\t253:\t253:\t254:\t254:\t254:\t254:\t254:\t255:\t255:\t255:\t255:\t255:\t256:\t256:\t256:\t256:\t256:\t257:\t257:\t257:\t257:\t257:\t258:\t258:\t258:\t258:\t258:\t259:\t259:\t259:\t259:\t259:\t260:\t260:\t260:\t260:\t260:\t261:\t261:\t261:\t261:\t261:\t262:\t262:\t262:\t262:\t262:\t263:\t263:\t263:\t263:\t263:\t264:\t264:\t264:\t264:\t264:\t265:\t265:\t265:\t265:\t265:\t266:\t266:\t266:\t266:\t266:\t267:\t267:\t267:\t267:\t267:\t268:\t268:\t268:\t268:\t268:\t269:\t269:\t269:\t269:\t269:\t270:\t270:\t270:\t270:\t270:\t271:\t271:\t271:\t271:\t271:\t272:\t272:\t272:\t272:\t272:\t273:\t273:\t273:\t273:\t273:\t274:\t274:\t274:\t274:\t274:\t275:\t275:\t275:\t275:\t275:\t276:\t276:\t276:\t276:\t276:\t277:\t277:\t277:\t277:\t277:\t278:\t278:\t278:\t278:\t278:\t279:\t279:\t279:\t279:\t279:\t280:\t280:\t280:\t280:\t280:\t281:\t281:\t281:\t281:\t281:\t282:\t282:\t282:\t282:\t282:\t283:\t283:\t283:\t283:\t283:\t284:\t284:\t284:\t284:\t284:\t285:\t285:\t285:\t285:\t285:\t286:\t286:\t286:\t286:\t286:\t287:\t287:\t287:\t287:\t287:\t288:\t288:\t288:\t288:\t288:\t289:\t289:\t289:\t289:\t289:\t290:\t290:\t290:\t290:\t290:\t291:\t291:\t291:\t291:\t291:\t292:\t292:\t292:\t292:\t292:\t293:\t293:\t293:\t293:\t293:\t294:\t294:\t294:\t294:\t294:\t295:\t295:\t295:\t295:\t295:\t296:\t296:\t296:\t296:\t296:\t297:\t297:\t297:\t297:\t297:\t298:\t298:\t298:\t298:\t298:\t299:\t299:\t299:\t299:\t299:\t300:\t300:\t300:\t300:\t300:\t301:\t301:\t301:\t301:\t301:\t302:\t302:\t302:\t302:\t302:\t303:\t303:\t303:\t303:\t303:\t304:\t304:\t304:\t304:\t304:\t305:\t305:\t305:\t305:\t305:\t306:\t306:\t306:\t306:\t306:\t307:\t307:\t307:\t307:\t307:\t308:\t308:\t308:\t308:\t308:\t309:\t309:\t309:\t309:\t309:\t310:\t310:\t310:\t310:\t310:\t311:\t311:\t311:\t311:\t311:\t312:\t312:\t312:\t312:\t312:\t313:\t313:\t313:\t313:\t313:\t314:\t314:\t314:\t314:\t314:\t315:\t315:\t315:\t315:\t315:\t316:\t316:\t316:\t316:\t316:\t317:\t317:\t317:\t317:\t317:\t318:\t318:\t318:\t318:\t318:\t319:\t319:\t319:\t319:\t319:\t320:\t320:\t320:\t320:\t320:\t321:\t321:\t321:\t321:\t321:\t322:\t322:\t322:\t322:\t322:\t323:\t323:\t323:\t323:\t323:\t324:\t324:\t324:\t324:\t324:\t325:\t325:\t325:\t325:\t325:\t326:\t326:\t326:\t326:\t326:\t327:\t327:\t327:\t327:\t327:\t328:\t328:\t328:\t328:\t328:\t329:\t329:\t329:\t329:\t329:\t330:\t330:\t330:\t330:\t330:\t331:\t331:\t331:\t331:\t331:\t332:\t332:\t332:\t332:\t332:\t333:\t333:\t333:\t333:\t333:\t334:\t334:\t334:\t334:\t334:\t335:\t335:\t335:\t335:\t335:\t336:\t336:\t336:\t336:\t336:\t337:\t337:\t337:\t337:\t337:\t338:\t338:\t338:\t338:\t338:\t339:\t339:\t339:\t339:\t339:\t340:\t340:\t340:\t340:\t340:\t341:\t341:\t341:\t341:\t341:\t342:\t342:\t342:\t342:\t342:\t343:\t343:\t343:\t343:\t343:\t344:\t344:\t344:\t344:\t344:\t345:\t345:\t345:\t345:\t345:\t346:\t346:\t346:\t346:\t346:\t347:\t347:\t347:\t347:\t347:\t348:\t348:\t348:\t348:\t348:\t349:\t349:\t349:\t349:\t349:\t350:\t350:\t350:\t350:\t350:\t351:\t351:\t351:\t351:\t351:\t352:\t352:\t352:\t352:\t352:\t353:\t353:\t353:\t353:\t353:\t354:\t354:\t354:\t354:\t354:\t355:\t355:\t355:\t355:\t355:\t356:\t356:\t356:\t356:\t356:\t357:\t357:\t357:\t357:\t357:\t358:\t358:\t358:\t358:\t358:\t359:\t359:\t359:\t359:\t359:\t360:\t360:\t360:\t360:\t360:\t361:\t361:\t361:\t361:\t361:\t362:\t362:\t362:\t362:\t362:\t363:\t363:\t363:\t363:\t363:\t364:\t364:\t364:\t364:\t364:\t365:\t365:\t365:\t365:\t365:\t366:\t366:\t366:\t366:\t366:\t367:\t367:\t367:\t367:\t367:\t368:\t368:\t368:\t368:\t368:\t369:\t369:\t369:\t369:\t369:\t370:\t370:\t370:\t370:\t370:\t371:\t371:\t371:\t371:\t371:\t372:\t372:\t372:\t372:\t372:\t373:\t373:\t373:\t373:\t373:\t374:\t374:\t374:\t374:\t374:\t375:\t375:\t375:\t375:\t375:\t376:\t376:\t376:\t376:\t376:\t377:\t377:\t377:\t377:\t377:\t378:\t378:\t378:\t378:\t378:\t379:\t379:\t379:\t379:\t379:\t380:\t380:\t380:\t380:\t380:\t381:\t381:\t381:\t381:\t381:\t382:\t382:\t382:\t382:\t382:\t383:\t383:\t383:\t383:\t383:\t384:\t384:\t384:\t384:\t384:\t385:\t385:\t385:\t385:\t385:\t386:\t386:\t386:\t386:\t386:\t387:\t387:\t387:\t387:\t387:\t388:\t388:\t388:\t388:\t388:\t389:\t389:\t389:\t389:\t389:\t390:\t390:\t390:\t390:\t390:\t391:\t391:\t391:\t391:\t391:\t392:\t392:\t392:\t392:\t392:\t393:\t393:\t393:\t393:\t393:\t394:\t394:\t394:\t394:\t394:\t395:\t395:\t395:\t395:\t395:\t396:\t396:\t396:\t396:\t396:\t397:\t397:\t397:\t397:\t397:\t398:\t398:\t398:\t398:\t398:\t399:\t399:\t399:\t399:\t399:\t400:\t400:\t400:\t400:\t400:\t401:\t401:\t401:\t401:\t401:\t402:\t402:\t402:\t402:\t402:\t403:\t403:\t403:\t403:\t403:\t404:\t404:\t404:\t404:\t404:\t405:\t405:\t405:\t405:\t405:\t406:\t406:\t406:\t406:\t406:\t407:\t407:\t407:\t407:\t407:\t408:\t408:\t408:\t408:\t408:\t409:\t409:\t409:\t409:\t409:\t410:\t410:\t410:\t410:\t410:\t411:\t411:\t411:\t411:\t411:\t412:\t412:\t412:\t412:\t412:\t413:\t413:\t413:\t413:\t413:\t414:\t414:\t414:\t414:\t414:\t415:\t415:\t415:\t415:\t415:\t416:\t416:\t416:\t416:\t416:\t417:\t417:\t417:\t417:\t417:\t418:\t418:\t418:\t418:\t418:\t419:\t419:\t419:\t419:\t419:\t420:\t420:\t420:\t420:\t420:\t421:\t421:\t421:\t421:\t421:\t422:\t422:\t422:\t422:\t422:\t423:\t423:\t423:\t423:\t423:\t424:\t424:\t424:\t424:\t424:\t425:\t425:\t425:\t425:\t425:\t426:\t426:\t426:\t426:\t426:\t427:\t427:\t427:\t427:\t427:\t428:\t428:\t428:\t428:\t428:\t429:\t429:\t429:\t429:\t429:\t430:\t430:\t430:\t430:\t430:\t431:\t431:\t431:\t431:\t431:\t432:\t432:\t432:\t432:\t432:\t433:\t433:\t433:\t433:\t433:\t434:\t434:\t434:\t434:\t434:\t435:\t435:\t435:\t435:\t435:\t436:\t436:\t436:\t436:\t436:\t437:\t437:\t437:\t437:\t437:\t438:\t438:\t438:\t438:\t438:\t439:\t439:\t439:\t439:\t439:\t440:\t440:\t440:\t440:\t440:\t441:\t441:\t441:\t441:\t441:\t442:\t442:\t442:\t442:\t442:\t443:\t443:\t443:\t443:\t443:\t444:\t444:\t444:\t444:\t444:\t445:\t445:\t445:\t445:\t445:\t446:\t446:\t446:\t446:\t446:\t447:\t447:\t447:\t447:\t447:\t448:\t448:\t448:\t448:\t448:\t449:\t449:\t449:\t449:\t449:\t450:\t450:\t450:\t450:\t450:\t451:\t451:\t451:\t451:\t451:\t452:\t452:\t452:\t452:\t452:\t453:\t453:\t453:\t453:\t453:\t454:\t454:\t454:\t454:\t454:\t455:\t455:\t455:\t455:\t455:\t456:\t456:\t456:\t456:\t456:\t457:\t457:\t457:\t457:\t457:\t458:\t458:\t458:\t458:\t458:\t459:\t459:\t459:\t459:\t459:\t460:\t460:\t460:\t460:\t460:\t461:\t461:\t461:\t461:\t461:\t462:\t462:\t462:\t462:\t462:\t463:\t463:\t463:\t463:\t463:\t464:\t464:\t464:\t464:\t464:\t465:\t465:\t465:\t465:\t465:\t466:\t466:\t466:\t466:\t466:\t467:\t467:\t467:\t467:\t467:\t468:\t468:\t468:\t468:\t468:\t469:\t469:\t469:\t469:\t469:\t470:\t470:\t470:\t470:\t470:\t471:\t471:\t471:\t471:\t471:\t472:\t472:\t472:\t472:\t472:\t473:\t473:\t473:\t473:\t473:\t474:\t474:\t474:\t474:\t474:\t475:\t475:\t475:\t475:\t475:\t476:\t476:\t476:\t476:\t476:\t477:\t477:\t477:\t477:\t477:\t478:\t478:\t478:\t478:\t478:\t479:\t479:\t479:\t479:\t479:\t480:\t480:\t480:\t480:\t480:\t481:\t481:\t481:\t481:\t481:\t482:\t482:\t482:\t482:\t482:\t483:\t483:\t483:\t483:\t483:\t484:\t484:\t484:\t484:\t484:\t485:\t485:\t485:\t485:\t485:\t486:\t486:\t486:\t486:\t486:\t487:\t487:\t487:\t487:\t487:\t488:\t488:\t488:\t488:\t488:\t489:\t489:\t489:\t489:\t489:\t490:\t490:\t490:\t490:\t490:\t491:\t491:\t491:\t491:\t491:\t492:\t492:\t492:\t492:\t492:\t493:\t493:\t493:\t493:\t493:\t494:\t494:\t494:\t494:\t494:\t495:\t495:\t495:\t495:\t495:\t496:\t496:\t496:\t496:\t496:\t497:\t497:\t497:\t497:\t497:\t498:\t498:\t498:\t498:\t498:\t499:\t499:\t499:\t499:\t499:\t500:\t500:\t500:\t500:\t500:\t501:\t501:\t501:\t501:\t501:\t502:\t502:\t502:\t502:\t502:\t503:\t503:\t503:\t503:\t503:\t504:\t504:\t504:\t504:\t504:\t505:\t505:\t505:\t505:\t505:\t506:\t506:\t506:\t506:\t506:\t507:\t507:\t507:\t507:\t507:\t508:\t508:\t508:\t508:\t508:\t509:\t509:\t509:\t509:\t509:\t510:\t510:\t510:\t510:\t510:\t511:\t511:\t511:\t511:\t511:\t512:\t512:\t512:\t512:\t512:\t513:\t513:\t513:\t513:\t513:\t514:\t514:\t514:\t514:\t514:\t515:\t515:\t515:\t515:\t515:\t516:\t516:\t516:\t516:\t516:\t517:\t517:\t517:\t517:\t517:\t518:\t518:\t518:\t518:\t518:\t519:\t519:\t519:\t519:\t519:\t520:\t520:\t520:\t520:\t520:\t521:\t521:\t521:\t521:\t521:\t522:\t522:\t522:\t522:\t522:\t523:\t523:\t523:\t523:\t523:\t524:\t524:\t524:\t524:\t524:\t525:\t525:\t525:\t525:\t525:\t526:\t526:\t526:\t526:\t526:\t527:\t527:\t527:\t527:\t527:\t528:\t528:\t528:\t528:\t528:\t529:\t529:\t529:\t529:\t529:\t530:\t530:\t530:\t530:\t530:\t531:\t531:\t531:\t531:\t531:\t532:\t532:\t532:\t532:\t532:\t533:\t533:\t533:\t533:\t533:\t534:\t534:\t534:\t534:\t534:\t535:\t535:\t535:\t535:\t535:\t536:\t536:\t536:\t536:\t536:\t537:\t537:\t537:\t537:\t537:\t538:\t538:\t538:\t538:\t538:\t539:\t539:\t539:\t539:\t539:\t540:\t540:\t540:\t540:\t540:\t541:\t541:\t541:\t541:\t541:\t542:\t542:\t542:\t542:\t542:\t543:\t543:\t543:\t543:\t543:\t544:\t544:\t544:\t544:\t544:\t545:\t545:\t545:\t545:\t545:\t546:\t546:\t546:\t546:\t546:\t547:\t547:\t547:\t547:\t547:\t548:\t548:\t548:\t548:\t548:\t549:\t549:\t549:\t549:\t549:\t550:\t550:\t550:\t550:\t550:\t551:\t551:\t551:\t551:\t551:\t552:\t552:\t552:\t552:\t552:\t553:\t553:\t553:\t553:\t553:\t554:\t554:\t554:\t554:\t554:\t555:\t555:\t555:\t555:\t555:\t556:\t556:\t556:\t556:\t556:\t557:\t557:\t557:\t557:\t557:\t558:\t558:\t558:\t558:\t558:\t559:\t559:\t559:\t559:\t559:\t560:\t560:\t560:\t560:\t560:\t561:\t561:\t561:\t561:\t561:\t562:\t562:\t562:\t562:\t562:\t563:\t563:\t563:\t563:\t563:\t564:\t564:\t564:\t564:\t564:\t565:\t565:\t565:\t565:\t565:\t566:\t566:\t566:\t566:\t566:\t567:\t567:\t567:\t567:\t567:\t568:\t568:\t568:\t568:\t568:\t569:\t569:\t569:\t569:\t569:\t570:\t570:\t570:\t570:\t570:\t571:\t571:\t571:\t571:\t571:\t572:\t572:\t572:\t572:\t572:\t573:\t573:\t573:\t573:\t573:\t574:\t574:\t574:\t574:\t574:\t575:\t575:\t575:\t575:\t575:\t576:\t576:\t576:\t576:\t576:\t577:\t577:\t577:\t577:\t577:\t578:\t578:\t578:\t578:\t578:\t579:\t579:\t579:\t579:\t579:\t580:\t580:\t580:\t580:\t580:\t581:\t581:\t581:\t581:\t581:\t582:\t582:\t582:\t582:\t582:\t583:\t583:\t583:\t583:\t583:\t584:\t584:\t584:\t584:\t584:\t585:\t585:\t585:\t585:\t585:\t586:\t586:\t586:\t586:\t586:\t587:\t587:\t587:\t587:\t587:\t588:\t588:\t588:\t588:\t588:\t589:\t589:\t589:\t589:\t589:\t590:\t590:\t590:\t590:\t590:\t591:\t591:\t591:\t591:\t591:\t592:\t592:\t592:\t592:\t592:\t593:\t593:\t593:\t593:\t593:\t594:\t594:\t594:\t594:\t594:\t595:\t595:\t595:\t595:\t595:\t596:\t596:\t596:\t596:\t596:\t597:\t597:\t597:\t597:\t597:\t598:\t598:\t598:\t598:\t598:\t599:\t599:\t599:\t599:\t599:\t600:\t600:\t600:\t600:\t600:\t601:\t601:\t601:\t601:\t601:\t602:\t602:\t602:\t602:\t602:\t603:\t603:\t603:\t603:\t603:\t604:\t604:\t604:\t604:\t604:\t605:\t605:\t605:\t605:\t605:\t606:\t606:\t606:\t606:\t606:\t607:\t607:\t607:\t607:\t607:\t608:\t608:\t608:\t608:\t608:\t609:\t609:\t609:\t609:\t609:\t610:\t610:\t610:\t610:\t610:\t611:\t611:\t611:\t611:\t611:\t612:\t612:\t612:\t612:\t612:\t613:\t613:\t613:\t613:\t613:\t614:\t614:\t614:\t614:\t614:\t615:\t615:\t615:\t615:\t615:\t616:\t616:\t616:\t616:\t616:\t617:\t617:\t617:\t617:\t617:\t618:\t618:\t618:\t618:\t618:\t619:\t619:\t619:\t619:\t619:\t620:\t620:\t620:\t620:\t620:\t621:\t621:\t621:\t621:\t621:\t622:\t622:\t622:\t622:\t622:\t623:\t623:\t623:\t623:\t623:\t624:\t624:\t624:\t624:\t624:\t625:\t625:\t625:\t625:\t625:\t626:\t626:\t626:\t626:\t626:\t627:\t627:\t627:\t627:\t627:\t628:\t628:\t628:\t628:\t628:\t629:\t629:\t629:\t629:\t629:\t630:\t630:\t630:\t630:\t630:\t631:\t631:\t631:\t631:\t631:\t632:\t632:\t632:\t632:\t632:\t633:\t633:\t633:\t633:\t633:\t634:\t634:\t634:\t634:\t634:\t635:\t635:\t635:\t635:\t635:\t636:\t636:\t636:\t636:\t636:\t637:\t637:\t637:\t637:\t637:\t638:\t638:\t638:\t638:\t638:\t639:\t639:\t639:\t639:\t639:\t640:\t640:\t640:\t640:\t640:\t641:\t641:\t641:\t641:\t641:\t642:\t642:\t642:\t642:\t642:\t643:\t643:\t643:\t643:\t643:\t644:\t644:\t644:\t644:\t644:\t645:\t645:\t645:\t645:\t645:\t646:\t646:\t646:\t646:\t646:\t647:\t647:\t647:\t647:\t647:\t648:\t648:\t648:\t648:\t648:\t649:\t649:\t649:\t649:\t649:\t650:\t650:\t650:\t650:\t650:\t651:\t651:\t651:\t651:\t651:\t652:\t652:\t652:\t652:\t652:\t653:\t653:\t653:\t653:\t653:\t654:\t654:\t654:\t654:\t654:\t655:\t655:\t655:\t655:\t655:\t656:\t656:\t656:\t656:\t656:\t657:\t657:\t657:\t657:\t657:\t658:\t658:\t658:\t658:\t658:\t659:\t659:\t659:\t659:\t659:\t660:\t660:\t660:\t660:\t660:\t661:\t661:\t661:\t661:\t661:\t662:\t662:\t662:\t662:\t662:\t663:\t663:\t663:\t663:\t663:\t664:\t664:\t664:\t664:\t664:\t665:\t665:\t665:\t665:\t665:\t666:\t666:\t666:\t666:\t666:\t667:\t667:\t667:\t667:\t667:\t668:\t668:\t668:\t668:\t668:\t669:\t669:\t669:\t669:\t669:\t670:\t670:\t670:\t670:\t670:\t671:\t671:\t671:\t671:\t671:\t672:\t672:\t672:\t672:\t672:\t673:\t673:\t673:\t673:\t673:\t674:\t674:\t674:\t674:\t674:\t675:\t675:\t675:\t675:\t675:\t676:\t676:\t676:\t676:\t676:\t677:\t677:\t677:\t677:\t677:\t678:\t678:\t678:\t678:\t678:\t679:\t679:\t679:\t679:\t679:\t680:\t680:\t680:\t680:\t680:\t681:\t681:\t681:\t681:\t681:\t682:\t682:\t682:\t682:\t682:\t683:\t683:\t683:\t683:\t683:\t684:\t684:\t684:\t684:\t684:\t685:\t685:\t685:\t685:\t685:\t686:\t686:\t686:\t686:\t686:\t687:\t687:\t687:\t687:\t687:\t688:\t688:\t688:\t688:\t688:\t689:\t689:\t689:\t689:\t689:\t690:\t690:\t690:\t690:\t690:\t691:\t691:\t691:\t691:\t691:\t692:\t692:\t692:\t692:\t692:\t693:\t693:\t693:\t693:\t693:\t694:\t694:\t694:\t694:\t694:\t695:\t695:\t695:\t695:\t695:\t696:\t696:\t696:\t696:\t696:\t697:\t697:\t697:\t697:\t697:\t698:\t698:\t698:\t698:\t698:\t699:\t699:\t699:\t699:\t699:\t700:\t700:\t700:\t700:\t700:\t701:\t701:\t701:\t701:\t701:\t702:\t702:\t702:\t702:\t702:\t703:\t703:\t703:\t703:\t703:\t704:\t704:\t704:\t704:\t704:\t705:\t705:\t705:\t705:\t705:\t706:\t706:\t706:\t706:\t706:\t707:\t707:\t707:\t707:\t707:\t708:\t708:\t708:\t708:\t708:\t709:\t709:\t709:\t709:\t709:\t710:\t710:\t710:\t710:\t710:\t711:\t711:\t711:\t711:\t711:\t712:\t712:\t712:\t712:\t712:\t713:\t713:\t713:\t713:\t713:\t714:\t714:\t714:\t714:\t714:\t715:\t715:\t715:\t715:\t715:\t716:\t716:\t716:\t716:\t716:\t717:\t717:\t717:\t717:\t717:\t718:\t718:\t718:\t718:\t718:\t719:\t719:\t719:\t719:\t719:\t720:\t720:\t720:\t720:\t720:\t721:\t721:\t721:\t721:\t721:\t722:\t722:\t722:\t722:\t722:\t723:\t723:\t723:\t723:\t723:\t724:\t724:\t724:\t724:\t724:\t725:\t725:\t725:\t725:\t725:\t726:\t726:\t726:\t726:\t726:\t727:\t727:\t727:\t727:\t727:\t728:\t728:\t728:\t728:\t728:\t729:\t729:\t729:\t729:\t729:\t730:\t730:\t730:\t730:\t730:\t731:\t731:\t731:\t731:\t731:\t732:\t732:\t732:\t732:\t732:\t733:\t733:\t733:\t733:\t733:\t734:\t734:\t734:\t734:\t734:\t735:\t735:\t735:\t735:\t735:\t736:\t736:\t736:\t736:\t736:\t737:\t737:\t737:\t737:\t737:\t738:\t738:\t738:\t738:\t738:\t739:\t739:\t739:\t739:\t739:\t740:\t740:\t740:\t740:\t740:\t741:\t741:\t741:\t741:\t741:\t742:\t742:\t742:\t742:\t742:\t743:\t743:\t743:\t743:\t743:\t744:\t744:\t744:\t744:\t744:\t745:\t745:\t745:\t745:\t745:\t746:\t746:\t746:\t746:\t746:\t747:\t747:\t747:\t747:\t747:\t748:\t748:\t748:\t748:\t748:\t749:\t749:\t749:\t749:\t749:\t750:\t750:\t750:\t750:\t750:\t751:\t751:\t751:\t751:\t751:\t752:\t752:\t752:\t752:\t752:\t753:\t753:\t753:\t753:\t753:\t754:\t754:\t754:\t754:\t754:\t755:\t755:\t755:\t755:\t755:\t756:\t756:\t756:\t756:\t756:\t757:\t757:\t757:\t757:\t757:\t758:\t758:\t758:\t758:\t758:\t759:\t759:\t759:\t759:\t759:\t760:\t760:\t760:\t760:\t760:\t761:\t761:\t761:\t761:\t761:\t762:\t762:\t762:\t762:\t762:\t763:\t763:\t763:\t763:\t763:\t764:\t764:\t764:\t764:\t764:\t765:\t765:\t765:\t765:\t765:\t766:\t766:\t766:\t766:\t766:\t767:\t767:\t767:\t767:\t767:\t768:\t768:\t768:\t768:\t768:\t769:\t769:\t769:\t769:\t769:\t770:\t770:\t770:\t770:\t770:\t771:\t771:\t771:\t771:\t771:\t772:\t772:\t772:\t772:\t772:\t773:\t773:\t773:\t773:\t773:\t774:\t774:\t774:\t774:\t774:\t775:\t775:\t775:\t775:\t775:\t776:\t776:\t776:\t776:\t776:\t777:\t777:\t777:\t777:\t777:\t778:\t778:\t778:\t778:\t778:\t779:\t779:\t779:\t779:\t779:\t780:\t780:\t780:\t780:\t780:\t781:\t781:\t781:\t781:\t781:\t782:\t782:\t782:\t782:\t782:\t783:\t783:\t783:\t783:\t783:\t784:\t784:\t784:\t784:\t784:\t785:\t785:\t785:\t785:\t785:\t786:\t786:\t786:\t786:\t786:\t787:\t787:\t787:\t787:\t787:\t788:\t788:\t788:\t788:\t788:\t789:\t789:\t789:\t789:\t789:\t790:\t790:\t790:\t790:\t790:\t791:\t791:\t791:\t791:\t791:\t792:\t792:\t792:\t792:\t792:\t793:\t793:\t793:\t793:\t793:\t794:\t794:\t794:\t794:\t794:\t795:\t795:\t795:\t795:\t795:\t796:\t796:\t796:\t796:\t796:\t797:\t797:\t797:\t797:\t797:\t798:\t798:\t798:\t798:\t798:\t799:\t799:\t799:\t799:\t799:\t800:\t800:\t800:\t800:\t800:\t801:\t801:\t801:\t801:\t801:\t802:\t802:\t802:\t802:\t802:\t803:\t803:\t803:\t803:\t803:\t804:\t804:\t804:\t804:\t804:\t805:\t805:\t805:\t805:\t805:\t806:\t806:\t806:\t806:\t806:\t807:\t807:\t807:\t807:\t807:\t808:\t808:\t808:\t808:\t808:\t809:\t809:\t809:\t809:\t809:\t810:\t810:\t810:\t810:\t810:\t811:\t811:\t811:\t811:\t811:\t812:\t812:\t812:\t812:\t812:\t813:\t813:\t813:\t813:\t813:\t814:\t814:\t814:\t814:\t814:\t815:\t815:\t815:\t815:\t815:\t816:\t816:\t816:\t816:\t816:\t817:\t817:\t817:\t817:\t817:\t818:\t818:\t818:\t818:\t818:\t819:\t819:\t819:\t819:\t819:\t820:\t820:\t820:\t820:\t820:\t821:\t821:\t821:\t821:\t821:\t822:\t822:\t822:\t822:\t822:\t823:\t823:\t823:\t823:\t823:\t824:\t824:\t824:\t824:\t824:\t825:\t825:\t825:\t825:\t825:\t826:\t826:\t826:\t826:\t826:\t827:\t827:\t827:\t827:\t827:\t828:\t828:\t828:\t828:\t828:\t829:\t829:\t829:\t829:\t829:\t830:\t830:\t830:\t830:\t830:\t831:\t831:\t831:\t831:\t831:\t832:\t832:\t832:\t832:\t832:\t833:\t833:\t833:\t833:\t833:\t834:\t834:\t834:\t834:\t834:\t835:\t835:\t835:\t835:\t835:\t836:\t836:\t836:\t836:\t836:\t837:\t837:\t837:\t837:\t837:\t838:\t838:\t838:\t838:\t838:\t839:\t839:\t839:\t839:\t839:\t840:\t840:\t840:\t840:\t840:\t841:\t841:\t841:\t841:\t841:\t842:\t842:\t842:\t842:\t842:\t843:\t843:\t843:\t843:\t843:\t844:\t844:\t844:\t844:\t844:\t845:\t845:\t845:\t845:\t845:\t846:\t846:\t846:\t846:\t846:\t847:\t847:\t847:\t847:\t847:\t848:\t848:\t848:\t848:\t848:\t849:\t849:\t849:\t849:\t849:\t850:\t850:\t850:\t850:\t850:\t851:\t851:\t851:\t851:\t851:\t852:\t852:\t852:\t852:\t852:\t853:\t853:\t853:\t853:\t853:\t854:\t854:\t854:\t854:\t854:\t855:\t855:\t855:\t855:\t855:\t856:\t856:\t856:\t856:\t856:\t857:\t857:\t857:\t857:\t857:\t858:\t858:\t858:\t858:\t858:\t859:\t859:\t859:\t859:\t859:\t860:\t860:\t860:\t860:\t860:\t861:\t861:\t861:\t861:\t861:\t862:\t862:\t862:\t862:\t862:\t863:\t863:\t863:\t863:\t863:\t864:\t864:\t864:\t864:\t864:\t865:\t865:\t865:\t865:\t865:\t866:\t866:\t866:\t866:\t866:\t867:\t867:\t867:\t867:\t867:\t868:\t868:\t868:\t868:\t868:\t869:\t869:\t869:\t869:\t869:\t870:\t870:\t870:\t870:\t870:\t871:\t871:\t871:\t871:\t871:\t872:\t872:\t872:\t872:\t872:\t873:\t873:\t873:\t873:\t873:\t874:\t874:\t874:\t874:\t874:\t875:\t875:\t875:\t875:\t875:\t876:\t876:\t876:\t876:\t876:\t877:\t877:\t877:\t877:\t877:\t878:\t878:\t878:\t878:\t878:\t879:\t879:\t879:\t879:\t879:\t880:\t880:\t880:\t880:\t880:\t881:\t881:\t881:\t881:\t881:\t882:\t882:\t882:\t882:\t882:\t883:\t883:\t883:\t883:\t883:\t884:\t884:\t884:\t884:\t884:\t885:\t885:\t885:\t885:\t885:\t886:\t886:\t886:\t886:\t886:\t887:\t887:\t887:\t887:\t887:\t888:\t888:\t888:\t888:\t888:\t889:\t889:\t889:\t889:\t889:\t890:\t890:\t890:\t890:\t890:\t891:\t891:\t891:\t891:\t891:\t892:\t892:\t892:\t892:\t892:\t893:\t893:\t893:\t893:\t893:\t894:\t894:\t894:\t894:\t894:\t895:\t895:\t895:\t895:\t895:\t896:\t896:\t896:\t896:\t896:\t897:\t897:\t897:\t897:\t897:\t898:\t898:\t898:\t898:\t898:\t899:\t899:\t899:\t899:\t899:\t900:\t900:\t900:\t900:\t900:\t901:\t901:\t901:\t901:\t901:\t902:\t902:\t902:\t902:\t902:\t903:\t903:\t903:\t903:\t903:\t904:\t904:\t904:\t904:\t904:\t905:\t905:\t905:\t905:\t905:\t906:\t906:\t906:\t906:\t906:\t907:\t907:\t907:\t907:\t907:\t908:\t908:\t908:\t908:\t908:\t909:\t909:\t909:\t909:\t909:\t910:\t910:\t910:\t910:\t910:\t911:\t911:\t911:\t911:\t911:\t912:\t912:\t912:\t912:\t912:\t913:\t913:\t913:\t913:\t913:\t914:\t914:\t914:\t914:\t914:\t915:\t915:\t915:\t915:\t915:\t916:\t916:\t916:\t916:\t916:\t917:\t917:\t917:\t917:\t917:\t918:\t918:\t918:\t918:\t918:\t919:\t919:\t919:\t919:\t919:\t920:\t920:\t920:\t920:\t920:\t921:\t921:\t921:\t921:\t921:\t922:\t922:\t922:\t922:\t922:\t923:\t923:\t923:\t923:\t923:\t924:\t924:\t924:\t924:\t924:\t925:\t925:\t925:\t925:\t925:\t926:\t926:\t926:\t926:\t926:\t927:\t927:\t927:\t927:\t927:\t928:\t928:\t928:\t928:\t928:\t929:\t929:\t929:\t929:\t929:\t930:\t930:\t930:\t930:\t930:\t931:\t931:\t931:\t931:\t931:\t932:\t932:\t932:\t932:\t932:\t933:\t933:\t933:\t933:\t933:\t934:\t934:\t934:\t934:\t934:\t935:\t935:\t935:\t935:\t935:\t936:\t936:\t936:\t936:\t936:\t937:\t937:\t937:\t937:\t937:\t938:\t938:\t938:\t938:\t938:\t939:\t939:\t939:\t939:\t939:\t940:\t940:\t940:\t940:\t940:\t941:\t941:\t941:\t941:\t941:\t942:\t942:\t942:\t942:\t942:\t943:\t943:\t943:\t943:\t943:\t944:\t944:\t944:\t944:\t944:\t945:\t945:\t945:\t945:\t945:\t946:\t946:\t946:\t946:\t946:\t947:\t947:\t947:\t947:\t947:\t948:\t948:\t948:\t948:\t948:\t949:\t949:\t949:\t949:\t949:\t950:\t950:\t950:\t950:\t950:\t951:\t951:\t951:\t951:\t951:\t952:\t952:\t952:\t952:\t952:\t953:\t953:\t953:\t953:\t953:\t954:\t954:\t954:\t954:\t954:\t955:\t955:\t955:\t955:\t955:\t956:\t956:\t956:\t956:\t956:\t957:\t957:\t957:\t957:\t957:\t958:\t958:\t958:\t958:\t958:\t959:\t959:\t959:\t959:\t959:\t960:\t960:\t960:\t960:\t960:\t961:\t961:\t961:\t961:\t961:\t962:\t962:\t962:\t962:\t962:\t963:\t963:\t963:\t963:\t963:\t964:\t964:\t964:\t964:\t964:\t965:\t965:\t965:\t965:\t965:\t966:\t966:\t966:\t966:\t966:\t967:\t967:\t967:\t967:\t967:\t968:\t968:\t968:\t968:\t968:\t969:\t969:\t969:\t969:\t969:\t970:\t970:\t970:\t970:\t970:\t971:\t971:\t971:\t971:\t971:\t972:\t972:\t972:\t972:\t972:\t973:\t973:\t973:\t973:\t973:\t974:\t974:\t974:\t974:\t974:\t975:\t975:\t975:\t975:\t975:\t976:\t976:\t976:\t976:\t976:\t977:\t977:\t977:\t977:\t977:\t978:\t978:\t978:\t978:\t978:\t979:\t979:\t979:\t979:\t979:\t980:\t980:\t980:\t980:\t980:\t981:\t981:\t981:\t981:\t981:\t982:\t982:\t982:\t982:\t982:\t983:\t983:\t983:\t983:\t983:\t984:\t984:\t984:\t984:\t984:\t985:\t985:\t985:\t985:\t985:\t986:\t986:\t986:\t986:\t986:\t987:\t987:\t987:\t987:\t987:\t988:\t988:\t988:\t988:\t988:\t989:\t989:\t989:\t989:\t989:\t990:\t990:\t990:\t990:\t990:\t991:\t991:\t991:\t991:\t991:\t992:\t992:\t992:\t992:\t992:\t993:\t993:\t993:\t993:\t993:\t994:\t994:\t994:\t994:\t994:\t995:\t995:\t995:\t995:\t995:\t996:\t996:\t996:\t996:\t996:\t997:\t997:\t997:\t997:\t997:\t998:\t998:\t998:\t998:\t998:\t999:\t999:\t999:\t999:\t999:\t"
     ]
    }
   ],
   "source": [
    "## run cv to get best iteration\n",
    "ctb_cv = cv(param, my_dt, fold_count=5, random_seed=2017)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# fetch best round\n",
    "best_round = ctb_cv['b\\'Accuracy\\'_test_avg'].index(np.max(ctb_cv['b\\'Accuracy\\'_test_avg']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## define the classifer model\n",
    "model = CatBoostClassifier(iterations=best_round, learning_rate=0.03,rsm = 0.3 ,depth=6, eval_metric='Accuracy', random_seed=2017)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<catboost.core.CatBoostClassifier at 0x7f3478039080>"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## train model\n",
    "model.fit(my_dt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## make predictions\n",
    "preds = model.predict(test_feats1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## make submission\n",
    "sub5 = pd.DataFrame({'User_ID':test.User_ID, 'Is_Response':preds})\n",
    "sub5['Is_Response'] = ['happy' if x == 1 else 'not_happy' for x in sub5['Is_Response']]\n",
    "sub5 = sub5[['User_ID','Is_Response']]\n",
    "sub5.to_csv('submissions/sub5_cb.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}