2.rental-listing-text-xgb.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This time we try some feature engineering on the text fields and combine the results with the otehr numerical columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "# plt.rcdefaults()\n",
    "mpl.style.use('additional')\n",
    "\n",
    "def print_df_max_colwidth(df, width=200):\n",
    "    with pd.option_context('display.max_colwidth', width):\n",
    "        print(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Warmup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>building_id</th>\n",
       "      <th>created</th>\n",
       "      <th>description</th>\n",
       "      <th>display_address</th>\n",
       "      <th>features</th>\n",
       "      <th>interest_level</th>\n",
       "      <th>latitude</th>\n",
       "      <th>listing_id</th>\n",
       "      <th>longitude</th>\n",
       "      <th>manager_id</th>\n",
       "      <th>photos</th>\n",
       "      <th>price</th>\n",
       "      <th>street_address</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.500000</td>\n",
       "      <td>3</td>\n",
       "      <td>53a5b119ba8f7b61d4e010512e0dfc85</td>\n",
       "      <td>2016-06-24 07:54:24</td>\n",
       "      <td>A Brand New 3 Bedroom 1.5 bath ApartmentEnjoy ...</td>\n",
       "      <td>Metropolitan Avenue</td>\n",
       "      <td>[]</td>\n",
       "      <td>medium</td>\n",
       "      <td>40.714500</td>\n",
       "      <td>7211212</td>\n",
       "      <td>-73.942500</td>\n",
       "      <td>5ba989232d0489da1b5f2c45f6688adc</td>\n",
       "      <td>[https://photos.renthop.com/2/7211212_1ed4542e...</td>\n",
       "      <td>3000</td>\n",
       "      <td>792 Metropolitan Avenue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10000</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>c5c8a357cba207596b04d1afd1e4f130</td>\n",
       "      <td>2016-06-12 12:19:27</td>\n",
       "      <td></td>\n",
       "      <td>Columbus Avenue</td>\n",
       "      <td>[Doorman, Elevator, Fitness Center, Cats Allow...</td>\n",
       "      <td>low</td>\n",
       "      <td>40.794700</td>\n",
       "      <td>7150865</td>\n",
       "      <td>-73.966700</td>\n",
       "      <td>7533621a882f71e25173b27e3139d83d</td>\n",
       "      <td>[https://photos.renthop.com/2/7150865_be3306c5...</td>\n",
       "      <td>5465</td>\n",
       "      <td>808 Columbus Avenue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100004</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>c3ba40552e2120b0acfc3cb5730bb2aa</td>\n",
       "      <td>2016-04-17 03:26:41</td>\n",
       "      <td>Top Top West Village location, beautiful Pre-w...</td>\n",
       "      <td>W 13 Street</td>\n",
       "      <td>[Laundry In Building, Dishwasher, Hardwood Flo...</td>\n",
       "      <td>high</td>\n",
       "      <td>40.738800</td>\n",
       "      <td>6887163</td>\n",
       "      <td>-74.001800</td>\n",
       "      <td>d9039c43983f6e564b1482b273bd7b01</td>\n",
       "      <td>[https://photos.renthop.com/2/6887163_de85c427...</td>\n",
       "      <td>2850</td>\n",
       "      <td>241 W 13 Street</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100007</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>28d9ad350afeaab8027513a3e52ac8d5</td>\n",
       "      <td>2016-04-18 02:22:02</td>\n",
       "      <td>Building Amenities - Garage - Garden - fitness...</td>\n",
       "      <td>East 49th Street</td>\n",
       "      <td>[Hardwood Floors, No Fee]</td>\n",
       "      <td>low</td>\n",
       "      <td>40.753900</td>\n",
       "      <td>6888711</td>\n",
       "      <td>-73.967700</td>\n",
       "      <td>1067e078446a7897d2da493d2f741316</td>\n",
       "      <td>[https://photos.renthop.com/2/6888711_6e660cee...</td>\n",
       "      <td>3275</td>\n",
       "      <td>333 East 49th Street</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100013</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2016-04-28 01:32:41</td>\n",
       "      <td>Beautifully renovated 3 bedroom flex 4 bedroom...</td>\n",
       "      <td>West 143rd Street</td>\n",
       "      <td>[Pre-War]</td>\n",
       "      <td>low</td>\n",
       "      <td>40.824100</td>\n",
       "      <td>6934781</td>\n",
       "      <td>-73.949300</td>\n",
       "      <td>98e13ad4b495b9613cef886d79a6291f</td>\n",
       "      <td>[https://photos.renthop.com/2/6934781_1fa4b41a...</td>\n",
       "      <td>3350</td>\n",
       "      <td>500 West 143rd Street</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        bathrooms  bedrooms                       building_id              created                                        description      display_address                                           features interest_level  latitude  listing_id  longitude                        manager_id                                             photos  price           street_address\n",
       "10       1.500000         3  53a5b119ba8f7b61d4e010512e0dfc85  2016-06-24 07:54:24  A Brand New 3 Bedroom 1.5 bath ApartmentEnjoy ...  Metropolitan Avenue                                                 []         medium 40.714500     7211212 -73.942500  5ba989232d0489da1b5f2c45f6688adc  [https://photos.renthop.com/2/7211212_1ed4542e...   3000  792 Metropolitan Avenue\n",
       "10000    1.000000         2  c5c8a357cba207596b04d1afd1e4f130  2016-06-12 12:19:27                                                         Columbus Avenue  [Doorman, Elevator, Fitness Center, Cats Allow...            low 40.794700     7150865 -73.966700  7533621a882f71e25173b27e3139d83d  [https://photos.renthop.com/2/7150865_be3306c5...   5465      808 Columbus Avenue\n",
       "100004   1.000000         1  c3ba40552e2120b0acfc3cb5730bb2aa  2016-04-17 03:26:41  Top Top West Village location, beautiful Pre-w...          W 13 Street  [Laundry In Building, Dishwasher, Hardwood Flo...           high 40.738800     6887163 -74.001800  d9039c43983f6e564b1482b273bd7b01  [https://photos.renthop.com/2/6887163_de85c427...   2850          241 W 13 Street\n",
       "100007   1.000000         1  28d9ad350afeaab8027513a3e52ac8d5  2016-04-18 02:22:02  Building Amenities - Garage - Garden - fitness...     East 49th Street                          [Hardwood Floors, No Fee]            low 40.753900     6888711 -73.967700  1067e078446a7897d2da493d2f741316  [https://photos.renthop.com/2/6888711_6e660cee...   3275     333 East 49th Street\n",
       "100013   1.000000         4                                 0  2016-04-28 01:32:41  Beautifully renovated 3 bedroom flex 4 bedroom...    West 143rd Street                                          [Pre-War]            low 40.824100     6934781 -73.949300  98e13ad4b495b9613cef886d79a6291f  [https://photos.renthop.com/2/6934781_1fa4b41a...   3350    500 West 143rd Street"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.read_json(\"train.json\")\n",
    "test_df = pd.read_json(\"test.json\")\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Redo feature engineering and random forest model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def engineer_features1(df):\n",
    "    price_max = np.percentile(df.price, 99)\n",
    "    df.loc[df.price > price_max, 'price'] = price_max\n",
    "    latitude_max = np.percentile(df.latitude, 99)\n",
    "    latitude_min = np.percentile(df.latitude, 1)\n",
    "    df.loc[df.latitude > latitude_max, 'latitude'] = latitude_max\n",
    "    df.loc[df.latitude < latitude_min, 'latitude'] = latitude_min\n",
    "    longitude_max = np.percentile(df.longitude, 99)\n",
    "    longitude_min = np.percentile(df.longitude, 1)\n",
    "    df.loc[df.longitude > longitude_max, 'longitude'] = longitude_max\n",
    "    df.loc[df.longitude < longitude_min, 'longitude'] = longitude_min\n",
    "    df[\"num_photos\"] = df.photos.apply(len)\n",
    "    df[\"num_features\"] = df.features.apply(len)\n",
    "    df[\"num_description_words\"] = df.description.apply(lambda x: len(x.split(\" \")))\n",
    "    df[\"created\"] = pd.to_datetime(df.created)\n",
    "    df[\"created_year\"] = df.created.dt.year\n",
    "    df[\"created_month\"] = df.created.dt.month\n",
    "    df[\"created_day\"] = df.created.dt.day\n",
    "\n",
    "engineer_features1(train_df)\n",
    "engineer_features1(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\n",
    "from sklearn.model_selection import GridSearchCV, cross_val_score, train_test_split\n",
    "from sklearn.metrics import classification_report, confusion_matrix, log_loss\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "X_vars = ['bathrooms', 'bedrooms', 'latitude', 'longitude', 'price', 'num_photos', 'num_features', 'num_description_words', 'created_year', 'created_month', 'created_day']\n",
    "X_train, X_test, y_train, y_test = train_test_split(train_df[X_vars], train_df['interest_level'], stratify=train_df['interest_level'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "n_estimators = np.logspace(1, 3, 5).astype(int)\n",
    "rf = GridSearchCV(RandomForestClassifier(n_jobs=-1), {'n_estimators': n_estimators},\n",
    "                 scoring='neg_log_loss').fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x14361bef0>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfYAAAGRCAYAAACXGpFeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAASdAAAEnQB3mYfeAAAIABJREFUeJzt3XmcXFWd9/HPt/eEkBBIIgiYSAKyicryjAGVCAIyuCA4\njytR1FFAcUZRQR+VODDiAiguLLNIjMuooAiishNAiWIURkGBEAirZE9IQtLpdP+eP+6tdHWleq26\nt6or3/frVa9bde5Sp06W3z3nnkURgZmZmTWGplpnwMzMzKrHgd3MzKyBOLCbmZk1EAd2MzOzBuLA\nbmZm1kAc2M3MzBqIA7uZmVkDcWA3MzNrIA7sZmZmDcSB3czMrIE4sJuZmTUQB3YzM7MG4sBuZmbW\nQFpqnYF6ImkCcCTwJLC5xtkxM7PtUxuwJ3BHRKwd7skO7H0dCVxb60yYmZkBbwauG+5JDux9PQnw\n85//nBkzZtQ6L2Zmth165JFHOPHEEyGNScPlwN7XZoAZM2ZwwAEH1DovZma2fRvRI2F3njMzM2sg\nDuxmZmYNxIHdzMysgTiwm5mZNZC6DeyS9pC0RtKsQY5rkXSOpEWSNki6T9LbcsqmmZlZXanLXvGS\n9gRuBCYM4fA5wKeBfwN+A7wF+JGkLRHx08wyOYCu7h4WLlnN2o2bmTCmjUOnTaS1uW7voczMrIHU\nVWCX1ATMBi4ENMTT3gf8MCK+kH6+VdIhwEeAXAN7V3cPl81fzLwFS1ixvneUwuRx7Zwycyqnz5ru\nAG9mZpmqtyhzEHA5MA84ZYjndADPlaStBHapYr4G1dXdwwfnLeTimx9m5fq+Qw9XrO/k4psf5kPf\n+yNd3T15ZsvMzLYz9RbYnwBmRMTHgeeHeM7XgdmSXi9pvKR3Aa8HvjfQSZKmSDqg+AVMH2nGL5u/\nmNsfWg5AlOwrfL7twWVcPn/xSL/CzMxsUHXVFB8Rq4BVwzzta8BM4NdFad+JiK8Oct4ZwLnD/K6y\nurp7mLdgCWLboF5MwLwFj3Oam+TNzCwjdRXYh0tSO3AXsBtwGvAgcDjwWUnrI+JfBjj9UuCqkrTp\njGARmIVLVvd5pt6fAJav72ThktXMnJ7rkwIzM9tOjOrADpwMvAw4JiJuSdPukLQW+Lak/4yI+8ud\nGBHLgGXFadJQ++v1tXbj8KbzHe7xZmZmQzXa24OnptvflqTfmW5zWcllwpi2TI83MzMbqtEe2B9M\nt68uST8i3T6aRyYOnTaRSePaBh2fJ5Khb4dOm5hHtszMbDs0qgJ72uv9lZImp0nXAb8Hvi/pdEmv\nlXQOcBFwXUT8IY98tTY3MXvmtAE7zkHyjH32zKnuOGdmZpkZbRHmYGABcAJARHQDxwI/Bj5H0jN+\nNnA+8E95Zuz0WdM5at8pwLYz6xQ+H7XvFE6bNeIRdWZmZoOq285zETGfkhjZT9pzwJnpq2Zam5u4\n4pRDuHz+YuYteJzl6zu37ps0rp3ZM6d6mJuZmWWubgP7aNTa3MSZR+/NabOmc+b//Ikb7l9KR2sT\nd3/6KAd0MzPLhaNNBlqbm3jxpHEAbOkOB3UzM8uNI05GOlqaAdjSE2zx/PBmZpYTB/aMdLT2Fm3n\nFgd2MzPLhwN7Rtpbeot2U1d3DXNiZmbbEwd2QNIcSQGUnX52JDpam7e+3+Qau5mZ5cSBHYiIOREh\n4MBqXbNPYHeN3czMcuLAnpE+z9i7XGM3M7N8OLBnpL1PU7xr7GZmlg8H9owUhruBm+LNzCw/DuwZ\naXdTvJmZ1YADe0ZcYzczs1pwYM9Icec5P2M3M7O8OLBnpHi4m5vizcwsLw7sGfE4djMzqwUH9oz0\nbYp3jd3MzPLhwJ6RdneeMzOzGnBgz0hzk2htFgCb/IzdzMxy4sBONovAQO+QN9fYzcwsLw7sZLMI\nDPROK+v12M3MLC8O7BkqdKDrdI3dzMxy4sCeofaWpHg9QY2ZmeXFgT1DhbHs7jxnZmZ5cWDPUG9g\nd43dzMzy4cCeoa3P2N15zszMcuLAniEPdzMzs7w5sGfITfFmZpY3B/YMbe0V785zZmaWEwf2DPVO\nUOMau5mZ5cOBPUOFznOusZuZWV4c2DPU4Rq7mZnlzIGd7BeB6eoOunuimpc2MzMry4Gd7BaBKTTF\ng3vGm5lZPhzYM1ToFQ8O7GZmlg8H9gwVnrEDbPLsc2ZmlgMH9gwVB3Yv3WpmZnlwYM9Q32fsrrGb\nmVn2HNgz1N6nKd41djMzy54De4bcec7MzPJWt4Fd0h6S1kiaNYRjT5B0j6SNkp6SdImkHXLI5oD6\nPmN3U7yZmWWvLgO7pD2Bm4AJQzj2jcB1wAPACcCXgFOB/8wyj0NRmKAGXGM3M7N8tNQ6A8UkNQGz\ngQsBDfG0rwFXR8Sp6efbJDUDH5U0NiKezyCrQ1Lcea7Tw93MzCwH9VZjPwi4HJgHnDLYwZJeAUwH\nvlmcHhGXRMT0WgZ1KBnH7hq7mZnloK5q7MATwIyIeGooz9aBl6fbTZKuB44GNpLcGJwdEZ39nShp\nCjC5JHn68LPcPwd2MzPLW10F9ohYBawaximFwHwN8EPgIuAw4AvAFOCdA5x7BnDuCLI5ZH16xbsp\n3szMclBXgX0E2tLtNRFxdvr+9vRZ/QWS5kTEw/2ceylwVUnadODaamXONXYzM8vbaA/s69Lt9SXp\nNwAXAK8Aygb2iFgGLCtOk4baX29omptEa7Po6g7PPGdmZrmot85zw7Uo3baXpLem24055qWswpC3\nTs88Z2ZmORjtgf1OYAPwjpL0NwFbgAW556hEYVpZ19jNzCwPo6opXtJ4YH9gcUQsj4j1kj4PXCRp\nNfAz4HDgbOCSiFhew+wCvWPZvbqbmZnlYbTV2A8mqYWfUEiIiIuB9wFHAr9K358LfKoWGSxV6Bnv\nRWDMzCwPdVtjj4j5lMw+Vy4tTb8SuDKXjA1Th5vizcwsR6Otxj7qFAK7O8+ZmVkeHNgzVnjG7hq7\nmZnlwYE9Y4Xhbp6gxszM8uDAnrH2rTV2B3YzM8ueA3vGemvsboo3M7PsObADkuZICuD+al+73Z3n\nzMwsRw7sQETMiQgBB1b72r0T1LjGbmZm2XNgz9jWceyusZuZWQ4c2DNWeMbe1R1090SNc2NmZo3O\ngT1jhV7x4J7xZmaWPQf2jHW0OLCbmVl+HNgzVnjGDtC5xR3ozMwsWw7sGSsO7K6xm5lZ1hzYM9bR\n5xm7a+xmZpYtB/aMtRfX2D3kzczMMubAnrF2d54zM7McObBnrE/nOTfFm5lZxhzYyXau+MIENeD5\n4s3MLHsO7OQzVzy485yZmWXPgT1jHu5mZmZ5cmDPmDvPmZlZnhzYM9anxu6Z58zMLGMO7BlzU7yZ\nmeXJgT1jzU2itVmA54o3M7PsObDnoDDkzTV2MzPLmgN7DgrTynq4m5mZZc2BPQeFnvGdrrGbmVnG\nHNhzUJikxovAmJlZ1hzYc1DoGe+54s3MLGsO7DkoBHbX2M3MLGsO7GS7CAwUNcW7xm5mZhlzYCfb\nRWDAw93MzCw/Duw5aN9aY3dgNzOzbDmw56C3xu6meDMzy5YDew4KE9R4SlkzM8uaA3sOCp3nPEGN\nmZllzYE9Bx7uZmZmeXFgz0FhStmu7qC7J2qcGzMza2R1G9gl7SFpjaRZwzinRdI9kuZnl7Ph85rs\nZmaWl7oM7JL2BG4CJgzz1HOAw6qfo8p0tPQWszvQmZlZluoqsEtqkvRe4F7gBcM892XAZ4BnM8ha\nRVxjNzOzvNRVYAcOAi4H5gGnDPUkSW3pOd8AHsomayPnwG5mZnlpqXUGSjwBzIiIp4bzbB34PNAK\nnAvcOJQTJE0BJpckTx/Gdw5ZYbgbeJIaMzPLVl0F9ohYBawazjmSDgM+AbwmIjolDfXUM0huBDLX\n3lJUY/eQNzMzy1BdBfbhktQBfBf4ekTcM8zTLwWuKkmbDlxbjbwVa+9TY3dgNzOz7IzqwA6cT9JP\n4DxJhd8iSIa+Ad0RUXbgeEQsA5YVpw2jtj8sxc/Y3SvezMyyNNoD+1uBqcD6Mvu6gFOBuXlmqJyO\noqZ4TytrZmZZGu2B/Y1Ae0naFen2Q8Bj+WanPHeeMzOzvIyqwC5pPLA/sDgilkfEX8ocsw4gIhbm\nnb/+eLibmZnlpd7GsQ/mYGABcEKtMzIc7S3uPGdmZvmo2xp7RMwn7Qg3UFqZ82ZllqkR6lNjd+c5\nMzPL0GirsY9KfXrF+xm7mZllyIE9B81NorU5aWjwBDVmZpYlB/acFIa8+Rm7mZllyYE9J4XZ5zzc\nzczMsuTAnpPCfPGeoMbMzLLkwA5ImiMpgPuz+o7CJDWeUtbMzLLkwA5ExJyIEHBgVt9R6BnvZ+xm\nZpYlB/acbA3s7hVvZmYZqiiwS9pL0n7p+wmSvinpOkmnVCd7jaPDnefMzCwHIw7sko4HHgTenyZd\nAZwG7AHMlfT+/s7dHrV7uJuZmeWgkhr754AbgS9I2gl4C3BBRBwMXAD8SxXy1zB6a+wO7GZmlp1K\nAvvLgK9HxDrgeJJ5569O990M7F1h3hpKYYIa94o3M7MsVRLYN9K7iMxxwNKI+HP6eVdgTSUZazTt\nW3vFO7CbmVl2Klnd7bfAJyRNBN4KzAWQdAhwLvCbinPXQLaOY3dTvJmZZaiSGvvHSDrK/RBYApyf\npv8S6ADOqShnDcbD3czMLA8jrrFHxKOS9gemRMTSol0nAvdGRGfFuWsg7S3JPVRXd9DdEzQ3Dbis\nvJmZ2YhUNI49IgJYX/gs6WTgcGDPCvPVcPqsye5au5mZZaSScewvkfQIaZO7pPOAnwAXAn+WdER1\nstgYOlp6i9od6MzMLCuV1Ni/DHQB10pqAz5MEth3Am6g95l73ctnEZjeGrvHspuZWVYqCeyvAT4d\nEQuBWcAE4IqIeA64HDi08uzlI89FYMCB3czMslNJYG8FVqfvjwc20DvErRnYUsG1G067m+LNzCwH\nlQT2+4GTJO0K/BNwU0RskdQKfAT4SzUy2Cj61Njdec7MzDJSyQQ1nwd+ThLEO4EvpemLgBcAb6ws\na42lvbX3HqrTNXYzM8tIJePYb5Z0IPB/gN9FxOPprq8Bt0WEa+xFXGM3M7M8VFJjJyIeAx6TtK+k\nVwIrIuKS6mStsRQWgQFPK2tmZtmpaIIaSe+Q9DTwAMnc8Q9JelrS7KrkroF0tLrznJmZZW/ENXZJ\nbwS+D9wGfAZ4Fngh8G7gSkkrI+KXVcllA2j3cDczM8tBJU3xnwWuioi3l6RfKelHwKdJFoQxSmee\nc2A3M7NsVNIU/1LSpVrLmAu8rIJrN5y+c8W7Kd7MzLJRSWBfAezcz75dgM0VXLvh9J15zoHdzMyy\nUUlgvwWYI2mP4kRJewLnAjdVkrFG09wkWpuTpVo93M3MzLJSSWD/DDAWWCTpVkk/kHQr8DCwA+mq\nb6NBHovAQO+QNz9jNzOzrIw4sEfEs8DBwDdIAvlh6fYbwMFFE9bUvTwWgYHe2efcFG9mZlmpdIKa\nZcDZVcpLw2tPa+ydboo3M7OMVDRBTX8knSnJnedKFCap8VzxZmaWlUwCe3rd5kGP2s4Uesb7GbuZ\nmWUlq8BuZWwN7G6KNzOzjDiw56i9xZ3nzMwsWw7sOXJTvJmZZW1YveIlvWiIh/Y3I91wvmsPknHl\nJ0bE/AGOawfOAmYDewJPAT8AvhQRddWBb2vnOU8pa2ZmGRnucLclQAzhOA3xuPInJ7PX3QhMGMLh\nlwCnAOcBfwAOJZn5birw/pHmIQueoMbMzLI23MD+PioI2IOR1ERS876Q5OZgsON3AT4InB0RX02T\nb5UE8CVJ50TE8qzyO1ztW5viXWM3M7NsDCuwR8TcjPJRcBBwOXApyVz0gy37Oj49/rqS9AfT7V5A\n3QT23nHsrrGbmVk2Kpp5LgNPADMi4ilJswY7OCIeA84os+tEoItk3vqyJE0BJpckTx96VoevMPOc\nh7uZmVlW6iqwR8QqYFUl15D0FuA9wLciYvUAh55B8iw+N4Uae1d30N0TNDcN+rTBzMxsWOoqsFdK\n0knAD4HfAJ8a5PBLgatK0qYD12aQNaDvmuydW7oZ29ZQxW9mZnWgYSKLpI+RdLqbTzJEbtNAx6cL\n2CwruUZm+QPoaOmdNmBTVw9j2zL9OjMz2w6N+glqlPgGcDHwY+D4iFhX42yVVVxj95A3MzPLwohr\n7JK+M8DuHmA9sAj4UUSsHOn3DMEXgTNJAvsnIiKz4XiVcmA3M7OsVdIUvydwBNBBMnHNUmAK8GKg\nG3g2/fxZSUdExKOVZRUkjQf2BxZHxHJJLydZD/4PJM/L/6GkOf2vEfFcpd9bLe0lTfFmZmbVVklT\n/C+AtcDhEbFXRMyMiOnAISTTup5HEtgfIalVV8PBwALghPTzSSQT2RyWppe+Dq7S91ZFaec5MzOz\naqukxv5x4JyI+F1xYkTcK+lzwPkR8R+SLiaZRGZY0vnhNVBaRHwe+Pzws14b7a2usZuZWbYqqbFP\nov9Z3VaT1NYBVgDjKviehtHnGbtr7GZmloFKAvufgE+mq6ttJamNZLW1+9Kkg0lmlNvuFRaBAU8r\na2Zm2aikKf5skvncH5X0K5Ix4S8AjidZle04Sa8CLgDOrzSjjcBN8WZmlrUR19gjYgFJR7lbgdeT\n1NJnATcAL4uI3wJtwOcjolqd50Y1D3czM7OsVTTzXEQ8SLLMan/7bwNuq+Q78iBpDjnMG18881zn\nFtfYzcys+iqaeU7SJElfkvQ7SQ9K+o2kC9KV00aNiJgTEQIOzPJ7XGM3M7OsjTiwS9oDuBf4V2Bj\n+n4LyTC4eyXtXpUcNpC+gd01djMzq75KmuK/TLLm+f7Fs8pJ2gu4Cfh34L0V5a7BNDeJ1mbR1R0e\n7mZmZpmopCn+OJKOcX2mik0/f4Gkd7yVaE+HvLkp3szMslBJYG8hmXymnOXA+Aqu3bA60iFv7jxn\nZmZZqCSw/xl4Vz/7TgH+UsG1G5Zr7GZmlqVKnrGfB9woaWfgRySrue0KvIOkmf6tlWev8Wytsbvz\nnJmZZWDEgT0ibpb0HpJOdMXP058F3hcR11SauUZU6BnvGruZmWWh0glqvifp+8BLgJ2BVcBDERHV\nyFwj2hrY3SvezMwyUFFgB0iD+IPFaZKOBt4WER+s9PqNpj2dfc7j2M3MLAsVzTw3gAOB92d07VGt\nUGPvdI3dzMwykFVgt34UOs+5xm5mZllwYCdZBEZSAPdn/V0dHu5mZmYZcmAnv0VgANq39op3jd3M\nzKrPgT1nvePYXWM3M7PqG1aveElDXVt9zxHkZbuwdeY5d54zM7MMDHe4WxMwlDHqT6UvK1GosXd1\nB909QXOTapwjMzNrJMMK7BExK6N8bDeK12Tv3NLN2LaKpxIwMzPbys/Yc9bR0lvk7kBnZmbV5sCe\ns+Iau4e8mZlZtTmw56y9tbjG7sBuZmbV5cCes8IENQCdW9wUb2Zm1eXAnjM3xZuZWZYc2HPWtyne\nNXYzM6suB/ac9amxe5IaMzOrMgd2arMIDHhaWTMzqz4HdvJeBMZN8WZmlh0H9pyVzjxnZmZWTQ7s\nOfPMc2ZmliUH9px5uJuZmWXJgT1nfQO7a+xmZlZdDuw5a24Src3JUq0e7mZmZtXmwF4D7emQt07X\n2M3MrMrqNrBL2kPSGkmzhnDsOyQ9IGmjpL9Jek8OWRyxjnTIm2vsZmZWbXUZ2CXtCdwETBjCsScD\nP0iPPxGYD8yV9PYs81iJQo3dnefMzKzaWmqdgWKSmoDZwIWAhnjaF4GrIuJj6ecbJe0MnAf8qPq5\nrFyhxu6meDMzq7Z6q7EfBFwOzANOGexgSdOAfYBrSnZdDcyQtHeV81cVhZ7xrrGbmVm11VWNHXgC\nmBERTw3l2TqwX7p9uCT9kXT7EmBRuRMlTQEmlyRPH2I+K9Le4mfsZmaWjboK7BGxClg1jFMKz+Cf\nK0lfl27HD3DuGcC5w/iuqinU2N0Ub2Zm1VZXgX0EBnuUMFDkvBS4qiRtOnBtRTkagq1N8a6xm5lZ\nlY32wL423e5Ykj6+ZP82ImIZsKw4TRpqf73KbB3u5hq7mZlVWb11nhuuh9LtjJL0wue/5ZiXIevw\ncDczM8vIqA7sEfEI8Bjw1pJdJwOLImJJ7pkagvatveJdYzczs+oaVU3xksYD+wOLI2J5mvxvwJWS\nVgLXAW8G/i9QxxPUFMaxu8ZuZmbVNdpq7AcDC4ATCgkRMRc4DTgG+DlwJDA7In5ciwwOxdZe8Vtc\nYzczs+qq2xp7RMynZPa5cmlp+hXAFblkrAoKnec2d/fQ3RM0N+XTac/MzBrfaKuxN4TiNdk7PeTN\nzMyqyIG9BjpaeovdHejMzKyaHNhroLjG7iFvZmZWTQ7sNdDe2lvs7kBnZmbV5MBeA4UJasA1djMz\nqy4HdkDSHEkB3J/H97kp3szMsuLADkTEnIgQcGAe31fcFO/Oc2ZmVk0O7DXQp8bu4W5mZlZFDuw1\n0F403M3TypqZWTU5sNdA3wlq3BRvZmbV48BeA+48Z2ZmWXFgrwHPPGdmZllxYK8B19jNzCwrDuw1\n0O4au5mZZcSBvQZamptoSZdq9epuZmZWTQ7sNVJojneN3czMqsmBvUbaW5Ia+6Jl61iweCVd3Q7w\nZmZWuZZaZ2B709Xdw2XzF7P6+S4A7lq0grsWrWDyuHZOmTmV02dNp7XZ91tmZjYyjiDktwhMV3cP\nH5y3kItvfpie6LtvxfpOLr75YT70vT+69m5mZiPmwE5+i8BcNn8xtz+0vHwe0u1tDy7j8vmLs8yG\nmZk1MAf2nHR19zBvwRI0yHEC5i143LV2MzMbEQf2nCxcspoV6zcTgxwXwPL1nSxcsjqPbJmZWYNx\nYM/J2o2bMz3ezMwMHNhzM2FMW6bHm5mZgQN7bg6dNpFJ49qG9Ix98rh2Dp02MY9smZlZg3Fgz0lr\ncxOzZ04b0jP22TOneiy7mZmNiKNHjk6fNZ2j9p0C0G/N/ah9p3DarOn5ZcrMzBqKA3uOWpubuOKU\nQzjrmH2YNK59m/2v2HMnrjjlENfWzcxsxDylbM5am5s48+i9OW3WdBYuWc3ajZv51m2PcP8zz/HA\nM8+xbF0nu+80ptbZNDOzUcpVwxppbW5i5vRdeP2Bu3H+W14KwObuHr5xy6Ia58zMzEYzB/Y68PI9\nd+J1+70AgKv/9BSPrdhQ4xyZmdlo5cBOfovADOSsY/dBgu6e4Gs3P1yrbJiZ2SjnwE5+i8AMZL/d\nxvOGg14IwC/+/AwPPvtcrbJiZmajmAN7HfnY6/amuUlEwEU3udZuZmbD58BeR/aaPI6TD94dgJv/\nupT7nlxT4xyZmdlo48BeZz569N60NifT11x000M1zo2ZmY02Dux1Zo+JY3nn/3kRAHctWsHvHl1Z\n4xyZmdlo4sBehz581Aw6WpM/mgtvfIiIwWaYNzMzS9RdYJd0rKQ/SHpe0mOSPiGp30XRJLVIOkfS\nIkkbJN0n6W155rnapuzYwXsOnwbAwsdXM//h5bXNkJmZjRp1FdglvRK4HngQOAn4AfAV4OwBTpsD\n/DvwfeBNwG+AH0k6OdPMZuy010xnx/Zkxt+LbnKt3czMhqauAjvwBeDeiDglIm6IiM8CXwU+I6m/\nCdTfB/wwIr4QEbdGxEeA3wEfySnPmZi4Qxvvf/WLAbj/6ee44f5na5wjMzMbDeomsEtqB2YB15Ts\nuhrYEXhVP6d2AKWzuawEdqlm/mrh/a96MRPHtgJw0c0P093jWruZmQ2sbgI7sBfQBpTOzPJIun1J\nP+d9HZgt6fWSxkt6F/B64HsDfZmkKZIOKH4BdbUQ+o4drZx2ZJKlR5at59r7nq5xjszMrN7VU2Cf\nkG5La9/r0u34fs77Gslz9V8Da0metX83Ir46yPedQTI3fPHr2mHmOXOzZ05j8o7J2u1fv2URXd09\nNc6RmZnVs3oK7IPlZZuIljbf3wUcDJxG0pT/GeDtki4Z5HqXkswNX/x68/CynL0xbc2cedQMAJ5Y\n9Tw/WfhkjXNkZmb1rKXWGSiyNt3uWJI+vmR/sZOBlwHHRMQtadodktYC35b0nxFRdsW2iFgGLCtO\nG2BUXU29/bAXccUdj/L0mo1849ZFnHzwHnS0Ntc6W2ZmVofqqca+GOgGZpSkFz7/rcw5U9Ptb0vS\n70y3B1Qna7XV1tLEv7xubwCWPtfJ93/3eI1zZGZm9apuAntEbCIJyCeVTEhzMklt/Z4ypz2Ybl9d\nkn5Eun20qpmsoZNesTt7Td4BgEvnL2Z955Ya58jMzOpR3QT21PnAPwA/kXS8pPOATwJfjIjn017v\nr5Q0OT3+OuD3wPclnS7ptZLOAS4CrouIP9TkV2SgpbmJj71uHwBWbdjMlb95rMY5MjOzelRXgT0i\nbiOpob8E+DnwLuCTEfGV9JCDgQXACenx3cCxwI+Bz5H0jJ9NcoPwT7lmPgcnvHQ39tst6XLwH3c9\nytrnu2qcIzMzqzd1FdgBIuKaiDgoItojYq+IuKho3/yIUETMLUp7LiLOjIgXRkRHROwfEV+KiM01\n+QEZamoSZx2T1NrXbdrCFXcurnGOzMys3tRdYLeBHb3fFF6+504AXPnbJSxf11njHJmZWT1xYB9l\nJPHJ45JJ+DZ2dXPp/EcGOcPMzLYnDuyj0BEzJjFzr2Qq/B/87gmeWbOxxjkyM7N64cA+Sn0irbVv\n7u7hm7ctqnFuzMysXjiwj1KHTJ3IUftOAeAnC59iyYoNNc6RmZnVAwd2QNIcSUGyEMyocdaxSQ/5\n7p7g67eULopnZmbbIwd2ICLmRIRIFoIZNQ544QROeOluAFz7v8/w0LPrBjnDzMwanQP7KPexY/ah\nSRABF9/8UK2zY2ZmNebAPsrNmDKOkw7eA4AbH1jKn59aU+McmZlZLTmwN4B/OXpvWpuTdXMuvMnP\n2s3MtmcO7A1gz53H8vbDXgTAnQ8v557HVtU4R2ZmVisO7A3iI0fNoL0l+eO88MaHiIga58jMzGrB\ngb1BvGB8B+85fBoA9yxZxZ2LVtQ2Q2ZmVhMO7A3ktCOnM669BYCLbnKt3cxse+TA3kB23qGN973q\nxQD8+altszERAAAXdklEQVS13PjA0hrnyMzM8ubA3mA+8OoXM2FMK5CMa+/uca3dzGx74sDeYMZ3\ntHLakdMBeHjpen7xv8/UOEdmZpYnB/YG9J7DpzJpXDuQ1Np/s2g5N9z/dxYsXklXd0+Nc2dmZllq\nqXUG6oGkOcC5tc5HtYxta+H0I/fivF/+jSdWbeTd/33P1n2Tx7VzysypnD5rOq3Nvq8zM2s0/p+d\n0bsITH+6unv6He62Yn0nF9/8MB/63h9dezcza0AO7A3osvmLuePh5WX3FbrS3fbgMi6fvzi/TJmZ\nWS4c2BtMV3cP8xYsQUM4du7dS+js6s46S2ZmliM/Y28wC5esZsX6zUM6duWGzRxw7o3sPnEMu+80\nhhfulGx3nziGPdLtbhPG0Nbi+z8zs9HCgb3BrN04tKBesKUneHzl8zy+8vmy+yWYsmN7b+AvCvq7\n7zSW3SeO2TrbnZmZ1Z7/R24wE8a0Dev4Nx60Gz3A06s38vSajSxf19lnfwQsfa6Tpc918qcnyq/1\nPmFM69bAv8fE3lp/YbvLDm1IQ3k4YGZmlXJgbzCHTpvIpHFtrFy/mYHmnBMwaVw7F7/t5X2GvW3q\n6ubZtZt4es1Gnl69kafS7dNrnufpNRv5+5pNbCmZzW7txi7Wbuzir39/rux3tbc09Q32JYF/1/Ed\ntHjonZlZVTiwN5jW5iZmz5zGxTc/POBxAcyeOXWbsewdrc1Mm7QD0ybtUPa87p5g+bpOnl7zPE+l\ntfynS7bPb+7bIa9zSw+PrtjAoys2lL1mk2DX8R19gn2hmb9wIzCmrXnohWBmth1zYG9Ap8+azn1P\nruG2B5ch6FNzL3w+at8pnDZr+rCv3dwkdp3Qwa4TOjhk6rb7I4K1G7vKBv1n1ibblRv69gPoCXhm\n7SaeWbuJP7C67PfuvENb2dp+IW2nsa2ZN/d3dfewcMlq1m7czIQxbRw6baIn+TGzuuPA3oBam5u4\n4pRDuHz+YuYteJzl63ufm08a187smVM5LaOZ5ySx09g2dhrbxoG7Tyh7zMbN3UmwL27mX72RZ9Yk\njwD+vnYjpWvXrNqwmVUbNvOXp9eWveYObc1bO/cVB/09JibP/qfs2EFz08gCf1d3D5fNX8y8BUv6\njDjwLH5mBvV30y+v2d1L0gHA/ffffz8HHHBArbNTFfX2F24otnT38Oxzm7ap7Re3AnRuGd6sea3N\nSUtDUsMf26d3/wt3GsMLd+qgvWXb5v6u7h4+OG8htz+0fMDWjytOOaTuy9XMqiurm/4HHniAAw88\nEODAiHhguOe7xt7gWpubmDl9l1pnY1hampvYY+JY9pg4tuz+iGDlhs1bA/8zazZu0/S/dmNXn3O6\nuoMnV23kyVUbgVVlrzs5HdZXHPT/+Phqbn8omcWv9Ba4dBa/M4/eu4JfbWajSelNf7HC1N33Pbmm\nJjf9Duw03iIwjU4Sk8a1M2lcOy/bc6eyx6zv3FLUm7+49p/07l+2rpPSxqrl6zpZvq6T+54sP6xv\nIN+8bRELH19FS1MTyaN+0aRkHgAhJGhKPiCS9+lHtPV94bje98lL6XHpeWXOEdDUlOyjz3HJMU3p\nh22/u++1SvO8zXcPkudtvrvwu9NzmkryrKJrbr1+n+8tl+e+7wt57i3zvnlmm/z3PUcleS4+Z5s8\nl5Z5cTk0leQpPY6S/Pf+Tg8BHc0um7+4bm/63RRfpBGb4q28zVt6+Pva8r36C8P6NnuRHMtB8c1I\n8c1f+Ru54puZ4huo8jd/AE1NfW8uCzd/29yYFOejqe8NTNkbsK03ciXXT+9Xem9wet/33kj1d/Pa\n95yBbv5Kb54LZQLb3rz2lusQbl6Ly7TMzauUdPi95JaH2bB54Cm5RdKv6e5PHzWsWrub4s1GoK2l\niam77MDUXcoP6+vpCa5a+CRn/+wvQ77mi3Yey/gxLfT0JHfsEUEEBIUt9KRvCu8L+3vSe4iIKNlX\nfJ3kfU/0Hhcl73vS9xS9L5xj9amn8IcHbFv3s9EsgOXrO1m4ZHWuj0Qd2M3KaGoSL+on6Pfnyycf\nVPf9GQo3CT0lNwOk74OSG4eebW9M+tysFL0v7IO+Nya96YXjyp8TZW9Giq7TM/DNTBRvo3Rfcq2e\n2PZ39vnubW7A+p5T9ru3uQHb9ndSkv/eG7Btb+T63PwV/+aS7+/73X2/vzjPfW/yytz8leS598+p\n+LtLrl/y9yJK8j9gmaQFMujNa/HfsbJ/3sV/ptGnvEp/c6FM+v6ZFl8nW8Od6rtSDuxm/RjuLH6H\nTpuYV9ZGrNC82DSk9f/Mtg/lb0z6uXkF7nlsFR/47sIhX3+4U31XyuNzzPpRmMVvsBv6oPwsfmY2\nOkiiqUk0N4nW5ibaWppob2mmo7WZMW3NjG1rYYf2FnbsaGV8RytH7jOZSePaBr09FsnQt7xv+v0/\nkdkATp81naP2nQKwzT/iwueRzuJnZqNTvd/0O7CbDaAwi99Zx+zDpHHtffZNGtfOWcfs48lpzLZD\n9XzTX3fP2CUdC/w7cACwFPg2cFEMMC5P0gkk49BfCqwEfgp8JiLKrzpiNgytzU2cefTenDZr+qib\nxc/MslHLqbsHU1eBXdIrgeuBHwOfA14FfIUkn1/q55w3Aj8H5gHnAPsDXwQmA+/MPte2vRiNs/iZ\nWXbq9aa/rgI78AXg3og4Jf18g6RW4DOSLomIjWXO+RpwdUScmn6+TVIz8FFJYyPi+RzybWZm26l6\nu+mvm3ZESe3ALOCakl1XAzuS1N5Lz3kFMB34ZnF6RFwSEdMd1M3MbHtTN4Ed2AtoAx4uSX8k3b6k\nzDkvT7ebJF0vaaOkVZK+nt4o9EvSFEkHFL9IbhLMzMxGrXpqii8s3v1cSfq6dDu+zDmT0+01wA+B\ni4DDSJr0pzDwM/Yz8MIvZmbWYOopsA/WelBuRY7CdD7XRMTZ6fvbJTUBF0iaExGlLQAFlwJXlaRN\nB64dUm7NzMzqUD0F9rXpdseS9PEl+4sVavPXl6TfAFwAvIJtm/YBiIhlwLLiNC+jaGZmo109PWNf\nDHQDM0rSC5//VuacRem29Hl6a7ot14vezMysYdVNjT0iNkm6EzhJ0oVFE9KcTFJbv6fMaXcCG4B3\nAL8oSn8TsAVYMMxstAE88sgjgx1nZmaWiaIYNKLVY+omsKfOB24BfiLpO8DhwCeBcyLieUnjSSag\nWRwRyyNivaTPAxdJWg38LD3nbOCSiFg+zO/fE+DEE0+s0s8xMzMbsT2Be4d7kgaYqbUmJL2FpFf7\nS4CngW9HxEXpvlnA7cCpETG36JxTgbOAvYFngP8AvhwR5TrcDfTdE4AjgSeBkSygW+h892aSRwuV\nuh84sArXqddrVbu8oD5/ZzWv5TIbvnr+d1nt67nManetapZZG0lQvyMiyvUvG1DdBfbRLB0Lfz9w\nYEQ8UIXrRURUpUdfPV6r2uWVXrPufmc1r+UyG9F16vbfZbWv5zKr3bWy+Lc5UvXUec629YXt4FrV\nVq+/02VW22tVU7Xz5TKr7fXqtcxGzDX2KqqnO7bRwOU1fC6z4XOZDZ/LbPjqqcxcYzczM2sgDuzV\ntZykWWe4vfG3Vy6v4XOZDZ/LbPhcZsNXN2XmpngzM7MG4hq7mZlZA3FgNzMzayAO7GZmZg3Egd3M\nzKyBOLBXiaRjJf1B0vOSHpP0CW2H68BKapJ0mqQ/S1ov6VFJX0vn+S8cM0PSLyStkbRC0mXF+9Nj\nxkn6tqRn0+v8StJL8v9F+ZP0M0lLStJcZiUkvVLS7ZI2SFoq6buSphTtd5mVkPTPkh5Iy+xvkj5c\n/P+UyywhaY+0DGaVpFetfCT9i6RHJG2U9CdJ/1i1HxARflX4Al5JMrf894DXkyxm00OyeE3N85dz\nWZxDsrLeBcDrgDOAlcDNgICdgCdIVut7M/DPwGrghpLrXAcsA94LnAT8L8k6ABNr/RszLr93AwEs\nKUpzmW1bToeQLMv8C+DY9Df/HbjbZdZvmX0g/bv1DeBoYE76/9RZLrM+v29P4K9pWc0qSq9a+QAf\nT/+f/BxwPHB1+vlVVfkNtS7ERngBNwK/L0n7MvAcMKbW+cuxHJrSv+jfLkl/W/qP5FDg0yRL7U4q\n2n98uv+I9PPM9PPxRcdMBtYD/6/WvzPD8nshsIpkEaIlRekus23L6lbgbqCpKO2ktOxe7DIrW2Z3\nA3eVpP0P8Jj/nm39/+u9wAqSykhpYK9K+QBj0v8nv1x0jEiWGb+5Gr/FTfEVktQOzAKuKdl1NbAj\n8Kq881RD40laLX5Ykv5gup0OHEfyn8uKov03AeuAQlPUcST/gG4qHBDJErx3FB3TiP6L5DffWpLu\nMisiaReSf3OXRtEKjhHxs4jYMyIew2VWTgdJZaPYSmCX9P32XmYHAZcD84BTyuyvVvn8A0nt/5qi\nY4Jk2fHXShpT6Q9xYK/cXiRL7D1ckv5Ium2oZ08DiYg1EfHRiPhtya7CAvcPAPtRUlYR0Q08Rm9Z\n7Qc8mqYXe4QGLU9JHyBpXv5Imd0us74OIvm/a7mkH0halz7LnCdpp/QYl9m2LgGOk/RuSRMkHQe8\nh+RmHFxmTwAzIuLjwPNl9lerfPZLt+ViRjNJBagiLZVewJiQbkvvhNel2/FsxyT9A8lz919ExP1K\n1rwvLStIyqtQVkM5pmFImgpcDJwaESvK9Ll0mfU1Od1+B/g1yY3j3iT9OvaS9GpcZuX8D0lLx/eK\n0m4E/jV9v12XWUSsInkU1p9qlU/mMcOBvXKDtXr0DLK/YUk6Arie5I721DR5oPLqGcYxDSHtkfwd\n4FcR8dN+DnOZ9dWWbv8YER9I398qaQ1J8DoGl1k515I8GvwUSQewl5J0oLtK0ltwmQ2mWuWTecxw\nYK/c2nS7Y0n6+JL92xVJbwPmkjQ3vT4iVqa71rJtWUFSXk8XHfOCfo5ptPL8MEnT8kslFf49CiD9\n3IPLrFShZnN9SfoN6fYVuMz6kHQ4yYidf46I/0qT75D0KPBL4ARcZoOpVvkUx4zVJccU7x8xP2Ov\n3GKgG5hRkl74/Ld8s1N7kj5BUnNaALwmIv5etPshSspKUjNJT+a/FR3zYkmlfz9n0Hjl+VZgEslQ\nra70NRuYmr7/PC6zUovSbXtJemu63YjLrNTUdFva/+XOdHsALrPBVKt8HipKKz1mM/BopRl1YK9Q\nRGwi+cdxUvFED8DJJHde99QkYzUi6UPAV4GfkNTUS+8+bwKOlDS5KO1YYBy9PUlvIrmbPa7oupOB\n1xQd0yg+BBxW8rqeJNAfBvwHLrNSfwOWAG8v+Tf3pnR7Fy6zUoWRKa8uST8i3T6Ky2ww1Sqfu0l6\nzr+16BiRDNe8IyI6K85prccONsILOIqkyfQqknGN56WfP1XrvOVcDruS9CZ9jORZ3itLXpPT13Lg\nPuAtJJNmrCJ5xlx8rdvT9A+kx/0v8BQNMgnGIOU4l77j2F1m25bRW9N/Yz8mmQjpoyRN9Fe7zPot\ns6tJxlOfTdKJ7sNpGS0keSzrMuv9jbPYdhx71cqH3smBzktjxlUkLXRHVCX/tS7ARnmlf4B/BjpJ\n7n7PqnWealAG70v/MfT3em963IHALSQ3AUuBK4AdS641EbiS5BnUWuBXwEtq/RtzKse5FAV2l1m/\n5fQGkhaxTSQze30VaHeZ9VtebcC/kdx4d5I80vgKMM5ltk1ZbRPYq1k+JK3lnyUZYrcR+CNFk9pU\n+lL6JWZmZtYA/IzdzMysgTiwm5mZNRAHdjMzswbiwG5mZtZAHNjNzMwaiAO7mZlZA3FgNzMzayAO\n7GZmZg3Egd3M6kbJ3O/bzXebVZMDu1kFJM2XtEXSof3sXyJpbs7Zqpik10p6SFKnpF/3c8xcSTHA\n69lhfN8ekn5J7ypkuZadpDcB383ju8yy5vXYzSrXDMyVdHBEbK51ZqrkqyS/6wSSObH78yzJOgnl\nDKcsXgf8Y0naW4DnhnGNSnw8p+8xy5wDu1nl1pKsZ30u8P9qnJdq2QW4MyJuGeS4zoj4XRYZiIh7\ns7iuWaNzU7xZ5e4D5gGfknTIQAemTdRzStLmSIqiz3Ml3SDpg5IWS9oo6beS9pH0Bkl/kfS8pN9L\nevlwMytpb0lXS3pW0gZJt0s6It03Lc3LNGB2mt9Zw/2OMt85XdJ1klameV8g6R/Tfe8lWQ0L4LFC\n83txU3whX5LeKunnab6XSvqspPGS/lvS2jTty8XPy9Nz50l6RlKXpGXp513S/fOBI0nW2t76eyXt\nJuk7kp5M/wzuSZvsi39XSDpX0sL0mM9LapJ0vqTH0kcZj0m6QFJrpeVoNhQO7GbV8a8kazVfKamt\nCtc7HPgISRPxqcD+JMs/fg34IvB24EXAD4ZzUUn7kywROQ04E3gnyfKUt0s6Evg7MJOkif1X6fs/\nDXLNln5eSvc3AdcDOwCnAG8GVgLXSZoB/BI4P73cSSRrVPfnv4C/AG8kWT7zPJKlWzem5/4U+BTJ\neu1IGgvMB/YjWX/8WOAS4B3Av6fXPAO4N33NBP4k6QXAH4DXAJ8BTgaWAD+X9K6SPH0G+GH6nT8l\nWe/8DJIlUo8FLgM+SbJMp1n2ar3urV9+jeYXSdCYn75/I0mQPL9o/xJgbtHnAOaUXGNO8k9x6+e5\n6XH7FqVdlqYdVZR2Vpq20zDy+2OSG5Adi9JagAeBe/rLdz/XKuSzv9cn0uN2TT+/s+jcCcDFwAHp\n5/emx0wrlweSG5EAflS0/wVp2p1FaSJ5NPL19PPLgbuAvUryfh3wYLk/x/Tzl0nWLJ9act4tJDc/\nTUV/nreUHHMDcFNJ2keAd9f676tf28fLz9jNqiQifiHp+8DZkn4WEQPWdAexOiIeLPpc6MD2+6K0\nlel2J2DNEK87C7g+ItYVEiJii6QfAZ+XNC4i1g8jn38H3tTPvifT7VLgr8B/SjoOuBH4dUSMpMPa\n3UX5Xpo2Cvy+KC0krSYpEyLiPuDVafP43sDeJK0f+zFwH6NZwN0R8XhJ+vdJHhvsm/4mSB7FFLsd\n+JKku0huIH4ZEd8azo80q4Sb4s2q66PACpIm+UqeqZbtDR4RGyq4JsDOJM3spZ4lqe2OH+b1NkfE\nwn5eS9M8B3AMyXCy40geHyyV9GNJE4f5feXKZcAykfRxYBnwMPAdkqC9geT39megcoL0xiFVeiP0\nVZIa+liSmv8Dku6X9NqB8mlWLQ7sZlUUEauB04CDgM/1c1hzyedxmWaqr1UkTeOldku3K8vsq1hE\nPBMRZ6Tf8wrgKyTPrc8f8MQKSXoncBFJgJ0cEbtGxBtIgvxABiunFf2dGBE9EfHtiDgkvcapQDvw\nsyr1vzAbkAO7WZVFxLUknak+DUwu2f0csEdJ2hF55Ct1B/AGSTsWEiQ1k3TG+0NEdFb7CyXNTHur\nHxaJ+yLisySd4AoT0nRX+3tTrwLWRMRXI2JFmp9xaXrx/3+l338HcLikqSXp7yaptT/S3xdKulvS\nJQARsSwi5gLfIqnlD7dFxGzY/IzdLBtnAkeTdPAqdj3wdkm/IwkO7wVmVOMLJY0neX68OCKW93PY\nF0gmgrld0pdIJpE5E5gOvH4EX9su6ZUD7P8zSW/z54HvpUP9niWZkOblJD3UobePwEmSflXSv6AS\n9wCnS7oI+AXwQpIe6rsCq4uOWwPMlHRUmt+LSXrw35rmeSXwHuAo4H0R0TPAd94BfELSUpI+AbuT\ndHS8o3BzYZYl19jNMhARq4DTy+z6OEmAuRC4muT57DlV+tqDgQUks8X1l68HSGqry0g6gX2f5Fnz\nrBh8Mppydk2/s7/XPhGxiWTY1wMkgfxG4ETgQ2ltFpIOZ7cAF5A0nVfLd0mGnf1f4Nfp+zuBDwE7\nS9ovPe5bQFd6zPER8SzJkMM/At8k+bN6EfDmiLiSgX2OZCjd+0h6yF9M8ptPrt7PMuufkn4tZmZm\n1ghcYzczM2sgDuxmZmYNxIHdzMysgTiwm5mZNRAHdjMzswbiwG5mZtZAHNjNzMwaiAO7mZlZA3Fg\nNzMzayAO7GZmZg3Egd3MzKyBOLCbmZk1EAd2MzOzBvL/AWQ9GarH7og8AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x143915d30>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "rf_cv_results = pd.DataFrame(rf.cv_results_)\n",
    "plt.plot(rf_cv_results['param_n_estimators'], -rf_cv_results['mean_test_score'], '-o')\n",
    "plt.xlabel('Num. of Estimators')\n",
    "plt.ylabel('Log Loss')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looks like 400 estimators are enough."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.644007760717\n"
     ]
    }
   ],
   "source": [
    "rf = RandomForestClassifier(n_jobs=-1, n_estimators=400).fit(X_train, y_train)\n",
    "y_probs_pred = rf.predict_proba(X_test)\n",
    "print(log_loss(y_test, y_probs_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "More feature engineering."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def engineer_features2(df):\n",
    "    df['all_rooms'] = df['bathrooms'] + df['bedrooms']\n",
    "    df['price_per_bed'] = df['price'] / df['bedrooms']    \n",
    "    df['price_per_bath'] = df['price'] / df['bathrooms']\n",
    "    df['price_per_room'] = df['price'] / df['all_rooms']\n",
    "\n",
    "    df['price_per_bed'] = df['price_per_bed'].replace(np.Inf, 5000)\n",
    "    df['price_per_bath'] = df['price_per_bath'].replace(np.Inf, 5000)\n",
    "    df['price_per_room'] = df['price_per_room'].replace(np.Inf, 5000)\n",
    "    \n",
    "engineer_features2(train_df)\n",
    "engineer_features2(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_vars1 = X_vars + ['all_rooms', 'price_per_bed', 'price_per_bath', 'price_per_room']\n",
    "X_train, X_test, y_train, y_test = train_test_split(train_df[X_vars1], train_df['interest_level'], stratify=train_df['interest_level'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.636847919769\n"
     ]
    }
   ],
   "source": [
    "rf = RandomForestClassifier(n_jobs=-1, n_estimators=1000).fit(X_train, y_train)\n",
    "y_probs_pred = rf.predict_proba(X_test)\n",
    "print(log_loss(y_test, y_probs_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Hmmm... Only minimal improvement.\n",
    "\n",
    "# Text Processing\n",
    "\n",
    "## `features` Field\n",
    "\n",
    "Try vectorizing the feature words. First look at the non-text characters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([' ', '-', '/', '(', ')', '*', '\\xa0', '&', '!', \"'\", '.', ':', '•',\n",
       "       '’', '$', '~', ';', '@', '^', '+', '<', '>', '\"', '®'], dtype=object)"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.features_clean.str.extractall('(\\W)')[0].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from itertools import chain\n",
    "\n",
    "def features_cleanup_star(x):\n",
    "    return list(chain.from_iterable(re.sub(r'\\s*\\*+\\s*\\**\\s*', r'*', i).strip('*').split('*') for i in x))\n",
    "\n",
    "def process_features(df):\n",
    "    df['features_clean'] = df['features']\\\n",
    "                    .apply(lambda x: ' '.join([re.sub(r'\\W', '', i) for i in x]).lower())\n",
    "\n",
    "    df.loc[df.features_clean.str.contains('\\*'), 'features_clean'] = \\\n",
    "            df.loc[df.features_clean.str.contains('\\*'), 'features']\\\n",
    "                    .apply(features_cleanup_star)\\\n",
    "                    .apply(lambda x: ' '.join([re.sub(r'\\W', '', i) for i in x]).lower())\n",
    "                \n",
    "process_features(train_df)\n",
    "process_features(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "?CountVectorizer(stop_words='english', max_features=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "countvec_features = CountVectorizer(stop_words='english', max_features=200)\n",
    "countvec_features.fit(train_df['features_clean'].tolist() + test_df['features_clean'].tolist())\n",
    "train_df_features = countvec_features.transform(train_df['features_clean'])\n",
    "test_df_features = countvec_features.transform(test_df['features_clean'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'247concierge': 0,\n",
       " '247doorman': 1,\n",
       " '247doormanconcierge': 2,\n",
       " '24hourdoorman': 3,\n",
       " 'actualaptphotos': 4,\n",
       " 'airconditioning': 5,\n",
       " 'allutilitiesincluded': 6,\n",
       " 'assignedparkingspace': 7,\n",
       " 'attendedlobby': 8,\n",
       " 'backyard': 9,\n",
       " 'balcony': 10,\n",
       " 'basementstorage': 11,\n",
       " 'bikeroom': 12,\n",
       " 'bikestorage': 13,\n",
       " 'billiardsroom': 14,\n",
       " 'breakfastbar': 15,\n",
       " 'brownstone': 16,\n",
       " 'buildingcommonoutdoorspace': 17,\n",
       " 'businesscenter': 18,\n",
       " 'cablesatellitetv': 19,\n",
       " 'calltextabrahamcaro9173730862': 20,\n",
       " 'catsallowed': 21,\n",
       " 'centralac': 22,\n",
       " 'centralair': 23,\n",
       " 'chefskitchen': 24,\n",
       " 'childrensplayroom': 25,\n",
       " 'cityview': 26,\n",
       " 'closetosubway': 27,\n",
       " 'closetsgalore': 28,\n",
       " 'commonbackyard': 29,\n",
       " 'commongarden': 30,\n",
       " 'commonoutdoorspace': 31,\n",
       " 'commonparkinggarage': 32,\n",
       " 'commonroofdeck': 33,\n",
       " 'commonterrace': 34,\n",
       " 'communityrecreationfacilities': 35,\n",
       " 'complimentarysundaybrunch': 36,\n",
       " 'concierge': 37,\n",
       " 'conciergeservice': 38,\n",
       " 'condofinishes': 39,\n",
       " 'courtyard': 40,\n",
       " 'deck': 41,\n",
       " 'decorativefireplace': 42,\n",
       " 'diningroom': 43,\n",
       " 'dishwasher': 44,\n",
       " 'dogsallowed': 45,\n",
       " 'doorman': 46,\n",
       " 'drycleaningservice': 47,\n",
       " 'dryerinunit': 48,\n",
       " 'duplex': 49,\n",
       " 'eatinkitchen': 50,\n",
       " 'elevator': 51,\n",
       " 'exclusive': 52,\n",
       " 'exposedbrick': 53,\n",
       " 'fireplace': 54,\n",
       " 'fireplaces': 55,\n",
       " 'fitnesscenter': 56,\n",
       " 'flex2': 57,\n",
       " 'flex3': 58,\n",
       " 'ftdoorman': 59,\n",
       " 'fullservicegarage': 60,\n",
       " 'fulltimedoorman': 61,\n",
       " 'furnished': 62,\n",
       " 'gameroom': 63,\n",
       " 'garage': 64,\n",
       " 'garden': 65,\n",
       " 'gardenpatio': 66,\n",
       " 'granitecountertops': 67,\n",
       " 'granitekitchen': 68,\n",
       " 'greenbuilding': 69,\n",
       " 'guarantorsaccepted': 70,\n",
       " 'gutrenovated': 71,\n",
       " 'gym': 72,\n",
       " 'gymfitness': 73,\n",
       " 'gyminbuilding': 74,\n",
       " 'hardwood': 75,\n",
       " 'hardwoodfloors': 76,\n",
       " 'healthclub': 77,\n",
       " 'highceiling': 78,\n",
       " 'highceilings': 79,\n",
       " 'highrise': 80,\n",
       " 'highspeedinternet': 81,\n",
       " 'hirise': 82,\n",
       " 'indoorpool': 83,\n",
       " 'inunitwasherdryer': 84,\n",
       " 'largelivingroom': 85,\n",
       " 'laundry': 86,\n",
       " 'laundryhousekeeping': 87,\n",
       " 'laundryinbuilding': 88,\n",
       " 'laundryinunit': 89,\n",
       " 'laundryonfloor': 90,\n",
       " 'laundryroom': 91,\n",
       " 'light': 92,\n",
       " 'liveinsuper': 93,\n",
       " 'liveinsuperintendent': 94,\n",
       " 'livework': 95,\n",
       " 'loft': 96,\n",
       " 'lounge': 97,\n",
       " 'loungeroom': 98,\n",
       " 'lowrise': 99,\n",
       " 'luxurybuilding': 100,\n",
       " 'mailroom': 101,\n",
       " 'marblebath': 102,\n",
       " 'marblebathroom': 103,\n",
       " 'mediaroom': 104,\n",
       " 'microwave': 105,\n",
       " 'midrise': 106,\n",
       " 'multilevel': 107,\n",
       " 'newconstruction': 108,\n",
       " 'newlyrenovated': 109,\n",
       " 'nofee': 110,\n",
       " 'nopets': 111,\n",
       " 'onsitegarage': 112,\n",
       " 'onsitelaundry': 113,\n",
       " 'onsiteparking': 114,\n",
       " 'onsiteparkingavailable': 115,\n",
       " 'onsiteparkinglot': 116,\n",
       " 'onsitesuper': 117,\n",
       " 'outdoorareas': 118,\n",
       " 'outdoorentertainmentspace': 119,\n",
       " 'outdoorpool': 120,\n",
       " 'outdoorroofdeckoverlookingnewyorkharborandbatterypark': 121,\n",
       " 'outdoorspace': 122,\n",
       " 'packageroom': 123,\n",
       " 'parking': 124,\n",
       " 'parkingavailable': 125,\n",
       " 'parkingspace': 126,\n",
       " 'parttimedoorman': 127,\n",
       " 'partyroom': 128,\n",
       " 'patio': 129,\n",
       " 'penthouse': 130,\n",
       " 'petfriendly': 131,\n",
       " 'pets': 132,\n",
       " 'petsallowed': 133,\n",
       " 'petsonapproval': 134,\n",
       " 'playroom': 135,\n",
       " 'playroomnursery': 136,\n",
       " 'pool': 137,\n",
       " 'postwar': 138,\n",
       " 'prewar': 139,\n",
       " 'privatebackyard': 140,\n",
       " 'privatebalcony': 141,\n",
       " 'privatedeck': 142,\n",
       " 'privatelaundryroomoneveryfloor': 143,\n",
       " 'privateoutdoorspace': 144,\n",
       " 'privateparking': 145,\n",
       " 'privateroofdeck': 146,\n",
       " 'privateterrace': 147,\n",
       " 'publicoutdoor': 148,\n",
       " 'queensizedrooms': 149,\n",
       " 'reducedfee': 150,\n",
       " 'renovated': 151,\n",
       " 'residentsgarden': 152,\n",
       " 'residentslounge': 153,\n",
       " 'roofaccess': 154,\n",
       " 'roofdeck': 155,\n",
       " 'rooftopdeck': 156,\n",
       " 'rooftopterrace': 157,\n",
       " 'sauna': 158,\n",
       " 'screeningroom': 159,\n",
       " 'sharedbackyard': 160,\n",
       " 'sharesok': 161,\n",
       " 'shorttermallowed': 162,\n",
       " 'simplex': 163,\n",
       " 'skylight': 164,\n",
       " 'ssappliances': 165,\n",
       " 'stainlesssteelappliances': 166,\n",
       " 'stateoftheartfitnesscenter': 167,\n",
       " 'storage': 168,\n",
       " 'storagefacilitiesavailable': 169,\n",
       " 'storageroom': 170,\n",
       " 'sublet': 171,\n",
       " 'subway': 172,\n",
       " 'sundeck': 173,\n",
       " 'swimmingpool': 174,\n",
       " 'tenantlounge': 175,\n",
       " 'terrace': 176,\n",
       " 'terracesbalconies': 177,\n",
       " 'tonsofnaturallight': 178,\n",
       " 'valet': 179,\n",
       " 'valetparking': 180,\n",
       " 'valetservices': 181,\n",
       " 'valetservicesincludingdrycleaning': 182,\n",
       " 'videointercom': 183,\n",
       " 'view': 184,\n",
       " 'virtualdoorman': 185,\n",
       " 'virtualtour': 186,\n",
       " 'walkincloset': 187,\n",
       " 'walkinclosets': 188,\n",
       " 'walkup': 189,\n",
       " 'washerdryer': 190,\n",
       " 'washerdryerhookup': 191,\n",
       " 'washerdryerinbuilding': 192,\n",
       " 'washerdryerinunit': 193,\n",
       " 'washerinunit': 194,\n",
       " 'wheelchairaccess': 195,\n",
       " 'wheelchairramp': 196,\n",
       " 'wifi': 197,\n",
       " 'wifiaccess': 198,\n",
       " 'woodburningfireplace': 199}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "countvec_features.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.051382209\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(train_df_features, train_df['interest_level'], stratify=train_df['interest_level'])\n",
    "rf = RandomForestClassifier(n_jobs=-1, n_estimators=500).fit(X_train, y_train)\n",
    "y_probs_pred = rf.predict_proba(X_test)\n",
    "print(log_loss(y_test, y_probs_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_df_combined = np.column_stack([train_df[X_vars1], train_df_features.todense()])\n",
    "X_train, X_test, y_train, y_test = train_test_split(train_df_combined, train_df['interest_level'], stratify=train_df['interest_level'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.606495224603\n"
     ]
    }
   ],
   "source": [
    "rf = RandomForestClassifier(n_jobs=-1, n_estimators=500).fit(X_train, y_train)\n",
    "y_probs_pred = rf.predict_proba(X_test)\n",
    "print(log_loss(y_test, y_probs_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Small improvement.\n",
    "\n",
    "## `description` Field"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "countvec_desc = CountVectorizer(stop_words='english', max_features=200)\n",
    "countvec_desc.fit(train_df['description'].tolist() + test_df['description'].tolist())\n",
    "train_df_desc = countvec_desc.transform(train_df['description'])\n",
    "test_df_desc = countvec_desc.transform(test_df['description'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'24': 0,\n",
       " '_blank': 1,\n",
       " 'access': 2,\n",
       " 'amazing': 3,\n",
       " 'amenities': 4,\n",
       " 'amp': 5,\n",
       " 'apartment': 6,\n",
       " 'apartments': 7,\n",
       " 'appliances': 8,\n",
       " 'appointment': 9,\n",
       " 'area': 10,\n",
       " 'art': 11,\n",
       " 'available': 12,\n",
       " 'away': 13,\n",
       " 'bars': 14,\n",
       " 'bath': 15,\n",
       " 'bathroom': 16,\n",
       " 'bathrooms': 17,\n",
       " 'beautiful': 18,\n",
       " 'bed': 19,\n",
       " 'bedroom': 20,\n",
       " 'bedrooms': 21,\n",
       " 'best': 22,\n",
       " 'block': 23,\n",
       " 'blocks': 24,\n",
       " 'bond': 25,\n",
       " 'br': 26,\n",
       " 'brand': 27,\n",
       " 'brick': 28,\n",
       " 'bright': 29,\n",
       " 'broker': 30,\n",
       " 'building': 31,\n",
       " 'cabinets': 32,\n",
       " 'ceiling': 33,\n",
       " 'ceilings': 34,\n",
       " 'center': 35,\n",
       " 'central': 36,\n",
       " 'city': 37,\n",
       " 'close': 38,\n",
       " 'closet': 39,\n",
       " 'closets': 40,\n",
       " 'com': 41,\n",
       " 'concierge': 42,\n",
       " 'contact': 43,\n",
       " 'coopercooper': 44,\n",
       " 'counter': 45,\n",
       " 'countertops': 46,\n",
       " 'custom': 47,\n",
       " 'deal': 48,\n",
       " 'deck': 49,\n",
       " 'dining': 50,\n",
       " 'dishwasher': 51,\n",
       " 'doorman': 52,\n",
       " 'dryer': 53,\n",
       " 'east': 54,\n",
       " 'easy': 55,\n",
       " 'elevator': 56,\n",
       " 'email': 57,\n",
       " 'enjoy': 58,\n",
       " 'equal': 59,\n",
       " 'estate': 60,\n",
       " 'exclusive': 61,\n",
       " 'exposed': 62,\n",
       " 'features': 63,\n",
       " 'fee': 64,\n",
       " 'feel': 65,\n",
       " 'finishes': 66,\n",
       " 'fitness': 67,\n",
       " 'flex': 68,\n",
       " 'floor': 69,\n",
       " 'flooring': 70,\n",
       " 'floors': 71,\n",
       " 'free': 72,\n",
       " 'friendly': 73,\n",
       " 'fully': 74,\n",
       " 'garage': 75,\n",
       " 'gorgeous': 76,\n",
       " 'granite': 77,\n",
       " 'great': 78,\n",
       " 'gym': 79,\n",
       " 'hardwood': 80,\n",
       " 'heart': 81,\n",
       " 'heat': 82,\n",
       " 'high': 83,\n",
       " 'home': 84,\n",
       " 'hour': 85,\n",
       " 'housing': 86,\n",
       " 'href': 87,\n",
       " 'huge': 88,\n",
       " 'include': 89,\n",
       " 'included': 90,\n",
       " 'including': 91,\n",
       " 'information': 92,\n",
       " 'just': 93,\n",
       " 'kagglemanager': 94,\n",
       " 'king': 95,\n",
       " 'kitchen': 96,\n",
       " 'large': 97,\n",
       " 'laundry': 98,\n",
       " 'li': 99,\n",
       " 'light': 100,\n",
       " 'listing': 101,\n",
       " 'listings': 102,\n",
       " 'live': 103,\n",
       " 'living': 104,\n",
       " 'lobby': 105,\n",
       " 'located': 106,\n",
       " 'location': 107,\n",
       " 'lots': 108,\n",
       " 'lounge': 109,\n",
       " 'luxury': 110,\n",
       " 'manhattan': 111,\n",
       " 'marble': 112,\n",
       " 'market': 113,\n",
       " 'microwave': 114,\n",
       " 'midtown': 115,\n",
       " 'modern': 116,\n",
       " 'month': 117,\n",
       " 'natural': 118,\n",
       " 'near': 119,\n",
       " 'neighborhood': 120,\n",
       " 'new': 121,\n",
       " 'newly': 122,\n",
       " 'offers': 123,\n",
       " 'open': 124,\n",
       " 'opportunity': 125,\n",
       " 'outdoor': 126,\n",
       " 'park': 127,\n",
       " 'parking': 128,\n",
       " 'perfect': 129,\n",
       " 'pet': 130,\n",
       " 'pets': 131,\n",
       " 'plenty': 132,\n",
       " 'pool': 133,\n",
       " 'price': 134,\n",
       " 'prime': 135,\n",
       " 'private': 136,\n",
       " 'queen': 137,\n",
       " 'real': 138,\n",
       " 'renovated': 139,\n",
       " 'rent': 140,\n",
       " 'renthop': 141,\n",
       " 'restaurants': 142,\n",
       " 'right': 143,\n",
       " 'river': 144,\n",
       " 'roof': 145,\n",
       " 'rooftop': 146,\n",
       " 'room': 147,\n",
       " 'schedule': 148,\n",
       " 'separate': 149,\n",
       " 'service': 150,\n",
       " 'shopping': 151,\n",
       " 'shops': 152,\n",
       " 'showing': 153,\n",
       " 'site': 154,\n",
       " 'size': 155,\n",
       " 'sized': 156,\n",
       " 'space': 157,\n",
       " 'spacious': 158,\n",
       " 'square': 159,\n",
       " 'st': 160,\n",
       " 'stainless': 161,\n",
       " 'state': 162,\n",
       " 'steel': 163,\n",
       " 'steps': 164,\n",
       " 'storage': 165,\n",
       " 'street': 166,\n",
       " 'studio': 167,\n",
       " 'stunning': 168,\n",
       " 'subway': 169,\n",
       " 'sun': 170,\n",
       " 'super': 171,\n",
       " 'supports': 172,\n",
       " 'target': 173,\n",
       " 'terrace': 174,\n",
       " 'text': 175,\n",
       " 'time': 176,\n",
       " 'today': 177,\n",
       " 'tons': 178,\n",
       " 'tops': 179,\n",
       " 'train': 180,\n",
       " 'trains': 181,\n",
       " 'transportation': 182,\n",
       " 'ul': 183,\n",
       " 'unit': 184,\n",
       " 'upper': 185,\n",
       " 'valet': 186,\n",
       " 'view': 187,\n",
       " 'viewing': 188,\n",
       " 'views': 189,\n",
       " 'village': 190,\n",
       " 'walk': 191,\n",
       " 'washer': 192,\n",
       " 'water': 193,\n",
       " 'website_redacted': 194,\n",
       " 'west': 195,\n",
       " 'white': 196,\n",
       " 'windows': 197,\n",
       " 'wood': 198,\n",
       " 'york': 199}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "countvec_desc.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_df_combined = np.column_stack([train_df[X_vars1], train_df_features.todense(), train_df_desc.todense()])\n",
    "X_train, X_test, y_train, y_test = train_test_split(train_df_combined, train_df['interest_level'], stratify=train_df['interest_level'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.621021205792\n"
     ]
    }
   ],
   "source": [
    "rf = RandomForestClassifier(n_jobs=-1, n_estimators=500).fit(X_train, y_train)\n",
    "y_probs_pred = rf.predict_proba(X_test)\n",
    "print(log_loss(y_test, y_probs_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apparently slightly worse. This level of deviation could be noice.\n",
    "\n",
    "# Boosting\n",
    "\n",
    "Try `GradientBoostingClassifier` in `scikit-learn`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.581470890804\n"
     ]
    }
   ],
   "source": [
    "# params = {'learning_rate': [0.05, 0.1, 0.2], 'subsample': [1, 0.5]}\n",
    "# gbc = GridSearchCV(GradientBoostingClassifier(n_estimators=1000), \n",
    "#                          params, scoring='neg_log_loss').fit(X_train, y_train)\n",
    "\n",
    "gbc = GradientBoostingClassifier(n_estimators=1000, learning_rate=0.1, subsample=0.5).fit(X_train, y_train)\n",
    "y_probs_pred = gbc.predict_proba(X_test)\n",
    "print(log_loss(y_test, y_probs_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now try `xgboost`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "\n",
    "def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=0, num_rounds=1000):\n",
    "    param = {}\n",
    "    param['objective'] = 'multi:softprob'\n",
    "    param['eta'] = 0.1\n",
    "    param['max_depth'] = 6\n",
    "    param['silent'] = 1\n",
    "    param['num_class'] = 3\n",
    "    param['eval_metric'] = \"mlogloss\"\n",
    "    param['min_child_weight'] = 1\n",
    "    param['subsample'] = 0.7\n",
    "    param['colsample_bytree'] = 0.7\n",
    "    param['seed'] = seed_val\n",
    "\n",
    "    plst = list(param.items())\n",
    "    xgtrain = xgb.DMatrix(train_X, label=train_y)\n",
    "\n",
    "    if test_y is not None:\n",
    "        xgtest = xgb.DMatrix(test_X, label=test_y)\n",
    "        watchlist = [ (xgtrain,'train'), (xgtest, 'test') ]\n",
    "        model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=20)\n",
    "    else:\n",
    "        xgtest = xgb.DMatrix(test_X)\n",
    "        model = xgb.train(plst, xgtrain, num_rounds)\n",
    "\n",
    "    pred_test_y = model.predict(xgtest)\n",
    "    return pred_test_y, model\n",
    "\n",
    "target_num_map = {'high':0, 'medium':1, 'low':2}\n",
    "X_train, X_test, y_train, y_test = train_test_split(train_df_combined, train_df['interest_level'].apply(lambda x: target_num_map[x]), stratify=train_df['interest_level'])\n",
    "preds, model = runXGB(X_train, y_train, X_test, num_rounds=400)\n",
    "print(log_loss(y_test, preds))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Result is faster and slightly better. So far the best model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_df_combined = np.column_stack([test_df[X_vars1], test_df_features.todense(), test_df_desc.todense()])\n",
    "test_y = model.predict(xgb.DMatrix(test_df_combined))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(test_y, index=test_df.listing_id, columns=['high', 'medium', 'low']).to_csv('submission_xgb.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}