session_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.cross_validation import LabelKFold\n",
    "\n",
    "from ml_metrics import rmsle\n",
    "\n",
    "from sklearn.dummy import DummyRegressor \n",
    "from sklearn.ensemble import RandomForestRegressor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv', parse_dates=['datetime'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def select_features(data):\n",
    "    black_list = ['casual', 'registered', 'count', 'is_test', 'datetime', 'count_log']\n",
    "    return [feat for feat in data.columns if feat not in black_list]\n",
    "    \n",
    "def get_X_y(data, target_var='count'):\n",
    "    features = select_features(data)\n",
    "    return data[features].values, data[target_var].values\n",
    "\n",
    "def _simple_modeling(X_train, X_test, y_train, y_test):\n",
    "    models = [\n",
    "        ('dummy-mean', DummyRegressor(strategy='mean')),\n",
    "        ('dummy-median', DummyRegressor(strategy='median')),\n",
    "        ('random-forest', RandomForestRegressor(random_state=0)),\n",
    "    ]\n",
    "    \n",
    "    results = []\n",
    "\n",
    "    for name, model in models:\n",
    "        model.fit(X_train, y_train)\n",
    "        y_pred = model.predict(X_test)\n",
    "        \n",
    "        results.append((name, y_test, y_pred))\n",
    "        \n",
    "    return results\n",
    "\n",
    "\n",
    "def simple_modeling(X_train, X_test, y_train, y_test):\n",
    "    results = _simple_modeling(X_train, X_test, y_train, y_test)\n",
    "    \n",
    "    return [ (r[0], rmsle(r[1], r[2]) ) for r in results]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('dummy-mean', 1.5691983019475926),\n",
       " ('dummy-median', 1.4725894242962372),\n",
       " ('random-forest', 0.84028175828825968)]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, y = get_X_y(train)\n",
    "simple_modeling(X, X, y, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def val(data, n_folds=3):\n",
    "    feats = select_features(data)\n",
    "    X = data[feats].values\n",
    "    y = data['count'].values\n",
    "\n",
    "    labels = data['datetime'].dt.month.values\n",
    "    \n",
    "    for train_idx, test_idx in LabelKFold(labels, n_folds=n_folds):\n",
    "        yield X[train_idx], X[test_idx], y[train_idx], y[test_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('dummy-mean', 1.5389717242983016), ('dummy-median', 1.4518285970779792), ('random-forest', 1.3898355462701857)]\n",
      "[('dummy-mean', 1.5973803133409017), ('dummy-median', 1.4947795712594232), ('random-forest', 1.3868563304622581)]\n",
      "[('dummy-mean', 1.5718600070726862), ('dummy-median', 1.4729884592466973), ('random-forest', 1.4279380440008111)]\n"
     ]
    }
   ],
   "source": [
    "for X_train, X_test, y_train, y_test in val(train):\n",
    "    print(simple_modeling(X_train, X_test, y_train, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's compare results with  validation and without one.\n",
    "\n",
    "| name        |  without validation | with validation  | diff in % |\n",
    "| ------------|:-------------------:|:----------------:|----------:|\n",
    "|dummy-mean   |     1.57            |      ~1.57        |   ~0%     |\n",
    "|dummy-median |     1.47            |      ~1.47        |   ~0%     |\n",
    "|random-forest|     0.84            |      ~1.40        |   ~40%    |\n",
    "\n",
    "## Questions\n",
    "1. Why the result for dumy mean/median is the same* with/without validation?\n",
    "2. Why the result for random-forest is very different with validation and without?\n",
    "3. Which result is better for us?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# For more advanced\n",
    "## Understand Better Target Variable\n",
    "\n",
    "count = registered + casual"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('count: ', 1.401543306911085, 0.018703485258713615, [1.3898355462701857, 1.3868563304622581, 1.4279380440008111])\n",
      "('registered + casual: ', 1.3946435992749022, 0.021842027802280947, [1.4016486462734967, 1.365087126839138, 1.4171950247120721])\n"
     ]
    }
   ],
   "source": [
    "def reg_cas_val(data, n_folds=3):\n",
    "    feats = select_features(data)\n",
    "    X = data[feats].values\n",
    "    y = data['count'].values\n",
    "    y_reg = data['registered'].values\n",
    "    y_cas = data['casual'].values\n",
    "\n",
    "    labels = data['datetime'].dt.month.values\n",
    "    \n",
    "    for train_idx, test_idx in LabelKFold(labels, n_folds=n_folds):\n",
    "        yield X[train_idx], X[test_idx], y[train_idx], y_reg[train_idx], y_cas[train_idx], y[test_idx]\n",
    "        \n",
    "        \n",
    "\n",
    "scores_sum = []\n",
    "scores_c = []\n",
    "\n",
    "for X_train, X_test, y_train_cnt, y_train_reg, y_train_cas, y_test in reg_cas_val(train): \n",
    "    model = RandomForestRegressor(random_state=0)\n",
    "    \n",
    "    model.fit(X_train, y_train_cnt)\n",
    "    y_pred_cnt = model.predict(X_test)\n",
    "    \n",
    "    model.fit(X_train, y_train_reg)\n",
    "    y_pred_reg = model.predict(X_test)\n",
    "\n",
    "    model.fit(X_train, y_train_cas)\n",
    "    y_pred_cas = model.predict(X_test)\n",
    "\n",
    "    y_pred = y_pred_reg + y_pred_cas\n",
    "    \n",
    "    scores_c.append( rmsle(y_test, y_pred_cnt) )\n",
    "    scores_sum.append( rmsle(y_test, y_pred) )\n",
    "    \n",
    "\n",
    "print(\"count: \", np.mean(scores_c), np.std(scores_c), scores_c)\n",
    "print(\"registered + casual: \", np.mean(scores_sum), np.std(scores_sum), scores_sum)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Questions:\n",
    "1. What does mean this experimetn for you?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}