experiments/02_BCI_GPU.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Experiment 02: BCI (GPU version)\n",
    "\n",
    "This experiment uses a Brain Computer Interface dataset. The purpose is to try and predict when the participant is paying attention. The dataset consists of recordings from a number of electrodes placed over the scalp.\n",
    "\n",
    "The details of the machine we used and the version of the libraries can be found in [experiment 01](01_airline.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "System version: 3.5.2 |Anaconda custom (64-bit)| (default, Jul  2 2016, 17:53:06) \n",
      "[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)]\n",
      "XGBoost version: 0.6\n",
      "LightGBM version: 0.2\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import sys\n",
    "import warnings\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pkg_resources\n",
    "from libs.loaders import load_bci\n",
    "from libs.timer import Timer\n",
    "from libs.metrics import classification_metrics_binary, classification_metrics_binary_prob, binarize_prediction\n",
    "import xgboost as xgb\n",
    "import lightgbm as lgb\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "print(\"System version: {}\".format(sys.version))\n",
    "print(\"XGBoost version: {}\".format(pkg_resources.get_distribution('xgboost').version))\n",
    "print(\"LightGBM version: {}\".format(pkg_resources.get_distribution('lightgbm').version))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Data loading and management\n",
    "\n",
    "\n",
    "The dataset has been preprepared by extracting 800ms epochs from each channel. The data was then lowpass filtered at 18Hz and downsampled by a factor of 6. This results is a feature vector of 2048. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:libs.loaders:MOUNT_POINT not found in environment. Defaulting to /fileshare\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.1 s, sys: 472 ms, total: 2.57 s\n",
      "Wall time: 18.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "X, y, X_test, y_test = load_bci()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(14519, 2048)\n",
      "(14519,)\n",
      "(5978, 2048)\n",
      "(5978,)\n"
     ]
    }
   ],
   "source": [
    "X_train = np.concatenate(X)\n",
    "y_train = np.concatenate(y)\n",
    "X_test = np.concatenate(X_test)\n",
    "y_test = np.concatenate(y_test)\n",
    "print(X_train.shape)\n",
    "print(y_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "dtrain = xgb.DMatrix(data=X_train, label=y_train)\n",
    "dtest = xgb.DMatrix(data=X_test, label=y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "lgb_train = lgb.Dataset(X_train, y_train, free_raw_data=False)\n",
    "lgb_test = lgb.Dataset(X_test, y_test, reference=lgb_train, free_raw_data=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict = dict()\n",
    "num_rounds = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "params = {'max_depth':3, \n",
    "          'objective':'binary:logistic', \n",
    "          'min_child_weight':1, \n",
    "          'eta':0.1, \n",
    "          'colsample_bytree':1, \n",
    "          'scale_pos_weight':2, \n",
    "          'gamma':0.1, \n",
    "          'reg_lamda':1, \n",
    "          'subsample':1,\n",
    "          'tree_method':'exact', \n",
    "          'updater':'grow_gpu'\n",
    "          }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer() as t_train:\n",
    "    xgb_clf_pipeline = xgb.train(params, dtrain, num_boost_round=num_rounds)\n",
    "    \n",
    "with Timer() as t_test:\n",
    "    y_prob_xgb = xgb_clf_pipeline.predict(dtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "y_pred_xgb = binarize_prediction(y_prob_xgb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "report_xgb = classification_metrics_binary(y_test, y_pred_xgb)\n",
    "report2_xgb = classification_metrics_binary_prob(y_test, y_prob_xgb)\n",
    "report_xgb.update(report2_xgb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['xgb']={\n",
    "    'train_time': t_train.interval,\n",
    "    'test_time': t_test.interval,\n",
    "    'performance': report_xgb \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "del xgb_clf_pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now let's try with XGBoost histogram."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "params = {'max_depth':0, \n",
    "          'objective':'binary:logistic', \n",
    "          'min_child_weight':1, \n",
    "          'eta':0.1, \n",
    "          'colsample_bytree':0.80, \n",
    "          'scale_pos_weight':2, \n",
    "          'gamma':0.1, \n",
    "          'reg_lamda':1, \n",
    "          'subsample':1,\n",
    "          'tree_method':'hist', \n",
    "          'max_leaves':2**3, \n",
    "          'grow_policy':'lossguide', \n",
    "         }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer() as t_train:\n",
    "    xgb_hist_clf_pipeline = xgb.train(params, dtrain, num_boost_round=num_rounds)\n",
    "    \n",
    "with Timer() as t_test:\n",
    "    y_prob_xgb_hist = xgb_hist_clf_pipeline.predict(dtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "y_pred_xgb_hist = binarize_prediction(y_prob_xgb_hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "report_xgb_hist = classification_metrics_binary(y_test, y_pred_xgb_hist)\n",
    "report2_xgb_hist = classification_metrics_binary_prob(y_test, y_prob_xgb_hist)\n",
    "report_xgb_hist.update(report2_xgb_hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['xgb_hist']={\n",
    "    'train_time': t_train.interval,\n",
    "    'test_time': t_test.interval,\n",
    "    'performance': report_xgb_hist\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "del xgb_hist_clf_pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "params = {'num_leaves': 2**3,\n",
    "         'learning_rate': 0.1,\n",
    "         'scale_pos_weight': 2,\n",
    "         'min_split_gain': 0.1,\n",
    "         'min_child_weight': 1,\n",
    "         'reg_lambda': 1,\n",
    "         'subsample': 1,\n",
    "         'objective':'binary',\n",
    "         'task': 'train'\n",
    "         }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer() as t_train:\n",
    "    lgbm_clf_pipeline = lgb.train(params, lgb_train, num_boost_round=num_rounds)\n",
    "    \n",
    "with Timer() as t_test:\n",
    "    y_prob_lgbm = lgbm_clf_pipeline.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "y_pred_lgbm = binarize_prediction(y_prob_lgbm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "report_lgbm = classification_metrics_binary(y_test, y_pred_lgbm)\n",
    "report2_lgbm = classification_metrics_binary_prob(y_test, y_prob_lgbm)\n",
    "report_lgbm.update(report2_lgbm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['lgbm']={\n",
    "    'train_time': t_train.interval,\n",
    "    'test_time': t_test.interval,\n",
    "    'performance': report_lgbm \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "del lgbm_clf_pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Finally, we show the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"lgbm\": {\n",
      "        \"performance\": {\n",
      "            \"AUC\": 0.7714542124542124,\n",
      "            \"Accuracy\": 0.8813984610237537,\n",
      "            \"F1\": 0.13851761846901578,\n",
      "            \"Precision\": 0.6,\n",
      "            \"Recall\": 0.0782967032967033\n",
      "        },\n",
      "        \"test_time\": 0.009907090001433971,\n",
      "        \"train_time\": 2.7659428379993187\n",
      "    },\n",
      "    \"xgb\": {\n",
      "        \"performance\": {\n",
      "            \"AUC\": 0.7716584249084248,\n",
      "            \"Accuracy\": 0.8798929407828705,\n",
      "            \"F1\": 0.09343434343434343,\n",
      "            \"Precision\": 0.578125,\n",
      "            \"Recall\": 0.050824175824175824\n",
      "        },\n",
      "        \"test_time\": 0.0064387769998575095,\n",
      "        \"train_time\": 12.934047714998997\n",
      "    },\n",
      "    \"xgb_hist\": {\n",
      "        \"performance\": {\n",
      "            \"AUC\": 0.7736170852956569,\n",
      "            \"Accuracy\": 0.8805620608899297,\n",
      "            \"F1\": 0.12068965517241378,\n",
      "            \"Precision\": 0.5833333333333334,\n",
      "            \"Recall\": 0.0673076923076923\n",
      "        },\n",
      "        \"test_time\": 0.00308577800024068,\n",
      "        \"train_time\": 42.69890288699935\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Results\n",
    "print(json.dumps(results_dict, indent=4, sort_keys=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.5",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}