experiments/05_FraudDetection_GPU.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Experiment 05: Credit card Fraud (GPU version)\n",
    "\n",
    "This experiment uses the data from the Kaggle dataset [Credit Card Fraud Detection](https://www.kaggle.com/dalpozz/creditcardfraud). The dataset is made up of a number of variables which are a result of PCA transformation.\n",
    "\n",
    "The details of the machine we used and the version of the libraries can be found in [experiment 01](01_airline.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "System version: 3.5.2 |Anaconda custom (64-bit)| (default, Jul  2 2016, 17:53:06) \n",
      "[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)]\n",
      "XGBoost version: 0.6\n",
      "LightGBM version: 0.2\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import sys\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import pkg_resources\n",
    "from libs.loaders import load_fraud\n",
    "from libs.timer import Timer\n",
    "from libs.metrics import classification_metrics_binary, classification_metrics_binary_prob, binarize_prediction\n",
    "import xgboost as xgb\n",
    "import lightgbm as lgb\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "print(\"System version: {}\".format(sys.version))\n",
    "print(\"XGBoost version: {}\".format(pkg_resources.get_distribution('xgboost').version))\n",
    "print(\"LightGBM version: {}\".format(pkg_resources.get_distribution('lightgbm').version))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "random_seed = 42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:libs.loaders:MOUNT_POINT not found in environment. Defaulting to /fileshare\n"
     ]
    }
   ],
   "source": [
    "df = load_fraud()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(284807, 31)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Time</th>\n",
       "      <th>V1</th>\n",
       "      <th>V2</th>\n",
       "      <th>V3</th>\n",
       "      <th>V4</th>\n",
       "      <th>V5</th>\n",
       "      <th>V6</th>\n",
       "      <th>V7</th>\n",
       "      <th>V8</th>\n",
       "      <th>V9</th>\n",
       "      <th>...</th>\n",
       "      <th>V21</th>\n",
       "      <th>V22</th>\n",
       "      <th>V23</th>\n",
       "      <th>V24</th>\n",
       "      <th>V25</th>\n",
       "      <th>V26</th>\n",
       "      <th>V27</th>\n",
       "      <th>V28</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.359807</td>\n",
       "      <td>-0.072781</td>\n",
       "      <td>2.536347</td>\n",
       "      <td>1.378155</td>\n",
       "      <td>-0.338321</td>\n",
       "      <td>0.462388</td>\n",
       "      <td>0.239599</td>\n",
       "      <td>0.098698</td>\n",
       "      <td>0.363787</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.018307</td>\n",
       "      <td>0.277838</td>\n",
       "      <td>-0.110474</td>\n",
       "      <td>0.066928</td>\n",
       "      <td>0.128539</td>\n",
       "      <td>-0.189115</td>\n",
       "      <td>0.133558</td>\n",
       "      <td>-0.021053</td>\n",
       "      <td>149.62</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.191857</td>\n",
       "      <td>0.266151</td>\n",
       "      <td>0.166480</td>\n",
       "      <td>0.448154</td>\n",
       "      <td>0.060018</td>\n",
       "      <td>-0.082361</td>\n",
       "      <td>-0.078803</td>\n",
       "      <td>0.085102</td>\n",
       "      <td>-0.255425</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.225775</td>\n",
       "      <td>-0.638672</td>\n",
       "      <td>0.101288</td>\n",
       "      <td>-0.339846</td>\n",
       "      <td>0.167170</td>\n",
       "      <td>0.125895</td>\n",
       "      <td>-0.008983</td>\n",
       "      <td>0.014724</td>\n",
       "      <td>2.69</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.358354</td>\n",
       "      <td>-1.340163</td>\n",
       "      <td>1.773209</td>\n",
       "      <td>0.379780</td>\n",
       "      <td>-0.503198</td>\n",
       "      <td>1.800499</td>\n",
       "      <td>0.791461</td>\n",
       "      <td>0.247676</td>\n",
       "      <td>-1.514654</td>\n",
       "      <td>...</td>\n",
       "      <td>0.247998</td>\n",
       "      <td>0.771679</td>\n",
       "      <td>0.909412</td>\n",
       "      <td>-0.689281</td>\n",
       "      <td>-0.327642</td>\n",
       "      <td>-0.139097</td>\n",
       "      <td>-0.055353</td>\n",
       "      <td>-0.059752</td>\n",
       "      <td>378.66</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.966272</td>\n",
       "      <td>-0.185226</td>\n",
       "      <td>1.792993</td>\n",
       "      <td>-0.863291</td>\n",
       "      <td>-0.010309</td>\n",
       "      <td>1.247203</td>\n",
       "      <td>0.237609</td>\n",
       "      <td>0.377436</td>\n",
       "      <td>-1.387024</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.108300</td>\n",
       "      <td>0.005274</td>\n",
       "      <td>-0.190321</td>\n",
       "      <td>-1.175575</td>\n",
       "      <td>0.647376</td>\n",
       "      <td>-0.221929</td>\n",
       "      <td>0.062723</td>\n",
       "      <td>0.061458</td>\n",
       "      <td>123.50</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>-1.158233</td>\n",
       "      <td>0.877737</td>\n",
       "      <td>1.548718</td>\n",
       "      <td>0.403034</td>\n",
       "      <td>-0.407193</td>\n",
       "      <td>0.095921</td>\n",
       "      <td>0.592941</td>\n",
       "      <td>-0.270533</td>\n",
       "      <td>0.817739</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.009431</td>\n",
       "      <td>0.798278</td>\n",
       "      <td>-0.137458</td>\n",
       "      <td>0.141267</td>\n",
       "      <td>-0.206010</td>\n",
       "      <td>0.502292</td>\n",
       "      <td>0.219422</td>\n",
       "      <td>0.215153</td>\n",
       "      <td>69.99</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Time        V1        V2        V3        V4        V5        V6        V7  \\\n",
       "0   0.0 -1.359807 -0.072781  2.536347  1.378155 -0.338321  0.462388  0.239599   \n",
       "1   0.0  1.191857  0.266151  0.166480  0.448154  0.060018 -0.082361 -0.078803   \n",
       "2   1.0 -1.358354 -1.340163  1.773209  0.379780 -0.503198  1.800499  0.791461   \n",
       "3   1.0 -0.966272 -0.185226  1.792993 -0.863291 -0.010309  1.247203  0.237609   \n",
       "4   2.0 -1.158233  0.877737  1.548718  0.403034 -0.407193  0.095921  0.592941   \n",
       "\n",
       "         V8        V9  ...         V21       V22       V23       V24  \\\n",
       "0  0.098698  0.363787  ...   -0.018307  0.277838 -0.110474  0.066928   \n",
       "1  0.085102 -0.255425  ...   -0.225775 -0.638672  0.101288 -0.339846   \n",
       "2  0.247676 -1.514654  ...    0.247998  0.771679  0.909412 -0.689281   \n",
       "3  0.377436 -1.387024  ...   -0.108300  0.005274 -0.190321 -1.175575   \n",
       "4 -0.270533  0.817739  ...   -0.009431  0.798278 -0.137458  0.141267   \n",
       "\n",
       "        V25       V26       V27       V28  Amount  Class  \n",
       "0  0.128539 -0.189115  0.133558 -0.021053  149.62      0  \n",
       "1  0.167170  0.125895 -0.008983  0.014724    2.69      0  \n",
       "2 -0.327642 -0.139097 -0.055353 -0.059752  378.66      0  \n",
       "3  0.647376 -0.221929  0.062723  0.061458  123.50      0  \n",
       "4 -0.206010  0.502292  0.219422  0.215153   69.99      0  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X = df[[col for col in df.columns if col.startswith('V')]].values\n",
    "y = df['Class'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(199364, 28)\n",
      "(199364,)\n",
      "(85443, 28)\n",
      "(85443,)\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=random_seed, test_size=0.3)\n",
    "print(X_train.shape)\n",
    "print(y_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "dtrain = xgb.DMatrix(data=X_train, label=y_train)\n",
    "dtest = xgb.DMatrix(data=X_test, label=y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "lgb_train = lgb.Dataset(X_train, y_train, free_raw_data=False)\n",
    "lgb_test = lgb.Dataset(X_test, y_test, reference=lgb_train, free_raw_data=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict = dict()\n",
    "num_rounds = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "params = {'max_depth':3, \n",
    "          'objective':'binary:logistic', \n",
    "          'min_child_weight':1, \n",
    "          'eta':0.1, \n",
    "          'colsample_bytree':1, \n",
    "          'scale_pos_weight':2, \n",
    "          'gamma':0.1, \n",
    "          'reg_lamda':1, \n",
    "          'subsample':1,\n",
    "          'tree_method':'exact', \n",
    "          'updater':'grow_gpu'\n",
    "          }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer() as t_train:\n",
    "    xgb_clf_pipeline = xgb.train(params, dtrain, num_boost_round=num_rounds)\n",
    "    \n",
    "with Timer() as t_test:\n",
    "    y_prob_xgb = xgb_clf_pipeline.predict(dtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "y_pred_xgb = binarize_prediction(y_prob_xgb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "report_xgb = classification_metrics_binary(y_test, y_pred_xgb)\n",
    "report2_xgb = classification_metrics_binary_prob(y_test, y_prob_xgb)\n",
    "report_xgb.update(report2_xgb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['xgb']={\n",
    "    'train_time': t_train.interval,\n",
    "    'test_time': t_test.interval,\n",
    "    'performance': report_xgb \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "del xgb_clf_pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now let's try with XGBoost histogram."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "params = {'max_depth':0, \n",
    "          'objective':'binary:logistic', \n",
    "          'min_child_weight':1, \n",
    "          'eta':0.1, \n",
    "          'colsample_bytree':0.80, \n",
    "          'scale_pos_weight':2, \n",
    "          'gamma':0.1, \n",
    "          'reg_lamda':1, \n",
    "          'subsample':1,\n",
    "          'tree_method':'hist', \n",
    "          'max_leaves':2**3, \n",
    "          'grow_policy':'lossguide', \n",
    "         }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer() as t_train:\n",
    "    xgb_hist_clf_pipeline = xgb.train(params, dtrain, num_boost_round=num_rounds)\n",
    "    \n",
    "with Timer() as t_test:\n",
    "    y_prob_xgb_hist = xgb_hist_clf_pipeline.predict(dtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "y_pred_xgb_hist = binarize_prediction(y_prob_xgb_hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "report_xgb_hist = classification_metrics_binary(y_test, y_pred_xgb_hist)\n",
    "report2_xgb_hist = classification_metrics_binary_prob(y_test, y_prob_xgb_hist)\n",
    "report_xgb_hist.update(report2_xgb_hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['xgb_hist']={\n",
    "    'train_time': t_train.interval,\n",
    "    'test_time': t_test.interval,\n",
    "    'performance': report_xgb_hist\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "del xgb_hist_clf_pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "params = {'num_leaves': 2**3,\n",
    "         'learning_rate': 0.1,\n",
    "         'scale_pos_weight': 2,\n",
    "         'min_split_gain': 0.1,\n",
    "         'min_child_weight': 1,\n",
    "         'reg_lambda': 1,\n",
    "         'subsample': 1,\n",
    "         'objective':'binary',\n",
    "         'task': 'train'\n",
    "         }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer() as t_train:\n",
    "    lgbm_clf_pipeline = lgb.train(params, lgb_train, num_boost_round=num_rounds)\n",
    "    \n",
    "with Timer() as t_test:\n",
    "    y_prob_lgbm = lgbm_clf_pipeline.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "y_pred_lgbm = binarize_prediction(y_prob_lgbm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "report_lgbm = classification_metrics_binary(y_test, y_pred_lgbm)\n",
    "report2_lgbm = classification_metrics_binary_prob(y_test, y_prob_lgbm)\n",
    "report_lgbm.update(report2_lgbm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['lgbm']={\n",
    "    'train_time': t_train.interval,\n",
    "    'test_time': t_test.interval,\n",
    "    'performance': report_lgbm \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "del lgbm_clf_pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Finally, we show the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"lgbm\": {\n",
      "        \"performance\": {\n",
      "            \"AUC\": 0.96572630283135,\n",
      "            \"Accuracy\": 0.9993914071369217,\n",
      "            \"F1\": 0.8115942028985507,\n",
      "            \"Precision\": 0.875,\n",
      "            \"Recall\": 0.7567567567567568\n",
      "        },\n",
      "        \"test_time\": 0.04103280100025586,\n",
      "        \"train_time\": 0.2938522020012897\n",
      "    },\n",
      "    \"xgb\": {\n",
      "        \"performance\": {\n",
      "            \"AUC\": 0.9699805365480376,\n",
      "            \"Accuracy\": 0.9994148145547324,\n",
      "            \"F1\": 0.8214285714285714,\n",
      "            \"Precision\": 0.8712121212121212,\n",
      "            \"Recall\": 0.777027027027027\n",
      "        },\n",
      "        \"test_time\": 0.03786925599706592,\n",
      "        \"train_time\": 5.8025254829990445\n",
      "    },\n",
      "    \"xgb_hist\": {\n",
      "        \"performance\": {\n",
      "            \"AUC\": 0.9692088902901377,\n",
      "            \"Accuracy\": 0.9994148145547324,\n",
      "            \"F1\": 0.8148148148148148,\n",
      "            \"Precision\": 0.9016393442622951,\n",
      "            \"Recall\": 0.7432432432432432\n",
      "        },\n",
      "        \"test_time\": 0.00477967400001944,\n",
      "        \"train_time\": 1.6418742589994508\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Results\n",
    "print(json.dumps(results_dict, indent=4, sort_keys=True))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.5",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}