experiments/06_HIGGS.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Experiment 06: HIGGS boson \n",
    "\n",
    "This experiment uses the data from the [HIGGS dataset](https://archive.ics.uci.edu/ml/datasets/HIGGS) to predict the appearance of the Higgs boson. The dataset consists of 11 million of observations. More information about the data can be found in [loaders.py](libs/loaders.py).  \n",
    "\n",
    "For details of virtual machine we used and the versions of LightGBM and XGBoost, please refer to [experiment 1](01_airline.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "System version: 3.5.3 |Anaconda 4.4.0 (64-bit)| (default, Mar  6 2017, 11:58:13) \n",
      "[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)]\n",
      "XGBoost version: 0.6\n",
      "LightGBM version: 0.2\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import sys\n",
    "import matplotlib.pyplot as plt\n",
    "import pkg_resources\n",
    "from libs.loaders import load_higgs\n",
    "from libs.timer import Timer\n",
    "from libs.utils import get_number_processors\n",
    "from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score, f1_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from xgboost import XGBClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "print(\"System version: {}\".format(sys.version))\n",
    "print(\"XGBoost version: {}\".format(pkg_resources.get_distribution('xgboost').version))\n",
    "print(\"LightGBM version: {}\".format(pkg_resources.get_distribution('lightgbm').version))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:libs.loaders:MOUNT_POINT not found in environment. Defaulting to /fileshare\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11000000, 29)\n",
      "CPU times: user 1min 12s, sys: 6.31 s, total: 1min 18s\n",
      "Wall time: 4min 15s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df = load_higgs()\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>boson</th>\n",
       "      <th>lepton_pT</th>\n",
       "      <th>lepton_eta</th>\n",
       "      <th>lepton_phi</th>\n",
       "      <th>missing_energy_magnitude</th>\n",
       "      <th>missing_energy_phi</th>\n",
       "      <th>jet_1_pt</th>\n",
       "      <th>jet_1_eta</th>\n",
       "      <th>jet_1_phi</th>\n",
       "      <th>jet_1_b-tag</th>\n",
       "      <th>...</th>\n",
       "      <th>jet_4_eta</th>\n",
       "      <th>jet_4_phi</th>\n",
       "      <th>jet_4_b-tag</th>\n",
       "      <th>m_jj</th>\n",
       "      <th>m_jjj</th>\n",
       "      <th>m_lv</th>\n",
       "      <th>m_jlv</th>\n",
       "      <th>m_bb</th>\n",
       "      <th>m_wbb</th>\n",
       "      <th>m_wwbb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.869293</td>\n",
       "      <td>-0.635082</td>\n",
       "      <td>0.225690</td>\n",
       "      <td>0.327470</td>\n",
       "      <td>-0.689993</td>\n",
       "      <td>0.754202</td>\n",
       "      <td>-0.248573</td>\n",
       "      <td>-1.092064</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.010455</td>\n",
       "      <td>-0.045767</td>\n",
       "      <td>3.101961</td>\n",
       "      <td>1.353760</td>\n",
       "      <td>0.979563</td>\n",
       "      <td>0.978076</td>\n",
       "      <td>0.920005</td>\n",
       "      <td>0.721657</td>\n",
       "      <td>0.988751</td>\n",
       "      <td>0.876678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.907542</td>\n",
       "      <td>0.329147</td>\n",
       "      <td>0.359412</td>\n",
       "      <td>1.497970</td>\n",
       "      <td>-0.313010</td>\n",
       "      <td>1.095531</td>\n",
       "      <td>-0.557525</td>\n",
       "      <td>-1.588230</td>\n",
       "      <td>2.173076</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.138930</td>\n",
       "      <td>-0.000819</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.302220</td>\n",
       "      <td>0.833048</td>\n",
       "      <td>0.985700</td>\n",
       "      <td>0.978098</td>\n",
       "      <td>0.779732</td>\n",
       "      <td>0.992356</td>\n",
       "      <td>0.798343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.798835</td>\n",
       "      <td>1.470639</td>\n",
       "      <td>-1.635975</td>\n",
       "      <td>0.453773</td>\n",
       "      <td>0.425629</td>\n",
       "      <td>1.104875</td>\n",
       "      <td>1.282322</td>\n",
       "      <td>1.381664</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.128848</td>\n",
       "      <td>0.900461</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.909753</td>\n",
       "      <td>1.108330</td>\n",
       "      <td>0.985692</td>\n",
       "      <td>0.951331</td>\n",
       "      <td>0.803252</td>\n",
       "      <td>0.865924</td>\n",
       "      <td>0.780118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.344385</td>\n",
       "      <td>-0.876626</td>\n",
       "      <td>0.935913</td>\n",
       "      <td>1.992050</td>\n",
       "      <td>0.882454</td>\n",
       "      <td>1.786066</td>\n",
       "      <td>-1.646778</td>\n",
       "      <td>-0.942383</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.678379</td>\n",
       "      <td>-1.360356</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.946652</td>\n",
       "      <td>1.028704</td>\n",
       "      <td>0.998656</td>\n",
       "      <td>0.728281</td>\n",
       "      <td>0.869200</td>\n",
       "      <td>1.026736</td>\n",
       "      <td>0.957904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.105009</td>\n",
       "      <td>0.321356</td>\n",
       "      <td>1.522401</td>\n",
       "      <td>0.882808</td>\n",
       "      <td>-1.205349</td>\n",
       "      <td>0.681466</td>\n",
       "      <td>-1.070464</td>\n",
       "      <td>-0.921871</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.373566</td>\n",
       "      <td>0.113041</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.755856</td>\n",
       "      <td>1.361057</td>\n",
       "      <td>0.986610</td>\n",
       "      <td>0.838085</td>\n",
       "      <td>1.133295</td>\n",
       "      <td>0.872245</td>\n",
       "      <td>0.808487</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   boson  lepton_pT  lepton_eta  lepton_phi  missing_energy_magnitude  \\\n",
       "0    1.0   0.869293   -0.635082    0.225690                  0.327470   \n",
       "1    1.0   0.907542    0.329147    0.359412                  1.497970   \n",
       "2    1.0   0.798835    1.470639   -1.635975                  0.453773   \n",
       "3    0.0   1.344385   -0.876626    0.935913                  1.992050   \n",
       "4    1.0   1.105009    0.321356    1.522401                  0.882808   \n",
       "\n",
       "   missing_energy_phi  jet_1_pt  jet_1_eta  jet_1_phi  jet_1_b-tag    ...     \\\n",
       "0           -0.689993  0.754202  -0.248573  -1.092064     0.000000    ...      \n",
       "1           -0.313010  1.095531  -0.557525  -1.588230     2.173076    ...      \n",
       "2            0.425629  1.104875   1.282322   1.381664     0.000000    ...      \n",
       "3            0.882454  1.786066  -1.646778  -0.942383     0.000000    ...      \n",
       "4           -1.205349  0.681466  -1.070464  -0.921871     0.000000    ...      \n",
       "\n",
       "   jet_4_eta  jet_4_phi  jet_4_b-tag      m_jj     m_jjj      m_lv     m_jlv  \\\n",
       "0  -0.010455  -0.045767     3.101961  1.353760  0.979563  0.978076  0.920005   \n",
       "1  -1.138930  -0.000819     0.000000  0.302220  0.833048  0.985700  0.978098   \n",
       "2   1.128848   0.900461     0.000000  0.909753  1.108330  0.985692  0.951331   \n",
       "3  -0.678379  -1.360356     0.000000  0.946652  1.028704  0.998656  0.728281   \n",
       "4  -0.373566   0.113041     0.000000  0.755856  1.361057  0.986610  0.838085   \n",
       "\n",
       "       m_bb     m_wbb    m_wwbb  \n",
       "0  0.721657  0.988751  0.876678  \n",
       "1  0.779732  0.992356  0.798343  \n",
       "2  0.803252  0.865924  0.780118  \n",
       "3  0.869200  1.026736  0.957904  \n",
       "4  1.133295  0.872245  0.808487  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24\n"
     ]
    }
   ],
   "source": [
    "num_rounds = 200\n",
    "number_processors = get_number_processors()\n",
    "print(number_processors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "xgb_clf_pipeline = XGBClassifier(max_depth=5, \n",
    "                                learning_rate=0.1, \n",
    "                                scale_pos_weight=2,\n",
    "                                n_estimators=num_rounds,\n",
    "                                gamma=0.1,\n",
    "                                min_child_weight=1,\n",
    "                                reg_lambda=1,\n",
    "                                subsample=1,\n",
    "                                nthread=number_processors\n",
    "                                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "xgb_hist_clf_pipeline = XGBClassifier(max_depth=0, \n",
    "                                     learning_rate=0.1, \n",
    "                                     scale_pos_weight=2,\n",
    "                                     n_estimators=num_rounds,\n",
    "                                     gamma=0.1,\n",
    "                                     min_child_weight=1,\n",
    "                                     reg_lambda=1,\n",
    "                                     subsample=1,\n",
    "                                     max_leaves=2**5,\n",
    "                                     grow_policy='lossguide',\n",
    "                                     tree_method='hist',\n",
    "                                     nthread=number_processors\n",
    "                                     )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "lgbm_clf_pipeline = LGBMClassifier(num_leaves=2**5, \n",
    "                                  learning_rate=0.1, \n",
    "                                  scale_pos_weight=2,\n",
    "                                  n_estimators=num_rounds,\n",
    "                                  min_split_gain=0.1,\n",
    "                                  min_child_weight=1,\n",
    "                                  reg_lambda=1,\n",
    "                                  subsample=1,\n",
    "                                  nthread=number_processors\n",
    "                                  )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "metrics_dict = {\n",
    "    'Accuracy': accuracy_score,\n",
    "    'Precision': precision_score,\n",
    "    'Recall': recall_score,\n",
    "    'AUC': roc_auc_score,\n",
    "    'F1': f1_score,\n",
    "}\n",
    "\n",
    "def classification_metrics(metrics, y_true, y_pred):\n",
    "    return {metric_name:metric(y_true, y_pred) for metric_name, metric in metrics.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def generate_feables(df):\n",
    "    X = df[df.columns.difference(['boson'])]\n",
    "    y = df['boson']\n",
    "    return X,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X, y = generate_feables(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=77, test_size=500000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict = dict()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer() as train_t:\n",
    "    xgb_clf_pipeline.fit(X_train,y_train)\n",
    "    \n",
    "with Timer() as test_t:\n",
    "    y_pred = xgb_clf_pipeline.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['xgb']={\n",
    "    'train_time': train_t.interval,\n",
    "    'test_time': test_t.interval,\n",
    "    'performance': classification_metrics(metrics_dict, \n",
    "                                          y_test, \n",
    "                                          y_pred) \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer() as t_train:\n",
    "    xgb_hist_clf_pipeline.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer() as t_test:\n",
    "    y_pred = xgb_hist_clf_pipeline.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['xgb_hist']={\n",
    "    'train_time': t_train.interval,\n",
    "    'test_time': t_test.interval,\n",
    "    'performance': classification_metrics(metrics_dict, \n",
    "                                          y_test, \n",
    "                                          y_pred) \n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with Timer() as train_t:\n",
    "    lgbm_clf_pipeline.fit(X_train, y_train)\n",
    "    \n",
    "with Timer() as test_t:\n",
    "    y_pred = lgbm_clf_pipeline.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "results_dict['lgbm']={\n",
    "    'train_time': train_t.interval,\n",
    "    'test_time': test_t.interval,\n",
    "    'performance': classification_metrics(metrics_dict, \n",
    "                                          y_test, \n",
    "                                          y_pred) \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"lgbm\": {\n",
      "        \"performance\": {\n",
      "            \"AUC\": 0.694682949690134,\n",
      "            \"Accuracy\": 0.707758,\n",
      "            \"F1\": 0.7680747894958216,\n",
      "            \"Precision\": 0.6627597069095391,\n",
      "            \"Recall\": 0.9131831219806763\n",
      "        },\n",
      "        \"test_time\": 0.7120589099995414,\n",
      "        \"train_time\": 119.34003880199998\n",
      "    },\n",
      "    \"xgb\": {\n",
      "        \"performance\": {\n",
      "            \"AUC\": 0.6859901403358623,\n",
      "            \"Accuracy\": 0.699694,\n",
      "            \"F1\": 0.7635493812093622,\n",
      "            \"Precision\": 0.6551156676187414,\n",
      "            \"Recall\": 0.9149984903381643\n",
      "        },\n",
      "        \"test_time\": 0.55617916600022,\n",
      "        \"train_time\": 2996.1667750769993\n",
      "    },\n",
      "    \"xgb_hist\": {\n",
      "        \"performance\": {\n",
      "            \"AUC\": 0.6941216899970567,\n",
      "            \"Accuracy\": 0.70721,\n",
      "            \"F1\": 0.767674555527519,\n",
      "            \"Precision\": 0.6623426413523601,\n",
      "            \"Recall\": 0.9128434480676328\n",
      "        },\n",
      "        \"test_time\": 0.6464068210007099,\n",
      "        \"train_time\": 121.21175534400027\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Results\n",
    "print(json.dumps(results_dict, indent=4, sort_keys=True))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python Strata",
   "language": "python",
   "name": "strata"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}