thuan-xgboost-pd.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "administrative-transfer",
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from util import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "charming-bolivia",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = load_data(\"data/data-202008.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "engaging-following",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\admin\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\pandas\\core\\frame.py:3191: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self[k1] = value[k2]\n"
     ]
    }
   ],
   "source": [
    "cat_features = [\"province_code\", \"district_code\",\"viettel_bank_code\",\"channel_code\"\n",
    "                ,\"staff_code\",\"process_code\",\"shop_code\"]\n",
    "\n",
    "data = preprocess_data(data, cat_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "electrical-excuse",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Label encoder\n",
    "for c in data.columns[data.dtypes == 'object']:\n",
    "    data[c] = data[c].factorize()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "related-fluid",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "int64      10\n",
       "float64     6\n",
       "bool        4\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dtypes.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "extensive-broadcast",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>province_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>date_diff</th>\n",
       "      <th>viettel_bank_code</th>\n",
       "      <th>channel_type_id</th>\n",
       "      <th>channel_code</th>\n",
       "      <th>staff_code</th>\n",
       "      <th>trans_amount</th>\n",
       "      <th>trans_fee</th>\n",
       "      <th>trans_type</th>\n",
       "      <th>process_code</th>\n",
       "      <th>channel_fee</th>\n",
       "      <th>shop_code</th>\n",
       "      <th>customer_fee</th>\n",
       "      <th>fee_partner</th>\n",
       "      <th>is_fraud</th>\n",
       "      <th>same_name</th>\n",
       "      <th>same_phone</th>\n",
       "      <th>same_phone_channel</th>\n",
       "      <th>same_phone_channel_ben</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>164.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>500000.0</td>\n",
       "      <td>8182.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5400.0</td>\n",
       "      <td>0</td>\n",
       "      <td>18000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>164.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3000000.0</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>5500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>11000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>600000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>197.0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>393692.0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>770.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1100.0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>349384.0</td>\n",
       "      <td>909.0</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>700.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   province_code  district_code  date_diff  viettel_bank_code  \\\n",
       "0              0              0         16                  0   \n",
       "1              1              1         16                  0   \n",
       "2              2              2         16                  1   \n",
       "3              1              3         16                  2   \n",
       "4              3              4         16                  2   \n",
       "\n",
       "   channel_type_id  channel_code  staff_code  trans_amount  trans_fee  \\\n",
       "0            164.0             0           0      500000.0     8182.0   \n",
       "1            164.0             1           1     3000000.0    10000.0   \n",
       "2              6.0             2           2      600000.0        0.0   \n",
       "3            197.0             3           3      393692.0     1000.0   \n",
       "4              1.0             4           4      349384.0      909.0   \n",
       "\n",
       "   trans_type  process_code  channel_fee  shop_code  customer_fee  \\\n",
       "0           1             0       5400.0          0       18000.0   \n",
       "1           2             1       5500.0          1       11000.0   \n",
       "2           2             2       1260.0          2           0.0   \n",
       "3           3             3        770.0          3           0.0   \n",
       "4           3             4        700.0          4           0.0   \n",
       "\n",
       "   fee_partner  is_fraud  same_name  same_phone  same_phone_channel  \\\n",
       "0          0.0         0      False        True               False   \n",
       "1          0.0         0      False        True               False   \n",
       "2          0.0         0       True        True               False   \n",
       "3       1100.0         0      False       False               False   \n",
       "4       1000.0         0      False       False                True   \n",
       "\n",
       "   same_phone_channel_ben  \n",
       "0                   False  \n",
       "1                   False  \n",
       "2                   False  \n",
       "3                   False  \n",
       "4                   False  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "offshore-railway",
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y, X_train, y_train, X_val, y_val, X_test, y_test = split_data(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "civil-victim",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from xgboost import XGBClassifier\n",
    "# from sklearn.model_selection import cross_val_score\n",
    "# from sklearn.model_selection import RepeatedStratifiedKFold\n",
    "# # define model\n",
    "# model = XGBClassifier()\n",
    "# # define evaluation procedure\n",
    "# cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=1)\n",
    "# # evaluate model\n",
    "# scores = cross_val_score(model, X, y, scoring='roc_auc', cv=cv, n_jobs=2)\n",
    "# # summarize performance\n",
    "# print('Mean ROC AUC: %.5f' % mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "defined-julian",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[16:52:54] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:541: \n",
      "Parameters: { missing, n_estimators, silent } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      "[16:52:59] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    }
   ],
   "source": [
    "D_train = xgb.DMatrix(X_train, label=y_train)\n",
    "D_val = xgb.DMatrix(X_val, label=y_val)\n",
    "D_test = xgb.DMatrix(X_test, label=y_test)\n",
    "\n",
    "# parameters = {'max_depth': 4,\n",
    "#          'eta': 0.01,\n",
    "#          'objective': 'binary:logistic',\n",
    "#          'eval_metric': 'error',\n",
    "#          'nthread': 4}\n",
    "\n",
    "parameters = {'nthread':4, #when use hyperthread, xgboost may become slower\n",
    "              'objective':'binary:logistic',\n",
    "              'learning_rate': 0.05, #so called `eta` value\n",
    "              'max_depth': 6,\n",
    "              'min_child_weight': 11,\n",
    "              'silent': 1,\n",
    "              'subsample': 0.8,\n",
    "              'colsample_bytree': 0.7,\n",
    "              'n_estimators': 5, #number of trees, change it to 1000 for better results\n",
    "              'missing':-999,\n",
    "              'seed': 1337,\n",
    "                'scale_pos_weight':10}\n",
    "\n",
    "steps = 20  # The number of training iterations\n",
    "\n",
    "model = xgb.train(parameters, D_train, steps)\n",
    "# #xgb.fit(X_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "remarkable-packet",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_model(model, 'xgb-model.saved')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "southern-craps",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "classification_report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00   1433272\n",
      "           1       0.38      0.67      0.49      5430\n",
      "\n",
      "    accuracy                           0.99   1438702\n",
      "   macro avg       0.69      0.83      0.74   1438702\n",
      "weighted avg       1.00      0.99      1.00   1438702\n",
      "\n",
      "confusion_matrix: \n",
      "[[1427400    5872]\n",
      " [   1775    3655]]\n",
      "roc_auc_score: 0.8345077096810526\n",
      "f1_score: 0.488734371866016\n",
      "precision_score: 0.38364647842972605\n",
      "recall_score: 0.6731123388581952\n"
     ]
    }
   ],
   "source": [
    "#eval_model(model, X_test, y_test)\n",
    "model = load_model('xgb-model.saved')\n",
    "y_pred = model.predict(D_test)\n",
    "print('classification_report: \\n{}'.format(classification_report(y_test, y_pred.round())))\n",
    "print('confusion_matrix: \\n{}'.format(confusion_matrix(y_test,  y_pred.round())))\n",
    "print('roc_auc_score: {}'.format(roc_auc_score(y_test,  y_pred.round())))\n",
    "print('f1_score: {}'.format(f1_score(y_test,  y_pred.round())))\n",
    "print('precision_score: {}'.format(precision_score(y_test,  y_pred.round())))\n",
    "print('recall_score: {}'.format(recall_score(y_test,  y_pred.round())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "universal-texture",
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'sklearn.grid_search'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-12-ecf37c95d388>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgrid_search\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mGridSearchCV\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcross_validation\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[1;33m*\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mxgb_model\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mxgb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mXGBClassifier\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'sklearn.grid_search'"
     ]
    }
   ],
   "source": [
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.cross_validation import *\n",
    "\n",
    "xgb_model = xgb.XGBClassifier()\n",
    "\n",
    "#brute force scan for all parameters, here are the tricks\n",
    "#usually max_depth is 6,7,8\n",
    "#learning rate is around 0.05, but small changes may make big diff\n",
    "#tuning min_child_weight subsample colsample_bytree can have \n",
    "#much fun of fighting against overfit \n",
    "#n_estimators is how many round of boosting\n",
    "#finally, ensemble xgboost with multiple seeds may reduce variance\n",
    "parameters = {'nthread':[4], #when use hyperthread, xgboost may become slower\n",
    "              'objective':['binary:logistic'],\n",
    "              'learning_rate': [0.05], #so called `eta` value\n",
    "              'max_depth': [6],\n",
    "              'min_child_weight': [11],\n",
    "              'silent': [1],\n",
    "              'subsample': [0.8],\n",
    "              'colsample_bytree': [0.7],\n",
    "              'n_estimators': [5], #number of trees, change it to 1000 for better results\n",
    "              'missing':[-999],\n",
    "              'seed': [1337]}\n",
    "\n",
    "\n",
    "clf = GridSearchCV(xgb_model, parameters, n_jobs=5, \n",
    "                   cv=StratifiedKFold(train['QuoteConversion_Flag'], n_folds=5, shuffle=True), \n",
    "                   scoring='roc_auc',\n",
    "                   verbose=2, refit=True)\n",
    "\n",
    "clf.fit(train[features], train[\"QuoteConversion_Flag\"])\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}