thuan-catboost-colab.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import seaborn as sns \n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report, confusion_matrix, roc_auc_score, precision_score, recall_score, f1_score\n",
    "import pickle\n",
    "from catboost import CatBoostClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import urllib.request\n",
    "urllib.request.urlretrieve('https://raw.githubusercontent.com/doanthuan/viettelpay/master/util.py', 'util.py')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from util import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('max_columns', None)\n",
    "#pd.set_option(\"max_rows\", None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>request_date</th>\n",
       "      <th>is_fraud</th>\n",
       "      <th>province_code</th>\n",
       "      <th>district_code</th>\n",
       "      <th>viettel_bank_code</th>\n",
       "      <th>channel_type_id</th>\n",
       "      <th>channel_code</th>\n",
       "      <th>staff_code</th>\n",
       "      <th>trans_amount</th>\n",
       "      <th>trans_fee</th>\n",
       "      <th>trans_type</th>\n",
       "      <th>process_code</th>\n",
       "      <th>channel_fee</th>\n",
       "      <th>shop_code</th>\n",
       "      <th>customer_fee</th>\n",
       "      <th>cust_name</th>\n",
       "      <th>ben_cust_name</th>\n",
       "      <th>msisdn</th>\n",
       "      <th>ben_msisdn</th>\n",
       "      <th>msisdn_channel</th>\n",
       "      <th>staff_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-07-20 12:00:48</td>\n",
       "      <td>0</td>\n",
       "      <td>B651</td>\n",
       "      <td>B651004</td>\n",
       "      <td>KPP</td>\n",
       "      <td>1</td>\n",
       "      <td>1000810100_00608_DB</td>\n",
       "      <td>1000810100_00608_DB</td>\n",
       "      <td>8955989.0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>KPP301</td>\n",
       "      <td>770.0</td>\n",
       "      <td>1000810100_VTP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0001f092d1091229f963</td>\n",
       "      <td>None</td>\n",
       "      <td>0001f092d1091229f963</td>\n",
       "      <td>Dien Luc Loc Ninh_Nguyen Quang Huy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-07-20 06:06:26</td>\n",
       "      <td>0</td>\n",
       "      <td>L072</td>\n",
       "      <td>L072008</td>\n",
       "      <td>KPP</td>\n",
       "      <td>1</td>\n",
       "      <td>1000239100_00658_DB</td>\n",
       "      <td>1000239100_00658_DB</td>\n",
       "      <td>270614.0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>KPP301</td>\n",
       "      <td>770.0</td>\n",
       "      <td>1000239100_VTP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>00019640e87c3bb7310b</td>\n",
       "      <td>None</td>\n",
       "      <td>000153defbfe4df71f2d</td>\n",
       "      <td>Tap hoa Thanh Thong-nguyen thi ngoc giau</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-07-20 11:25:25</td>\n",
       "      <td>0</td>\n",
       "      <td>T008</td>\n",
       "      <td>T00802</td>\n",
       "      <td>MB</td>\n",
       "      <td>164</td>\n",
       "      <td>LAMHV5_VTPOST_HCM</td>\n",
       "      <td>LAMHV5_VTPOST_HCM</td>\n",
       "      <td>2000000.0</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>2</td>\n",
       "      <td>400300</td>\n",
       "      <td>5500.0</td>\n",
       "      <td>1800590048</td>\n",
       "      <td>11000.0</td>\n",
       "      <td>nguyen van tiep</td>\n",
       "      <td>NGUYEN VAN TIEP</td>\n",
       "      <td>0001115a8f19769edb47</td>\n",
       "      <td>0001115a8f19769edb47</td>\n",
       "      <td>000165dac633c3b694c7</td>\n",
       "      <td>Huỳnh Văn Lắm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020-07-20 14:17:50</td>\n",
       "      <td>0</td>\n",
       "      <td>T008</td>\n",
       "      <td>T00801</td>\n",
       "      <td>KPP</td>\n",
       "      <td>163</td>\n",
       "      <td>4400590001</td>\n",
       "      <td>4400590001</td>\n",
       "      <td>800000.0</td>\n",
       "      <td>8182.0</td>\n",
       "      <td>4</td>\n",
       "      <td>517000</td>\n",
       "      <td>7200.0</td>\n",
       "      <td>4400590001</td>\n",
       "      <td>18000.0</td>\n",
       "      <td>Ho Thi Loan</td>\n",
       "      <td>Tran Duy Khanh</td>\n",
       "      <td>00014835e613e06fa003</td>\n",
       "      <td>0001058a41c5c3c31af9</td>\n",
       "      <td>0001db3e096d46c51b2f</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-07-20 02:49:25</td>\n",
       "      <td>0</td>\n",
       "      <td>H004</td>\n",
       "      <td>H004009</td>\n",
       "      <td>VIETTEL</td>\n",
       "      <td>164</td>\n",
       "      <td>THOMTT4_VTPOST_HNI</td>\n",
       "      <td>THOMTT4_VTPOST_HNI</td>\n",
       "      <td>912391.0</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>3</td>\n",
       "      <td>666003</td>\n",
       "      <td>660.0</td>\n",
       "      <td>1800240011</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Ngo Vi Thanh</td>\n",
       "      <td>Ngo Vi Thanh</td>\n",
       "      <td>000183d8f83cdd4fa068</td>\n",
       "      <td>None</td>\n",
       "      <td>00010d9d7268603b25b2</td>\n",
       "      <td>Trần Thị Thơm</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         request_date  is_fraud province_code district_code viettel_bank_code  \\\n",
       "0 2020-07-20 12:00:48         0          B651       B651004               KPP   \n",
       "1 2020-07-20 06:06:26         0          L072       L072008               KPP   \n",
       "2 2020-07-20 11:25:25         0          T008        T00802                MB   \n",
       "3 2020-07-20 14:17:50         0          T008        T00801               KPP   \n",
       "4 2020-07-20 02:49:25         0          H004       H004009           VIETTEL   \n",
       "\n",
       "  channel_type_id         channel_code           staff_code  trans_amount  \\\n",
       "0               1  1000810100_00608_DB  1000810100_00608_DB     8955989.0   \n",
       "1               1  1000239100_00658_DB  1000239100_00658_DB      270614.0   \n",
       "2             164    LAMHV5_VTPOST_HCM    LAMHV5_VTPOST_HCM     2000000.0   \n",
       "3             163           4400590001           4400590001      800000.0   \n",
       "4             164   THOMTT4_VTPOST_HNI   THOMTT4_VTPOST_HNI      912391.0   \n",
       "\n",
       "   trans_fee trans_type process_code  channel_fee       shop_code  \\\n",
       "0     1000.0          3       KPP301        770.0  1000810100_VTP   \n",
       "1     1000.0          3       KPP301        770.0  1000239100_VTP   \n",
       "2    10000.0          2       400300       5500.0      1800590048   \n",
       "3     8182.0          4       517000       7200.0      4400590001   \n",
       "4     1200.0          3       666003        660.0      1800240011   \n",
       "\n",
       "   customer_fee        cust_name    ben_cust_name                msisdn  \\\n",
       "0           0.0             None             None  0001f092d1091229f963   \n",
       "1           0.0             None             None  00019640e87c3bb7310b   \n",
       "2       11000.0  nguyen van tiep  NGUYEN VAN TIEP  0001115a8f19769edb47   \n",
       "3       18000.0      Ho Thi Loan   Tran Duy Khanh  00014835e613e06fa003   \n",
       "4           0.0     Ngo Vi Thanh     Ngo Vi Thanh  000183d8f83cdd4fa068   \n",
       "\n",
       "             ben_msisdn        msisdn_channel  \\\n",
       "0                  None  0001f092d1091229f963   \n",
       "1                  None  000153defbfe4df71f2d   \n",
       "2  0001115a8f19769edb47  000165dac633c3b694c7   \n",
       "3  0001058a41c5c3c31af9  0001db3e096d46c51b2f   \n",
       "4                  None  00010d9d7268603b25b2   \n",
       "\n",
       "                                 staff_name  \n",
       "0        Dien Luc Loc Ninh_Nguyen Quang Huy  \n",
       "1  Tap hoa Thanh Thong-nguyen thi ngoc giau  \n",
       "2                             Huỳnh Văn Lắm  \n",
       "3                                      None  \n",
       "4                             Trần Thị Thơm  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "use_cols = [\n",
    "    \"request_date\",\n",
    "    \"is_fraud\",\n",
    "    \"province_code\",\n",
    "    \"district_code\",\n",
    "    \"viettel_bank_code\",\n",
    "    \"channel_type_id\",\n",
    "    \"channel_code\",\n",
    "    \"staff_code\",\n",
    "    \"trans_amount\",\n",
    "    \"trans_fee\",\n",
    "    \"trans_type\",\n",
    "    \"process_code\",\n",
    "    \"channel_fee\",\n",
    "    \"shop_code\",\n",
    "    \"customer_fee\",\n",
    "    \"cust_name\",\n",
    "    \"ben_cust_name\",\n",
    "    \"msisdn\",\n",
    "    \"ben_msisdn\",\n",
    "    \"msisdn_channel\",\n",
    "    \"staff_name\"\n",
    "]\n",
    "train = pd.read_parquet(\"/content/drive/MyDrive/projects/viettelpay/code/data/data-2m.snappy.parquet\", columns=use_cols)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "account = pd.read_parquet(\"/content/drive/MyDrive/projects/viettelpay/code/data/account_sale_mapping.snappy.parquet\")\n",
    "sale = pd.read_parquet(\"/content/drive/MyDrive/projects/viettelpay/code/data/account_sale.snappy.parquet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trans_hist = pd.read_parquet(\"/content/drive/MyDrive/projects/viettelpay/code/data/trans_hist_2m.snappy.parquet\")\n",
    "trans_hist = trans_hist.groupby(['msisdn'], as_index=False).agg({'trans_amount':['sum', 'mean']})\n",
    "trans_hist.columns = ['_'.join(col) for col in trans_hist.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = [\n",
    "    \"province_code\",\n",
    "    \"district_code\",\n",
    "    \"viettel_bank_code\",\n",
    "    \"channel_type_id\",\n",
    "    \"channel_code\",\n",
    "    \"staff_code\",\n",
    "    \"trans_amount\",\n",
    "    \"trans_fee\",\n",
    "    \"trans_type\",\n",
    "    \"process_code\",\n",
    "    \"channel_fee\",\n",
    "    \"shop_code\",\n",
    "    \"customer_fee\",\n",
    "    \"cust_name\",\n",
    "    \"ben_cust_name\",\n",
    "    \"msisdn\",\n",
    "    \"ben_msisdn\",\n",
    "    \"msisdn_channel\",\n",
    "    \"staff_name\",\n",
    "    \n",
    "    \"msisdn_\",\n",
    "    \"trans_amount_sum\",\n",
    "    \"trans_amount_mean\",\n",
    "    \"RW000076\",\n",
    "    \"date_diff\"\n",
    "]\n",
    "\n",
    "cat_features = [\n",
    "    'viettel_bank_code',\n",
    "    'msisdn_channel',\n",
    "    'msisdn_',\n",
    "    'msisdn',\n",
    "    'district_code',\n",
    "    'process_code',\n",
    "    'staff_code',\n",
    "    'ben_msisdn',\n",
    "    'shop_code',\n",
    "    'province_code',\n",
    "    'channel_code',\n",
    "    'cust_name',\n",
    "    'trans_type',\n",
    "    'staff_name',\n",
    "    'ben_cust_name',\n",
    "    'channel_type_id'\n",
    "]\n",
    "\n",
    "X, y = preprocess_data(train, account, sale, trans_hist, features, cat_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y = train['is_fraud']\n",
    "# X = train.drop(['is_fraud'], axis = 1)\n",
    "# y_test = test['is_fraud']\n",
    "# X_test = test.drop(['is_fraud'], axis = 1)\n",
    "    \n",
    "X_train, X_val, y_train, y_val = train_test_split( X, y, test_size=0.2, random_state=42)\n",
    "#X_train, X_val, y_train, y_val = train_test_split( X_train, y_train, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_model = CatBoostClassifier(\n",
    "    iterations=1000,\n",
    "    max_ctr_complexity=1,\n",
    "    random_seed=0,\n",
    "    od_type='Iter',\n",
    "    od_wait=25,\n",
    "    verbose=25,\n",
    "    depth=6,\n",
    "    task_type=\"GPU\",\n",
    "    loss_function=\"Logloss\",\n",
    "    eval_metric=\"AUC\"\n",
    ")\n",
    "\n",
    "cat_model.fit(\n",
    "    X_train, y_train,\n",
    "    cat_features=cat_features,\n",
    "    eval_set=(X_val, y_val)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save_model(cat_model, 'cat-model.saved')\n",
    "import gc\n",
    "pickle.dump(cat_model, open('cat-model.saved', 'wb'))\n",
    "del X, y, X_train, X_val, y_train, y_val\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loaded_cat_model = pickle.load(open('cat-model.saved', 'rb'))\n",
    "test = pd.read_parquet(\"/content/drive/MyDrive/projects/viettelpay/code/data/data-202008.snappy.parquet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trans_hist = pd.read_parquet(\"/content/drive/MyDrive/projects/viettelpay/code/data/trans_hist_202008.snappy.parquet\")\n",
    "trans_hist = trans_hist.groupby(['msisdn'], as_index=False).agg({'trans_amount':['sum', 'mean']})\n",
    "trans_hist.columns = ['_'.join(col) for col in trans_hist.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test, y_test = preprocess_data(test, account, sale, trans_hist, features, cat_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_model(loaded_cat_model, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feature_importance = pd.DataFrame(loaded_cat_model.feature_importances_, index=features, columns=['feature importance']).sort_values('feature importance', ascending=False)\n",
    "df_feature_importance.plot(kind='bar');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_feature_importance(model, features):\n",
    "    feature_importances = pd.DataFrame(sorted(zip(model.feature_importances_, features)), columns=['Value', 'Feature'])\n",
    "    plt.figure(figsize=(20,10))\n",
    "    sns.barplot(x=\"Value\", y=\"Feature\", data=feature_importances.sort_values(by=\"Value\", ascending=False).iloc[:50])\n",
    "    plt.title(\"Feature Importance\")\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "plot_feature_importance(loaded_cat_model, features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}