catboost_models/fraud-fs-fe9477-catboost-no-cat.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "\n",
    "import datetime\n",
    "import gc\n",
    "import lightgbm as lgb\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    "import random\n",
    "import seaborn as sns\n",
    "\n",
    "from catboost import CatBoostClassifier\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "sns.set()\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def seed_everything(seed=13):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    \n",
    "    \n",
    "def read_from_disk(path, filename):\n",
    "    with open(os.path.join(path, filename), 'rb') as handle:\n",
    "        return pickle.load(handle)\n",
    "    \n",
    "    \n",
    "def save_to_disk(obj, filename):\n",
    "    with open(filename, 'wb') as handle:\n",
    "        pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = read_from_disk('../input/fraud-fs-fe9477-no-day-data-preparation', 'df_train.pkl')\n",
    "df_test = read_from_disk('../input/fraud-fs-fe9477-no-day-data-preparation', 'df_test.pkl')\n",
    "y = read_from_disk('../input/fraud-fs-fe9477-no-day-data-preparation', 'y.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['TransactionDT', 'TransactionAmt', 'ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'addr1', 'addr2', 'dist1', 'dist2', 'P_emaildomain', 'R_emaildomain', 'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12', 'C13', 'C14', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D7', 'D8', 'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'M1', 'M2', 'M3', 'M4', 'M5', 'M6', 'M7', 'M8', 'M9', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V15', 'V16', 'V17', 'V18', 'V19', 'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V29', 'V30', 'V31', 'V32', 'V33', 'V34', 'V35', 'V36', 'V37', 'V38', 'V39', 'V40', 'V42', 'V43', 'V44', 'V45', 'V46', 'V47', 'V48', 'V49', 'V50', 'V51', 'V52', 'V53', 'V54', 'V55', 'V56', 'V57', 'V58', 'V59', 'V60', 'V61', 'V62', 'V63', 'V64', 'V66', 'V67', 'V69', 'V70', 'V71', 'V72', 'V73', 'V74', 'V75', 'V76', 'V77', 'V78', 'V79', 'V80', 'V81', 'V82', 'V83', 'V84', 'V85', 'V86', 'V87', 'V90', 'V91', 'V92', 'V93', 'V94', 'V95', 'V96', 'V97', 'V98', 'V99', 'V100', 'V101', 'V102', 'V103', 'V104', 'V105', 'V106', 'V108', 'V109', 'V110', 'V111', 'V112', 'V114', 'V115', 'V116', 'V122', 'V123', 'V124', 'V125', 'V126', 'V127', 'V128', 'V129', 'V130', 'V131', 'V132', 'V133', 'V134', 'V135', 'V136', 'V137', 'V138', 'V139', 'V140', 'V141', 'V142', 'V143', 'V144', 'V145', 'V146', 'V147', 'V148', 'V149', 'V150', 'V151', 'V152', 'V153', 'V154', 'V155', 'V156', 'V157', 'V158', 'V159', 'V160', 'V161', 'V162', 'V163', 'V164', 'V165', 'V166', 'V167', 'V168', 'V169', 'V170', 'V171', 'V172', 'V173', 'V174', 'V175', 'V176', 'V177', 'V178', 'V179', 'V180', 'V181', 'V182', 'V183', 'V184', 'V185', 'V186', 'V187', 'V188', 'V189', 'V190', 'V191', 'V192', 'V193', 'V194', 'V195', 'V196', 'V197', 'V198', 'V199', 'V200', 'V201', 'V202', 'V203', 'V204', 'V205', 'V206', 'V207', 'V208', 'V209', 'V210', 'V211', 'V212', 'V213', 'V214', 'V215', 'V216', 'V217', 'V218', 'V219', 'V220', 'V221', 'V222', 'V223', 'V224', 'V225', 'V226', 'V227', 'V228', 'V229', 'V230', 'V231', 'V232', 'V233', 'V234', 'V235', 'V236', 'V237', 'V238', 'V239', 'V240', 'V242', 'V243', 'V244', 'V245', 'V246', 'V247', 'V248', 'V249', 'V250', 'V251', 'V252', 'V253', 'V254', 'V255', 'V256', 'V257', 'V258', 'V259', 'V260', 'V261', 'V262', 'V263', 'V264', 'V265', 'V266', 'V267', 'V268', 'V270', 'V271', 'V272', 'V273', 'V274', 'V275', 'V276', 'V277', 'V278', 'V279', 'V280', 'V281', 'V282', 'V283', 'V284', 'V285', 'V286', 'V287', 'V288', 'V289', 'V290', 'V291', 'V292', 'V293', 'V294', 'V295', 'V296', 'V297', 'V298', 'V299', 'V300', 'V301', 'V302', 'V303', 'V304', 'V306', 'V307', 'V308', 'V309', 'V310', 'V311', 'V312', 'V313', 'V314', 'V315', 'V316', 'V317', 'V318', 'V319', 'V320', 'V321', 'V322', 'V323', 'V324', 'V325', 'V326', 'V327', 'V328', 'V329', 'V330', 'V331', 'V332', 'V333', 'V334', 'V335', 'V336', 'V337', 'V338', 'V339', 'id_01', 'id_02', 'id_03', 'id_04', 'id_05', 'id_06', 'id_07', 'id_08', 'id_09', 'id_10', 'id_11', 'id_12', 'id_13', 'id_14', 'id_15', 'id_16', 'id_17', 'id_18', 'id_19', 'id_20', 'id_21', 'id_22', 'id_23', 'id_24', 'id_25', 'id_26', 'id_28', 'id_29', 'id_30', 'id_31', 'id_32', 'id_33', 'id_34', 'id_35', 'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo', 'Cents', 'Hour', 'DayOfWeek', 'card1__addr1', 'card1__addr1_count', 'card1__card5', 'card1__card5_count', 'card2__dist1', 'card2__id_20', 'card5__P_emaildomain', 'card1_count', 'card2_count', 'card5_count', 'addr1_count', 'C1_count', 'C13_count', 'P_emaildomain_count', 'D2_na', 'D3_na', 'D5_na', 'D6_na', 'D7_na', 'D13_na', 'D14_na', 'M4_na', 'M5_na', 'M6_na', 'M7_na', 'M8_na', 'P_emaildomain_parts', 'P_emaildomain_sfx2', 'P_emaildomain_sfx1', 'P_emaildomain_bin', 'R_emaildomain_parts', 'R_emaildomain_sfx2', 'R_emaildomain_sfx1', 'R_emaildomain_bin', 'Browser', 'OS', 'id_16_na', 'id_33_0', 'id_33_1', 'uid', 'uid2', 'uid3', 'card3_count', 'C2_count', 'C3_count', 'C4_count', 'C5_count', 'C6_count', 'C7_count', 'C8_count', 'C9_count', 'C10_count', 'C11_count', 'C12_count', 'C14_count', 'D1_count', 'D2_count', 'D3_count', 'D4_count', 'D5_count', 'D6_count', 'D7_count', 'D8_count', 'addr2_count', 'dist1_count', 'dist2_count', 'R_emaildomain_count', 'DeviceInfo_count', 'DeviceType_count', 'id_30_count', 'id_33_count', 'uid_count', 'uid2_count', 'uid3_count', 'card1_TransactionAmt_mean', 'card1_TransactionAmt_std', 'card2_TransactionAmt_mean', 'card2_TransactionAmt_std', 'card3_TransactionAmt_mean', 'card3_TransactionAmt_std', 'card5_TransactionAmt_mean', 'card5_TransactionAmt_std', 'uid_TransactionAmt_mean', 'uid_TransactionAmt_std', 'uid2_TransactionAmt_mean', 'uid2_TransactionAmt_std', 'uid3_TransactionAmt_mean', 'uid3_TransactionAmt_std', 'Cents_ProductCD_W']\n"
     ]
    }
   ],
   "source": [
    "print(list(df_train.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_kg_hide-output": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Browser', 'C1', 'C10', 'C10_count', 'C11', 'C11_count', 'C12', 'C12_count', 'C13', 'C13_count', 'C14', 'C14_count', 'C1_count', 'C2', 'C2_count', 'C3', 'C3_count', 'C4', 'C4_count', 'C5', 'C5_count', 'C6', 'C6_count', 'C7', 'C7_count', 'C8', 'C8_count', 'C9', 'C9_count', 'Cents', 'Cents_ProductCD_W', 'D1', 'D10', 'D11', 'D12', 'D13', 'D13_na', 'D14', 'D14_na', 'D15', 'D1_count', 'D2', 'D2_count', 'D2_na', 'D3', 'D3_count', 'D3_na', 'D4', 'D4_count', 'D5', 'D5_count', 'D5_na', 'D6', 'D6_count', 'D6_na', 'D7', 'D7_count', 'D7_na', 'D8', 'D8_count', 'D9', 'DayOfWeek', 'DeviceInfo_count', 'DeviceType_count', 'Hour', 'M4_na', 'M5_na', 'M6_na', 'M7_na', 'M8_na', 'OS', 'P_emaildomain_count', 'P_emaildomain_parts', 'R_emaildomain_count', 'R_emaildomain_parts', 'TransactionAmt', 'TransactionDT', 'V1', 'V10', 'V100', 'V101', 'V102', 'V103', 'V104', 'V105', 'V106', 'V108', 'V109', 'V11', 'V110', 'V111', 'V112', 'V114', 'V115', 'V116', 'V12', 'V122', 'V123', 'V124', 'V125', 'V126', 'V127', 'V128', 'V129', 'V13', 'V130', 'V131', 'V132', 'V133', 'V134', 'V135', 'V136', 'V137', 'V138', 'V139', 'V140', 'V141', 'V142', 'V143', 'V144', 'V145', 'V146', 'V147', 'V148', 'V149', 'V15', 'V150', 'V151', 'V152', 'V153', 'V154', 'V155', 'V156', 'V157', 'V158', 'V159', 'V16', 'V160', 'V161', 'V162', 'V163', 'V164', 'V165', 'V166', 'V167', 'V168', 'V169', 'V17', 'V170', 'V171', 'V172', 'V173', 'V174', 'V175', 'V176', 'V177', 'V178', 'V179', 'V18', 'V180', 'V181', 'V182', 'V183', 'V184', 'V185', 'V186', 'V187', 'V188', 'V189', 'V19', 'V190', 'V191', 'V192', 'V193', 'V194', 'V195', 'V196', 'V197', 'V198', 'V199', 'V2', 'V20', 'V200', 'V201', 'V202', 'V203', 'V204', 'V205', 'V206', 'V207', 'V208', 'V209', 'V21', 'V210', 'V211', 'V212', 'V213', 'V214', 'V215', 'V216', 'V217', 'V218', 'V219', 'V22', 'V220', 'V221', 'V222', 'V223', 'V224', 'V225', 'V226', 'V227', 'V228', 'V229', 'V23', 'V230', 'V231', 'V232', 'V233', 'V234', 'V235', 'V236', 'V237', 'V238', 'V239', 'V24', 'V240', 'V242', 'V243', 'V244', 'V245', 'V246', 'V247', 'V248', 'V249', 'V25', 'V250', 'V251', 'V252', 'V253', 'V254', 'V255', 'V256', 'V257', 'V258', 'V259', 'V26', 'V260', 'V261', 'V262', 'V263', 'V264', 'V265', 'V266', 'V267', 'V268', 'V270', 'V271', 'V272', 'V273', 'V274', 'V275', 'V276', 'V277', 'V278', 'V279', 'V280', 'V281', 'V282', 'V283', 'V284', 'V285', 'V286', 'V287', 'V288', 'V289', 'V29', 'V290', 'V291', 'V292', 'V293', 'V294', 'V295', 'V296', 'V297', 'V298', 'V299', 'V3', 'V30', 'V300', 'V301', 'V302', 'V303', 'V304', 'V306', 'V307', 'V308', 'V309', 'V31', 'V310', 'V311', 'V312', 'V313', 'V314', 'V315', 'V316', 'V317', 'V318', 'V319', 'V32', 'V320', 'V321', 'V322', 'V323', 'V324', 'V325', 'V326', 'V327', 'V328', 'V329', 'V33', 'V330', 'V331', 'V332', 'V333', 'V334', 'V335', 'V336', 'V337', 'V338', 'V339', 'V34', 'V35', 'V36', 'V37', 'V38', 'V39', 'V4', 'V40', 'V42', 'V43', 'V44', 'V45', 'V46', 'V47', 'V48', 'V49', 'V5', 'V50', 'V51', 'V52', 'V53', 'V54', 'V55', 'V56', 'V57', 'V58', 'V59', 'V6', 'V60', 'V61', 'V62', 'V63', 'V64', 'V66', 'V67', 'V69', 'V7', 'V70', 'V71', 'V72', 'V73', 'V74', 'V75', 'V76', 'V77', 'V78', 'V79', 'V8', 'V80', 'V81', 'V82', 'V83', 'V84', 'V85', 'V86', 'V87', 'V9', 'V90', 'V91', 'V92', 'V93', 'V94', 'V95', 'V96', 'V97', 'V98', 'V99', 'addr1_count', 'addr2_count', 'card1_TransactionAmt_mean', 'card1_TransactionAmt_std', 'card1__addr1_count', 'card1__card5_count', 'card1_count', 'card2_TransactionAmt_mean', 'card2_TransactionAmt_std', 'card2_count', 'card3_TransactionAmt_mean', 'card3_TransactionAmt_std', 'card3_count', 'card5_TransactionAmt_mean', 'card5_TransactionAmt_std', 'card5_count', 'dist1', 'dist1_count', 'dist2', 'dist2_count', 'id_01', 'id_02', 'id_03', 'id_04', 'id_05', 'id_06', 'id_07', 'id_08', 'id_09', 'id_10', 'id_11', 'id_16_na', 'id_30_count', 'id_33_0', 'id_33_1', 'id_33_count', 'uid', 'uid2', 'uid2_TransactionAmt_mean', 'uid2_TransactionAmt_std', 'uid2_count', 'uid3', 'uid3_TransactionAmt_mean', 'uid3_TransactionAmt_std', 'uid3_count', 'uid_TransactionAmt_mean', 'uid_TransactionAmt_std', 'uid_count']\n"
     ]
    }
   ],
   "source": [
    "categorical = ['ProductCD', 'DeviceType', 'DeviceInfo', 'P_emaildomain', 'R_emaildomain', 'addr1', 'addr2', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'M1', 'M2', 'M3', 'M4', 'M5', 'M6', 'M7', 'M8', 'M9', 'id_12', 'id_13', 'id_14', 'id_15', 'id_16', 'id_17', 'id_18', 'id_19', 'id_20', 'id_21', 'id_22', 'id_23', 'id_24', 'id_25', 'id_26', 'id_28', 'id_29', 'id_30', 'id_31', 'id_32', 'id_33', 'id_34', 'id_35', 'id_36', 'id_37', 'id_38', 'card1__addr1', 'card1__card5', 'card2__dist1', 'card2__id_20', 'card5__P_emaildomain', 'P_emaildomain_sfx2', 'P_emaildomain_sfx1', 'P_emaildomain_bin', 'R_emaildomain_sfx2', 'R_emaildomain_sfx1', 'R_emaildomain_bin', 'P_emaildomain_sfx2', 'P_emaildomain_sfx1', 'P_emaildomain_bin', 'R_emaildomain_sfx2', 'R_emaildomain_sfx1', 'R_emaildomain_bin']\n",
    "numerical = sorted(list(set(df_train.columns) - set(categorical)))\n",
    "print(numerical)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d')\n",
    "TransactionDT1 = df_train['TransactionDT'].apply(lambda x: (START_DATE + datetime.timedelta(seconds = x)))\n",
    "dt_m = TransactionDT1.dt.date.astype('str').str[:7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((590540, 504), (506691, 504))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = df_train.drop(['TransactionDT'], axis=1)\n",
    "df_test = df_test.drop(['TransactionDT'], axis=1)\n",
    "df_train.shape, df_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "month = '2017-12'\n",
    "X_train, y_train = df_train[~dt_m.isin([month])], y[~dt_m.isin([month])]\n",
    "X_valid, y_valid = df_train[dt_m.isin([month])], y[dt_m.isin([month])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'n_estimators': 2500,\n",
    "    'early_stopping_rounds': 500,\n",
    "    'depth': 8,\n",
    "    'eval_metric': 'AUC',\n",
    "    'loss_function': 'Logloss',\n",
    "    'random_seed': 13,\n",
    "    'thread_count': 7,\n",
    "    'border_count': 32,\n",
    "    'bootstrap_type': 'Poisson',\n",
    "    'grow_policy': 'Lossguide',\n",
    "    'max_leaves': 20,\n",
    "    'min_data_in_leaf': 3,\n",
    "    'task_type': 'GPU',\n",
    "    'verbose': False\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "_kg_hide-output": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 1 | AUC: 0.9133559064202742\n",
      "Fold 2 | AUC: 0.9323482753133613\n",
      "Fold 3 | AUC: 0.9363597725626474\n",
      "Fold 4 | AUC: 0.937291416833934\n",
      "Fold 5 | AUC: 0.9453792853785589\n",
      "Fold 6 | AUC: 0.9303716725429071\n",
      "\n",
      "Mean AUC = 0.9325177215086138\n"
     ]
    }
   ],
   "source": [
    "y_preds = np.zeros(df_test.shape[0])\n",
    "score = 0\n",
    "\n",
    "NFOLDS = len(np.unique(dt_m)) # 6\n",
    "\n",
    "for fold_n, month in enumerate(np.unique(dt_m)):\n",
    "        X_train, y_train = df_train[~dt_m.isin([month])], y[~dt_m.isin([month])]\n",
    "        X_valid, y_valid = df_train[dt_m.isin([month])], y[dt_m.isin([month])]\n",
    "\n",
    "        clf = CatBoostClassifier(**params)\n",
    "        clf.fit(\n",
    "            X_train, y_train,\n",
    "            eval_set=(X_valid, y_valid),\n",
    "#             cat_features=categorical,\n",
    "            use_best_model=True,\n",
    "            verbose=False\n",
    "        )\n",
    "\n",
    "        y_pred_valid = clf.predict_proba(X_valid)[:, 1]\n",
    "        save_to_disk(y_pred_valid, 'y_pred_valid_fold{}.pkl'.format(fold_n))\n",
    "        print(f\"Fold {fold_n + 1} | AUC: {roc_auc_score(y_valid, y_pred_valid)}\")\n",
    "        \n",
    "        score += roc_auc_score(y_valid, y_pred_valid) / NFOLDS\n",
    "        \n",
    "        y_pred_test = clf.predict_proba(df_test)[:, 1]\n",
    "        save_to_disk(y_pred_test, 'y_pred_test_fold{}.pkl'.format(fold_n))\n",
    "        y_preds += y_pred_test / NFOLDS\n",
    "        \n",
    "        del X_train, X_valid, y_train, y_valid\n",
    "        gc.collect()\n",
    "\n",
    "print(f\"\\nMean AUC = {score}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.read_csv('../input/ieee-fraud-detection/sample_submission.csv')\n",
    "sub['isFraud'] = y_preds\n",
    "sub.to_csv('submission.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}