ligthgbm_models/fraud-lgb-kfold-newfeatures-seed-111.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# safe downcast\n",
    "def sd(col, max_loss_limit=0.001, avg_loss_limit=0.001, na_loss_limit=0, n_uniq_loss_limit=0, fillna=0):\n",
    "    \"\"\"\n",
    "    max_loss_limit - don't allow any float to lose precision more than this value. Any values are ok for GBT algorithms as long as you don't unique values.\n",
    "                     See https://en.wikipedia.org/wiki/Half-precision_floating-point_format#Precision_limitations_on_decimal_values_in_[0,_1]\n",
    "    avg_loss_limit - same but calculates avg throughout the series.\n",
    "    na_loss_limit - not really useful.\n",
    "    n_uniq_loss_limit - very important parameter. If you have a float field with very high cardinality you can set this value to something like n_records * 0.01 in order to allow some field relaxing.\n",
    "    \"\"\"\n",
    "    is_float = str(col.dtypes)[:5] == 'float'\n",
    "    na_count = col.isna().sum()\n",
    "    n_uniq = col.nunique(dropna=False)\n",
    "    try_types = ['float16', 'float32']\n",
    "\n",
    "    if na_count <= na_loss_limit:\n",
    "        try_types = ['int8', 'int16', 'float16', 'int32', 'float32']\n",
    "\n",
    "    for type in try_types:\n",
    "        col_tmp = col\n",
    "\n",
    "        # float to int conversion => try to round to minimize casting error\n",
    "        if is_float and (str(type)[:3] == 'int'):\n",
    "            col_tmp = col_tmp.copy().fillna(fillna).round()\n",
    "\n",
    "        col_tmp = col_tmp.astype(type)\n",
    "        max_loss = (col_tmp - col).abs().max()\n",
    "        avg_loss = (col_tmp - col).abs().mean()\n",
    "        na_loss = np.abs(na_count - col_tmp.isna().sum())\n",
    "        n_uniq_loss = np.abs(n_uniq - col_tmp.nunique(dropna=False))\n",
    "\n",
    "        if max_loss <= max_loss_limit and avg_loss <= avg_loss_limit and na_loss <= na_loss_limit and n_uniq_loss <= n_uniq_loss_limit:\n",
    "            return col_tmp\n",
    "\n",
    "    # field can't be converted\n",
    "    return col\n",
    "\n",
    "\n",
    "def reduce_mem_usage(df, deep=True, verbose=False, obj_to_cat=False):\n",
    "    numerics = ['int16', 'uint16', 'int32', 'uint32', 'int64', 'uint64', 'float16', 'float32', 'float64']\n",
    "    start_mem = df.memory_usage(deep=deep).sum() / 1024 ** 2\n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtypes\n",
    "\n",
    "        # collect stats\n",
    "        na_count = df[col].isna().sum()\n",
    "        n_uniq = df[col].nunique(dropna=False)\n",
    "        \n",
    "        # numerics\n",
    "        if col_type in numerics:\n",
    "            df[col] = sd(df[col])\n",
    "\n",
    "        # strings\n",
    "        if (col_type == 'object') and obj_to_cat:\n",
    "            df[col] = df[col].astype('category')\n",
    "        \n",
    "        if verbose:\n",
    "            print(f'Column {col}: {col_type} -> {df[col].dtypes}, na_count={na_count}, n_uniq={n_uniq}')\n",
    "        new_na_count = df[col].isna().sum()\n",
    "        if (na_count != new_na_count):\n",
    "            print(f'Warning: column {col}, {col_type} -> {df[col].dtypes} lost na values. Before: {na_count}, after: {new_na_count}')\n",
    "        new_n_uniq = df[col].nunique(dropna=False)\n",
    "        if (n_uniq != new_n_uniq):\n",
    "            print(f'Warning: column {col}, {col_type} -> {df[col].dtypes} lost unique values. Before: {n_uniq}, after: {new_n_uniq}')\n",
    "\n",
    "    end_mem = df.memory_usage(deep=deep).sum() / 1024 ** 2\n",
    "    percent = 100 * (start_mem - end_mem) / start_mem\n",
    "    if verbose:\n",
    "        print('Mem. usage decreased from {:5.2f} Mb to {:5.2f} Mb ({:.1f}% reduction)'.format(start_mem, end_mem, percent))\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " From notebooks of\n",
    " https://www.kaggle.com/xhlulu/ieee-fraud-xgboost-with-gpu-fit-in-40s  \n",
    " https://www.kaggle.com/davidcairuz/feature-engineering-lightgbm\n",
    "https://www.kaggle.com/kyakovlev/ieee-fe-with-some-eda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading libs...\n"
     ]
    }
   ],
   "source": [
    "print('loading libs...')\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import os\n",
    "import gc\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import preprocessing\n",
    "import xgboost as xgb\n",
    "from tqdm import tqdm\n",
    "from sklearn.model_selection import KFold\n",
    "import lightgbm as lgb\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import datetime\n",
    "import time\n",
    "\n",
    "import pickle\n",
    "def save_to_disk(obj, filename):\n",
    "    with open(filename, 'wb') as handle:\n",
    "        pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading data...\n",
      "done\n",
      "CPU times: user 2.83 s, sys: 14 s, total: 16.8 s\n",
      "Wall time: 18.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "print('loading data...')\n",
    "train = pd.read_pickle('../input/ieee-fe-with-some-eda/train_df.pkl')\n",
    "test = pd.read_pickle('../input/ieee-fe-with-some-eda/test_df.pkl')\n",
    "remove_features = pd.read_pickle('../input/ieee-fe-with-some-eda/remove_features.pkl')\n",
    "sample_submission = pd.read_csv('../input/ieee-fraud-detection/sample_submission.csv')\n",
    "print('done')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dropping target...\n",
      "selecting features...\n",
      "Done\n",
      "CPU times: user 3.04 s, sys: 17.4 s, total: 20.5 s\n",
      "Wall time: 12.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "print('dropping target...')\n",
    "y_train = train['isFraud'].copy()\n",
    "X_train = train.drop('isFraud', axis=1)\n",
    "X_test = test.copy()\n",
    "train_cols = list(train.columns)\n",
    "del train, test\n",
    "gc.collect()\n",
    "print('selecting features...')\n",
    "remove_features = list(remove_features['features_to_remove'].values)\n",
    "features_columns = [col for col in train_cols if col not in set(remove_features) - set(['TransactionDT'])]\n",
    "X_train = X_train[features_columns]\n",
    "X_test = X_test[features_columns]\n",
    "print('Done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = reduce_mem_usage(X_train)\n",
    "X_test = reduce_mem_usage(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "          'objective':'binary',\n",
    "          'boosting_type':'gbdt',\n",
    "          'metric':'auc',\n",
    "          'n_jobs':-1,\n",
    "          'max_depth':-1,\n",
    "          'tree_learner':'serial',\n",
    "          'min_data_in_leaf':30,\n",
    "          'n_estimators':1800,\n",
    "          'max_bin':255,\n",
    "          'verbose':-1,\n",
    "          'seed': 111,\n",
    "          'learning_rate': 0.01,\n",
    "          'early_stopping_rounds':200,\n",
    "          'colsample_bytree': 0.5,          \n",
    "          'num_leaves': 256, \n",
    "          'reg_alpha': 0.35, \n",
    "         }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((590540, 772), (506691, 772))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d')\n",
    "TransactionDT1 = X_train['TransactionDT'].apply(lambda x: (START_DATE + datetime.timedelta(seconds = x)))\n",
    "dt_m = TransactionDT1.dt.date.astype('str').str[:7]\n",
    "X_train = X_train.drop(['TransactionDT'], axis=1)\n",
    "X_test = X_test.drop(['TransactionDT'], axis=1)\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.996373\tvalid_1's auc: 0.907944\n",
      "[1000]\ttraining's auc: 0.999899\tvalid_1's auc: 0.91612\n",
      "[1500]\ttraining's auc: 0.999998\tvalid_1's auc: 0.918131\n",
      "Early stopping, best iteration is:\n",
      "[1371]\ttraining's auc: 0.999994\tvalid_1's auc: 0.918271\n",
      "Fold 1 | AUC: 0.9182213496872573\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.995926\tvalid_1's auc: 0.939965\n",
      "[1000]\ttraining's auc: 0.99988\tvalid_1's auc: 0.943916\n",
      "Early stopping, best iteration is:\n",
      "[1109]\ttraining's auc: 0.999943\tvalid_1's auc: 0.944308\n",
      "Fold 2 | AUC: 0.9443139728952652\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.995705\tvalid_1's auc: 0.949197\n",
      "[1000]\ttraining's auc: 0.999847\tvalid_1's auc: 0.950821\n",
      "Early stopping, best iteration is:\n",
      "[855]\ttraining's auc: 0.999608\tvalid_1's auc: 0.95093\n",
      "Fold 3 | AUC: 0.9508621040561281\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.996356\tvalid_1's auc: 0.943705\n",
      "[1000]\ttraining's auc: 0.999896\tvalid_1's auc: 0.947951\n",
      "[1500]\ttraining's auc: 0.999997\tvalid_1's auc: 0.9481\n",
      "Early stopping, best iteration is:\n",
      "[1384]\ttraining's auc: 0.999994\tvalid_1's auc: 0.948281\n",
      "Fold 4 | AUC: 0.9482709223634439\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.995425\tvalid_1's auc: 0.954715\n",
      "[1000]\ttraining's auc: 0.999827\tvalid_1's auc: 0.959514\n",
      "[1500]\ttraining's auc: 0.999994\tvalid_1's auc: 0.960175\n",
      "Early stopping, best iteration is:\n",
      "[1338]\ttraining's auc: 0.999981\tvalid_1's auc: 0.960409\n",
      "Fold 5 | AUC: 0.9603951821314489\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.995787\tvalid_1's auc: 0.939171\n",
      "[1000]\ttraining's auc: 0.999864\tvalid_1's auc: 0.942971\n",
      "Early stopping, best iteration is:\n",
      "[1121]\ttraining's auc: 0.999939\tvalid_1's auc: 0.943104\n",
      "Fold 6 | AUC: 0.943123046499077\n",
      "\n",
      "Mean AUC = 0.94419776293877\n",
      "CPU times: user 13h 44min 1s, sys: 4min 8s, total: 13h 48min 9s\n",
      "Wall time: 3h 32min 25s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "y_preds = np.zeros(X_test.shape[0])\n",
    "score = 0\n",
    "\n",
    "feature_importances = pd.DataFrame()\n",
    "feature_importances['feature'] = X_train.columns\n",
    "\n",
    "NFOLDS = len(np.unique(dt_m)) # 6\n",
    "  \n",
    "for fold_n, month in enumerate(np.unique(dt_m)):\n",
    "    X_tr, X_val = X_train[~dt_m.isin([month])], X_train[dt_m.isin([month])]\n",
    "    y_tr, y_val = y_train[~dt_m.isin([month])], y_train[dt_m.isin([month])]  \n",
    "    dtrain = lgb.Dataset(X_tr, label=y_tr)\n",
    "    dvalid = lgb.Dataset(X_val, label=y_val)\n",
    "    clf = lgb.train(params, dtrain,  valid_sets = [dtrain, dvalid], verbose_eval=500)      \n",
    "    \n",
    "    feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()\n",
    "    \n",
    "    y_pred_valid = clf.predict(X_val)\n",
    "    save_to_disk(y_pred_valid, 'y_pred_valid_fold{}.pkl'.format(fold_n))\n",
    "    print(f\"Fold {fold_n + 1} | AUC: {roc_auc_score(y_val, y_pred_valid)}\")   \n",
    "    \n",
    "    score += roc_auc_score(y_val, y_pred_valid) / NFOLDS\n",
    "    \n",
    "    y_pred_test = clf.predict(X_test)\n",
    "    save_to_disk(y_pred_test, 'y_pred_test_fold{}.pkl'.format(fold_n))\n",
    "    y_preds += y_pred_test / NFOLDS\n",
    "\n",
    "    del X_tr, X_val, y_tr, y_val\n",
    "    gc.collect()\n",
    "\n",
    "print(f\"\\nMean AUC = {score}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.read_csv('../input/ieee-fraud-detection/sample_submission.csv')\n",
    "sub['isFraud'] = y_preds\n",
    "sub.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'sns' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-385fb9fa8a6e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfigsize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m16\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m16\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0msns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbarplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeature_importances\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mby\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'average'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mascending\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m50\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'average'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'feature'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'50 TOP feature importance over {} folds average'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mNFOLDS\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'sns' is not defined"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1152x1152 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "feature_importances['average'] = feature_importances[[f'fold_{fold_n + 1}' for fold_n in range(NFOLDS)]].mean(axis=1)\n",
    "feature_importances.to_csv('feature_importances.csv')\n",
    "\n",
    "plt.figure(figsize=(16, 16))\n",
    "sns.barplot(data=feature_importances.sort_values(by='average', ascending=False).head(50), x='average', y='feature');\n",
    "plt.title('50 TOP feature importance over {} folds average'.format(NFOLDS));"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}