catboost_models/fraud-dima-cb-hope-v1-944414.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import datetime\n",
    "import math\n",
    "import os\n",
    "import pickle\n",
    "import random\n",
    "\n",
    "from functools import partial\n",
    "from time import time\n",
    "\n",
    "import lightgbm as lgb\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.model_selection import GroupKFold, KFold, TimeSeriesSplit\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "from catboost import CatBoostClassifier, Pool, cv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_PATH = '/kaggle/input/fraud-just-data'\n",
    "\n",
    "COL_ID = 'TransactionID'\n",
    "COL_DT = 'TransactionDT'\n",
    "COL_AMOUNT = 'TransactionAmt'\n",
    "COL_TARGET = 'isFraud'\n",
    "\n",
    "START_DATE = '2017-11-30'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def seed_everything(seed=13):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "\n",
    "def save_to_disk(df, filename):\n",
    "    with open(filename, 'wb') as handle:\n",
    "        pickle.dump(df, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "        \n",
    "        \n",
    "def read_from_disk(path, filename):\n",
    "    with open(os.path.join(path, filename), 'rb') as handle:\n",
    "        return pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_datetime(df):\n",
    "    start_date = datetime.datetime.strptime(START_DATE, '%Y-%m-%d')\n",
    "    return df[COL_DT].apply(lambda x: (start_date + datetime.timedelta(seconds = x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((590540, 503), (590540,), (590540,), (506691, 503))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train, y_train, df_test, dt_m = map(partial(read_from_disk, INPUT_PATH), \n",
    "                                 ['df_train.pkl', 'y_train.pkl', 'df_test.pkl', 'dt_m.pkl'])\n",
    "\n",
    "df_train.shape, y_train.shape, dt_m.shape, df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2017-12', '2018-01', '2018-02', '2018-03', '2018-04', '2018-05'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt_m.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_categorical_cols = [\n",
    "    'card1__addr1', 'card1__card5', 'card2__dist1', 'card5__P_emaildomain', \n",
    "    'card2__id_19', 'card2__id_20', 'DeviceInfo__id_30', 'D2_na', 'D3_na', \n",
    "    'D5_na', 'D6_na', 'D7_na', 'D13_na', 'D14_na', 'M4_na', 'M5_na', 'M6_na', \n",
    "    'id_16_na', 'D2_na', 'D3_na', 'D5_na', 'D6_na', 'D7_na', 'D13_na', 'D14_na', \n",
    "    'M4_na', 'M5_na', 'M6_na', 'id_16_na', 'P_emaildomain_sfx2', \n",
    "    'P_emaildomain_sfx1', 'P_emaildomain_bin', 'R_emaildomain_sfx2', \n",
    "    'R_emaildomain_sfx1', 'R_emaildomain_bin', 'Browser', 'OS'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ProductCD', 'DeviceType', 'DeviceInfo', 'P_emaildomain', 'R_emaildomain', 'addr1', 'addr2', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'M1', 'M2', 'M3', 'M4', 'M5', 'M6', 'M7', 'M8', 'M9', 'id_12', 'id_13', 'id_14', 'id_15', 'id_16', 'id_17', 'id_18', 'id_19', 'id_20', 'id_21', 'id_22', 'id_23', 'id_24', 'id_25', 'id_26', 'id_27', 'id_28', 'id_29', 'id_30', 'id_31', 'id_32', 'id_33', 'id_34', 'id_35', 'id_36', 'id_37', 'id_38', 'card1__addr1', 'card1__card5', 'card2__dist1', 'card5__P_emaildomain', 'card2__id_19', 'card2__id_20', 'DeviceInfo__id_30', 'D2_na', 'D3_na', 'D5_na', 'D6_na', 'D7_na', 'D13_na', 'D14_na', 'M4_na', 'M5_na', 'M6_na', 'id_16_na', 'D2_na', 'D3_na', 'D5_na', 'D6_na', 'D7_na', 'D13_na', 'D14_na', 'M4_na', 'M5_na', 'M6_na', 'id_16_na', 'P_emaildomain_sfx2', 'P_emaildomain_sfx1', 'P_emaildomain_bin', 'R_emaildomain_sfx2', 'R_emaildomain_sfx1', 'R_emaildomain_bin', 'Browser', 'OS']\n"
     ]
    }
   ],
   "source": [
    "def col_names(prefix, interval):\n",
    "    assert len(interval) == 2\n",
    "    return map(lambda i: prefix + str(i), range(interval[0], interval[1] + 1))\n",
    "\n",
    "\n",
    "categorical = [\n",
    "    'ProductCD', \n",
    "    'DeviceType', 'DeviceInfo', \n",
    "    'P_emaildomain', 'R_emaildomain', \n",
    "    'addr1', 'addr2',\n",
    "    *col_names('card', (1, 6)),\n",
    "    *col_names('M', (1, 9)),\n",
    "    *col_names('id_', (12, 38)),\n",
    "    *new_categorical_cols,\n",
    "]\n",
    "print(categorical)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['C1', 'C10', 'C11', 'C11_count', 'C12', 'C13', 'C13_count', 'C14', 'C14_count', 'C1_count', 'C2', 'C2_count', 'C3', 'C4', 'C5', 'C5_count', 'C6', 'C6_count', 'C7', 'C7_count', 'C8', 'C9', 'C9_count', 'Cents', 'D1', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'D2', 'D3', 'D4', 'D5', 'D6', 'D7', 'D8', 'D9', 'DayOfMonth', 'DayOfWeek', 'DeviceInfo__id_30_count', 'DeviceInfo_count', 'Hour', 'MonthFraction', 'P_emaildomain_count', 'P_emaildomain_parts', 'ProductCD__Cents_magic', 'R_emaildomain_parts', 'TransactionAmt', 'TransactionAmt_log', 'TransactionAmt_mean_by_card1', 'TransactionAmt_mean_by_uuid2_ProductCD', 'TransactionAmt_mean_by_uuid3', 'TransactionAmt_std_by_uuid3', 'V1', 'V10', 'V100', 'V101', 'V102', 'V103', 'V104', 'V105', 'V106', 'V107', 'V108', 'V109', 'V11', 'V110', 'V111', 'V112', 'V113', 'V114', 'V115', 'V116', 'V117', 'V118', 'V119', 'V12', 'V120', 'V121', 'V122', 'V123', 'V124', 'V125', 'V126', 'V127', 'V128', 'V129', 'V13', 'V130', 'V131', 'V132', 'V133', 'V134', 'V135', 'V136', 'V137', 'V138', 'V139', 'V14', 'V140', 'V141', 'V142', 'V143', 'V144', 'V145', 'V146', 'V147', 'V148', 'V149', 'V15', 'V150', 'V151', 'V152', 'V153', 'V154', 'V155', 'V156', 'V157', 'V158', 'V159', 'V16', 'V160', 'V161', 'V162', 'V163', 'V164', 'V165', 'V166', 'V167', 'V168', 'V169', 'V17', 'V170', 'V171', 'V172', 'V173', 'V174', 'V175', 'V176', 'V177', 'V178', 'V179', 'V18', 'V180', 'V181', 'V182', 'V183', 'V184', 'V185', 'V186', 'V187', 'V188', 'V189', 'V19', 'V190', 'V191', 'V192', 'V193', 'V194', 'V195', 'V196', 'V197', 'V198', 'V199', 'V2', 'V20', 'V200', 'V201', 'V202', 'V203', 'V204', 'V205', 'V206', 'V207', 'V208', 'V209', 'V21', 'V210', 'V211', 'V212', 'V213', 'V214', 'V215', 'V216', 'V217', 'V218', 'V219', 'V22', 'V220', 'V221', 'V222', 'V223', 'V224', 'V225', 'V226', 'V227', 'V228', 'V229', 'V23', 'V230', 'V231', 'V232', 'V233', 'V234', 'V235', 'V236', 'V237', 'V238', 'V239', 'V24', 'V240', 'V241', 'V242', 'V243', 'V244', 'V245', 'V246', 'V247', 'V248', 'V249', 'V25', 'V250', 'V251', 'V252', 'V253', 'V254', 'V255', 'V256', 'V257', 'V258', 'V259', 'V26', 'V260', 'V261', 'V262', 'V263', 'V264', 'V265', 'V266', 'V267', 'V268', 'V269', 'V27', 'V270', 'V271', 'V272', 'V273', 'V274', 'V275', 'V276', 'V277', 'V278', 'V279', 'V28', 'V280', 'V281', 'V282', 'V283', 'V283_count', 'V284', 'V285', 'V285_count', 'V286', 'V287', 'V288', 'V289', 'V29', 'V290', 'V291', 'V292', 'V293', 'V294', 'V295', 'V296', 'V297', 'V298', 'V299', 'V3', 'V30', 'V300', 'V301', 'V302', 'V303', 'V304', 'V305', 'V306', 'V307', 'V307_count', 'V308', 'V308_count', 'V309', 'V31', 'V310', 'V311', 'V312', 'V313', 'V313_mean_by_V307', 'V314', 'V315', 'V316', 'V317', 'V318', 'V319', 'V32', 'V320', 'V321', 'V322', 'V323', 'V324', 'V325', 'V326', 'V327', 'V328', 'V329', 'V33', 'V330', 'V331', 'V332', 'V333', 'V334', 'V335', 'V336', 'V337', 'V338', 'V339', 'V34', 'V35', 'V36', 'V37', 'V38', 'V39', 'V4', 'V40', 'V41', 'V42', 'V43', 'V44', 'V45', 'V46', 'V47', 'V48', 'V49', 'V5', 'V50', 'V51', 'V52', 'V53', 'V54', 'V55', 'V56', 'V57', 'V58', 'V59', 'V6', 'V60', 'V61', 'V62', 'V63', 'V64', 'V65', 'V66', 'V67', 'V68', 'V69', 'V7', 'V70', 'V71', 'V72', 'V73', 'V74', 'V75', 'V76', 'V77', 'V78', 'V79', 'V8', 'V80', 'V81', 'V82', 'V83', 'V84', 'V85', 'V86', 'V87', 'V88', 'V89', 'V9', 'V90', 'V91', 'V92', 'V93', 'V94', 'V95', 'V96', 'V97', 'V98', 'V99', 'addr1_count', 'addr2_count', 'card1__addr1_count', 'card1__card5_count', 'card1_count', 'card2__id_19_count', 'card2__id_20_count', 'card2_count', 'card3_count', 'card5_count', 'dist1', 'dist2', 'dist2_mean_by_uuid2_addr1', 'id_01', 'id_02', 'id_03', 'id_04', 'id_05', 'id_06', 'id_07', 'id_08', 'id_09', 'id_10', 'id_11', 'id_33_0', 'id_33_1', 'uuid2', 'uuid3', 'uuid3_count']\n"
     ]
    }
   ],
   "source": [
    "numerical = sorted(list(set(df_train.columns) - set(categorical)))\n",
    "print(numerical)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'loss_function': 'Logloss',\n",
    "    'custom_loss': ['AUC'],\n",
    "    'logging_level': 'Silent',\n",
    "    'use_best_model': True,\n",
    "    'task_type': 'GPU',\n",
    "    'early_stopping_rounds': 100,\n",
    "    'iterations': 2000,\n",
    "    'learning_rate': 0.075,\n",
    "    'depth': 8,\n",
    "    'l2_leaf_reg': 4.0,\n",
    "    'bagging_temperature': 0.6,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training on fold 0\n",
      "Train: 453219, Validation: 137321\n",
      "OOF score for 0 fold: 0.9241696672924647\n",
      "Fold 0 finished in 0:13:57.487541\n",
      "Training on fold 1\n",
      "Train: 497955, Validation: 92585\n",
      "OOF score for 1 fold: 0.9446720502009717\n",
      "Fold 1 finished in 0:10:55.513951\n",
      "Training on fold 2\n",
      "Train: 504519, Validation: 86021\n",
      "OOF score for 2 fold: 0.9502988832149776\n",
      "Fold 2 finished in 0:08:32.170632\n",
      "Training on fold 3\n",
      "Train: 488908, Validation: 101632\n",
      "OOF score for 3 fold: 0.9477688151432214\n",
      "Fold 3 finished in 0:09:50.526684\n",
      "Training on fold 4\n",
      "Train: 506885, Validation: 83655\n",
      "OOF score for 4 fold: 0.9626949496566513\n",
      "Fold 4 finished in 0:13:54.785128\n",
      "Training on fold 5\n",
      "Train: 501214, Validation: 89326\n",
      "OOF score for 5 fold: 0.9368805709669232\n",
      "Fold 5 finished in 0:11:03.818745\n",
      "\n",
      "DONE\n",
      "Total time: 1:08:15.013472\n"
     ]
    }
   ],
   "source": [
    "seeds = [11, 19, 23, 27, 31, 37]\n",
    "scores = []\n",
    "test_preds = []\n",
    "feat_imps = []\n",
    "\n",
    "training_start_time = time()\n",
    "\n",
    "for fold, month in enumerate(np.unique(dt_m)):\n",
    "    seed = seeds[fold]\n",
    "    seed_everything(seed)\n",
    "    params['random_seed'] = seed\n",
    "    \n",
    "    start_time = time()\n",
    "    print('Training on fold {}'.format(fold))\n",
    "    \n",
    "    is_val = dt_m.isin([month])\n",
    "    x_train, y_train0 = df_train[~is_val], y_train[~is_val]\n",
    "    x_val, y_val = df_train[is_val], y_train[is_val]\n",
    "    print('Train: {}, Validation: {}'.format(len(x_train), len(x_val)))\n",
    "    \n",
    "    train_pool = Pool(x_train, y_train0, cat_features=categorical)\n",
    "    validate_pool = Pool(x_val, y_val, cat_features=categorical)\n",
    "    \n",
    "    model = CatBoostClassifier(**params)\n",
    "    model.fit(train_pool, eval_set=validate_pool)\n",
    "    \n",
    "    y_pred_val = model.predict_proba(x_val)[:,1]\n",
    "    auc = roc_auc_score(y_val, y_pred_val)\n",
    "    scores.append(auc)\n",
    "    print('OOF score for {} fold: {}'.format(fold, auc))\n",
    "    save_to_disk(y_pred_val, 'y_pred_valid_fold{}.pkl'.format(fold))\n",
    "    \n",
    "    y_pred_test = model.predict_proba(df_test)[:,1]\n",
    "    test_preds.append(y_pred_test)\n",
    "    save_to_disk(y_pred_test, 'y_pred_test_fold{}.pkl'.format(fold))\n",
    "    \n",
    "    end_time = str(datetime.timedelta(seconds=time() - start_time))\n",
    "    print('Fold {} finished in {}'.format(fold, end_time))\n",
    "    \n",
    "    feature_importances = model.get_feature_importance(train_pool)\n",
    "    feat_imps.append(feature_importances)\n",
    "    \n",
    "print('\\nDONE')\n",
    "print('Total time: {}'.format(str(datetime.timedelta(seconds=time() - training_start_time))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OOF score: avg 0.944414, std 0.011879, min 0.924170\n"
     ]
    }
   ],
   "source": [
    "print('OOF score: avg %f, std %f, min %f' % (np.mean(scores), np.std(scores), np.min(scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "card1__addr1                                       6.943026554687289\n",
      "C13                                                4.953747895620424\n",
      "C13_count                                          2.6399361268829176\n",
      "M5                                                 2.567435951950587\n",
      "C14                                                2.5300220740950317\n",
      "TransactionAmt                                     2.4515523772016703\n",
      "card2__dist1                                       2.269861617800595\n",
      "C1_count                                           2.245285176910478\n",
      "C1                                                 2.1320141355837743\n",
      "TransactionAmt_log                                 1.690064398771516\n",
      "card5__P_emaildomain                               1.665033945815811\n",
      "DeviceInfo__id_30                                  1.6535974716766535\n",
      "P_emaildomain                                      1.6137248665647004\n",
      "M6                                                 1.6043979868797031\n",
      "M4                                                 1.3736830399169409\n",
      "id_31                                              1.2885955461322163\n",
      "D2                                                 1.2861373102763178\n",
      "R_emaildomain                                      1.1831657095167232\n",
      "D1                                                 1.1619497531230196\n",
      "id_19                                              1.0701335692123535\n",
      "id_20                                              1.0382549889523414\n",
      "C11_count                                          1.016939927423066\n",
      "card2__id_20                                       1.00053870138231\n",
      "dist1                                              0.9849705195541039\n",
      "dist2_mean_by_uuid2_addr1                          0.8654038324422862\n",
      "Cents                                              0.8394478120698089\n",
      "D15                                                0.82614733792308\n",
      "V281                                               0.7446409850123378\n",
      "card2__id_19                                       0.6969347226173124\n",
      "DeviceInfo                                         0.6737581152197653\n",
      "ProductCD                                          0.6674540485274713\n",
      "D8                                                 0.632111480188322\n",
      "V307                                               0.6231845740801302\n",
      "P_emaildomain_bin                                  0.6154728023629115\n",
      "card1                                              0.6149054503481587\n",
      "uuid3_count                                        0.6098351521303758\n",
      "id_02                                              0.6093060848494519\n",
      "C8                                                 0.6047535937970321\n",
      "card1__addr1_count                                 0.591837068641048\n",
      "C12                                                0.5911297855041204\n",
      "id_13                                              0.5787262933319182\n",
      "Hour                                               0.5729618319024665\n",
      "C11                                                0.5507889255401544\n",
      "D4                                                 0.5462622876105919\n",
      "V317                                               0.5416218961210543\n",
      "V308                                               0.538882117226171\n",
      "card6                                              0.5031760467063421\n",
      "C6_count                                           0.5029484316632532\n",
      "R_emaildomain_bin                                  0.49734832883000896\n",
      "MonthFraction                                      0.4777614110059754\n",
      "C14_count                                          0.4572652035563134\n",
      "addr1                                              0.42292508458955796\n",
      "id_33                                              0.42074235799622545\n",
      "V308_count                                         0.4181405491517081\n",
      "card1__card5                                       0.4177488938069538\n",
      "V45                                                0.41627111898155444\n",
      "V258                                               0.41485454993937426\n",
      "V283_count                                         0.4049721017524977\n",
      "C7_count                                           0.4035328222834315\n",
      "V62                                                0.39768957170184455\n",
      "M3                                                 0.39614675676400424\n",
      "V283                                               0.38056059486671207\n",
      "D3                                                 0.3682510891875856\n",
      "TransactionAmt_mean_by_uuid2_ProductCD             0.3655317136979785\n",
      "card2                                              0.360501257303758\n",
      "V70                                                0.35940629303823984\n",
      "V294                                               0.3565308682626452\n",
      "C4                                                 0.3545021005583018\n",
      "card1__card5_count                                 0.3534387421396188\n",
      "card3                                              0.3451963808214127\n",
      "card1_count                                        0.3319993975912136\n",
      "id_05                                              0.3282331764202955\n",
      "V315                                               0.32596119014527863\n",
      "DayOfMonth                                         0.32539258131204757\n",
      "D10                                                0.3210388166535918\n",
      "id_38                                              0.3167903730089841\n",
      "M2                                                 0.30976921184909795\n",
      "TransactionAmt_std_by_uuid3                        0.3036298470701444\n",
      "V67                                                0.3030430541997796\n",
      "C2_count                                           0.3009700734870775\n",
      "D2_na                                              0.29765055640760657\n",
      "C2                                                 0.2971679662278243\n",
      "V310                                               0.27700739198717633\n",
      "D12                                                0.27161719959149105\n",
      "TransactionAmt_mean_by_card1                       0.26778477668269646\n",
      "V274                                               0.2655823387198532\n",
      "id_25                                              0.26418036010590956\n",
      "C9_count                                           0.2585137766562731\n",
      "C10                                                0.2547306629584067\n",
      "TransactionAmt_mean_by_uuid3                       0.24778349218297713\n",
      "V189                                               0.24696610285843865\n",
      "V156                                               0.2454508309499246\n",
      "uuid3                                              0.24134337785454094\n",
      "M5_na                                              0.2413130881390448\n",
      "C9                                                 0.23288560701451674\n",
      "V264                                               0.23081600570313596\n",
      "V313                                               0.21931552379039587\n",
      "uuid2                                              0.21830584693327384\n",
      "C7                                                 0.21809275957464644\n",
      "C6                                                 0.2158344910468379\n",
      "V307_count                                         0.21156167273152274\n",
      "card5                                              0.21133592507891194\n",
      "M8                                                 0.20751882286422305\n",
      "V275                                               0.20729382445889866\n",
      "V128                                               0.20693332087697677\n",
      "card4                                              0.20379124122711914\n",
      "Browser                                            0.19962654309236083\n",
      "V30                                                0.19650685430722317\n",
      "M7                                                 0.1963495810951034\n",
      "V309                                               0.19366203568087892\n",
      "addr1_count                                        0.19161710792067568\n",
      "id_14                                              0.1909053488463047\n",
      "id_06                                              0.18874264209940317\n",
      "id_18                                              0.18731738069808399\n",
      "V87                                                0.1870529142589514\n",
      "D11                                                0.18694486105939997\n",
      "V201                                               0.18585932093860302\n",
      "V245                                               0.18490763383069278\n",
      "V312                                               0.18374618309493743\n",
      "card2__id_19_count                                 0.18224517552757583\n",
      "V149                                               0.18073044195571172\n",
      "DayOfWeek                                          0.17688794375917194\n",
      "id_12                                              0.17378924428417686\n",
      "V316                                               0.16569510338640395\n",
      "V165                                               0.1631725972874465\n",
      "id_01                                              0.16065063429431028\n",
      "R_emaildomain_sfx2                                 0.15570770780578252\n",
      "V133                                               0.15561800203705894\n",
      "V306                                               0.15463586005294275\n",
      "id_15                                              0.15439521493339595\n",
      "id_21                                              0.1518334139132189\n",
      "DeviceType                                         0.15154851666118416\n",
      "V314                                               0.15120977114870002\n",
      "card2_count                                        0.15109693906963548\n",
      "D5                                                 0.15019947200387648\n",
      "R_emaildomain_sfx1                                 0.1482059880568022\n",
      "id_29                                              0.1476755676400536\n",
      "M9                                                 0.14674327120542457\n",
      "V256                                               0.14325409389582774\n",
      "id_16                                              0.14300439272297746\n",
      "dist2                                              0.14185728530599057\n",
      "V211                                               0.13982953298493253\n",
      "addr2                                              0.13913489650095015\n",
      "P_emaildomain_count                                0.139129892479312\n",
      "V311                                               0.13716659066257372\n",
      "P_emaildomain_sfx2                                 0.1369833734199952\n",
      "id_35                                              0.13629857723848424\n",
      "C5_count                                           0.13620877999367997\n",
      "id_09                                              0.13492082916157375\n",
      "D14                                                0.1341040517940675\n",
      "V83                                                0.13268078943292502\n",
      "card2__id_20_count                                 0.1294344357644325\n",
      "id_36                                              0.1289367959560983\n",
      "P_emaildomain_sfx1                                 0.12581024860681186\n",
      "D3_na                                              0.12452396342677748\n",
      "V323                                               0.12335553657825243\n",
      "V313_mean_by_V307                                  0.12315721421158517\n",
      "id_28                                              0.12124872282886678\n",
      "V69                                                0.1206919001090383\n",
      "V277                                               0.1204468856750834\n",
      "V280                                               0.11910677912392936\n",
      "V303                                               0.11825793255893413\n",
      "id_17                                              0.11749385816041147\n",
      "D6                                                 0.11543979436895775\n",
      "DeviceInfo_count                                   0.11503404830581206\n",
      "card3_count                                        0.11440066547426213\n",
      "id_37                                              0.11340273103605097\n",
      "V318                                               0.1110856240472259\n",
      "V82                                                0.11090352916700986\n",
      "ProductCD__Cents_magic                             0.11089165544109163\n",
      "D7                                                 0.10958217014101157\n",
      "id_30                                              0.10941710332912286\n",
      "DeviceInfo__id_30_count                            0.10853915416767185\n",
      "V273                                               0.10823685471705924\n",
      "V127                                               0.10793946198988946\n",
      "V129                                               0.10461172838941152\n",
      "V102                                               0.10399209860019966\n",
      "V290                                               0.10332871764296726\n",
      "OS                                                 0.10258801730305878\n",
      "id_04                                              0.09671375077263412\n",
      "V12                                                0.09648555810390586\n",
      "V194                                               0.0944718331943271\n",
      "V257                                               0.09363767829561008\n",
      "V143                                               0.09311644894246927\n",
      "V131                                               0.09197189630688414\n",
      "M1                                                 0.08961435208332054\n",
      "V54                                                0.08812219376583645\n",
      "V265                                               0.08581458293372927\n",
      "id_03                                              0.08217186237502241\n",
      "V291                                               0.07996081184866989\n",
      "V212                                               0.07872978377171343\n",
      "V282                                               0.07741355816712532\n",
      "M4_na                                              0.07717037398857497\n",
      "V263                                               0.07709824456058588\n",
      "card5_count                                        0.07700912763406974\n",
      "V78                                                0.07486330365909856\n",
      "id_33_1                                            0.07481615542261821\n",
      "V16                                                0.073651509680474\n",
      "V91                                                0.07239942070009188\n",
      "V66                                                0.07227272770583414\n",
      "V187                                               0.07174423024149379\n",
      "V126                                               0.07120680970867342\n",
      "C5                                                 0.07028366825030037\n",
      "V85                                                0.07018646991170403\n",
      "V199                                               0.06864801750165554\n",
      "C3                                                 0.0685752040644065\n",
      "V279                                               0.06835942824572491\n",
      "V324                                               0.06795755379503521\n",
      "D13                                                0.06790216339637295\n",
      "id_34                                              0.06774236204111996\n",
      "V164                                               0.06754868784411187\n",
      "V134                                               0.06636698936300442\n",
      "V320                                               0.06603809136825892\n",
      "V96                                                0.06523346174286704\n",
      "V86                                                0.06502126997759328\n",
      "V298                                               0.06479905853141874\n",
      "V130                                               0.06383276498284433\n",
      "V44                                                0.06369558840194564\n",
      "V249                                               0.06289000646081057\n",
      "V136                                               0.06204176120589608\n",
      "V295                                               0.06172115893786876\n",
      "V38                                                0.05962246932269588\n",
      "V285                                               0.059612017315942296\n",
      "id_23                                              0.05942492055497831\n",
      "V29                                                0.05720095236937578\n",
      "id_11                                              0.05561857834391667\n",
      "V61                                                0.05537234385389193\n",
      "V132                                               0.05420979273589432\n",
      "V19                                                0.05300668536627174\n",
      "V203                                               0.05294193649684393\n",
      "id_32                                              0.051395218809089836\n",
      "V72                                                0.04998671960732709\n",
      "D9                                                 0.0482947232500753\n",
      "V59                                                0.0481615631309116\n",
      "V296                                               0.04754468669867388\n",
      "V234                                               0.0472583418446069\n",
      "V190                                               0.046674155227497904\n",
      "D5_na                                              0.0465594870281108\n",
      "id_24                                              0.04640801900176595\n",
      "V162                                               0.046123526643032674\n",
      "V152                                               0.04539992127053913\n",
      "V77                                                0.04524433452405638\n",
      "V40                                                0.045074927346582916\n",
      "V209                                               0.04458561505069988\n",
      "V319                                               0.04455243502043477\n",
      "V271                                               0.044465452030942716\n",
      "V108                                               0.04336783497344465\n",
      "V197                                               0.04278855301679493\n",
      "V74                                                0.042617877231239415\n",
      "V25                                                0.04170132527942368\n",
      "V292                                               0.04047852265412145\n",
      "V293                                               0.04004489440278675\n",
      "V58                                                0.03976687388431304\n",
      "V79                                                0.03967497847653192\n",
      "V169                                               0.03937382919789399\n",
      "id_27                                              0.03908295805173371\n",
      "V200                                               0.038759746787316474\n",
      "V243                                               0.03750993916992628\n",
      "V206                                               0.036466527888352994\n",
      "V139                                               0.0349250879332258\n",
      "V53                                                0.03446515244368998\n",
      "V301                                               0.03434175111390927\n",
      "V210                                               0.03425489400522899\n",
      "id_07                                              0.03423858319809537\n",
      "V24                                                0.03351142535014074\n",
      "V179                                               0.032942276925090705\n",
      "V285_count                                         0.03272650427142215\n",
      "V202                                               0.03249967672017682\n",
      "V225                                               0.03240867682401171\n",
      "id_33_0                                            0.03213908591253278\n",
      "V251                                               0.03180425517832159\n",
      "V266                                               0.031733992611223075\n",
      "V47                                                0.03130878320268928\n",
      "V105                                               0.030783630546160127\n",
      "V207                                               0.030448397938626387\n",
      "V208                                               0.030394187800525302\n",
      "V218                                               0.030393202111253223\n",
      "V23                                                0.03035824820274709\n",
      "id_22                                              0.0297486291464331\n",
      "V135                                               0.029374273789052486\n",
      "V172                                               0.029227697018141898\n",
      "V55                                                0.029224190842797444\n",
      "V174                                               0.028862173569224964\n",
      "V215                                               0.028795749643171765\n",
      "V76                                                0.02819304447391871\n",
      "V244                                               0.02813185881904277\n",
      "V151                                               0.027836324277644948\n",
      "V269                                               0.027031164035013527\n",
      "V100                                               0.02702724551824548\n",
      "P_emaildomain_parts                                0.02694392182460735\n",
      "V99                                                0.02679596842449977\n",
      "V116                                               0.026644794465339544\n",
      "V20                                                0.026623104189665067\n",
      "V124                                               0.026469389266534693\n",
      "V140                                               0.026452665038081715\n",
      "V60                                                0.02642091256017734\n",
      "V159                                               0.0262838233864405\n",
      "V205                                               0.025990151157561977\n",
      "V321                                               0.025901681046503738\n",
      "V160                                               0.02556534305958971\n",
      "V278                                               0.025397984625647813\n",
      "V270                                               0.025171862181383908\n",
      "V219                                               0.024477722076156425\n",
      "V75                                                0.023917578509552714\n",
      "V259                                               0.02387043135296496\n",
      "V115                                               0.023837143644084003\n",
      "V272                                               0.023675678605947218\n",
      "V26                                                0.023134683263079738\n",
      "V188                                               0.02279185959488553\n",
      "V125                                               0.02211761414634957\n",
      "V204                                               0.022102248029299536\n",
      "V304                                               0.022079197609388854\n",
      "V5                                                 0.021245890068310135\n",
      "V97                                                0.021170523601893337\n",
      "V226                                               0.02112410244740044\n",
      "V198                                               0.02108530541464808\n",
      "V220                                               0.020683726697515184\n",
      "V261                                               0.020532785268121842\n",
      "V109                                               0.020201754497181024\n",
      "V289                                               0.02003434730588968\n",
      "V180                                               0.019879575199159675\n",
      "V56                                                0.018991543869182056\n",
      "V338                                               0.01859395256444726\n",
      "V178                                               0.01798005987146206\n",
      "V168                                               0.01747286367086713\n",
      "V137                                               0.017350335169710646\n",
      "V300                                               0.016745224253934606\n",
      "V332                                               0.016465518855404756\n",
      "V4                                                 0.016431128145189492\n",
      "id_26                                              0.01607732723195963\n",
      "id_10                                              0.016028330645853284\n",
      "V123                                               0.016024735078904485\n",
      "V161                                               0.015642426299169808\n",
      "V288                                               0.015523668618145414\n",
      "V37                                                0.01535071532798155\n",
      "V267                                               0.01504330501712924\n",
      "V166                                               0.01490333021452739\n",
      "V43                                                0.014821266514343914\n",
      "V73                                                0.014814095519370291\n",
      "V52                                                0.014791647211790162\n",
      "V260                                               0.01457168151707805\n",
      "V14                                                0.014487513678468929\n",
      "V222                                               0.014464050987694618\n",
      "V232                                               0.013820781864944016\n",
      "V64                                                0.013669523015856543\n",
      "V213                                               0.013575173134718148\n",
      "V98                                                0.013555656733480054\n",
      "V214                                               0.012999622202074297\n",
      "V84                                                0.012890559827570999\n",
      "V3                                                 0.012783675678650318\n",
      "V216                                               0.012746930202976292\n",
      "V28                                                0.012671526400395863\n",
      "V95                                                0.012645377396628318\n",
      "V36                                                0.012627842565163637\n",
      "V111                                               0.011771537387186637\n",
      "V171                                               0.011628898903569572\n",
      "V253                                               0.011451391979650193\n",
      "V276                                               0.011419167586507956\n",
      "V284                                               0.011008525558699411\n",
      "V246                                               0.010909840243445582\n",
      "V239                                               0.010548292999083093\n",
      "V223                                               0.010386198720984044\n",
      "V93                                                0.010128464177352607\n",
      "V63                                                0.010086051499238908\n",
      "V13                                                0.009830499884276277\n",
      "V81                                                0.00975972531191193\n",
      "V299                                               0.00957435769779408\n",
      "V339                                               0.00949585807390164\n",
      "V240                                               0.009312693422308763\n",
      "V268                                               0.008829597106032075\n",
      "V229                                               0.008740618006265684\n",
      "V217                                               0.00861232815805396\n",
      "V337                                               0.008608151393056155\n",
      "V90                                                0.008594755390436087\n",
      "V34                                                0.008524966723918328\n",
      "V221                                               0.008463790126710707\n",
      "V163                                               0.008459220205495804\n",
      "V333                                               0.00844578080330239\n",
      "V71                                                0.008332655355865639\n",
      "V247                                               0.008059526343450243\n",
      "V101                                               0.007975102242831232\n",
      "V18                                                0.007970980603036415\n",
      "D14_na                                             0.007966022586387015\n",
      "V287                                               0.007888332907755086\n",
      "D13_na                                             0.00766394224537066\n",
      "V80                                                0.007584616077845767\n",
      "V32                                                0.007475369906901771\n",
      "V175                                               0.007441859119304701\n",
      "V11                                                0.007428935999365279\n",
      "V170                                               0.007297697708321799\n",
      "V150                                               0.007229356685023229\n",
      "V248                                               0.007170373690093297\n",
      "V302                                               0.007071883608681439\n",
      "V1                                                 0.006724681525093187\n",
      "V94                                                0.006649571597454366\n",
      "V286                                               0.006361947158339587\n",
      "V114                                               0.006347761549806825\n",
      "V255                                               0.006015417727218312\n",
      "V237                                               0.00598490815264207\n",
      "V334                                               0.005973962713829562\n",
      "V250                                               0.005889014040774797\n",
      "V110                                               0.005800672767313319\n",
      "V193                                               0.005772029158498526\n",
      "V262                                               0.005541284551318164\n",
      "V297                                               0.005487820848091703\n",
      "V224                                               0.005375429791163706\n",
      "V141                                               0.005258748065726065\n",
      "V228                                               0.005135232782332886\n",
      "R_emaildomain_parts                                0.00512461008805899\n",
      "V242                                               0.005039307612461646\n",
      "V33                                                0.004993975454064489\n",
      "V48                                                0.004954191560413283\n",
      "V39                                                0.004915853465226068\n",
      "V103                                               0.004889014439325311\n",
      "V231                                               0.00482684466771298\n",
      "V181                                               0.004808096767945229\n",
      "D6_na                                              0.00468139539299038\n",
      "V233                                               0.004677426202719373\n",
      "V57                                                0.004677297253029999\n",
      "V148                                               0.004664986906225082\n",
      "V238                                               0.00457730003729516\n",
      "V112                                               0.004401928582069739\n",
      "V177                                               0.004346993769431721\n",
      "V254                                               0.004307044291480458\n",
      "V106                                               0.004177934095973231\n",
      "id_16_na                                           0.004159263274819775\n",
      "V118                                               0.004052358153800363\n",
      "V51                                                0.003912263708199624\n",
      "V186                                               0.003899657754060252\n",
      "V335                                               0.003872300602202908\n",
      "V154                                               0.003841229020959387\n",
      "V146                                               0.0037645994023599878\n",
      "V49                                                0.0037413210747559798\n",
      "V9                                                 0.003673294735330825\n",
      "id_08                                              0.0036489163966161205\n",
      "V230                                               0.0036389029497136964\n",
      "V42                                                0.003588824131850144\n",
      "addr2_count                                        0.003500256240296045\n",
      "V167                                               0.0034746326335084164\n",
      "V145                                               0.0034642984807561074\n",
      "V113                                               0.003463951969294021\n",
      "V176                                               0.003462989766411863\n",
      "V330                                               0.0034101695651306174\n",
      "V192                                               0.003351408996715327\n",
      "V46                                                0.003256273594344302\n",
      "V10                                                0.003237542086474573\n",
      "V227                                               0.0032222355539575057\n",
      "V35                                                0.003213509422597115\n",
      "V183                                               0.003078345151415991\n",
      "V331                                               0.0030664607035916377\n",
      "V144                                               0.0029732939606102377\n",
      "V8                                                 0.002803675375268655\n",
      "V326                                               0.0026456714573552787\n",
      "V17                                                0.0024802966358342996\n",
      "V2                                                 0.0022053975913569886\n",
      "V104                                               0.002205327748828746\n",
      "V196                                               0.002129998787284577\n",
      "V322                                               0.0020776537219982403\n",
      "V336                                               0.0020032564955923342\n",
      "V88                                                0.001954828302400032\n",
      "V21                                                0.001765461780594981\n",
      "V184                                               0.0017341398345297368\n",
      "V122                                               0.0016978012710901179\n",
      "V92                                                0.0016977956076675005\n",
      "V155                                               0.0016882429004842602\n",
      "V117                                               0.0015306812128942592\n",
      "V235                                               0.001402369835959326\n",
      "V182                                               0.0013418098024336829\n",
      "V50                                                0.0010992058044290464\n",
      "V142                                               0.0010916243906584494\n",
      "V121                                               0.0010898168433292213\n",
      "V120                                               0.0010647863320819796\n",
      "V119                                               0.0009398163208256742\n",
      "V241                                               0.0009283110472142329\n",
      "V195                                               0.0009268397557476831\n",
      "V158                                               0.0007989821265361246\n",
      "V191                                               0.0007411660342901467\n",
      "V65                                                0.0007161773975032935\n",
      "V173                                               0.0006663467132663223\n",
      "V107                                               0.0005990326041969892\n",
      "V7                                                 0.0004442889230630198\n",
      "V89                                                0.00041156191556216077\n",
      "V236                                               0.0003629535416478812\n",
      "V329                                               0.0003520574110330425\n",
      "V327                                               0.00023498455310897218\n",
      "D7_na                                              0.00019651133231268935\n",
      "V31                                                9.20737962744106e-05\n",
      "V68                                                0.0\n",
      "V6                                                 0.0\n",
      "V41                                                0.0\n",
      "V328                                               0.0\n",
      "V325                                               0.0\n",
      "V305                                               0.0\n",
      "V27                                                0.0\n",
      "V252                                               0.0\n",
      "V22                                                0.0\n",
      "V185                                               0.0\n",
      "V157                                               0.0\n",
      "V153                                               0.0\n",
      "V15                                                0.0\n",
      "V147                                               0.0\n",
      "V138                                               0.0\n",
      "M6_na                                              0.0\n"
     ]
    }
   ],
   "source": [
    "for score, name in sorted(zip(np.average(feat_imps, axis=0), df_train.columns), reverse=True):\n",
    "    print('{:50s} {}'.format(name, score))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.read_csv('../input/ieee-fraud-detection/sample_submission.csv')\n",
    "sub['isFraud'] = np.average(test_preds, axis=0)\n",
    "sub.to_csv('submission.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}