04_feature_engineering/000_tvt_code_split.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, math, subprocess\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from IPython.display import display\n",
    "\n",
    "# some settings for displaying Pandas results\n",
    "pd.set_option('display.width', 2000)\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.precision', 4)\n",
    "pd.set_option('display.max_colwidth', -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load train and test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('(rows, columns)', (307511, 122))\n",
      "First 5 rows\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>NAME_CONTRACT_TYPE</th>\n",
       "      <th>CODE_GENDER</th>\n",
       "      <th>FLAG_OWN_CAR</th>\n",
       "      <th>FLAG_OWN_REALTY</th>\n",
       "      <th>CNT_CHILDREN</th>\n",
       "      <th>AMT_INCOME_TOTAL</th>\n",
       "      <th>AMT_CREDIT</th>\n",
       "      <th>AMT_ANNUITY</th>\n",
       "      <th>AMT_GOODS_PRICE</th>\n",
       "      <th>NAME_TYPE_SUITE</th>\n",
       "      <th>NAME_INCOME_TYPE</th>\n",
       "      <th>NAME_EDUCATION_TYPE</th>\n",
       "      <th>NAME_FAMILY_STATUS</th>\n",
       "      <th>NAME_HOUSING_TYPE</th>\n",
       "      <th>REGION_POPULATION_RELATIVE</th>\n",
       "      <th>DAYS_BIRTH</th>\n",
       "      <th>DAYS_EMPLOYED</th>\n",
       "      <th>DAYS_REGISTRATION</th>\n",
       "      <th>DAYS_ID_PUBLISH</th>\n",
       "      <th>OWN_CAR_AGE</th>\n",
       "      <th>FLAG_MOBIL</th>\n",
       "      <th>FLAG_EMP_PHONE</th>\n",
       "      <th>FLAG_WORK_PHONE</th>\n",
       "      <th>FLAG_CONT_MOBILE</th>\n",
       "      <th>FLAG_PHONE</th>\n",
       "      <th>FLAG_EMAIL</th>\n",
       "      <th>OCCUPATION_TYPE</th>\n",
       "      <th>CNT_FAM_MEMBERS</th>\n",
       "      <th>REGION_RATING_CLIENT</th>\n",
       "      <th>REGION_RATING_CLIENT_W_CITY</th>\n",
       "      <th>WEEKDAY_APPR_PROCESS_START</th>\n",
       "      <th>HOUR_APPR_PROCESS_START</th>\n",
       "      <th>REG_REGION_NOT_LIVE_REGION</th>\n",
       "      <th>REG_REGION_NOT_WORK_REGION</th>\n",
       "      <th>LIVE_REGION_NOT_WORK_REGION</th>\n",
       "      <th>REG_CITY_NOT_LIVE_CITY</th>\n",
       "      <th>REG_CITY_NOT_WORK_CITY</th>\n",
       "      <th>LIVE_CITY_NOT_WORK_CITY</th>\n",
       "      <th>ORGANIZATION_TYPE</th>\n",
       "      <th>EXT_SOURCE_1</th>\n",
       "      <th>EXT_SOURCE_2</th>\n",
       "      <th>EXT_SOURCE_3</th>\n",
       "      <th>APARTMENTS_AVG</th>\n",
       "      <th>BASEMENTAREA_AVG</th>\n",
       "      <th>YEARS_BEGINEXPLUATATION_AVG</th>\n",
       "      <th>YEARS_BUILD_AVG</th>\n",
       "      <th>COMMONAREA_AVG</th>\n",
       "      <th>ELEVATORS_AVG</th>\n",
       "      <th>ENTRANCES_AVG</th>\n",
       "      <th>FLOORSMAX_AVG</th>\n",
       "      <th>FLOORSMIN_AVG</th>\n",
       "      <th>LANDAREA_AVG</th>\n",
       "      <th>LIVINGAPARTMENTS_AVG</th>\n",
       "      <th>LIVINGAREA_AVG</th>\n",
       "      <th>NONLIVINGAPARTMENTS_AVG</th>\n",
       "      <th>NONLIVINGAREA_AVG</th>\n",
       "      <th>APARTMENTS_MODE</th>\n",
       "      <th>BASEMENTAREA_MODE</th>\n",
       "      <th>YEARS_BEGINEXPLUATATION_MODE</th>\n",
       "      <th>YEARS_BUILD_MODE</th>\n",
       "      <th>COMMONAREA_MODE</th>\n",
       "      <th>ELEVATORS_MODE</th>\n",
       "      <th>ENTRANCES_MODE</th>\n",
       "      <th>FLOORSMAX_MODE</th>\n",
       "      <th>FLOORSMIN_MODE</th>\n",
       "      <th>LANDAREA_MODE</th>\n",
       "      <th>LIVINGAPARTMENTS_MODE</th>\n",
       "      <th>LIVINGAREA_MODE</th>\n",
       "      <th>NONLIVINGAPARTMENTS_MODE</th>\n",
       "      <th>NONLIVINGAREA_MODE</th>\n",
       "      <th>APARTMENTS_MEDI</th>\n",
       "      <th>BASEMENTAREA_MEDI</th>\n",
       "      <th>YEARS_BEGINEXPLUATATION_MEDI</th>\n",
       "      <th>YEARS_BUILD_MEDI</th>\n",
       "      <th>COMMONAREA_MEDI</th>\n",
       "      <th>ELEVATORS_MEDI</th>\n",
       "      <th>ENTRANCES_MEDI</th>\n",
       "      <th>FLOORSMAX_MEDI</th>\n",
       "      <th>FLOORSMIN_MEDI</th>\n",
       "      <th>LANDAREA_MEDI</th>\n",
       "      <th>LIVINGAPARTMENTS_MEDI</th>\n",
       "      <th>LIVINGAREA_MEDI</th>\n",
       "      <th>NONLIVINGAPARTMENTS_MEDI</th>\n",
       "      <th>NONLIVINGAREA_MEDI</th>\n",
       "      <th>FONDKAPREMONT_MODE</th>\n",
       "      <th>HOUSETYPE_MODE</th>\n",
       "      <th>TOTALAREA_MODE</th>\n",
       "      <th>WALLSMATERIAL_MODE</th>\n",
       "      <th>EMERGENCYSTATE_MODE</th>\n",
       "      <th>OBS_30_CNT_SOCIAL_CIRCLE</th>\n",
       "      <th>DEF_30_CNT_SOCIAL_CIRCLE</th>\n",
       "      <th>OBS_60_CNT_SOCIAL_CIRCLE</th>\n",
       "      <th>DEF_60_CNT_SOCIAL_CIRCLE</th>\n",
       "      <th>DAYS_LAST_PHONE_CHANGE</th>\n",
       "      <th>FLAG_DOCUMENT_2</th>\n",
       "      <th>FLAG_DOCUMENT_3</th>\n",
       "      <th>FLAG_DOCUMENT_4</th>\n",
       "      <th>FLAG_DOCUMENT_5</th>\n",
       "      <th>FLAG_DOCUMENT_6</th>\n",
       "      <th>FLAG_DOCUMENT_7</th>\n",
       "      <th>FLAG_DOCUMENT_8</th>\n",
       "      <th>FLAG_DOCUMENT_9</th>\n",
       "      <th>FLAG_DOCUMENT_10</th>\n",
       "      <th>FLAG_DOCUMENT_11</th>\n",
       "      <th>FLAG_DOCUMENT_12</th>\n",
       "      <th>FLAG_DOCUMENT_13</th>\n",
       "      <th>FLAG_DOCUMENT_14</th>\n",
       "      <th>FLAG_DOCUMENT_15</th>\n",
       "      <th>FLAG_DOCUMENT_16</th>\n",
       "      <th>FLAG_DOCUMENT_17</th>\n",
       "      <th>FLAG_DOCUMENT_18</th>\n",
       "      <th>FLAG_DOCUMENT_19</th>\n",
       "      <th>FLAG_DOCUMENT_20</th>\n",
       "      <th>FLAG_DOCUMENT_21</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_HOUR</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_DAY</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_WEEK</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_MON</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_QRT</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_YEAR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100002</td>\n",
       "      <td>1</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>202500.0</td>\n",
       "      <td>406597.5</td>\n",
       "      <td>24700.5</td>\n",
       "      <td>351000.0</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Working</td>\n",
       "      <td>Secondary / secondary special</td>\n",
       "      <td>Single / not married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.0188</td>\n",
       "      <td>-9461</td>\n",
       "      <td>-637</td>\n",
       "      <td>-3648.0</td>\n",
       "      <td>-2120</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Laborers</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>WEDNESDAY</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Business Entity Type 3</td>\n",
       "      <td>0.0830</td>\n",
       "      <td>0.2629</td>\n",
       "      <td>0.1394</td>\n",
       "      <td>0.0247</td>\n",
       "      <td>0.0369</td>\n",
       "      <td>0.9722</td>\n",
       "      <td>0.6192</td>\n",
       "      <td>0.0143</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0690</td>\n",
       "      <td>0.0833</td>\n",
       "      <td>0.1250</td>\n",
       "      <td>0.0369</td>\n",
       "      <td>0.0202</td>\n",
       "      <td>0.0190</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0252</td>\n",
       "      <td>0.0383</td>\n",
       "      <td>0.9722</td>\n",
       "      <td>0.6341</td>\n",
       "      <td>0.0144</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0690</td>\n",
       "      <td>0.0833</td>\n",
       "      <td>0.1250</td>\n",
       "      <td>0.0377</td>\n",
       "      <td>0.022</td>\n",
       "      <td>0.0198</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0250</td>\n",
       "      <td>0.0369</td>\n",
       "      <td>0.9722</td>\n",
       "      <td>0.6243</td>\n",
       "      <td>0.0144</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0690</td>\n",
       "      <td>0.0833</td>\n",
       "      <td>0.1250</td>\n",
       "      <td>0.0375</td>\n",
       "      <td>0.0205</td>\n",
       "      <td>0.0193</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>reg oper account</td>\n",
       "      <td>block of flats</td>\n",
       "      <td>0.0149</td>\n",
       "      <td>Stone, brick</td>\n",
       "      <td>No</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-1134.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100003</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>0</td>\n",
       "      <td>270000.0</td>\n",
       "      <td>1293502.5</td>\n",
       "      <td>35698.5</td>\n",
       "      <td>1129500.0</td>\n",
       "      <td>Family</td>\n",
       "      <td>State servant</td>\n",
       "      <td>Higher education</td>\n",
       "      <td>Married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.0035</td>\n",
       "      <td>-16765</td>\n",
       "      <td>-1188</td>\n",
       "      <td>-1186.0</td>\n",
       "      <td>-291</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Core staff</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>MONDAY</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>School</td>\n",
       "      <td>0.3113</td>\n",
       "      <td>0.6222</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0959</td>\n",
       "      <td>0.0529</td>\n",
       "      <td>0.9851</td>\n",
       "      <td>0.7960</td>\n",
       "      <td>0.0605</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.0345</td>\n",
       "      <td>0.2917</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>0.0130</td>\n",
       "      <td>0.0773</td>\n",
       "      <td>0.0549</td>\n",
       "      <td>0.0039</td>\n",
       "      <td>0.0098</td>\n",
       "      <td>0.0924</td>\n",
       "      <td>0.0538</td>\n",
       "      <td>0.9851</td>\n",
       "      <td>0.8040</td>\n",
       "      <td>0.0497</td>\n",
       "      <td>0.0806</td>\n",
       "      <td>0.0345</td>\n",
       "      <td>0.2917</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>0.0128</td>\n",
       "      <td>0.079</td>\n",
       "      <td>0.0554</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0968</td>\n",
       "      <td>0.0529</td>\n",
       "      <td>0.9851</td>\n",
       "      <td>0.7987</td>\n",
       "      <td>0.0608</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.0345</td>\n",
       "      <td>0.2917</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>0.0132</td>\n",
       "      <td>0.0787</td>\n",
       "      <td>0.0558</td>\n",
       "      <td>0.0039</td>\n",
       "      <td>0.01</td>\n",
       "      <td>reg oper account</td>\n",
       "      <td>block of flats</td>\n",
       "      <td>0.0714</td>\n",
       "      <td>Block</td>\n",
       "      <td>No</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-828.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100004</td>\n",
       "      <td>0</td>\n",
       "      <td>Revolving loans</td>\n",
       "      <td>M</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>67500.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>6750.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Working</td>\n",
       "      <td>Secondary / secondary special</td>\n",
       "      <td>Single / not married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.0100</td>\n",
       "      <td>-19046</td>\n",
       "      <td>-225</td>\n",
       "      <td>-4260.0</td>\n",
       "      <td>-2531</td>\n",
       "      <td>26.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Laborers</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>MONDAY</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Government</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.5559</td>\n",
       "      <td>0.7296</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-815.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100006</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>312682.5</td>\n",
       "      <td>29686.5</td>\n",
       "      <td>297000.0</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Working</td>\n",
       "      <td>Secondary / secondary special</td>\n",
       "      <td>Civil marriage</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.0080</td>\n",
       "      <td>-19005</td>\n",
       "      <td>-3039</td>\n",
       "      <td>-9833.0</td>\n",
       "      <td>-2437</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Laborers</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>WEDNESDAY</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Business Entity Type 3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.6504</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-617.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100007</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>121500.0</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>21865.5</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Working</td>\n",
       "      <td>Secondary / secondary special</td>\n",
       "      <td>Single / not married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.0287</td>\n",
       "      <td>-19932</td>\n",
       "      <td>-3038</td>\n",
       "      <td>-4311.0</td>\n",
       "      <td>-3458</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Core staff</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>THURSDAY</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Religion</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.3227</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1106.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR  TARGET NAME_CONTRACT_TYPE CODE_GENDER FLAG_OWN_CAR FLAG_OWN_REALTY  CNT_CHILDREN  AMT_INCOME_TOTAL  AMT_CREDIT  AMT_ANNUITY  AMT_GOODS_PRICE NAME_TYPE_SUITE NAME_INCOME_TYPE            NAME_EDUCATION_TYPE    NAME_FAMILY_STATUS  NAME_HOUSING_TYPE  REGION_POPULATION_RELATIVE  DAYS_BIRTH  DAYS_EMPLOYED  DAYS_REGISTRATION  DAYS_ID_PUBLISH  OWN_CAR_AGE  FLAG_MOBIL  FLAG_EMP_PHONE  FLAG_WORK_PHONE  FLAG_CONT_MOBILE  FLAG_PHONE  FLAG_EMAIL OCCUPATION_TYPE  CNT_FAM_MEMBERS  REGION_RATING_CLIENT  REGION_RATING_CLIENT_W_CITY WEEKDAY_APPR_PROCESS_START  HOUR_APPR_PROCESS_START  REG_REGION_NOT_LIVE_REGION  REG_REGION_NOT_WORK_REGION  LIVE_REGION_NOT_WORK_REGION  REG_CITY_NOT_LIVE_CITY  REG_CITY_NOT_WORK_CITY  LIVE_CITY_NOT_WORK_CITY       ORGANIZATION_TYPE  EXT_SOURCE_1  EXT_SOURCE_2  EXT_SOURCE_3  APARTMENTS_AVG  BASEMENTAREA_AVG  YEARS_BEGINEXPLUATATION_AVG  YEARS_BUILD_AVG  COMMONAREA_AVG  ELEVATORS_AVG  ENTRANCES_AVG  FLOORSMAX_AVG  FLOORSMIN_AVG  LANDAREA_AVG  LIVINGAPARTMENTS_AVG  LIVINGAREA_AVG  NONLIVINGAPARTMENTS_AVG  NONLIVINGAREA_AVG  APARTMENTS_MODE  BASEMENTAREA_MODE  YEARS_BEGINEXPLUATATION_MODE  YEARS_BUILD_MODE  COMMONAREA_MODE  ELEVATORS_MODE  ENTRANCES_MODE  FLOORSMAX_MODE  FLOORSMIN_MODE  LANDAREA_MODE  LIVINGAPARTMENTS_MODE  LIVINGAREA_MODE  NONLIVINGAPARTMENTS_MODE  NONLIVINGAREA_MODE  APARTMENTS_MEDI  BASEMENTAREA_MEDI  YEARS_BEGINEXPLUATATION_MEDI  YEARS_BUILD_MEDI  COMMONAREA_MEDI  ELEVATORS_MEDI  ENTRANCES_MEDI  FLOORSMAX_MEDI  FLOORSMIN_MEDI  LANDAREA_MEDI  LIVINGAPARTMENTS_MEDI  LIVINGAREA_MEDI  NONLIVINGAPARTMENTS_MEDI  NONLIVINGAREA_MEDI FONDKAPREMONT_MODE  HOUSETYPE_MODE  TOTALAREA_MODE WALLSMATERIAL_MODE EMERGENCYSTATE_MODE  OBS_30_CNT_SOCIAL_CIRCLE  DEF_30_CNT_SOCIAL_CIRCLE  OBS_60_CNT_SOCIAL_CIRCLE  DEF_60_CNT_SOCIAL_CIRCLE  DAYS_LAST_PHONE_CHANGE  FLAG_DOCUMENT_2  FLAG_DOCUMENT_3  FLAG_DOCUMENT_4  FLAG_DOCUMENT_5  FLAG_DOCUMENT_6  FLAG_DOCUMENT_7  FLAG_DOCUMENT_8  FLAG_DOCUMENT_9  FLAG_DOCUMENT_10  FLAG_DOCUMENT_11  \\\n",
       "0  100002      1       Cash loans         M           N            Y               0             202500.0          406597.5    24700.5      351000.0         Unaccompanied   Working          Secondary / secondary special  Single / not married  House / apartment  0.0188                     -9461       -637           -3648.0            -2120            NaN           1           1               0                1                 1           0           Laborers        1.0              2                     2                            WEDNESDAY                  10                       0                           0                           0                            0                       0                       0                        Business Entity Type 3  0.0830        0.2629        0.1394        0.0247          0.0369            0.9722                       0.6192           0.0143          0.00           0.0690         0.0833         0.1250         0.0369        0.0202                0.0190          0.0000                   0.0000             0.0252           0.0383             0.9722                        0.6341            0.0144           0.0000          0.0690          0.0833          0.1250          0.0377         0.022                  0.0198           0.0                       0.0                 0.0250           0.0369             0.9722                        0.6243            0.0144           0.00            0.0690          0.0833          0.1250          0.0375         0.0205                 0.0193           0.0000                    0.00                reg oper account   block of flats  0.0149          Stone, brick       No                  2.0                       2.0                       2.0                       2.0                      -1134.0                  0                1                0                0                0                0                0                0                0                 0                  \n",
       "1  100003      0       Cash loans         F           N            N               0             270000.0          1293502.5   35698.5      1129500.0        Family          State servant    Higher education               Married               House / apartment  0.0035                     -16765      -1188          -1186.0            -291             NaN           1           1               0                1                 1           0           Core staff      2.0              1                     1                            MONDAY                     11                       0                           0                           0                            0                       0                       0                        School                  0.3113        0.6222       NaN            0.0959          0.0529            0.9851                       0.7960           0.0605          0.08           0.0345         0.2917         0.3333         0.0130        0.0773                0.0549          0.0039                   0.0098             0.0924           0.0538             0.9851                        0.8040            0.0497           0.0806          0.0345          0.2917          0.3333          0.0128         0.079                  0.0554           0.0                       0.0                 0.0968           0.0529             0.9851                        0.7987            0.0608           0.08            0.0345          0.2917          0.3333          0.0132         0.0787                 0.0558           0.0039                    0.01                reg oper account   block of flats  0.0714          Block              No                  1.0                       0.0                       1.0                       0.0                      -828.0                   0                1                0                0                0                0                0                0                0                 0                  \n",
       "2  100004      0       Revolving loans    M           Y            Y               0             67500.0           135000.0    6750.0       135000.0         Unaccompanied   Working          Secondary / secondary special  Single / not married  House / apartment  0.0100                     -19046      -225           -4260.0            -2531             26.0         1           1               1                1                 1           0           Laborers        1.0              2                     2                            MONDAY                     9                        0                           0                           0                            0                       0                       0                        Government             NaN            0.5559        0.7296       NaN             NaN               NaN                          NaN              NaN             NaN            NaN            NaN            NaN            NaN           NaN                   NaN             NaN                      NaN                NaN              NaN                NaN                           NaN               NaN              NaN             NaN             NaN             NaN             NaN            NaN                    NaN              NaN                       NaN                 NaN              NaN                NaN                           NaN               NaN              NaN             NaN             NaN             NaN             NaN            NaN                    NaN              NaN                       NaN                  NaN                NaN            NaN              NaN                NaN                 0.0                       0.0                       0.0                       0.0                      -815.0                   0                0                0                0                0                0                0                0                0                 0                  \n",
       "3  100006      0       Cash loans         F           N            Y               0             135000.0          312682.5    29686.5      297000.0         Unaccompanied   Working          Secondary / secondary special  Civil marriage        House / apartment  0.0080                     -19005      -3039          -9833.0            -2437            NaN           1           1               0                1                 0           0           Laborers        2.0              2                     2                            WEDNESDAY                  17                       0                           0                           0                            0                       0                       0                        Business Entity Type 3 NaN            0.6504       NaN           NaN             NaN               NaN                          NaN              NaN             NaN            NaN            NaN            NaN            NaN           NaN                   NaN             NaN                      NaN                NaN              NaN                NaN                           NaN               NaN              NaN             NaN             NaN             NaN             NaN            NaN                    NaN              NaN                       NaN                 NaN              NaN                NaN                           NaN               NaN              NaN             NaN             NaN             NaN             NaN            NaN                    NaN              NaN                       NaN                  NaN                NaN            NaN              NaN                NaN                 2.0                       0.0                       2.0                       0.0                      -617.0                   0                1                0                0                0                0                0                0                0                 0                  \n",
       "4  100007      0       Cash loans         M           N            Y               0             121500.0          513000.0    21865.5      513000.0         Unaccompanied   Working          Secondary / secondary special  Single / not married  House / apartment  0.0287                     -19932      -3038          -4311.0            -3458            NaN           1           1               0                1                 0           0           Core staff      1.0              2                     2                            THURSDAY                   11                       0                           0                           0                            0                       1                       1                        Religion               NaN            0.3227       NaN           NaN             NaN               NaN                          NaN              NaN             NaN            NaN            NaN            NaN            NaN           NaN                   NaN             NaN                      NaN                NaN              NaN                NaN                           NaN               NaN              NaN             NaN             NaN             NaN             NaN            NaN                    NaN              NaN                       NaN                 NaN              NaN                NaN                           NaN               NaN              NaN             NaN             NaN             NaN             NaN            NaN                    NaN              NaN                       NaN                  NaN                NaN            NaN              NaN                NaN                 0.0                       0.0                       0.0                       0.0                      -1106.0                  0                0                0                0                0                0                1                0                0                 0                  \n",
       "\n",
       "   FLAG_DOCUMENT_12  FLAG_DOCUMENT_13  FLAG_DOCUMENT_14  FLAG_DOCUMENT_15  FLAG_DOCUMENT_16  FLAG_DOCUMENT_17  FLAG_DOCUMENT_18  FLAG_DOCUMENT_19  FLAG_DOCUMENT_20  FLAG_DOCUMENT_21  AMT_REQ_CREDIT_BUREAU_HOUR  AMT_REQ_CREDIT_BUREAU_DAY  AMT_REQ_CREDIT_BUREAU_WEEK  AMT_REQ_CREDIT_BUREAU_MON  AMT_REQ_CREDIT_BUREAU_QRT  AMT_REQ_CREDIT_BUREAU_YEAR  \n",
       "0  0                 0                 0                 0                 0                 0                 0                 0                 0                 0                 0.0                         0.0                        0.0                         0.0                        0.0                        1.0                         \n",
       "1  0                 0                 0                 0                 0                 0                 0                 0                 0                 0                 0.0                         0.0                        0.0                         0.0                        0.0                        0.0                         \n",
       "2  0                 0                 0                 0                 0                 0                 0                 0                 0                 0                 0.0                         0.0                        0.0                         0.0                        0.0                        0.0                         \n",
       "3  0                 0                 0                 0                 0                 0                 0                 0                 0                 0                NaN                         NaN                        NaN                         NaN                        NaN                        NaN                          \n",
       "4  0                 0                 0                 0                 0                 0                 0                 0                 0                 0                 0.0                         0.0                        0.0                         0.0                        0.0                        0.0                         "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('(rows, columns)', (48744, 121))\n",
      "First 5 rows\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>NAME_CONTRACT_TYPE</th>\n",
       "      <th>CODE_GENDER</th>\n",
       "      <th>FLAG_OWN_CAR</th>\n",
       "      <th>FLAG_OWN_REALTY</th>\n",
       "      <th>CNT_CHILDREN</th>\n",
       "      <th>AMT_INCOME_TOTAL</th>\n",
       "      <th>AMT_CREDIT</th>\n",
       "      <th>AMT_ANNUITY</th>\n",
       "      <th>AMT_GOODS_PRICE</th>\n",
       "      <th>NAME_TYPE_SUITE</th>\n",
       "      <th>NAME_INCOME_TYPE</th>\n",
       "      <th>NAME_EDUCATION_TYPE</th>\n",
       "      <th>NAME_FAMILY_STATUS</th>\n",
       "      <th>NAME_HOUSING_TYPE</th>\n",
       "      <th>REGION_POPULATION_RELATIVE</th>\n",
       "      <th>DAYS_BIRTH</th>\n",
       "      <th>DAYS_EMPLOYED</th>\n",
       "      <th>DAYS_REGISTRATION</th>\n",
       "      <th>DAYS_ID_PUBLISH</th>\n",
       "      <th>OWN_CAR_AGE</th>\n",
       "      <th>FLAG_MOBIL</th>\n",
       "      <th>FLAG_EMP_PHONE</th>\n",
       "      <th>FLAG_WORK_PHONE</th>\n",
       "      <th>FLAG_CONT_MOBILE</th>\n",
       "      <th>FLAG_PHONE</th>\n",
       "      <th>FLAG_EMAIL</th>\n",
       "      <th>OCCUPATION_TYPE</th>\n",
       "      <th>CNT_FAM_MEMBERS</th>\n",
       "      <th>REGION_RATING_CLIENT</th>\n",
       "      <th>REGION_RATING_CLIENT_W_CITY</th>\n",
       "      <th>WEEKDAY_APPR_PROCESS_START</th>\n",
       "      <th>HOUR_APPR_PROCESS_START</th>\n",
       "      <th>REG_REGION_NOT_LIVE_REGION</th>\n",
       "      <th>REG_REGION_NOT_WORK_REGION</th>\n",
       "      <th>LIVE_REGION_NOT_WORK_REGION</th>\n",
       "      <th>REG_CITY_NOT_LIVE_CITY</th>\n",
       "      <th>REG_CITY_NOT_WORK_CITY</th>\n",
       "      <th>LIVE_CITY_NOT_WORK_CITY</th>\n",
       "      <th>ORGANIZATION_TYPE</th>\n",
       "      <th>EXT_SOURCE_1</th>\n",
       "      <th>EXT_SOURCE_2</th>\n",
       "      <th>EXT_SOURCE_3</th>\n",
       "      <th>APARTMENTS_AVG</th>\n",
       "      <th>BASEMENTAREA_AVG</th>\n",
       "      <th>YEARS_BEGINEXPLUATATION_AVG</th>\n",
       "      <th>YEARS_BUILD_AVG</th>\n",
       "      <th>COMMONAREA_AVG</th>\n",
       "      <th>ELEVATORS_AVG</th>\n",
       "      <th>ENTRANCES_AVG</th>\n",
       "      <th>FLOORSMAX_AVG</th>\n",
       "      <th>FLOORSMIN_AVG</th>\n",
       "      <th>LANDAREA_AVG</th>\n",
       "      <th>LIVINGAPARTMENTS_AVG</th>\n",
       "      <th>LIVINGAREA_AVG</th>\n",
       "      <th>NONLIVINGAPARTMENTS_AVG</th>\n",
       "      <th>NONLIVINGAREA_AVG</th>\n",
       "      <th>APARTMENTS_MODE</th>\n",
       "      <th>BASEMENTAREA_MODE</th>\n",
       "      <th>YEARS_BEGINEXPLUATATION_MODE</th>\n",
       "      <th>YEARS_BUILD_MODE</th>\n",
       "      <th>COMMONAREA_MODE</th>\n",
       "      <th>ELEVATORS_MODE</th>\n",
       "      <th>ENTRANCES_MODE</th>\n",
       "      <th>FLOORSMAX_MODE</th>\n",
       "      <th>FLOORSMIN_MODE</th>\n",
       "      <th>LANDAREA_MODE</th>\n",
       "      <th>LIVINGAPARTMENTS_MODE</th>\n",
       "      <th>LIVINGAREA_MODE</th>\n",
       "      <th>NONLIVINGAPARTMENTS_MODE</th>\n",
       "      <th>NONLIVINGAREA_MODE</th>\n",
       "      <th>APARTMENTS_MEDI</th>\n",
       "      <th>BASEMENTAREA_MEDI</th>\n",
       "      <th>YEARS_BEGINEXPLUATATION_MEDI</th>\n",
       "      <th>YEARS_BUILD_MEDI</th>\n",
       "      <th>COMMONAREA_MEDI</th>\n",
       "      <th>ELEVATORS_MEDI</th>\n",
       "      <th>ENTRANCES_MEDI</th>\n",
       "      <th>FLOORSMAX_MEDI</th>\n",
       "      <th>FLOORSMIN_MEDI</th>\n",
       "      <th>LANDAREA_MEDI</th>\n",
       "      <th>LIVINGAPARTMENTS_MEDI</th>\n",
       "      <th>LIVINGAREA_MEDI</th>\n",
       "      <th>NONLIVINGAPARTMENTS_MEDI</th>\n",
       "      <th>NONLIVINGAREA_MEDI</th>\n",
       "      <th>FONDKAPREMONT_MODE</th>\n",
       "      <th>HOUSETYPE_MODE</th>\n",
       "      <th>TOTALAREA_MODE</th>\n",
       "      <th>WALLSMATERIAL_MODE</th>\n",
       "      <th>EMERGENCYSTATE_MODE</th>\n",
       "      <th>OBS_30_CNT_SOCIAL_CIRCLE</th>\n",
       "      <th>DEF_30_CNT_SOCIAL_CIRCLE</th>\n",
       "      <th>OBS_60_CNT_SOCIAL_CIRCLE</th>\n",
       "      <th>DEF_60_CNT_SOCIAL_CIRCLE</th>\n",
       "      <th>DAYS_LAST_PHONE_CHANGE</th>\n",
       "      <th>FLAG_DOCUMENT_2</th>\n",
       "      <th>FLAG_DOCUMENT_3</th>\n",
       "      <th>FLAG_DOCUMENT_4</th>\n",
       "      <th>FLAG_DOCUMENT_5</th>\n",
       "      <th>FLAG_DOCUMENT_6</th>\n",
       "      <th>FLAG_DOCUMENT_7</th>\n",
       "      <th>FLAG_DOCUMENT_8</th>\n",
       "      <th>FLAG_DOCUMENT_9</th>\n",
       "      <th>FLAG_DOCUMENT_10</th>\n",
       "      <th>FLAG_DOCUMENT_11</th>\n",
       "      <th>FLAG_DOCUMENT_12</th>\n",
       "      <th>FLAG_DOCUMENT_13</th>\n",
       "      <th>FLAG_DOCUMENT_14</th>\n",
       "      <th>FLAG_DOCUMENT_15</th>\n",
       "      <th>FLAG_DOCUMENT_16</th>\n",
       "      <th>FLAG_DOCUMENT_17</th>\n",
       "      <th>FLAG_DOCUMENT_18</th>\n",
       "      <th>FLAG_DOCUMENT_19</th>\n",
       "      <th>FLAG_DOCUMENT_20</th>\n",
       "      <th>FLAG_DOCUMENT_21</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_HOUR</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_DAY</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_WEEK</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_MON</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_QRT</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_YEAR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100001</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>568800.0</td>\n",
       "      <td>20560.5</td>\n",
       "      <td>450000.0</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Working</td>\n",
       "      <td>Higher education</td>\n",
       "      <td>Married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.0188</td>\n",
       "      <td>-19241</td>\n",
       "      <td>-2329</td>\n",
       "      <td>-5170.0</td>\n",
       "      <td>-812</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>TUESDAY</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Kindergarten</td>\n",
       "      <td>0.7526</td>\n",
       "      <td>0.7897</td>\n",
       "      <td>0.1595</td>\n",
       "      <td>0.0660</td>\n",
       "      <td>0.0590</td>\n",
       "      <td>0.9732</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.1379</td>\n",
       "      <td>0.125</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0505</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0672</td>\n",
       "      <td>0.0612</td>\n",
       "      <td>0.9732</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.1379</td>\n",
       "      <td>0.125</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0526</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0666</td>\n",
       "      <td>0.0590</td>\n",
       "      <td>0.9732</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.1379</td>\n",
       "      <td>0.125</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0514</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>block of flats</td>\n",
       "      <td>0.0392</td>\n",
       "      <td>Stone, brick</td>\n",
       "      <td>No</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1740.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100005</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>99000.0</td>\n",
       "      <td>222768.0</td>\n",
       "      <td>17370.0</td>\n",
       "      <td>180000.0</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Working</td>\n",
       "      <td>Secondary / secondary special</td>\n",
       "      <td>Married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.0358</td>\n",
       "      <td>-18064</td>\n",
       "      <td>-4469</td>\n",
       "      <td>-9118.0</td>\n",
       "      <td>-1623</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Low-skill Laborers</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>FRIDAY</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Self-employed</td>\n",
       "      <td>0.5650</td>\n",
       "      <td>0.2917</td>\n",
       "      <td>0.4330</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100013</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>202500.0</td>\n",
       "      <td>663264.0</td>\n",
       "      <td>69777.0</td>\n",
       "      <td>630000.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Working</td>\n",
       "      <td>Higher education</td>\n",
       "      <td>Married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.0191</td>\n",
       "      <td>-20038</td>\n",
       "      <td>-4458</td>\n",
       "      <td>-2175.0</td>\n",
       "      <td>-3503</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Drivers</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>MONDAY</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Transport: type 3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.6998</td>\n",
       "      <td>0.6110</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-856.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100028</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>2</td>\n",
       "      <td>315000.0</td>\n",
       "      <td>1575000.0</td>\n",
       "      <td>49018.5</td>\n",
       "      <td>1575000.0</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Working</td>\n",
       "      <td>Secondary / secondary special</td>\n",
       "      <td>Married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.0264</td>\n",
       "      <td>-13976</td>\n",
       "      <td>-1866</td>\n",
       "      <td>-2000.0</td>\n",
       "      <td>-4208</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Sales staff</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>WEDNESDAY</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Business Entity Type 3</td>\n",
       "      <td>0.5257</td>\n",
       "      <td>0.5097</td>\n",
       "      <td>0.6127</td>\n",
       "      <td>0.3052</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.9970</td>\n",
       "      <td>0.9592</td>\n",
       "      <td>0.1165</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.2759</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.0417</td>\n",
       "      <td>0.2042</td>\n",
       "      <td>0.2404</td>\n",
       "      <td>0.3673</td>\n",
       "      <td>0.0386</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.3109</td>\n",
       "      <td>0.2049</td>\n",
       "      <td>0.9970</td>\n",
       "      <td>0.9608</td>\n",
       "      <td>0.1176</td>\n",
       "      <td>0.3222</td>\n",
       "      <td>0.2759</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.0417</td>\n",
       "      <td>0.2089</td>\n",
       "      <td>0.2626</td>\n",
       "      <td>0.3827</td>\n",
       "      <td>0.0389</td>\n",
       "      <td>0.0847</td>\n",
       "      <td>0.3081</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.9970</td>\n",
       "      <td>0.9597</td>\n",
       "      <td>0.1173</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.2759</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.0417</td>\n",
       "      <td>0.2078</td>\n",
       "      <td>0.2446</td>\n",
       "      <td>0.3739</td>\n",
       "      <td>0.0388</td>\n",
       "      <td>0.0817</td>\n",
       "      <td>reg oper account</td>\n",
       "      <td>block of flats</td>\n",
       "      <td>0.3700</td>\n",
       "      <td>Panel</td>\n",
       "      <td>No</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1805.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100038</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>Y</td>\n",
       "      <td>N</td>\n",
       "      <td>1</td>\n",
       "      <td>180000.0</td>\n",
       "      <td>625500.0</td>\n",
       "      <td>32067.0</td>\n",
       "      <td>625500.0</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Working</td>\n",
       "      <td>Secondary / secondary special</td>\n",
       "      <td>Married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.0100</td>\n",
       "      <td>-13040</td>\n",
       "      <td>-2191</td>\n",
       "      <td>-4000.0</td>\n",
       "      <td>-4262</td>\n",
       "      <td>16.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>FRIDAY</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Business Entity Type 3</td>\n",
       "      <td>0.2021</td>\n",
       "      <td>0.4257</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-821.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR NAME_CONTRACT_TYPE CODE_GENDER FLAG_OWN_CAR FLAG_OWN_REALTY  CNT_CHILDREN  AMT_INCOME_TOTAL  AMT_CREDIT  AMT_ANNUITY  AMT_GOODS_PRICE NAME_TYPE_SUITE NAME_INCOME_TYPE            NAME_EDUCATION_TYPE NAME_FAMILY_STATUS  NAME_HOUSING_TYPE  REGION_POPULATION_RELATIVE  DAYS_BIRTH  DAYS_EMPLOYED  DAYS_REGISTRATION  DAYS_ID_PUBLISH  OWN_CAR_AGE  FLAG_MOBIL  FLAG_EMP_PHONE  FLAG_WORK_PHONE  FLAG_CONT_MOBILE  FLAG_PHONE  FLAG_EMAIL     OCCUPATION_TYPE  CNT_FAM_MEMBERS  REGION_RATING_CLIENT  REGION_RATING_CLIENT_W_CITY WEEKDAY_APPR_PROCESS_START  HOUR_APPR_PROCESS_START  REG_REGION_NOT_LIVE_REGION  REG_REGION_NOT_WORK_REGION  LIVE_REGION_NOT_WORK_REGION  REG_CITY_NOT_LIVE_CITY  REG_CITY_NOT_WORK_CITY  LIVE_CITY_NOT_WORK_CITY       ORGANIZATION_TYPE  EXT_SOURCE_1  EXT_SOURCE_2  EXT_SOURCE_3  APARTMENTS_AVG  BASEMENTAREA_AVG  YEARS_BEGINEXPLUATATION_AVG  YEARS_BUILD_AVG  COMMONAREA_AVG  ELEVATORS_AVG  ENTRANCES_AVG  FLOORSMAX_AVG  FLOORSMIN_AVG  LANDAREA_AVG  LIVINGAPARTMENTS_AVG  LIVINGAREA_AVG  NONLIVINGAPARTMENTS_AVG  NONLIVINGAREA_AVG  APARTMENTS_MODE  BASEMENTAREA_MODE  YEARS_BEGINEXPLUATATION_MODE  YEARS_BUILD_MODE  COMMONAREA_MODE  ELEVATORS_MODE  ENTRANCES_MODE  FLOORSMAX_MODE  FLOORSMIN_MODE  LANDAREA_MODE  LIVINGAPARTMENTS_MODE  LIVINGAREA_MODE  NONLIVINGAPARTMENTS_MODE  NONLIVINGAREA_MODE  APARTMENTS_MEDI  BASEMENTAREA_MEDI  YEARS_BEGINEXPLUATATION_MEDI  YEARS_BUILD_MEDI  COMMONAREA_MEDI  ELEVATORS_MEDI  ENTRANCES_MEDI  FLOORSMAX_MEDI  FLOORSMIN_MEDI  LANDAREA_MEDI  LIVINGAPARTMENTS_MEDI  LIVINGAREA_MEDI  NONLIVINGAPARTMENTS_MEDI  NONLIVINGAREA_MEDI FONDKAPREMONT_MODE  HOUSETYPE_MODE  TOTALAREA_MODE WALLSMATERIAL_MODE EMERGENCYSTATE_MODE  OBS_30_CNT_SOCIAL_CIRCLE  DEF_30_CNT_SOCIAL_CIRCLE  OBS_60_CNT_SOCIAL_CIRCLE  DEF_60_CNT_SOCIAL_CIRCLE  DAYS_LAST_PHONE_CHANGE  FLAG_DOCUMENT_2  FLAG_DOCUMENT_3  FLAG_DOCUMENT_4  FLAG_DOCUMENT_5  FLAG_DOCUMENT_6  FLAG_DOCUMENT_7  FLAG_DOCUMENT_8  FLAG_DOCUMENT_9  FLAG_DOCUMENT_10  FLAG_DOCUMENT_11  FLAG_DOCUMENT_12  \\\n",
       "0  100001      Cash loans         F           N            Y               0             135000.0          568800.0    20560.5      450000.0         Unaccompanied   Working          Higher education               Married            House / apartment  0.0188                     -19241      -2329          -5170.0            -812             NaN           1           1               0                1                 0           1           NaN                 2.0              2                     2                            TUESDAY                    18                       0                           0                           0                            0                       0                       0                        Kindergarten            0.7526        0.7897        0.1595        0.0660          0.0590            0.9732                      NaN              NaN             NaN             0.1379         0.125         NaN            NaN           NaN                    0.0505         NaN                      NaN                 0.0672           0.0612             0.9732                       NaN               NaN              NaN              0.1379          0.125          NaN             NaN            NaN                     0.0526          NaN                       NaN                  0.0666           0.0590             0.9732                       NaN               NaN              NaN              0.1379          0.125          NaN             NaN            NaN                     0.0514          NaN                       NaN                  NaN                block of flats  0.0392          Stone, brick       No                  0.0                       0.0                       0.0                       0.0                      -1740.0                  0                1                0                0                0                0                0                0                0                 0                 0                  \n",
       "1  100005      Cash loans         M           N            Y               0             99000.0           222768.0    17370.0      180000.0         Unaccompanied   Working          Secondary / secondary special  Married            House / apartment  0.0358                     -18064      -4469          -9118.0            -1623            NaN           1           1               0                1                 0           0           Low-skill Laborers  2.0              2                     2                            FRIDAY                     9                        0                           0                           0                            0                       0                       0                        Self-employed           0.5650        0.2917        0.4330       NaN             NaN               NaN                          NaN              NaN             NaN            NaN            NaN            NaN            NaN           NaN                   NaN             NaN                      NaN                NaN              NaN                NaN                           NaN               NaN              NaN             NaN             NaN             NaN             NaN            NaN                    NaN              NaN                       NaN                 NaN              NaN                NaN                           NaN               NaN              NaN             NaN             NaN             NaN             NaN            NaN                    NaN              NaN                       NaN                  NaN                NaN            NaN              NaN                NaN                 0.0                       0.0                       0.0                       0.0                       0.0                     0                1                0                0                0                0                0                0                0                 0                 0                  \n",
       "2  100013      Cash loans         M           Y            Y               0             202500.0          663264.0    69777.0      630000.0         NaN             Working          Higher education               Married            House / apartment  0.0191                     -20038      -4458          -2175.0            -3503             5.0          1           1               0                1                 0           0           Drivers             2.0              2                     2                            MONDAY                     14                       0                           0                           0                            0                       0                       0                        Transport: type 3      NaN            0.6998        0.6110       NaN             NaN               NaN                          NaN              NaN             NaN            NaN            NaN            NaN            NaN           NaN                   NaN             NaN                      NaN                NaN              NaN                NaN                           NaN               NaN              NaN             NaN             NaN             NaN             NaN            NaN                    NaN              NaN                       NaN                 NaN              NaN                NaN                           NaN               NaN              NaN             NaN             NaN             NaN             NaN            NaN                    NaN              NaN                       NaN                  NaN                NaN            NaN              NaN                NaN                 0.0                       0.0                       0.0                       0.0                      -856.0                   0                0                0                0                0                0                1                0                0                 0                 0                  \n",
       "3  100028      Cash loans         F           N            Y               2             315000.0          1575000.0   49018.5      1575000.0        Unaccompanied   Working          Secondary / secondary special  Married            House / apartment  0.0264                     -13976      -1866          -2000.0            -4208            NaN           1           1               0                1                 1           0           Sales staff         4.0              2                     2                            WEDNESDAY                  11                       0                           0                           0                            0                       0                       0                        Business Entity Type 3  0.5257        0.5097        0.6127        0.3052          0.1974            0.9970                       0.9592           0.1165          0.32           0.2759         0.375          0.0417         0.2042        0.2404                0.3673          0.0386                   0.08               0.3109           0.2049             0.9970                        0.9608            0.1176           0.3222          0.2759          0.375           0.0417          0.2089         0.2626                 0.3827           0.0389                    0.0847              0.3081           0.1974             0.9970                        0.9597            0.1173           0.32            0.2759          0.375           0.0417          0.2078         0.2446                 0.3739           0.0388                    0.0817              reg oper account   block of flats  0.3700          Panel              No                  0.0                       0.0                       0.0                       0.0                      -1805.0                  0                1                0                0                0                0                0                0                0                 0                 0                  \n",
       "4  100038      Cash loans         M           Y            N               1             180000.0          625500.0    32067.0      625500.0         Unaccompanied   Working          Secondary / secondary special  Married            House / apartment  0.0100                     -13040      -2191          -4000.0            -4262             16.0         1           1               1                1                 0           0           NaN                 3.0              2                     2                            FRIDAY                     5                        0                           0                           0                            0                       1                       1                        Business Entity Type 3  0.2021        0.4257       NaN           NaN             NaN               NaN                          NaN              NaN             NaN            NaN            NaN            NaN            NaN           NaN                   NaN             NaN                      NaN                NaN              NaN                NaN                           NaN               NaN              NaN             NaN             NaN             NaN             NaN            NaN                    NaN              NaN                       NaN                 NaN              NaN                NaN                           NaN               NaN              NaN             NaN             NaN             NaN             NaN            NaN                    NaN              NaN                       NaN                  NaN                NaN            NaN              NaN                NaN                 0.0                       0.0                       0.0                       0.0                      -821.0                   0                1                0                0                0                0                0                0                0                 0                 0                  \n",
       "\n",
       "   FLAG_DOCUMENT_13  FLAG_DOCUMENT_14  FLAG_DOCUMENT_15  FLAG_DOCUMENT_16  FLAG_DOCUMENT_17  FLAG_DOCUMENT_18  FLAG_DOCUMENT_19  FLAG_DOCUMENT_20  FLAG_DOCUMENT_21  AMT_REQ_CREDIT_BUREAU_HOUR  AMT_REQ_CREDIT_BUREAU_DAY  AMT_REQ_CREDIT_BUREAU_WEEK  AMT_REQ_CREDIT_BUREAU_MON  AMT_REQ_CREDIT_BUREAU_QRT  AMT_REQ_CREDIT_BUREAU_YEAR  \n",
       "0  0                 0                 0                 0                 0                 0                 0                 0                 0                 0.0                         0.0                        0.0                         0.0                        0.0                        0.0                         \n",
       "1  0                 0                 0                 0                 0                 0                 0                 0                 0                 0.0                         0.0                        0.0                         0.0                        0.0                        3.0                         \n",
       "2  0                 0                 0                 0                 0                 0                 0                 0                 0                 0.0                         0.0                        0.0                         0.0                        1.0                        4.0                         \n",
       "3  0                 0                 0                 0                 0                 0                 0                 0                 0                 0.0                         0.0                        0.0                         0.0                        0.0                        3.0                         \n",
       "4  0                 0                 0                 0                 0                 0                 0                 0                 0                NaN                         NaN                        NaN                         NaN                        NaN                        NaN                          "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# train\n",
    "train_path = \"home-credit-default-risk/application_train.csv\"\n",
    "pdf_train = pd.read_csv(train_path)\n",
    "print(\"(rows, columns)\", pdf_train.shape)\n",
    "print(\"First 5 rows\")\n",
    "display(pdf_train.head(5))\n",
    "\n",
    "# test\n",
    "test_path = \"home-credit-default-risk/application_test.csv\"\n",
    "pdf_test = pd.read_csv(test_path)\n",
    "print(\"(rows, columns)\", pdf_test.shape)\n",
    "print(\"First 5 rows\")\n",
    "display(pdf_test.head(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of overlap ids: 0\n"
     ]
    }
   ],
   "source": [
    "# check overlap\n",
    "train_ids = pdf_train[\"SK_ID_CURR\"]\n",
    "test_ids = pdf_test[\"SK_ID_CURR\"]\n",
    "n_overlap = len(set(train_ids) & set(test_ids))\n",
    "print(\"Number of overlap ids: {}\".format(n_overlap))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data no duplicate\n",
      "Test data no duplicate\n"
     ]
    }
   ],
   "source": [
    "# check duplicate\n",
    "if len(train_ids.drop_duplicates()) == pdf_train.shape[0]:\n",
    "    print(\"Train data no duplicate\")\n",
    "if len(test_ids.drop_duplicates()) == pdf_test.shape[0]:\n",
    "    print(\"Test data no duplicate\")\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train validation test (tvt) split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>tvt_code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100002</td>\n",
       "      <td>1</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100003</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100004</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100006</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100007</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR  TARGET tvt_code\n",
       "0  100002      1       train  \n",
       "1  100003      0       train  \n",
       "2  100004      0       train  \n",
       "3  100006      0       train  \n",
       "4  100007      0       train  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get ids and target label\n",
    "pdf_tvt = pdf_train[[\"SK_ID_CURR\", \"TARGET\"]].copy()\n",
    "pdf_tvt[\"tvt_code\"] = \"kaggle_test\"\n",
    "\n",
    "# target\n",
    "y = pdf_tvt[\"TARGET\"]\n",
    "\n",
    "# index\n",
    "X = pdf_tvt[\"SK_ID_CURR\"]\n",
    "\n",
    "# train, val, test set will be 70%, 15%, 15% of the dataset respectively.\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15, random_state=1)\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.17, random_state=1)\n",
    "\n",
    "# check validity\n",
    "assert X.shape[0] == X_train.shape[0] + X_val.shape[0] + X_test.shape[0], \"TVT split not correct\"\n",
    "\n",
    "# assign code\n",
    "pdf_tvt.loc[X_train.index, \"tvt_code\"] = \"train\"\n",
    "pdf_tvt.loc[X_val.index, \"tvt_code\"] = \"val\"\n",
    "pdf_tvt.loc[X_test.index, \"tvt_code\"] = \"test\"\n",
    "\n",
    "pdf_tvt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>tvt_code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100002</td>\n",
       "      <td>1</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100003</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100004</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100006</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100007</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR  TARGET tvt_code\n",
       "0  100002      1       train  \n",
       "1  100003      0       train  \n",
       "2  100004      0       train  \n",
       "3  100006      0       train  \n",
       "4  100007      0       train  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extend for kaggle test\n",
    "pdf_test[\"TARGET\"] = -1\n",
    "pdf_test[\"tvt_code\"] = \"kaggle_test\"\n",
    "\n",
    "pdf_tvt_extend = pd.concat([pdf_tvt, pdf_test[[\"SK_ID_CURR\", \"TARGET\", \"tvt_code\"]]])\n",
    "pdf_tvt_extend = pdf_tvt_extend.reset_index(drop=True)\n",
    "pdf_tvt_extend.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check validity\n",
    "assert pdf_tvt_extend.shape[0] == pdf_tvt_extend[\"SK_ID_CURR\"].nunique(), \"Extend TVT overlaped\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cnt</th>\n",
       "      <th>percentage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>216948</td>\n",
       "      <td>70.5497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kaggle_test</th>\n",
       "      <td>48744</td>\n",
       "      <td>15.8511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test</th>\n",
       "      <td>46127</td>\n",
       "      <td>15.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>44436</td>\n",
       "      <td>14.4502</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                cnt  percentage\n",
       "train        216948  70.5497   \n",
       "kaggle_test  48744   15.8511   \n",
       "test         46127   15.0001   \n",
       "val          44436   14.4502   "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check tvt percentage\n",
    "pdf_tvt_check = pdf_tvt_extend[\"tvt_code\"].value_counts().to_frame(\"cnt\")\n",
    "pdf_tvt_check[\"percentage\"] = pdf_tvt_check[\"cnt\"] * 100.0 / pdf_tvt.shape[0]\n",
    "pdf_tvt_check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save tvt split\n",
    "pdf_tvt_extend.to_pickle(\"pdf_tvt_extend.pkl\", compression=\"bz2\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}