3. Missing data impute.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1295896 entries, 0 to 1295895\n",
      "Data columns (total 72 columns):\n",
      "loan_amnt                     1295864 non-null float64\n",
      "funded_amnt                   1295864 non-null float64\n",
      "funded_amnt_inv               1295864 non-null float64\n",
      "term                          1295864 non-null object\n",
      "int_rate                      1295864 non-null object\n",
      "installment                   1295864 non-null float64\n",
      "grade                         1295864 non-null object\n",
      "sub_grade                     1295864 non-null object\n",
      "emp_title                     1208774 non-null object\n",
      "emp_length                    1212582 non-null object\n",
      "home_ownership                1295864 non-null object\n",
      "annual_inc                    1295864 non-null float64\n",
      "verification_status           1295864 non-null object\n",
      "purpose                       1295864 non-null object\n",
      "title                         1276095 non-null object\n",
      "zip_code                      1295863 non-null object\n",
      "addr_state                    1295864 non-null object\n",
      "dti                           1295256 non-null float64\n",
      "delinq_2yrs                   1295864 non-null float64\n",
      "earliest_cr_line              1295864 non-null object\n",
      "inq_last_6mths                1295863 non-null float64\n",
      "open_acc                      1295864 non-null float64\n",
      "pub_rec                       1295864 non-null float64\n",
      "revol_bal                     1295864 non-null float64\n",
      "revol_util                    1295014 non-null object\n",
      "total_acc                     1295864 non-null float64\n",
      "initial_list_status           1295864 non-null object\n",
      "collections_12_mths_ex_med    1295864 non-null float64\n",
      "application_type              1295864 non-null object\n",
      "acc_now_delinq                1295864 non-null float64\n",
      "tot_coll_amt                  1295864 non-null float64\n",
      "tot_cur_bal                   1295864 non-null float64\n",
      "total_rev_hi_lim              1295864 non-null float64\n",
      "acc_open_past_24mths          1295864 non-null float64\n",
      "avg_cur_bal                   1295840 non-null float64\n",
      "bc_open_to_buy                1281191 non-null float64\n",
      "bc_util                       1280415 non-null float64\n",
      "chargeoff_within_12_mths      1295864 non-null float64\n",
      "delinq_amnt                   1295864 non-null float64\n",
      "mo_sin_old_il_acct            1258212 non-null float64\n",
      "mo_sin_old_rev_tl_op          1295864 non-null float64\n",
      "mo_sin_rcnt_rev_tl_op         1295864 non-null float64\n",
      "mo_sin_rcnt_tl                1295864 non-null float64\n",
      "mort_acc                      1295864 non-null float64\n",
      "mths_since_recent_bc          1282046 non-null float64\n",
      "mths_since_recent_inq         1170717 non-null float64\n",
      "num_accts_ever_120_pd         1295864 non-null float64\n",
      "num_actv_bc_tl                1295864 non-null float64\n",
      "num_actv_rev_tl               1295864 non-null float64\n",
      "num_bc_sats                   1295864 non-null float64\n",
      "num_bc_tl                     1295864 non-null float64\n",
      "num_il_tl                     1295864 non-null float64\n",
      "num_op_rev_tl                 1295864 non-null float64\n",
      "num_rev_accts                 1295863 non-null float64\n",
      "num_rev_tl_bal_gt_0           1295864 non-null float64\n",
      "num_sats                      1295864 non-null float64\n",
      "num_tl_120dpd_2m              1238650 non-null float64\n",
      "num_tl_30dpd                  1295864 non-null float64\n",
      "num_tl_90g_dpd_24m            1295864 non-null float64\n",
      "num_tl_op_past_12m            1295864 non-null float64\n",
      "pct_tl_nvr_dlq                1295863 non-null float64\n",
      "percent_bc_gt_75              1280772 non-null float64\n",
      "pub_rec_bankruptcies          1295864 non-null float64\n",
      "tax_liens                     1295864 non-null float64\n",
      "tot_hi_cred_lim               1295864 non-null float64\n",
      "total_bal_ex_mort             1295864 non-null float64\n",
      "total_bc_limit                1295864 non-null float64\n",
      "total_il_high_credit_limit    1295864 non-null float64\n",
      "hardship_flag                 1295864 non-null object\n",
      "debt_settlement_flag          1295864 non-null object\n",
      "year                          1295896 non-null int64\n",
      "target                        1295896 non-null int64\n",
      "dtypes: float64(52), int64(2), object(18)\n",
      "memory usage: 711.9+ MB\n"
     ]
    }
   ],
   "source": [
    "# import the preliminary cleaned data in last section\n",
    "data = pd.read_csv('./data.csv')\n",
    "data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split into train and test datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2015    383794\n",
      "2016    347678\n",
      "2014    228978\n",
      "2017    215424\n",
      "2018    108659\n",
      "2019     11363\n",
      "Name: year, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "count_year=data['year'].value_counts()\n",
    "print(count_year)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1gAAAGoCAYAAABbkkSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xu0nVV97//3l4QY5U4IcgmYKLESQQOES0sPUqDczhHQwjnBHyUW2lQLv6JW02h/AxXqOQpWeqiAcgoKFogUSsnPwUVEwBu3BMMliZRw34FAuFORS8L3/LFmYGWz9s4OmWuvvdd+v8ZYY601n+eZz3eODUw+65nrWZGZSJIkSZLW3XqdLkCSJEmSuoUBS5IkSZIqMWBJkiRJUiUGLEmSJEmqxIAlSZIkSZUYsCRJkiSpEgOWJEmSJFViwJIkSZKkSgxYkiRJklTJ6E4XMFRsscUWOXHixE6XIUlqYf78+U9l5vhO19FuzkWSNHQNdC4yYBUTJ05k3rx5nS5DktRCRDzc6RoGg3ORJA1dA52LXCIoSZIkSZUYsCRJkiSpEgOWJEmSJFXid7AkaYR47bXX6Onp4eWXX+50KX0aO3YsEyZMYP311+90KZIkvS0GLEkaIXp6ethoo42YOHEiEdHpct4iM3n66afp6elh0qRJnS5HkqS3xSWCkjRCvPzyy4wbN25IhiuAiGDcuHFD+gqbJElrYsCSpBFkqIarVYZ6fZIkrYkBS5IkSZIqMWBJ0gj33HPPcfbZZ7f9PDfeeCO/+tWv2n4eSZI6yYAlSSPc2gaszOT1119f6/MYsCRJI4EBS5JGuNmzZ3P//fczdepUPvvZz7L//vuz6667svPOO3PllVcC8NBDD7HjjjvyV3/1V+y66648+uijnHfeebz//e9n33335S/+4i848cQTAVi+fDl/8id/wu67787uu+/OL3/5Sx566CG+853vcMYZZzB16lR+/vOfd3LIkiS1jbdpl6QR7utf/zr33HMPCxYsYMWKFbz00ktsvPHGPPXUU+y1114cdthhANx7771873vf4+yzz+axxx7j1FNP5Y477mCjjTZiv/3248Mf/jAAJ510Ep/97Gf5wz/8Qx555BEOOuggFi9ezKc+9Sk23HBDPv/5z3dyuJIktZUBS5L0hszkS1/6Ej/72c9Yb731WLp0KU888QQA73nPe9hrr70AuO222/jIRz7C5ptvDsBRRx3Ff/zHfwDwk5/8hEWLFr3R5wsvvMCLL744yCORJKkzDFiSpDdcdNFFLF++nPnz57P++uszceLEN36XaoMNNnhjv8zss4/XX3+dm2++mXe+851tr1eSpKHG72BJ0gi30UYbvXGF6fnnn2fLLbdk/fXX54YbbuDhhx9uecwee+zBTTfdxLPPPsuKFSu4/PLL39h24IEH8u1vf/uN9wsWLHjLeYaLiDg4Iu6NiCURMbvF9k9FxN0RsSAifhERU5q2fbEcd29EHDS4lUuSOsWAJUkj3Lhx49h7773ZaaedWLBgAfPmzWPatGlcdNFFfOADH2h5zLbbbsuXvvQl9txzTw444ACmTJnCJptsAsCZZ57JvHnz+NCHPsSUKVP4zne+A8BHP/pRrrjiimFzk4uIGAWcBRwCTAGObg5QxcWZuXNmTgVOA75Vjp0CTAc+CBwMnF36k6SqZs2axbHHHsusWbM6XYoKlwhqNbt94cK29j//9GPb2r+kt+fiiy9e4z733HPPau8/8YlPMHPmTFasWMHHPvYxDjzwQAC22GILfvjDH77l+Pe///3cdddddQoeHHsASzLzAYCImAMcDrzxBbPMfKFp/w2AVWsnDwfmZOYrwIMRsaT0d/NgFC5p5Fi2bBlLly7tdBlq4hUsSdLb8pWvfIWpU6ey0047MWnSJI444ohOl1TbtsCjTe97SttqIuKEiLifxhWsv16bY8vxMyNiXkTMW758eZXCJUmd07aAFRFjI+K2iLgzIhZGxFdL+6SIuDUi7ouIH0bEmNL+jvJ+Sdk+samvluvY+1ob39c5JEn1fPOb32TBggX85je/4cwzzyQiOl1Sba0G9Ja7e2TmWZn5PuBvgf9vbY4tx5+bmdMyc9r48ePfdrGSpKGhnUsEXwH2y8z/jIj1gV9ExNXA54AzMnNORHwHOB44pzw/m5k7RMR04BvA/+i1jn0b4CcR8f5yjrOAP6bxyeDtETE3MxeVY1udQ5KkgeoBtmt6PwF4rJ/95/DmXLO2x0rqYt/+m/+/bX0/99Rv33hu13lO/IePtqXfbtW2K1jZ8J/l7frlkcB+wGWl/QJg1ZqSw8t7yvb9o/Fx6Bvr2DPzQWDVOvY31sZn5qs0JrbDyzF9nUOSpIG6HZhcVkWMofFh39zmHSJictPb/wrcV17PBaaX1RmTgMnAbYNQsySpw9p6k4tyx6T5wA40rjbdDzyXmSvKLs1r0t9Yr56ZKyLieWBcab+lqdvmY3qvb9+zHNPXOXrXNxOYCbD99tu/vUFKkrpSmYtOBK4FRgHnZ+bCiDgFmJeZc4ETI+IA4DXgWWBGOXZhRFxK44YYK4ATMnNlRwYiSRpUbQ1YZTKZGhGbAlcAO7barTz3tV69r/ZWV9/6279VfecC5wJMmzat71/NlCSNSJl5FXBVr7aTm16f1M+xXwO+1r7qJElD0aDcpj0zn4uIG4G9gE0jYnS5wtS8Jn3VevWeiBgNbAI8Q//r2Fu1P9XPOSRJRe2fZRjIzzBcc801nHTSSaxcuZI///M/Z/bst/x2ryRpLWwwZuPVntV57byL4Phy5YqIeCdwALAYuAE4suw2A7iyvJ5b3lO2/zQzk77XsbdcG1+O6esckqQOWblyJSeccAJXX301ixYt4pJLLmHRokVrPlCS1Ke93/dxDtzxk+z9vo93uhQV7fwdrK2BGyLiLhph6LrM/BGN29h+rvzo4jjgvLL/ecC40v45YDY01rEDq9axX0NZx16uTq1aG78YuLTsSz/nkCR1yG233cYOO+zAe9/7XsaMGcP06dO58ko//5IkdZe2LRHMzLuAXVq0P0DjDoC9218Gjuqjr5br2Futje/vHJKkzlm6dCnbbffmyu4JEyZw6623drAiSZLqa+cVLEmS3tBYwb26LvxxYknSCGfAkiQNigkTJvDoo2/+ukZPTw/bbLNNByuSJKk+A5YkaVDsvvvu3HfffTz44IO8+uqrzJkzh8MOO6zTZUmSVNWg3KZdkjT0DOS26jWNHj2ab3/72xx00EGsXLmS4447jg9+8IODWoMkSe1mwJIkDZpDDz2UQw89tNNlSJLUNi4RlCRJkqRKDFiSJEmSVIkBS5IkSZIqMWBJkiRJUiUGLEmSJEmqxIAlSZIkSZV4m3ZJGqEeOWXnqv1tf/Lda9znuOOO40c/+hFbbrkl99xzT9XzS5I0FHgFS5I0aD75yU9yzTXXdLoMSZLaxoAlSRo0++yzD5tvvnmny5AkqW0MWJIkSZJUiQFLkiRJkirxJheSJEnqGrNmzWLZsmVstdVWnHbaaZ0uRyOQAUuSJEldY9myZSxdurTTZWgEM2BJ0gg1kNuq13b00Udz44038tRTTzFhwgS++tWvcvzxxw96HZIktYsBS5I0aC655JJOlyCpw27a5yNt7f93o0dBBL/r6WnbuT7ys5va0q/WbDgsATVgSZIkSRoWhsMSUO8iKEmSJEmVeAVLkkaQzCQiOl1GnzKz0yVIGuY2Lf8d2dT/nqhDDFiSNEKMHTuWp59+mnHjxg3JkJWZPP3004wdO7bTpUgaxo5Z+XqnS9AIZ8CSpBFiwoQJ9PT0sHz58k6X0qexY8cyYcKETpchSdLbZsCSpBFi/fXXZ9KkSZ0uQ5KkruZNLiRJkiSpEgOWJEmSJFViwJIkSZKkSgxYkiRJklSJN7nQiLbbFy5sa//zTz+2rf1LkiQNNV875si29f3Mk883npc93rbz/N2/XLZOx3sFS5IkSZIqMWBJkiRJUiUGLEmSJEmqxIAlSZIkSZUYsCRJkiSpEgOWJEmSJFViwJIkSZKkSgxYkiRJklSJAUuSJEmSKjFgSZIkSVIlBixJkiRJqmR0pwuQJEmSpIEYO2q91Z6HIgOWJEmSpGFhl3EbdbqENRq60U+SJEmShhkDliRJkiRVYsCSJKkPEXFwRNwbEUsiYnaL7Z+LiEURcVdEXB8R72natjIiFpTH3MGtXJLUKX4HS5KkFiJiFHAW8MdAD3B7RMzNzEVNu/0amJaZL0XEp4HTgP9Rtv0uM6cOatGSpI7zCpYkSa3tASzJzAcy81VgDnB48w6ZeUNmvlTe3gJMGOQaJUlDjAFLkqTWtgUebXrfU9r6cjxwddP7sRExLyJuiYgj+jooImaW/eYtX7583SqWJHVc2wJWRGwXETdExOKIWBgRJ5X2r0TE0qZ16Yc2HfPFss793og4qKm95Rr4iJgUEbdGxH0R8cOIGFPa31HeLynbJ7ZrnJKkrhUt2rLljhHHANOA05uat8/MacAngH+MiPe1OjYzz83MaZk5bfz48etasySpw9p5BWsF8DeZuSOwF3BCREwp287IzKnlcRVA2TYd+CBwMHB2RIxqWgN/CDAFOLqpn2+UviYDz9L49JDy/Gxm7gCcUfaTJGlt9ADbNb2fADzWe6eIOAD4O+CwzHxlVXtmPlaeHwBuBHZpZ7GSpKGhbQErMx/PzDvK6xeBxfS/tOJwYE5mvpKZDwJLaKx/b7kGPiIC2A+4rBx/AXBEU18XlNeXAfuX/SVJGqjbgclltcQYGh8CrnY3wIjYBfgujXD1ZFP7ZhHxjvJ6C2BvoPnmGJKkLjUo38EqS/R2AW4tTSeWW9qeHxGblba+1rr31T4OeC4zV/RqX62vsv35sn/vulz3LklqqcwfJwLX0viQ8NLMXBgRp0TEYWW304ENgX/tdTv2HYF5EXEncAPw9V53H5Qkdam236Y9IjYELgc+k5kvRMQ5wKk01rGfCvwDcBx9r3VvFQKzn/1Zw7Y3GzLPBc4FmDZtWst19ZKkkassY7+qV9vJTa8P6OO4XwE7t7c6SdJQ1NYrWBGxPo1wdVFm/htAZj6RmSsz83Xg/9BYAgh9r3Xvq/0pYNOIGN2rfbW+yvZNgGfqjk6SJEmSVtfOuwgGcB6wODO/1dS+ddNuHwPuKa/nAtPLHQAnAZOB2+hjDXxmJo1lF0eW42cAVzb1NaO8PhL4adlfkiRJktqmnUsE9wb+FLg7IhaUti/RuAvgVBpL9h4C/hKgrGu/lMaXgFcAJ2TmSoCIWLUGfhRwfmYuLP39LTAnIv4e+DWNQEd5/kFELKFx5Wp6G8cpSZIkSUAbA1Zm/oLW34W6qkXbqmO+BnytRftb1sCX9gd4c4lhc/vLwFFrU68kSZIkratBuYugJEmSJI0EBixJkiRJqsSAJUmSJEmVGLAkSZIkqRIDliRJkiRVYsCSJEmSpEoMWJIkSZJUiQFLkiRJkioxYEmSJElSJQYsSZIkSarEgCVJkiRJlRiwJEmSJKkSA5YkSZIkVWLAkiRJkqRKDFiSJEmSVIkBS5IkSZIqMWBJkiRJUiUGLEmSJEmqxIAlSZIkSZWM7nQBkiRJao9Zs2axbNkyttpqK0477bROlyONCAYsSZKkLrVs2TKWLl3a6TKkEcWAJUmS1EF7/9Pebet7zHNjWI/1ePS5R9t2nl/+v79sS7/ScOV3sCRJkiSpEgOWJEmSJFXiEkFJkqQule9KXud18l3Z6VKkEcOAJUmS1KVe2/u1TpcgjTguEZQkSZKkSgxYkiRJklSJAUuSJEmSKjFgSZIkSVIlBixJkiRJqsSAJUmSJEmVGLAkSZIkqRIDliRJkiRVYsCSJEmSpEoMWJIkSZJUiQFLkiRJkioxYEmSJElSJQYsSZIkSarEgCVJkiRJlYzudAGSJEmDZdasWSxbtoytttqK0047rdPlSOpCBixJkjRiLFu2jKVLl3a6DEldzIAlSZKGlEdO2bltfa94ZnNgNCueebht59n+5Lvb0q+k4cHvYEmSJElSJQYsSZIkSarEJYKSJGnE2GLs68CK8ixJ9RmwJEnSiPH5Dz3X6RIkdTmXCEqS1IeIODgi7o2IJRExu8X2z0XEooi4KyKuj4j3NG2bERH3lceMwa1cktQpBixJklqIiFHAWcAhwBTg6IiY0mu3XwPTMvNDwGXAaeXYzYEvA3sCewBfjojNBqt2SVLnGLAkSWptD2BJZj6Qma8Cc4DDm3fIzBsy86Xy9hZgQnl9EHBdZj6Tmc8C1wEHD1LdkqQOalvAiojtIuKGiFgcEQsj4qTSvnlEXFeWTFy36hO9aDizLMO4KyJ2beqr5TKLiNgtIu4ux5wZEdHfOSRJWgvbAo82ve8pbX05Hrh6bY+NiJkRMS8i5i1fvnwdypUkDQXtvIK1AvibzNwR2As4oSytmA1cn5mTgevLe2gswZhcHjOBc2CNyyzOKfuuOm7Vp4N9nUOSpIGKFm3ZcseIY4BpwOlre2xmnpuZ0zJz2vjx499WoZKkoaNtASszH8/MO8rrF4HFND69Oxy4oOx2AXBEeX04cGE23AJsGhFb08cyi7Jt48y8OTMTuLBXX63OIUnSQPUA2zW9nwA81nuniDgA+DvgsMx8ZW2OlSR1n0H5DlZETAR2AW4F3p2Zj0MjhAFblt36Wk7RX3tPi3b6OUfvulyWIUnqy+3A5IiYFBFjgOnA3OYdImIX4Ls0wtWTTZuuBQ6MiM3KqosDS5skqcu1PWBFxIbA5cBnMvOF/nZt0ZZvo33AXJYhSepLZq4ATqQRjBYDl2bmwog4JSIOK7udDmwI/GtELIiIueXYZ4BTaYS024FTSpskqcu19YeGI2J9GuHqosz8t9L8RERsnZmPl2V+qz7x62s5RQ+wb6/2G0v7hBb793cOSZIGLDOvAq7q1XZy0+sD+jn2fOD89lUnSRqK2nkXwQDOAxZn5reaNs0FVt0JcAZwZVP7seVugnsBz5flfS2XWZRtL0bEXuVcx/bqq9U5JEmSJKlt2nkFa2/gT4G7I2JBafsS8HXg0og4HngEOKpsuwo4FFgCvAT8GTSWWUTEqmUWsPoyi08D3wfeSePWuKtuj9vXOSRJkiSpbdoWsDLzF7T+nhTA/i32T+CEPvpqucwiM+cBO7Vof7rVOSRJkiSpnQblLoKSJEmSNBIYsCRJkiSpEgOWJEmSJFViwJIkSZKkSgxYkiRJklSJAUuSJEmSKjFgSZIkSVIlBixJkiRJqqRtPzQsSZKGvlmzZrFs2TK22morTjvttE6XI0nDngFrDXb7woVt7X/+6ce2tX9JkvqzbNkyli5d2ukyJKlruERQkiRJkirxCpYkSUNcO1dTbPTUi4wCHnnqxbadx9UakkYSr2BJkiRJUiUGLEmSJEmqxCWCkiSNYK+P2WC1Z0nSujFgSZI0gv128oGdLkGSuopLBCVJkiSpEgOWJEmSJFViwJIkSZKkSgxYkiRJklSJAUuSJEmSKjFgSZIkSVIlBixJkiRJqsSAJUmSJEmVGLAkSZIkqRIDliRJkiRVYsCSJEmSpEoMWJIkSZJUiQFLktT1ImLvgbRJkrSuDFiSpJHgnwbYJknSOhk9kJ0i4vrM3H9NbZIkDSUR8fvAHwDjI+JzTZs2BkZ1pipJUjfrN2BFxFjgXcAWEbEZEGXTxsA2ba5NkqR1NQbYkMZ8t1FT+wvAkR2pSJLU1dZ0Besvgc/QCFPzeTNgvQCc1ca6JElaZ5l5E3BTRHw/Mx+OiA0y87edrkuS1L36/Q5WZv7vzJwEfD4z35uZk8rjw5n57UGqUZKkdbVNRCwCFgNExIcj4uwO1yRJ6kID+g5WZv5TRPwBMLH5mMy8sE11SZJU0z8CBwFzATLzzojYp7MlSZK60UBvcvED4H3AAmBlaU7AgCVJGhYy89GIaG5a2de+kiS9XQMKWMA0YEpmZjuLkSSpTR4tKzEyIsYAf01ZLihJUk0D/R2se4Ct2lmIJElt9CngBGBboAeYWt5LklTVQK9gbQEsiojbgFdWNWbmYW2pSpKkijLzKeD/6XQdkqTuN9CA9ZV2FiFJUjtFxJktmp8H5mXmlYNdjySpew30LoI3tbsQSZLaaCzwAeBfy/s/ARYCx0fEH2XmZzpWmSSpqwz0LoIv0rhrIMAYYH3gt5m5cbsKkySpoh2A/TJzBUBEnAP8GPhj4O5OFiZJ6i4DvYK1UfP7iDgC2KMtFUmSVN+2wAY0lgVSXm+TmSsj4pW+D5Mkae0M9DtYq8nMf4+I2bWLkSSpTU4DFkTEjUAA+wD/MyI2AH7SycIkSd1loEsEP970dj0av4vlb2JJkoa8aPy68I+Bq2isvgjgS5n5WNnlC52qTZLUfQZ6BeujTa9XAA8Bh1evRlLb7faFC9va//zTj21r/9LaysyMiH/PzN0A7xgoSWqrgX4H68/aXYgkSW10S0Tsnpm3d7oQSVJ3W28gO0XEhIi4IiKejIgnIuLyiJjQ7uIkSarkj4CbI+L+iLgrIu6OiLvWdFBEHBwR90bEklbfPY6IfSLijohYERFH9tq2MiIWlMfcimORJA1hA10i+D3gYuCo8v6Y0vbH7ShKkqTKDlnbAyJiFHAWjbmuB7g9IuZm5qKm3R4BPgl8vkUXv8vMqW+jVknSMDagK1jA+Mz8XmauKI/vA+P7OyAizi9XvO5pavtKRCxt+kTv0KZtXyyfEN4bEQc1tbf89DAiJkXErRFxX0T8MCLGlPZ3lPdLyvaJAxyjJKlLZebDmfkw8DsaN2la9ejPHsCSzHwgM18F5tDr+8eZ+VBm3gW83oayJUnD0EAD1lMRcUxEjCqPY4Cn13DM94GDW7SfkZlTy+MqgIiYAkwHPliOOXvVuWh8engIMAU4uuwL8I3S12TgWeD40n488Gxm7gCcUfaTJI1gEXFYRNwHPAjcRONmTVev4bBtgUeb3veUtoEaGxHzIuKW8vuRfdU2s+w3b/ny5WvRvSRpKBpowDoO+O/AMuBx4Eig3xtfZObPgGcG2P/hwJzMfCUzHwSW0PjksOWnh+WWu/sBl5XjLwCOaOrrgvL6MmD/sr8kaeQ6FdgL+I/MnATsD/xyDce0mjvW5idKts/MacAngH+MiPe12ikzz83MaZk5bfz4fheHSJKGgYEGrFOBGZk5PjO3pBG4vvI2z3li+YLx+RGxWWnr61PCvtrHAc9l5ope7av1VbY/X/aXJI1cr2Xm08B6EbFeZt4ArOn7UT3Adk3vJwCP9bHvW6z6na3MfAC4EdhlrSqWJA1LAw1YH8rMZ1e9ycxneHsTxTnA+2hMao8D/1Da+/qUcG3b++vrLVyWIUkjxnMRsSHwM+CiiPjfwGtrOOZ2YHL5zu8YGkvZB3Q3wIjYLCLeUV5vAewNLOr/KElSNxhowFqv6WoTEbE5A78D4Rsy84nMXJmZrwP/h8YSQOj7U8K+2p8CNo2I0b3aV+urbN+EPpYquixDkkaMO4GXgM8C1wD3A7/p74CyCuJE4FpgMXBpZi6MiFMi4jCAiNg9Inpo3GX3uxGxsBy+IzAvIu4EbgC+3uvug5KkLjXQkPQPwK8i4jIaV4P+O/C1tT1ZRGydmY+Xtx8DVt1hcC5wcUR8C9gGmAzcRuNq1OSImAQspfHp4ScyMyPiBhrfBZsDzACubOprBnBz2f7TzFybNfOSpO7zR+XDvdcp39MdyO9glZsxXdWr7eSm17fT+JCv93G/AnZex5olScPQgAJWZl4YEfNo3FgigI+v6ZO4iLgE2BfYony692Vg34iYSiOkPQT8Zel/YURcSmP5xArghMxcWfpZ9enhKOD8zFz16eDfAnMi4u+BXwPnlfbzgB9ExBIaV66mD2SMkqTuExGfBv4KeF+vQLURa77JhSRJa23Ay/xKoBrw8obMPLpF83kt2lbt/zVaXBVr9elhaX+AN5cYNre/zJs/iCxJGtkupnE79v8FzG5qf7F8n1iSpKrW+ntUkiQNF5n5PI27ybb60E+SpOoGepMLSZIkSdIaGLAkSZIkqRIDliRJkiRVYsCSJEmSpEoMWJIkSZJUiQFLkiRJkioxYEmSJElSJQYsSZIkSarEgCVJkiRJlRiwJEmSJKkSA5YkSZIkVWLAkiRJkqRKDFiSJEmSVIkBS5IkSZIqMWBJkiRJUiUGLEmSJEmqxIAlSZIkSZUYsCRJkiSpEgOWJEmSJFViwJIkSZKkSgxYkiRJklSJAUuSJEmSKjFgSZIkSVIlBixJkiRJqsSAJUmSJEmVGLAkSZIkqRIDliRJkiRVYsCSJEmSpEoMWJIkSZJUiQFLkiRJkioxYEmSJElSJQYsSZIkSarEgCVJkiRJlRiwJEmSJKkSA5YkSZIkVWLAkiRJkqRKDFiSJEmSVIkBS5IkSZIqMWBJkiRJUiUGLEmSJEmqxIAlSZIkSZUYsCRJkiSpEgOWJEmSJFViwJIkSZKkSgxYkiRJklSJAUuSJEmSKjFgSZIkSVIlBixJkiRJqsSAJUlSHyLi4Ii4NyKWRMTsFtv3iYg7ImJFRBzZa9uMiLivPGYMXtWSpE5qW8CKiPMj4smIuKepbfOIuK5MNtdFxGalPSLizDKB3RURuzYd03KCiojdIuLucsyZERH9nUOSpLUREaOAs4BDgCnA0RExpddujwCfBC7udezmwJeBPYE9gC87H0nSyNDOK1jfBw7u1TYbuD4zJwPXl/fQmLwml8dM4BxY4wR1Ttl31XEHr+EckiStjT2AJZn5QGa+CswBDm/eITMfysy7gNd7HXsQcF1mPpOZzwLX8dY5UZLUhdoWsDLzZ8AzvZoPBy4ory8AjmhqvzAbbgE2jYit6WOCKts2zsybMzOBC3v11eockiStjW2BR5ve95S2qsdGxMyImBcR85YvX/62CpUkDR2D/R2sd2fm4wDlecvS3tdE1F97T4v2/s7xFk5qkqR+RIu2rH1sZp6bmdMyc9r48eMHXJwkaWgaKje56GsiWtv2teKkJknqRw+wXdP7CcBjg3CsJGkYG+yA9URZ3kd5frK09zUR9dc+oUV7f+eQJGlt3A5MjohJETEGmA7MHeCx1wIHRsRm5bvDB5Y2SVKXG+yANRdYdSfAGcCVTe0WcqVVAAAP7ElEQVTHlrsJ7gU8X5b3tZygyrYXI2KvcvfAY3v11eockiQNWGauAE6kMRctBi7NzIURcUpEHAYQEbtHRA9wFPDdiFhYjn0GOJVGSLsdOKW0SZK63Oh2dRwRlwD7AluUyefLwNeBSyPieBq3tj2q7H4VcCiwBHgJ+DNoTFARsWqCgtUnqE/TuFPhO4Gry4N+ziFJ0lrJzKtozFHNbSc3vb6d1VdUNO93PnB+WwuUJA05bQtYmXl0H5v2b7FvAif00U/LCSoz5wE7tWh/utU5JEmSJKndhspNLiRJkiRp2DNgSZIkSVIlBixJkiRJqsSAJUmSJEmVGLAkSZIkqRIDliRJkiRVYsCSJEmSpEoMWJIkSZJUiQFLkiRJkioxYEmSJElSJQYsSZIkSarEgCVJkiRJlRiwJEmSJKkSA5YkSZIkVWLAkiRJkqRKDFiSJEmSVIkBS5IkSZIqMWBJkiRJUiUGLEmSJEmqZHSnC9DI8sgpO7e1/+1Pvrut/UuSJEn9MWBJbdTuQAmGSkmSpKHEJYKSJEmSVIlXsCR1td2+cGFb+59/+rFt7V+SJA0vXsGSJEmSpEoMWJIkSZJUiQFLkiRJkioxYEmSJElSJQYsSZIkSarEgCVJkiRJlRiwJEmSJKkSA5YkSZIkVWLAkiRJkqRKDFiSJEmSVIkBS5IkSZIqMWBJkiRJUiUGLEmSJEmqxIAlSZIkSZUYsCRJkiSpEgOWJEmSJFViwJIkSZKkSgxYkiRJklSJAUuSJEmSKhnd6QIkdZdHTtm5rf1vf/Ldbe1fkiRpXXgFS5IkSZIqMWBJkiRJUiUGLEmSJEmqxIAlSZIkSZUYsCRJkiSpEgOWJEmSJFViwJIkqQ8RcXBE3BsRSyJidovt74iIH5btt0bExNI+MSJ+FxELyuM7g127JKkzOhKwIuKhiLi7TDrzStvmEXFdRNxXnjcr7RERZ5bJ666I2LWpnxll//siYkZT+26l/yXl2Bj8UUqShrOIGAWcBRwCTAGOjogpvXY7Hng2M3cAzgC+0bTt/sycWh6fGpSiJUkd18krWH9UJp1p5f1s4PrMnAxcX95DY2KbXB4zgXOgEciALwN7AnsAX14Vyso+M5uOO7j9w5EkdZk9gCWZ+UBmvgrMAQ7vtc/hwAXl9WXA/n6oJ0kj21BaItg8SV0AHNHUfmE23AJsGhFbAwcB12XmM5n5LHAdcHDZtnFm3pyZCVzY1JckSQO1LfBo0/ue0tZyn8xcATwPjCvbJkXEryPipoj4L32dJCJmRsS8iJi3fPnyetVLkjqiUwErgR9HxPyImFna3p2ZjwOU5y1Le18TXH/tPS3a38JJTZLUj1ZXonKA+zwObJ+ZuwCfAy6OiI1bnSQzz83MaZk5bfz48etUsCSp8zoVsPbOzF1pLP87ISL26WffviavtW1/a6OTmiSpbz3Adk3vJwCP9bVPRIwGNgGeycxXMvNpgMycD9wPvL/tFUuSOq4jASszHyvPTwJX0Fjn/kRZ3kd5frLs3tcE11/7hBbtkiStjduByRExKSLGANOBub32mQususnSkcBPMzMjYny5SQYR8V4a3wd+YJDqliR10KAHrIjYICI2WvUaOBC4h9UnqRnAleX1XODYcjfBvYDnyxLCa4EDI2KzcnOLA4Fry7YXI2Kv8kXjY5v6kiRpQMp3qk6kMd8sBi7NzIURcUpEHFZ2Ow8YFxFLaCwFXHWDpn2AuyLiTho3v/hUZj4zuCOQJHXC6A6c893AFeUmS6OBizPzmoi4Hbg0Io4HHgGOKvtfBRwKLAFeAv4MIDOfiYhTaXzCCHBK0+T1aeD7wDuBq8tDkqS1kplX0ZiHmttObnr9Mm/OV837XA5c3vYCJUlDzqAHrMx8APhwi/angf1btCdwQh99nQ+c36J9HrDTOhcrSZIkSWthKN2mXZIkSZKGNQOWJEmSJFViwJIkSZKkSgxYkiRJklRJJ+4iqCaPnLJzW/vf/uS729q/JEmSpDd5BUuSJEmSKjFgSZIkSVIlBixJkiRJqsSAJUmSJEmVGLAkSZIkqRIDliRJkiRVYsCSJEmSpEoMWJIkSZJUiQFLkiRJkioxYEmSJElSJaM7XYAkDWePnLJzW/vf/uS729q/JEmqyytYkiRJklSJAUuSJEmSKjFgSZIkSVIlBixJkiRJqsSAJUmSJEmVGLAkSZIkqRIDliRJkiRVYsCSJEmSpEoMWJIkSZJUiQFLkiRJkioxYEmSJElSJQYsSZIkSarEgCVJkiRJlRiwJEmSJKkSA5YkSZIkVWLAkiRJkqRKDFiSJEmSVIkBS5IkSZIqMWBJkiRJUiUGLEmSJEmqxIAlSZIkSZUYsCRJkiSpEgOWJEmSJFViwJIkSZKkSgxYkiRJklSJAUuSJEmSKjFgSZIkSVIloztdgCSpnt2+cGFb+59/+rFt7V+SpOHOK1iSJEmSVIkBS5IkSZIqMWBJkiRJUiUGLEmSJEmqxIAlSZIkSZV4F0FJ0oA9csrObe1/+5Pvbmv/kiS1W9dewYqIgyPi3ohYEhGzO12PJGn4WdNcEhHviIgflu23RsTEpm1fLO33RsRBg1m3JKlzujJgRcQo4CzgEGAKcHRETOlsVZKk4WSAc8nxwLOZuQNwBvCNcuwUYDrwQeBg4OzSnySpy3VlwAL2AJZk5gOZ+SowBzi8wzVJkoaXgcwlhwMXlNeXAftHRJT2OZn5SmY+CCwp/UmSulxkZqdrqC4ijgQOzsw/L+//FNgzM0/std9MYGZ5+3vAvYNaaMMWwFMdOG+nON7uN9LG7HgHx3syc/xgnnAgc0lE3FP26Snv7wf2BL4C3JKZ/1LazwOuzszLWpzHuWjwOd7uN9LG7HgHx4Dmom69yUW0aHtLkszMc4Fz219O3yJiXmZO62QNg8nxdr+RNmbH29UGMpf0tc+A5iFwLuoEx9v9RtqYHe/Q0q1LBHuA7ZreTwAe61AtkqThaSBzyRv7RMRoYBPgmQEeK0nqQt0asG4HJkfEpIgYQ+OLxnM7XJMkaXgZyFwyF5hRXh8J/DQba+/nAtPLXQYnAZOB2wapbklSB3XlEsHMXBERJwLXAqOA8zNzYYfL6ktHl4V0gOPtfiNtzI63S/U1l0TEKcC8zJwLnAf8ICKW0LhyNb0cuzAiLgUWASuAEzJzZUcGMjAj5u9aON7uN9LG7HiHkK68yYUkSZIkdUK3LhGUJEmSpEFnwJIkSZKkSgxYlUXEdhFxQ0QsjoiFEXFSad88Iq6LiPvK82al/QMRcXNEvBIRn2/R36iI+HVE/GiwxzIQNccbEQ9FxN0RsSAi5nViPANRecybRsRlEfGb0t/vd2JM/ak13oj4vfK3XfV4ISI+06lx9aXy3/ezpY97IuKSiBjbiTH1p/J4TypjXTgU/7YjiXNRd89FI20eAuci56LhNRf5HazKImJrYOvMvCMiNgLmA0cAnwSeycyvR8RsYLPM/NuI2BJ4T9nn2cz8Zq/+PgdMAzbOzP82mGMZiJrjjYiHgGmZOaR/KK/ymC8Afp6Z/xyNu5S9KzOfG+wx9af2P9Olz1HAUho/2vrwYI1lIGqNNyK2BX4BTMnM30XjhgdXZeb3B39Ufas43p2AOcAewKvANcCnM/O+QR+UnIu6fC4aafMQOBfhXDSs5iKvYFWWmY9n5h3l9YvAYmBb4HDggrLbBTT+gSAzn8zM24HXevcVEROA/wr88yCU/rbUHO9wUWvMEbExsA+Nu5CRma8OxUmtTX/j/YH7h9qEBtXHOxp4ZzR+H+ldDMHfQao43h2BWzLzpcxcAdwEfGwQhqAWnIu6ey4aafMQOBfhXATDaC4yYLVRREwEdgFuBd6dmY9D4x8iYMsBdPGPwCzg9TaVWFWF8Sbw44iYHxEz21VnTes45vcCy4HvRWPpzT9HxAZtLHedVfgbrzIduKR2fbWty3gzcynwTeAR4HHg+cz8cTvrXVfr+Pe9B9gnIsZFxLuAQ1n9h3bVIc5F3T0XjbR5CJyLnIv6NSTmIgNWm0TEhsDlwGcy84W3cfx/A57MzPnVi2uDdR1vsXdm7gocApwQEftUK7ANKox5NLArcE5m7gL8FphdscSqKv2NKUtQDgP+tVZt7VDh3+HNaHzyNgnYBtggIo6pW2U96zrezFwMfAO4jsaSjDtp/P6TOsi56G0ZNnPRSJuHwLnobRzvXNSBuciA1QYRsT6Nfzguysx/K81PlPWlq9aZPrmGbvYGDitrwecA+0XEv7Sp5HVSabxk5mPl+UngChrrZ4ekSmPuAXoy89by/jIaE92QU+tvXBwC3JGZT9SvtI5K4z0AeDAzl2fma8C/AX/QrprXRcV/h8/LzF0zcx8aP7rr9686yLkI6OK5aKTNQ+BcVJqdi9ZgKMxFBqzKIiJorGVenJnfato0F5hRXs8Aruyvn8z8YmZOyMyJNC5h/zQzh9wnDrXGGxEblC81UpYnHEjjMu+QU/FvvAx4NCJ+rzTtDyyqXO46qzXeJkczhJdkVBzvI8BeEfGu0uf+NNaUDyk1/77lS8dExPbAxxnCf+du51z0hq6ci0baPATORU2bnIvW3Ffn56LM9FHxAfwhjfXbdwELyuNQYBxwPY0UfT2wedl/KxqfIL0APFdeb9yrz32BH3V6bO0cL4114HeWx0Lg7zo9tsH4GwNTgXmlr3+ncXecjo+xjeN9F/A0sEmnxzVI4/0q8Bsa/4P2A+AdnR5fm8f7cxr/c3YnsH+nxzaSHzX/rk197otz0ZB4VP73dsjPQ20Ys3PREBhjG8fb8bnI27RLkiRJUiUuEZQkSZKkSgxYkiRJklSJAUuSJEmSKjFgSZIkSVIlBixJkiRJqsSAJUmSJEmVGLCkES4iRnW6BknSyOZcpG5iwJKGkYg4NSJOanr/tYj464j4QkTcHhF3RcRXm7b/e0TMj4iFETGzqf0/I+KUiLgV+P1BHoYkaRhzLpL6Z8CShpfzgBkAEbEeMB14ApgM7AFMBXaLiH3K/sdl5m7ANOCvI2Jcad8AuCcz98zMXwzmACRJw55zkdSP0Z0uQNLAZeZDEfF0ROwCvBv4NbA7cGB5DbAhjUnuZzQmso+V9u1K+9PASuDywaxdktQdnIuk/hmwpOHnn4FPAlsB5wP7A/8rM7/bvFNE7AscAPx+Zr4UETcCY8vmlzNz5WAVLEnqOs5FUh9cIigNP1cAB9P4tPDa8jguIjYEiIhtI2JLYBPg2TKhfQDYq1MFS5K6jnOR1AevYEnDTGa+GhE3AM+VT/5+HBE7AjdHBMB/AscA1wCfioi7gHuBWzpVsySpuzgXSX2LzOx0DZLWQvlC8R3AUZl5X6frkSSNPM5FUt9cIigNIxExBVgCXO+EJknqBOciqX9ewZIkSZKkSryCJUmSJEmVGLAkSZIkqRIDliRJkiRVYsCSJEmSpEoMWJIkSZJUyf8FrKC7/+ZaytcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# To see the percentage of bad loan for each year is independent or not?\n",
    "fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(12,6))\n",
    "sns.countplot(x='year', hue='target', data=data, ax=ax[0])\n",
    "sns.barplot(x='year', y='target', data=data, ax=ax[1])\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The percentage of default loan for each year is found to be at relative close range. \n",
    "Therefore, we choose Year 2014 - 2017, as the training data, and Year 2018-2019 as the test data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data shape: (1175874, 72)\n",
      "Test data shape: (120022, 72)\n"
     ]
    }
   ],
   "source": [
    "train = data.loc[data['year'] .isin(['2014','2015','2016','2017'])]\n",
    "test = data.loc[data['year'] .isin(['2018','2019'])]\n",
    "print('Train data shape:',train.shape)\n",
    "print('Test data shape:',test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    917612\n",
       "1    258262\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['target'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    84607\n",
       "1    35415\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test['target'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature engineering\n",
    "\n",
    "### Missing data imputation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Your selected dataframe has 72 columns.\n",
      "There are 70 columns that have missing values.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Missing Values</th>\n",
       "      <th>% of Total Values</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mths_since_recent_inq</th>\n",
       "      <td>114776</td>\n",
       "      <td>9.760910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emp_title</th>\n",
       "      <td>74217</td>\n",
       "      <td>6.311646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emp_length</th>\n",
       "      <td>72553</td>\n",
       "      <td>6.170134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_tl_120dpd_2m</th>\n",
       "      <td>53751</td>\n",
       "      <td>4.571153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mo_sin_old_il_acct</th>\n",
       "      <td>33544</td>\n",
       "      <td>2.852687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>title</th>\n",
       "      <td>19789</td>\n",
       "      <td>1.682918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bc_util</th>\n",
       "      <td>13404</td>\n",
       "      <td>1.139918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>percent_bc_gt_75</th>\n",
       "      <td>13127</td>\n",
       "      <td>1.116361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bc_open_to_buy</th>\n",
       "      <td>12711</td>\n",
       "      <td>1.080983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mths_since_recent_bc</th>\n",
       "      <td>11991</td>\n",
       "      <td>1.019752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>revol_util</th>\n",
       "      <td>698</td>\n",
       "      <td>0.059360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dti</th>\n",
       "      <td>307</td>\n",
       "      <td>0.026108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>avg_cur_bal</th>\n",
       "      <td>33</td>\n",
       "      <td>0.002806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zip_code</th>\n",
       "      <td>21</td>\n",
       "      <td>0.001786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_rev_accts</th>\n",
       "      <td>21</td>\n",
       "      <td>0.001786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>inq_last_6mths</th>\n",
       "      <td>21</td>\n",
       "      <td>0.001786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_tl_90g_dpd_24m</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_bal_ex_mort</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mo_sin_rcnt_rev_tl_op</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mo_sin_rcnt_tl</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hardship_flag</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mort_acc</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tot_hi_cred_lim</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pct_tl_nvr_dlq</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_il_high_credit_limit</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_accts_ever_120_pd</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_actv_bc_tl</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_actv_rev_tl</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_bc_sats</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_bc_tl</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>loan_amnt</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delinq_amnt</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>earliest_cr_line</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>funded_amnt_inv</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>term</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>int_rate</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installment</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>grade</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sub_grade</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>home_ownership</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>annual_inc</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>verification_status</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purpose</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>addr_state</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delinq_2yrs</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>open_acc</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chargeoff_within_12_mths</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pub_rec</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>revol_bal</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_acc</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>initial_list_status</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>application_type</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tot_coll_amt</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tot_cur_bal</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_rev_hi_lim</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>acc_open_past_24mths</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>funded_amnt</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>debt_settlement_flag</th>\n",
       "      <td>20</td>\n",
       "      <td>0.001701</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>70 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                            Missing Values  % of Total Values\n",
       "mths_since_recent_inq               114776           9.760910\n",
       "emp_title                            74217           6.311646\n",
       "emp_length                           72553           6.170134\n",
       "num_tl_120dpd_2m                     53751           4.571153\n",
       "mo_sin_old_il_acct                   33544           2.852687\n",
       "title                                19789           1.682918\n",
       "bc_util                              13404           1.139918\n",
       "percent_bc_gt_75                     13127           1.116361\n",
       "bc_open_to_buy                       12711           1.080983\n",
       "mths_since_recent_bc                 11991           1.019752\n",
       "revol_util                             698           0.059360\n",
       "dti                                    307           0.026108\n",
       "avg_cur_bal                             33           0.002806\n",
       "zip_code                                21           0.001786\n",
       "num_rev_accts                           21           0.001786\n",
       "inq_last_6mths                          21           0.001786\n",
       "num_tl_90g_dpd_24m                      20           0.001701\n",
       "total_bal_ex_mort                       20           0.001701\n",
       "mo_sin_rcnt_rev_tl_op                   20           0.001701\n",
       "mo_sin_rcnt_tl                          20           0.001701\n",
       "hardship_flag                           20           0.001701\n",
       "mort_acc                                20           0.001701\n",
       "tot_hi_cred_lim                         20           0.001701\n",
       "pct_tl_nvr_dlq                          20           0.001701\n",
       "total_il_high_credit_limit              20           0.001701\n",
       "num_accts_ever_120_pd                   20           0.001701\n",
       "num_actv_bc_tl                          20           0.001701\n",
       "num_actv_rev_tl                         20           0.001701\n",
       "num_bc_sats                             20           0.001701\n",
       "num_bc_tl                               20           0.001701\n",
       "...                                    ...                ...\n",
       "loan_amnt                               20           0.001701\n",
       "delinq_amnt                             20           0.001701\n",
       "earliest_cr_line                        20           0.001701\n",
       "funded_amnt_inv                         20           0.001701\n",
       "term                                    20           0.001701\n",
       "int_rate                                20           0.001701\n",
       "installment                             20           0.001701\n",
       "grade                                   20           0.001701\n",
       "sub_grade                               20           0.001701\n",
       "home_ownership                          20           0.001701\n",
       "annual_inc                              20           0.001701\n",
       "verification_status                     20           0.001701\n",
       "purpose                                 20           0.001701\n",
       "addr_state                              20           0.001701\n",
       "delinq_2yrs                             20           0.001701\n",
       "open_acc                                20           0.001701\n",
       "chargeoff_within_12_mths                20           0.001701\n",
       "pub_rec                                 20           0.001701\n",
       "revol_bal                               20           0.001701\n",
       "total_acc                               20           0.001701\n",
       "initial_list_status                     20           0.001701\n",
       "collections_12_mths_ex_med              20           0.001701\n",
       "application_type                        20           0.001701\n",
       "acc_now_delinq                          20           0.001701\n",
       "tot_coll_amt                            20           0.001701\n",
       "tot_cur_bal                             20           0.001701\n",
       "total_rev_hi_lim                        20           0.001701\n",
       "acc_open_past_24mths                    20           0.001701\n",
       "funded_amnt                             20           0.001701\n",
       "debt_settlement_flag                    20           0.001701\n",
       "\n",
       "[70 rows x 2 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# missing value proportions \n",
    "def missing_values_table(df):\n",
    "        # Total missing values\n",
    "        mis_val = df.isnull().sum()\n",
    "        \n",
    "        # Percentage of missing values\n",
    "        mis_val_percent = 100 * df.isnull().sum() / len(df)\n",
    "        \n",
    "        # Make a table with the results\n",
    "        mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)\n",
    "        \n",
    "        # Rename the columns\n",
    "        mis_val_table_ren_columns = mis_val_table.rename(\n",
    "        columns = {0 : 'Missing Values', 1 : '% of Total Values'})\n",
    "        \n",
    "        # Sort the table by percentage of missing descending\n",
    "        mis_val_table_ren_columns = mis_val_table_ren_columns[\n",
    "            mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(\n",
    "        '% of Total Values', ascending=False).round(6)\n",
    "        \n",
    "        # Print some summary information\n",
    "        print (\"Your selected dataframe has \" + str(df.shape[1]) + \" columns.\\n\"      \n",
    "            \"There are \" + str(mis_val_table_ren_columns.shape[0]) +\n",
    "              \" columns that have missing values.\")\n",
    "        \n",
    "        # Return the dataframe with missing information greater than 20%\n",
    "        return mis_val_table_ren_columns\n",
    "    \n",
    "missing_values_table(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Your selected dataframe has 72 columns.\n",
      "There are 70 columns that have missing values.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Missing Values</th>\n",
       "      <th>% of Total Values</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>emp_title</th>\n",
       "      <td>12905</td>\n",
       "      <td>10.752195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emp_length</th>\n",
       "      <td>10761</td>\n",
       "      <td>8.965856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mths_since_recent_inq</th>\n",
       "      <td>10403</td>\n",
       "      <td>8.667578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mo_sin_old_il_acct</th>\n",
       "      <td>4140</td>\n",
       "      <td>3.449368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_tl_120dpd_2m</th>\n",
       "      <td>3495</td>\n",
       "      <td>2.911966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bc_util</th>\n",
       "      <td>2077</td>\n",
       "      <td>1.730516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>percent_bc_gt_75</th>\n",
       "      <td>1997</td>\n",
       "      <td>1.663862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bc_open_to_buy</th>\n",
       "      <td>1994</td>\n",
       "      <td>1.661362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mths_since_recent_bc</th>\n",
       "      <td>1859</td>\n",
       "      <td>1.548883</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dti</th>\n",
       "      <td>333</td>\n",
       "      <td>0.277449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>revol_util</th>\n",
       "      <td>184</td>\n",
       "      <td>0.153305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>avg_cur_bal</th>\n",
       "      <td>23</td>\n",
       "      <td>0.019163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pct_tl_nvr_dlq</th>\n",
       "      <td>13</td>\n",
       "      <td>0.010831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tax_liens</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hardship_flag</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mo_sin_old_rev_tl_op</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mo_sin_rcnt_rev_tl_op</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mo_sin_rcnt_tl</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mort_acc</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_il_high_credit_limit</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_accts_ever_120_pd</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_actv_bc_tl</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_actv_rev_tl</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_bc_sats</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_bc_tl</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tot_hi_cred_lim</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_il_tl</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_op_rev_tl</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_rev_accts</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delinq_amnt</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delinq_2yrs</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>funded_amnt_inv</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>term</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>int_rate</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installment</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>grade</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sub_grade</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>home_ownership</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>annual_inc</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>verification_status</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purpose</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>title</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zip_code</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>addr_state</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>earliest_cr_line</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>funded_amnt</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>inq_last_6mths</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>open_acc</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pub_rec</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>revol_bal</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_acc</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>initial_list_status</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>application_type</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tot_coll_amt</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tot_cur_bal</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_rev_hi_lim</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>acc_open_past_24mths</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>debt_settlement_flag</th>\n",
       "      <td>12</td>\n",
       "      <td>0.009998</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>70 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                            Missing Values  % of Total Values\n",
       "emp_title                            12905          10.752195\n",
       "emp_length                           10761           8.965856\n",
       "mths_since_recent_inq                10403           8.667578\n",
       "mo_sin_old_il_acct                    4140           3.449368\n",
       "num_tl_120dpd_2m                      3495           2.911966\n",
       "bc_util                               2077           1.730516\n",
       "percent_bc_gt_75                      1997           1.663862\n",
       "bc_open_to_buy                        1994           1.661362\n",
       "mths_since_recent_bc                  1859           1.548883\n",
       "dti                                    333           0.277449\n",
       "revol_util                             184           0.153305\n",
       "avg_cur_bal                             23           0.019163\n",
       "pct_tl_nvr_dlq                          13           0.010831\n",
       "tax_liens                               12           0.009998\n",
       "hardship_flag                           12           0.009998\n",
       "mo_sin_old_rev_tl_op                    12           0.009998\n",
       "mo_sin_rcnt_rev_tl_op                   12           0.009998\n",
       "mo_sin_rcnt_tl                          12           0.009998\n",
       "mort_acc                                12           0.009998\n",
       "total_il_high_credit_limit              12           0.009998\n",
       "num_accts_ever_120_pd                   12           0.009998\n",
       "num_actv_bc_tl                          12           0.009998\n",
       "num_actv_rev_tl                         12           0.009998\n",
       "num_bc_sats                             12           0.009998\n",
       "num_bc_tl                               12           0.009998\n",
       "tot_hi_cred_lim                         12           0.009998\n",
       "num_il_tl                               12           0.009998\n",
       "num_op_rev_tl                           12           0.009998\n",
       "num_rev_accts                           12           0.009998\n",
       "delinq_amnt                             12           0.009998\n",
       "...                                    ...                ...\n",
       "delinq_2yrs                             12           0.009998\n",
       "funded_amnt_inv                         12           0.009998\n",
       "term                                    12           0.009998\n",
       "int_rate                                12           0.009998\n",
       "installment                             12           0.009998\n",
       "grade                                   12           0.009998\n",
       "sub_grade                               12           0.009998\n",
       "home_ownership                          12           0.009998\n",
       "annual_inc                              12           0.009998\n",
       "verification_status                     12           0.009998\n",
       "purpose                                 12           0.009998\n",
       "title                                   12           0.009998\n",
       "zip_code                                12           0.009998\n",
       "addr_state                              12           0.009998\n",
       "earliest_cr_line                        12           0.009998\n",
       "funded_amnt                             12           0.009998\n",
       "inq_last_6mths                          12           0.009998\n",
       "open_acc                                12           0.009998\n",
       "pub_rec                                 12           0.009998\n",
       "revol_bal                               12           0.009998\n",
       "total_acc                               12           0.009998\n",
       "initial_list_status                     12           0.009998\n",
       "collections_12_mths_ex_med              12           0.009998\n",
       "application_type                        12           0.009998\n",
       "acc_now_delinq                          12           0.009998\n",
       "tot_coll_amt                            12           0.009998\n",
       "tot_cur_bal                             12           0.009998\n",
       "total_rev_hi_lim                        12           0.009998\n",
       "acc_open_past_24mths                    12           0.009998\n",
       "debt_settlement_flag                    12           0.009998\n",
       "\n",
       "[70 rows x 2 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "missing_values_table(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>funded_amnt</th>\n",
       "      <th>funded_amnt_inv</th>\n",
       "      <th>term</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>grade</th>\n",
       "      <th>sub_grade</th>\n",
       "      <th>emp_title</th>\n",
       "      <th>emp_length</th>\n",
       "      <th>home_ownership</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>verification_status</th>\n",
       "      <th>purpose</th>\n",
       "      <th>title</th>\n",
       "      <th>zip_code</th>\n",
       "      <th>addr_state</th>\n",
       "      <th>dti</th>\n",
       "      <th>delinq_2yrs</th>\n",
       "      <th>earliest_cr_line</th>\n",
       "      <th>inq_last_6mths</th>\n",
       "      <th>open_acc</th>\n",
       "      <th>pub_rec</th>\n",
       "      <th>revol_bal</th>\n",
       "      <th>revol_util</th>\n",
       "      <th>total_acc</th>\n",
       "      <th>initial_list_status</th>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <th>application_type</th>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <th>tot_coll_amt</th>\n",
       "      <th>tot_cur_bal</th>\n",
       "      <th>total_rev_hi_lim</th>\n",
       "      <th>acc_open_past_24mths</th>\n",
       "      <th>avg_cur_bal</th>\n",
       "      <th>bc_open_to_buy</th>\n",
       "      <th>bc_util</th>\n",
       "      <th>chargeoff_within_12_mths</th>\n",
       "      <th>delinq_amnt</th>\n",
       "      <th>mo_sin_old_il_acct</th>\n",
       "      <th>mo_sin_old_rev_tl_op</th>\n",
       "      <th>mo_sin_rcnt_rev_tl_op</th>\n",
       "      <th>mo_sin_rcnt_tl</th>\n",
       "      <th>mort_acc</th>\n",
       "      <th>mths_since_recent_bc</th>\n",
       "      <th>mths_since_recent_inq</th>\n",
       "      <th>num_accts_ever_120_pd</th>\n",
       "      <th>num_actv_bc_tl</th>\n",
       "      <th>num_actv_rev_tl</th>\n",
       "      <th>num_bc_sats</th>\n",
       "      <th>num_bc_tl</th>\n",
       "      <th>num_il_tl</th>\n",
       "      <th>num_op_rev_tl</th>\n",
       "      <th>num_rev_accts</th>\n",
       "      <th>num_rev_tl_bal_gt_0</th>\n",
       "      <th>num_sats</th>\n",
       "      <th>num_tl_120dpd_2m</th>\n",
       "      <th>num_tl_30dpd</th>\n",
       "      <th>num_tl_90g_dpd_24m</th>\n",
       "      <th>num_tl_op_past_12m</th>\n",
       "      <th>pct_tl_nvr_dlq</th>\n",
       "      <th>percent_bc_gt_75</th>\n",
       "      <th>pub_rec_bankruptcies</th>\n",
       "      <th>tax_liens</th>\n",
       "      <th>tot_hi_cred_lim</th>\n",
       "      <th>total_bal_ex_mort</th>\n",
       "      <th>total_bc_limit</th>\n",
       "      <th>total_il_high_credit_limit</th>\n",
       "      <th>hardship_flag</th>\n",
       "      <th>debt_settlement_flag</th>\n",
       "      <th>year</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>228976</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2014</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>228977</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2014</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>612770</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>612771</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>733491</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>733492</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>821887</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>821888</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>893845</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>893846</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>960448</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>960449</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1016468</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1016469</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1071660</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1071661</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1128547</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1128548</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1175872</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1175873</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         loan_amnt  funded_amnt  funded_amnt_inv term int_rate  installment  \\\n",
       "228976         NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "228977         NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "612770         NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "612771         NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "733491         NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "733492         NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "821887         NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "821888         NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "893845         NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "893846         NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "960448         NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "960449         NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "1016468        NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "1016469        NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "1071660        NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "1071661        NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "1128547        NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "1128548        NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "1175872        NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "1175873        NaN          NaN              NaN  NaN      NaN          NaN   \n",
       "\n",
       "        grade sub_grade emp_title emp_length home_ownership  annual_inc  \\\n",
       "228976    NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "228977    NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "612770    NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "612771    NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "733491    NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "733492    NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "821887    NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "821888    NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "893845    NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "893846    NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "960448    NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "960449    NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "1016468   NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "1016469   NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "1071660   NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "1071661   NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "1128547   NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "1128548   NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "1175872   NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "1175873   NaN       NaN       NaN        NaN            NaN         NaN   \n",
       "\n",
       "        verification_status purpose title zip_code addr_state  dti  \\\n",
       "228976                  NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "228977                  NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "612770                  NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "612771                  NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "733491                  NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "733492                  NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "821887                  NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "821888                  NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "893845                  NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "893846                  NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "960448                  NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "960449                  NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "1016468                 NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "1016469                 NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "1071660                 NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "1071661                 NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "1128547                 NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "1128548                 NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "1175872                 NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "1175873                 NaN     NaN   NaN      NaN        NaN  NaN   \n",
       "\n",
       "         delinq_2yrs earliest_cr_line  inq_last_6mths  open_acc  pub_rec  \\\n",
       "228976           NaN              NaN             NaN       NaN      NaN   \n",
       "228977           NaN              NaN             NaN       NaN      NaN   \n",
       "612770           NaN              NaN             NaN       NaN      NaN   \n",
       "612771           NaN              NaN             NaN       NaN      NaN   \n",
       "733491           NaN              NaN             NaN       NaN      NaN   \n",
       "733492           NaN              NaN             NaN       NaN      NaN   \n",
       "821887           NaN              NaN             NaN       NaN      NaN   \n",
       "821888           NaN              NaN             NaN       NaN      NaN   \n",
       "893845           NaN              NaN             NaN       NaN      NaN   \n",
       "893846           NaN              NaN             NaN       NaN      NaN   \n",
       "960448           NaN              NaN             NaN       NaN      NaN   \n",
       "960449           NaN              NaN             NaN       NaN      NaN   \n",
       "1016468          NaN              NaN             NaN       NaN      NaN   \n",
       "1016469          NaN              NaN             NaN       NaN      NaN   \n",
       "1071660          NaN              NaN             NaN       NaN      NaN   \n",
       "1071661          NaN              NaN             NaN       NaN      NaN   \n",
       "1128547          NaN              NaN             NaN       NaN      NaN   \n",
       "1128548          NaN              NaN             NaN       NaN      NaN   \n",
       "1175872          NaN              NaN             NaN       NaN      NaN   \n",
       "1175873          NaN              NaN             NaN       NaN      NaN   \n",
       "\n",
       "         revol_bal revol_util  total_acc initial_list_status  \\\n",
       "228976         NaN        NaN        NaN                 NaN   \n",
       "228977         NaN        NaN        NaN                 NaN   \n",
       "612770         NaN        NaN        NaN                 NaN   \n",
       "612771         NaN        NaN        NaN                 NaN   \n",
       "733491         NaN        NaN        NaN                 NaN   \n",
       "733492         NaN        NaN        NaN                 NaN   \n",
       "821887         NaN        NaN        NaN                 NaN   \n",
       "821888         NaN        NaN        NaN                 NaN   \n",
       "893845         NaN        NaN        NaN                 NaN   \n",
       "893846         NaN        NaN        NaN                 NaN   \n",
       "960448         NaN        NaN        NaN                 NaN   \n",
       "960449         NaN        NaN        NaN                 NaN   \n",
       "1016468        NaN        NaN        NaN                 NaN   \n",
       "1016469        NaN        NaN        NaN                 NaN   \n",
       "1071660        NaN        NaN        NaN                 NaN   \n",
       "1071661        NaN        NaN        NaN                 NaN   \n",
       "1128547        NaN        NaN        NaN                 NaN   \n",
       "1128548        NaN        NaN        NaN                 NaN   \n",
       "1175872        NaN        NaN        NaN                 NaN   \n",
       "1175873        NaN        NaN        NaN                 NaN   \n",
       "\n",
       "         collections_12_mths_ex_med application_type  acc_now_delinq  \\\n",
       "228976                          NaN              NaN             NaN   \n",
       "228977                          NaN              NaN             NaN   \n",
       "612770                          NaN              NaN             NaN   \n",
       "612771                          NaN              NaN             NaN   \n",
       "733491                          NaN              NaN             NaN   \n",
       "733492                          NaN              NaN             NaN   \n",
       "821887                          NaN              NaN             NaN   \n",
       "821888                          NaN              NaN             NaN   \n",
       "893845                          NaN              NaN             NaN   \n",
       "893846                          NaN              NaN             NaN   \n",
       "960448                          NaN              NaN             NaN   \n",
       "960449                          NaN              NaN             NaN   \n",
       "1016468                         NaN              NaN             NaN   \n",
       "1016469                         NaN              NaN             NaN   \n",
       "1071660                         NaN              NaN             NaN   \n",
       "1071661                         NaN              NaN             NaN   \n",
       "1128547                         NaN              NaN             NaN   \n",
       "1128548                         NaN              NaN             NaN   \n",
       "1175872                         NaN              NaN             NaN   \n",
       "1175873                         NaN              NaN             NaN   \n",
       "\n",
       "         tot_coll_amt  tot_cur_bal  total_rev_hi_lim  acc_open_past_24mths  \\\n",
       "228976            NaN          NaN               NaN                   NaN   \n",
       "228977            NaN          NaN               NaN                   NaN   \n",
       "612770            NaN          NaN               NaN                   NaN   \n",
       "612771            NaN          NaN               NaN                   NaN   \n",
       "733491            NaN          NaN               NaN                   NaN   \n",
       "733492            NaN          NaN               NaN                   NaN   \n",
       "821887            NaN          NaN               NaN                   NaN   \n",
       "821888            NaN          NaN               NaN                   NaN   \n",
       "893845            NaN          NaN               NaN                   NaN   \n",
       "893846            NaN          NaN               NaN                   NaN   \n",
       "960448            NaN          NaN               NaN                   NaN   \n",
       "960449            NaN          NaN               NaN                   NaN   \n",
       "1016468           NaN          NaN               NaN                   NaN   \n",
       "1016469           NaN          NaN               NaN                   NaN   \n",
       "1071660           NaN          NaN               NaN                   NaN   \n",
       "1071661           NaN          NaN               NaN                   NaN   \n",
       "1128547           NaN          NaN               NaN                   NaN   \n",
       "1128548           NaN          NaN               NaN                   NaN   \n",
       "1175872           NaN          NaN               NaN                   NaN   \n",
       "1175873           NaN          NaN               NaN                   NaN   \n",
       "\n",
       "         avg_cur_bal  bc_open_to_buy  bc_util  chargeoff_within_12_mths  \\\n",
       "228976           NaN             NaN      NaN                       NaN   \n",
       "228977           NaN             NaN      NaN                       NaN   \n",
       "612770           NaN             NaN      NaN                       NaN   \n",
       "612771           NaN             NaN      NaN                       NaN   \n",
       "733491           NaN             NaN      NaN                       NaN   \n",
       "733492           NaN             NaN      NaN                       NaN   \n",
       "821887           NaN             NaN      NaN                       NaN   \n",
       "821888           NaN             NaN      NaN                       NaN   \n",
       "893845           NaN             NaN      NaN                       NaN   \n",
       "893846           NaN             NaN      NaN                       NaN   \n",
       "960448           NaN             NaN      NaN                       NaN   \n",
       "960449           NaN             NaN      NaN                       NaN   \n",
       "1016468          NaN             NaN      NaN                       NaN   \n",
       "1016469          NaN             NaN      NaN                       NaN   \n",
       "1071660          NaN             NaN      NaN                       NaN   \n",
       "1071661          NaN             NaN      NaN                       NaN   \n",
       "1128547          NaN             NaN      NaN                       NaN   \n",
       "1128548          NaN             NaN      NaN                       NaN   \n",
       "1175872          NaN             NaN      NaN                       NaN   \n",
       "1175873          NaN             NaN      NaN                       NaN   \n",
       "\n",
       "         delinq_amnt  mo_sin_old_il_acct  mo_sin_old_rev_tl_op  \\\n",
       "228976           NaN                 NaN                   NaN   \n",
       "228977           NaN                 NaN                   NaN   \n",
       "612770           NaN                 NaN                   NaN   \n",
       "612771           NaN                 NaN                   NaN   \n",
       "733491           NaN                 NaN                   NaN   \n",
       "733492           NaN                 NaN                   NaN   \n",
       "821887           NaN                 NaN                   NaN   \n",
       "821888           NaN                 NaN                   NaN   \n",
       "893845           NaN                 NaN                   NaN   \n",
       "893846           NaN                 NaN                   NaN   \n",
       "960448           NaN                 NaN                   NaN   \n",
       "960449           NaN                 NaN                   NaN   \n",
       "1016468          NaN                 NaN                   NaN   \n",
       "1016469          NaN                 NaN                   NaN   \n",
       "1071660          NaN                 NaN                   NaN   \n",
       "1071661          NaN                 NaN                   NaN   \n",
       "1128547          NaN                 NaN                   NaN   \n",
       "1128548          NaN                 NaN                   NaN   \n",
       "1175872          NaN                 NaN                   NaN   \n",
       "1175873          NaN                 NaN                   NaN   \n",
       "\n",
       "         mo_sin_rcnt_rev_tl_op  mo_sin_rcnt_tl  mort_acc  \\\n",
       "228976                     NaN             NaN       NaN   \n",
       "228977                     NaN             NaN       NaN   \n",
       "612770                     NaN             NaN       NaN   \n",
       "612771                     NaN             NaN       NaN   \n",
       "733491                     NaN             NaN       NaN   \n",
       "733492                     NaN             NaN       NaN   \n",
       "821887                     NaN             NaN       NaN   \n",
       "821888                     NaN             NaN       NaN   \n",
       "893845                     NaN             NaN       NaN   \n",
       "893846                     NaN             NaN       NaN   \n",
       "960448                     NaN             NaN       NaN   \n",
       "960449                     NaN             NaN       NaN   \n",
       "1016468                    NaN             NaN       NaN   \n",
       "1016469                    NaN             NaN       NaN   \n",
       "1071660                    NaN             NaN       NaN   \n",
       "1071661                    NaN             NaN       NaN   \n",
       "1128547                    NaN             NaN       NaN   \n",
       "1128548                    NaN             NaN       NaN   \n",
       "1175872                    NaN             NaN       NaN   \n",
       "1175873                    NaN             NaN       NaN   \n",
       "\n",
       "         mths_since_recent_bc  mths_since_recent_inq  num_accts_ever_120_pd  \\\n",
       "228976                    NaN                    NaN                    NaN   \n",
       "228977                    NaN                    NaN                    NaN   \n",
       "612770                    NaN                    NaN                    NaN   \n",
       "612771                    NaN                    NaN                    NaN   \n",
       "733491                    NaN                    NaN                    NaN   \n",
       "733492                    NaN                    NaN                    NaN   \n",
       "821887                    NaN                    NaN                    NaN   \n",
       "821888                    NaN                    NaN                    NaN   \n",
       "893845                    NaN                    NaN                    NaN   \n",
       "893846                    NaN                    NaN                    NaN   \n",
       "960448                    NaN                    NaN                    NaN   \n",
       "960449                    NaN                    NaN                    NaN   \n",
       "1016468                   NaN                    NaN                    NaN   \n",
       "1016469                   NaN                    NaN                    NaN   \n",
       "1071660                   NaN                    NaN                    NaN   \n",
       "1071661                   NaN                    NaN                    NaN   \n",
       "1128547                   NaN                    NaN                    NaN   \n",
       "1128548                   NaN                    NaN                    NaN   \n",
       "1175872                   NaN                    NaN                    NaN   \n",
       "1175873                   NaN                    NaN                    NaN   \n",
       "\n",
       "         num_actv_bc_tl  num_actv_rev_tl  num_bc_sats  num_bc_tl  num_il_tl  \\\n",
       "228976              NaN              NaN          NaN        NaN        NaN   \n",
       "228977              NaN              NaN          NaN        NaN        NaN   \n",
       "612770              NaN              NaN          NaN        NaN        NaN   \n",
       "612771              NaN              NaN          NaN        NaN        NaN   \n",
       "733491              NaN              NaN          NaN        NaN        NaN   \n",
       "733492              NaN              NaN          NaN        NaN        NaN   \n",
       "821887              NaN              NaN          NaN        NaN        NaN   \n",
       "821888              NaN              NaN          NaN        NaN        NaN   \n",
       "893845              NaN              NaN          NaN        NaN        NaN   \n",
       "893846              NaN              NaN          NaN        NaN        NaN   \n",
       "960448              NaN              NaN          NaN        NaN        NaN   \n",
       "960449              NaN              NaN          NaN        NaN        NaN   \n",
       "1016468             NaN              NaN          NaN        NaN        NaN   \n",
       "1016469             NaN              NaN          NaN        NaN        NaN   \n",
       "1071660             NaN              NaN          NaN        NaN        NaN   \n",
       "1071661             NaN              NaN          NaN        NaN        NaN   \n",
       "1128547             NaN              NaN          NaN        NaN        NaN   \n",
       "1128548             NaN              NaN          NaN        NaN        NaN   \n",
       "1175872             NaN              NaN          NaN        NaN        NaN   \n",
       "1175873             NaN              NaN          NaN        NaN        NaN   \n",
       "\n",
       "         num_op_rev_tl  num_rev_accts  num_rev_tl_bal_gt_0  num_sats  \\\n",
       "228976             NaN            NaN                  NaN       NaN   \n",
       "228977             NaN            NaN                  NaN       NaN   \n",
       "612770             NaN            NaN                  NaN       NaN   \n",
       "612771             NaN            NaN                  NaN       NaN   \n",
       "733491             NaN            NaN                  NaN       NaN   \n",
       "733492             NaN            NaN                  NaN       NaN   \n",
       "821887             NaN            NaN                  NaN       NaN   \n",
       "821888             NaN            NaN                  NaN       NaN   \n",
       "893845             NaN            NaN                  NaN       NaN   \n",
       "893846             NaN            NaN                  NaN       NaN   \n",
       "960448             NaN            NaN                  NaN       NaN   \n",
       "960449             NaN            NaN                  NaN       NaN   \n",
       "1016468            NaN            NaN                  NaN       NaN   \n",
       "1016469            NaN            NaN                  NaN       NaN   \n",
       "1071660            NaN            NaN                  NaN       NaN   \n",
       "1071661            NaN            NaN                  NaN       NaN   \n",
       "1128547            NaN            NaN                  NaN       NaN   \n",
       "1128548            NaN            NaN                  NaN       NaN   \n",
       "1175872            NaN            NaN                  NaN       NaN   \n",
       "1175873            NaN            NaN                  NaN       NaN   \n",
       "\n",
       "         num_tl_120dpd_2m  num_tl_30dpd  num_tl_90g_dpd_24m  \\\n",
       "228976                NaN           NaN                 NaN   \n",
       "228977                NaN           NaN                 NaN   \n",
       "612770                NaN           NaN                 NaN   \n",
       "612771                NaN           NaN                 NaN   \n",
       "733491                NaN           NaN                 NaN   \n",
       "733492                NaN           NaN                 NaN   \n",
       "821887                NaN           NaN                 NaN   \n",
       "821888                NaN           NaN                 NaN   \n",
       "893845                NaN           NaN                 NaN   \n",
       "893846                NaN           NaN                 NaN   \n",
       "960448                NaN           NaN                 NaN   \n",
       "960449                NaN           NaN                 NaN   \n",
       "1016468               NaN           NaN                 NaN   \n",
       "1016469               NaN           NaN                 NaN   \n",
       "1071660               NaN           NaN                 NaN   \n",
       "1071661               NaN           NaN                 NaN   \n",
       "1128547               NaN           NaN                 NaN   \n",
       "1128548               NaN           NaN                 NaN   \n",
       "1175872               NaN           NaN                 NaN   \n",
       "1175873               NaN           NaN                 NaN   \n",
       "\n",
       "         num_tl_op_past_12m  pct_tl_nvr_dlq  percent_bc_gt_75  \\\n",
       "228976                  NaN             NaN               NaN   \n",
       "228977                  NaN             NaN               NaN   \n",
       "612770                  NaN             NaN               NaN   \n",
       "612771                  NaN             NaN               NaN   \n",
       "733491                  NaN             NaN               NaN   \n",
       "733492                  NaN             NaN               NaN   \n",
       "821887                  NaN             NaN               NaN   \n",
       "821888                  NaN             NaN               NaN   \n",
       "893845                  NaN             NaN               NaN   \n",
       "893846                  NaN             NaN               NaN   \n",
       "960448                  NaN             NaN               NaN   \n",
       "960449                  NaN             NaN               NaN   \n",
       "1016468                 NaN             NaN               NaN   \n",
       "1016469                 NaN             NaN               NaN   \n",
       "1071660                 NaN             NaN               NaN   \n",
       "1071661                 NaN             NaN               NaN   \n",
       "1128547                 NaN             NaN               NaN   \n",
       "1128548                 NaN             NaN               NaN   \n",
       "1175872                 NaN             NaN               NaN   \n",
       "1175873                 NaN             NaN               NaN   \n",
       "\n",
       "         pub_rec_bankruptcies  tax_liens  tot_hi_cred_lim  total_bal_ex_mort  \\\n",
       "228976                    NaN        NaN              NaN                NaN   \n",
       "228977                    NaN        NaN              NaN                NaN   \n",
       "612770                    NaN        NaN              NaN                NaN   \n",
       "612771                    NaN        NaN              NaN                NaN   \n",
       "733491                    NaN        NaN              NaN                NaN   \n",
       "733492                    NaN        NaN              NaN                NaN   \n",
       "821887                    NaN        NaN              NaN                NaN   \n",
       "821888                    NaN        NaN              NaN                NaN   \n",
       "893845                    NaN        NaN              NaN                NaN   \n",
       "893846                    NaN        NaN              NaN                NaN   \n",
       "960448                    NaN        NaN              NaN                NaN   \n",
       "960449                    NaN        NaN              NaN                NaN   \n",
       "1016468                   NaN        NaN              NaN                NaN   \n",
       "1016469                   NaN        NaN              NaN                NaN   \n",
       "1071660                   NaN        NaN              NaN                NaN   \n",
       "1071661                   NaN        NaN              NaN                NaN   \n",
       "1128547                   NaN        NaN              NaN                NaN   \n",
       "1128548                   NaN        NaN              NaN                NaN   \n",
       "1175872                   NaN        NaN              NaN                NaN   \n",
       "1175873                   NaN        NaN              NaN                NaN   \n",
       "\n",
       "         total_bc_limit  total_il_high_credit_limit hardship_flag  \\\n",
       "228976              NaN                         NaN           NaN   \n",
       "228977              NaN                         NaN           NaN   \n",
       "612770              NaN                         NaN           NaN   \n",
       "612771              NaN                         NaN           NaN   \n",
       "733491              NaN                         NaN           NaN   \n",
       "733492              NaN                         NaN           NaN   \n",
       "821887              NaN                         NaN           NaN   \n",
       "821888              NaN                         NaN           NaN   \n",
       "893845              NaN                         NaN           NaN   \n",
       "893846              NaN                         NaN           NaN   \n",
       "960448              NaN                         NaN           NaN   \n",
       "960449              NaN                         NaN           NaN   \n",
       "1016468             NaN                         NaN           NaN   \n",
       "1016469             NaN                         NaN           NaN   \n",
       "1071660             NaN                         NaN           NaN   \n",
       "1071661             NaN                         NaN           NaN   \n",
       "1128547             NaN                         NaN           NaN   \n",
       "1128548             NaN                         NaN           NaN   \n",
       "1175872             NaN                         NaN           NaN   \n",
       "1175873             NaN                         NaN           NaN   \n",
       "\n",
       "        debt_settlement_flag  year  target  \n",
       "228976                   NaN  2014       1  \n",
       "228977                   NaN  2014       1  \n",
       "612770                   NaN  2015       1  \n",
       "612771                   NaN  2015       1  \n",
       "733491                   NaN  2016       1  \n",
       "733492                   NaN  2016       1  \n",
       "821887                   NaN  2016       1  \n",
       "821888                   NaN  2016       1  \n",
       "893845                   NaN  2016       1  \n",
       "893846                   NaN  2016       1  \n",
       "960448                   NaN  2016       1  \n",
       "960449                   NaN  2016       1  \n",
       "1016468                  NaN  2017       1  \n",
       "1016469                  NaN  2017       1  \n",
       "1071660                  NaN  2017       1  \n",
       "1071661                  NaN  2017       1  \n",
       "1128547                  NaN  2017       1  \n",
       "1128548                  NaN  2017       1  \n",
       "1175872                  NaN  2017       1  \n",
       "1175873                  NaN  2017       1  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# There're 20 samples in train and 12 samples in test that all features are Null \n",
    "pd.set_option('display.max_columns', None)\n",
    "train[train['loan_amnt'] .isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Locate these rows\n",
    "Index1 = train[np.isnan(train['loan_amnt'])].index\n",
    "Index2 = test[np.isnan(test['loan_amnt'])].index\n",
    "# Delete them\n",
    "train.drop(Index1,inplace=True)\n",
    "test.drop(Index2,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Your selected dataframe has 72 columns.\n",
      "There are 16 columns that have missing values.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Missing Values</th>\n",
       "      <th>% of Total Values</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mths_since_recent_inq</th>\n",
       "      <td>114756</td>\n",
       "      <td>9.759375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emp_title</th>\n",
       "      <td>74197</td>\n",
       "      <td>6.310052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emp_length</th>\n",
       "      <td>72533</td>\n",
       "      <td>6.168538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_tl_120dpd_2m</th>\n",
       "      <td>53731</td>\n",
       "      <td>4.569530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mo_sin_old_il_acct</th>\n",
       "      <td>33524</td>\n",
       "      <td>2.851034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>title</th>\n",
       "      <td>19769</td>\n",
       "      <td>1.681246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bc_util</th>\n",
       "      <td>13384</td>\n",
       "      <td>1.138237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>percent_bc_gt_75</th>\n",
       "      <td>13107</td>\n",
       "      <td>1.114679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bc_open_to_buy</th>\n",
       "      <td>12691</td>\n",
       "      <td>1.079301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mths_since_recent_bc</th>\n",
       "      <td>11971</td>\n",
       "      <td>1.018069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>revol_util</th>\n",
       "      <td>678</td>\n",
       "      <td>0.057660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dti</th>\n",
       "      <td>287</td>\n",
       "      <td>0.024408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>avg_cur_bal</th>\n",
       "      <td>13</td>\n",
       "      <td>0.001106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zip_code</th>\n",
       "      <td>1</td>\n",
       "      <td>0.000085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>inq_last_6mths</th>\n",
       "      <td>1</td>\n",
       "      <td>0.000085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_rev_accts</th>\n",
       "      <td>1</td>\n",
       "      <td>0.000085</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       Missing Values  % of Total Values\n",
       "mths_since_recent_inq          114756           9.759375\n",
       "emp_title                       74197           6.310052\n",
       "emp_length                      72533           6.168538\n",
       "num_tl_120dpd_2m                53731           4.569530\n",
       "mo_sin_old_il_acct              33524           2.851034\n",
       "title                           19769           1.681246\n",
       "bc_util                         13384           1.138237\n",
       "percent_bc_gt_75                13107           1.114679\n",
       "bc_open_to_buy                  12691           1.079301\n",
       "mths_since_recent_bc            11971           1.018069\n",
       "revol_util                        678           0.057660\n",
       "dti                               287           0.024408\n",
       "avg_cur_bal                        13           0.001106\n",
       "zip_code                            1           0.000085\n",
       "inq_last_6mths                      1           0.000085\n",
       "num_rev_accts                       1           0.000085"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "missing_values_table(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Your selected dataframe has 72 columns.\n",
      "There are 13 columns that have missing values.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Missing Values</th>\n",
       "      <th>% of Total Values</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>emp_title</th>\n",
       "      <td>12893</td>\n",
       "      <td>10.743271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emp_length</th>\n",
       "      <td>10749</td>\n",
       "      <td>8.956754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mths_since_recent_inq</th>\n",
       "      <td>10391</td>\n",
       "      <td>8.658445</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mo_sin_old_il_acct</th>\n",
       "      <td>4128</td>\n",
       "      <td>3.439713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_tl_120dpd_2m</th>\n",
       "      <td>3483</td>\n",
       "      <td>2.902258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bc_util</th>\n",
       "      <td>2065</td>\n",
       "      <td>1.720690</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>percent_bc_gt_75</th>\n",
       "      <td>1985</td>\n",
       "      <td>1.654029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bc_open_to_buy</th>\n",
       "      <td>1982</td>\n",
       "      <td>1.651529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mths_since_recent_bc</th>\n",
       "      <td>1847</td>\n",
       "      <td>1.539038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dti</th>\n",
       "      <td>321</td>\n",
       "      <td>0.267478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>revol_util</th>\n",
       "      <td>172</td>\n",
       "      <td>0.143321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>avg_cur_bal</th>\n",
       "      <td>11</td>\n",
       "      <td>0.009166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pct_tl_nvr_dlq</th>\n",
       "      <td>1</td>\n",
       "      <td>0.000833</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       Missing Values  % of Total Values\n",
       "emp_title                       12893          10.743271\n",
       "emp_length                      10749           8.956754\n",
       "mths_since_recent_inq           10391           8.658445\n",
       "mo_sin_old_il_acct               4128           3.439713\n",
       "num_tl_120dpd_2m                 3483           2.902258\n",
       "bc_util                          2065           1.720690\n",
       "percent_bc_gt_75                 1985           1.654029\n",
       "bc_open_to_buy                   1982           1.651529\n",
       "mths_since_recent_bc             1847           1.539038\n",
       "dti                               321           0.267478\n",
       "revol_util                        172           0.143321\n",
       "avg_cur_bal                        11           0.009166\n",
       "pct_tl_nvr_dlq                      1           0.000833"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "missing_values_table(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Analyze from bottom to top:<br>\n",
    "For **zip_code**, theoretically, we can try to add latitude, longitude, local population and local economic data to make it more informative. However, as I am not quite familiar with US postal code distribution, I will delete this feature anyway."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train.drop(labels='zip_code', axis=1)\n",
    "test = test.drop(labels='zip_code', axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For **inq_last_6mths**(The number of inquiries in past 6 months (excluding auto and mortgage inquiries).):\n",
    "Since there's only one missing value, and it varies from 0 to 6. Use the mode (most frequent value) to replace it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0    693809\n",
       "1.0    316570\n",
       "2.0    110438\n",
       "3.0     38401\n",
       "4.0     11894\n",
       "5.0      4199\n",
       "6.0       542\n",
       "Name: inq_last_6mths, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['inq_last_6mths'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['inq_last_6mths'] = train['inq_last_6mths'].fillna(value=0.0)\n",
    "test['pct_tl_nvr_dlq'] = test['pct_tl_nvr_dlq'].fillna(value=100.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For **num_rev_accts**(Number of revolving accounts):<br>\n",
    "It's a continuous feature, so it is reasonable to fill with a median value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_rev_accts_median = train['num_rev_accts'].median()\n",
    "train['num_rev_accts'] = train['num_rev_accts'].fillna(value=num_rev_accts_median)\n",
    "#train['num_rev_accts'] = train['num_rev_accts'].fillna(value=train['num_rev_accts'].median)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For **avg_cur_bal**(Average current balance of all accounts): use median value to fill in"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "avg_cur_bal_median = train['avg_cur_bal'].median()\n",
    "train['avg_cur_bal'] = train['avg_cur_bal'].fillna(value=avg_cur_bal_median)\n",
    "test['avg_cur_bal'] = test['avg_cur_bal'].fillna(value=avg_cur_bal_median)\n",
    "#train['avg_cur_bal'] = train['avg_cur_bal'].fillna(value=train['avg_cur_bal'].median)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For **dti**(A ratio calculated using the borrower’s total monthly debt payments on the total debt obligations, excluding mortgage and the requested LC loan, divided by the borrower’s self-reported monthly income):\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fill the missing value with the median value\n",
    "dti_median = train['dti'].median()\n",
    "train['dti'] = train['dti'].fillna(value=dti_median)\n",
    "test['dti'] = test['dti'].fillna(value=dti_median)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For **revol_util**(Revolving line utilization rate, or the amount of credit the borrower is using relative to all available revolving credit.):<br>\n",
    "The data type of this feature is object, and it is a percentage variable. Need to transfer to float, and parse the data to delete the '%'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['revol_util'] = train['revol_util'].str.strip(\"%\").astype(float)/100\n",
    "test['revol_util'] = test['revol_util'].str.strip(\"%\").astype(float)/100\n",
    "revol_util_median = train['revol_util'].median()\n",
    "train['revol_util'] = train['revol_util'].fillna(value=revol_util_median)\n",
    "test['revol_util'] = test['revol_util'].fillna(value=revol_util_median)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For **mths_since_recent_bc**(Months since most recent bankcard account opened.) <br>**mths_since_recent_inq**(Months since most recent inquire.)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "mths_recent_bc_median = train['mths_since_recent_bc'].median()\n",
    "mths_recent_inq_median = train['mths_since_recent_inq'].median()\n",
    "train['mths_since_recent_bc'] = train['mths_since_recent_bc'].fillna(value = mths_recent_bc_median)\n",
    "train['mths_since_recent_inq'] = train['mths_since_recent_inq'].fillna(value = mths_recent_inq_median)\n",
    "test['mths_since_recent_bc'] = test['mths_since_recent_bc'].fillna(value = mths_recent_bc_median)\n",
    "test['mths_since_recent_inq'] = test['mths_since_recent_inq'].fillna(value = mths_recent_inq_median)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For **num_tl_120dpd_2m** (Number of accounts currently 120 days past due (updated in past 2 months)), use most frequent value 0 to fill in the missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['num_tl_120dpd_2m'] = train['num_tl_120dpd_2m'].fillna(value=0)\n",
    "test['num_tl_120dpd_2m'] = test['num_tl_120dpd_2m'].fillna(value=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For **bc_open_to_buy** (Total open to buy on revolving bankcards),<br> **bc_util** (Ratio of total current balance to high credit/credit limit for all bankcard accounts.), <br> **percent_bc_gt_75** (Percentage of all bankcard accounts > 75% of limit), <br> **mo_sin_old_il_acct**(Months since oldest bank installment account opened.), <br>  use the fill in with median value strategy :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "bc_open_to_buy_median = train['bc_open_to_buy'].median()\n",
    "train['bc_open_to_buy'] = train['bc_open_to_buy'].fillna(value=bc_open_to_buy_median)\n",
    "test['bc_open_to_buy'] = test['bc_open_to_buy'].fillna(value=bc_open_to_buy_median)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "bc_util_median = train['bc_util'].median()\n",
    "train['bc_util'] = train['bc_util'].fillna(value=bc_util_median)\n",
    "test['bc_util'] = test['bc_util'].fillna(value=bc_util_median)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "percent_bc_gt_75_median = train['percent_bc_gt_75'].median()\n",
    "train['percent_bc_gt_75'] = train['percent_bc_gt_75'].fillna(value=percent_bc_gt_75_median)\n",
    "test['percent_bc_gt_75'] = test['percent_bc_gt_75'].fillna(value=percent_bc_gt_75_median)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "mo_sin_old_il_acct_median = train['mo_sin_old_il_acct'].median()\n",
    "train['mo_sin_old_il_acct'] = train['mo_sin_old_il_acct'].fillna(value=mo_sin_old_il_acct_median)\n",
    "test['mo_sin_old_il_acct'] = test['mo_sin_old_il_acct'].fillna(value=mo_sin_old_il_acct_median)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# emp_title contains too many categories, it is better to delete it\n",
    "train = train.drop(labels='emp_title', axis=1)\n",
    "test = test.drop(labels='emp_title', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for 'title', we will fill the missing value with another category, named as \"missing\"\n",
    "train['title'] = train['title'].fillna(value='Missing')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For **employ_length**, usually it is a positive relationship with annual income based on common sense. Though it is not absolute, but let's explore to see the correlation of these two variables below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_parser(x):\n",
    "    \"\"\" function to parse `int_rate` and `revol_util` \"\"\"\n",
    "    if 'years' in str(x):\n",
    "        return str(x).replace('years', 'year')\n",
    "    if '< 1' in str(x):\n",
    "        return str(x).replace('< 1', '0')\n",
    "    if '+' in str(x):\n",
    "        return str(x).replace('+', '')\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['emp_length'] = train['emp_length'].apply(str_parser)\n",
    "test['emp_length'] = test['emp_length'].apply(str_parser)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['emp_length'] = train['emp_length'].str.strip(\"year\").astype(float)\n",
    "test['emp_length'] = test['emp_length'].str.strip(\"year\").astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>x</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.03832</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>y</th>\n",
       "      <td>0.03832</td>\n",
       "      <td>1.00000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         x        y\n",
       "x  1.00000  0.03832\n",
       "y  0.03832  1.00000"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check their correlation first\n",
    "employ_length_salary = pd.DataFrame({'x':train['emp_length'], 'y':train['annual_inc']})\n",
    "employ_length_salary.corr()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As shown, surprisingly, we find that these two columns have few relationship. Choose mode method to fill in the missing value might be better here. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "emp_length_mode = train['emp_length'].mode()[0]\n",
    "train['emp_length'] = train['emp_length'].fillna(value=emp_length_mode)\n",
    "test['emp_length'] = test['emp_length'].fillna(value=emp_length_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Your selected dataframe has 70 columns.\n",
      "There are 0 columns that have missing values.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Missing Values</th>\n",
       "      <th>% of Total Values</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Missing Values, % of Total Values]\n",
       "Index: []"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "missing_values_table(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Your selected dataframe has 70 columns.\n",
      "There are 0 columns that have missing values.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Missing Values</th>\n",
       "      <th>% of Total Values</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Missing Values, % of Total Values]\n",
       "Index: []"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "missing_values_table(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save to disk\n",
    "train.to_csv('./train_clean.csv', index=False)\n",
    "test.to_csv('./test_clean.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}