data_cleaning v5.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import dependencies\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from sklearn.metrics import confusion_matrix, classification_report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load in data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/zt/x756b70j3v75355fl48s3dzc0000gn/T/ipykernel_10263/1073337694.py:4: DtypeWarning: Columns (26) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df_train = pd.read_csv(file_path)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Customer_ID</th>\n",
       "      <th>Month</th>\n",
       "      <th>Name</th>\n",
       "      <th>Age</th>\n",
       "      <th>SSN</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Monthly_Inhand_Salary</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>...</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "      <th>Credit_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0x1602</td>\n",
       "      <td>CUS_0xd40</td>\n",
       "      <td>January</td>\n",
       "      <td>Aaron Maashoh</td>\n",
       "      <td>23</td>\n",
       "      <td>821-00-0265</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>1824.843333</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>809.98</td>\n",
       "      <td>26.822620</td>\n",
       "      <td>22 Years and 1 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>80.41529543900253</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>312.49408867943663</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0x1603</td>\n",
       "      <td>CUS_0xd40</td>\n",
       "      <td>February</td>\n",
       "      <td>Aaron Maashoh</td>\n",
       "      <td>23</td>\n",
       "      <td>821-00-0265</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>31.944960</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>118.28022162236736</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>284.62916249607184</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0x1604</td>\n",
       "      <td>CUS_0xd40</td>\n",
       "      <td>March</td>\n",
       "      <td>Aaron Maashoh</td>\n",
       "      <td>-500</td>\n",
       "      <td>821-00-0265</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>28.609352</td>\n",
       "      <td>22 Years and 3 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>81.699521264648</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>331.2098628537912</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0x1605</td>\n",
       "      <td>CUS_0xd40</td>\n",
       "      <td>April</td>\n",
       "      <td>Aaron Maashoh</td>\n",
       "      <td>23</td>\n",
       "      <td>821-00-0265</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>31.377862</td>\n",
       "      <td>22 Years and 4 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>199.4580743910713</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>223.45130972736786</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0x1606</td>\n",
       "      <td>CUS_0xd40</td>\n",
       "      <td>May</td>\n",
       "      <td>Aaron Maashoh</td>\n",
       "      <td>23</td>\n",
       "      <td>821-00-0265</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>1824.843333</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>24.797347</td>\n",
       "      <td>22 Years and 5 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>41.420153086217326</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>341.48923103222177</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>0x25fe9</td>\n",
       "      <td>CUS_0x942c</td>\n",
       "      <td>April</td>\n",
       "      <td>Nicks</td>\n",
       "      <td>25</td>\n",
       "      <td>078-73-5990</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>3359.415833</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.663572</td>\n",
       "      <td>31 Years and 6 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>60.97133255718485</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>479.866228</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>0x25fea</td>\n",
       "      <td>CUS_0x942c</td>\n",
       "      <td>May</td>\n",
       "      <td>Nicks</td>\n",
       "      <td>25</td>\n",
       "      <td>078-73-5990</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>3359.415833</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>40.565631</td>\n",
       "      <td>31 Years and 7 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>54.18595028760385</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>496.65161</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>0x25feb</td>\n",
       "      <td>CUS_0x942c</td>\n",
       "      <td>June</td>\n",
       "      <td>Nicks</td>\n",
       "      <td>25</td>\n",
       "      <td>078-73-5990</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>3359.415833</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>41.255522</td>\n",
       "      <td>31 Years and 8 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>24.02847744864441</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>516.809083</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>0x25fec</td>\n",
       "      <td>CUS_0x942c</td>\n",
       "      <td>July</td>\n",
       "      <td>Nicks</td>\n",
       "      <td>25</td>\n",
       "      <td>078-73-5990</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>3359.415833</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>33.638208</td>\n",
       "      <td>31 Years and 9 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>251.67258219721603</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>319.164979</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>0x25fed</td>\n",
       "      <td>CUS_0x942c</td>\n",
       "      <td>August</td>\n",
       "      <td>Nicks</td>\n",
       "      <td>25</td>\n",
       "      <td>078-73-5990</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99_</td>\n",
       "      <td>3359.415833</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.192463</td>\n",
       "      <td>31 Years and 10 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>167.1638651610451</td>\n",
       "      <td>!@9#%8</td>\n",
       "      <td>393.673696</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100000 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            ID Customer_ID     Month           Name   Age          SSN  \\\n",
       "0       0x1602   CUS_0xd40   January  Aaron Maashoh    23  821-00-0265   \n",
       "1       0x1603   CUS_0xd40  February  Aaron Maashoh    23  821-00-0265   \n",
       "2       0x1604   CUS_0xd40     March  Aaron Maashoh  -500  821-00-0265   \n",
       "3       0x1605   CUS_0xd40     April  Aaron Maashoh    23  821-00-0265   \n",
       "4       0x1606   CUS_0xd40       May  Aaron Maashoh    23  821-00-0265   \n",
       "...        ...         ...       ...            ...   ...          ...   \n",
       "99995  0x25fe9  CUS_0x942c     April          Nicks    25  078-73-5990   \n",
       "99996  0x25fea  CUS_0x942c       May          Nicks    25  078-73-5990   \n",
       "99997  0x25feb  CUS_0x942c      June          Nicks    25  078-73-5990   \n",
       "99998  0x25fec  CUS_0x942c      July          Nicks    25  078-73-5990   \n",
       "99999  0x25fed  CUS_0x942c    August          Nicks    25  078-73-5990   \n",
       "\n",
       "      Occupation Annual_Income  Monthly_Inhand_Salary  Num_Bank_Accounts  ...  \\\n",
       "0      Scientist      19114.12            1824.843333                  3  ...   \n",
       "1      Scientist      19114.12                    NaN                  3  ...   \n",
       "2      Scientist      19114.12                    NaN                  3  ...   \n",
       "3      Scientist      19114.12                    NaN                  3  ...   \n",
       "4      Scientist      19114.12            1824.843333                  3  ...   \n",
       "...          ...           ...                    ...                ...  ...   \n",
       "99995   Mechanic      39628.99            3359.415833                  4  ...   \n",
       "99996   Mechanic      39628.99            3359.415833                  4  ...   \n",
       "99997   Mechanic      39628.99            3359.415833                  4  ...   \n",
       "99998   Mechanic      39628.99            3359.415833                  4  ...   \n",
       "99999   Mechanic     39628.99_            3359.415833                  4  ...   \n",
       "\n",
       "       Credit_Mix  Outstanding_Debt Credit_Utilization_Ratio  \\\n",
       "0               _            809.98                26.822620   \n",
       "1            Good            809.98                31.944960   \n",
       "2            Good            809.98                28.609352   \n",
       "3            Good            809.98                31.377862   \n",
       "4            Good            809.98                24.797347   \n",
       "...           ...               ...                      ...   \n",
       "99995           _            502.38                34.663572   \n",
       "99996           _            502.38                40.565631   \n",
       "99997        Good            502.38                41.255522   \n",
       "99998        Good            502.38                33.638208   \n",
       "99999        Good            502.38                34.192463   \n",
       "\n",
       "           Credit_History_Age  Payment_of_Min_Amount Total_EMI_per_month  \\\n",
       "0       22 Years and 1 Months                     No           49.574949   \n",
       "1                         NaN                     No           49.574949   \n",
       "2       22 Years and 3 Months                     No           49.574949   \n",
       "3       22 Years and 4 Months                     No           49.574949   \n",
       "4       22 Years and 5 Months                     No           49.574949   \n",
       "...                       ...                    ...                 ...   \n",
       "99995   31 Years and 6 Months                     No           35.104023   \n",
       "99996   31 Years and 7 Months                     No           35.104023   \n",
       "99997   31 Years and 8 Months                     No           35.104023   \n",
       "99998   31 Years and 9 Months                     No           35.104023   \n",
       "99999  31 Years and 10 Months                     No           35.104023   \n",
       "\n",
       "      Amount_invested_monthly                 Payment_Behaviour  \\\n",
       "0           80.41529543900253   High_spent_Small_value_payments   \n",
       "1          118.28022162236736    Low_spent_Large_value_payments   \n",
       "2             81.699521264648   Low_spent_Medium_value_payments   \n",
       "3           199.4580743910713    Low_spent_Small_value_payments   \n",
       "4          41.420153086217326  High_spent_Medium_value_payments   \n",
       "...                       ...                               ...   \n",
       "99995       60.97133255718485   High_spent_Large_value_payments   \n",
       "99996       54.18595028760385  High_spent_Medium_value_payments   \n",
       "99997       24.02847744864441   High_spent_Large_value_payments   \n",
       "99998      251.67258219721603    Low_spent_Large_value_payments   \n",
       "99999       167.1638651610451                            !@9#%8   \n",
       "\n",
       "          Monthly_Balance Credit_Score  \n",
       "0      312.49408867943663         Good  \n",
       "1      284.62916249607184         Good  \n",
       "2       331.2098628537912         Good  \n",
       "3      223.45130972736786         Good  \n",
       "4      341.48923103222177         Good  \n",
       "...                   ...          ...  \n",
       "99995          479.866228         Poor  \n",
       "99996           496.65161         Poor  \n",
       "99997          516.809083         Poor  \n",
       "99998          319.164979     Standard  \n",
       "99999          393.673696         Poor  \n",
       "\n",
       "[100000 rows x 28 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read in CSV to df\n",
    "\n",
    "file_path = Path(\"Resources/train.csv\")\n",
    "df_train = pd.read_csv(file_path)\n",
    "\n",
    "df_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "General Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ID                              0\n",
       "Customer_ID                     0\n",
       "Month                           0\n",
       "Name                         9985\n",
       "Age                             0\n",
       "SSN                             0\n",
       "Occupation                      0\n",
       "Annual_Income                   0\n",
       "Monthly_Inhand_Salary       15002\n",
       "Num_Bank_Accounts               0\n",
       "Num_Credit_Card                 0\n",
       "Interest_Rate                   0\n",
       "Num_of_Loan                     0\n",
       "Type_of_Loan                11408\n",
       "Delay_from_due_date             0\n",
       "Num_of_Delayed_Payment       7002\n",
       "Changed_Credit_Limit            0\n",
       "Num_Credit_Inquiries         1965\n",
       "Credit_Mix                      0\n",
       "Outstanding_Debt                0\n",
       "Credit_Utilization_Ratio        0\n",
       "Credit_History_Age           9030\n",
       "Payment_of_Min_Amount           0\n",
       "Total_EMI_per_month             0\n",
       "Amount_invested_monthly      4479\n",
       "Payment_Behaviour               0\n",
       "Monthly_Balance              1200\n",
       "Credit_Score                    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dtm # of NaN values in each column\n",
    "list_of_columns = ['ID', 'Customer_ID', 'Month', 'Name', 'Age', 'SSN', 'Occupation', 'Annual_Income', 'Monthly_Inhand_Salary',\n",
    "                   'Num_Bank_Accounts', 'Num_Credit_Card', 'Interest_Rate', 'Num_of_Loan', 'Type_of_Loan', 'Delay_from_due_date',\n",
    "                   'Num_of_Delayed_Payment', 'Changed_Credit_Limit', 'Num_Credit_Inquiries', 'Credit_Mix', 'Outstanding_Debt',\n",
    "                   'Credit_Utilization_Ratio', 'Credit_History_Age', 'Payment_of_Min_Amount', 'Total_EMI_per_month', 'Amount_invested_monthly',\n",
    "                   'Payment_Behaviour', 'Monthly_Balance', 'Credit_Score']\n",
    "\n",
    "\n",
    "nan_counts = df_train[list_of_columns].isna().sum()\n",
    "nan_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>...</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "      <th>Credit_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>809.98</td>\n",
       "      <td>26.822620</td>\n",
       "      <td>22 Years and 1 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>80.41529543900253</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>312.49408867943663</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>31.944960</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>118.28022162236736</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>284.62916249607184</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-500</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>_</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>28.609352</td>\n",
       "      <td>22 Years and 3 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>81.699521264648</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>331.2098628537912</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>6.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>31.377862</td>\n",
       "      <td>22 Years and 4 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>199.4580743910713</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>223.45130972736786</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>24.797347</td>\n",
       "      <td>22 Years and 5 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>41.420153086217326</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>341.48923103222177</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "      <td>7</td>\n",
       "      <td>11.5</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.663572</td>\n",
       "      <td>31 Years and 6 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>60.97133255718485</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>479.866228</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>7</td>\n",
       "      <td>11.5</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>40.565631</td>\n",
       "      <td>31 Years and 7 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>54.18595028760385</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>496.65161</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>5729</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>6</td>\n",
       "      <td>11.5</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>41.255522</td>\n",
       "      <td>31 Years and 8 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>24.02847744864441</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>516.809083</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11.5</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>33.638208</td>\n",
       "      <td>31 Years and 9 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>251.67258219721603</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>319.164979</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99_</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>6</td>\n",
       "      <td>11.5</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.192463</td>\n",
       "      <td>31 Years and 10 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>167.1638651610451</td>\n",
       "      <td>!@9#%8</td>\n",
       "      <td>393.673696</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100000 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Age Occupation Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0        23  Scientist      19114.12                  3                4   \n",
       "1        23  Scientist      19114.12                  3                4   \n",
       "2      -500  Scientist      19114.12                  3                4   \n",
       "3        23  Scientist      19114.12                  3                4   \n",
       "4        23  Scientist      19114.12                  3                4   \n",
       "...     ...        ...           ...                ...              ...   \n",
       "99995    25   Mechanic      39628.99                  4                6   \n",
       "99996    25   Mechanic      39628.99                  4                6   \n",
       "99997    25   Mechanic      39628.99                  4                6   \n",
       "99998    25   Mechanic      39628.99                  4                6   \n",
       "99999    25   Mechanic     39628.99_                  4                6   \n",
       "\n",
       "       Interest_Rate Num_of_Loan  Delay_from_due_date Num_of_Delayed_Payment  \\\n",
       "0                  3           4                    3                      7   \n",
       "1                  3           4                   -1                    NaN   \n",
       "2                  3           4                    3                      7   \n",
       "3                  3           4                    5                      4   \n",
       "4                  3           4                    6                    NaN   \n",
       "...              ...         ...                  ...                    ...   \n",
       "99995              7           2                   23                      7   \n",
       "99996              7           2                   18                      7   \n",
       "99997           5729           2                   27                      6   \n",
       "99998              7           2                   20                    NaN   \n",
       "99999              7           2                   18                      6   \n",
       "\n",
       "      Changed_Credit_Limit  ...  Credit_Mix Outstanding_Debt  \\\n",
       "0                    11.27  ...           _           809.98   \n",
       "1                    11.27  ...        Good           809.98   \n",
       "2                        _  ...        Good           809.98   \n",
       "3                     6.27  ...        Good           809.98   \n",
       "4                    11.27  ...        Good           809.98   \n",
       "...                    ...  ...         ...              ...   \n",
       "99995                 11.5  ...           _           502.38   \n",
       "99996                 11.5  ...           _           502.38   \n",
       "99997                 11.5  ...        Good           502.38   \n",
       "99998                 11.5  ...        Good           502.38   \n",
       "99999                 11.5  ...        Good           502.38   \n",
       "\n",
       "      Credit_Utilization_Ratio      Credit_History_Age Payment_of_Min_Amount  \\\n",
       "0                    26.822620   22 Years and 1 Months                    No   \n",
       "1                    31.944960                     NaN                    No   \n",
       "2                    28.609352   22 Years and 3 Months                    No   \n",
       "3                    31.377862   22 Years and 4 Months                    No   \n",
       "4                    24.797347   22 Years and 5 Months                    No   \n",
       "...                        ...                     ...                   ...   \n",
       "99995                34.663572   31 Years and 6 Months                    No   \n",
       "99996                40.565631   31 Years and 7 Months                    No   \n",
       "99997                41.255522   31 Years and 8 Months                    No   \n",
       "99998                33.638208   31 Years and 9 Months                    No   \n",
       "99999                34.192463  31 Years and 10 Months                    No   \n",
       "\n",
       "      Total_EMI_per_month  Amount_invested_monthly  \\\n",
       "0               49.574949        80.41529543900253   \n",
       "1               49.574949       118.28022162236736   \n",
       "2               49.574949          81.699521264648   \n",
       "3               49.574949        199.4580743910713   \n",
       "4               49.574949       41.420153086217326   \n",
       "...                   ...                      ...   \n",
       "99995           35.104023        60.97133255718485   \n",
       "99996           35.104023        54.18595028760385   \n",
       "99997           35.104023        24.02847744864441   \n",
       "99998           35.104023       251.67258219721603   \n",
       "99999           35.104023        167.1638651610451   \n",
       "\n",
       "                      Payment_Behaviour     Monthly_Balance Credit_Score  \n",
       "0       High_spent_Small_value_payments  312.49408867943663         Good  \n",
       "1        Low_spent_Large_value_payments  284.62916249607184         Good  \n",
       "2       Low_spent_Medium_value_payments   331.2098628537912         Good  \n",
       "3        Low_spent_Small_value_payments  223.45130972736786         Good  \n",
       "4      High_spent_Medium_value_payments  341.48923103222177         Good  \n",
       "...                                 ...                 ...          ...  \n",
       "99995   High_spent_Large_value_payments          479.866228         Poor  \n",
       "99996  High_spent_Medium_value_payments           496.65161         Poor  \n",
       "99997   High_spent_Large_value_payments          516.809083         Poor  \n",
       "99998    Low_spent_Large_value_payments          319.164979     Standard  \n",
       "99999                            !@9#%8          393.673696         Poor  \n",
       "\n",
       "[100000 rows x 21 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop duplicate or unnecessary info for classsification model\n",
    "# Name and SSN are duplicate due to Customer_ID\n",
    "columns_to_drop = ['Month', 'Name', 'SSN', 'Monthly_Inhand_Salary', 'Type_of_Loan', 'Customer_ID', 'ID']\n",
    "df_train.drop(columns=columns_to_drop, inplace=True)\n",
    "\n",
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>...</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "      <th>Credit_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>809.98</td>\n",
       "      <td>26.822620</td>\n",
       "      <td>22 Years and 1 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>80.41529543900253</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>312.49408867943663</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-500</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>_</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>28.609352</td>\n",
       "      <td>22 Years and 3 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>81.699521264648</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>331.2098628537912</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>6.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>31.377862</td>\n",
       "      <td>22 Years and 4 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>199.4580743910713</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>223.45130972736786</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>9.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>27.262259</td>\n",
       "      <td>22 Years and 6 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>62.430172331195294</td>\n",
       "      <td>!@9#%8</td>\n",
       "      <td>340.4792117872438</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>8_</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>22.537593</td>\n",
       "      <td>22 Years and 7 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>178.3440674122349</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>244.5653167062043</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99994</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>9.5</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.323569</td>\n",
       "      <td>31 Years and 5 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>140.58140274528395</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>410.256158</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "      <td>7</td>\n",
       "      <td>11.5</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.663572</td>\n",
       "      <td>31 Years and 6 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>60.97133255718485</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>479.866228</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>7</td>\n",
       "      <td>11.5</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>40.565631</td>\n",
       "      <td>31 Years and 7 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>54.18595028760385</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>496.65161</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>5729</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>6</td>\n",
       "      <td>11.5</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>41.255522</td>\n",
       "      <td>31 Years and 8 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>24.02847744864441</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>516.809083</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99_</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>6</td>\n",
       "      <td>11.5</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.192463</td>\n",
       "      <td>31 Years and 10 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>167.1638651610451</td>\n",
       "      <td>!@9#%8</td>\n",
       "      <td>393.673696</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>78287 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Age Occupation Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0        23  Scientist      19114.12                  3                4   \n",
       "2      -500  Scientist      19114.12                  3                4   \n",
       "3        23  Scientist      19114.12                  3                4   \n",
       "5        23  Scientist      19114.12                  3                4   \n",
       "6        23  Scientist      19114.12                  3                4   \n",
       "...     ...        ...           ...                ...              ...   \n",
       "99994    25   Mechanic      39628.99                  4                6   \n",
       "99995    25   Mechanic      39628.99                  4                6   \n",
       "99996    25   Mechanic      39628.99                  4                6   \n",
       "99997    25   Mechanic      39628.99                  4                6   \n",
       "99999    25   Mechanic     39628.99_                  4                6   \n",
       "\n",
       "       Interest_Rate Num_of_Loan  Delay_from_due_date Num_of_Delayed_Payment  \\\n",
       "0                  3           4                    3                      7   \n",
       "2                  3           4                    3                      7   \n",
       "3                  3           4                    5                      4   \n",
       "5                  3           4                    8                      4   \n",
       "6                  3           4                    3                     8_   \n",
       "...              ...         ...                  ...                    ...   \n",
       "99994              7           2                   20                      6   \n",
       "99995              7           2                   23                      7   \n",
       "99996              7           2                   18                      7   \n",
       "99997           5729           2                   27                      6   \n",
       "99999              7           2                   18                      6   \n",
       "\n",
       "      Changed_Credit_Limit  ...  Credit_Mix Outstanding_Debt  \\\n",
       "0                    11.27  ...           _           809.98   \n",
       "2                        _  ...        Good           809.98   \n",
       "3                     6.27  ...        Good           809.98   \n",
       "5                     9.27  ...        Good           809.98   \n",
       "6                    11.27  ...        Good           809.98   \n",
       "...                    ...  ...         ...              ...   \n",
       "99994                  9.5  ...           _           502.38   \n",
       "99995                 11.5  ...           _           502.38   \n",
       "99996                 11.5  ...           _           502.38   \n",
       "99997                 11.5  ...        Good           502.38   \n",
       "99999                 11.5  ...        Good           502.38   \n",
       "\n",
       "      Credit_Utilization_Ratio      Credit_History_Age Payment_of_Min_Amount  \\\n",
       "0                    26.822620   22 Years and 1 Months                    No   \n",
       "2                    28.609352   22 Years and 3 Months                    No   \n",
       "3                    31.377862   22 Years and 4 Months                    No   \n",
       "5                    27.262259   22 Years and 6 Months                    No   \n",
       "6                    22.537593   22 Years and 7 Months                    No   \n",
       "...                        ...                     ...                   ...   \n",
       "99994                39.323569   31 Years and 5 Months                    No   \n",
       "99995                34.663572   31 Years and 6 Months                    No   \n",
       "99996                40.565631   31 Years and 7 Months                    No   \n",
       "99997                41.255522   31 Years and 8 Months                    No   \n",
       "99999                34.192463  31 Years and 10 Months                    No   \n",
       "\n",
       "      Total_EMI_per_month  Amount_invested_monthly  \\\n",
       "0               49.574949        80.41529543900253   \n",
       "2               49.574949          81.699521264648   \n",
       "3               49.574949        199.4580743910713   \n",
       "5               49.574949       62.430172331195294   \n",
       "6               49.574949        178.3440674122349   \n",
       "...                   ...                      ...   \n",
       "99994           35.104023       140.58140274528395   \n",
       "99995           35.104023        60.97133255718485   \n",
       "99996           35.104023        54.18595028760385   \n",
       "99997           35.104023        24.02847744864441   \n",
       "99999           35.104023        167.1638651610451   \n",
       "\n",
       "                      Payment_Behaviour     Monthly_Balance Credit_Score  \n",
       "0       High_spent_Small_value_payments  312.49408867943663         Good  \n",
       "2       Low_spent_Medium_value_payments   331.2098628537912         Good  \n",
       "3        Low_spent_Small_value_payments  223.45130972736786         Good  \n",
       "5                                !@9#%8   340.4792117872438         Good  \n",
       "6        Low_spent_Small_value_payments   244.5653167062043         Good  \n",
       "...                                 ...                 ...          ...  \n",
       "99994  High_spent_Medium_value_payments          410.256158         Poor  \n",
       "99995   High_spent_Large_value_payments          479.866228         Poor  \n",
       "99996  High_spent_Medium_value_payments           496.65161         Poor  \n",
       "99997   High_spent_Large_value_payments          516.809083         Poor  \n",
       "99999                            !@9#%8          393.673696         Poor  \n",
       "\n",
       "[78287 rows x 21 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop any row with null value NaN\n",
    "df_train.dropna(inplace=True)\n",
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>...</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "      <th>Credit_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>809.98</td>\n",
       "      <td>26.82</td>\n",
       "      <td>22 Years and 1 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>80.41529543900253</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>312.49408867943663</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-500</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>_</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>28.61</td>\n",
       "      <td>22 Years and 3 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>81.699521264648</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>331.2098628537912</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>6.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>31.38</td>\n",
       "      <td>22 Years and 4 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>199.4580743910713</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>223.45130972736786</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Age Occupation Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0    23  Scientist      19114.12                  3                4   \n",
       "2  -500  Scientist      19114.12                  3                4   \n",
       "3    23  Scientist      19114.12                  3                4   \n",
       "\n",
       "   Interest_Rate Num_of_Loan  Delay_from_due_date Num_of_Delayed_Payment  \\\n",
       "0              3           4                    3                      7   \n",
       "2              3           4                    3                      7   \n",
       "3              3           4                    5                      4   \n",
       "\n",
       "  Changed_Credit_Limit  ...  Credit_Mix Outstanding_Debt  \\\n",
       "0                11.27  ...           _           809.98   \n",
       "2                    _  ...        Good           809.98   \n",
       "3                 6.27  ...        Good           809.98   \n",
       "\n",
       "  Credit_Utilization_Ratio     Credit_History_Age Payment_of_Min_Amount  \\\n",
       "0                    26.82  22 Years and 1 Months                    No   \n",
       "2                    28.61  22 Years and 3 Months                    No   \n",
       "3                    31.38  22 Years and 4 Months                    No   \n",
       "\n",
       "  Total_EMI_per_month  Amount_invested_monthly  \\\n",
       "0           49.574949        80.41529543900253   \n",
       "2           49.574949          81.699521264648   \n",
       "3           49.574949        199.4580743910713   \n",
       "\n",
       "                 Payment_Behaviour     Monthly_Balance Credit_Score  \n",
       "0  High_spent_Small_value_payments  312.49408867943663         Good  \n",
       "2  Low_spent_Medium_value_payments   331.2098628537912         Good  \n",
       "3   Low_spent_Small_value_payments  223.45130972736786         Good  \n",
       "\n",
       "[3 rows x 21 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# clean Credit_Utilization_Ratio\n",
    "\n",
    "# cast as float & round\n",
    "df_train[\"Credit_Utilization_Ratio\"] = df_train[\"Credit_Utilization_Ratio\"].astype(float)\n",
    "df_train[\"Credit_Utilization_Ratio\"] = df_train[\"Credit_Utilization_Ratio\"].round(2)\n",
    "\n",
    "df_train.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>...</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "      <th>Credit_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>809.98</td>\n",
       "      <td>26.82</td>\n",
       "      <td>22.1</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>80.41529543900253</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>312.49408867943663</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-500</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>_</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>28.61</td>\n",
       "      <td>22.2</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>81.699521264648</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>331.2098628537912</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>6.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>31.38</td>\n",
       "      <td>22.3</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>199.4580743910713</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>223.45130972736786</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Age Occupation Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0    23  Scientist      19114.12                  3                4   \n",
       "2  -500  Scientist      19114.12                  3                4   \n",
       "3    23  Scientist      19114.12                  3                4   \n",
       "\n",
       "   Interest_Rate Num_of_Loan  Delay_from_due_date Num_of_Delayed_Payment  \\\n",
       "0              3           4                    3                      7   \n",
       "2              3           4                    3                      7   \n",
       "3              3           4                    5                      4   \n",
       "\n",
       "  Changed_Credit_Limit  ...  Credit_Mix Outstanding_Debt  \\\n",
       "0                11.27  ...           _           809.98   \n",
       "2                    _  ...        Good           809.98   \n",
       "3                 6.27  ...        Good           809.98   \n",
       "\n",
       "  Credit_Utilization_Ratio  Credit_History_Age  Payment_of_Min_Amount  \\\n",
       "0                    26.82                22.1                     No   \n",
       "2                    28.61                22.2                     No   \n",
       "3                    31.38                22.3                     No   \n",
       "\n",
       "  Total_EMI_per_month  Amount_invested_monthly  \\\n",
       "0           49.574949        80.41529543900253   \n",
       "2           49.574949          81.699521264648   \n",
       "3           49.574949        199.4580743910713   \n",
       "\n",
       "                 Payment_Behaviour     Monthly_Balance Credit_Score  \n",
       "0  High_spent_Small_value_payments  312.49408867943663         Good  \n",
       "2  Low_spent_Medium_value_payments   331.2098628537912         Good  \n",
       "3   Low_spent_Small_value_payments  223.45130972736786         Good  \n",
       "\n",
       "[3 rows x 21 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# convert credit history to numerical\n",
    "\n",
    "# Function to convert age string to total years\n",
    "def convert_age_to_years(age_str):\n",
    "    years, months = map(str.strip, age_str.split('Years and'))\n",
    "    total_months = int(years) * 12 + int(months.split()[0])\n",
    "    total_years = total_months / 12  # Convert total months to years\n",
    "    return round(total_years, 1)  # Round to 1 decimal place\n",
    "\n",
    "# Convert age strings to total years\n",
    "df_train['Credit_History_Age'] = df_train['Credit_History_Age'].apply(convert_age_to_years).astype(float)\n",
    "\n",
    "df_train.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>...</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "      <th>Credit_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>809.98</td>\n",
       "      <td>26.82</td>\n",
       "      <td>22.1</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>80.41529543900253</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>312.49408867943663</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-500</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>_</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>28.61</td>\n",
       "      <td>22.2</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>81.699521264648</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>331.2098628537912</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>6.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>31.38</td>\n",
       "      <td>22.3</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>199.4580743910713</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>223.45130972736786</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>9.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>27.26</td>\n",
       "      <td>22.5</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>62.430172331195294</td>\n",
       "      <td>!@9#%8</td>\n",
       "      <td>340.4792117872438</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>8_</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>22.54</td>\n",
       "      <td>22.6</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>178.3440674122349</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>244.5653167062043</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99994</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>9.5</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.32</td>\n",
       "      <td>31.4</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>140.58140274528395</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>410.256158</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "      <td>7</td>\n",
       "      <td>11.5</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.66</td>\n",
       "      <td>31.5</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>60.97133255718485</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>479.866228</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>7</td>\n",
       "      <td>11.5</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>40.57</td>\n",
       "      <td>31.6</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>54.18595028760385</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>496.65161</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>5729</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>6</td>\n",
       "      <td>11.5</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>41.26</td>\n",
       "      <td>31.7</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>24.02847744864441</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>516.809083</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99_</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>6</td>\n",
       "      <td>11.5</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.19</td>\n",
       "      <td>31.8</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>167.1638651610451</td>\n",
       "      <td>!@9#%8</td>\n",
       "      <td>393.673696</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>78287 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Age Occupation Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0        23  Scientist      19114.12                  3                4   \n",
       "2      -500  Scientist      19114.12                  3                4   \n",
       "3        23  Scientist      19114.12                  3                4   \n",
       "5        23  Scientist      19114.12                  3                4   \n",
       "6        23  Scientist      19114.12                  3                4   \n",
       "...     ...        ...           ...                ...              ...   \n",
       "99994    25   Mechanic      39628.99                  4                6   \n",
       "99995    25   Mechanic      39628.99                  4                6   \n",
       "99996    25   Mechanic      39628.99                  4                6   \n",
       "99997    25   Mechanic      39628.99                  4                6   \n",
       "99999    25   Mechanic     39628.99_                  4                6   \n",
       "\n",
       "       Interest_Rate Num_of_Loan  Delay_from_due_date Num_of_Delayed_Payment  \\\n",
       "0                  3           4                    3                      7   \n",
       "2                  3           4                    3                      7   \n",
       "3                  3           4                    5                      4   \n",
       "5                  3           4                    8                      4   \n",
       "6                  3           4                    3                     8_   \n",
       "...              ...         ...                  ...                    ...   \n",
       "99994              7           2                   20                      6   \n",
       "99995              7           2                   23                      7   \n",
       "99996              7           2                   18                      7   \n",
       "99997           5729           2                   27                      6   \n",
       "99999              7           2                   18                      6   \n",
       "\n",
       "      Changed_Credit_Limit  ...  Credit_Mix Outstanding_Debt  \\\n",
       "0                    11.27  ...           _           809.98   \n",
       "2                        _  ...        Good           809.98   \n",
       "3                     6.27  ...        Good           809.98   \n",
       "5                     9.27  ...        Good           809.98   \n",
       "6                    11.27  ...        Good           809.98   \n",
       "...                    ...  ...         ...              ...   \n",
       "99994                  9.5  ...           _           502.38   \n",
       "99995                 11.5  ...           _           502.38   \n",
       "99996                 11.5  ...           _           502.38   \n",
       "99997                 11.5  ...        Good           502.38   \n",
       "99999                 11.5  ...        Good           502.38   \n",
       "\n",
       "      Credit_Utilization_Ratio  Credit_History_Age  Payment_of_Min_Amount  \\\n",
       "0                        26.82                22.1                     No   \n",
       "2                        28.61                22.2                     No   \n",
       "3                        31.38                22.3                     No   \n",
       "5                        27.26                22.5                     No   \n",
       "6                        22.54                22.6                     No   \n",
       "...                        ...                 ...                    ...   \n",
       "99994                    39.32                31.4                     No   \n",
       "99995                    34.66                31.5                     No   \n",
       "99996                    40.57                31.6                     No   \n",
       "99997                    41.26                31.7                     No   \n",
       "99999                    34.19                31.8                     No   \n",
       "\n",
       "      Total_EMI_per_month  Amount_invested_monthly  \\\n",
       "0               49.574949        80.41529543900253   \n",
       "2               49.574949          81.699521264648   \n",
       "3               49.574949        199.4580743910713   \n",
       "5               49.574949       62.430172331195294   \n",
       "6               49.574949        178.3440674122349   \n",
       "...                   ...                      ...   \n",
       "99994           35.104023       140.58140274528395   \n",
       "99995           35.104023        60.97133255718485   \n",
       "99996           35.104023        54.18595028760385   \n",
       "99997           35.104023        24.02847744864441   \n",
       "99999           35.104023        167.1638651610451   \n",
       "\n",
       "                      Payment_Behaviour     Monthly_Balance Credit_Score  \n",
       "0       High_spent_Small_value_payments  312.49408867943663         Good  \n",
       "2       Low_spent_Medium_value_payments   331.2098628537912         Good  \n",
       "3        Low_spent_Small_value_payments  223.45130972736786         Good  \n",
       "5                                !@9#%8   340.4792117872438         Good  \n",
       "6        Low_spent_Small_value_payments   244.5653167062043         Good  \n",
       "...                                 ...                 ...          ...  \n",
       "99994  High_spent_Medium_value_payments          410.256158         Poor  \n",
       "99995   High_spent_Large_value_payments          479.866228         Poor  \n",
       "99996  High_spent_Medium_value_payments           496.65161         Poor  \n",
       "99997   High_spent_Large_value_payments          516.809083         Poor  \n",
       "99999                            !@9#%8          393.673696         Poor  \n",
       "\n",
       "[78287 rows x 21 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use Regular Expressions to Remove Typos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>...</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "      <th>Credit_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>809.98</td>\n",
       "      <td>26.82</td>\n",
       "      <td>22.1</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>80.415295</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>312.494089</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-500</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>_</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>28.61</td>\n",
       "      <td>22.2</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>81.699521</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>331.209863</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>31.38</td>\n",
       "      <td>22.3</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>199.458074</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>223.451310</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>27.26</td>\n",
       "      <td>22.5</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>62.430172</td>\n",
       "      <td>!@9#%8</td>\n",
       "      <td>340.479212</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>22.54</td>\n",
       "      <td>22.6</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>178.344067</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>244.565317</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99994</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>9.5</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.32</td>\n",
       "      <td>31.4</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>140.581403</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>410.256158</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.5</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.66</td>\n",
       "      <td>31.5</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>60.971333</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>479.866228</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.5</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>40.57</td>\n",
       "      <td>31.6</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>54.185950</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>496.651610</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>5729</td>\n",
       "      <td>2.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11.5</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>41.26</td>\n",
       "      <td>31.7</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>24.028477</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>516.809083</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11.5</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.19</td>\n",
       "      <td>31.8</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>167.163865</td>\n",
       "      <td>!@9#%8</td>\n",
       "      <td>393.673696</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>78287 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0       23  Scientist       19114.12                  3                4   \n",
       "2     -500  Scientist       19114.12                  3                4   \n",
       "3       23  Scientist       19114.12                  3                4   \n",
       "5       23  Scientist       19114.12                  3                4   \n",
       "6       23  Scientist       19114.12                  3                4   \n",
       "...    ...        ...            ...                ...              ...   \n",
       "99994   25   Mechanic       39628.99                  4                6   \n",
       "99995   25   Mechanic       39628.99                  4                6   \n",
       "99996   25   Mechanic       39628.99                  4                6   \n",
       "99997   25   Mechanic       39628.99                  4                6   \n",
       "99999   25   Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "0                  3          4.0                  3.0   \n",
       "2                  3          4.0                  3.0   \n",
       "3                  3          4.0                  5.0   \n",
       "5                  3          4.0                  8.0   \n",
       "6                  3          4.0                  3.0   \n",
       "...              ...          ...                  ...   \n",
       "99994              7          2.0                 20.0   \n",
       "99995              7          2.0                 23.0   \n",
       "99996              7          2.0                 18.0   \n",
       "99997           5729          2.0                 27.0   \n",
       "99999              7          2.0                 18.0   \n",
       "\n",
       "       Num_of_Delayed_Payment Changed_Credit_Limit  ...  Credit_Mix  \\\n",
       "0                         7.0                11.27  ...           _   \n",
       "2                         7.0                    _  ...        Good   \n",
       "3                         4.0                 6.27  ...        Good   \n",
       "5                         4.0                 9.27  ...        Good   \n",
       "6                         8.0                11.27  ...        Good   \n",
       "...                       ...                  ...  ...         ...   \n",
       "99994                     6.0                  9.5  ...           _   \n",
       "99995                     7.0                 11.5  ...           _   \n",
       "99996                     7.0                 11.5  ...           _   \n",
       "99997                     6.0                 11.5  ...        Good   \n",
       "99999                     6.0                 11.5  ...        Good   \n",
       "\n",
       "      Outstanding_Debt  Credit_Utilization_Ratio  Credit_History_Age  \\\n",
       "0               809.98                     26.82                22.1   \n",
       "2               809.98                     28.61                22.2   \n",
       "3               809.98                     31.38                22.3   \n",
       "5               809.98                     27.26                22.5   \n",
       "6               809.98                     22.54                22.6   \n",
       "...                ...                       ...                 ...   \n",
       "99994           502.38                     39.32                31.4   \n",
       "99995           502.38                     34.66                31.5   \n",
       "99996           502.38                     40.57                31.6   \n",
       "99997           502.38                     41.26                31.7   \n",
       "99999           502.38                     34.19                31.8   \n",
       "\n",
       "       Payment_of_Min_Amount Total_EMI_per_month  Amount_invested_monthly  \\\n",
       "0                         No           49.574949                80.415295   \n",
       "2                         No           49.574949                81.699521   \n",
       "3                         No           49.574949               199.458074   \n",
       "5                         No           49.574949                62.430172   \n",
       "6                         No           49.574949               178.344067   \n",
       "...                      ...                 ...                      ...   \n",
       "99994                     No           35.104023               140.581403   \n",
       "99995                     No           35.104023                60.971333   \n",
       "99996                     No           35.104023                54.185950   \n",
       "99997                     No           35.104023                24.028477   \n",
       "99999                     No           35.104023               167.163865   \n",
       "\n",
       "                      Payment_Behaviour Monthly_Balance  Credit_Score  \n",
       "0       High_spent_Small_value_payments      312.494089          Good  \n",
       "2       Low_spent_Medium_value_payments      331.209863          Good  \n",
       "3        Low_spent_Small_value_payments      223.451310          Good  \n",
       "5                                !@9#%8      340.479212          Good  \n",
       "6        Low_spent_Small_value_payments      244.565317          Good  \n",
       "...                                 ...             ...           ...  \n",
       "99994  High_spent_Medium_value_payments      410.256158          Poor  \n",
       "99995   High_spent_Large_value_payments      479.866228          Poor  \n",
       "99996  High_spent_Medium_value_payments      496.651610          Poor  \n",
       "99997   High_spent_Large_value_payments      516.809083          Poor  \n",
       "99999                            !@9#%8      393.673696          Poor  \n",
       "\n",
       "[78287 rows x 21 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "columns_to_regex = ['Annual_Income', 'Age', 'Delay_from_due_date', 'Num_of_Loan','Amount_invested_monthly', 'Monthly_Balance', 'Num_of_Delayed_Payment', 'Outstanding_Debt']\n",
    "\n",
    "for column in columns_to_regex:\n",
    "    df_train[column] = df_train[column].astype(str).str.replace('_', '')\n",
    "    df_train[column] = df_train[column].astype(float)  # Convert the column back to float\n",
    "\n",
    "df_train['Age'] = df_train['Age'].astype(int)\n",
    "\n",
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 78287 entries, 0 to 99999\n",
      "Data columns (total 21 columns):\n",
      " #   Column                    Non-Null Count  Dtype  \n",
      "---  ------                    --------------  -----  \n",
      " 0   Age                       78287 non-null  int64  \n",
      " 1   Occupation                78287 non-null  object \n",
      " 2   Annual_Income             78287 non-null  float64\n",
      " 3   Num_Bank_Accounts         78287 non-null  int64  \n",
      " 4   Num_Credit_Card           78287 non-null  int64  \n",
      " 5   Interest_Rate             78287 non-null  int64  \n",
      " 6   Num_of_Loan               78287 non-null  float64\n",
      " 7   Delay_from_due_date       78287 non-null  float64\n",
      " 8   Num_of_Delayed_Payment    78287 non-null  float64\n",
      " 9   Changed_Credit_Limit      78287 non-null  object \n",
      " 10  Num_Credit_Inquiries      78287 non-null  float64\n",
      " 11  Credit_Mix                78287 non-null  object \n",
      " 12  Outstanding_Debt          78287 non-null  float64\n",
      " 13  Credit_Utilization_Ratio  78287 non-null  float64\n",
      " 14  Credit_History_Age        78287 non-null  float64\n",
      " 15  Payment_of_Min_Amount     78287 non-null  object \n",
      " 16  Total_EMI_per_month       78287 non-null  float64\n",
      " 17  Amount_invested_monthly   78287 non-null  float64\n",
      " 18  Payment_Behaviour         78287 non-null  object \n",
      " 19  Monthly_Balance           78287 non-null  float64\n",
      " 20  Credit_Score              78287 non-null  object \n",
      "dtypes: float64(11), int64(4), object(6)\n",
      "memory usage: 13.1+ MB\n"
     ]
    }
   ],
   "source": [
    "df_train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace nonnumeric values with numeric values (to allow casting of variables as float in next block)\n",
    "df_train['Changed_Credit_Limit'] = df_train['Changed_Credit_Limit'].replace('-', '')\n",
    "df_train['Changed_Credit_Limit'] = pd.to_numeric(df_train['Changed_Credit_Limit'], errors='coerce')\n",
    "df_train['Changed_Credit_Limit'] = df_train['Changed_Credit_Limit'].fillna(0) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>...</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "      <th>Credit_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>809.98</td>\n",
       "      <td>26.82</td>\n",
       "      <td>22.1</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>80.42</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>312.49</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-500</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>28.61</td>\n",
       "      <td>22.2</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>81.70</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>331.21</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>31.38</td>\n",
       "      <td>22.3</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>199.46</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>223.45</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>27.26</td>\n",
       "      <td>22.5</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>62.43</td>\n",
       "      <td>!@9#%8</td>\n",
       "      <td>340.48</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>22.54</td>\n",
       "      <td>22.6</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>178.34</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>244.57</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99994</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>9.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.32</td>\n",
       "      <td>31.4</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>140.58</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>410.26</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.66</td>\n",
       "      <td>31.5</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>60.97</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>479.87</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>40.57</td>\n",
       "      <td>31.6</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>54.19</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>496.65</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>5729</td>\n",
       "      <td>2.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>41.26</td>\n",
       "      <td>31.7</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>24.03</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>516.81</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.19</td>\n",
       "      <td>31.8</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>167.16</td>\n",
       "      <td>!@9#%8</td>\n",
       "      <td>393.67</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>78287 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0       23  Scientist       19114.12                  3                4   \n",
       "2     -500  Scientist       19114.12                  3                4   \n",
       "3       23  Scientist       19114.12                  3                4   \n",
       "5       23  Scientist       19114.12                  3                4   \n",
       "6       23  Scientist       19114.12                  3                4   \n",
       "...    ...        ...            ...                ...              ...   \n",
       "99994   25   Mechanic       39628.99                  4                6   \n",
       "99995   25   Mechanic       39628.99                  4                6   \n",
       "99996   25   Mechanic       39628.99                  4                6   \n",
       "99997   25   Mechanic       39628.99                  4                6   \n",
       "99999   25   Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "0                  3          4.0                  3.0   \n",
       "2                  3          4.0                  3.0   \n",
       "3                  3          4.0                  5.0   \n",
       "5                  3          4.0                  8.0   \n",
       "6                  3          4.0                  3.0   \n",
       "...              ...          ...                  ...   \n",
       "99994              7          2.0                 20.0   \n",
       "99995              7          2.0                 23.0   \n",
       "99996              7          2.0                 18.0   \n",
       "99997           5729          2.0                 27.0   \n",
       "99999              7          2.0                 18.0   \n",
       "\n",
       "       Num_of_Delayed_Payment  Changed_Credit_Limit  ...  Credit_Mix  \\\n",
       "0                         7.0                 11.27  ...           _   \n",
       "2                         7.0                  0.00  ...        Good   \n",
       "3                         4.0                  6.27  ...        Good   \n",
       "5                         4.0                  9.27  ...        Good   \n",
       "6                         8.0                 11.27  ...        Good   \n",
       "...                       ...                   ...  ...         ...   \n",
       "99994                     6.0                  9.50  ...           _   \n",
       "99995                     7.0                 11.50  ...           _   \n",
       "99996                     7.0                 11.50  ...           _   \n",
       "99997                     6.0                 11.50  ...        Good   \n",
       "99999                     6.0                 11.50  ...        Good   \n",
       "\n",
       "      Outstanding_Debt  Credit_Utilization_Ratio  Credit_History_Age  \\\n",
       "0               809.98                     26.82                22.1   \n",
       "2               809.98                     28.61                22.2   \n",
       "3               809.98                     31.38                22.3   \n",
       "5               809.98                     27.26                22.5   \n",
       "6               809.98                     22.54                22.6   \n",
       "...                ...                       ...                 ...   \n",
       "99994           502.38                     39.32                31.4   \n",
       "99995           502.38                     34.66                31.5   \n",
       "99996           502.38                     40.57                31.6   \n",
       "99997           502.38                     41.26                31.7   \n",
       "99999           502.38                     34.19                31.8   \n",
       "\n",
       "       Payment_of_Min_Amount Total_EMI_per_month  Amount_invested_monthly  \\\n",
       "0                         No               49.57                    80.42   \n",
       "2                         No               49.57                    81.70   \n",
       "3                         No               49.57                   199.46   \n",
       "5                         No               49.57                    62.43   \n",
       "6                         No               49.57                   178.34   \n",
       "...                      ...                 ...                      ...   \n",
       "99994                     No               35.10                   140.58   \n",
       "99995                     No               35.10                    60.97   \n",
       "99996                     No               35.10                    54.19   \n",
       "99997                     No               35.10                    24.03   \n",
       "99999                     No               35.10                   167.16   \n",
       "\n",
       "                      Payment_Behaviour Monthly_Balance  Credit_Score  \n",
       "0       High_spent_Small_value_payments          312.49          Good  \n",
       "2       Low_spent_Medium_value_payments          331.21          Good  \n",
       "3        Low_spent_Small_value_payments          223.45          Good  \n",
       "5                                !@9#%8          340.48          Good  \n",
       "6        Low_spent_Small_value_payments          244.57          Good  \n",
       "...                                 ...             ...           ...  \n",
       "99994  High_spent_Medium_value_payments          410.26          Poor  \n",
       "99995   High_spent_Large_value_payments          479.87          Poor  \n",
       "99996  High_spent_Medium_value_payments          496.65          Poor  \n",
       "99997   High_spent_Large_value_payments          516.81          Poor  \n",
       "99999                            !@9#%8          393.67          Poor  \n",
       "\n",
       "[78287 rows x 21 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# List of columns to round and cast as floats\n",
    "columns_to_round = ['Num_of_Loan', 'Delay_from_due_date', 'Amount_invested_monthly', 'Monthly_Balance', 'Changed_Credit_Limit', 'Total_EMI_per_month', 'Annual_Income']\n",
    "\n",
    "# Round the columns to 2 decimal places and cast them as floats\n",
    "df_train[columns_to_round] = df_train[columns_to_round].round(2).astype(float)\n",
    "\n",
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 78287 entries, 0 to 99999\n",
      "Data columns (total 21 columns):\n",
      " #   Column                    Non-Null Count  Dtype  \n",
      "---  ------                    --------------  -----  \n",
      " 0   Age                       78287 non-null  int64  \n",
      " 1   Occupation                78287 non-null  object \n",
      " 2   Annual_Income             78287 non-null  float64\n",
      " 3   Num_Bank_Accounts         78287 non-null  int64  \n",
      " 4   Num_Credit_Card           78287 non-null  int64  \n",
      " 5   Interest_Rate             78287 non-null  int64  \n",
      " 6   Num_of_Loan               78287 non-null  float64\n",
      " 7   Delay_from_due_date       78287 non-null  float64\n",
      " 8   Num_of_Delayed_Payment    78287 non-null  float64\n",
      " 9   Changed_Credit_Limit      78287 non-null  float64\n",
      " 10  Num_Credit_Inquiries      78287 non-null  float64\n",
      " 11  Credit_Mix                78287 non-null  object \n",
      " 12  Outstanding_Debt          78287 non-null  float64\n",
      " 13  Credit_Utilization_Ratio  78287 non-null  float64\n",
      " 14  Credit_History_Age        78287 non-null  float64\n",
      " 15  Payment_of_Min_Amount     78287 non-null  object \n",
      " 16  Total_EMI_per_month       78287 non-null  float64\n",
      " 17  Amount_invested_monthly   78287 non-null  float64\n",
      " 18  Payment_Behaviour         78287 non-null  object \n",
      " 19  Monthly_Balance           78287 non-null  float64\n",
      " 20  Credit_Score              78287 non-null  object \n",
      "dtypes: float64(12), int64(4), object(5)\n",
      "memory usage: 13.1+ MB\n"
     ]
    }
   ],
   "source": [
    "df_train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        11.27\n",
       "2         0.00\n",
       "3         6.27\n",
       "5         9.27\n",
       "6        11.27\n",
       "         ...  \n",
       "99994     9.50\n",
       "99995    11.50\n",
       "99996    11.50\n",
       "99997    11.50\n",
       "99999    11.50\n",
       "Name: Changed_Credit_Limit, Length: 78287, dtype: float64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train['Changed_Credit_Limit']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove Outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Num_Bank_Accounts\n",
       " 6       10232\n",
       " 7       10112\n",
       " 8        9937\n",
       " 4        9499\n",
       " 5        9464\n",
       " 3        9388\n",
       " 9        4234\n",
       " 10       4087\n",
       " 1        3522\n",
       " 2        3395\n",
       " 0        3372\n",
       "-1          19\n",
       " 105         5\n",
       " 803         5\n",
       " 11          5\n",
       " 857         4\n",
       " 791         4\n",
       " 1668        4\n",
       " 1481        4\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of bank acct outliers\n",
    "\n",
    "# value counts\n",
    "values_counted = df_train['Num_Bank_Accounts'].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find and remove outlier, typo rows for # of bank accts\n",
    "\n",
    "# Define the range of normal values\n",
    "lower_bound = 0\n",
    "upper_bound = 15\n",
    "\n",
    "column = 'Num_Bank_Accounts'\n",
    "\n",
    "# testing variable\n",
    "within_range_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "\n",
    "# used this to play with range and see where cutoff for normal values were\n",
    "outliers_df = df_train[(df_train[column] < lower_bound) | (df_train[column] > upper_bound)]\n",
    "\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "\n",
    "df_train = cleaned_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Num_Credit_Card\n",
       "5       14219\n",
       "7       12890\n",
       "6       12751\n",
       "4       10933\n",
       "3       10236\n",
       "        ...  \n",
       "1178        4\n",
       "1043        4\n",
       "1156        4\n",
       "1077        4\n",
       "249         4\n",
       "Name: count, Length: 65, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of credit card outliers\n",
    "\n",
    "# value counts\n",
    "values_counted = df_train['Num_Credit_Card'].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find and remove outlier, typo rows by # of credit cards\n",
    "\n",
    "# Define the range of normal values\n",
    "lower_bound = 0\n",
    "upper_bound = 16\n",
    "\n",
    "column = 'Num_Credit_Card'\n",
    "\n",
    "# testing variable\n",
    "within_range_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "\n",
    "# used this to play with range and see where cutoff for normal values were\n",
    "outliers_df = df_train[(df_train[column] < lower_bound) | (df_train[column] > upper_bound)]\n",
    "\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "\n",
    "df_train = cleaned_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Interest_Rate\n",
       "8      3790\n",
       "5      3769\n",
       "6      3555\n",
       "12     3427\n",
       "9      3422\n",
       "7      3404\n",
       "10     3401\n",
       "11     3327\n",
       "18     3084\n",
       "15     3056\n",
       "20     2952\n",
       "17     2895\n",
       "16     2778\n",
       "19     2751\n",
       "3      2060\n",
       "1      2045\n",
       "4      1954\n",
       "2      1871\n",
       "13     1788\n",
       "14     1667\n",
       "32     1296\n",
       "24     1287\n",
       "30     1280\n",
       "23     1260\n",
       "29     1258\n",
       "22     1249\n",
       "28     1222\n",
       "27     1209\n",
       "21     1179\n",
       "34     1178\n",
       "25     1172\n",
       "26     1119\n",
       "31     1093\n",
       "33     1081\n",
       "498       4\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of interest rate\n",
    "\n",
    "column = 'Interest_Rate'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_train[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>...</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "      <th>Credit_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>809.98</td>\n",
       "      <td>26.82</td>\n",
       "      <td>22.1</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>80.42</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>312.49</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-500</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>28.61</td>\n",
       "      <td>22.2</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>81.70</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>331.21</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>31.38</td>\n",
       "      <td>22.3</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>199.46</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>223.45</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>27.26</td>\n",
       "      <td>22.5</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>62.43</td>\n",
       "      <td>!@9#%8</td>\n",
       "      <td>340.48</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>22.54</td>\n",
       "      <td>22.6</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>178.34</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>244.57</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99991</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>...</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>37.14</td>\n",
       "      <td>6.2</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>34.66</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>337.36</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99994</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>9.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.32</td>\n",
       "      <td>31.4</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>140.58</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>410.26</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.66</td>\n",
       "      <td>31.5</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>60.97</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>479.87</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>40.57</td>\n",
       "      <td>31.6</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>54.19</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>496.65</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.19</td>\n",
       "      <td>31.8</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>167.16</td>\n",
       "      <td>!@9#%8</td>\n",
       "      <td>393.67</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>73879 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0       23  Scientist       19114.12                  3                4   \n",
       "2     -500  Scientist       19114.12                  3                4   \n",
       "3       23  Scientist       19114.12                  3                4   \n",
       "5       23  Scientist       19114.12                  3                4   \n",
       "6       23  Scientist       19114.12                  3                4   \n",
       "...    ...        ...            ...                ...              ...   \n",
       "99991   29  Architect       20002.88                 10                8   \n",
       "99994   25   Mechanic       39628.99                  4                6   \n",
       "99995   25   Mechanic       39628.99                  4                6   \n",
       "99996   25   Mechanic       39628.99                  4                6   \n",
       "99999   25   Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "0                  3          4.0                  3.0   \n",
       "2                  3          4.0                  3.0   \n",
       "3                  3          4.0                  5.0   \n",
       "5                  3          4.0                  8.0   \n",
       "6                  3          4.0                  3.0   \n",
       "...              ...          ...                  ...   \n",
       "99991             29          5.0                 33.0   \n",
       "99994              7          2.0                 20.0   \n",
       "99995              7          2.0                 23.0   \n",
       "99996              7          2.0                 18.0   \n",
       "99999              7          2.0                 18.0   \n",
       "\n",
       "       Num_of_Delayed_Payment  Changed_Credit_Limit  ...  Credit_Mix  \\\n",
       "0                         7.0                 11.27  ...           _   \n",
       "2                         7.0                  0.00  ...        Good   \n",
       "3                         4.0                  6.27  ...        Good   \n",
       "5                         4.0                  9.27  ...        Good   \n",
       "6                         8.0                 11.27  ...        Good   \n",
       "...                       ...                   ...  ...         ...   \n",
       "99991                    25.0                 18.31  ...         Bad   \n",
       "99994                     6.0                  9.50  ...           _   \n",
       "99995                     7.0                 11.50  ...           _   \n",
       "99996                     7.0                 11.50  ...           _   \n",
       "99999                     6.0                 11.50  ...        Good   \n",
       "\n",
       "      Outstanding_Debt  Credit_Utilization_Ratio  Credit_History_Age  \\\n",
       "0               809.98                     26.82                22.1   \n",
       "2               809.98                     28.61                22.2   \n",
       "3               809.98                     31.38                22.3   \n",
       "5               809.98                     27.26                22.5   \n",
       "6               809.98                     22.54                22.6   \n",
       "...                ...                       ...                 ...   \n",
       "99991          3571.70                     37.14                 6.2   \n",
       "99994           502.38                     39.32                31.4   \n",
       "99995           502.38                     34.66                31.5   \n",
       "99996           502.38                     40.57                31.6   \n",
       "99999           502.38                     34.19                31.8   \n",
       "\n",
       "       Payment_of_Min_Amount Total_EMI_per_month  Amount_invested_monthly  \\\n",
       "0                         No               49.57                    80.42   \n",
       "2                         No               49.57                    81.70   \n",
       "3                         No               49.57                   199.46   \n",
       "5                         No               49.57                    62.43   \n",
       "6                         No               49.57                   178.34   \n",
       "...                      ...                 ...                      ...   \n",
       "99991                    Yes               60.96                    34.66   \n",
       "99994                     No               35.10                   140.58   \n",
       "99995                     No               35.10                    60.97   \n",
       "99996                     No               35.10                    54.19   \n",
       "99999                     No               35.10                   167.16   \n",
       "\n",
       "                      Payment_Behaviour Monthly_Balance  Credit_Score  \n",
       "0       High_spent_Small_value_payments          312.49          Good  \n",
       "2       Low_spent_Medium_value_payments          331.21          Good  \n",
       "3        Low_spent_Small_value_payments          223.45          Good  \n",
       "5                                !@9#%8          340.48          Good  \n",
       "6        Low_spent_Small_value_payments          244.57          Good  \n",
       "...                                 ...             ...           ...  \n",
       "99991   High_spent_Large_value_payments          337.36      Standard  \n",
       "99994  High_spent_Medium_value_payments          410.26          Poor  \n",
       "99995   High_spent_Large_value_payments          479.87          Poor  \n",
       "99996  High_spent_Medium_value_payments          496.65          Poor  \n",
       "99999                            !@9#%8          393.67          Poor  \n",
       "\n",
       "[73879 rows x 21 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  find and remove outlier, typo rows by interest rate\n",
    "\n",
    "# Define the range of normal values\n",
    "lower_bound = 0\n",
    "upper_bound = 40\n",
    "\n",
    "column = 'Interest_Rate'\n",
    "\n",
    "# testing variable\n",
    "within_range_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "\n",
    "# used this to play with range and dtm where cutoff for normal values were\n",
    "outliers_df = df_train[(df_train[column] < lower_bound) | (df_train[column] > upper_bound)]\n",
    "\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "\n",
    "df_train = cleaned_df\n",
    "\n",
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Age\n",
       " 28      2210\n",
       " 26      2181\n",
       " 31      2160\n",
       " 35      2155\n",
       " 38      2154\n",
       " 32      2112\n",
       " 36      2101\n",
       " 27      2094\n",
       " 22      2090\n",
       " 39      2086\n",
       " 41      2084\n",
       " 44      2082\n",
       " 19      2077\n",
       " 25      2068\n",
       " 34      2066\n",
       " 43      2052\n",
       " 20      2044\n",
       " 37      2018\n",
       " 21      2014\n",
       " 23      2011\n",
       " 30      2006\n",
       " 29      1999\n",
       " 24      1995\n",
       " 45      1941\n",
       " 42      1913\n",
       " 33      1874\n",
       " 40      1868\n",
       " 18      1795\n",
       " 46      1202\n",
       " 15      1145\n",
       " 17      1126\n",
       " 16      1092\n",
       " 48      1016\n",
       " 55      1011\n",
       " 49      1009\n",
       " 53      1004\n",
       " 52      1001\n",
       " 50       962\n",
       " 54       959\n",
       " 51       957\n",
       " 47       911\n",
       " 14       891\n",
       "-500      668\n",
       " 56       262\n",
       " 6799       4\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of age\n",
    "\n",
    "column = 'Age'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_train[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>...</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "      <th>Credit_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>809.98</td>\n",
       "      <td>26.82</td>\n",
       "      <td>22.1</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>80.42</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>312.49</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>31.38</td>\n",
       "      <td>22.3</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>199.46</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>223.45</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>27.26</td>\n",
       "      <td>22.5</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>62.43</td>\n",
       "      <td>!@9#%8</td>\n",
       "      <td>340.48</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>22.54</td>\n",
       "      <td>22.6</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>178.34</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>244.57</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>24.46</td>\n",
       "      <td>26.6</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>104.29</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>470.69</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99991</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>...</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>37.14</td>\n",
       "      <td>6.2</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>34.66</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>337.36</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99994</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>9.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.32</td>\n",
       "      <td>31.4</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>140.58</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>410.26</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.66</td>\n",
       "      <td>31.5</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>60.97</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>479.87</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>40.57</td>\n",
       "      <td>31.6</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>54.19</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>496.65</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.19</td>\n",
       "      <td>31.8</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>167.16</td>\n",
       "      <td>!@9#%8</td>\n",
       "      <td>393.67</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>67544 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0       23  Scientist       19114.12                  3                4   \n",
       "3       23  Scientist       19114.12                  3                4   \n",
       "5       23  Scientist       19114.12                  3                4   \n",
       "6       23  Scientist       19114.12                  3                4   \n",
       "8       28    _______       34847.84                  2                4   \n",
       "...    ...        ...            ...                ...              ...   \n",
       "99991   29  Architect       20002.88                 10                8   \n",
       "99994   25   Mechanic       39628.99                  4                6   \n",
       "99995   25   Mechanic       39628.99                  4                6   \n",
       "99996   25   Mechanic       39628.99                  4                6   \n",
       "99999   25   Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "0                  3          4.0                  3.0   \n",
       "3                  3          4.0                  5.0   \n",
       "5                  3          4.0                  8.0   \n",
       "6                  3          4.0                  3.0   \n",
       "8                  6          1.0                  3.0   \n",
       "...              ...          ...                  ...   \n",
       "99991             29          5.0                 33.0   \n",
       "99994              7          2.0                 20.0   \n",
       "99995              7          2.0                 23.0   \n",
       "99996              7          2.0                 18.0   \n",
       "99999              7          2.0                 18.0   \n",
       "\n",
       "       Num_of_Delayed_Payment  Changed_Credit_Limit  ...  Credit_Mix  \\\n",
       "0                         7.0                 11.27  ...           _   \n",
       "3                         4.0                  6.27  ...        Good   \n",
       "5                         4.0                  9.27  ...        Good   \n",
       "6                         8.0                 11.27  ...        Good   \n",
       "8                         4.0                  5.42  ...        Good   \n",
       "...                       ...                   ...  ...         ...   \n",
       "99991                    25.0                 18.31  ...         Bad   \n",
       "99994                     6.0                  9.50  ...           _   \n",
       "99995                     7.0                 11.50  ...           _   \n",
       "99996                     7.0                 11.50  ...           _   \n",
       "99999                     6.0                 11.50  ...        Good   \n",
       "\n",
       "      Outstanding_Debt  Credit_Utilization_Ratio  Credit_History_Age  \\\n",
       "0               809.98                     26.82                22.1   \n",
       "3               809.98                     31.38                22.3   \n",
       "5               809.98                     27.26                22.5   \n",
       "6               809.98                     22.54                22.6   \n",
       "8               605.03                     24.46                26.6   \n",
       "...                ...                       ...                 ...   \n",
       "99991          3571.70                     37.14                 6.2   \n",
       "99994           502.38                     39.32                31.4   \n",
       "99995           502.38                     34.66                31.5   \n",
       "99996           502.38                     40.57                31.6   \n",
       "99999           502.38                     34.19                31.8   \n",
       "\n",
       "       Payment_of_Min_Amount Total_EMI_per_month  Amount_invested_monthly  \\\n",
       "0                         No               49.57                    80.42   \n",
       "3                         No               49.57                   199.46   \n",
       "5                         No               49.57                    62.43   \n",
       "6                         No               49.57                   178.34   \n",
       "8                         No               18.82                   104.29   \n",
       "...                      ...                 ...                      ...   \n",
       "99991                    Yes               60.96                    34.66   \n",
       "99994                     No               35.10                   140.58   \n",
       "99995                     No               35.10                    60.97   \n",
       "99996                     No               35.10                    54.19   \n",
       "99999                     No               35.10                   167.16   \n",
       "\n",
       "                      Payment_Behaviour Monthly_Balance  Credit_Score  \n",
       "0       High_spent_Small_value_payments          312.49          Good  \n",
       "3        Low_spent_Small_value_payments          223.45          Good  \n",
       "5                                !@9#%8          340.48          Good  \n",
       "6        Low_spent_Small_value_payments          244.57          Good  \n",
       "8        Low_spent_Small_value_payments          470.69      Standard  \n",
       "...                                 ...             ...           ...  \n",
       "99991   High_spent_Large_value_payments          337.36      Standard  \n",
       "99994  High_spent_Medium_value_payments          410.26          Poor  \n",
       "99995   High_spent_Large_value_payments          479.87          Poor  \n",
       "99996  High_spent_Medium_value_payments          496.65          Poor  \n",
       "99999                            !@9#%8          393.67          Poor  \n",
       "\n",
       "[67544 rows x 21 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  find and remove outlier, typo rows - Age\n",
    "\n",
    "# Define the range of normal values\n",
    "lower_bound = 18\n",
    "upper_bound = 90\n",
    "\n",
    "column = 'Age'\n",
    "\n",
    "# testing variable\n",
    "within_range_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "\n",
    "# used this to play with range and dtm where cutoff for normal values were\n",
    "outliers_df = df_train[(df_train[column] < lower_bound) | (df_train[column] > upper_bound)]\n",
    "\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "\n",
    "df_train = cleaned_df\n",
    "\n",
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Payment_Behaviour\n",
       "Low_spent_Small_value_payments      16738\n",
       "High_spent_Medium_value_payments    12059\n",
       "High_spent_Large_value_payments      9564\n",
       "Low_spent_Medium_value_payments      9348\n",
       "High_spent_Small_value_payments      7722\n",
       "Low_spent_Large_value_payments       7073\n",
       "!@9#%8                               5040\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of Payment_Behaviour\n",
    "\n",
    "column = 'Payment_Behaviour'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_train[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = df_train[df_train['Payment_Behaviour'] != '!@9#%8']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Amount_invested_monthly\n",
       "10000.00    2816\n",
       "0.00         106\n",
       "54.48         11\n",
       "63.67         11\n",
       "80.57         10\n",
       "            ... \n",
       "51.49          4\n",
       "38.11          4\n",
       "44.88          4\n",
       "125.83         4\n",
       "71.80          4\n",
       "Name: count, Length: 3808, dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of Amount_invested_monthly\n",
    "\n",
    "column = 'Amount_invested_monthly'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_train[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>...</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "      <th>Credit_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>809.98</td>\n",
       "      <td>26.82</td>\n",
       "      <td>22.1</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>80.42</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>312.49</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>31.38</td>\n",
       "      <td>22.3</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>199.46</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>223.45</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>22.54</td>\n",
       "      <td>22.6</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>178.34</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>244.57</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>24.46</td>\n",
       "      <td>26.6</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>104.29</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>470.69</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>38.55</td>\n",
       "      <td>26.7</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>40.39</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>484.59</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99988</th>\n",
       "      <td>28</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>28.51</td>\n",
       "      <td>6.0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>213.98</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>208.05</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99991</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>...</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>37.14</td>\n",
       "      <td>6.2</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>34.66</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>337.36</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99994</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>9.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.32</td>\n",
       "      <td>31.4</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>140.58</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>410.26</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.66</td>\n",
       "      <td>31.5</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>60.97</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>479.87</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>40.57</td>\n",
       "      <td>31.6</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>54.19</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>496.65</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>59688 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0       23  Scientist       19114.12                  3                4   \n",
       "3       23  Scientist       19114.12                  3                4   \n",
       "6       23  Scientist       19114.12                  3                4   \n",
       "8       28    _______       34847.84                  2                4   \n",
       "9       28    Teacher       34847.84                  2                4   \n",
       "...    ...        ...            ...                ...              ...   \n",
       "99988   28  Architect       20002.88                 10                8   \n",
       "99991   29  Architect       20002.88                 10                8   \n",
       "99994   25   Mechanic       39628.99                  4                6   \n",
       "99995   25   Mechanic       39628.99                  4                6   \n",
       "99996   25   Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "0                  3          4.0                  3.0   \n",
       "3                  3          4.0                  5.0   \n",
       "6                  3          4.0                  3.0   \n",
       "8                  6          1.0                  3.0   \n",
       "9                  6          1.0                  7.0   \n",
       "...              ...          ...                  ...   \n",
       "99988             29          5.0                 33.0   \n",
       "99991             29          5.0                 33.0   \n",
       "99994              7          2.0                 20.0   \n",
       "99995              7          2.0                 23.0   \n",
       "99996              7          2.0                 18.0   \n",
       "\n",
       "       Num_of_Delayed_Payment  Changed_Credit_Limit  ...  Credit_Mix  \\\n",
       "0                         7.0                 11.27  ...           _   \n",
       "3                         4.0                  6.27  ...        Good   \n",
       "6                         8.0                 11.27  ...        Good   \n",
       "8                         4.0                  5.42  ...        Good   \n",
       "9                         1.0                  7.42  ...        Good   \n",
       "...                       ...                   ...  ...         ...   \n",
       "99988                    25.0                 18.31  ...           _   \n",
       "99991                    25.0                 18.31  ...         Bad   \n",
       "99994                     6.0                  9.50  ...           _   \n",
       "99995                     7.0                 11.50  ...           _   \n",
       "99996                     7.0                 11.50  ...           _   \n",
       "\n",
       "      Outstanding_Debt  Credit_Utilization_Ratio  Credit_History_Age  \\\n",
       "0               809.98                     26.82                22.1   \n",
       "3               809.98                     31.38                22.3   \n",
       "6               809.98                     22.54                22.6   \n",
       "8               605.03                     24.46                26.6   \n",
       "9               605.03                     38.55                26.7   \n",
       "...                ...                       ...                 ...   \n",
       "99988          3571.70                     28.51                 6.0   \n",
       "99991          3571.70                     37.14                 6.2   \n",
       "99994           502.38                     39.32                31.4   \n",
       "99995           502.38                     34.66                31.5   \n",
       "99996           502.38                     40.57                31.6   \n",
       "\n",
       "       Payment_of_Min_Amount Total_EMI_per_month  Amount_invested_monthly  \\\n",
       "0                         No               49.57                    80.42   \n",
       "3                         No               49.57                   199.46   \n",
       "6                         No               49.57                   178.34   \n",
       "8                         No               18.82                   104.29   \n",
       "9                         No               18.82                    40.39   \n",
       "...                      ...                 ...                      ...   \n",
       "99988                    Yes               60.96                   213.98   \n",
       "99991                    Yes               60.96                    34.66   \n",
       "99994                     No               35.10                   140.58   \n",
       "99995                     No               35.10                    60.97   \n",
       "99996                     No               35.10                    54.19   \n",
       "\n",
       "                      Payment_Behaviour Monthly_Balance  Credit_Score  \n",
       "0       High_spent_Small_value_payments          312.49          Good  \n",
       "3        Low_spent_Small_value_payments          223.45          Good  \n",
       "6        Low_spent_Small_value_payments          244.57          Good  \n",
       "8        Low_spent_Small_value_payments          470.69      Standard  \n",
       "9       High_spent_Large_value_payments          484.59          Good  \n",
       "...                                 ...             ...           ...  \n",
       "99988    Low_spent_Small_value_payments          208.05      Standard  \n",
       "99991   High_spent_Large_value_payments          337.36      Standard  \n",
       "99994  High_spent_Medium_value_payments          410.26          Poor  \n",
       "99995   High_spent_Large_value_payments          479.87          Poor  \n",
       "99996  High_spent_Medium_value_payments          496.65          Poor  \n",
       "\n",
       "[59688 rows x 21 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  find and remove outlier, typo rows - Amount_invested_monthly\n",
    "\n",
    "# Define the range of normal values\n",
    "lower_bound = 0\n",
    "upper_bound = 9000\n",
    "\n",
    "column = 'Amount_invested_monthly'\n",
    "\n",
    "# testing variable\n",
    "within_range_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "\n",
    "# used this to play with range and dtm where cutoff for normal values were\n",
    "outliers_df = df_train[(df_train[column] < lower_bound) | (df_train[column] > upper_bound)]\n",
    "\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "\n",
    "df_train = cleaned_df\n",
    "\n",
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Num_of_Loan\n",
       " 2.0      9265\n",
       " 3.0      9160\n",
       " 4.0      8876\n",
       " 0.0      6882\n",
       " 1.0      6681\n",
       " 6.0      4334\n",
       " 5.0      4058\n",
       " 7.0      4030\n",
       "-100.0    2293\n",
       " 9.0      2040\n",
       " 8.0      1797\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of Num_of_Loan\n",
    "\n",
    "column = 'Num_of_Loan'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_train[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>...</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "      <th>Credit_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>809.98</td>\n",
       "      <td>26.82</td>\n",
       "      <td>22.1</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>80.42</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>312.49</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>31.38</td>\n",
       "      <td>22.3</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>199.46</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>223.45</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>22.54</td>\n",
       "      <td>22.6</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>178.34</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>244.57</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>24.46</td>\n",
       "      <td>26.6</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>104.29</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>470.69</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>38.55</td>\n",
       "      <td>26.7</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>40.39</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>484.59</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99988</th>\n",
       "      <td>28</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>28.51</td>\n",
       "      <td>6.0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>213.98</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>208.05</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99991</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>...</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>37.14</td>\n",
       "      <td>6.2</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>34.66</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>337.36</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99994</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>9.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.32</td>\n",
       "      <td>31.4</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>140.58</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>410.26</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.66</td>\n",
       "      <td>31.5</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>60.97</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>479.87</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>40.57</td>\n",
       "      <td>31.6</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>54.19</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>496.65</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>57123 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0       23  Scientist       19114.12                  3                4   \n",
       "3       23  Scientist       19114.12                  3                4   \n",
       "6       23  Scientist       19114.12                  3                4   \n",
       "8       28    _______       34847.84                  2                4   \n",
       "9       28    Teacher       34847.84                  2                4   \n",
       "...    ...        ...            ...                ...              ...   \n",
       "99988   28  Architect       20002.88                 10                8   \n",
       "99991   29  Architect       20002.88                 10                8   \n",
       "99994   25   Mechanic       39628.99                  4                6   \n",
       "99995   25   Mechanic       39628.99                  4                6   \n",
       "99996   25   Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "0                  3          4.0                  3.0   \n",
       "3                  3          4.0                  5.0   \n",
       "6                  3          4.0                  3.0   \n",
       "8                  6          1.0                  3.0   \n",
       "9                  6          1.0                  7.0   \n",
       "...              ...          ...                  ...   \n",
       "99988             29          5.0                 33.0   \n",
       "99991             29          5.0                 33.0   \n",
       "99994              7          2.0                 20.0   \n",
       "99995              7          2.0                 23.0   \n",
       "99996              7          2.0                 18.0   \n",
       "\n",
       "       Num_of_Delayed_Payment  Changed_Credit_Limit  ...  Credit_Mix  \\\n",
       "0                         7.0                 11.27  ...           _   \n",
       "3                         4.0                  6.27  ...        Good   \n",
       "6                         8.0                 11.27  ...        Good   \n",
       "8                         4.0                  5.42  ...        Good   \n",
       "9                         1.0                  7.42  ...        Good   \n",
       "...                       ...                   ...  ...         ...   \n",
       "99988                    25.0                 18.31  ...           _   \n",
       "99991                    25.0                 18.31  ...         Bad   \n",
       "99994                     6.0                  9.50  ...           _   \n",
       "99995                     7.0                 11.50  ...           _   \n",
       "99996                     7.0                 11.50  ...           _   \n",
       "\n",
       "      Outstanding_Debt  Credit_Utilization_Ratio  Credit_History_Age  \\\n",
       "0               809.98                     26.82                22.1   \n",
       "3               809.98                     31.38                22.3   \n",
       "6               809.98                     22.54                22.6   \n",
       "8               605.03                     24.46                26.6   \n",
       "9               605.03                     38.55                26.7   \n",
       "...                ...                       ...                 ...   \n",
       "99988          3571.70                     28.51                 6.0   \n",
       "99991          3571.70                     37.14                 6.2   \n",
       "99994           502.38                     39.32                31.4   \n",
       "99995           502.38                     34.66                31.5   \n",
       "99996           502.38                     40.57                31.6   \n",
       "\n",
       "       Payment_of_Min_Amount Total_EMI_per_month  Amount_invested_monthly  \\\n",
       "0                         No               49.57                    80.42   \n",
       "3                         No               49.57                   199.46   \n",
       "6                         No               49.57                   178.34   \n",
       "8                         No               18.82                   104.29   \n",
       "9                         No               18.82                    40.39   \n",
       "...                      ...                 ...                      ...   \n",
       "99988                    Yes               60.96                   213.98   \n",
       "99991                    Yes               60.96                    34.66   \n",
       "99994                     No               35.10                   140.58   \n",
       "99995                     No               35.10                    60.97   \n",
       "99996                     No               35.10                    54.19   \n",
       "\n",
       "                      Payment_Behaviour Monthly_Balance  Credit_Score  \n",
       "0       High_spent_Small_value_payments          312.49          Good  \n",
       "3        Low_spent_Small_value_payments          223.45          Good  \n",
       "6        Low_spent_Small_value_payments          244.57          Good  \n",
       "8        Low_spent_Small_value_payments          470.69      Standard  \n",
       "9       High_spent_Large_value_payments          484.59          Good  \n",
       "...                                 ...             ...           ...  \n",
       "99988    Low_spent_Small_value_payments          208.05      Standard  \n",
       "99991   High_spent_Large_value_payments          337.36      Standard  \n",
       "99994  High_spent_Medium_value_payments          410.26          Poor  \n",
       "99995   High_spent_Large_value_payments          479.87          Poor  \n",
       "99996  High_spent_Medium_value_payments          496.65          Poor  \n",
       "\n",
       "[57123 rows x 21 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  find and remove outlier, typo rows - Num_of_Loan\n",
    "\n",
    "# Define the range of normal values\n",
    "lower_bound = 0\n",
    "upper_bound = 10\n",
    "\n",
    "column = 'Num_of_Loan'\n",
    "\n",
    "# testing variable\n",
    "within_range_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "\n",
    "# used this to play with range and dtm where cutoff for normal values were\n",
    "outliers_df = df_train[(df_train[column] < lower_bound) | (df_train[column] > upper_bound)]\n",
    "\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "\n",
    "df_train = cleaned_df\n",
    "\n",
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Num_of_Delayed_Payment\n",
       " 10.0    3312\n",
       " 19.0    3271\n",
       " 17.0    3240\n",
       " 15.0    3178\n",
       " 16.0    3165\n",
       " 12.0    3155\n",
       " 18.0    3112\n",
       " 9.0     3096\n",
       " 8.0     3067\n",
       " 20.0    3060\n",
       " 11.0    2979\n",
       " 14.0    2570\n",
       " 13.0    2503\n",
       " 7.0     1578\n",
       " 21.0    1505\n",
       " 6.0     1488\n",
       " 22.0    1376\n",
       " 5.0     1358\n",
       " 3.0     1262\n",
       " 4.0     1205\n",
       " 2.0     1173\n",
       " 23.0    1159\n",
       " 1.0     1072\n",
       " 0.0     1047\n",
       " 24.0     969\n",
       " 25.0     962\n",
       "-1.0      203\n",
       " 26.0     186\n",
       "-2.0      158\n",
       " 27.0     126\n",
       " 28.0      64\n",
       "-3.0       61\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of Num_of_Delayed_Payment\n",
    "\n",
    "column = 'Num_of_Delayed_Payment'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_train[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>...</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "      <th>Credit_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>809.98</td>\n",
       "      <td>26.82</td>\n",
       "      <td>22.1</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>80.42</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>312.49</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>31.38</td>\n",
       "      <td>22.3</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>199.46</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>223.45</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>22.54</td>\n",
       "      <td>22.6</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>178.34</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>244.57</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>24.46</td>\n",
       "      <td>26.6</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>104.29</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>470.69</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>38.55</td>\n",
       "      <td>26.7</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>40.39</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>484.59</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99988</th>\n",
       "      <td>28</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>28.51</td>\n",
       "      <td>6.0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>213.98</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>208.05</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99991</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>...</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>37.14</td>\n",
       "      <td>6.2</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>34.66</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>337.36</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99994</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>9.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.32</td>\n",
       "      <td>31.4</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>140.58</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>410.26</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.66</td>\n",
       "      <td>31.5</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>60.97</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>479.87</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>40.57</td>\n",
       "      <td>31.6</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>54.19</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>496.65</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>56238 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0       23  Scientist       19114.12                  3                4   \n",
       "3       23  Scientist       19114.12                  3                4   \n",
       "6       23  Scientist       19114.12                  3                4   \n",
       "8       28    _______       34847.84                  2                4   \n",
       "9       28    Teacher       34847.84                  2                4   \n",
       "...    ...        ...            ...                ...              ...   \n",
       "99988   28  Architect       20002.88                 10                8   \n",
       "99991   29  Architect       20002.88                 10                8   \n",
       "99994   25   Mechanic       39628.99                  4                6   \n",
       "99995   25   Mechanic       39628.99                  4                6   \n",
       "99996   25   Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "0                  3          4.0                  3.0   \n",
       "3                  3          4.0                  5.0   \n",
       "6                  3          4.0                  3.0   \n",
       "8                  6          1.0                  3.0   \n",
       "9                  6          1.0                  7.0   \n",
       "...              ...          ...                  ...   \n",
       "99988             29          5.0                 33.0   \n",
       "99991             29          5.0                 33.0   \n",
       "99994              7          2.0                 20.0   \n",
       "99995              7          2.0                 23.0   \n",
       "99996              7          2.0                 18.0   \n",
       "\n",
       "       Num_of_Delayed_Payment  Changed_Credit_Limit  ...  Credit_Mix  \\\n",
       "0                         7.0                 11.27  ...           _   \n",
       "3                         4.0                  6.27  ...        Good   \n",
       "6                         8.0                 11.27  ...        Good   \n",
       "8                         4.0                  5.42  ...        Good   \n",
       "9                         1.0                  7.42  ...        Good   \n",
       "...                       ...                   ...  ...         ...   \n",
       "99988                    25.0                 18.31  ...           _   \n",
       "99991                    25.0                 18.31  ...         Bad   \n",
       "99994                     6.0                  9.50  ...           _   \n",
       "99995                     7.0                 11.50  ...           _   \n",
       "99996                     7.0                 11.50  ...           _   \n",
       "\n",
       "      Outstanding_Debt  Credit_Utilization_Ratio  Credit_History_Age  \\\n",
       "0               809.98                     26.82                22.1   \n",
       "3               809.98                     31.38                22.3   \n",
       "6               809.98                     22.54                22.6   \n",
       "8               605.03                     24.46                26.6   \n",
       "9               605.03                     38.55                26.7   \n",
       "...                ...                       ...                 ...   \n",
       "99988          3571.70                     28.51                 6.0   \n",
       "99991          3571.70                     37.14                 6.2   \n",
       "99994           502.38                     39.32                31.4   \n",
       "99995           502.38                     34.66                31.5   \n",
       "99996           502.38                     40.57                31.6   \n",
       "\n",
       "       Payment_of_Min_Amount Total_EMI_per_month  Amount_invested_monthly  \\\n",
       "0                         No               49.57                    80.42   \n",
       "3                         No               49.57                   199.46   \n",
       "6                         No               49.57                   178.34   \n",
       "8                         No               18.82                   104.29   \n",
       "9                         No               18.82                    40.39   \n",
       "...                      ...                 ...                      ...   \n",
       "99988                    Yes               60.96                   213.98   \n",
       "99991                    Yes               60.96                    34.66   \n",
       "99994                     No               35.10                   140.58   \n",
       "99995                     No               35.10                    60.97   \n",
       "99996                     No               35.10                    54.19   \n",
       "\n",
       "                      Payment_Behaviour Monthly_Balance  Credit_Score  \n",
       "0       High_spent_Small_value_payments          312.49          Good  \n",
       "3        Low_spent_Small_value_payments          223.45          Good  \n",
       "6        Low_spent_Small_value_payments          244.57          Good  \n",
       "8        Low_spent_Small_value_payments          470.69      Standard  \n",
       "9       High_spent_Large_value_payments          484.59          Good  \n",
       "...                                 ...             ...           ...  \n",
       "99988    Low_spent_Small_value_payments          208.05      Standard  \n",
       "99991   High_spent_Large_value_payments          337.36      Standard  \n",
       "99994  High_spent_Medium_value_payments          410.26          Poor  \n",
       "99995   High_spent_Large_value_payments          479.87          Poor  \n",
       "99996  High_spent_Medium_value_payments          496.65          Poor  \n",
       "\n",
       "[56238 rows x 21 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  find and remove outlier, typo rows - Num_of_Delayed_Payment\n",
    "\n",
    "# Define the range of normal values\n",
    "lower_bound = 0\n",
    "upper_bound = 30\n",
    "\n",
    "column = 'Num_of_Delayed_Payment'\n",
    "\n",
    "# testing variable\n",
    "within_range_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "\n",
    "# used this to play with range and dtm where cutoff for normal values were\n",
    "outliers_df = df_train[(df_train[column] < lower_bound) | (df_train[column] > upper_bound)]\n",
    "\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "\n",
    "df_train = cleaned_df\n",
    "\n",
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Payment_of_Min_Amount\n",
       "Yes    28404\n",
       "No     21119\n",
       "NM      6715\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of Payment_of_Min_Amount\n",
    "\n",
    "column = 'Payment_of_Min_Amount'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_train[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop NM from Payment_of_Min_Amount\n",
    "df_train = df_train[df_train['Payment_of_Min_Amount'] != 'NM']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Outstanding_Debt\n",
       "1456.12    14\n",
       "460.46     13\n",
       "1072.42    13\n",
       "562.34     13\n",
       "559.35     13\n",
       "           ..\n",
       "2626.18     4\n",
       "286.94      4\n",
       "1465.90     4\n",
       "1961.87     4\n",
       "537.23      4\n",
       "Name: count, Length: 8036, dtype: int64"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of Outstanding_Debt\n",
    "\n",
    "column = 'Outstanding_Debt'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_train[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/zt/x756b70j3v75355fl48s3dzc0000gn/T/ipykernel_10263/3412360804.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_train['Num_of_Delayed_Payment'] = df_train['Num_of_Delayed_Payment'].astype(int)\n",
      "/var/folders/zt/x756b70j3v75355fl48s3dzc0000gn/T/ipykernel_10263/3412360804.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_train['Num_Credit_Inquiries'] = df_train['Num_Credit_Inquiries'].astype(int)\n",
      "/var/folders/zt/x756b70j3v75355fl48s3dzc0000gn/T/ipykernel_10263/3412360804.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_train['Num_of_Loan'] = df_train['Num_of_Loan'].astype(int)\n",
      "/var/folders/zt/x756b70j3v75355fl48s3dzc0000gn/T/ipykernel_10263/3412360804.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_train['Delay_from_due_date'] = df_train['Delay_from_due_date'].astype(int)\n"
     ]
    }
   ],
   "source": [
    "# cast some columns as int when appropriate\n",
    "\n",
    "df_train['Num_of_Delayed_Payment'] = df_train['Num_of_Delayed_Payment'].astype(int)\n",
    "df_train['Num_Credit_Inquiries'] = df_train['Num_Credit_Inquiries'].astype(int)\n",
    "df_train['Num_of_Loan'] = df_train['Num_of_Loan'].astype(int)\n",
    "df_train['Delay_from_due_date'] = df_train['Delay_from_due_date'].astype(int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Delay_from_due_date\n",
       " 15    1800\n",
       " 13    1743\n",
       " 8     1717\n",
       " 14    1674\n",
       " 10    1633\n",
       "       ... \n",
       " 64      26\n",
       " 65      26\n",
       " 66      17\n",
       "-5       17\n",
       " 67      11\n",
       "Name: count, Length: 73, dtype: int64"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of Delay_from_due_date\n",
    "\n",
    "column = 'Delay_from_due_date'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_train[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts\n",
    "\n",
    "# values_counted = df_train[column].describe()\n",
    "# values_counted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    4.952300e+04\n",
       "mean     1.833540e+05\n",
       "std      1.473172e+06\n",
       "min      7.006040e+03\n",
       "25%      1.951957e+04\n",
       "50%      3.736734e+04\n",
       "75%      7.343607e+04\n",
       "max      2.419806e+07\n",
       "Name: Annual_Income, dtype: float64"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of Annual_Income\n",
    "\n",
    "column = 'Annual_Income'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_train[column].describe()\n",
    "values_counted\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>...</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "      <th>Credit_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>24.46</td>\n",
       "      <td>26.6</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>104.29</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>470.69</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>7.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>38.55</td>\n",
       "      <td>26.7</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>40.39</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>484.59</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>39.18</td>\n",
       "      <td>26.8</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>99.31</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>465.68</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>6.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>34.98</td>\n",
       "      <td>26.9</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>130.12</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>444.87</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>33.38</td>\n",
       "      <td>27.0</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>43.48</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>481.51</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99988</th>\n",
       "      <td>28</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>25</td>\n",
       "      <td>18.31</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>28.51</td>\n",
       "      <td>6.0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>213.98</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>208.05</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99991</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>25</td>\n",
       "      <td>18.31</td>\n",
       "      <td>...</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>37.14</td>\n",
       "      <td>6.2</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>34.66</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>337.36</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99994</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>9.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.32</td>\n",
       "      <td>31.4</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>140.58</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>410.26</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "      <td>7</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.66</td>\n",
       "      <td>31.5</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>60.97</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>479.87</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>7</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>40.57</td>\n",
       "      <td>31.6</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>54.19</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>496.65</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>35926 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "8       28    _______       34847.84                  2                4   \n",
       "9       28    Teacher       34847.84                  2                4   \n",
       "11      28    Teacher       34847.84                  2                4   \n",
       "12      28    Teacher       34847.84                  2                4   \n",
       "13      28    Teacher       34847.84                  2                4   \n",
       "...    ...        ...            ...                ...              ...   \n",
       "99988   28  Architect       20002.88                 10                8   \n",
       "99991   29  Architect       20002.88                 10                8   \n",
       "99994   25   Mechanic       39628.99                  4                6   \n",
       "99995   25   Mechanic       39628.99                  4                6   \n",
       "99996   25   Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "8                  6            1                    3   \n",
       "9                  6            1                    7   \n",
       "11                 6            1                    3   \n",
       "12                 6            1                    3   \n",
       "13                 6            1                    3   \n",
       "...              ...          ...                  ...   \n",
       "99988             29            5                   33   \n",
       "99991             29            5                   33   \n",
       "99994              7            2                   20   \n",
       "99995              7            2                   23   \n",
       "99996              7            2                   18   \n",
       "\n",
       "       Num_of_Delayed_Payment  Changed_Credit_Limit  ...  Credit_Mix  \\\n",
       "8                           4                  5.42  ...        Good   \n",
       "9                           1                  7.42  ...        Good   \n",
       "11                          3                  5.42  ...        Good   \n",
       "12                          1                  6.42  ...        Good   \n",
       "13                          0                  5.42  ...        Good   \n",
       "...                       ...                   ...  ...         ...   \n",
       "99988                      25                 18.31  ...           _   \n",
       "99991                      25                 18.31  ...         Bad   \n",
       "99994                       6                  9.50  ...           _   \n",
       "99995                       7                 11.50  ...           _   \n",
       "99996                       7                 11.50  ...           _   \n",
       "\n",
       "      Outstanding_Debt  Credit_Utilization_Ratio  Credit_History_Age  \\\n",
       "8               605.03                     24.46                26.6   \n",
       "9               605.03                     38.55                26.7   \n",
       "11              605.03                     39.18                26.8   \n",
       "12              605.03                     34.98                26.9   \n",
       "13              605.03                     33.38                27.0   \n",
       "...                ...                       ...                 ...   \n",
       "99988          3571.70                     28.51                 6.0   \n",
       "99991          3571.70                     37.14                 6.2   \n",
       "99994           502.38                     39.32                31.4   \n",
       "99995           502.38                     34.66                31.5   \n",
       "99996           502.38                     40.57                31.6   \n",
       "\n",
       "       Payment_of_Min_Amount Total_EMI_per_month  Amount_invested_monthly  \\\n",
       "8                         No               18.82                   104.29   \n",
       "9                         No               18.82                    40.39   \n",
       "11                        No               18.82                    99.31   \n",
       "12                        No               18.82                   130.12   \n",
       "13                        No               18.82                    43.48   \n",
       "...                      ...                 ...                      ...   \n",
       "99988                    Yes               60.96                   213.98   \n",
       "99991                    Yes               60.96                    34.66   \n",
       "99994                     No               35.10                   140.58   \n",
       "99995                     No               35.10                    60.97   \n",
       "99996                     No               35.10                    54.19   \n",
       "\n",
       "                      Payment_Behaviour Monthly_Balance  Credit_Score  \n",
       "8        Low_spent_Small_value_payments          470.69      Standard  \n",
       "9       High_spent_Large_value_payments          484.59          Good  \n",
       "11      Low_spent_Medium_value_payments          465.68          Good  \n",
       "12       Low_spent_Small_value_payments          444.87          Good  \n",
       "13      High_spent_Large_value_payments          481.51          Good  \n",
       "...                                 ...             ...           ...  \n",
       "99988    Low_spent_Small_value_payments          208.05      Standard  \n",
       "99991   High_spent_Large_value_payments          337.36      Standard  \n",
       "99994  High_spent_Medium_value_payments          410.26          Poor  \n",
       "99995   High_spent_Large_value_payments          479.87          Poor  \n",
       "99996  High_spent_Medium_value_payments          496.65          Poor  \n",
       "\n",
       "[35926 rows x 21 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  find and remove outlier, typo rows - Annual_Income\n",
    "\n",
    "# Define the range of normal values\n",
    "lower_bound = 20000\n",
    "upper_bound = 1000000\n",
    "\n",
    "column = 'Annual_Income'\n",
    "\n",
    "# testing variable\n",
    "within_range_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "\n",
    "# used this to play with range and dtm where cutoff for normal values were\n",
    "outliers_df = df_train[(df_train[column] < lower_bound) | (df_train[column] > upper_bound)]\n",
    "\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "\n",
    "df_train = cleaned_df\n",
    "\n",
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Num_Credit_Inquiries\n",
       "4      4837\n",
       "3      4089\n",
       "2      3650\n",
       "1      3404\n",
       "0      3094\n",
       "6      2699\n",
       "7      2597\n",
       "8      2447\n",
       "5      2245\n",
       "9      1515\n",
       "11     1309\n",
       "10     1283\n",
       "12     1176\n",
       "13      364\n",
       "14      258\n",
       "15      199\n",
       "16      110\n",
       "17       67\n",
       "285       4\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of Delay_from_due_date\n",
    "\n",
    "column = 'Num_Credit_Inquiries'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_train[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>...</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "      <th>Credit_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>24.46</td>\n",
       "      <td>26.6</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>104.29</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>470.69</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>7.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>38.55</td>\n",
       "      <td>26.7</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>40.39</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>484.59</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>39.18</td>\n",
       "      <td>26.8</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>99.31</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>465.68</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>6.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>34.98</td>\n",
       "      <td>26.9</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>130.12</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>444.87</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>33.38</td>\n",
       "      <td>27.0</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>43.48</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>481.51</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99988</th>\n",
       "      <td>28</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>25</td>\n",
       "      <td>18.31</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>28.51</td>\n",
       "      <td>6.0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>213.98</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>208.05</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99991</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>25</td>\n",
       "      <td>18.31</td>\n",
       "      <td>...</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>37.14</td>\n",
       "      <td>6.2</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>34.66</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>337.36</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99994</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>9.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.32</td>\n",
       "      <td>31.4</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>140.58</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>410.26</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "      <td>7</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.66</td>\n",
       "      <td>31.5</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>60.97</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>479.87</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>7</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>40.57</td>\n",
       "      <td>31.6</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>54.19</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>496.65</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>35343 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "8       28    _______       34847.84                  2                4   \n",
       "9       28    Teacher       34847.84                  2                4   \n",
       "11      28    Teacher       34847.84                  2                4   \n",
       "12      28    Teacher       34847.84                  2                4   \n",
       "13      28    Teacher       34847.84                  2                4   \n",
       "...    ...        ...            ...                ...              ...   \n",
       "99988   28  Architect       20002.88                 10                8   \n",
       "99991   29  Architect       20002.88                 10                8   \n",
       "99994   25   Mechanic       39628.99                  4                6   \n",
       "99995   25   Mechanic       39628.99                  4                6   \n",
       "99996   25   Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "8                  6            1                    3   \n",
       "9                  6            1                    7   \n",
       "11                 6            1                    3   \n",
       "12                 6            1                    3   \n",
       "13                 6            1                    3   \n",
       "...              ...          ...                  ...   \n",
       "99988             29            5                   33   \n",
       "99991             29            5                   33   \n",
       "99994              7            2                   20   \n",
       "99995              7            2                   23   \n",
       "99996              7            2                   18   \n",
       "\n",
       "       Num_of_Delayed_Payment  Changed_Credit_Limit  ...  Credit_Mix  \\\n",
       "8                           4                  5.42  ...        Good   \n",
       "9                           1                  7.42  ...        Good   \n",
       "11                          3                  5.42  ...        Good   \n",
       "12                          1                  6.42  ...        Good   \n",
       "13                          0                  5.42  ...        Good   \n",
       "...                       ...                   ...  ...         ...   \n",
       "99988                      25                 18.31  ...           _   \n",
       "99991                      25                 18.31  ...         Bad   \n",
       "99994                       6                  9.50  ...           _   \n",
       "99995                       7                 11.50  ...           _   \n",
       "99996                       7                 11.50  ...           _   \n",
       "\n",
       "      Outstanding_Debt  Credit_Utilization_Ratio  Credit_History_Age  \\\n",
       "8               605.03                     24.46                26.6   \n",
       "9               605.03                     38.55                26.7   \n",
       "11              605.03                     39.18                26.8   \n",
       "12              605.03                     34.98                26.9   \n",
       "13              605.03                     33.38                27.0   \n",
       "...                ...                       ...                 ...   \n",
       "99988          3571.70                     28.51                 6.0   \n",
       "99991          3571.70                     37.14                 6.2   \n",
       "99994           502.38                     39.32                31.4   \n",
       "99995           502.38                     34.66                31.5   \n",
       "99996           502.38                     40.57                31.6   \n",
       "\n",
       "       Payment_of_Min_Amount Total_EMI_per_month  Amount_invested_monthly  \\\n",
       "8                         No               18.82                   104.29   \n",
       "9                         No               18.82                    40.39   \n",
       "11                        No               18.82                    99.31   \n",
       "12                        No               18.82                   130.12   \n",
       "13                        No               18.82                    43.48   \n",
       "...                      ...                 ...                      ...   \n",
       "99988                    Yes               60.96                   213.98   \n",
       "99991                    Yes               60.96                    34.66   \n",
       "99994                     No               35.10                   140.58   \n",
       "99995                     No               35.10                    60.97   \n",
       "99996                     No               35.10                    54.19   \n",
       "\n",
       "                      Payment_Behaviour Monthly_Balance  Credit_Score  \n",
       "8        Low_spent_Small_value_payments          470.69      Standard  \n",
       "9       High_spent_Large_value_payments          484.59          Good  \n",
       "11      Low_spent_Medium_value_payments          465.68          Good  \n",
       "12       Low_spent_Small_value_payments          444.87          Good  \n",
       "13      High_spent_Large_value_payments          481.51          Good  \n",
       "...                                 ...             ...           ...  \n",
       "99988    Low_spent_Small_value_payments          208.05      Standard  \n",
       "99991   High_spent_Large_value_payments          337.36      Standard  \n",
       "99994  High_spent_Medium_value_payments          410.26          Poor  \n",
       "99995   High_spent_Large_value_payments          479.87          Poor  \n",
       "99996  High_spent_Medium_value_payments          496.65          Poor  \n",
       "\n",
       "[35343 rows x 21 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  find and remove outlier, typo rows - Annual_Income\n",
    "\n",
    "# Define the range of normal values\n",
    "lower_bound = 0\n",
    "upper_bound = 20\n",
    "\n",
    "column = 'Num_Credit_Inquiries'\n",
    "\n",
    "# testing variable\n",
    "within_range_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "\n",
    "# used this to play with range and dtm where cutoff for normal values were\n",
    "outliers_df = df_train[(df_train[column] < lower_bound) | (df_train[column] > upper_bound)]\n",
    "\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "\n",
    "df_train = cleaned_df\n",
    "\n",
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>...</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "      <th>Credit_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>24.46</td>\n",
       "      <td>26.6</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>104.29</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>470.69</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>7.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>38.55</td>\n",
       "      <td>26.7</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>40.39</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>484.59</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>39.18</td>\n",
       "      <td>26.8</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>99.31</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>465.68</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>6.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>34.98</td>\n",
       "      <td>26.9</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>130.12</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>444.87</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>...</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>33.38</td>\n",
       "      <td>27.0</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>43.48</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>481.51</td>\n",
       "      <td>Good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99988</th>\n",
       "      <td>28</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>25</td>\n",
       "      <td>18.31</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>28.51</td>\n",
       "      <td>6.0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>213.98</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>208.05</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99991</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>25</td>\n",
       "      <td>18.31</td>\n",
       "      <td>...</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>37.14</td>\n",
       "      <td>6.2</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>34.66</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>337.36</td>\n",
       "      <td>Standard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99994</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>9.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.32</td>\n",
       "      <td>31.4</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>140.58</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>410.26</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "      <td>7</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.66</td>\n",
       "      <td>31.5</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>60.97</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>479.87</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>7</td>\n",
       "      <td>11.50</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>40.57</td>\n",
       "      <td>31.6</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>54.19</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>496.65</td>\n",
       "      <td>Poor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>35338 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "8       28    _______       34847.84                  2                4   \n",
       "9       28    Teacher       34847.84                  2                4   \n",
       "11      28    Teacher       34847.84                  2                4   \n",
       "12      28    Teacher       34847.84                  2                4   \n",
       "13      28    Teacher       34847.84                  2                4   \n",
       "...    ...        ...            ...                ...              ...   \n",
       "99988   28  Architect       20002.88                 10                8   \n",
       "99991   29  Architect       20002.88                 10                8   \n",
       "99994   25   Mechanic       39628.99                  4                6   \n",
       "99995   25   Mechanic       39628.99                  4                6   \n",
       "99996   25   Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "8                  6            1                    3   \n",
       "9                  6            1                    7   \n",
       "11                 6            1                    3   \n",
       "12                 6            1                    3   \n",
       "13                 6            1                    3   \n",
       "...              ...          ...                  ...   \n",
       "99988             29            5                   33   \n",
       "99991             29            5                   33   \n",
       "99994              7            2                   20   \n",
       "99995              7            2                   23   \n",
       "99996              7            2                   18   \n",
       "\n",
       "       Num_of_Delayed_Payment  Changed_Credit_Limit  ...  Credit_Mix  \\\n",
       "8                           4                  5.42  ...        Good   \n",
       "9                           1                  7.42  ...        Good   \n",
       "11                          3                  5.42  ...        Good   \n",
       "12                          1                  6.42  ...        Good   \n",
       "13                          0                  5.42  ...        Good   \n",
       "...                       ...                   ...  ...         ...   \n",
       "99988                      25                 18.31  ...           _   \n",
       "99991                      25                 18.31  ...         Bad   \n",
       "99994                       6                  9.50  ...           _   \n",
       "99995                       7                 11.50  ...           _   \n",
       "99996                       7                 11.50  ...           _   \n",
       "\n",
       "      Outstanding_Debt  Credit_Utilization_Ratio  Credit_History_Age  \\\n",
       "8               605.03                     24.46                26.6   \n",
       "9               605.03                     38.55                26.7   \n",
       "11              605.03                     39.18                26.8   \n",
       "12              605.03                     34.98                26.9   \n",
       "13              605.03                     33.38                27.0   \n",
       "...                ...                       ...                 ...   \n",
       "99988          3571.70                     28.51                 6.0   \n",
       "99991          3571.70                     37.14                 6.2   \n",
       "99994           502.38                     39.32                31.4   \n",
       "99995           502.38                     34.66                31.5   \n",
       "99996           502.38                     40.57                31.6   \n",
       "\n",
       "       Payment_of_Min_Amount Total_EMI_per_month  Amount_invested_monthly  \\\n",
       "8                         No               18.82                   104.29   \n",
       "9                         No               18.82                    40.39   \n",
       "11                        No               18.82                    99.31   \n",
       "12                        No               18.82                   130.12   \n",
       "13                        No               18.82                    43.48   \n",
       "...                      ...                 ...                      ...   \n",
       "99988                    Yes               60.96                   213.98   \n",
       "99991                    Yes               60.96                    34.66   \n",
       "99994                     No               35.10                   140.58   \n",
       "99995                     No               35.10                    60.97   \n",
       "99996                     No               35.10                    54.19   \n",
       "\n",
       "                      Payment_Behaviour Monthly_Balance  Credit_Score  \n",
       "8        Low_spent_Small_value_payments          470.69      Standard  \n",
       "9       High_spent_Large_value_payments          484.59          Good  \n",
       "11      Low_spent_Medium_value_payments          465.68          Good  \n",
       "12       Low_spent_Small_value_payments          444.87          Good  \n",
       "13      High_spent_Large_value_payments          481.51          Good  \n",
       "...                                 ...             ...           ...  \n",
       "99988    Low_spent_Small_value_payments          208.05      Standard  \n",
       "99991   High_spent_Large_value_payments          337.36      Standard  \n",
       "99994  High_spent_Medium_value_payments          410.26          Poor  \n",
       "99995   High_spent_Large_value_payments          479.87          Poor  \n",
       "99996  High_spent_Medium_value_payments          496.65          Poor  \n",
       "\n",
       "[35338 rows x 21 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  find and remove outlier, typo rows - Monthly_Balance\n",
    "# Define the range of normal values\n",
    "lower_bound = 0\n",
    "upper_bound = 2000\n",
    "column = 'Monthly_Balance'\n",
    "# testing variable\n",
    "within_range_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "# used this to play with range and dtm where cutoff for normal values were\n",
    "outliers_df = df_train[(df_train[column] < lower_bound) | (df_train[column] > upper_bound)]\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_train[(df_train[column] >= lower_bound) & (df_train[column] <= upper_bound)]\n",
    "df_train = cleaned_df\n",
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export the DataFrame to a CSV file\n",
    "df_train.to_csv('cleaned_data_v2.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}