Predicting_renewal_probability.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing dependencies\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train =  pd.read_csv('train_ZoGVYWq.csv')\n",
    "test = pd.read_csv('test_66516Ee.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: (79853, 13)\n",
      "Test: (34224, 12)\n"
     ]
    }
   ],
   "source": [
    "print('Train:', train.shape)\n",
    "print('Test:', test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for easy cleaning\n",
    "dataset = [train,test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>perc_premium_paid_by_cash_credit</th>\n",
       "      <th>age_in_days</th>\n",
       "      <th>Income</th>\n",
       "      <th>Count_3-6_months_late</th>\n",
       "      <th>Count_6-12_months_late</th>\n",
       "      <th>Count_more_than_12_months_late</th>\n",
       "      <th>application_underwriting_score</th>\n",
       "      <th>no_of_premiums_paid</th>\n",
       "      <th>sourcing_channel</th>\n",
       "      <th>residence_area_type</th>\n",
       "      <th>premium</th>\n",
       "      <th>renewal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>110936</td>\n",
       "      <td>0.429</td>\n",
       "      <td>12058</td>\n",
       "      <td>355060</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99.02</td>\n",
       "      <td>13</td>\n",
       "      <td>C</td>\n",
       "      <td>Urban</td>\n",
       "      <td>3300</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>41492</td>\n",
       "      <td>0.010</td>\n",
       "      <td>21546</td>\n",
       "      <td>315150</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99.89</td>\n",
       "      <td>21</td>\n",
       "      <td>A</td>\n",
       "      <td>Urban</td>\n",
       "      <td>18000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>31300</td>\n",
       "      <td>0.917</td>\n",
       "      <td>17531</td>\n",
       "      <td>84140</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>98.69</td>\n",
       "      <td>7</td>\n",
       "      <td>C</td>\n",
       "      <td>Rural</td>\n",
       "      <td>3300</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>19415</td>\n",
       "      <td>0.049</td>\n",
       "      <td>15341</td>\n",
       "      <td>250510</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99.57</td>\n",
       "      <td>9</td>\n",
       "      <td>A</td>\n",
       "      <td>Urban</td>\n",
       "      <td>9600</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>99379</td>\n",
       "      <td>0.052</td>\n",
       "      <td>31400</td>\n",
       "      <td>198680</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99.87</td>\n",
       "      <td>12</td>\n",
       "      <td>B</td>\n",
       "      <td>Urban</td>\n",
       "      <td>9600</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  perc_premium_paid_by_cash_credit  age_in_days  Income  \\\n",
       "0  110936                             0.429        12058  355060   \n",
       "1   41492                             0.010        21546  315150   \n",
       "2   31300                             0.917        17531   84140   \n",
       "3   19415                             0.049        15341  250510   \n",
       "4   99379                             0.052        31400  198680   \n",
       "\n",
       "   Count_3-6_months_late  Count_6-12_months_late  \\\n",
       "0                    0.0                     0.0   \n",
       "1                    0.0                     0.0   \n",
       "2                    2.0                     3.0   \n",
       "3                    0.0                     0.0   \n",
       "4                    0.0                     0.0   \n",
       "\n",
       "   Count_more_than_12_months_late  application_underwriting_score  \\\n",
       "0                             0.0                           99.02   \n",
       "1                             0.0                           99.89   \n",
       "2                             1.0                           98.69   \n",
       "3                             0.0                           99.57   \n",
       "4                             0.0                           99.87   \n",
       "\n",
       "   no_of_premiums_paid sourcing_channel residence_area_type  premium  renewal  \n",
       "0                   13                C               Urban     3300        1  \n",
       "1                   21                A               Urban    18000        1  \n",
       "2                    7                C               Rural     3300        0  \n",
       "3                    9                A               Urban     9600        1  \n",
       "4                   12                B               Urban     9600        1  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>perc_premium_paid_by_cash_credit</th>\n",
       "      <th>age_in_days</th>\n",
       "      <th>Income</th>\n",
       "      <th>Count_3-6_months_late</th>\n",
       "      <th>Count_6-12_months_late</th>\n",
       "      <th>Count_more_than_12_months_late</th>\n",
       "      <th>application_underwriting_score</th>\n",
       "      <th>no_of_premiums_paid</th>\n",
       "      <th>sourcing_channel</th>\n",
       "      <th>residence_area_type</th>\n",
       "      <th>premium</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>649</td>\n",
       "      <td>0.001</td>\n",
       "      <td>27384</td>\n",
       "      <td>51150</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99.89</td>\n",
       "      <td>7</td>\n",
       "      <td>A</td>\n",
       "      <td>Rural</td>\n",
       "      <td>3300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>81136</td>\n",
       "      <td>0.124</td>\n",
       "      <td>23735</td>\n",
       "      <td>285140</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>98.93</td>\n",
       "      <td>19</td>\n",
       "      <td>A</td>\n",
       "      <td>Urban</td>\n",
       "      <td>11700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>70762</td>\n",
       "      <td>1.000</td>\n",
       "      <td>17170</td>\n",
       "      <td>186030</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>B</td>\n",
       "      <td>Urban</td>\n",
       "      <td>11700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53935</td>\n",
       "      <td>0.198</td>\n",
       "      <td>16068</td>\n",
       "      <td>123540</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99.00</td>\n",
       "      <td>11</td>\n",
       "      <td>B</td>\n",
       "      <td>Rural</td>\n",
       "      <td>5400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15476</td>\n",
       "      <td>0.041</td>\n",
       "      <td>10591</td>\n",
       "      <td>200020</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99.17</td>\n",
       "      <td>14</td>\n",
       "      <td>A</td>\n",
       "      <td>Rural</td>\n",
       "      <td>9600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id  perc_premium_paid_by_cash_credit  age_in_days  Income  \\\n",
       "0    649                             0.001        27384   51150   \n",
       "1  81136                             0.124        23735  285140   \n",
       "2  70762                             1.000        17170  186030   \n",
       "3  53935                             0.198        16068  123540   \n",
       "4  15476                             0.041        10591  200020   \n",
       "\n",
       "   Count_3-6_months_late  Count_6-12_months_late  \\\n",
       "0                    0.0                     0.0   \n",
       "1                    0.0                     0.0   \n",
       "2                    0.0                     0.0   \n",
       "3                    0.0                     0.0   \n",
       "4                    1.0                     0.0   \n",
       "\n",
       "   Count_more_than_12_months_late  application_underwriting_score  \\\n",
       "0                             0.0                           99.89   \n",
       "1                             0.0                           98.93   \n",
       "2                             0.0                             NaN   \n",
       "3                             0.0                           99.00   \n",
       "4                             0.0                           99.17   \n",
       "\n",
       "   no_of_premiums_paid sourcing_channel residence_area_type  premium  \n",
       "0                    7                A               Rural     3300  \n",
       "1                   19                A               Urban    11700  \n",
       "2                    2                B               Urban    11700  \n",
       "3                   11                B               Rural     5400  \n",
       "4                   14                A               Rural     9600  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['id', 'perc_premium_paid_by_cash_credit', 'age_in_days', 'Income',\n",
       "       'Count_3-6_months_late', 'Count_6-12_months_late',\n",
       "       'Count_more_than_12_months_late', 'application_underwriting_score',\n",
       "       'no_of_premiums_paid', 'sourcing_channel', 'residence_area_type',\n",
       "       'premium', 'renewal'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# their is a problem with labeling of the dataset\n",
    "#this columns are; Count_3-6_months_late, Count_6-12_months_late\n",
    "for df in dataset:\n",
    "    df.rename(index = str, columns = {'Count_3-6_months_late':'Count_3_6_months_late','Count_6-12_months_late':'Count_6_12_months_late'},inplace = True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['id', 'perc_premium_paid_by_cash_credit', 'age_in_days', 'Income',\n",
       "       'Count_3_6_months_late', 'Count_6_12_months_late',\n",
       "       'Count_more_than_12_months_late', 'application_underwriting_score',\n",
       "       'no_of_premiums_paid', 'sourcing_channel', 'residence_area_type',\n",
       "       'premium', 'renewal'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['id', 'perc_premium_paid_by_cash_credit', 'age_in_days', 'Income',\n",
       "       'Count_3_6_months_late', 'Count_6_12_months_late',\n",
       "       'Count_more_than_12_months_late', 'application_underwriting_score',\n",
       "       'no_of_premiums_paid', 'sourcing_channel', 'residence_area_type',\n",
       "       'premium'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Handle null values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                                  False\n",
       "perc_premium_paid_by_cash_credit    False\n",
       "age_in_days                         False\n",
       "Income                              False\n",
       "Count_3_6_months_late                True\n",
       "Count_6_12_months_late               True\n",
       "Count_more_than_12_months_late       True\n",
       "application_underwriting_score       True\n",
       "no_of_premiums_paid                 False\n",
       "sourcing_channel                    False\n",
       "residence_area_type                 False\n",
       "premium                             False\n",
       "renewal                             False\n",
       "dtype: bool"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.isnull().any(axis = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                                  False\n",
       "perc_premium_paid_by_cash_credit    False\n",
       "age_in_days                         False\n",
       "Income                              False\n",
       "Count_3_6_months_late                True\n",
       "Count_6_12_months_late               True\n",
       "Count_more_than_12_months_late       True\n",
       "application_underwriting_score       True\n",
       "no_of_premiums_paid                 False\n",
       "sourcing_channel                    False\n",
       "residence_area_type                 False\n",
       "premium                             False\n",
       "dtype: bool"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.isnull().any(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fill null values\n",
    "for df in dataset:\n",
    "    df['Count_3_6_months_late'].fillna(train.Count_3_6_months_late.median(), inplace = True)\n",
    "    df['Count_6_12_months_late'].fillna(train.Count_6_12_months_late.median(), inplace = True)\n",
    "    df['Count_more_than_12_months_late'].fillna(train.Count_more_than_12_months_late.median(), inplace = True)\n",
    "    df['application_underwriting_score'].fillna(train.application_underwriting_score.median(), inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                                  False\n",
       "perc_premium_paid_by_cash_credit    False\n",
       "age_in_days                         False\n",
       "Income                              False\n",
       "Count_3_6_months_late               False\n",
       "Count_6_12_months_late              False\n",
       "Count_more_than_12_months_late      False\n",
       "application_underwriting_score      False\n",
       "no_of_premiums_paid                 False\n",
       "sourcing_channel                    False\n",
       "residence_area_type                 False\n",
       "premium                             False\n",
       "dtype: bool"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.isnull().any(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                                  False\n",
       "perc_premium_paid_by_cash_credit    False\n",
       "age_in_days                         False\n",
       "Income                              False\n",
       "Count_3_6_months_late               False\n",
       "Count_6_12_months_late              False\n",
       "Count_more_than_12_months_late      False\n",
       "application_underwriting_score      False\n",
       "no_of_premiums_paid                 False\n",
       "sourcing_channel                    False\n",
       "residence_area_type                 False\n",
       "premium                             False\n",
       "renewal                             False\n",
       "dtype: bool"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.isnull().any(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# a little feature engineering\n",
    "#lets calculate monthly premium paid per number of premium paid\n",
    "train['premium_per_premium_paid'] = train.premium/train.no_of_premiums_paid\n",
    "test['premium_per_premium_paid'] = test.premium/test.no_of_premiums_paid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>perc_premium_paid_by_cash_credit</th>\n",
       "      <th>age_in_days</th>\n",
       "      <th>Income</th>\n",
       "      <th>Count_3_6_months_late</th>\n",
       "      <th>Count_6_12_months_late</th>\n",
       "      <th>Count_more_than_12_months_late</th>\n",
       "      <th>application_underwriting_score</th>\n",
       "      <th>no_of_premiums_paid</th>\n",
       "      <th>sourcing_channel</th>\n",
       "      <th>residence_area_type</th>\n",
       "      <th>premium</th>\n",
       "      <th>renewal</th>\n",
       "      <th>premium_per_premium_paid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>110936</td>\n",
       "      <td>0.429</td>\n",
       "      <td>12058</td>\n",
       "      <td>355060</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99.02</td>\n",
       "      <td>13</td>\n",
       "      <td>C</td>\n",
       "      <td>Urban</td>\n",
       "      <td>3300</td>\n",
       "      <td>1</td>\n",
       "      <td>253.846154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>41492</td>\n",
       "      <td>0.010</td>\n",
       "      <td>21546</td>\n",
       "      <td>315150</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99.89</td>\n",
       "      <td>21</td>\n",
       "      <td>A</td>\n",
       "      <td>Urban</td>\n",
       "      <td>18000</td>\n",
       "      <td>1</td>\n",
       "      <td>857.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>31300</td>\n",
       "      <td>0.917</td>\n",
       "      <td>17531</td>\n",
       "      <td>84140</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>98.69</td>\n",
       "      <td>7</td>\n",
       "      <td>C</td>\n",
       "      <td>Rural</td>\n",
       "      <td>3300</td>\n",
       "      <td>0</td>\n",
       "      <td>471.428571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>19415</td>\n",
       "      <td>0.049</td>\n",
       "      <td>15341</td>\n",
       "      <td>250510</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99.57</td>\n",
       "      <td>9</td>\n",
       "      <td>A</td>\n",
       "      <td>Urban</td>\n",
       "      <td>9600</td>\n",
       "      <td>1</td>\n",
       "      <td>1066.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>99379</td>\n",
       "      <td>0.052</td>\n",
       "      <td>31400</td>\n",
       "      <td>198680</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99.87</td>\n",
       "      <td>12</td>\n",
       "      <td>B</td>\n",
       "      <td>Urban</td>\n",
       "      <td>9600</td>\n",
       "      <td>1</td>\n",
       "      <td>800.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  perc_premium_paid_by_cash_credit  age_in_days  Income  \\\n",
       "0  110936                             0.429        12058  355060   \n",
       "1   41492                             0.010        21546  315150   \n",
       "2   31300                             0.917        17531   84140   \n",
       "3   19415                             0.049        15341  250510   \n",
       "4   99379                             0.052        31400  198680   \n",
       "\n",
       "   Count_3_6_months_late  Count_6_12_months_late  \\\n",
       "0                    0.0                     0.0   \n",
       "1                    0.0                     0.0   \n",
       "2                    2.0                     3.0   \n",
       "3                    0.0                     0.0   \n",
       "4                    0.0                     0.0   \n",
       "\n",
       "   Count_more_than_12_months_late  application_underwriting_score  \\\n",
       "0                             0.0                           99.02   \n",
       "1                             0.0                           99.89   \n",
       "2                             1.0                           98.69   \n",
       "3                             0.0                           99.57   \n",
       "4                             0.0                           99.87   \n",
       "\n",
       "   no_of_premiums_paid sourcing_channel residence_area_type  premium  renewal  \\\n",
       "0                   13                C               Urban     3300        1   \n",
       "1                   21                A               Urban    18000        1   \n",
       "2                    7                C               Rural     3300        0   \n",
       "3                    9                A               Urban     9600        1   \n",
       "4                   12                B               Urban     9600        1   \n",
       "\n",
       "   premium_per_premium_paid  \n",
       "0                253.846154  \n",
       "1                857.142857  \n",
       "2                471.428571  \n",
       "3               1066.666667  \n",
       "4                800.000000  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>perc_premium_paid_by_cash_credit</th>\n",
       "      <th>age_in_days</th>\n",
       "      <th>Income</th>\n",
       "      <th>Count_3_6_months_late</th>\n",
       "      <th>Count_6_12_months_late</th>\n",
       "      <th>Count_more_than_12_months_late</th>\n",
       "      <th>application_underwriting_score</th>\n",
       "      <th>no_of_premiums_paid</th>\n",
       "      <th>sourcing_channel</th>\n",
       "      <th>residence_area_type</th>\n",
       "      <th>premium</th>\n",
       "      <th>premium_per_premium_paid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>649</td>\n",
       "      <td>0.001</td>\n",
       "      <td>27384</td>\n",
       "      <td>51150</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99.89</td>\n",
       "      <td>7</td>\n",
       "      <td>A</td>\n",
       "      <td>Rural</td>\n",
       "      <td>3300</td>\n",
       "      <td>471.428571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>81136</td>\n",
       "      <td>0.124</td>\n",
       "      <td>23735</td>\n",
       "      <td>285140</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>98.93</td>\n",
       "      <td>19</td>\n",
       "      <td>A</td>\n",
       "      <td>Urban</td>\n",
       "      <td>11700</td>\n",
       "      <td>615.789474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>70762</td>\n",
       "      <td>1.000</td>\n",
       "      <td>17170</td>\n",
       "      <td>186030</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99.21</td>\n",
       "      <td>2</td>\n",
       "      <td>B</td>\n",
       "      <td>Urban</td>\n",
       "      <td>11700</td>\n",
       "      <td>5850.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53935</td>\n",
       "      <td>0.198</td>\n",
       "      <td>16068</td>\n",
       "      <td>123540</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99.00</td>\n",
       "      <td>11</td>\n",
       "      <td>B</td>\n",
       "      <td>Rural</td>\n",
       "      <td>5400</td>\n",
       "      <td>490.909091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15476</td>\n",
       "      <td>0.041</td>\n",
       "      <td>10591</td>\n",
       "      <td>200020</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99.17</td>\n",
       "      <td>14</td>\n",
       "      <td>A</td>\n",
       "      <td>Rural</td>\n",
       "      <td>9600</td>\n",
       "      <td>685.714286</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id  perc_premium_paid_by_cash_credit  age_in_days  Income  \\\n",
       "0    649                             0.001        27384   51150   \n",
       "1  81136                             0.124        23735  285140   \n",
       "2  70762                             1.000        17170  186030   \n",
       "3  53935                             0.198        16068  123540   \n",
       "4  15476                             0.041        10591  200020   \n",
       "\n",
       "   Count_3_6_months_late  Count_6_12_months_late  \\\n",
       "0                    0.0                     0.0   \n",
       "1                    0.0                     0.0   \n",
       "2                    0.0                     0.0   \n",
       "3                    0.0                     0.0   \n",
       "4                    1.0                     0.0   \n",
       "\n",
       "   Count_more_than_12_months_late  application_underwriting_score  \\\n",
       "0                             0.0                           99.89   \n",
       "1                             0.0                           98.93   \n",
       "2                             0.0                           99.21   \n",
       "3                             0.0                           99.00   \n",
       "4                             0.0                           99.17   \n",
       "\n",
       "   no_of_premiums_paid sourcing_channel residence_area_type  premium  \\\n",
       "0                    7                A               Rural     3300   \n",
       "1                   19                A               Urban    11700   \n",
       "2                    2                B               Urban    11700   \n",
       "3                   11                B               Rural     5400   \n",
       "4                   14                A               Rural     9600   \n",
       "\n",
       "   premium_per_premium_paid  \n",
       "0                471.428571  \n",
       "1                615.789474  \n",
       "2               5850.000000  \n",
       "3                490.909091  \n",
       "4                685.714286  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separete training set and testing set\n",
    "X = train.drop(('renewal'),axis=1)\n",
    "X = X.drop(('id'),axis=1).values\n",
    "\n",
    "y = train.iloc[:,12].values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.429, 12058, 355060, ..., 'Urban', 3300, 253.84615384615384],\n",
       "       [0.01, 21546, 315150, ..., 'Urban', 18000, 857.1428571428571],\n",
       "       [0.917, 17531, 84140, ..., 'Rural', 3300, 471.42857142857144],\n",
       "       ...,\n",
       "       [0.0, 23372, 305020, ..., 'Rural', 9600, 800.0],\n",
       "       [0.39799999999999996, 22641, 39330, ..., 'Rural', 5700, 712.5],\n",
       "       [0.55, 15709, 280140, ..., 'Urban', 9600, 1200.0]], dtype=object)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 0, ..., 1, 1, 0])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encoding categorical data\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
    "labelencoder = LabelEncoder()\n",
    "X[:, 9] = labelencoder.fit_transform(X[:, 9])\n",
    "X[:, 8] = labelencoder.fit_transform(X[:, 8])\n",
    "onehotencoder = OneHotEncoder(categorical_features = [8])\n",
    "X = onehotencoder.fit_transform(X).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 2., ..., 0., 0., 1.])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[:, 8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# splitting data into training set and testing set\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.20,random_state = 0)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train (63882, 16)\n",
      "X_test (15971, 16)\n",
      "y_train (63882,)\n",
      "y_test (15971,)\n"
     ]
    }
   ],
   "source": [
    "print('X_train',X_train.shape)\n",
    "print('X_test',X_test.shape)\n",
    "print('y_train',y_train.shape)\n",
    "print('y_test',y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.00e+00, 0.00e+00, 0.00e+00, ..., 1.00e+00, 1.20e+03, 3.00e+02],\n",
       "       [0.00e+00, 0.00e+00, 1.00e+00, ..., 1.00e+00, 3.30e+03, 1.32e+02],\n",
       "       [0.00e+00, 0.00e+00, 0.00e+00, ..., 1.00e+00, 1.17e+04, 7.80e+02],\n",
       "       ...,\n",
       "       [1.00e+00, 0.00e+00, 0.00e+00, ..., 1.00e+00, 1.17e+04, 1.17e+03],\n",
       "       [0.00e+00, 1.00e+00, 0.00e+00, ..., 1.00e+00, 1.17e+04, 1.30e+03],\n",
       "       [0.00e+00, 0.00e+00, 1.00e+00, ..., 1.00e+00, 1.17e+04, 9.75e+02]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scaling the dataset\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modeling "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression \n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from time import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Applying k-Fold Cross Validation to evaluate the model\n",
    "def k_fold(model,X_train,y_train):\n",
    "    accuracies = cross_val_score(estimator = model, X = X_train, y = y_train, cv = 10)\n",
    "    print('K-Fold cross validation mean:',accuracies.mean())\n",
    "    print('K-Fold cross validation std:',accuracies.std()) \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LogisticRegression classifier\n",
    "def logistic_classfier(features, labels, **kwargs):\n",
    "    ''' a function of logistic regression that will fit the classier to the data,\n",
    "    make prediction and print a classification report'''\n",
    "    \n",
    "    print('==================================')\n",
    "    print('Fitting logistic classifier')\n",
    "    print('==================================')\n",
    "    \n",
    "    # instantiate model\n",
    "    model_1 = LogisticRegression(random_state=0)\n",
    "    \n",
    "    # train model\n",
    "    start = time()\n",
    "    model_1.fit(features, labels)\n",
    "    end = time()\n",
    "    \n",
    "    # Training time\n",
    "    print('Training time:',end-start)\n",
    "    \n",
    "    # Making predictions\n",
    "    y_pred = model_1.predict(X_test)\n",
    "    \n",
    "    # get a sense of performance\n",
    "    accuracy_1 = classification_report(y_test,y_pred)\n",
    "    print('Accuracy:',accuracy_1)\n",
    "    \n",
    "    cm = confusion_matrix(y_test,y_pred)\n",
    "    print('Confusion matrix:')\n",
    "    print(cm)\n",
    "    \n",
    "    k_fold(model_1,features,labels)\n",
    "    \n",
    "\n",
    "    \n",
    "    return model_1\n",
    "\n",
    "# RandomForest classifier\n",
    "def random_forest_classfier(features, labels, **kwargs):\n",
    "    ''' a function of Random Forest that will fit the classier to the data,\n",
    "    make prediction and print a classification report'''  \n",
    "    \n",
    "    print('==================================')\n",
    "    print('Fitting RandomForest classifier')\n",
    "    print('==================================')\n",
    "    \n",
    "    # instantiate model\n",
    "    model_2 = RandomForestClassifier(n_estimators = 10, criterion = 'entropy', random_state = 0)\n",
    "    \n",
    "    # train model\n",
    "    start = time()\n",
    "    model_2.fit(features, labels) \n",
    "    end = time()\n",
    "    \n",
    "    # Training time\n",
    "    print('Training time:',end-start)  \n",
    "    # Making prediction\n",
    "    \n",
    "    y_pred = model_2.predict(X_test)\n",
    "    \n",
    "    # get a  sense of performance\n",
    "    accuracy_2 = classification_report(y_test,y_pred)\n",
    "    print('Accuracy:',accuracy_2)\n",
    "    \n",
    "    cm = confusion_matrix(y_test,y_pred)\n",
    "    print('Confusion matrix:')\n",
    "    print(cm)\n",
    "    \n",
    "    k_fold(model_2,features,labels)\n",
    "\n",
    "    \n",
    "    return model_2\n",
    "    \n",
    "# Decision tree classifier\n",
    "def decision_tree_classifier(features, labels, **kwargs):\n",
    "    ''' a function of xgboost that will fit the classier to the data,\n",
    "    make prediction and print a classification report'''  \n",
    "    \n",
    "    print('==================================')\n",
    "    print('Decision classfier classifier')\n",
    "    print('==================================')\n",
    "    \n",
    "    # instantiate model\n",
    "    model_3 = DecisionTreeClassifier(criterion = 'entropy', random_state = 0)\n",
    "    \n",
    "    # train model\n",
    "    start = time()\n",
    "    model_3.fit(features, labels) \n",
    "    end = time()\n",
    "       \n",
    "    # Training time\n",
    "    print('Training time:',end-start) \n",
    "    \n",
    "    # Making prediction\n",
    "    y_pred = model_3.predict(X_test)\n",
    "    \n",
    "    # get a sense of performance\n",
    "    accuracy_3 = classification_report(y_test,y_pred)\n",
    "    print('Decision tree accuracy:',accuracy_3)\n",
    "    \n",
    "    cm = confusion_matrix(y_test,y_pred)\n",
    "    print('Confusion matrix:')\n",
    "    print(cm)\n",
    "    \n",
    "    k_fold(model_3,features,labels)\n",
    "    \n",
    "    return model_3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==================================\n",
      "Fitting logistic classifier\n",
      "==================================\n",
      "Training time: 0.4453239440917969\n",
      "Accuracy:              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.55      0.13      0.21       977\n",
      "          1       0.95      0.99      0.97     14994\n",
      "\n",
      "avg / total       0.92      0.94      0.92     15971\n",
      "\n",
      "Confusion matrix:\n",
      "[[  125   852]\n",
      " [  103 14891]]\n",
      "K-Fold cross validation mean: 0.9390594747781197\n",
      "K-Fold cross validation std: 0.0011440454089608045\n",
      "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
      "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
      "          penalty='l2', random_state=0, solver='liblinear', tol=0.0001,\n",
      "          verbose=0, warm_start=False)\n"
     ]
    }
   ],
   "source": [
    "model_a =  logistic_classfier(X_train,y_train)\n",
    "print(model_a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==================================\n",
      "Fitting RandomForest classifier\n",
      "==================================\n",
      "Training time: 2.3179612159729004\n",
      "Accuracy:              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.39      0.17      0.24       977\n",
      "          1       0.95      0.98      0.96     14994\n",
      "\n",
      "avg / total       0.91      0.93      0.92     15971\n",
      "\n",
      "Confusion matrix:\n",
      "[[  166   811]\n",
      " [  263 14731]]\n",
      "K-Fold cross validation mean: 0.933611970998047\n",
      "K-Fold cross validation std: 0.0016588756448864222\n",
      "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',\n",
      "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
      "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "            min_samples_leaf=1, min_samples_split=2,\n",
      "            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
      "            oob_score=False, random_state=0, verbose=0, warm_start=False)\n"
     ]
    }
   ],
   "source": [
    "model_b = random_forest_classfier(X_train,y_train)\n",
    "print(model_b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==================================\n",
      "Decision classfier classifier\n",
      "==================================\n",
      "Training time: 1.2136297225952148\n",
      "Decision tree accuracy:              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.21      0.25      0.23       977\n",
      "          1       0.95      0.94      0.95     14994\n",
      "\n",
      "avg / total       0.91      0.90      0.90     15971\n",
      "\n",
      "Confusion matrix:\n",
      "[[  242   735]\n",
      " [  889 14105]]\n",
      "K-Fold cross validation mean: 0.8978901755345262\n",
      "K-Fold cross validation std: 0.004041463226566219\n",
      "DecisionTreeClassifier(class_weight=None, criterion='entropy', max_depth=None,\n",
      "            max_features=None, max_leaf_nodes=None,\n",
      "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "            min_samples_leaf=1, min_samples_split=2,\n",
      "            min_weight_fraction_leaf=0.0, presort=False, random_state=0,\n",
      "            splitter='best')\n"
     ]
    }
   ],
   "source": [
    "model_c = decision_tree_classifier(X_train,y_train)\n",
    "print(model_c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best accuracy 0.9390908237062083\n",
      "best parameters {'C': 0.1, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "# seems we will be working with logistic regression\n",
    "# Applying Grid Search to find the best model and the best parameters\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "parameters = [{'C': [0.1,1, 10, 100],'penalty': ['l1','l2']}]\n",
    "grid_search = GridSearchCV(estimator = model_a,\n",
    "                           param_grid = parameters,\n",
    "                           scoring = 'accuracy',\n",
    "                           cv = 10,\n",
    "                           n_jobs = -1)\n",
    "grid_search = grid_search.fit(X_train, y_train)\n",
    "best_accuracy = grid_search.best_score_\n",
    "best_parameters = grid_search.best_params_\n",
    "best_model = grid_search.best_estimator_\n",
    "\n",
    "print('best accuracy',best_accuracy)\n",
    "print('best parameters',best_parameters)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final predictions for submissions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['id', 'perc_premium_paid_by_cash_credit', 'age_in_days', 'Income',\n",
       "       'Count_3_6_months_late', 'Count_6_12_months_late',\n",
       "       'Count_more_than_12_months_late', 'application_underwriting_score',\n",
       "       'no_of_premiums_paid', 'sourcing_channel', 'residence_area_type',\n",
       "       'premium', 'premium_per_premium_paid'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "Validation = test.drop(('id'),axis = 1).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>perc_premium_paid_by_cash_credit</th>\n",
       "      <th>age_in_days</th>\n",
       "      <th>Income</th>\n",
       "      <th>Count_3_6_months_late</th>\n",
       "      <th>Count_6_12_months_late</th>\n",
       "      <th>Count_more_than_12_months_late</th>\n",
       "      <th>application_underwriting_score</th>\n",
       "      <th>no_of_premiums_paid</th>\n",
       "      <th>sourcing_channel</th>\n",
       "      <th>residence_area_type</th>\n",
       "      <th>premium</th>\n",
       "      <th>premium_per_premium_paid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>649</td>\n",
       "      <td>0.001</td>\n",
       "      <td>27384</td>\n",
       "      <td>51150</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99.89</td>\n",
       "      <td>7</td>\n",
       "      <td>A</td>\n",
       "      <td>Rural</td>\n",
       "      <td>3300</td>\n",
       "      <td>471.428571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>81136</td>\n",
       "      <td>0.124</td>\n",
       "      <td>23735</td>\n",
       "      <td>285140</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>98.93</td>\n",
       "      <td>19</td>\n",
       "      <td>A</td>\n",
       "      <td>Urban</td>\n",
       "      <td>11700</td>\n",
       "      <td>615.789474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>70762</td>\n",
       "      <td>1.000</td>\n",
       "      <td>17170</td>\n",
       "      <td>186030</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99.21</td>\n",
       "      <td>2</td>\n",
       "      <td>B</td>\n",
       "      <td>Urban</td>\n",
       "      <td>11700</td>\n",
       "      <td>5850.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53935</td>\n",
       "      <td>0.198</td>\n",
       "      <td>16068</td>\n",
       "      <td>123540</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99.00</td>\n",
       "      <td>11</td>\n",
       "      <td>B</td>\n",
       "      <td>Rural</td>\n",
       "      <td>5400</td>\n",
       "      <td>490.909091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15476</td>\n",
       "      <td>0.041</td>\n",
       "      <td>10591</td>\n",
       "      <td>200020</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99.17</td>\n",
       "      <td>14</td>\n",
       "      <td>A</td>\n",
       "      <td>Rural</td>\n",
       "      <td>9600</td>\n",
       "      <td>685.714286</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id  perc_premium_paid_by_cash_credit  age_in_days  Income  \\\n",
       "0    649                             0.001        27384   51150   \n",
       "1  81136                             0.124        23735  285140   \n",
       "2  70762                             1.000        17170  186030   \n",
       "3  53935                             0.198        16068  123540   \n",
       "4  15476                             0.041        10591  200020   \n",
       "\n",
       "   Count_3_6_months_late  Count_6_12_months_late  \\\n",
       "0                    0.0                     0.0   \n",
       "1                    0.0                     0.0   \n",
       "2                    0.0                     0.0   \n",
       "3                    0.0                     0.0   \n",
       "4                    1.0                     0.0   \n",
       "\n",
       "   Count_more_than_12_months_late  application_underwriting_score  \\\n",
       "0                             0.0                           99.89   \n",
       "1                             0.0                           98.93   \n",
       "2                             0.0                           99.21   \n",
       "3                             0.0                           99.00   \n",
       "4                             0.0                           99.17   \n",
       "\n",
       "   no_of_premiums_paid sourcing_channel residence_area_type  premium  \\\n",
       "0                    7                A               Rural     3300   \n",
       "1                   19                A               Urban    11700   \n",
       "2                    2                B               Urban    11700   \n",
       "3                   11                B               Rural     5400   \n",
       "4                   14                A               Rural     9600   \n",
       "\n",
       "   premium_per_premium_paid  \n",
       "0                471.428571  \n",
       "1                615.789474  \n",
       "2               5850.000000  \n",
       "3                490.909091  \n",
       "4                685.714286  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.001, 27384, 51150, ..., 'Rural', 3300, 471.42857142857144],\n",
       "       [0.124, 23735, 285140, ..., 'Urban', 11700, 615.7894736842105],\n",
       "       [1.0, 17170, 186030, ..., 'Urban', 11700, 5850.0],\n",
       "       ...,\n",
       "       [0.325, 18264, 196570, ..., 'Rural', 1200, 85.71428571428571],\n",
       "       [0.983, 16798, 212150, ..., 'Urban', 13800, 1380.0],\n",
       "       [0.005, 24467, 89190, ..., 'Urban', 5400, 490.90909090909093]],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encoding validation dataset\n",
    "Validation[:, 9] = labelencoder.fit_transform(Validation[:, 9])\n",
    "Validation[:, 8] = labelencoder.fit_transform(Validation[:, 8])\n",
    "onehotencoder = OneHotEncoder(categorical_features = [8])\n",
    "Validation = onehotencoder.fit_transform(Validation).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scaling validation dataset\n",
    "Validation = scaler.transform(Validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_proba = best_model.predict_proba(Validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.01025108, 0.98974892],\n",
       "       [0.02187682, 0.97812318],\n",
       "       [0.12342406, 0.87657594],\n",
       "       ...,\n",
       "       [0.04028193, 0.95971807],\n",
       "       [0.25364865, 0.74635135],\n",
       "       [0.02658734, 0.97341266]])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction_proba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_prediction = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_prediction['id'] = test.id\n",
    "final_prediction['renewal'] = prediction_proba[:,1]  #probability of renewal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>renewal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>649</td>\n",
       "      <td>0.989749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>81136</td>\n",
       "      <td>0.978123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>70762</td>\n",
       "      <td>0.876576</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53935</td>\n",
       "      <td>0.970145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15476</td>\n",
       "      <td>0.956428</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id   renewal\n",
       "0    649  0.989749\n",
       "1  81136  0.978123\n",
       "2  70762  0.876576\n",
       "3  53935  0.970145\n",
       "4  15476  0.956428"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_prediction.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculating incentives from the given formulae"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating a dataset a with values of delta_p\n",
    "import numpy as np\n",
    "x = pd.DataFrame(np.arange(0,1000,10), columns=['incentive'])\n",
    "x['delta_p'] = 0.2*(1-np.exp(-2*(1-np.exp(-x['incentive']/400))))\n",
    "\n",
    "# calculating optimised premium\n",
    "final_prediction['opt_premium'] = test['premium']*final_prediction['renewal']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>incentive</th>\n",
       "      <th>delta_p</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>0.009636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20</td>\n",
       "      <td>0.018587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>30</td>\n",
       "      <td>0.026911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>40</td>\n",
       "      <td>0.034662</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   incentive   delta_p\n",
       "0          0  0.000000\n",
       "1         10  0.009636\n",
       "2         20  0.018587\n",
       "3         30  0.026911\n",
       "4         40  0.034662"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>renewal</th>\n",
       "      <th>opt_premium</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>649</td>\n",
       "      <td>0.989749</td>\n",
       "      <td>3266.171430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>81136</td>\n",
       "      <td>0.978123</td>\n",
       "      <td>11444.041233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>70762</td>\n",
       "      <td>0.876576</td>\n",
       "      <td>10255.938473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53935</td>\n",
       "      <td>0.970145</td>\n",
       "      <td>5238.781085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15476</td>\n",
       "      <td>0.956428</td>\n",
       "      <td>9181.706263</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id   renewal   opt_premium\n",
       "0    649  0.989749   3266.171430\n",
       "1  81136  0.978123  11444.041233\n",
       "2  70762  0.876576  10255.938473\n",
       "3  53935  0.970145   5238.781085\n",
       "4  15476  0.956428   9181.706263"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_prediction.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "#finding values with maximum delta revenue\n",
    "f = []\n",
    "r = []\n",
    "for i in range(test.shape[0]):\n",
    "    n = test.iloc[i,-1]\n",
    "    p = test.iloc[i,-2]\n",
    "    x['total_revenue'] = (n + (x['delta_p'])*n) - (x['incentive']) \n",
    "    x['tot_p'] = p + (p * (x['delta_p']))\n",
    "    #y = x[x.tot_p <=1.000000]\n",
    "    y = x.sort_values(['total_revenue'], ascending=False)\n",
    "    incen = y.iloc[0, 0]\n",
    "    f.append(incen)\n",
    "    rev = y.iloc[0,2]\n",
    "    r.append(rev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "#appending final prediction dataset\n",
    "final_prediction['incentive'] = f\n",
    "final_prediction['reven'] =r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>renewal</th>\n",
       "      <th>opt_premium</th>\n",
       "      <th>incentive</th>\n",
       "      <th>reven</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>649</td>\n",
       "      <td>0.989749</td>\n",
       "      <td>3266.171430</td>\n",
       "      <td>0</td>\n",
       "      <td>471.428571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>81136</td>\n",
       "      <td>0.978123</td>\n",
       "      <td>11444.041233</td>\n",
       "      <td>0</td>\n",
       "      <td>615.789474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>70762</td>\n",
       "      <td>0.876576</td>\n",
       "      <td>10255.938473</td>\n",
       "      <td>290</td>\n",
       "      <td>6312.865728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53935</td>\n",
       "      <td>0.970145</td>\n",
       "      <td>5238.781085</td>\n",
       "      <td>0</td>\n",
       "      <td>490.909091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15476</td>\n",
       "      <td>0.956428</td>\n",
       "      <td>9181.706263</td>\n",
       "      <td>0</td>\n",
       "      <td>685.714286</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id   renewal   opt_premium  incentive        reven\n",
       "0    649  0.989749   3266.171430          0   471.428571\n",
       "1  81136  0.978123  11444.041233          0   615.789474\n",
       "2  70762  0.876576  10255.938473        290  6312.865728\n",
       "3  53935  0.970145   5238.781085          0   490.909091\n",
       "4  15476  0.956428   9181.706263          0   685.714286"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_prediction.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# will only submit prediction with id,renewal and incentive\n",
    "submission = final_prediction.iloc[:,[0,1,3]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>renewal</th>\n",
       "      <th>incentive</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>649</td>\n",
       "      <td>0.989749</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>81136</td>\n",
       "      <td>0.978123</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>70762</td>\n",
       "      <td>0.876576</td>\n",
       "      <td>290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53935</td>\n",
       "      <td>0.970145</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15476</td>\n",
       "      <td>0.956428</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id   renewal  incentive\n",
       "0    649  0.989749          0\n",
       "1  81136  0.978123          0\n",
       "2  70762  0.876576        290\n",
       "3  53935  0.970145          0\n",
       "4  15476  0.956428          0"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating submission file\n",
    "submission.to_csv(\"submission.csv\", index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}