24th AUGUST - Credit Classifications .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1000 entries, 0 to 999\n",
      "Data columns (total 14 columns):\n",
      " #   Column             Non-Null Count  Dtype \n",
      "---  ------             --------------  ----- \n",
      " 0   checkin_acc        1000 non-null   object\n",
      " 1   duration           1000 non-null   int64 \n",
      " 2   credit_history     1000 non-null   object\n",
      " 3   amount             1000 non-null   int64 \n",
      " 4   savings_acc        1000 non-null   object\n",
      " 5   present_emp_since  1000 non-null   object\n",
      " 6   inst_rate          1000 non-null   int64 \n",
      " 7   personal_status    1000 non-null   object\n",
      " 8   residing_since     1000 non-null   int64 \n",
      " 9   age                1000 non-null   int64 \n",
      " 10  inst_plans         1000 non-null   object\n",
      " 11  num_credits        1000 non-null   int64 \n",
      " 12  job                1000 non-null   object\n",
      " 13  status             1000 non-null   int64 \n",
      "dtypes: int64(7), object(7)\n",
      "memory usage: 109.5+ KB\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "credit_df = pd.read_csv( r\"C:\\Users\\Home\\Desktop\\MLAP - Getting Started with Python\\LOGISTIC REGRESSION\\Classification Model - LR\\German Credit Data.csv\" )\n",
    "credit_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>credit_history</th>\n",
       "      <th>amount</th>\n",
       "      <th>savings_acc</th>\n",
       "      <th>present_emp_since</th>\n",
       "      <th>inst_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>A34</td>\n",
       "      <td>1169</td>\n",
       "      <td>A65</td>\n",
       "      <td>A75</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>48</td>\n",
       "      <td>A32</td>\n",
       "      <td>5951</td>\n",
       "      <td>A61</td>\n",
       "      <td>A73</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12</td>\n",
       "      <td>A34</td>\n",
       "      <td>2096</td>\n",
       "      <td>A61</td>\n",
       "      <td>A74</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>42</td>\n",
       "      <td>A32</td>\n",
       "      <td>7882</td>\n",
       "      <td>A61</td>\n",
       "      <td>A74</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>24</td>\n",
       "      <td>A33</td>\n",
       "      <td>4870</td>\n",
       "      <td>A61</td>\n",
       "      <td>A73</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   duration credit_history  amount savings_acc present_emp_since  inst_rate\n",
       "0         6            A34    1169         A65               A75          4\n",
       "1        48            A32    5951         A61               A73          2\n",
       "2        12            A34    2096         A61               A74          2\n",
       "3        42            A32    7882         A61               A74          2\n",
       "4        24            A33    4870         A61               A73          3"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "credit_df.iloc[0:5,1:7]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>personal_status</th>\n",
       "      <th>residing_since</th>\n",
       "      <th>age</th>\n",
       "      <th>inst_plans</th>\n",
       "      <th>num_credits</th>\n",
       "      <th>job</th>\n",
       "      <th>status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A93</td>\n",
       "      <td>4</td>\n",
       "      <td>67</td>\n",
       "      <td>A143</td>\n",
       "      <td>2</td>\n",
       "      <td>A173</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A92</td>\n",
       "      <td>2</td>\n",
       "      <td>22</td>\n",
       "      <td>A143</td>\n",
       "      <td>1</td>\n",
       "      <td>A173</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A93</td>\n",
       "      <td>3</td>\n",
       "      <td>49</td>\n",
       "      <td>A143</td>\n",
       "      <td>1</td>\n",
       "      <td>A172</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A93</td>\n",
       "      <td>4</td>\n",
       "      <td>45</td>\n",
       "      <td>A143</td>\n",
       "      <td>1</td>\n",
       "      <td>A173</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A93</td>\n",
       "      <td>4</td>\n",
       "      <td>53</td>\n",
       "      <td>A143</td>\n",
       "      <td>2</td>\n",
       "      <td>A173</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  personal_status  residing_since  age inst_plans  num_credits   job  status\n",
       "0             A93               4   67       A143            2  A173       0\n",
       "1             A92               2   22       A143            1  A173       1\n",
       "2             A93               3   49       A143            1  A172       0\n",
       "3             A93               4   45       A143            1  A173       0\n",
       "4             A93               4   53       A143            2  A173       1"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "credit_df.iloc[0:5,7:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    700\n",
       "1    300\n",
       "Name: status, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "credit_df.status.value_counts() \n",
    "# min number of sample in each class\n",
    "# \n",
    "## 0 - Good Credit and 1 - Bad Credit (Success Class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['checkin_acc',\n",
       " 'duration',\n",
       " 'credit_history',\n",
       " 'amount',\n",
       " 'savings_acc',\n",
       " 'present_emp_since',\n",
       " 'inst_rate',\n",
       " 'personal_status',\n",
       " 'residing_since',\n",
       " 'age',\n",
       " 'inst_plans',\n",
       " 'num_credits',\n",
       " 'job']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_features = list( credit_df.columns )\n",
    "X_features.remove( 'status' )\n",
    "X_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded_credit_df = pd.get_dummies( credit_df[X_features],drop_first = True )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['duration',\n",
       " 'amount',\n",
       " 'inst_rate',\n",
       " 'residing_since',\n",
       " 'age',\n",
       " 'num_credits',\n",
       " 'checkin_acc_A12',\n",
       " 'checkin_acc_A13',\n",
       " 'checkin_acc_A14',\n",
       " 'credit_history_A31',\n",
       " 'credit_history_A32',\n",
       " 'credit_history_A33',\n",
       " 'credit_history_A34',\n",
       " 'savings_acc_A62',\n",
       " 'savings_acc_A63',\n",
       " 'savings_acc_A64',\n",
       " 'savings_acc_A65',\n",
       " 'present_emp_since_A72',\n",
       " 'present_emp_since_A73',\n",
       " 'present_emp_since_A74',\n",
       " 'present_emp_since_A75',\n",
       " 'personal_status_A92',\n",
       " 'personal_status_A93',\n",
       " 'personal_status_A94',\n",
       " 'inst_plans_A142',\n",
       " 'inst_plans_A143',\n",
       " 'job_A172',\n",
       " 'job_A173',\n",
       " 'job_A174']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(encoded_credit_df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>checkin_acc_A12</th>\n",
       "      <th>checkin_acc_A13</th>\n",
       "      <th>checkin_acc_A14</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   checkin_acc_A12  checkin_acc_A13  checkin_acc_A14\n",
       "0                0                0                0\n",
       "1                1                0                0\n",
       "2                0                0                1\n",
       "3                0                0                0\n",
       "4                0                0                0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_credit_df[['checkin_acc_A12',\n",
    "'checkin_acc_A13',\n",
    "'checkin_acc_A14']].head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import statsmodels.api as sm\n",
    "Y = credit_df.status\n",
    "X = sm.add_constant( encoded_credit_df )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Splitting into Train and Validation Sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,Y,test_size = 0.3,random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.488938\n",
      "         Iterations 6\n"
     ]
    }
   ],
   "source": [
    "import statsmodels.api as sm\n",
    "logit = sm.Logit(y_train, X_train)\n",
    "logit_model = logit.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "        <td>Model:</td>              <td>Logit</td>      <td>Pseudo R-squared:</td>    <td>0.198</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <td>Dependent Variable:</td>      <td>status</td>            <td>AIC:</td>         <td>744.5132</td> \n",
       "</tr>\n",
       "<tr>\n",
       "         <td>Date:</td>        <td>2020-08-24 14:31</td>       <td>BIC:</td>         <td>881.0456</td> \n",
       "</tr>\n",
       "<tr>\n",
       "   <td>No. Observations:</td>         <td>700</td>        <td>Log-Likelihood:</td>    <td>-342.26</td> \n",
       "</tr>\n",
       "<tr>\n",
       "       <td>Df Model:</td>             <td>29</td>            <td>LL-Null:</td>        <td>-426.75</td> \n",
       "</tr>\n",
       "<tr>\n",
       "     <td>Df Residuals:</td>           <td>670</td>         <td>LLR p-value:</td>    <td>1.0630e-21</td>\n",
       "</tr>\n",
       "<tr>\n",
       "      <td>Converged:</td>           <td>1.0000</td>           <td>Scale:</td>         <td>1.0000</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "    <td>No. Iterations:</td>        <td>6.0000</td>              <td></td>               <td></td>     \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "            <td></td>             <th>Coef.</th>  <th>Std.Err.</th>    <th>z</th>     <th>P>|z|</th> <th>[0.025</th>  <th>0.975]</th> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>const</th>                 <td>-0.1511</td>  <td>1.1349</td>  <td>-0.1331</td> <td>0.8941</td> <td>-2.3754</td> <td>2.0733</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>duration</th>              <td>0.0206</td>   <td>0.0104</td>  <td>1.9927</td>  <td>0.0463</td> <td>0.0003</td>  <td>0.0409</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>amount</th>                <td>0.0001</td>   <td>0.0000</td>  <td>2.3765</td>  <td>0.0175</td> <td>0.0000</td>  <td>0.0002</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>inst_rate</th>             <td>0.3064</td>   <td>0.0986</td>  <td>3.1083</td>  <td>0.0019</td> <td>0.1132</td>  <td>0.4996</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>residing_since</th>        <td>0.0967</td>   <td>0.0920</td>  <td>1.0511</td>  <td>0.2932</td> <td>-0.0836</td> <td>0.2771</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>age</th>                   <td>-0.0227</td>  <td>0.0103</td>  <td>-2.2131</td> <td>0.0269</td> <td>-0.0428</td> <td>-0.0026</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>num_credits</th>           <td>0.2854</td>   <td>0.2139</td>  <td>1.3342</td>  <td>0.1821</td> <td>-0.1338</td> <td>0.7045</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>checkin_acc_A12</th>       <td>-0.4126</td>  <td>0.2391</td>  <td>-1.7260</td> <td>0.0843</td> <td>-0.8812</td> <td>0.0559</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>checkin_acc_A13</th>       <td>-0.9053</td>  <td>0.4338</td>  <td>-2.0868</td> <td>0.0369</td> <td>-1.7556</td> <td>-0.0550</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>checkin_acc_A14</th>       <td>-1.6052</td>  <td>0.2586</td>  <td>-6.2073</td> <td>0.0000</td> <td>-2.1120</td> <td>-1.0983</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>credit_history_A31</th>    <td>0.1532</td>   <td>0.5795</td>  <td>0.2643</td>  <td>0.7916</td> <td>-0.9827</td> <td>1.2890</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>credit_history_A32</th>    <td>-0.4960</td>  <td>0.4411</td>  <td>-1.1245</td> <td>0.2608</td> <td>-1.3604</td> <td>0.3685</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>credit_history_A33</th>    <td>-0.8881</td>  <td>0.5022</td>  <td>-1.7683</td> <td>0.0770</td> <td>-1.8724</td> <td>0.0962</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>credit_history_A34</th>    <td>-1.4124</td>  <td>0.4528</td>  <td>-3.1190</td> <td>0.0018</td> <td>-2.2999</td> <td>-0.5249</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>savings_acc_A62</th>       <td>-0.0496</td>  <td>0.3208</td>  <td>-0.1545</td> <td>0.8772</td> <td>-0.6782</td> <td>0.5791</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>savings_acc_A63</th>       <td>-0.6640</td>  <td>0.4818</td>  <td>-1.3779</td> <td>0.1682</td> <td>-1.6084</td> <td>0.2804</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>savings_acc_A64</th>       <td>-1.1099</td>  <td>0.6019</td>  <td>-1.8439</td> <td>0.0652</td> <td>-2.2896</td> <td>0.0699</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>savings_acc_A65</th>       <td>-0.6061</td>  <td>0.2745</td>  <td>-2.2080</td> <td>0.0272</td> <td>-1.1441</td> <td>-0.0681</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>present_emp_since_A72</th> <td>0.0855</td>   <td>0.4722</td>  <td>0.1810</td>  <td>0.8564</td> <td>-0.8401</td> <td>1.0110</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>present_emp_since_A73</th> <td>-0.0339</td>  <td>0.4492</td>  <td>-0.0754</td> <td>0.9399</td> <td>-0.9142</td> <td>0.8465</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>present_emp_since_A74</th> <td>-0.3789</td>  <td>0.4790</td>  <td>-0.7910</td> <td>0.4289</td> <td>-1.3178</td> <td>0.5600</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>present_emp_since_A75</th> <td>-0.2605</td>  <td>0.4554</td>  <td>-0.5721</td> <td>0.5673</td> <td>-1.1532</td> <td>0.6321</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>personal_status_A92</th>   <td>-0.0069</td>  <td>0.4841</td>  <td>-0.0142</td> <td>0.9887</td> <td>-0.9557</td> <td>0.9419</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>personal_status_A93</th>   <td>-0.4426</td>  <td>0.4764</td>  <td>-0.9291</td> <td>0.3528</td> <td>-1.3762</td> <td>0.4911</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>personal_status_A94</th>   <td>-0.3080</td>  <td>0.5554</td>  <td>-0.5546</td> <td>0.5792</td> <td>-1.3967</td> <td>0.7806</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>inst_plans_A142</th>       <td>-0.2976</td>  <td>0.5157</td>  <td>-0.5772</td> <td>0.5638</td> <td>-1.3084</td> <td>0.7131</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>inst_plans_A143</th>       <td>-0.4458</td>  <td>0.2771</td>  <td>-1.6086</td> <td>0.1077</td> <td>-0.9889</td> <td>0.0974</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>job_A172</th>              <td>-0.0955</td>  <td>0.7681</td>  <td>-0.1243</td> <td>0.9011</td> <td>-1.6009</td> <td>1.4100</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>job_A173</th>              <td>-0.0198</td>  <td>0.7378</td>  <td>-0.0269</td> <td>0.9786</td> <td>-1.4658</td> <td>1.4262</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>job_A174</th>              <td>-0.0428</td>  <td>0.7371</td>  <td>-0.0581</td> <td>0.9537</td> <td>-1.4876</td> <td>1.4019</td> \n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary2.Summary'>\n",
       "\"\"\"\n",
       "                           Results: Logit\n",
       "=====================================================================\n",
       "Model:                Logit             Pseudo R-squared:  0.198     \n",
       "Dependent Variable:   status            AIC:               744.5132  \n",
       "Date:                 2020-08-24 14:31  BIC:               881.0456  \n",
       "No. Observations:     700               Log-Likelihood:    -342.26   \n",
       "Df Model:             29                LL-Null:           -426.75   \n",
       "Df Residuals:         670               LLR p-value:       1.0630e-21\n",
       "Converged:            1.0000            Scale:             1.0000    \n",
       "No. Iterations:       6.0000                                         \n",
       "---------------------------------------------------------------------\n",
       "                       Coef.  Std.Err.    z    P>|z|   [0.025  0.975]\n",
       "---------------------------------------------------------------------\n",
       "const                 -0.1511   1.1349 -0.1331 0.8941 -2.3754  2.0733\n",
       "duration               0.0206   0.0104  1.9927 0.0463  0.0003  0.0409\n",
       "amount                 0.0001   0.0000  2.3765 0.0175  0.0000  0.0002\n",
       "inst_rate              0.3064   0.0986  3.1083 0.0019  0.1132  0.4996\n",
       "residing_since         0.0967   0.0920  1.0511 0.2932 -0.0836  0.2771\n",
       "age                   -0.0227   0.0103 -2.2131 0.0269 -0.0428 -0.0026\n",
       "num_credits            0.2854   0.2139  1.3342 0.1821 -0.1338  0.7045\n",
       "checkin_acc_A12       -0.4126   0.2391 -1.7260 0.0843 -0.8812  0.0559\n",
       "checkin_acc_A13       -0.9053   0.4338 -2.0868 0.0369 -1.7556 -0.0550\n",
       "checkin_acc_A14       -1.6052   0.2586 -6.2073 0.0000 -2.1120 -1.0983\n",
       "credit_history_A31     0.1532   0.5795  0.2643 0.7916 -0.9827  1.2890\n",
       "credit_history_A32    -0.4960   0.4411 -1.1245 0.2608 -1.3604  0.3685\n",
       "credit_history_A33    -0.8881   0.5022 -1.7683 0.0770 -1.8724  0.0962\n",
       "credit_history_A34    -1.4124   0.4528 -3.1190 0.0018 -2.2999 -0.5249\n",
       "savings_acc_A62       -0.0496   0.3208 -0.1545 0.8772 -0.6782  0.5791\n",
       "savings_acc_A63       -0.6640   0.4818 -1.3779 0.1682 -1.6084  0.2804\n",
       "savings_acc_A64       -1.1099   0.6019 -1.8439 0.0652 -2.2896  0.0699\n",
       "savings_acc_A65       -0.6061   0.2745 -2.2080 0.0272 -1.1441 -0.0681\n",
       "present_emp_since_A72  0.0855   0.4722  0.1810 0.8564 -0.8401  1.0110\n",
       "present_emp_since_A73 -0.0339   0.4492 -0.0754 0.9399 -0.9142  0.8465\n",
       "present_emp_since_A74 -0.3789   0.4790 -0.7910 0.4289 -1.3178  0.5600\n",
       "present_emp_since_A75 -0.2605   0.4554 -0.5721 0.5673 -1.1532  0.6321\n",
       "personal_status_A92   -0.0069   0.4841 -0.0142 0.9887 -0.9557  0.9419\n",
       "personal_status_A93   -0.4426   0.4764 -0.9291 0.3528 -1.3762  0.4911\n",
       "personal_status_A94   -0.3080   0.5554 -0.5546 0.5792 -1.3967  0.7806\n",
       "inst_plans_A142       -0.2976   0.5157 -0.5772 0.5638 -1.3084  0.7131\n",
       "inst_plans_A143       -0.4458   0.2771 -1.6086 0.1077 -0.9889  0.0974\n",
       "job_A172              -0.0955   0.7681 -0.1243 0.9011 -1.6009  1.4100\n",
       "job_A173              -0.0198   0.7378 -0.0269 0.9786 -1.4658  1.4262\n",
       "job_A174              -0.0428   0.7371 -0.0581 0.9537 -1.4876  1.4019\n",
       "=====================================================================\n",
       "\n",
       "\"\"\""
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logit_model.summary2()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Dignostics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_significant_vars( lm ):\n",
    "    var_p_vals_df = pd.DataFrame( lm.pvalues )\n",
    "    var_p_vals_df['vars'] = var_p_vals_df.index\n",
    "    var_p_vals_df.columns = ['pvals', 'vars']\n",
    "    return list( var_p_vals_df[var_p_vals_df.pvals <= 0.05]['vars'] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['duration',\n",
       " 'amount',\n",
       " 'inst_rate',\n",
       " 'age',\n",
       " 'checkin_acc_A13',\n",
       " 'checkin_acc_A14',\n",
       " 'credit_history_A34',\n",
       " 'savings_acc_A65']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "significant_vars = get_significant_vars( logit_model )\n",
    "significant_vars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.511350\n",
      "         Iterations 6\n"
     ]
    }
   ],
   "source": [
    "final_logit = sm.Logit( y_train,sm.add_constant( X_train[significant_vars] ) ).fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "        <td>Model:</td>              <td>Logit</td>      <td>Pseudo R-squared:</td>    <td>0.161</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <td>Dependent Variable:</td>      <td>status</td>            <td>AIC:</td>         <td>733.8898</td> \n",
       "</tr>\n",
       "<tr>\n",
       "         <td>Date:</td>        <td>2020-08-24 14:32</td>       <td>BIC:</td>         <td>774.8495</td> \n",
       "</tr>\n",
       "<tr>\n",
       "   <td>No. Observations:</td>         <td>700</td>        <td>Log-Likelihood:</td>    <td>-357.94</td> \n",
       "</tr>\n",
       "<tr>\n",
       "       <td>Df Model:</td>              <td>8</td>            <td>LL-Null:</td>        <td>-426.75</td> \n",
       "</tr>\n",
       "<tr>\n",
       "     <td>Df Residuals:</td>           <td>691</td>         <td>LLR p-value:</td>    <td>7.4185e-26</td>\n",
       "</tr>\n",
       "<tr>\n",
       "      <td>Converged:</td>           <td>1.0000</td>           <td>Scale:</td>         <td>1.0000</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "    <td>No. Iterations:</td>        <td>6.0000</td>              <td></td>               <td></td>     \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "           <td></td>           <th>Coef.</th>  <th>Std.Err.</th>    <th>z</th>     <th>P>|z|</th> <th>[0.025</th>  <th>0.975]</th> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>const</th>              <td>-0.8969</td>  <td>0.4364</td>  <td>-2.0551</td> <td>0.0399</td> <td>-1.7523</td> <td>-0.0415</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>duration</th>           <td>0.0197</td>   <td>0.0098</td>  <td>2.0033</td>  <td>0.0451</td> <td>0.0004</td>  <td>0.0390</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>amount</th>             <td>0.0001</td>   <td>0.0000</td>  <td>2.3205</td>  <td>0.0203</td> <td>0.0000</td>  <td>0.0002</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>inst_rate</th>          <td>0.2811</td>   <td>0.0929</td>  <td>3.0264</td>  <td>0.0025</td> <td>0.0991</td>  <td>0.4632</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>age</th>                <td>-0.0216</td>  <td>0.0089</td>  <td>-2.4207</td> <td>0.0155</td> <td>-0.0392</td> <td>-0.0041</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>checkin_acc_A13</th>    <td>-0.8038</td>  <td>0.4081</td>  <td>-1.9697</td> <td>0.0489</td> <td>-1.6037</td> <td>-0.0040</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>checkin_acc_A14</th>    <td>-1.5452</td>  <td>0.2187</td>  <td>-7.0649</td> <td>0.0000</td> <td>-1.9738</td> <td>-1.1165</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>credit_history_A34</th> <td>-0.8781</td>  <td>0.2319</td>  <td>-3.7858</td> <td>0.0002</td> <td>-1.3327</td> <td>-0.4235</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>savings_acc_A65</th>    <td>-0.5448</td>  <td>0.2581</td>  <td>-2.1108</td> <td>0.0348</td> <td>-1.0507</td> <td>-0.0389</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary2.Summary'>\n",
       "\"\"\"\n",
       "                          Results: Logit\n",
       "==================================================================\n",
       "Model:               Logit            Pseudo R-squared: 0.161     \n",
       "Dependent Variable:  status           AIC:              733.8898  \n",
       "Date:                2020-08-24 14:32 BIC:              774.8495  \n",
       "No. Observations:    700              Log-Likelihood:   -357.94   \n",
       "Df Model:            8                LL-Null:          -426.75   \n",
       "Df Residuals:        691              LLR p-value:      7.4185e-26\n",
       "Converged:           1.0000           Scale:            1.0000    \n",
       "No. Iterations:      6.0000                                       \n",
       "------------------------------------------------------------------\n",
       "                    Coef.  Std.Err.    z    P>|z|   [0.025  0.975]\n",
       "------------------------------------------------------------------\n",
       "const              -0.8969   0.4364 -2.0551 0.0399 -1.7523 -0.0415\n",
       "duration            0.0197   0.0098  2.0033 0.0451  0.0004  0.0390\n",
       "amount              0.0001   0.0000  2.3205 0.0203  0.0000  0.0002\n",
       "inst_rate           0.2811   0.0929  3.0264 0.0025  0.0991  0.4632\n",
       "age                -0.0216   0.0089 -2.4207 0.0155 -0.0392 -0.0041\n",
       "checkin_acc_A13    -0.8038   0.4081 -1.9697 0.0489 -1.6037 -0.0040\n",
       "checkin_acc_A14    -1.5452   0.2187 -7.0649 0.0000 -1.9738 -1.1165\n",
       "credit_history_A34 -0.8781   0.2319 -3.7858 0.0002 -1.3327 -0.4235\n",
       "savings_acc_A65    -0.5448   0.2581 -2.1108 0.0348 -1.0507 -0.0389\n",
       "==================================================================\n",
       "\n",
       "\"\"\""
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_logit.summary2()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predicting on Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_df = pd.DataFrame( { \"actual\": y_test,\"predicted_prob\": final_logit.predict(sm.add_constant( X_test[significant_vars] ) ) } )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>predicted_prob</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>557</th>\n",
       "      <td>1</td>\n",
       "      <td>0.080493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>798</th>\n",
       "      <td>0</td>\n",
       "      <td>0.076653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>977</th>\n",
       "      <td>0</td>\n",
       "      <td>0.345979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>0</td>\n",
       "      <td>0.249919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>575</th>\n",
       "      <td>0</td>\n",
       "      <td>0.062264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544</th>\n",
       "      <td>0</td>\n",
       "      <td>0.040768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332</th>\n",
       "      <td>1</td>\n",
       "      <td>0.833093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>917</th>\n",
       "      <td>1</td>\n",
       "      <td>0.370667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>678</th>\n",
       "      <td>0</td>\n",
       "      <td>0.388392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363</th>\n",
       "      <td>0</td>\n",
       "      <td>0.088952</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     actual  predicted_prob\n",
       "557       1        0.080493\n",
       "798       0        0.076653\n",
       "977       0        0.345979\n",
       "136       0        0.249919\n",
       "575       0        0.062264\n",
       "544       0        0.040768\n",
       "332       1        0.833093\n",
       "917       1        0.370667\n",
       "678       0        0.388392\n",
       "363       0        0.088952"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_df.sample(10, random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>predicted_prob</th>\n",
       "      <th>predicted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>557</th>\n",
       "      <td>1</td>\n",
       "      <td>0.080493</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>798</th>\n",
       "      <td>0</td>\n",
       "      <td>0.076653</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>977</th>\n",
       "      <td>0</td>\n",
       "      <td>0.345979</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>0</td>\n",
       "      <td>0.249919</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>575</th>\n",
       "      <td>0</td>\n",
       "      <td>0.062264</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544</th>\n",
       "      <td>0</td>\n",
       "      <td>0.040768</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332</th>\n",
       "      <td>1</td>\n",
       "      <td>0.833093</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>917</th>\n",
       "      <td>1</td>\n",
       "      <td>0.370667</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>678</th>\n",
       "      <td>0</td>\n",
       "      <td>0.388392</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363</th>\n",
       "      <td>0</td>\n",
       "      <td>0.088952</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     actual  predicted_prob  predicted\n",
       "557       1        0.080493          0\n",
       "798       0        0.076653          0\n",
       "977       0        0.345979          0\n",
       "136       0        0.249919          0\n",
       "575       0        0.062264          0\n",
       "544       0        0.040768          0\n",
       "332       1        0.833093          1\n",
       "917       1        0.370667          0\n",
       "678       0        0.388392          0\n",
       "363       0        0.088952          0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_df['predicted'] = y_pred_df.predicted_prob.map(lambda x: 1 if x > 0.5 else 0)\n",
    "y_pred_df.sample(10, random_state = 42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating a Confusion Matrix\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sn\n",
    "%matplotlib inline\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_cm( actual, predicted ):\n",
    "## Cret\n",
    "    cm = metrics.confusion_matrix( actual, predicted, [1,0] )\n",
    "    sn.heatmap(cm, annot=True, fmt='.2f',\n",
    "    xticklabels = [\"Bad credit\", \"Good Credit\"] ,\n",
    "    yticklabels = [\"Bad credit\", \"Good Credit\"] )\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAEGCAYAAABFBX+4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd5xU1f3G8c+zFBUVUESkChrUqFEENagYQY1dQU1ijVgihhBLjDUajSa22KKx/bDEligYe+wlgiYCAQUBBcWCIggIAgoW2P3+/rh3cYQts7tz2Z3lefu6r5175t5zzsD65cy5pygiMDOz4lFS3xUwM7OaceA2MysyDtxmZkXGgdvMrMg4cJuZFZmm9V2ByrRttbmHu9hK9mm9VX1XwRqge6Y/pLrmsfTT9/KOOc022KTO5dWFW9xmZkWmwba4zcxWqbLS+q5B3hy4zcwASpfVdw3y5sBtZgZElNV3FfLmwG1mBlDmwG1mVlzc4jYzKzJF9HDSwwHNzCBpced7VEPSHZLmSJqUk9ZD0ihJ4yWNlbRjmi5J10uaJukNST2ry9+B28wMiNJleR95uBPYZ4W0PwMXRUQP4IL0HGBfoHt6DAJuri5zB24zM0geTuZ7VCMiRgLzV0wGWqavWwEz09f9gbsjMQpoLal9Vfm7j9vMDGr0cFLSIJLWcbmhETG0mttOA56RdBVJo3nnNL0j8FHOdTPStFmVZeTAbWYGNXo4mQbp6gL1igYDv4mIByX9DLgd2BOoaN2TKtdNcVeJmRkU9OFkJQYCD6WvHwB2TF/PADrnXNeJb7tRKuTAbWYGyZT3fI/amQnslr7eHXgnff0YcEw6uqQ3sDAiKu0mAXeVmJklCjhzUtJ9QF9gA0kzgAuBE4HrJDUFvuLbPvIngf2AacAS4Ljq8nfgNjMDIgo3AScijqjkrV4VXBvAkJrk78BtZgae8m5mVnS8yJSZWZFxi9vMrMiULq3vGuTNgdvMDNxVYmZWdNxVYmZWZNziNjMrMg7cZmbFJfxw0sysyLiP28ysyLirxMysyLjFbWZWZNziNjMrMm5xm5kVmWW13iBhlXPgNjMDt7jNzIpOEfVxe89JMzMo6GbBku6QNEfSpBXST5Y0VdJkSX/OST9X0rT0vb2ry98tbjMzKHSL+07gBuDu8gRJ/YD+wDYR8bWkDdP0LYHDga2ADsDzkjaLKvZSc4vbzAwK2uKOiJHA/BWSBwOXR8TX6TVz0vT+wP0R8XVEvE+yafCOVeXvwG1mBsmokjwPSYMkjc05BlVfAJsBu0oaLWmEpB3S9I7ARznXzUjTKuWuEjMzgIgaXBpDgaE1LKEpsB7QG9gBGC5pE0AVFVFdRmZmlv2okhnAQxERwBhJZcAGaXrnnOs6ATOryshdJWZmkATufI/aeQTYHUDSZkBz4FPgMeBwSWtI6gZ0B8ZUlZFb3GZmUNAJOJLuA/oCG0iaAVwI3AHckQ4R/AYYmLa+J0saDrwJLAOGVDWiBBy4zcwSpVXGyhqJiCMqeevoSq6/BLgk3/wduM3MoKhmTjpwm5mBA7eZWdHxIlNmZsUlyvIfx13fHLjNzMBdJWZmRaeAo0qy5sBtZgZucZuZFZ0iCtye8r6KrbFGc5558QH+/cqjvDzqX5x17skAdNm4E0+/MJzRrz3DrX+7lmbNmlV4/6mnD2LM68/y6tin6bdHn+Xpu++xK6+OfZoxrz/LKb85cZV8FiusFi1bcPLNZ3LFC9dz+QvX872em7Hjfjtx2XN/4a73/0m3H2xa6b0/2G07/vziX7lqxI0cMPjg5eltO2/IHx65nCtfuoEhN/yWJs3cVqtURP5HPcsscKdz7qtNW918/fU3HHLgQPr16U+/PgPYfc9d6bX9tlxw0RncctOd/LDn3ixYsIijjvnJSvdutvmmDDhkf/r8cH8OO/QXXHH1hZSUlFBSUsLlV1/A4T/5BbvsuD8HH3oAm21e+f/k1jAdfeEJvDHidc7e4xTO2+d0Zk6bwYy3P+S6k/7M1NFvVnqfSkoY+McTuXLgnzh7z1PZ6aBd6dC9EwCHnfNznr79cc7s+2sWL/yCvoftsao+TvHJfq2Sgsmyxf1gBWn/zLC8orF48RIAmjVrSrNmTYkI+vyoN48/8gwAw/7xMPvtv/L/YPvuvwePPPQE33yzlA+nz+CD96bTs9c29Oy1DR+8N53pH8xg6dKlPPLQE+xbwf3WcK25zlps8cMtGXH/8wCULl3GkkVLmDntYz55r8qF4ti0x/eY/cEs5n40m9Klyxj1+Cv0+nGyDv+WO/+AMU++CsArD/6bnntVuT7/6q0s8j/qWcG/N0nagmQLnlaSDsl5qyWwZqHLK0YlJSW8MOIhum3Shdtv+wcfvP8RixYuojR9qj1z5ids1L7dSve1b9+Osf+bsPx85szZtO+QXPfxx598m/7xbHptv03Gn8IKacMu7Vg0bxGDrvo1nbfsygcT3+PeP9zO119+Xe29623Uhvmz5i0/nz9rHptu15111luXJYsWU1Zatjx9/Y3aZPYZil4RjSrJosW9OXAA0Bo4MOfoCVTZ+Zq7q8RX3yzIoGoNQ1lZGf12HcA2W+5Gz57bsNnmm6x0TVTQjyatvN56RFSabsWjSZMmdN16E1649xl+v98ZfL3kKw741SHV30glq/BH5b8vVrEoK8v7qG8Fb3FHxKPAo5J2iohXa3jv8l0l2rbavNH/hi1a+Dn/eWU0vXboQctWLWnSpAmlpaV06LARsz+Zs9L1M2d+QsdOGy0/79ChHZ/MSq7r2DEnvWM7Pqngfmu45n8yj/mz5vHu+HcAGPPkqxyYZ+Ce/8k81m//bUt6/fZtWDB7Pp/PX0SLlmtT0qSEstIy1m/fhs9mr7gNoi3XALpA8lXwFreks9KXR0q6fsWj0OUVmzZt1qNlq3UBWHPNNdit7868PfVd/vPyaA4csDcAhx15ME89+eJK9z795IsMOGR/mjdvRpeNO9Ft0668Nu4NXn9tIt027UqXjTvRrFkzBhyyP09XcL81XAvnLmD+rE/ZaJMOAGy1yzZ8/M5H1dyVeG/CNDbq1p62nTekSbOm9D6wD6899z8A3np1EjvutxMAfQ7ttzzdKlDAzYKzlsXYoLfSn2MzyLvotdtoQ2645XJKSppQUiIeffhpnnvmJd6eOo2hd1zL784/jYlvvMXf734AgL333Z0e223NFZdez9Qp03jskad4ZcyTlC4r5ZzfXkxZ+rXt3DMuZvhDt1HSpAn33fsgU6dMq8+PabVw94W3Mfi602jarClzP5zN0DNuoNfeP+SYi37Buuu35Ld/O4/pb77Plcf8kdYbrscv/vwrrjr2EspKy7j7gts48+4LKGlSwsjhLywP+vdfdg9Dbjidn5xxJNMnv8+IYc/X86dswIqoxa2G2ue1OnSVWM3t03qr+q6CNUD3TH+ooq7+Gll8weF5x5y1L76/zuXVRRajSh6nih2KI+KgQpdpZlZnhd267A6SQRpzImLrFd47A7gSaBsRnyp5inwdsB+wBDg2Il6rKv8sRpVcBVwNvA98CdyaHl8AkzIoz8ys7go7jvtOYJ8VEyV1Bn4MfJiTvC/JBsHdgUHAzdVlnsWokhFpBf8YET/KeetxSSMLXZ6ZWSEUcphfRIyU1LWCt64FzgIezUnrD9ydbhw8SlJrSe0jYlZl+Wc5c7KtpOUDlNPp7m0zLM/MrPZq0OLOnXOSHoOqy17SQcDHETFhhbc6ArlDiGakaZXKcsWZ3wAvSXovPe8KnJRheWZmtVeDUSW5c07yIakFcB6wV0VvV1REVfllFrgj4mlJ3YEt0qQpEVH9/F0zs/qQ7ZT3TYFuwIR0Rmsn4DVJO5K0sDvnXNsJqHKBmixXB2wBnAn8Ov1q0EXSAVmVZ2ZWF1EWeR81zjtiYkRsGBFdI6IrSbDuGRGfAI8BxyjRG1hYVf82ZNvH/TfgG2Cn9HwG8KcMyzMzq70CjiqRdB/wKrC5pBmSTqji8ieB94BpJCPwflVd/ln2cW8aEYdJOgIgIr5URavemJk1BIUdVXJENe93zXkdwJCa5J9l4P5G0lqkneySNgXcx21mDVMRTXnPMnBfCDwNdJb0d2AX4NgMyzMzq73VPXCnXSJTgEOA3iTDXU6NiE+zKM/MrK6itP5X/ctXJoE7IkLSIxHRC3giizLMzAqqiFrcWY4qGSVphwzzNzMrmCyHAxZaln3c/YCTJE0HFpN0l0REeDNEM2t4GkBAzleWgXvfDPM2Myus4unizjRwtwcmR8TnAJLWBbYEpmdYpplZrcSy4oncWfZx30yyBne5xeSxzqyZWb0oq8FRz7JscSty9kWLiDJJWZZnZlZrDeGhY76ybHG/J+kUSc3S41SS+fhmZg1PEbW4swzcvwR2Bj4mWWDqhyTb8piZNTgeDghExBzg8KzyNzMrqAbQks6X+5zNzIBYVt81yJ8Dt5kZEG5xm5kVmdU5cEs6var3I+KaQpdpZlZXxdTirnRUiaSWVR1V5LluemwPDCbZZr4jySiTLQtZeTOzQomy/I/qSLpD0hxJk3LSrpQ0RdIbkh6W1DrnvXMlTZM0VdLe1eVfVYt7MsnuNbnbjZWfB9Clopsi4qK0Is+SbIZZPuX9D8AD1VXIzKw+RGlBd1a8E7gBuDsn7Tng3IhYJukK4FzgbElbkozA2wroADwvabOIqHTb+UoDd0R0ruy9PHUh2Sy43DdA1zrmaWaWiUJ2lUTESEldV0h7Nud0FPCT9HV/4P6I+Bp4X9I0YEeSzYYrlFcft6TDgU0i4lJJnYB2ETGumtvuAcZIepikhX4w3/3Xx8yswYiy/Fvckgbx3QmFQyNiaA2KOx4Ylr7uSBLIy81I0ypVbeCWdAPQDPgRcCmwBLgFqHKThIi4RNLTQJ806biIeL268szM6kNNWtxpkK5JoF5O0nnAMuDv5UkVFVFVHvm0uHeOiJ6SXgeIiPmSmudTwYgYJ+kjYM20wl0i4sN87jUzW5UiCtrHXSFJA4EDgD1yFuGbAeR2TXcCZlaVTz5rlSyVVEL6L4CkNuQx4lHSQZLeAd4HRqQ/n8qjPDOzVa6Qo0oqImkf4GzgoIhYkvPWY8DhktaQ1A3oDoypKq98AveNwINAW0kXAa8AV+Rx3x9Jdnh/OyK6AXsC/8njPjOzVa6sVHkf1ZF0H8nDxc0lzZB0Askok3WB5ySNl3QLQERMBoYDbwJPA0OqGlECeXSVRMTdksaRBF6An0bEpKruSS2NiHmSSiSVRMS/0yEwZmYNTk0eTlabV8QRFSTfXsX1lwCX5Jt/vjMnmwBLSbpL8l0KdoGkdYCRwN8lzSHpkDcza3AKGbizVm0QTp+A3kcyMLwT8A9J5+aRd3+SESi/IWn+vwscWPuqmpllJyL/o77l0+I+GuhV3pku6RJgHHBZVTdFxOL0ZZmkJ4B5uVuZmZk1JI2qxU2yK3tugG9KFVuQSeot6SVJD0naLp2rPwmYnT5VNTNrcCKU91HfKm1xS7qWpE97CTBZ0jPp+V4kI0sqcwPwO6AV8CKwb0SMkrQFSZfL0wWqu5lZwZQWdq2STFXVVVI+cmQy8ERO+qgKrv1OnuVz8iVdHBGjACJiilQ8fzBmtnppCC3pfFW1yFSlQ1eqkTs8/csVs61lnmZmmSqmPu581irZlGR84ZakU9cBImKzSm7ZVtIikvn3a6WvSc/XrOQeM7N6VUxDJ/IZVXIn8CfgKmBf4DiqmPIeEU0KUjMzs1WomFrc+YwqaRERzwBExLsRcT7QL9tqmZmtWqVlJXkf9S2fFvfXSp4qvivpl8DHwIbZVsvMbNVqbF0lvwHWAU4h6etuRbIIuJlZo1HWGEaVlIuI0enLz4GfZ1sdM7P60SiGA+ZsOVahiDgkkxqZmdWDxtJVcsMqq0UFPvvyi/os3hqoO969qr6rYI1Uo+gqiYgXVmVFzMzqU0MYLZKvfNfjNjNr1IqopyTvTRHMzBq1slDeR3Uk3SFpTro6anna+pKek/RO+nO9NF2Srpc0TdIbknpWl3/egVvSGvlea2ZWbAq8rOudwIrLWJ8DvBAR3YEX0nNIZqR3T49BwM3VZZ7PDjg7SpoIvJOebyvpr/nU3MysWJTV4KhORIwE5q+Q3B+4K319FzAgJ/3uSIwCWktqX1X++bS4rwcOAOalFZqAp7ybWSMTKO9D0iBJY3OOQXkU0S4iZgGkP8tnoHcEPsq5bkaaVql8Hk6WRMT0FdbSrnLreDOzYrOsBsMBI2IoMLRARVdUcJXPSvNpcX8kaUcgJDWRdBrwdm1qZ2bWUNWkxV1Ls8u7QNKfc9L0GUDnnOs6ATOryiifwD0YOB3oAswGeqdpZmaNRiH7uCvxGDAwfT0QeDQn/Zh0dElvYGF5l0pl8lmrZA5weO3rambW8NWhJb0SSfcBfYENJM0ALgQuB4ZLOgH4EPhpevmTwH7ANJI9fo+rLv98dsC5lQr6WyIin854M7OiUIeW9Eoi4ohK3tqjgmsDGFKT/PN5OPl8zus1gYP57hNQM7OiV1rAFnfW8ukqGZZ7Luke4LnMamRmVg+KaOeyWq1V0g3YuNAVMTOrT2WNqcUt6TO+7eMuIZkNdE7ld5iZFZ9iWmSqysCd7jW5Lck+kwBlaUe6mVmjUsiHk1mrchx3GqQfjojS9HDQNrNGqUzK+6hv+UzAGZPPMoNmZsWstAZHfatqz8mmEbEM6AOcKOldYDHJvPqICAdzM2s0GsuokjFAT75detDMrNFqLKNKBBAR766iupiZ1ZtieoBXVeBuK+n0yt6MiGsyqI+ZWb1oLF0lTYB1qHitWDOzRqWYhgNWFbhnRcTFq6wmZmb1qLSImqjV9nGbma0OGkuLe6XlB83MGqtGEbgjYsUdis3MGq0abDlZ72qzOqCZWaNTTC3ufKa8m5k1eoWc8i7pN5ImS5ok6T5Ja0rqJmm0pHckDZPUvLZ1deA2MyMZx53vURVJHYFTgO0jYmuSodWHA1cA10ZEd+Az4ITa1tWB28yMgu/y3hRYS1JToAUwC9gd+Gf6/l3UYTkRB24zM2oWuCUNkjQ251i+eXpEfAxcRbKT+yxgITAOWJAu3AcwA+hY27r64aSZGTVbqyQihgJDK3pP0npAf5JtHhcADwD71rHI73DgNjOjoGuV7Am8HxFzASQ9BOwMtM5ZLrsTMLO2BbirxMyMgo4q+RDoLalFuv3jHsCbwL+Bn6TXDAQerW1dHbjNzIAyIu+jKhExmuQh5GvARJI4OxQ4Gzhd0jSgDXB7bevqrhIzMwo7ASciLgQuXCH5PWDHQuTvwG1mRuPZSMHMbLVRTFPeHbjNzIBlKp42twO3mRnuKjEzKzruKjEzKzLVDfNrSBy4zcxwV4mZWdFxV4mZWZEpLaI2twO3mRlucZuZFZ1wi9vMrLgUU4vbqwOuYp06deD5Zx9g4hsvMWH8i5z862TbuUMPPYAJ41/km68+olfPbSq9f++9+jJ50kimvPkKZ505ZHl6166d+e8rj/PW5Ff4x99vplmzZpl/Fqub8y+9hh/tfzgDjv7l8rQpb7/LkSeexqEDh/Cz409h4ptTAfj8i8UMOetCDhn4K/ofdRIPP/FshXlOnvIOB/98MPv+7HguvfZmIpJW5MJFn/OLU3/HfoedwC9O/R0LF32e/QcsMoVaHXBVcOBexZYtW8aZZ13ED7bpyy59DmTw4GP5/ve7M3nyFH76sxN5+eVRld5bUlLC9dddwgEHHs0Ptu3HYYcN4Pvf7w7AZZeex1+uv5Xvb9WHzz5byPHHHbGqPpLV0oD9fswt1/zpO2lX33Q7g48/igfvupFf/+Jorr4pWfnzvgcfZ9OuXXjorpv42w1XcOVfb2Xp0qUr5fnHq27gwrNP4clht/PhjJm8MmosALfdM5ze2/fgyWG303v7Htx+7/DsP2CRiRoc9S2zwC1pl3zSVjeffDKH18dPAuCLLxYzZco7dOywEVOmTOPtt9+t8t4dd9iOd9/9gPff/5ClS5cyfPijHHTg3gD067sLDz74BAD33PMA/Q/aO9sPYnW2fY8f0Krlut9Jk8QXi5cA8MXiJWy4QZvl6YuXfElEsOTLr2jVcl2aNGnynXvnfjqfxYuX0GPr7yOJg/bZgxdffhWAf7/8Kv333ROA/vvuyYsjX8364xWdZUTeR33Lso/7r0DPPNJWWxtv3Ike227N6DGv53V9h44b8dGMb3c7mvHxLHbcYTvatFmPBQsWUlpaujy9Q8eNMqmzZevsU0/ipNPP56obbyPKgnv/72oAjjz0QH599kX0638Ui5d8yVUXn0tJyXfbXbPnfkq7DTdYft6u7QbMnjsPgHmfLaDtBusD0HaD9Zm/YOEq+kTFY7V+OClpJ5L91dpKOj3nrZZAk4rvWn7vIGAQgJq0oqRk7UJXr8FYe+0WDB92K6efcSGff/5FXvckuyB9V0RUmm7FZ9jDT3D2yYP4cb8+PP3CSC647C/cdt1l/GfMOLbovgl3/PVyPvp4Fiee9jt6bbsV66z97f8jFQWeCn41rBKr+8PJ5sA6JP8orJtzLOLb/dYqFBFDI2L7iNi+MQftpk2b8sCwW7nvvod55JGn8r7v4xmz6Nypw/LzTh3bM2vWbD79dD6tW7da/tW5U8f2zJo5u+D1tuw99tTz7Nk36VHce/ddlz+cfPiJ59hzt12QRJdOHejYfiPenz7jO/du1LYts+d8uvx89txPl3e1tFmvNXM/nQ8kXSrrt261Kj5OUYka/FcdSa0l/VPSFElvSdpJ0vqSnpP0TvpzvdrWteCBOyJGRMRFQO+IuCjnuCYi3il0ecXo1qFX89aUafzluqE1uu9/Y8fzve91o2vXzjRr1oyf/aw/j/8rGV3w0oj/cuih+wPw85//lMcer3jUgTVsbTdow/9enwjA6HHj2bhzRwDat2vLqHHjAfh0/md88OEMOnXYaIV716dFi7WYMOktIoLHnn6Bfn16A9C3T28efep5AB596nn67brTqvpIRaOsBkcergOejogtgG2Bt4BzgBciojvwQnpeKyr0V2pJf4mI0yQ9TgUPYCPioHzyadq8Y6P8rr/Lzjsw4qVHeGPim5SVJR/x97+/nOZrNOe6a/9E27brs2DBIiZMmMx+BxxF+/btGHrLlRzY/xgA9t1nd66++iKalJRw513DuOzy6wHo1q0L/7j3JtZbrzXjJ0zmmIEn880339Tb58zKlzNfru8qFMyZF17O/15/gwULFtFm/db86oSf061LRy6/7v9YVlrKGs2bc/5vh7DVFt2ZM3ce511yNZ/O+4yI4ISf/4wD994dgEMHDuHBu24EYNJbb3P+Jdfw1ddfs2vvHfjd6YORxIKFi/jt7y9l1uy5tG/Xlmv+dN5KD0aLWbMNNqlzp9DRGx+Sd8y5d/pDlZYnqSUwAdgkcgKspKlA34iYJak98FJEbF6bumYRuHtFxDhJu1X0fkSMyCefxhq4rW4aU+C2wilE4D5y44Pzjjn3ffjISaTP41JDI2IogKQeJLu6v0nS2h4HnAp8HBGty2+Q9FlE1Kq7pOAPJyNiXPozrwBtZtYQ1GRUSRqkK+vrbEoyeu7kiBgt6Trq0C1SWQEFJWkiVYxRj4jKpwWamdWTAo4qmQHMiIjR6fk/SQL3bEntc7pK5tS2gCzGcR+Q/iyfj31P+vMoYEkG5ZmZ1VmhprJHxCeSPpK0eURMBfYg6TZ5ExgIXJ7+fLS2ZWTRVTIdklmSEZE7U/IcSf8BLi50mWZmdVXgCTgnA3+X1Bx4DziOZBTfcEknAB8CP61t5lnOnFxbUp+IeAVA0s5A4x2cbWZFrbSAAzUiYjywfQVv7VGI/LMM3CcAd0hqRdLnvRA4PsPyzMxqrSGs+pevzAJ3Orpk23RMoyLCiyOYWYO1uk95B0BSO0m3A8MiYqGkLdO+HTOzBqeQU96zluV63HcCzwDli2u8DZyWYXlmZrXmjRQSG0TEcNJvIBGxDCjNsDwzs1qLiLyP+pblw8nFktqQTsaR1JvkAaWZWYNT2gBa0vnKMnCfDjwGbJqO325LNcu6mpnVl4bQBZKvTAK3pBJgTWA3YHNAwNSIWHmTPDOzBqAhdIHkK5PAHRFlkq6OiJ2AyVmUYWZWSMXU4s7y4eSzkg5VRftqmZk1MMU0HDDrPu61gWWSviLpLomIaJlhmWZmtVLIKe9Zy3LmZOPZXsPMGr3VuqtE0t6SVho9IulIST8udHlmZoWwuk/AuQioaPebF/GSrmbWQK3uE3BaRMTcFRPTxcW9rKuZNUgNoSWdrywC95qSmqZT3JeT1AxYK4PyzMzqrCGMFslXFl0lDwG35rau09e3pO+ZmTU4pVGW91Hfsgjc5wOzgemSxkkaB3wAzE3fMzNrcArdxy2piaTXJf0rPe8mabSkdyQNS7c1q5WCB+6IWBYR5wCdgWPTo0tEnOMp72bWUGUwquRU4K2c8yuAayOiO/AZyS5htZLZzMmI+DIiJqbHl1mVY2ZWCIWcOSmpE7A/cFt6LmB34J/pJXcBA2pb1yynvJuZFY2yiLwPSYMkjc05Bq2Q3V+As/h2R7Q2wIKcQRszgI61rWuWU97NzIpGTUaVRMRQYGhF70k6AJgTEeMk9S1PrrDIWip44JbUs6r3I+K1QpdpZlZXBRwtsgtwkKT9SJa3bknSAm+dM1S6EzCztgVk0eK+Ov25JrA9MIHkX5ttgNFAnwzKNDOrk7ICzYiMiHOBcwHSFvcZEXGUpAdINpO5HxgIPFrbMrIYVdIvIvoB04GeEbF9RPQCtgOmFbo8M7NCWAXLup4NnC5pGkmf9+21zSjLPu4tImJi+UlETJLUI8PyzMxqrVAt7lwR8RLwUvr6PWDHQuSbZeB+S9JtwL0knfBH890xjWZmDUYxTXnPMnAfBwwmGYQOMBK4OcPyzMxqrTRK67sKectyI4WvJN0IPE/S4vZmwWbWYDWE5VrzlVngTp+m3kWyTomAzpIGRsTIrMo0M6ut1X1Z13JXA3tFxFQASZsB9wG9MizTzKxW3OJONCsP2gAR8Xa6JreZWYOTxaiSrGQZuMdKuh24Jz0/ChiXYXlmZrXmUSWJwcAQ4BSSPu6RwE0Zlgeyvr8AAAiISURBVGdmVmsNYYOEfGU5quRrSTcAz+FRJWbWwLmPG48qMbPi4j7uhEeVmFnRcIs74VElZlY0PI474VElZlY03OJOeFSJmRUNjyohGVUCXJMeZmYNWjE9nCz4RgqS+ksaknM+WtJ76fHTQpdnZlYIEZH3Ud+y2OX9LOCxnPM1gB2AvsAvMyjPzKzOVsEOOAWTRVdJ84j4KOf8lYiYB8yTtHYG5ZmZ1VlDaEnnK4vAvV7uSUT8Oue0bQblmZnVWTH1cWcRuEdLOjEibs1NlHQSMCbfTJZ987EKXrMiJWlQRAyt73pYw+Lfi8IqppijQn89kLQh8AjwNfBamtyLpK97QETMLmiBqwFJYyNi+/quhzUs/r1YfRW8xR0Rc4CdJe0ObJUmPxERLxa6LDOz1VGW47hfBByszcwKLIvhgFZ47se0ivj3YjVV8D5uMzPLllvcZmZFxoHbzKzIOHDXkaRSSeMlTZD0mqSda3j/HySdkVX90jL6SvpX+vogSeekrwdI2jLLslcXktpJ+ke6Js84Sa9KOrhAeb8kaaVhf5KaSbpc0juSJkkaI2nfOpTTVdKk9PX2kq5PX/et6e+1ZSvLZV1XF19GRA8ASXsDlwG7ZV2oJJE8o6jRWpQR8RjfriUzAPgX8GaBq7daSf8uHgHuiogj07SNgYMyLvqPQHtg63SP13ZU8LsnqUlElNYk44gYC4xNT/sCXwD/rVt1rVDc4i6slsBnAJLWkfRC2gqfKKl/+UWSzpM0VdLzwOYVZZS24B5OW/ITJO2ctojeknQTyeSmzpL2Slt3r0l6QNI66f37SJoi6RXgkJx8j5V0Q9qCOgi4Mv3GsGlmfyqN3+7ANxFxS3lCREyPiL8CSFpT0t/S34PXJfWrJn0tSfdLekPSMGCtFQuU1AI4ETg5XUKZiJgdEcPT97+QdLGk0cBOknpJGpF+G3hGUvv0ul7p79erJOvnl+ffV9K/JHUlWRzuN+nvya6F/+OzmnKLu+7WkjQeWJOk9bN7mv4VcHBELJK0ATBK0mNAT+BwYDuSP//XqHhnoOuBERFxsKQmwDok68BsDhwXEb9K8z0f2DMiFks6Gzhd0p+BW9O6TAOGrZh5RPw3rc+/IuKfhfmjWG1txbezhCsyBCAifiBpC+BZJXuwVpY+GFgSEdtI2qaSvL8HfBgRiyopc21gUkRcoGTLwBFA/4iYK+kw4BLgeOBvJMF/hKQrV8wkIj6QdAvwRURcVe2fhK0SDtx1l9tVshNwt6StSXb9uVTSj4AyoCPQDtgVeDgilqT3PFZxtuwOHAOQfs1dKGk9YHpEjEqv6Q1sCfwn+bZOc+BVYAvg/Yh4Jy3jXmBQQT+1VUrSjUAfklb4DunrvwJExBRJ04HNqkj/Eck/3ETEG5LeqEU1SoEH09ebA1sDz6W/J02AWZJaAa0jYkR63T1ArfvIbdVx4C6giHg1bQW3BfZLf/aKiKWSPiBplQN1WtB3cc5rAc9FxBG5F0jqUccyrGYmA4eWn0TEkPT3oLyPuLLFi6pa1Ki6v79pQBdJ60bE5xW8/1VOv7aAyRGx03cKl1rnUY41QO7jLqD0624TYB7QCpiTBu1+wMbpZSOBg9N+zHWBAyvJ7gWSr8xIaiKpZQXXjAJ2kfS99LoW6VftKUC3nH7rIyq4F+BzYN0afUiryIvAmpIG56S1yHk9kmSzbNK/ny7A1DzTtwa2WbHA9Bvb7cD1kpqn17aXdHQF9ZsKtE2/EZaPRtkqIhaQfJPrk153VCWfz78nDYwDd92tlT60GU/Slzwwben8Hdhe0liS/yGmAETEa+l140m+yr5cSb6nAv0kTSTpA99qxQsiYi5wLHBf+nV6FLBFRHxF0jXyRPpwcnolZdwPnJk+GPPDyVqKZPrxAGA3Se9LGgPcBZydXnIT0CT9uxwGHJs+UKws/WZgnfTv9CwqXw75fGAu8KaSYXyPpOcr1u8b4CfAFZImkPzulQ/vOw64MX04+WUl5TxO0tjww8kGwlPezcyKjFvcZmZFxoHbzKzIOHCbmRUZB24zsyLjwG1mVmQcuK1K+nb1w0npWigtqr+r0rwqXKWwkmtbS/pVLcqocLXFytJXuOZOST+pQVnLV9MzW5UcuK06X0ZEj4jYGviGZMGh5ZSo8e9RRDwWEZdXcUlroMaB22x14MBtNfEy8D0VcJXC9PVKKyEClwObpq39K9PrzpT0PyWr5l2Uk1e1qy3mknRims8ESQ+u8C1iT0kvS3pb0gHp9U0kXZlT9kl1/YM0qwsHbsuLpKYkCxBNTJM2B+6OiO1I1k8pX6WwJ8kaHadLWpNklcIDSRbX2qiS7MtXQtyWZPXEycA5wLtpa/9MSXsB3YEdgR5AL0k/ktSLb1dbPATYIY+P81BE7JCW9xZwQs57XUnWtN4fuCX9DCcAC9MFo3YATpTULY9yzDLhRaasOuXL1kLS4r4d6EBhVymsbCXEXHulx+vp+TokgXxd8lttMdfWkv5E0h2zDvBMznvD080p3pH0XvoZ9gK2yen/bpWW/XYeZZkVnAO3VWf5srXl0uC8qlcpFHBZRPzfCmWcVosy7gQGRMQESceS7PBSbsW8Ii375IjIDfAo2WTAbJVzV4kVQl1XKaxoJcQVV6R7Bjg+p++8o6QNyX+1xVzrkqxH3YyVV8T7qaSStM6bkKys9wwwOL0eSZtJWjuPcswy4Ra31Vm6q8qxJKsUrpEmnx8Rb0sqX6XwU+AVkgX9V3QqMFTSCSQbAAxO1zb/Tzrc7qm0n/v7wKtpi/8L4OiIeE3J9l7jSVZBrGy1xVy/B0an10/ku/9ATCXZLaYd8MuI+ErSbSR9368pKXwuyWqAZvXCqwOamRUZd5WYmRUZB24zsyLjwG1mVmQcuM3MiowDt5lZkXHgNjMrMg7cZmZF5v8B0lQcULnwdzUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_cm( y_pred_df.actual,\n",
    "y_pred_df.predicted )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Measuring Accuracies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.76      0.90      0.82       209\n",
      "           1       0.59      0.33      0.42        91\n",
      "\n",
      "    accuracy                           0.73       300\n",
      "   macro avg       0.67      0.61      0.62       300\n",
      "weighted avg       0.70      0.73      0.70       300\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print( metrics.classification_report( y_pred_df.actual,y_pred_df.predicted ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeAAAAF0CAYAAAD7I+rAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAcTElEQVR4nO3de5iWdb3v8fdXDiIeUhFNIx1amYoII4wkZWBq2q5U2laeVqF5iZZby1YHtDI7XVnbva3MrdFJMk1dlmGWq0xBwMgENUTxtAyJZOFA4iGlwL77j7mZRTI4zzDPzA8e3q/rmuu5n/v53ff9nd/FNR/uw/P7RWYiSZJ611alC5AkaUtkAEuSVIABLElSAQawJEkFGMCSJBVgAEuSVEDf3jzYLrvskk1NTb15SEmSipk3b97yzBzc0We9GsBNTU3MnTu3Nw8pSVIxEfHEhj7zErQkSQUYwJIkFWAAS5JUQK/eA5Yk9azVq1ezZMkSVq1aVbqULcqAAQMYMmQI/fr1q3kbA1iSGsiSJUvYfvvtaWpqIiJKl7NFyExWrFjBkiVLGDp0aM3beQlakhrIqlWrGDRokOHbiyKCQYMGdfmqgwEsSQ3G8O19G9PnBrAkqa769OlDc3MzI0eOZNSoUfz2t7/t0vYXXnghF198cYef/fCHP2T48OHsv//+DBs2bIPtatXU1MTy5csBeNOb3gTAokWLuOaaa7q131p4D1iSGtiUKfXd36RJnbfZZpttuO+++wD41a9+xXnnnccdd9zR7WPfcsstfP3rX+fXv/41e+yxB6tWreKqq65ar92aNWvo27fr8bb2PwprA/ikk07qds2vxDNgSVKPefbZZ9lpp50AeP755zn88MMZNWoUBxxwANOmTWtv9+Uvf5l99tmHI444gocffrjDfX3lK1/h4osvZo899gDanjw+/fTTATj00EM5//zzGT9+PN/4xjdobW3luOOO46CDDuKggw7izjvvBGDFihUceeSRHHjggZxxxhlkZvv+t9tuOwAmT57MrFmzaG5u5pJLLql/p1Q8A5Yk1dWLL75Ic3Mzq1atYunSpdx+++1AW2DeeOON7LDDDixfvpyDDz6YY445hnvuuYdrr72We++9lzVr1jBq1ChGjx693n4XLFjQ4fq1Vq5c2X6mfdJJJ3HuuedyyCGHsHjxYo466igWLlzI5z//eQ455BAuuOACfvGLXzClg0sEF110ERdffDE333xznXqkYwawJKmu1r0EPWfOHD7wgQ+wYMECMpPzzz+fmTNnstVWW/HnP/+ZZcuWMWvWLN797nczcOBAAI455piNOu7xxx/fvvyb3/yGBx98sP39s88+y3PPPcfMmTP56U9/CsA73/nO9rPzEgxgSVKPGTt2LMuXL6e1tZVf/vKXtLa2Mm/ePPr160dTU1P7V3dqeYp4//33Z968eRx22GEdfr7tttu2L//jH/9gzpw5bLPNNuu121SeEt+sA3jKvDo/XVAHk0bX8ISCJG0hHnroIV566SUGDRrEM888w6677kq/fv2YPn06TzzRNlHQuHHjOOWUU5g8eTJr1qzh5z//OWecccZ6+zrvvPP45Cc/yc0338yrX/1q/va3v/Htb3+bc845Z722Rx55JN/61rf4xCc+AcB9991Hc3Mz48aN4+qrr+Yzn/kMt9xyC08//fR6226//fY899xzde6J9W3WASxJ2vSsvQcMbaNETZ06lT59+nDyySdz9NFH09LSQnNzM/vuuy8Ao0aN4vjjj6e5uZm99tqLt7zlLR3u9x3veAfLli3jiCOOIDOJCD74wQ922Pab3/wmZ511FiNGjGDNmjWMGzeOK664gs997nOceOKJjBo1ivHjx7Pnnnuut+2IESPo27cvI0eO5JRTTuHcc8+tU8/8s1j3CbCe1tLSkvWcD9gzYEn6ZwsXLmS//fYrXcYWqaO+j4h5mdnSUXu/hiRJUgEGsCRJBRjAkiQVYABLklSAASxJUgEGsCRJBRjAkqS6WrZsGSeddBKve93rGD16NGPHjuXGG2+sy74PPfRQOvo66+rVq5k8eTJ77703w4cPZ8yYMdxyyy0bfZxFixYxfPhwAObOnds+2MeMGTO6PL3ihjgQhyQ1sHqPl9DZWAeZyYQJE5g4cWL7nLpPPPEEN910U13reLnPfvazLF26lAULFrD11luzbNmyDqdAfOmll+jTp0+X9t3S0kJLS9tXeWfMmMF2223XPndwd3gGLEmqm9tvv53+/ftz5plntq/ba6+9OPvsswFYtWoVp556KgcccAAHHngg06dPf8X1L774IieccAIjRozg+OOP58UXX1zvmC+88ALf+c53uPTSS9l6660B2G233Xjf+94HtE0zeMEFF/DGN76ROXPmMG/ePMaPH8/o0aM56qijWLp0KQDz5s1j5MiRjB07lssuu6x9/zNmzOBd73oXixYt4oorruCSSy6hubmZWbNmdauvPAOWJNXNAw88wKhRozb4+dpgu//++3nooYc48sgjeeSRRza4/vLLL2fgwIHMnz+f+fPnd7jvxx57jD333JMddtihw2P+9a9/Zfjw4XzhC19g9erVjB8/nmnTpjF48GCuu+46Pv3pT/P973+fU089lUsvvZTx48e3jyG9rqamJs4880y22247Pv7xj29M9/wTA1iS1GPOOussZs+eTf/+/bn77ruZPXt2+9nwvvvuy1577cUjjzyywfUzZ85sv/86YsQIRowY0eUa+vTpw3HHHQfAww8/zIIFC3jb294GtF2S3n333XnmmWdYuXIl48ePB+D9739/t+4h18IAliTVzf77789PfvKT9veXXXYZy5cvb7+HuqH5B15pXoLOpg98/etfz+LFi3nuuefYfvvt1/t8wIAB7fd9M5P999+fOXPm/FOblStX9vo0hd4DliTVzWGHHcaqVau4/PLL29e98MIL7ctrpwMEeOSRR1i8eDH77LNPTesXLFjA/Pnz1zvmwIEDOe200zjnnHP4+9//DsDSpUv50Y9+tF7bffbZh9bW1vYAXr16NQ888AA77rgjr3rVq5g9ezZA+zFfrp5TFdYUwBGxY0TcEBEPRcTCiBgbETtHxK0R8Wj1ulNdKpIkbbYigp/97GfccccdDB06lDFjxjBx4kS++tWvAvDhD3+Yl156iQMOOIDjjz+eK6+8kq233nqD6z/0oQ/x/PPPM2LECL72ta8xZsyYDo/7pS99icGDBzNs2DCGDx/OhAkTGDx48Hrt+vfvzw033MCnPvUpRo4cSXNzc/vXin7wgx9w1llnMXbsWLbZZpsOj3P00Udz44031uUhrJqmI4yIqcCszPxuRPQHBgLnA3/JzIsiYjKwU2Z+6pX243SEktSznI6wnLpPRxgROwDjgO8BZObfM3MlcCwwtWo2FZjQjbolSdqi1HIJ+nVAK/CDiLg3Ir4bEdsCu2XmUoDqddcerFOSpIZSSwD3BUYBl2fmgcBfgcm1HiAiJkXE3IiY29raupFlSpLUWGoJ4CXAksy8q3p/A22BvCwidgeoXp/qaOPMnJKZLZnZ0tENcUlSfdXybI/qa2P6vNMAzsz/Av4UEftUqw4HHgRuAiZW6yYC07p8dElSXQ0YMIAVK1YYwr0oM1mxYgUDBgzo0na1DsRxNnB19QT048CptIX39RFxGrAYeG+XjixJqrshQ4awZMkSvOXXuwYMGMCQIUO6tE1NAZyZ9wEdPUZ9eJeOJknqUf369WPo0KGly1ANHAlLkqQCDGBJkgowgCVJKsAAliSpAANYkqQCDGBJkgowgCVJKsAAliSpAANYkqQCDGBJkgowgCVJKsAAliSpAANYkqQCDGBJkgowgCVJKsAAliSpAANYkqQCDGBJkgowgCVJKsAAliSpAANYkqQCDGBJkgowgCVJKsAAliSpAANYkqQCDGBJkgowgCVJKsAAliSpAANYkqQCDGBJkgowgCVJKsAAliSpAANYkqQCDGBJkgowgCVJKsAAliSpAANYkqQCDGBJkgowgCVJKsAAliSpgL61NIqIRcBzwEvAmsxsiYidgeuAJmAR8L7MfLpnypQkqbF05Qz4rZnZnJkt1fvJwG2ZuTdwW/VekiTVoDuXoI8FplbLU4EJ3S9HkqQtQ60BnMCvI2JeREyq1u2WmUsBqtddO9owIiZFxNyImNva2tr9iiVJagA13QMG3pyZT0bErsCtEfFQrQfIzCnAFICWlpbciBolSWo4NZ0BZ+aT1etTwI3AGGBZROwOUL0+1VNFSpLUaDoN4IjYNiK2X7sMHAksAG4CJlbNJgLTeqpISZIaTS2XoHcDboyIte2vycz/iIi7gesj4jRgMfDenitTkqTG0mkAZ+bjwMgO1q8ADu+JojZnU+ZNKV1ChyaNntR5I0lSr3EkLEmSCjCAJUkqwACWJKkAA1iSpAIMYEmSCjCAJUkqwACWJKkAA1iSpAIMYEmSCjCAJUkqwACWJKkAA1iSpAIMYEmSCjCAJUkqwACWJKkAA1iSpAIMYEmSCjCAJUkqwACWJKkAA1iSpAIMYEmSCjCAJUkqwACWJKkAA1iSpAIMYEmSCjCAJUkqwACWJKkAA1iSpAIMYEmSCjCAJUkqwACWJKkAA1iSpAIMYEmSCjCAJUkqwACWJKkAA1iSpAIMYEmSCjCAJUkqwACWJKmAmgM4IvpExL0RcXP1fmhE3BURj0bEdRHRv+fKlCSpsXTlDPgjwMJ13n8VuCQz9waeBk6rZ2GSJDWymgI4IoYA7wS+W70P4DDghqrJVGBCTxQoSVIjqvUM+OvAJ4F/VO8HASszc031fgnwmjrXJklSw+o0gCPiXcBTmTlv3dUdNM0NbD8pIuZGxNzW1taNLFOSpMZSyxnwm4FjImIRcC1tl56/DuwYEX2rNkOAJzvaODOnZGZLZrYMHjy4DiVLkrT56zSAM/O8zBySmU3ACcDtmXkyMB14T9VsIjCtx6qUJKnBdOd7wJ8CPhYRj9F2T/h79SlJkqTG17fzJv8tM2cAM6rlx4Ex9S9JkqTG50hYkiQVYABLklSAASxJUgEGsCRJBRjAkiQVYABLklSAASxJUgEGsCRJBRjAkiQVYABLklSAASxJUgEGsCRJBRjAkiQVYABLklSAASxJUgEGsCRJBRjAkiQVYABLklSAASxJUgEGsCRJBRjAkiQVYABLklSAASxJUgEGsCRJBRjAkiQVYABLklSAASxJUgEGsCRJBRjAkiQVYABLklSAASxJUgEGsCRJBRjAkiQVYABLklSAASxJUgEGsCRJBRjAkiQVYABLklSAASxJUgEGsCRJBXQawBExICJ+HxF/iIgHIuLz1fqhEXFXRDwaEddFRP+eL1eSpMZQyxnw34DDMnMk0Ay8PSIOBr4KXJKZewNPA6f1XJmSJDWWTgM42zxfve1X/SRwGHBDtX4qMKFHKpQkqQHVdA84IvpExH3AU8CtwH8CKzNzTdVkCfCaDWw7KSLmRsTc1tbWetQsSdJmr6YAzsyXMrMZGAKMAfbrqNkGtp2SmS2Z2TJ48OCNr1SSpAbSpaegM3MlMAM4GNgxIvpWHw0BnqxvaZIkNa5anoIeHBE7VsvbAEcAC4HpwHuqZhOBaT1VpCRJjaZv503YHZgaEX1oC+zrM/PmiHgQuDYivgTcC3yvB+uUJKmhdBrAmTkfOLCD9Y/Tdj9YkiR1kSNhSZJUgAEsSVIBBrAkSQUYwJIkFWAAS5JUgAEsSVIBBrAkSQUYwJIkFWAAS5JUgAEsSVIBBrAkSQUYwJIkFWAAS5JUgAEsSVIBBrAkSQUYwJIkFWAAS5JUgAEsSVIBBrAkSQUYwJIkFWAAS5JUgAEsSVIBBrAkSQX0LV2ApFc2ZUrpCnrOpEmlK5DK8QxYkqQCDGBJkgowgCVJKsAAliSpAANYkqQCDGBJkgowgCVJKsAAliSpAANYkqQCHAlLxUyZt+kN8TRptEMzSeodngFLklSAASxJUgEGsCRJBRjAkiQV4ENYW4hN8YEnSdqSdXoGHBGvjYjpEbEwIh6IiI9U63eOiFsj4tHqdaeeL1eSpMZQyyXoNcC/ZeZ+wMHAWRExDJgM3JaZewO3Ve8lSVINOg3gzFyamfdUy88BC4HXAMcCU6tmU4EJPVWkJEmNpksPYUVEE3AgcBewW2YuhbaQBnatd3GSJDWqmgM4IrYDfgJ8NDOf7cJ2kyJibkTMbW1t3ZgaJUlqODUFcET0oy18r87Mn1arl0XE7tXnuwNPdbRtZk7JzJbMbBk8eHA9apYkabNXy1PQAXwPWJiZ/3edj24CJlbLE4Fp9S9PkqTGVMv3gN8MvB+4PyLuq9adD1wEXB8RpwGLgff2TImSJDWeTgM4M2cDsYGPD69vOZIkbRkcCUtax6Y5YphTJEqNyLGgJUkqwACWJKkAA1iSpAIMYEmSCjCAJUkqwACWJKkAA1iSpAIMYEmSCnAgji3EzFmlK+g5495SugJJ6jrPgCVJKsAAliSpAANYkqQCDGBJkgowgCVJKsAAliSpAANYkqQCDGBJkgowgCVJKsCRsCQVM2VK6Qp6zqRJpSvQps4zYEmSCjCAJUkqwACWJKkAA1iSpAJ8CEubvUaearHNpvek0riBPmEkdZdnwJIkFWAAS5JUgAEsSVIBBrAkSQX4EJakLpv5wqb3YBj4cJg2L54BS5JUgAEsSVIBBrAkSQUYwJIkFeBDWOto/BGVJEmbCs+AJUkqwACWJKkAA1iSpAIMYEmSCjCAJUkqoNMAjojvR8RTEbFgnXU7R8StEfFo9bpTz5YpSVJjqeUM+Erg7S9bNxm4LTP3Bm6r3kuSpBp1GsCZORP4y8tWHwtMrZanAhPqXJckSQ1tY+8B75aZSwGq11031DAiJkXE3IiY29raupGHkySpsfT4Q1iZOSUzWzKzZfDgwT19OEmSNgsbG8DLImJ3gOr1qfqVJElS49vYAL4JmFgtTwSm1accSZK2DLV8DenHwBxgn4hYEhGnARcBb4uIR4G3Ve8lSVKNOp0NKTNP3MBHh9e5FkmSthiOhCVJUgEGsCRJBRjAkiQVYABLklSAASxJUgEGsCRJBRjAkiQVYABLklSAASxJUgEGsCRJBRjAkiQVYABLklSAASxJUgEGsCRJBRjAkiQVYABLklSAASxJUgEGsCRJBRjAkiQV0Ld0AZLUiKZMKV1Bz5k0qXQFjcEzYEmSCjCAJUkqwACWJKkA7wFLahgzX9j0bryOG9h4N0y9v10fngFLklSAASxJUgEGsCRJBRjAkiQV4ENYktSDfDBMG+IZsCRJBRjAkiQVYABLklSAASxJUgEGsCRJBRjAkiQVYABLklSAASxJUgEGsCRJBRjAkiQV0K0Ajoi3R8TDEfFYREyuV1GSJDW6jQ7giOgDXAb8D2AYcGJEDKtXYZIkNbLunAGPAR7LzMcz8+/AtcCx9SlLkqTG1p0Afg3wp3XeL6nWSZKkTnRnOsLoYF2u1yhiErB27qvnI+LhGva9C7C8G7Wp6+zz3mV/9z77vHI1Z/TGYTbL/j6j/l2z14Y+6E4ALwFeu877IcCTL2+UmVOALk2IGRFzM7OlG7Wpi+zz3mV/9z77vHfZ353rziXou4G9I2JoRPQHTgBuqk9ZkiQ1to0+A87MNRHxv4BfAX2A72fmA3WrTJKkBtadS9Bk5i+BX9aplnV16ZK16sI+7132d++zz3uX/d2JyFzvuSlJktTDHIpSkqQCigZwZ0NZRsTWEXFd9fldEdHU+1U2jhr6+2MR8WBEzI+I2yJig4/Pqza1DtcaEe+JiIwInxrthlr6OyLeV/07fyAiruntGhtNDX9X9oyI6RFxb/W35R0l6twkZWaRH9oe3PpP4HVAf+APwLCXtfkwcEW1fAJwXal6N/efGvv7rcDAavlD9nfP93nVbntgJvA7oKV03ZvrT43/xvcG7gV2qt7vWrruzfmnxj6fAnyoWh4GLCpd96byU/IMuJahLI8FplbLNwCHR0RHA4Coc532d2ZOz8wXqre/o+273dp4tQ7X+kXga8Cq3iyuAdXS36cDl2Xm0wCZ+VQv19hoaunzBHaoll9FB+NFbKlKBnAtQ1m2t8nMNcAzwKBeqa7xdHXo0NOAW3q0osbXaZ9HxIHAazPz5t4srEHV8m/8DcAbIuLOiPhdRLy916prTLX0+YXAv0bEEtq+NXN275S26evW15C6qZahLGsa7lI1qbkvI+JfgRZgfI9W1Phesc8jYivgEuCU3iqowdXyb7wvbZehD6XtCs+siBiemSt7uLZGVUufnwhcmZn/JyLGAldVff6Pni9v01byDLiWoSzb20REX9ouX/ylV6prPDUNHRoRRwCfBo7JzL/1Um2NqrM+3x4YDsyIiEXAwcBNPoi10Wr9mzItM1dn5h+Bh2kLZG2cWvr8NOB6gMycAwygbZzoLV7JAK5lKMubgInV8nuA27O6k68u67S/q8uh36YtfL031n2v2OeZ+Uxm7pKZTZnZRNt992Myc26Zcjd7tfxN+RltDxsSEbvQdkn68V6tsrHU0ueLgcMBImI/2gK4tVer3EQVC+Dqnu7aoSwXAtdn5gMR8YWIOKZq9j1gUEQ8BnwM2ODXOPTKauzv/w1sB/x7RNwXEY7t3Q019rnqpMb+/hWwIiIeBKYDn8jMFWUq3vzV2Of/BpweEX8Afgyc4olUG0fCkiSpAEfCkiSpAANYkqQCDGBJkgowgCVJKsAAliSpAANYkqQCDGBpExcRz1eve0TEDZ20/WhEDOzi/g+NiB4bizoimiJiQU/tX9pcGcBSARHRp6vbZOaTmfmeTpp9FOhSANfDxvw+0pbOAJbqrDrjeygiplYTkN8QEQMjYlFEXBARs4H3RsS/RMR/RMS8iJgVEftW2w+NiDkRcXdEfPFl+11QLfeJiIsj4v7qGGdHxDnAHsD0iJhetTuy2tc9EfHvEbFdtf7tVY2zgf/Zye9zYURcFRG3R8SjEXF6tf7QaqL1a4D7q3Ufi4gF1c9H19lN35f3R736W9pcGcBSz9gHmJKZI4BngQ9X61dl5iGZeS1tE5WfnZmjgY8D/69q8w3g8sw8CPivDex/EjAUOLA6xtWZ+U3aBsJ/a2a+tRrr+DPAEZk5CpgLfCwiBgDfAY4G3gK8uobfZwTwTmAscEFE7FGtHwN8OjOHRcRo4FTgjbRNLHF6Nb74K/WHtMUygKWe8afMvLNa/hFwSLV8HUB1JvomqnG3aZsEY/eqzZtpGzMX4KoN7P8I4IpqLF4ys6NZwg4GhgF3VseYCOwF7Av8MTMfrcbk/VENv8+0zHwxM5fTNobymGr976tZhah+xxsz86+Z+TzwU9oCHjbcH9IWq+R8wFIje/kg62vf/7V63QpYmZnNNW7/clFjm1sz88R/WhnRXMO2ndXz8t9n7fG6ur20xfIMWOoZe1aTj0PbhOSz1/0wM58F/hgR7wWINiOrj++kbVo3gJM3sP9fA2dW82QTETtX65+jbZ5haJve8M0R8fqqzcCIeAPwEDA0Iv5lnfo6c2xEDIiIQbRNZn93B21mAhOq42wLvBuYVX32iv0hbYkMYKlnLAQmRsR8YGfg8g7anAycVk3T9gBwbLX+I8BZEXE38KoN7P+7tM2zOr/a/qRq/RTgloiYnpmtwCnAj6s6fgfsm5mraLuH/IvqIawnavh9fg/8otrHFzPz5ZOuk5n3AFdWbe8CvpuZ93ahP6QtitMRSnUWEU3AzZk5vHApdRERFwLPZ+bFpWuRGolnwJIkFeAZsCQAIuJU2i5/r+vOzDyrRD1SozOAJUkqwEvQkiQVYABLklSAASxJUgEGsCRJBRjAkiQV8P8Bp7i58SCdZGAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure( figsize = (8,6) )\n",
    "sn.distplot( y_pred_df[y_pred_df.actual == 1][\"predicted_prob\"],\n",
    "kde=False, color = 'b',\n",
    "label = 'Bad Credit' )\n",
    "sn.distplot( y_pred_df[y_pred_df.actual == 0][\"predicted_prob\"],\n",
    "kde=False, color = 'g',\n",
    "label = 'Good Credit' )\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ROC & AUC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_roc( actual, probs ):\n",
    "    fpr, \\\n",
    "    tpr, \\\n",
    "    thresholds = metrics.roc_curve( actual,probs,drop_intermediate = False )\n",
    "    auc_score = metrics.roc_auc_score( actual, probs )\n",
    "    plt.figure(figsize=(8, 6))\n",
    "    plt.plot( fpr, tpr, label='ROC curve (area = %0.2f)' % auc_score )\n",
    "    plt.plot([0, 1], [0, 1], 'k--')\n",
    "    plt.xlim([0.0, 1.0])\n",
    "    plt.ylim([0.0, 1.05])\n",
    "    plt.xlabel('False Positive Rate or [1 - True Negative Rate]')\n",
    "    plt.ylabel('True Positive Rate')\n",
    "    plt.legend(loc=\"lower right\")\n",
    "    plt.show()\n",
    "    return fpr, tpr, thresholds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfkAAAFzCAYAAAA9nXBaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdeZyNdf/H8dfHECpr0S0S0mJsI0shbku6payVfVIREsmakpK0aFOiooUoUcpSt1KWUhKNDGNJSWW9IzRkneX7+2OG35jGzBnmnOucM+/n4zGP+yzXuc7b1dw+vsv1/ZpzDhEREQk/ebwOICIiIv6hIi8iIhKmVORFRETClIq8iIhImFKRFxERCVMq8iIiImEqr9cBsuvCCy905cqV8zqGiIhIQKxatepP51yJM/lsyBX5cuXKERMT43UMERGRgDCz38/0s+quFxERCVMq8iIiImFKRV5ERCRMqciLiIiEKRV5ERGRMKUiLyIiEqZU5EVERMKUiryIiEiYUpEXEREJUyryIiIiYcpvRd7M3jKz3Wa27jTvm5mNM7PNZrbWzK72VxYREZHcyJ8t+SlA80zevxG4PPWnJ/CqH7OIiIjkOn7boMY5t9TMymVySGtgqnPOAd+ZWVEzK+Wc2+WvTCIiElqmr9jK3NgdXsfwzKE/z64kejkmXxrYlub59tTX/sHMeppZjJnF7NmzJyDhRETEe3Njd7Bh1wGvYwScc47fls9nwejoszqPl1vNWgavuYwOdM5NAiYB1KpVK8NjREQkPEWWKszMXnW9jhFQU6ZM4c63R9OwYUOWLl16xufxsshvBy5J87wMsNOjLCIiIp47cOAAhQsXpmPHjiQkJHDXXXeRN++Zl2ovi/w8oK+ZzQCuAeI1Hi8iEjoCMV6+YdcBIksV9ut3BINDhw4xZMgQPvvsM2JjYylcuDB33333WZ/Xb0XezN4DGgEXmtl24FEgH4Bz7jVgPtAC2AwcBu70VxYREcl5J8bL/VmEI0sVpnVUhtO1wsaKFSuIjo5m8+bNDBo0iHPOOSfHzu3P2fWdsnjfAff66/tFRMT/cuN4eU5JTExk9OjRjB49mtKlS7N48WIaNWqUo9/hZXe9iIj4QaBuO8stXen+YmZ8+eWXdO7cmZdffpkiRYrk+HdoWVsRkTATqNvOckNXek5zzjFp0iR27dpFREQEn376KVOnTvVLgQe15EVEwpK60YPPrl27uOuuu/jss8/YtWsXjz76KAULFvTrd6rIi4gEkGak504ffvghPXv25MiRI0yYMIF77rknIN+r7noRkQAKRFe6utGDy2uvvcatt97KZZddxurVq+nTpw9mGa0Hl/PUkhcRCTB1pecOx48f55xzzqF9+/bEx8czcOBA8uXLF9AMasmLiIjkoGPHjjFkyBAaNGhAQkICxYsX54EHHgh4gQe15EVETuHvMXONl4e3tWvX0rVrV+Li4ujduzeJiYmeFPcT1JIXEUnD32PmGi8PT0lJSTz33HPUrl2b3bt388knn/Dqq6/6ffZ8VtSSFxFJR2Pmkl3Hjx9n8uTJtGjRgkmTJlGiRAmvIwEq8iKSi/jSFa/udPGVc47333+fFi1aUKhQIZYuXUrx4sUDNnPeF+quF5Fcw5eueHWniy/+/PNPbrvtNjp27Mirr74KwAUXXBBUBR7UkheRXEZd8XK2PvvsM+6880727t3LmDFjGDRokNeRTksteRERER+9/PLL3HjjjVx44YV8//33DB06lIiICK9jnZZa8iIS1HLyljaNt8uZcs5hZrRs2ZIdO3YwcuRIChQo4HWsLKklLyJBLSdvadN4u2RXQkICjz32GO3atcM5R7ly5Xj66adDosCDWvIiEgI0ji5e+Omnn4iOjmblypV07dqVY8eOhUxxP0FFXkSCwum65dXFLoHmnOO1115j0KBBFChQgJkzZ9K+fXuvY50RddeLSFA4Xbe8utgl0OLj43n88cdp0KABcXFxIVvgQS15EQki6pYXL33++ec0btyYokWLsnz5csqWLRt0971nl4q8iHgiffe8uuXFK/Hx8fTv35+3336bV155hXvuuYdLL73U61g5Qt31IuKJ9N3z6pYXLyxdupTq1aszbdo0RowYQY8ePbyOlKPUkhcRz6h7Xrw0btw47r//fi677DKWLVvGtdde63WkHKeWvIiI5Er169enV69erF69OiwLPKglLyIBkNHtcRqDl0BLTk5m7NixbN26lZdeeomaNWtSs2ZNr2P5lVryIuJ3Gd0epzF4CaTff/+dpk2bMnjwYLZu3UpiYqLXkQJCLXkRCQiNv4sXnHO888479O3bl+TkZN566y3uuOOOkL81zlcq8iJy1rLaREZd8+KVP/74gz59+hAVFcXUqVMpX76815ECSt31InLWstpERl3zEmgxMTE45/jXv/7FN998w5dffpnrCjyoJS8iOUTd8RIMDh8+zNChQ5kwYQLvvfceHTt2pHr16l7H8oyKvIiIhIXvv/+erl278tNPPzFgwADatGnjdSTPqciLyCmyGl/PiMbcxWvjxo1j4MCBlCpVioULF9K0aVOvIwUFjcmLyCmyGl/PiMbcxWsVK1akQ4cOxMXFqcCnoZa8iPyDxtcl2DnnmDRpEgcPHmTw4MG0aNGCFi1aeB0r6KjIi4SwM+laz4q63iXY/e9//6N79+7Mnz+fFi1aMHDgQPLkUcd0RnRVRELYmXStZ0Vd7xLMPvroI6pUqcLixYsZN24cH3/8sQp8JtSSFwlx6lqX3OLXX3+lffv2REVFMW3aNCpVquR1pKCnf/6IiEhQ++WXXwAoX748X3zxBcuXL1eB95Fa8iJBypfxdo2fSzg7duwYjz76KM8++yyfffYZzZo1o3Hjxl7HCikq8iJB6sR4e2ZFXOPnEq7WrVtH165dWbNmDT179qRuXQ1JnQkVeZEgpvF2yY1eeeUVBgwYQNGiRfn444+5+eabvY4UslTkRQIoO7e8qStecqu8efPSvHlzXn/9dUqWLOl1nJCmiXciAZSdW97UFS+5xYk936dPnw7A3XffzZw5c1Tgc4Ba8iIBpi54kf+3b98+evfuzQcffMCNN95Ip06dMDOvY4UNteRFRMQTCxYsoEqVKsyZM4ennnqKjz/+WAU+h6klLyIiAbd+/XqaN29OZGQk//3vf6lRo4bXkcKSWvIiIhIwe/bsAaBy5crMnDmTVatWqcD7kYq8iIj4XWJiIqNHj+bSSy/lhx9+AKB9+/YUKFDA42ThTd31IiLiV5s3byY6OprvvvuOzp07U758ea8j5RpqyYuIiN+88cYbVK9enR9//JH33nuPd999l2LFinkdK9dQS15ERPxm27Zt1K1blylTplCmTBmv4+Q6KvIiIpKj5syZQ6FChWjatCkjRowgT5482vPdI7rqIiKSIw4ePEj37t1p27YtL774IpCyRK0KvHf8euXNrLmZbTKzzWY2LIP3y5rZEjNbbWZrzayFP/OIiIh/fPPNN1SvXp0pU6YwfPhwPvzwQ68jCX7srjezCGAC0AzYDnxvZvOccxvSHPYw8L5z7lUziwTmA+X8lUlERHLeypUradiwIeXLl+frr7+mXr16XkeSVP5sydcBNjvntjjnjgMzgNbpjnHAiW22igA7/ZhHRERy0KFDhwCoXbs2L730EmvWrFGBDzL+LPKlgW1pnm9PfS2tkUBXM9tOSiu+X0YnMrOeZhZjZjEnVksSERFvJCcnM3bsWCpUqMBvv/2GmdGvXz/OP/98r6NJOv4s8hntMuDSPe8ETHHOlQFaANPM7B+ZnHOTnHO1nHO1SpQo4YeoIiLii23bttGsWTMGDhzItddey7nnnut1JMmEP4v8duCSNM/L8M/u+O7A+wDOueVAAeBCP2YS8cT0FVvpMHG5z3vJiwSj6dOnU7VqVVasWMHrr7+uPd9DgD+L/PfA5WZW3szOAToC89IdsxVoCmBmlUgp8uqPl7AzN3YHG3YdILJUYVpHpR+1EgkNixYtIjIykjVr1tCjRw9tCxsC/Da73jmXaGZ9gQVABPCWc269mY0CYpxz84BBwOtmNoCUrvw7nHPpu/RFwkJkqcLM7FXX6xgi2fLFF19QsmRJqlevzvjx48mXLx9582odtVDh1/9Szrn5pEyoS/vaI2kebwDq+zODiJemr9h6SiteJFQcOXKEYcOGMW7cOG655RZmzZpFwYIFvY4l2aRliET8SN30EopWrVrF1Vdfzbhx4+jfvz/Tpk3zOpKcIfW5iPiZuukllHz11Vdcf/31XHTRRXzxxRdcf/31XkeSs6CWvIiIkJiYCEC9evV44IEHiIuLU4EPA2rJi+SgE2PwJ2gsXoKdc4433niD559/nm+//ZbixYszevRor2NJDlFLXiQHnRiDP0Fj8RLM/vjjD1q1akXPnj0pU6YMx44d8zqS5DC15EVymMbgJRTMmTOHu+++m4MHD/Liiy/Sr18/bQkbhlTkRVKl72o/E+qel1DgnGPixIlccsklvPPOO0RGRnodSfxERV4kVU7cz67ueQlmy5Yt45JLLqFs2bK8++67nH/++ZxzzjlexxI/UpEXSUNd7RKOjh8/zsiRIxkzZgxdu3bl7bffpnjx4l7HkgBQkRcRCWPr16+na9euxMbG0r17d8aOHet1JAkgFXnJ9bT0rISrL774gpYtW1K4cGHmzJlD69atvY4kAaaplJLraelZCTcn9vm65ppr6NatG3FxcSrwuZSKvAj/Pxbf+ZqyXkcROSvTp0+nUaNGHD16lMKFCzNx4kQuuugir2OJR9RdL7lKRrfJqZtewsG+ffu49957mTFjBnXr1mX//v2UKlXK61jiMbXkJVdJvyId6LY3CX1ffPEF1apVY9asWYwePZqlS5eqwAuglrzkQrpNTsJJcnIyw4YNo1ChQsydO5eaNWt6HUmCiIq8iEgIWr16NeXKlaNYsWLMnj2bEiVKULBgQa9jSZBRd73kCtNXbKXDxOX/6KoXCTWJiYk8+eST1KlTh0ceeQSAsmXLqsBLhtSSl1xBt8lJOPjll1+4/fbb+fbbb2nfvj2PPfaY15EkyKnIS66hsXgJZZ9++im33XYbefPm5d1336VTp06YmdexJMipu17CmrrpJVxUq1aN//znP8TFxdG5c2cVePGJiryENXXTSyibN28enTt3Jjk5mdKlS/Phhx9yySWXeB1LQoiKvIQ9rWYnoebgwYPcfffdtG7dmo0bN7J3716vI0mIUpEXEQkiy5YtIyoqijfffJNhw4axYsUKSpQo4XUsCVGaeCciEiQSEhKIjo7GOcfSpUu57rrrvI4kIU5FXkTEY5s2baJcuXLkz5+fefPmUbZsWQoX1n4KcvbUXS8i4pHk5GTGjRtHVFQUTz75JABVqlRRgZcco5a8hKUTu81phzkJVtu3b+fOO+9k4cKF3HzzzfTp08frSBKGVOQlLOnWOQlmn376KZ07d+b48eNMnDiRu+++W/e9i1+oyEvY0gp3EqzKlClDVFQUr7/+OhUrVvQ6joQxFXnx3Imu9ZykbnoJNosWLWLBggU888wzVK1alSVLlngdSXIBTbwTz53oWs9J6qaXYHHkyBEGDBjA9ddfz7x584iPj/c6kuQiaslLUFDXuoSj1atX07VrVzZs2EC/fv14+umnOffcc72OJbmIiryIiB8cOXKE5s2bkzdvXhYsWMANN9zgdSTJhVTkRURy0Pbt27n44ospWLAgs2bNonLlyhQvXtzrWJJLaUxeRCQHOOd48803qVSpEq+++ioADRo0UIEXT/lU5M3sHDPTfR4iIhnYvXs3bdu2pUePHtSuXZuWLVt6HUkE8KG73sxuAl4AzgHKm1kU8Khzrq2/w0n4yOw2Od3uJqHs888/Jzo6mvj4eF544QX69+9PnjzqJJXg4MuY/CjgGmAJgHMuVq16ya7MlpjV7W4SyvLmzUvp0qVZtGgRVapU8TqOyCl8KfIJzrm/0i256PyUR8KYbpOTcLF8+XJWrlxJ//79adKkCTExMWq9S1Dy5bdyo5m1B/KYWXkzexH4zs+5JExMX7GVDhOX5/hiNyJeSEhIYMSIEVx33XW8/PLLHD58GEAFXoKWL7+ZfYGaQDLwEXAU6O/PUBI+tFGMhIuNGzdSt25dRo8eTbdu3fjhhx+0sI0EPV+66//jnHsAeODEC2bWjpSCL5IlddNLqIuPj6du3brky5ePjz76iLZtNe9YQoMvLfmHM3hteE4HEREJNvv37wegSJEiTJ48mbi4OBV4CSmnbcmb2X+A5kBpM3shzVuFSem6FznpdLfI6fY4CVUzZ87knnvu4Y033qBdu3Yq7hKSMmvJ7wbWkTIGvz7Nz+fAjf6PJqHkdDvJaSxeQs3+/fvp0qULHTt25IorrqBq1apeRxI5Y6dtyTvnVgOrzexd59zRAGaSEKWxdwl1S5Ys4fbbb2fXrl2MGjWKBx98kLx5tcWHhC5ffntLm9kTQCRQ4MSLzrkr/JZKRMQDO3bs4LzzzmP58uXUrl3b6zgiZ82XiXdTgMmAkdJN/z4ww4+ZREQCJjY2lpkzZwLQpUsX1qxZowIvYcOXIn+uc24BgHPuF+fcw0Bj/8YSEfGvpKQkxowZQ506dRg+fDgJCQmYGfnz5/c6mkiO8aXIH7OUNW1/MbPeZtYSKOnnXCIifvPrr7/SqFEjhg0bRuvWrVmxYgX58uXzOpZIjvNlTH4AcD5wH/AEUAS4y5+hJLhldLucbpWTULF7926ioqIAmDZtGl26dCHd3hwiYSPLIu+cW5H68CAQDWBmZfwZSoJbRjvK6VY5CXbHjh0jf/78lCxZkjFjxtCiRQvKli3rdSwRv8q0yJtZbaA08I1z7k8zq0zK8rZNgCwLvZk1B14CIoA3nHNPZ3BMe2AkKTvbrXHOdc7uH0ICT7fLSSj55JNP6NWrFx988AH16tWjd+/eXkcSCYjTjsmb2VPAu0AX4DMzG07KnvJrgCxvnzOzCGACKTPyI4FOZhaZ7pjLgQeB+s65ysD9Z/jnEBH5h7///ptevXrRsmVLLrzwQgoX1pCS5C6ZteRbA9Wdc0fMrDiwM/X5Jh/PXQfY7JzbAmBmM1LPuSHNMXcDE5xz+wGcc7uz+wcQEcnId999R3R0NL/88gtDhw5l1KhRmjkvuU5mRf6oc+4IgHNun5n9mI0CDynd/NvSPN8OXJPumCsAzGwZKV36I51zn6U/kZn1BHoCGkMTEZ98+eWXJCQk8OWXX9KwYUOv44h4IrMiX8HMTmwna0C5NM9xzrXL4twZTVd1GXz/5UAjUsb4vzazKs65v075kHOTgEkAtWrVSn8OySGn22QmPc2kl2D1448/snPnTpo0acKQIUPo06ePuuglV8usyN+S7vn4bJ57O3BJmudlSOnyT3/Md865BOBXM9tEStH/PpvfJTkgo1nzGdFMegk2zjkmTJjAkCFDKFeuHOvXryciIkIFXnK9zDaoWXSW5/4euNzMygM7gI5A+pnzc4BOwBQzu5CU7vstZ/m9chY0a15Czc6dO7nzzjv5/PPPufHGG3nzzTfJk8eXdb5Ewp/ftldyziWaWV9gASnj7W8559ab2Sggxjk3L/W9G8xsA5AEDHHO7fVXJhEJL1u3biUqKopjx47x6quv0qtXLy1sI5KGX/dQdM7NB+ane+2RNI8dMDD1R0TEJ8nJyeTJk4dLLrmEvn370rVrV664QhtjiqTnc5+WmeneExHx3JIlS6hcuTI//fQTZsaoUaNU4EVOI8sib2Z1zCwO+Dn1eXUze9nvyURE0jh69CiDBg2iSZMmJCUlcfjwYa8jiQQ9X7rrxwE3kzJJDufcGjPTVrMhTpvMSCiJjY2la9eurF+/nj59+vDMM89w3nnneR1LJOj5UuTzOOd+TzeZJclPeSRAtMmMhJLJkyezd+9e5s+fz4033uh1HJGQ4UuR32ZmdQCXuh59P+An/8aSQNDtchLMfv31V+Lj44mKiuKpp55ixIgRXHjhhV7HEgkpvky8u4eU2e9lgT+Aa1NfExHJcc45Jk+eTLVq1ejRowfOOc4991wVeJEz4EtLPtE519HvSSQgTozFa/xdgtGePXvo1asXs2fPpmHDhkydOlX3vYucBV+K/Pepy83OBD5yzh30cybxo7QFXuPvEkx++uknGjZsyP79+3n22WcZMGAAERERXscSCWlZFnnn3GVmVo+UZWkfM7NYYIZzbobf04lfaCxeglGFChVo2bIl/fr1o1q1al7HEQkLPq1455z7FvjWzEYCLwLvAiryISD9rXLqppdgsmLFCgYNGsRHH31EyZIlef31172OJBJWfFkM53wz62JmHwMrgT1APb8nkxxxonv+BHXTSzBISEjg0UcfpX79+mzbto2dO9NvUCkiOcGXlvw64GPgGefc137OI36g7nkJJps2baJr167ExMTQrVs3XnrpJYoUKeJ1LJGw5EuRr+CcS/Z7EhHJFUaOHMmWLVuYNWsWt9xyi9dxRMLaaYu8mT3vnBsEfGhmLv37zrl2fk0mImFj586dHD9+nHLlyvHyyy+TkJBAqVKlvI4lEvYya8nPTP3f8YEIIiLh6YMPPqB3795ERUWxaNEiLWojEkCnnXjnnFuZ+rCSc25R2h+gUmDiiUioio+PJzo6mvbt23PZZZfx6quveh1JJNfxZUz+Lv7Zmu+ewWsSIBntIHc6umVOvLBhwwZuvPFGduzYwaOPPsrw4cPJly+f17FEcp3MxuQ7kLIATnkz+yjNW4WAv/wdTE4vO8vS6pY58cKll15K1apVef/997nmmmu8jiOSa2XWkl8J7AXKABPSvH4QWO3PUJI13RYnwWbt2rU89thjTJs2jfPOO49PPvnE60giud5pi7xz7lfgV2Bh4OKISKhJSkrihRde4OGHH6Z48eJs3rxZy9KKBInTTrwzs69S/3e/me1L87PfzPYFLqKIBKvffvuNJk2aMHToUG6++Wbi4uJU4EWCSGbd9Y1T/1f3u4hIhnr37s3q1at5++23iY6O1rawIkEms+76E6vcXQLsdM4dN7PrgGrAO8CB031WRMLXn3/+iZlxwQUX8Oqrr2JmlCtXzutYIpKBLDeoAeYAzswuA6aSco/8dL+mEpGgNH/+fKpUqUKfPn0AKF++vAq8SBDzpcgnO+cSgHbAi865foDuyRLJRQ4dOsQ999zDTTfdRIkSJRg+fLjXkUTEB74U+UQzuw2IBk7cE6NVLURyifXr11OjRg0mTpzI4MGD+f777zW5TiRE+LriXR9StprdYmblgff8G0tEgkWJEiUoWrQoixcvplGjRl7HEZFsyLIl75xbB9wHxJjZVcA259wTfk8mIp756aefuPfee0lKSqJkyZKsWLFCBV4kBGVZ5M2sAbAZeBN4C/jJzOr7O5iIBJ5zjldeeYWoqChmzJjBpk2bAHRrnEiI8mVMfizQwjlX3zlXD7gJeMm/sUQk0Hbt2kWLFi249957adiwIXFxcURGRnodS0TOgi9j8uc45zaceOKc22hm5/gxk4gEmHOOdu3asWbNGiZMmMA999yj1rtIGPClyP9gZhOBaanPu6ANakTCQnx8PPny5ePcc8/llVde4dxzz+XKK6/0OpaI5BBfuut7A78AQ4EHgC1AL3+GEhH/++qrr6hWrRrDhg0DoEaNGirwImEm0yJvZlWB5sBs51wr51xL59yzzrmjgYknIjnt2LFjDBkyhMaNG3POOefQuXNnryOJiJ9ktgvdQ6QsadsF+MLM7gpYKhHxiw0bNlC7dm2ee+45evXqRWxsLNdee63XsUTETzIbk+8CVHPOHTKzEsB8Um6hE5EQlTdvXg4dOsQnn3zCTTfd5HUcEfGzzIr8MefcIQDn3B4z82X8Xvxg+oqtzI3dcfL5hl0HiCxV2MNEEkp+//13pk2bxvDhw7niiivYtGkTefP6MudWREJdZv9Pr2BmH6U+NuCyNM9xzrXzazI5aW7sjlMKe2SpwrSO0h5BkjnnHNOmTaNfv3445+jcuTMVKlRQgRfJRTL7f/st6Z6P92cQyVxkqcLM7FXX6xgSIv7880969+7Nhx9+SIMGDXj77bcpX76817FEJMBOW+Sdc4sCGUREcoZzjqZNm7Jx40bGjBnDoEGDiIiI8DqWiHhA/XYiYeLw4cPkz5+fiIgInn/+eUqUKEH16tW9jiUiHtJkOpEwsHLlSmrUqMFzzz0HwPXXX68CLyK+F3kzy+/PICKSfYmJiTz22GPUq1ePI0eOUKdOHa8jiUgQ8WWr2TpmFgf8nPq8upm97PdkIpKpn3/+mfr16zNy5Eg6derE2rVrady4sdexRCSI+DImPw64mZTV73DOrTEz/U0i4rHdu3ezZcsWZs6cSfv27b2OIyJByJfu+jzOud/TvZbkjzAikrldu3bx5ptvAlC/fn1+++03FXgROS1fivw2M6sDODOLMLP7gZ/8nEtE0vnwww+pWrUq9913H7t27QLgvPPO8ziViAQzX4r8PcBAoCzwB3Bt6msiEgDx8fF069aNW2+9lfLly7Nq1SpKlSrldSwRCQFZjsk753YDHQOQRUTSSUxMpG7dumzatIkRI0YwYsQI8uXL53UsEQkRWRZ5M3sdcOlfd8719EsiESEhIYG8efOSN29eHn74YSpUqKAtYUUk23zprl8ILEr9WQaUBI75M5RIbhYXF0etWrV47733AOjcubMKvIicEV+662emfW5m04Av/JZIJJdKTk5m7NixPPTQQxQrVoxixYp5HUlEQtyZrF1fHrg0p4OI5Ga///47d9xxB19++SVt2rRh0qRJlChRwutYIhLifFnxbr+Z7Uv9+YuUVvxDvpzczJqb2SYz22xmwzI57lYzc2ZWy/foIuHjhx9+ICYmhrfeeouPPvpIBV5EckSmLXkzM6A6sCP1pWTn3D8m4Z3msxHABKAZsB343szmOec2pDuuEHAfsCKb2UVC2t69e/n2229p2bIlbdu2ZcuWLSruIpKjMm3Jpxb02c65pNQfnwp8qjrAZufcFufccWAG0DqD4x4HngGOZuPcIiHts88+o2rVqnTu3Jn9+/cDqMCLSI7zZXb9SjO7+gzOXRrYlub59spzUe8AACAASURBVNTXTjKzGsAlzrlPzuD8IiHn8OHD9O3blxtvvJFixYqxdOlSTbATEb85bXe9meV1ziUC1wF3m9kvwCHASGnkZ1X4LYPXTvYEmFkeYCxwR1Yhzawn0BOgbNmyWR0uEpSOHj1KrVq12LhxIwMGDODJJ5+kQIECXscSkTCW2Zj8SuBqoM0Znns7cEma52WAnWmeFwKqAF+mDP3zL2CembVyzsWkPZFzbhIwCaBWrVrZGTIQ8ZxzDjOjQIECdO/enaioKJo2bep1LBHJBTLrrjcA59wvGf34cO7vgcvNrLyZnUPK0rjzTrzpnIt3zl3onCvnnCsHfAf8o8CLhLITe74vXrwYgEGDBqnAi0jAZNaSL2FmA0/3pnPuhcxO7JxLNLO+wAIgAnjLObfezEYBMc65eZl9XiSUOeeYNGkSAwcOJH/+/Pz9999eRxKRXCizIh8BnE/GY+s+cc7NB+ane+2R0xzb6Ey/RySY/O9//6N79+7Mnz+fZs2aMXnyZEqXLp31B0VEclhmRX6Xc25UwJKIhImPPvqIxYsXM27cOO69917y5PHlJhYRkZyXWZE/4xa8SG5z4MAB1q1bR7169ejduzfNmzenQoUKXscSkVwusyaGZgeJ+ODrr7+mevXqtGrVikOHDpEnTx4VeBEJCqct8s65fYEMIhJqjh07xrBhw/j3v/9NREQEH3/8Meedd57XsURETjqTXehEcr2DBw/SoEED1qxZQ8+ePXn++ec5//zzvY4lInIKFfkgNH3FVubG7jj5fMOuA0SWKuxhIkmvUKFCNGnShMcff5yWLVt6HUdEJEOa9huE5sbuYMOuAyefR5YqTOso3YLlta1bt9KiRQvWrVsHwAsvvKACLyJBTS35IBVZqjAze9X1OoaQsrDNu+++y7333ktycjKbN2+mSpUqXscSEcmSWvIimdi3bx8dOnQgOjqaqlWrsmbNGtq0OdPtHEREAksteY+lH38HjcEHk/HjxzNnzhyeeuophgwZQkREhNeRRER8ppa8x9KPv4PG4L12+PBhNm7cCMADDzzAqlWrGDZsmAq8iIQcteSDgMbfg0dMTAxdu3bl6NGjbNq0ifz581O1alWvY4mInBEV+QDJqFse1DUfLBITE3nqqacYNWoU//rXv5gyZQr58+f3OpaIyFlRkQ+QE93y6Qu6uua9t2/fPm666Sa+++47OnfuzPjx4ylWrJjXsUREzpqKfACpWz44FS1alDJlyvDee+/RsWNHr+OIiOQYFXk/0ap1we2PP/5g0KBBjBkzhtKlS/PBBx94HUlEJMdpdr2faNW64DVnzhyqVKnChx9+SExMjNdxRET8Ri15P1L3fHA5cOAAAwYM4K233uLqq69m2rRpREZGeh1LRMRv1JKXXGPkyJFMmTKF4cOHs3z5chV4EQl7asnnsBNj8RqDDw7Hjx9nz549lC5dmkceeYTbbruNunXVuyIiuYNa8jksbYHXGLy31q9fT506dWjZsiVJSUkULVpUBV5EchW15P1AY/HeSk5O5qWXXuLBBx+kcOHCvPHGG1qSVkRyJRV5CSt79uyhY8eOLF68mFatWvH6669TsmRJr2OJiHhCRV7CSqFChThy5AhvvPEGd911F2bmdSQREc9oTF5C3r59++jfvz8HDhygQIECLFu2jO7du6vAi0iupyIvIe2LL76gWrVqvPLKK3z99dcAKu4iIqnUXX8GTrejHGj52kA5cuQIw4YNY9y4cVSqVIl58+Zx9dVXex1LRCSoqCV/BtIvWZuWbp0LjPvuu49x48bRv39/Vq1apQIvIpIBteTPkG6TC7zExET+/vtvihYtyogRI+jQoQPXX3+917FERIKWiryEhF9++YXo6GgKFy7Mp59+StmyZSlbtqzXsUREgpq66yWoOed4/fXXqV69Ohs3bqRbt26aWCci4iO15CVo/fnnn9x11118/PHHNG3alMmTJ3PJJZd4HUtEJGSoyPsg/Wx6zaAPjDx58rBhwwZefPFF+vXrR5486ngSEckO/a3pg/Sz6TWD3n8OHjzI448/zvHjxylevDgbNmygf//+KvAiImdALXkfaTa9/y1btozo6Gh+//13rr32Wpo1a8Y555zjdSwRkZCl5pF47vjx4zz00EM0bNgQgK+++opmzZp5nEpEJPSpJS+eu/POO5k+fTrdu3dn7NixFCpUyOtIIiJhQUVePJGcnMzx48cpUKAAQ4YMoX379rRu3drrWCIiYUXd9RJw27Zto1mzZvTr1w+AqKgoFXgRET9QkZeAmj59OlWrVmXFihVcc801XscREQlrKvISEPv376dTp0506dKFyMhIYmNj6dGjh9exRETCmoq8BMSBAwf44osvGD16NEuXLqVixYpeRxIRCXuaeCd+c+TIEaZOnUrPnj259NJL2bJlC4ULa6VAEZFAUUte/OKHH36gZs2a9O7dm2XLlgGowIuIBJiKvOSoxMREnnzySa655hri4+NZsGAB1113ndexRERyJXXXS47q1KkTs2bNon379rz66qsUL17c60giIrlWriny6XeSyw7tOpc55xzJyclERETQs2dP2rZtS6dOnbTvu4iIx3JNd336neSyQ7vOnd4ff/xB69atefzxxwFo1qwZnTt3VoEXEQkCuaYlD9pJLqfNmzePHj16cODAAa6//nqv44iISDq5piUvOefgwYP06NGD1q1bU7p0aVatWsV9993ndSwREUlHRV6ybfPmzbzzzjsMGzaMFStWULlyZa8jiYhIBnJVd72cuePHjzN//nzatGlDjRo12LJlCxdffLHXsUREJBNqyUuWNmzYwLXXXkvbtm2JjY0FUIEXEQkBfi3yZtbczDaZ2WYzG5bB+wPNbIOZrTWzRWZ2qT/zSPYkJyczbtw4atasybZt25g9ezZRUVFexxIRER/5rcibWQQwAbgRiAQ6mVlkusNWA7Wcc9WAWcAz/soj2XfbbbfRv39/mjZtyrp162jTpo3XkUREJBv82ZKvA2x2zm1xzh0HZgCt0x7gnFvinDuc+vQ7oIwf84iPnHMAtG3blokTJ/Lxxx9z0UUXeZxKRESyy58T70oD29I83w5ck8nx3YFPczrEiZXutGpd1vbv30/fvn1p3LgxPXr0oGvXrl5HEhGRs+DPlnxGS565DA806wrUAp49zfs9zSzGzGL27NmTrRBpC7xWrTu9RYsWUa1aNd5//33++usvr+OIiEgO8GdLfjtwSZrnZYCd6Q8ys+uB4cC/nXPHMjqRc24SMAmgVq1aGf5DITNa6e70jhw5wkMPPcSLL77IVVddxfLly6lVq5bXsUREJAf4syX/PXC5mZU3s3OAjsC8tAeYWQ1gItDKObfbj1nkNJYvX85LL71Ev379WLVqlQq8iEgY8VtL3jmXaGZ9gQVABPCWc269mY0CYpxz80jpnj8f+CB1Q5OtzrlW/sokKZKSkli+fDnXXXcdTZo0Yf369VSqVMnrWCIiksP8uuKdc24+MD/da4+keaxdTQJsy5YtREdHs2LFCjZu3Mjll1+uAi8iEqa04l0u4ZzjzTffpHr16qxfv563336bihUreh1LRET8SGvX5wLOOdq3b8+sWbNo3LgxU6ZMoWzZsl7HEhERP1ORzwXMjFq1alGvXj369+9PnjzqwBERyQ1U5MPU33//zaBBg2jVqhU33XQTDzzwgNeRREQkwNSkC0PLly8nKiqK119/nXXr1nkdR0REPKIiH0YSEhIYMWIE1113HUlJSXz11VdqwYuI5GIq8mFk7ty5jB49mm7durFmzRoaNGjgdSQREfGQxuRDXHJyMj/++CORkZHccsstfPPNN9SvX9/rWCIiEgTUkg9hO3bsoHnz5lxzzTXs2LEDM1OBFxGRk1TkQ9TMmTOpWrUqy5Yt47nnnuPiiy/2OpKIiAQZFfkQk5SURNeuXenYsSNXXHEFsbGx9OrVi9S1/0VERE4K2yI/fcVWOkxczoZdB7yOkqMiIiIoUqQIo0aN4ptvvuHyyy/3OpKIiASpsJ14Nzd2Bxt2HSCyVGFaR5X2Os5ZOXr0KMOHD6dz587UrFmT8ePHq+UuIiJZCtsiDxBZqjAze9X1OsZZWb16NdHR0axfv56SJUtSs2ZNFXgREfFJ2HbXh7qkpCSefvpprrnmGvbt28enn36qhW1ERCRbVOSD1OTJk3nwwQdp3bo1cXFxNG/e3OtIIiISYsK6uz7UOOfYuXMnpUuXplu3bpQsWZKWLVuqe15ERM6IWvJBYs+ePbRr147atWuzf/9+8uXLR6tWrVTgRUTkjKnIB4FPPvmEKlWqMH/+fAYPHkyRIkW8jiQiImFA3fUeOn78OP369WPSpElUq1aNhQsXUrVqVa9jiYhImFBL3kP58uXjf//7H0OHDmXlypUq8CIikqPUkg+whIQEnnzySW6//XbKly/PRx99REREhNexREQkDKklH0A//vgjdevWZeTIkcyaNQtABV5ERPxGRT4AkpOTGT9+PDVq1OC3335j1qxZDBkyxOtYIiIS5lTkA2DcuHH069ePxo0bExcXxy233OJ1JBERyQXCYkx++oqtzI3dccprJzan8VJ8fDxFihShe/fuFC1alG7duum+dxERCZiwaMmf2HEuLS93n/vrr7+Ijo6mXr16HDlyhEKFCnHHHXeowIuISECFRUsegmfHuSVLltCtWzd27tzJI488Qr58+byOJCIiuVRYtOSDwbFjxxg0aBBNmjShYMGCLF++nEceeYS8ecPm31EiIhJiVORzSJ48efj666/p06cPq1evpnbt2l5HEhGRXE7NzLOQlJTE+PHjiY6Opnjx4nz11VcULFjQ61giIiKAivwZ+/XXX7n99tv55ptvcM5x//33q8CLiEhQCekif+LWuUDeLuecY8qUKdx3333kyZOHqVOn0rVr14B8t4iISHaE9Jh82gIfqNvlxowZw1133cXVV1/N2rVriY6O1q1xIiISlEK6JQ+Bu3Xu2LFj5M+fn27dulGgQAH69eundedFwlhCQgLbt2/n6NGjXkeRXKJAgQKUKVMmR2+9Nudcjp0sEIpfWsk1e+gt4P9XtfNnkT906BCDBw/mxx9/ZNGiReTJE9KdHyLio19//ZVChQpxwQUXqLdO/M45x969ezl48CDly5c/5T0zW+Wcq3Um5w25inUkIenkY393069YsYKoqCgmTpxI7dq1SUpKyvpDIhIWjh49qgIvAWNmXHDBBTnecxRy3fUF80X4vXs+ISGB0aNH88QTT1C6dGmWLFnCv//9b79+p4gEHxV4CSR//L6FXEs+EI4ePcrUqVPp0qULa9euVYEXEZGQpCKfyjnH1KlTOXr0KIUKFWLVqlW8/fbbFClSxOtoIpJLRUREEBUVRZUqVWjZsiV//fXXyffWr19PkyZNuOKKK7j88st5/PHHSTvH6tNPP6VWrVpUqlSJq666isGDB3vxR8jU6tWr6dGjh9cxMvXUU09RsWJFrrzyShYsWJDhMQ0aNCAqKoqoqCguvvhi2rRpA6TsRNqyZUuqV69O5cqVmTx5MgB79uyhefPmgfkDOOdC6qdY2atcTtuxY4e74YYbHOAmTpyY4+cXkdCzYcMGryO488477+Tj22+/3Y0ePdo559zhw4ddhQoV3IIFC5xzzh06dMg1b97cjR8/3jnnXFxcnKtQoYLbuHGjc865hIQEN2HChBzNlpCQcNbnuPXWW11sbGxAvzM71q9f76pVq+aOHj3qtmzZ4ipUqOASExMz/Uy7du3c22+/7Zxz7oknnnBDhw51zjm3e/duV6xYMXfs2DHnnHN33HGH++abb/7x+Yx+74AYd4Y1M+TG5HPaBx98QO/evTly5AivvPIKd999t9eRRCTIPPbxejbsPJD1gdkQeXFhHm1Z2efj69aty9q1awGYPn069evX54YbbgDg3HPPZfz48TRq1Ih7772XZ555huHDh3PVVVcBkDdvXvr06fOPc/7999/069ePmJgYzIxHH32UW265hfPPP5+///4bgFmzZvHJJ58wZcoU7rjjDooXL87q1auJiopi9uzZxMbGUrRoUQAqVqzIsmXLyJMnD71792br1q0AvPjii9SvX/+U7z548CBr166levXqAKxcuZL777+fI0eOULBgQSZPnsyVV17JlClT+O9//8vRo0c5dOgQixcv5tlnn+X999/n2LFjtG3blsceewyANm3asG3bNo4ePUr//v3p2bOnz9c3I3PnzqVjx47kz5+f8uXLU7FiRVauXEnduhnPCzt48CCLFy8+2WI3Mw4ePIhzjr///pvixYuf3LSsTZs2vPvuu/+4LjktVxf5xx9/nEceeYTatWszbdo0rrzySq8jiYj8Q1JSEosWLaJ79+5ASld9zZo1Tznmsssu4++//+bAgQOsW7eOQYMGZXnexx9/nCJFihAXFwfA/v37s/zMTz/9xMKFC4mIiCA5OZnZs2dz5513smLFCsqVK8dFF11E586dGTBgANdddx1bt27lP//5Dxs3bjzlPDExMVSpUuXk86uuuoqlS5eSN29eFi5cyEMPPcSHH34IwPLly1m7di3Fixfn888/5+eff2blypU452jVqhVLly6lYcOGvPXWWxQvXpwjR45Qu3ZtbrnlFi644IJTvnfAgAEsWbLkH3+ujh07MmzYsFNe27FjB9dee+3J52XKlGHHjh2nvTazZ8+madOmFC6csgJr3759adWqFRdffDEHDx5k5syZJ2/DrlWrFg8//HCW1/ts5coin5ycTJ48ebj11ltJTk7moYce0r7vInJa2Wlx56QjR44QFRXFb7/9Rs2aNWnWrBmQMsx6upnY2ZmhvXDhQmbMmHHyebFixbL8zG233XZyIbAOHTowatQo7rzzTmbMmEGHDh1OnnfDhg0nP3PgwAEOHjxIoUKFTr62a9cuSpQocfJ5fHw83bp14+eff8bMSEhIOPles2bNKF68OACff/45n3/+OTVq1ABSeiN+/vlnGjZsyLhx45g9ezYA27Zt4+eff/5HkR87dqxvFwdOmeNwQmbX97333jtljsGCBQuIiopi8eLF/PLLLzRr1owGDRpQuHBhSpYsyc6dO33OcqZy1cS7o0ePMnjwYLp16wZApUqVePTRR1XgRSQoFSxYkNjYWH7//XeOHz/OhAkTAKhcuTIxMTGnHLtlyxbOP/98ChUqROXKlVm1alWW5z/dPxbSvpb+vu3zzjvv5OO6deuyefNm9uzZw5w5c2jXrh2Q0pBavnw5sbGxxMbGsmPHjlMK/Ik/W9pzjxgxgsaNG7Nu3To+/vjjU95L+53OOR588MGT5968eTPdu3fnyy+/ZOHChSxfvpw1a9ZQo0aNDO85HzBgwMlJcml/nn766X8cW6ZMGbZt23by+fbt27n44ov/eSGBvXv3snLlSm666aaTr02ePJl27dphZlSsWJHy5cvz448/nryugdjULNcU+bVr11KnTh2ef/55ChUqRGJioteRRER8UqRIEcaNG8dzzz1HQkICXbp04ZtvvmHhwoVASov/vvvuY+jQoQAMGTKEJ598kp9++glIKbovvPDCP857ww03MH78+JPPT3TXX3TRRWzcuPFkd/zpmBlt27Zl4MCBVKpU6WSrOf15Y2Nj//HZSpUqsXnz5pPP4+PjKV06ZXGzKVOmnPY7//Of//DWW2+dnDOwY8cOdu/eTXx8PMWKFePcc8/lxx9/5Lvvvsvw82PHjj35D4S0P+m76gFatWrFjBkzOHbsGL/++is///wzderUyfC8H3zwATfffDMFChQ4+VrZsmVZtGgRAH/88QebNm2iQoUKQMqwR9rhCn8J+yKflJTEs88+S+3atdmzZw/z58/nlVdeOTn5QUQkFNSoUYPq1aszY8YMChYsyNy5cxk9ejRXXnklVatWpXbt2vTt2xeAatWq8eKLL9KpUycqVapElSpV2LVr1z/O+fDDD7N//36qVKlC9erVT45VP/3009x88800adKEUqVKZZqrQ4cOvPPOOye76gHGjRtHTEwM1apVIzIyktdee+0fn7vqqquIj4/n4MGDAAwdOpQHH3yQ+vXrZ7q66A033EDnzp2pW7cuVatW5dZbb+XgwYM0b96cxMREqlWrxogRI04ZSz9TlStXpn379kRGRtK8eXMmTJhwcqiiRYsWp3S3z5gxg06dOp3y+REjRvDtt99StWpVmjZtypgxY7jwwgsBWLJkySmtfn8JybXr9/2+MesDU/3xxx9ERkbSqFEjJk6cePICi4hkZuPGjVSqVMnrGGFt7NixFCpUKOjvlfeHhg0bMnfu3H/Mg8jo9y5XrV3vC+cc//3vf0lOTuaiiy5i9erVzJo1SwVeRCSI3HPPPeTPn9/rGAG3Z88eBg4c6NNEx7MVdkX+zz//5NZbb+Xmm28+OWu0bNmyWoNaRCTIFChQgOjoaK9jBFyJEiVOrornb2E1MD1//nzuuusu9u/fz7PPPnvKGJGISHZldquaSE7zx/B52LTkR40axU033UTJkiX5/vvvGTx48MkJEiIi2VWgQAH27t3rl794RdJzqfvJp52dnxPCpiXfqFEjBg0axOjRo3P8IolI7lOmTBm2b9/Onj17vI4iuUSBAgUoU6ZMjp7Tr7Przaw58BIQAbzhnHs63fv5galATWAv0ME591tm5zwxuz4hIYEnnniC48eP8+STT/rnDyAiIuKxoJxdb2YRwATgRiAS6GRmkekO6w7sd85VBMYCY3w596ZNm6hfvz6PPfYYO3fuVHeaiIhIBvw5Jl8H2Oyc2+KcOw7MAFqnO6Y18Hbq41lAU8tilsuxg/upUaMGv/zyC++//z5TpkzRxBgREZEM+LPIlwa2pXm+PfW1DI9xziUC8cAFZOLw/t00bNiQuLg4brvtthyMKyIiEl78OfEuo+Z1+n51X47BzHoCJzYGPrZgwYJ1J9Y4Fr+4EPjT6xC5gK6z/+ka+5+usf+d8T7o/izy24FL0jwvA6TfV+/EMdvNLC9QBNiX/kTOuUnAJAAziznTCQjiG13jwNB19j9dY//TNfY/M4vJ+qiM+bO7/nvgcjMrb2bnAB2BeemOmQd0S318K7DYaRadiIhIjvBbS945l2hmfYEFpNxC95Zzbr2ZjQJinHPzgDeBaWa2mZQWfEd/5REREclt/LoYjnNuPjA/3WuPpHl8FMju7LlJORBNMqdrHBi6zv6na+x/usb+d8bXOOS2mhURERHfhM3a9SIiInKqoC3yZtbczDaZ2WYzG5bB+/nNbGbq+yvMrFzgU4Y2H67xQDPbYGZrzWyRmV3qRc5QltU1TnPcrWbmzEyzlM+AL9fZzNqn/j6vN7Ppgc4Y6nz4+6KsmS0xs9Wpf2e08CJnKDOzt8xst5mtO837ZmbjUv8brDWzq7M8qXMu6H5Imaj3C1ABOAdYA0SmO6YP8Frq447ATK9zh9KPj9e4MXBu6uN7dI1z/hqnHlcIWAp8B9TyOneo/fj4u3w5sBoolvq8pNe5Q+nHx2s8Cbgn9XEk8JvXuUPtB2gIXA2sO837LYBPSVlj5lpgRVbnDNaWvF+WxJVTZHmNnXNLnHOHU59+R8paB+I7X36PAR4HngGOBjJcGPHlOt8NTHDO7Qdwzu0OcMZQ58s1dkDh1MdF+Oe6KJIF59xSMlgrJo3WwFSX4jugqJmVyuycwVrk/bIkrpzCl2ucVndS/gUpvsvyGptZDeAS59wngQwWZnz5Xb4CuMLMlpnZd6k7ZIrvfLnGI4GuZradlLuq+gUmWq6S3b+3g3Y/+RxbEldOy+frZ2ZdgVrAv/2aKPxkeo3NLA8puy/eEahAYcqX3+W8pHTZNyKlR+prM6vinPvLz9nChS/XuBMwxTn3vJnVJWUNlCrOuWT/x8s1sl33grUln50lcclsSVw5LV+uMWZ2PTAcaOWcOxagbOEiq2tcCKgCfGlmv5EyxjZPk++yzde/L+Y65xKcc78Cm0gp+uIbX65xd+B9AOfccqAAKevaS87x6e/ttIK1yGtJXP/L8hqndiVPJKXAawwz+zK9xs65eOfchc65cs65cqTMe2jlnDvjdapzKV/+vphDykRSzOxCUrrvtwQ0ZWjz5RpvBZoCmFklUor8noCmDH/zgNtTZ9lfC8Q753Zl9oGg7K53WhLX73y8xs8C5wMfpM5p3Oqca+VZ6BDj4zWWs+TjdV4A3GBmG4AkYIhzbq93qUOLj9d4EPC6mQ0gpQv5DjW8ssfM3iNlSOnC1LkNjwL5AJxzr5Ey16EFsBk4DNyZ5Tn130BERCQ8BWt3vYiIiJwlFXkREZEwpSIvIiISplTkRUREwpSKvIiISJhSkRe/M7MkM4tN81Muk2PLnW4Hpmx+55epO2atSV3K9MozOEdvM7s99fEdZnZxmvfeMLPIHM75vZlF+fCZ+83s3LP9bh++Z6SZ7Ui9TQozu8rMlpvZMTMbnM1zVU3z33+fmf2a+nihH3L3MLNkM6uc5rUfzSxH914ws6vTLo9rZm3NbEgOnLeHme1JvT4/mtl9PnymSep901kd1yV1B7M5Z5tTQkNQ3icvYeeIcy7L4uUHXZxzMWbWk5R7/rN1j3/qfakn3AGsI3V1Kedcj5wKyf/nvJOUnM2yOP5+4B1S7pPNMWaWN3UfiLTGOueeS328D7gPaJPdczvn4oCo1O+ZAnzinJvlY4YzsR14COiSA+c6natJWbHwMwDn3OwcPPe7zrn7zawEsMnMPshi0ZMmwJ+kLKh0Ws65d83sD6BvDmaVIKaWvHgitcX+tZn9kPpTL4NjKpvZytQWzVozuzz19a5pXp9oZhFZfN1SoGLqZ5tayn7XcZayd3P+1NeftpS9xtea2XOpr400s8Fmdispa/e/m/qdBVNb4LXM7B4zeyZN5jvM7OUzzLmcNJtNmNmrZhZjKfufP5b62n3AxcASM1uS+toNoF7iGAAABpZJREFUqS3sH8zsAzM7P4NrGWUpG7OsNbPZZlYs9fUvzexJM/sK6J9ZOOfcbufc90BCFn+ObDGz681soZnNAFabWUUzi03z/jAzezj18eVmtsDMVpnZUjO74jSnnQNcbWYVM/i+G9Ncr5lmdl7q661Se1W+NrOXT7R2zeza1ONXW0qv0OVmVhB4BOiS+t/31tQW+ItmVjy1p8JSP3++mW01s7zZyA+Ac24PKSvzlUo9V2szW5Ga5XMzK2lmlwE9gCGpWeqZ2UVm9lHq789KX1r5Ep5U5CUQCtr/d9WeaO3sBpo5564GOgDjMvhcb+Cl1F6AWsB2S1kuswNQP/X1JLJurbUE4sysADAF6OCcq0pKT9Y9ZlYcaAtUds5V4//aO9cQq6oojv/+0UMzGisqCdLqQ0WQTg9lKCNMi6IgEsnETAOLXhCVmINCIhZWWFEqZQ/KKJkeWlKmPZnSyUoItcyM7EFlUjRYlobV6sNaN4+XO869mU3dWb9Pc/fZZ+219z1z9l5r7bsXTC/eHBbnStzibjSzrYXLzwDDC59HAi1/U89z8cmpxGQzOxXoD5wpqb+Z3Yt7E4aY2RD5Ea1TgGExliuBGyvIngfcHP1bg5+kVaK3mZ1pZjM70W9P0gRMjO9lV8wFrjGzU4BmYFYH9f7AvSLNxUJJhwGTgKExXquB6+XhjznAOXhO7z6F2z4CBpvZSXha4OnxDEzDLe7GolfCzH4A1gKDo+hCYHF4KKrVv6TvUfgJc6UQ1ptAU+iyALjJzD4FHgLuDF3a8P+nO+L5uTiuJ92QdNcn/waV3PX7ALPkMejf8bPEy3kbmCyPpS4ws08kDQVOAd4LQ6knvmCoxBOStgKf42kvjwM+M7P1cf0x4Fr8RbsNeEjSi0DVaV/N7DtJG8JS+iTaWB5ya9GzF/4yP7lQfrE81LA3bsmdgE9KRZqifHm0sy8+bn8hqQGfyFsL/X66UKWl2v7uQd42sy93VUFSb7y/z0ZfYdfvsMeBZkl9C2Wn4ePVVhivZVH2sZl9EW3NBy6Le3oD88JirpYWfJH3Fn7k9l016j9a0tn483R55HAH6As8JakPsB+wvoP7hwHHFdo5SFLPsgVq0g3IST7pKm4ANgEDcI/StvIKZvakpHeA84GlksbjqRYfM7Pm8voVGF1M9iLpkEqV4lzuQXhyjUvweOVZNfSlBbeW1gELzczCVVu1nsAqYAYwGxgu6WhgAjDQzNrlceweFe4V8IqZjapB33J+3o17d1ZGuogdXoLxNSTbKerwGzt7GXtEmYDvq93fYWbbJd0NTCyqCCwxszFleg/chahbgaVmNifc/0uqaP45YJqkW4ATgVY8U2a1+pdi8oPxzIRLI0nUbOA2M1sszxA5qYP7BQwqLA6Sbkq665OuogHYGLmmx+BW7E5IOgbYEC7qRbjb+jVgRLhdifhnvyrbXAccVYjTjgFaI4bdYGaL8U1tlV7CP+GpYSuxAN+MNoodVnFNeprZdtzt3hSu/gPxiW+zpMOB8zrQZQVweqlPkvYvj/Oa2WagXdIZxX53pMvuYGYLw2XcuBvZ9L4FjpB0UIRYzg/Z7cDGWEggaS9JAzqR9TA+dgfH5zY89HFMyOgl3+vxIW75HhkLtJEFGQ3A1/H3uEJ5h8+Emf0IvA/cAywysz/+jv5mtgyYj3ui/tIldBxbqFquy6u4N4loqys2vib/AXKST7qKOcBYSStwV30la3Ik8IF8E9bxwDwzW4tPhi9LWg28QmxK6gwz24ZnbXpa0ho8bns//nJ8IeS14l6Gch4F7o99BT3L5LbjMdh+ZvZulNWsZ7hSZwITzGwVPkl8CDyChwBKzAVekvRGbMwaB8yPdlbgY1XOWODOqNOIx5NrQlIfeWasG4Epkr6SdGCtcjojvqfb8PSmi/CxLXEJcJWkVfjYXNCJrF9x6/fQ+LwJz3veEjLagGPN7Bfcg/Mq7mL/BtgcYm7Hx255mfjXgQGxCW5EheZbgEvZORxSk/7BDGB8hHSmAgvx53RToc7zeHjnffkm1mvxxd9qeea9K6poJ6lDMgtdkiQVkTQV2FL4CV1dI+kAM9sSVvIDwBozu6+r9fqnCTf/dWZW808hk/8facknSdIRW4ArFYfhdAOuDq/RWnyj5INdrM8/jqTR+M779q7WJfl3SEs+SZIkSeqUtOSTJEmSpE7JST5JkiRJ6pSc5JMkSZKkTslJPkmSJEnqlJzkkyRJkqROyUk+SZIkSeqUPwGXlL4qwIadbQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fpr, tpr, thresholds = draw_roc( y_pred_df.actual,y_pred_df.predicted_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.78"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc_score = metrics.roc_auc_score( y_pred_df.actual,y_pred_df.predicted_prob )\n",
    "round( float( auc_score ), 2 )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Finding Optimal Cutoff"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Youden's index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tpr</th>\n",
       "      <th>fpr</th>\n",
       "      <th>thresholds</th>\n",
       "      <th>diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>0.846154</td>\n",
       "      <td>0.397129</td>\n",
       "      <td>0.221534</td>\n",
       "      <td>0.449025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161</th>\n",
       "      <td>0.846154</td>\n",
       "      <td>0.401914</td>\n",
       "      <td>0.216531</td>\n",
       "      <td>0.444240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162</th>\n",
       "      <td>0.846154</td>\n",
       "      <td>0.406699</td>\n",
       "      <td>0.215591</td>\n",
       "      <td>0.439455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159</th>\n",
       "      <td>0.835165</td>\n",
       "      <td>0.397129</td>\n",
       "      <td>0.223980</td>\n",
       "      <td>0.438036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166</th>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.421053</td>\n",
       "      <td>0.207107</td>\n",
       "      <td>0.436090</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          tpr       fpr  thresholds      diff\n",
       "160  0.846154  0.397129    0.221534  0.449025\n",
       "161  0.846154  0.401914    0.216531  0.444240\n",
       "162  0.846154  0.406699    0.215591  0.439455\n",
       "159  0.835165  0.397129    0.223980  0.438036\n",
       "166  0.857143  0.421053    0.207107  0.436090"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tpr_fpr = pd.DataFrame( { 'tpr': tpr,'fpr': fpr,'thresholds': thresholds } )\n",
    "tpr_fpr['diff'] = tpr_fpr.tpr - tpr_fpr.fpr\n",
    "tpr_fpr.sort_values( 'diff', ascending = False )[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_df['predicted_new'] = y_pred_df.predicted_prob.map(\n",
    "lambda x: 1 if x > 0.22 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAEGCAYAAABFBX+4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deZwU1b3+8c8zAwKyqqyCW9yNMSqgoEZxX2LcEm+Mxp/bDdGoSTQ3xiW5JkaiJmoSY6IhkrhexRjXGFeMICoY3FfcURABFRRBhZn5/v6oGmhhlp6eLnpq5nnzqld3na465/RM853Tp06do4jAzMzyo6rSFTAzs5Zx4DYzyxkHbjOznHHgNjPLGQduM7Oc6VTpCjTmza339HAXW8lGL7xQ6SpYG1SzZJZam8fS914vOuZ07vuFVpfXGm5xm5nlTJttcZuZrVJ1tZWuQdEcuM3MAGprKl2Dojlwm5kBEXWVrkLRHLjNzADqHLjNzPLFLW4zs5zxxUkzs5zJUYvb47jNzICorSl6a46kv0qaK+m5grTfSHpJ0jOSbpHUp+C1MyS9Kmm6pL2by9+B28wMkouTxW7NuxLYZ4W0+4AtI2Ir4GXgDABJWwCHAV9Mz/mTpOqmMnfgNjODpKuk2K25rCImAR+skHZvRNQ316cAQ9LnBwI3RMRnEfEG8CqwXVP5O3CbmUFycbLITdJoSdMKttEtLO1Y4K70+WDg7YLXZqZpjfLFSTMzaNHFyYgYC4wtpRhJZwE1wHX1SQ0V0VQeDtxmZrBKbnmXdBSwP7B7LF/wdyawTsFhQ4B3msrHXSVmZlDui5MrkbQP8BPggIhYXPDS7cBhkrpI2gDYGHisqbzc4jYzAyLKdwOOpOuBUUBfSTOBs0lGkXQB7pMEMCUijo+I5yXdCLxA0oVyYjRTGQduMzMo6w04EfGtBpLHNXH8GGBMsfk7cJuZgSeZMjPLnRzd8u7AbWYGULu00jUomgO3mRm4q8TMLHfcVWJmljNucZuZ5YwDt5lZvoQvTpqZ5Yz7uM3McsZdJWZmOeMWt5lZzrjFbWaWM25xm5nlTE32CymUiwO3mRm4xW1mljvu4zYzyxm3uM3McsYtbjOznHGL28wsZzyqxMwsZyIqXYOiOXCbmYH7uM3McseB28wsZ3xx0swsZ2prK12Dojlwm5mBu0rMzHLHgdvMLGfcx21mli9R53HcZmb5kqOukqpKV8DMrE2orS1+a4akv0qaK+m5grQ1Jd0n6ZX0cY00XZIukfSqpGckbdtc/g7cZmaQtLiL3Zp3JbDPCmmnAxMiYmNgQroPsC+wcbqNBi5rLnMHbjMzKGvgjohJwAcrJB8IXJU+vwo4qCD96khMAfpIGtRU/g7cq1in9Yaw9vjLl23rTr6VXkccTL8LzlqWNuRf17D2+MsbPL/bDsMYfOtfGXz7lfQ+5pvL8117IIOuuYTBt19JvwvOgk6+fJEnfxl7Ee/MfJqnnpyw0munnvJdapbMYq211mjw3COPPJQXn5/Mi89P5sgjD12Wvu02X+LJJ+7npRcm89uLz8ms7u1GRNGbpNGSphVso4soYUBEzE6KitlA/zR9MPB2wXEz07RGZRa4JW1QTFpHUzNjJu988/hk+9b3iE8/Y9EDDzPvJ2OWpS+6fzKLJkxe+eSqKtY842TmnHgmsw75b7rvsyudv7AuAGv88L/56NqbmXXA0dR99DE9D17xW5q1ZVdffSNf3f+IldKHDFmbPXbfmRkzZjZ43hpr9OFnZ53CDjvtz8gdv8rPzjqFPn16A/DHS8/jhBN+wmZb7MTGG23APnvvmul7yL0WtLgjYmxEDCvYxraiZDWQ1uQQlyxb3P9oIO2mDMvLna7bb8PSmbOpnT33c+nd99qZRXf/e6Xju2y5KTVvv0PNrHehpoZF9zzI6qN2SPIavjWL7p8EwMd33Mvqu+6Y/Ruwsnlo8lQ+mL9gpfSLLvw5p585hmhkytG99tqF+yc8xPz5C1iw4EPun/AQe+89ioED+9OzV0+mTH0cgGuuu4kDDvAf8ybVRfFbaebUd4Gkj/X/8WcC6xQcNwR4p6mMyv59WtJmwBeB3pIOKXipF9C13OXlWfe9R7Hors8H6C7bfona9xdQ89aslY6v7t+XmnfnLduvmfMeXb60GVV9elG38GOorVuWXt1/rWwrb5nbf/89mTVrNs8880KjxwxeeyAzZy7/Pz5r1mwGrz2QwWsPZNbM2cvTZybp1oTs5yq5HTgKOD99vK0g/SRJNwDbAx/Wd6k0JouO0E2B/YE+wNcK0hcC32nqxLSfaDTAmCGbcfhaQzKoXhvRqROr7zKS+ZeM+1xy9312bbC1DYAa+EYV0Uh6GepoFdOtW1fOPP377LPf4U0epwZ+98lHooF0fyiaFGUcxy3pemAU0FfSTOBskoB9o6TjgLeA+gsS/wL2A14FFgPHNJd/2QN3RNwG3CZpZEQ82sJzxwJjAd7ces92/SnrttNwlrz0KnUfFHw9rq6i++478c63vtfgObVz5tFpYL9l+50G9KV23vvUzf+Qqp49oLoKauuWpVt+bbjh+qy//ro8Me0+AIYMGcR/pt7DyB2/ypw5y791zZw1m1123mHZ/uDBg5g46RFmzprN4CHLByYMHjKId96Zs+reQB6V8c7JiPhWIy/t3sCxAZzYkvzL3sct6bT06eHpoPLPbeUuL696NNCy7rb9tix9421q577X4DmfPT+dTusOptPaA6FTJ7rvPYrFE5O/jZ9Oe5rue+yc5P21vVj84CPZvgHL1HPPvcTaQ77MRpuMYKNNRjBz5myGb7/354I2wL33TmTPPXamT5/e9OnTmz332Jl7753Iu+/OZeHCj9l+u+RejiOP+AZ33HFPJd5KfkRd8VuFZXFx8sX0cRrweANbh6euXeg6YiiLJjz0ufSGukmq+61F/0vHJDu1dXxw/qUMuOw8Bt8yjkX3TWLpazMAmP+7v9DryK8z+PYrqerTi4W33L1K3ouVx7XX/JHJk25n00025M3Xp3HM0Yc1euzQbbfiz5f/BoD58xcw5le/Y8ojdzLlkTs5d8xvmZ9e5DzppDP4859/w/QXH+a112dw190PrJL3klvZX5wsGzV2tbrS2ntXiZVmoxcav1BnHVfNklkNDalrkUX/e1jRMaf7OTe0urzWyGJUyR00cWksIg4od5lmZq3WBrpAipXFqJIL08dDgIHAten+t4A3MyjPzKz12kAXSLGyGFUyEUDSLyNi54KX7pA0qdzlmZmVQzmHA2Ytyzsn+0n6Qv1Oert7vyaONzOrnBxdnMxyJqJTgAclvZ7urw98N8PyzMxK1wYCcrEyC9wRcbekjYHN0qSXIuKzrMozM2uV7G95L5vMArek1YFTgfUi4juSNpa0aUT8M6syzcxKlac1J7Ps4/4bsAQYme7PBM7NsDwzs9LlqI87y8C9YUT8GlgKEBGf0PC8s2ZmlVfepcsyleXFySWSupHejCNpQ8B93GbWNrWBlnSxsgzcZwN3A+tIug7YETg6w/LMzErX0QO3ksmAXyK5e3IESRfJDyKi4WnvzMwqLGor3wVSrEwCd0SEpFsjYihwZxZlmJmVVY5a3FlenJwiaXiG+ZuZlU3URdFbpWXZx70r8F1JM4BFJN0lERFbZVimmVlp2kBALlaWgXvfDPM2Myuv/HRxZxq4BwHPR8RCAEk9gS2AGRmWaWZWkqjJT+TOso/7MuDjgv1FaZqZWdtT14KtwrJscSsK1kWLiDpJWZZnZlaytnDRsVhZtrhfl/R9SZ3T7QfA682eZWZWCTlqcWcZuI8HdgBmkUwwtT0wOsPyzMxK5uGAQETMBQ7LKn8zs7JqAy3pYrnP2cwMiJpK16B4DtxmZkC4xW1mljMdOXBLOrWp1yPi4nKXaWbWWu2ixS2pV1MnRsRHjbzUM33cFBgO3J7ufw2Y1NIKmpmtCu0icAPPk6xeU7jcWP1+AOs2dFJE/AJA0r3AtgW3vP8c+Hvrq2xmVn5RW76VFSWdAvw3Sax8FjiGZBqQG4A1gSeAIyNiSSn5Nxq4I2KdUjIssC7JYsH1lgDrtzJPM7NMlKvFLWkw8H1gi4j4RNKNJEOj9wN+GxE3SLocOI4SpwEp6gYcSYdJOjN9PkTS0CJOuwZ4TNLPJZ0NTAWuLqWSZmZZizoVvRWhE9AtneZjdWA2sBtwU/r6VcBBpda12cAt6VKSubWPTJMWA5c3d15EjAGOBeYDC4BjIuJXpVbUzCxLUVf8Jmm0pGkF27K7wiNiFnAh8BZJwP4QeBxYELFstPhMYHCpdS1mVMkOEbGtpCfTSn0gabViMo+IxyW9DXQFkLRuRLxVamXNzLISUXwfd0SMBcY29JqkNYADgQ1IGq1/p+H1CUq+d76YrpKlkqrqC5G0FkWMeJR0gKRXgDeAienjXaVW1MwsSy1pcTdjD+CNiJgXEUuBm0nmbepTMEPqEOCdUutaTOD+I/APoJ+kXwCTgQuKOO+XJCu8vxwRG5C8mYdLraiZWZbqalX01oy3gBGSVpckYHfgBeDfwDfSY44Cbiu1rs12lUTE1ZIeJwm8AIdGxHNF5L00It6XVCWpKiL+LamYgG9mtsoVedGx+Xwipkq6iWTIXw3wJEm3yp3ADZLOTdPGlVpGsXdOVgNLSbpLip0KdoGkHiQ33VwnaS7JmzAza3PKFbgBIuJs4OwVkl8HtitH/sWMKjkLuB5Ym6Rf5v8knVFE3geSjEA5BbgbeI3k7kkzszYnovit0oppcX8bGBoRiwEkjSEZ2nJeUydFxKL0aZ2kO4H3C5cyMzNrS8rZ4s5aMd0eM/h8gO9EE0uQSRoh6UFJN0vaRtJzwHPAHEn7tK66ZmbZiFDRW6U1NcnUb0n6tBcDz0u6J93fi2RkSWMuBc4EegMPAPtGxBRJm5F0udxdprqbmZVNbRnnKslaU10l9SNHnie5GlpvSnN5RsS9AJLOiYgpABHxUjIyxsys7WkLLeliNTXJVKlDVQqHp3+yYrYl5mlmlqk89XE3e3FS0obAGGAL0lvXASJik0ZO+bKkj0imf+2WPifd79rIOWZmFZWnoRPFjCq5EjiXZNKUfUnmlW30ps+IqC5LzczMVqE8tbiLGVWyekTcAxARr0XET0lmCzQzazdq66qK3iqtmBb3Z+n99q9JOh6YBfTPtlpmZqtWe+sqOQXoQbKiwxiSYX7HZlkpM7NVra49jCqpFxFT06cLWb6YgplZu9IuhgNKuoUmhu9FxCGZ1MjMrALaS1fJpausFg3oMaCkxY+tnfvk/ocqXQVrp9pFV0lETFiVFTEzq6S2MFqkWMXOx21m1q7lqKfEgdvMDNpJV8mKJHWJiM+yrIyZWaXkaVRJMSvgbCfpWeCVdP/Lkv6Qec3MzFahuhZslVZMb/wlwP7A+wAR8TS+5d3M2plARW+VVkxXSVVEzFhhLu3ajOpjZlYRNTnqKikmcL8taTsgJFUDJwMvZ1stM7NVqy20pItVTOA+gaS7ZF1gDnB/mmZm1m60hb7rYhUzV8lc4LBVUBczs4ppVy1uSX+hgbHpETE6kxqZmVVAu2pxk3SN1OsKHAy8nU11zMwqo7Y9tbgjYnzhvqRrgPsyq5GZWQXkaOWykm553wBYr9wVMTOrpLr21OKWNJ/lfdxVwAfA6VlWysxsVWs3k0yla01+mWSdSYC6iDxNN25mVpxyXpyU1Ae4AtiS5G/CscB0YDywPvAm8F8RMb+U/Ju85T0N0rdERG26OWibWbtUJxW9FeH3wN0RsRlJ4/dFkp6KCRGxMTCBVvRcFDNXyWOSti21ADOzPKhtwdYUSb2AnYFxABGxJCIWAAcCV6WHXQUcVGpdGw3ckuq7UXYiCd7TJT0h6UlJT5RaoJlZW1Sn4jdJoyVNK9gK72v5AjAP+FsaL6+Q1B0YEBGzAdLH/qXWtak+7seAbWnFXwUzs7xoyaiSiBgLjG3k5U4ksfPkiJgq6feUeUBHU4FbaQVfK2eBZmZtURkv4M0EZkbE1HT/JpLAPUfSoIiYLWkQMLfUApoK3P0kndrYixFxcamFmpm1NeW6ASci3pX0tqRNI2I6sDvwQrodBZyfPt5WahlNBe5qoAfkaFS6mVmJyjxXycnAdZJWA14HjiG5pnijpOOAt4BDS828qcA9OyLOKTVjM7M8qS1jEzUingKGNfDS7uXIv9k+bjOzjqC9zA5Ylr8MZmZ50C4Cd0R8sCorYmZWSTlacrKk2QHNzNqddtHiNjPrSJq7lb0tceA2M6P9L6RgZtbuuKvEzCxnHLjNzHImT4sNOHCbmeE+bjOz3PGoEjOznKnLUWeJA7eZGb44aWaWO/lpbztwm5kBbnGbmeVOjfLT5nbgNjPDXSVmZrnjrhIzs5zxcEAzs5zJT9h24DYzA9xVYmaWO7U5anM7cJuZ4Ra3mVnuhFvcZmb54ha3NanrwYfSdd+vQgS1b7zBwovOp8dJP6TTJpsConbW2yy88Hz49JOVzu32zSPous9+RG0diy67hKWP/weAzsO2o/vxJ6PqKj69604+ufH/VvG7spb66a8uZtLDj7HmGn249drLAbjw0iuY+PBUOnXuxDqDB3HumafSq2cPAKa/+gbn/PoSPl60mKqqKm644vd06bLa5/L88KOF/Ohn5/HOu3NYe+AALvrlGfTu1ZOI4LzfXc5Dj/6Hrl27MOasH7HFphut8vfcluVpOGBVpSvQ0VSt1ZduB32dBSeNZsF3j4HqKrqM2o1Ff76UBSccx4ITjqVu7ly6HXDwSudWr7seXUbtxvzRR/PRWT+mx0mnQFUVVFXR48Qf8tFPT2P+d46iy667U73uehV4d9YSB+23J5dffO7n0kYO34ZbrrmcW66+jPXXGcwV14wHoKamltPP+TU/+/HJ3Hbdn/nbpRfQqVP1Snlecc2NjBi2Nf8aP44Rw7Zm3LU3AvDQo//hrZnv8K/x4/j5ad/nlxdemv0bzJlowVZpmQVuSTsWk9YhVVejLl2gKnmse/89YvHi5a936UJDH4/VRu7EZw8+AEuXUjfnXWrfmUWnTTen06abU/vOLOrenQ01NXz24AOsNnKnVfd+rCTDtv4SvXv1/FzajtsPXRaQt/riZsyZ+x4Ajzz2OJtsuAGbbfwFAPr07kV19cqB+98PPcqB++4BwIH77sEDkx5N0idP4YB9dkcSX95ycxYu/Jh5732Q2XvLoxqi6K3Ssmxx/6HItA6l7v33+OSmG1jzmhtZ8/qbqVu0iKVPTAOgx49OZ80bbqF6nXX55LabVzq3qm9f6ubNXZ7Xe/OoWqsvVWs1kN63b/ZvxjJ1y533stPI4QDMeHsWkhh9ylkcesxJ/PW6vzd4zvvzF9Cv75oA9Ou7Jh8s+BCAOfPeZ2D/5Z+JAf37Mmfeexm/g3yJFvyrtLIHbkkjJf0I6Cfp1ILt58DKTYTPnzta0jRJ066eObvcVWsT1KMHq43ciQ+OOowPDj8Ede1Kl932BODji87ng8O/Tu1bM+iyy24Nnd1AWoAaSK/8Z8ta4c9XXU91dTX777UrADW1tTz5zPNccPZpXH3ZhUyY+AhTpj1ZdH4RK38g1NDnpgOra8FWaVm0uFcDepBc+OxZsH0EfKOpEyNibEQMi4hh/2/IoAyqVnmdtxlG3buziQ8/hNpaljz8EJ222HL5AXV1fDbxAbrstPNK59a9N4+qfv2X7Vf17Ufd++83ku7WVF7d9q/7mPTwY1xw9mnLguuA/n0ZtvWXWKNPb7p17cpXRg7nhemvrXTuWmv0WdYFMu+9D1izT28ABvbvy7tzl38m5sx9j/5911oF7yY/yt3illQt6UlJ/0z3N5A0VdIrksZLWq25PBpT9sAdERMj4hfAiIj4RcF2cUS8Uu7y8qZu7hw6bb5F2o8Nnbfeltq3ZlC19uBlx6w2Ygdq3n5rpXOXTHmYLqN2g86dqRowkOrBQ6iZ/iI101+ievAQqgYMhE6d6DJqN5ZMeXiVvScrn8lTpjHuur/zhwvOplvXrsvSd9xuKC+/9gaffPopNTW1THvqWTbcYN2Vzh+10whuu+t+AG676352/crIZem33z2BiODp516kR4/uy7pULJFBi/sHwIsF+xcAv42IjYH5wHGl1lUNfYVqDUm/i4gfSrqDBr6wR8QBxeTz3t67tNsv+6sfeQyr7bIr1NZS8+qrfPy7X9P7gt+i1buDoOb111j0h4uJxYtZbcQOdNpkMxZf/VcAun3r23Tdaz+itpZFl1/K0mlTAeg8fHt6HH8yVFXx6b3/4pPrr63kW8xM7+v+VukqlM2Pzz6f/zz5DAsWfMRaa/bhe8cdyRXXjGfJ0qX06dULSC5Qnn3ayQDccc8DXHH1eCTxlZHD+dGJyf/7/z3vd/zXQfux5eabsODDj/jRz37F7DnzGDSgHxefe9ay4YBjLv4Tk6dMo1vXrvzyzFPYcvNNKvbey61z3y+0ut/n2+sdUnTMuXbGzU2WJ2kIcBUwBjgV+BowDxgYETWSRgI/j4i9S6lrFoF7aEQ8LmmXhl6PiInF5NOeA7eVrj0FbiufcgTuw9c7uOiYc/1bt34XGF2QNDYixtbvSLoJOI+km/h/gKOBKRGxUfr6OsBdEVHQT1q8st+AExGPp49FBWgzs7agJaNF0iA9tqHXJO0PzE0bsKPqkxssskRlD9ySnqWJCkXEVuUu08ystco4WmRH4ABJ+wFdgV7A74A+kjpFRA0wBHin1AKyuOV9//TxxPTxmvTxCGDxyoebmVVeuW55j4gzgDMA0hb3/0TEEZL+TjKy7gbgKOC2UsvIYlTJjIiYAewYEadFxLPpdjpQUke8mVnWVsENOD8BTpX0KrAWMK7UjLKcZKq7pJ0iYjKApB2A7hmWZ2ZWstoyD9QAiIgHgQfT568D25Uj3ywD93HAXyX1Junz/hA4NsPyzMxKlqfZATML3Onoki9L6kUy7PDDrMoyM2uttnAre7GynB1wgKRxwPiI+FDSFpJKvlPIzCxLHXqSqQJXAvcAa6f7LwM/zLA8M7OS1RFFb5WWZeDuGxE3kn4DSccu1mZYnplZySKi6K3Ssrw4uUjSWqQ340gaQXKB0syszaltAy3pYmUZuE8Fbgc2lPQw0I9mpnU1M6uUttAFUqxMArekKpJbPXcBkhVwYXpELM2iPDOz1moLXSDFyiRwR0SdpIsiYiTwfBZlmJmVU55a3FlenLxX0tfl9ZHMLAfyNBww6z7u7kCNpE9JuksiInplWKaZWUmyuOU9K1neOdkzq7zNzMqtQ3eVSNpb0kqjRyQdLmnPcpdnZlYOHf0GnF8ADa1+8wBwTgblmZm1Wke/AWf1iJi3YmJEvCvJ07qaWZvUFlrSxcoicHctWJ5nGUmdgW4ZlGdm1mptYbRIsbLoKrkZ+Eth6zp9fnn6mplZm1MbdUVvlZZF4P4pMAeYIelxSY8DbwLz0tfMzNqcDt3HnXaRnC7pF8BGafKrEfFJucsyMyuXjt7HDUAaqJ/NKn8zs3LKUx93lndOmpnlRl0b6AIplgO3mRkdvMUtadumXo+IJ8pdpplZa7WF0SLFyqLFfVH62BUYBjxNMsHUVsBUYKcMyjQza5U8dZWUfThgROwaEbsCM4BtI2JYRAwFtgFeLXd5Zmbl4GldE5tFxLJRJRHxnKStMyzPzKxkeWpxZxm4X5R0BXAtyYLB3wZezLA8M7OStYWWdLGyDNzHACcAP0j3JwGXZViemVnJaqO20lUoWpY34Hwq6Y/A/SQtbi8WbGZtVlu4lb1YmQVuSaOAq0jmKRGwjqSjImJSVmWamZXKt7wnLgL2iojpAJI2Aa4HhmZYpplZScrV4pa0DnA1MBCoA8ZGxO8lrQmMB9YnadD+V0TML6WMLFd571wftAEi4mWgc4blmZmVrC6i6K0ZNcCPImJzYARwoqQtgNOBCRGxMTAh3S9Jli3uaZLGAdek+0cAj2dYnplZyco1qiQiZgOz0+cLJb0IDAYOBEalh10FPAj8pJQysgzcJwAnAt8n6eOeBPwpw/LMzErWklveJY0GRhckjY2IsQ0ctz7JzYdTgQFpUCciZkvqX2pdsxxV8pmkS4H78KgSM2vjWtLHnQbplQJ1IUk9gH8AP4yIjyS1roIFPKrEzIzy3jmZrrH7D+C6iKhfsnGOpEFpa3sQMLfU/D2qxMyMso4qETAOeDEiLi546XbgKOD89PG2UsvIMnCvNKok/StkZtbmlHEc947AkcCzkp5K084kCdg3SjoOeAs4tNQCPKrEzIzytbgjYjJJ93BDdi9HGR5VYmaGF1IAklElwMXpZmbWpuVpWtey3zkp6UBJJxbsT5X0erqV3KdjZpaliCh6q7Qsbnk/jeTqab0uwHCSO4aOz6A8M7NW6+gr4KwWEW8X7E+OiPeB9yV1z6A8M7NWawst6WJlEbjXKNyJiJMKdvtlUJ6ZWavlqY9b5f4rI+k64MGI+MsK6d8FRkXEt8paYAcgaXRD8yBYx+bPRceVReDuD9wKfAY8kSYPJenrPigi5pS1wA5A0rSIGFbpeljb4s9Fx1X2rpKImAvsIGk34Itp8p0R8UC5yzIz64iyHMf9AOBgbWZWZlmugGPl435Ma4g/Fx1U2fu4zcwsW25xm5nljAO3mVnOOHC3kqRaSU9JelrSE5J2aOH5P5f0P1nVLy1jlKR/ps8PkHR6+vygdPVpayVJAyT9Xzonz+OSHpV0cJnyflDSSsP+JHWWdL6kVyQ9J+kxSfu2opz1JT2XPh8m6ZL0+aiWfq4tW1lO69pRfBIRWwNI2hs4D9gl60LTVTYU0bK5KCPidpbPJXMQ8E/ghTJXr0NJfxe3AldFxOFp2nrAARkX/UtgELBlusbrABr47EmqjojalmQcEdOAaenuKOBj4JHWVdfKxS3u8uoFzIdkoVBJE9JW+LOSDqw/SNJZkqZLuh/YtKGM0hbcLWlL/mlJO6Qtohcl/Ynk5qZ1JO2Vtu6ekPT3dIFSJO0j6SVJk4FDCvI9WtKlaQvqAOA36TeGDTP7qbR/uwFLIuLy+oSImBERfwCQ1FXS39LPwZOSdm0mvZukGyQ9I2k80G3FAiWtDnwHODmdQpmImBMRN6avfyzpHMT81hMAAAXOSURBVElTgZGShkqamH4buCdd85A0/WlJj5LMn1+f/yhJ/1SySvnxwCnp5+Qr5f/xWUu5xd163dLlibqStH52S9M/BQ5OV3fuC0yRdDuwLXAYsA3Jz/8JGl4Z6BJgYkQcLKka6EEyD8ymwDER8b00358Ce0TEIkk/AU6V9GvgL2ldXgXGr5h5RDyS1uefEXFTeX4UHdYXWX6XcENOBIiIL0naDLhXyRqsjaWfACyOiK0kbdVI3hsBb0XER42U2R14LiL+V8mSgROBAyNinqRvAmOAY4G/kQT/iZJ+s2ImEfGmpMuBjyPiwmZ/ErZKOHC3XmFXyUjgaklbkqz68ytJOwN1wGBgAPAV4JaIWJyec3vD2bIb8P8A0q+5H0paA5gREVPSY0YAWwAPJ9/WWQ14FNgMeCMiXknLuBYYXdZ3bY2S9EdgJ5JW+PD0+R8AIuIlSTOATZpI35nkDzcR8YykZ0qoRi3JKuOQ/LHfErgv/ZxUA7Ml9Qb6RMTE9LhrgJL7yG3VceAuo4h4NG0F9wP2Sx+HRsRSSW+StMqBVk3ou6jguYD7Vpy4S9LWrSzDWuZ54Ov1OxFxYvo5qO8jbmz9wcbSofnf36vAupJ6RsTCBl7/tKBfW8DzETHyc4VLfYoox9og93GXUfp1txp4H+gNzE2D9q7Aeulhk4CD037MnsDXGsluAslXZiRVS+rVwDFTgB0lbZQet3r6VfslYIOCfuvGZmRcCPRs0Zu0hjwAdJV0QkHa6gXPJ5Eslk36+1kXmF5k+pbAVisWmH5jGwdcImm19NhBkr7dQP2mA/3Sb4T1o1G+GBELSL7J7ZQed0Qj78+fkzbGgbv1uqUXbZ4i6Us+Km3pXAcMkzSN5D/ESwAR8UR63FMkX2UfaiTfHwC7SnqWpA/8iyseEBHzgKOB69Ov01OAzSLiU5KukTvTi5MzGinjBuDH6YUxX5wsUSS3Hx8E7CLpDUmPAVcBP0kP+RNQnf4uxwNHpxcUG0u/DOiR/k5PAx5rpOifAvOAF5QM47s13V+xfkuAbwAXSHqa5LNXP7zvGOCP6cXJTxop5w6SxoYvTrYRvuXdzCxn3OI2M8sZB24zs5xx4DYzyxkHbjOznHHgNjPLGQdua5KWz374XDoXyurNn9VoXg3OUtjIsX0kfa+EMhqcbbGx9BWOuVLSN1pQ1rLZ9MxWJQdua84nEbF1RGwJLCGZcGgZJVr8OYqI2yPi/CYO6QO0OHCbdQQO3NYSDwEbqYyzFKbPV5oJETgf2DBt7f8mPe7Hkv6jZNa8XxTk1exsi4UkfSfN52lJ/1jhW8Qekh6S9LKk/dPjqyX9pqDs77b2B2nWGg7cVhRJnUgmIHo2TdoUuDoitiGZP6V+lsJtSeboOFVSV5JZCr9GMrnWwEayr58J8csksyc+D5wOvJa29n8saS9gY2A7YGtgqKSdJQ1l+WyLhwDDi3g7N0fE8LS8F4HjCl5bn2RO668Cl6fv4Tjgw3TCqOHAdyRtUEQ5ZpnwJFPWnPppayFpcY8D1qa8sxQ2NhNiob3S7cl0vwdJIO9JcbMtFtpS0rkk3TE9gHsKXrsxXZziFUmvp+9hL2Crgv7v3mnZLxdRllnZOXBbc5ZNW1svDc6repZCAedFxJ9XKOOHJZRxJXBQRDwt6WiSFV7qrZhXpGWfHBGFAR4liwyYrXLuKrFyaO0shQ3NhLjijHT3AMcW9J0PltSf4mdbLNSTZD7qzqw8I96hkqrSOn+BZGa9e4AT0uORtImk7kWUY5YJt7it1dJVVY4mmaWwS5r804h4WVL9LIXvAZNJJvRf0Q+AsZKOI1kA4IR0bvOH0+F2d6X93JsDj6Yt/o+Bb0fEE0qW93qKZBbExmZbLPQzYGp6/LN8/g/EdJLVYgYAx0fEp5KuIOn7fkJJ4fNIZgM0qwjPDmhmljPuKjEzyxkHbjOznHHgNjPLGQduM7OcceA2M8sZB24zs5xx4DYzy5n/DwoDoAPIm+5qAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_cm( y_pred_df.actual,y_pred_df.predicted_new)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.60      0.72       209\n",
      "           1       0.48      0.85      0.61        91\n",
      "\n",
      "    accuracy                           0.68       300\n",
      "   macro avg       0.69      0.72      0.67       300\n",
      "weighted avg       0.77      0.68      0.69       300\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(metrics.classification_report( y_pred_df.actual,y_pred_df.predicted_new ))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cost Based Approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_total_cost( actual, predicted, cost_FPs, cost_FNs ):\n",
    "    cm = metrics.confusion_matrix( actual, predicted, [1,0] )\n",
    "    cm_mat = np.array( cm )\n",
    "    return cm_mat[0,1] * cost_FNs + cm_mat[1,0] * cost_FPs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "cost_df = pd.DataFrame( columns = ['prob', 'cost'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = 0\n",
    "## iterate cut-off probability values between 0.1 and 0.5\n",
    "for each_prob in range( 10, 50):\n",
    "    cost = get_total_cost( y_pred_df.actual,y_pred_df.predicted_prob.map(lambda x: 1 if x > (each_prob/100) else 0), 1, 5 )\n",
    "    cost_df.loc[idx] = [(each_prob/100), cost]\n",
    "    idx += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prob</th>\n",
       "      <th>cost</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.14</td>\n",
       "      <td>150.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.22</td>\n",
       "      <td>153.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.12</td>\n",
       "      <td>154.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.20</td>\n",
       "      <td>154.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.19</td>\n",
       "      <td>156.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    prob   cost\n",
       "4   0.14  150.0\n",
       "12  0.22  153.0\n",
       "2   0.12  154.0\n",
       "10  0.20  154.0\n",
       "9   0.19  156.0"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cost_df.sort_values( 'cost', ascending = True )[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_df['predicted_using_cost'] = y_pred_df.predicted_prob.map(lambda x: 1 if x > 0.14 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAEKCAYAAAAyx7/DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd5xdVbn/8c93SnqDECAkQToRMEBCr6FIu3T1JxZERCLIVcq9CChXBEHxiqAIChFEQC5FAWkCoUgRSSIh9ITeAiEJIQXSZ+b5/bH3hJNkypkzZ8+cPfm+ee3XnL3O3mutkwxP1ll77WcrIjAzs/yo6uwOmJlZ2zhwm5nljAO3mVnOOHCbmeWMA7eZWc44cJuZ5YwDt5lZmUn6o6SZkl4oKPulpKmSnpN0u6QBBe+dJek1SS9L2r+1+h24zczK70/AASuVPQBsFREjgFeAswAkbQEcBWyZnvM7SdUtVe7AbWZWZhHxGPDRSmXjIqIu3R0PDE1fHwbcFBFLIuJN4DVgh5bqrylzf8vm4+8f7Fs6bRVrXDG5s7tgFahu6Xtqbx3LPnyj6JjTbdDG3wHGFBSNjYixbWjuW8DN6eshJIG80bS0rFkVG7jNzCpVGqTbEqiXk/QjoA64obGoqSZaqsOB28wMoKE+8yYkHQMcDOwTnyaKmgYMKzhsKPB+S/V4jtvMDKC+rvitBJIOAM4ADo2IhQVv3QkcJam7pA2BTYGJLdXlEbeZGRDRULa6JN0IjAbWkjQNOIdkFUl34AFJAOMj4oSIeFHSLcBLJFMoJ0VEi8N/B24zM4CG8gXuiPhKE8VXt3D8BcAFxdbvwG1mBlDGEXfWHLjNzKBDLk6WiwO3mRl4xG1mljdR4mqRzuDAbWYGZb04mTUHbjMz8FSJmVnu+OKkmVnOeMRtZpYzvjhpZpYzvjhpZpYvraQHqSgO3GZm4DluM7Pc8VSJmVnOeMRtZpYz9cs6uwdFc+A2MwNPlZiZ5Y6nSszMcsYjbjOznHHgNjPLl/DFSTOznPEct5lZzniqxMwsZzziNjPLGY+4zcxyxiNuM7OcqfODFMzM8sUjbjOznPEct5lZzuRoxF3V2R0wM6sIDQ3Fb62Q9EdJMyW9UFC2pqQHJL2a/lwjLZekSyW9Juk5SSNbq9+B28wMkhF3sVvr/gQcsFLZmcBDEbEp8FC6D3AgsGm6jQF+31rlDtxmZpCsKil2a0VEPAZ8tFLxYcC16etrgcMLyq+LxHhggKTBLdXvwG1mBhBR/FaadSJietJUTAfWTsuHAO8WHDctLWuWL06amUGbVpVIGkMyrdFobESMLbFlNVHW4r8ODtxmZtCmwJ0G6bYG6hmSBkfE9HQqZGZaPg0YVnDcUOD9liryVImZGZT74mRT7gSOSV8fA9xRUP6NdHXJTsC8ximV5njEbWYGUF9ftqok3QiMBtaSNA04B7gQuEXSccA7wJfSw/8OHAS8BiwEjm2tfgduMzMo652TEfGVZt7ap4ljAzipLfU7cJuZgW95NzPLnRzd8u7AbWYGREPJ67M7nAO3mRl4qsTMLHfKuKokaw7cZmbgEbeZWe44cFtLakcfRu3O+0FAw/S3WHzDr+nx5ZOo3mQrYtFCABbfcAkN7725yrk1O+xN9/2OAmDJuJuom/gwAFXDNqbH105Ftd2oe+kpltxaatoEqwT9+/dj7JUXseWWmxMRHH/8fzF+wqQVjrnk4vM48IC9WbhoEccddyqTn0lSPx999Jf44ZknA/CzC3/D9df/pcP7n0ulJ4/qcJkFbkkbRsSbrZWtbtR/IN32PIQFP/suLFtKj2PPoGbkHgAsueMa6p55ovmTe/Wh+wFfZcFFp0AEvU//DXXPT4BFC+jx/05i8U2X0fDWVHqe8BOqPzuK+imTmq/LKtolF5/H/ff/gy8fNYba2lp69eq5wvsHHrA3m26yIcO32I0ddxjJ5Zf9nF12O4Q11hjA//zoVHbc+SAigonj7+Wuu8Yxd+68TvokOZKjEXeWuUpubaLsrxm2lx9V1VDbDaqqUG13Yv7KaXubVjN8JHUvT4aFn8CiBdS9PJmaz45C/daAHj1peGsqAMsmPkzNiJ2y/ASWob59+7D7bjvyx2tuBGDZsmXMmzd/hWMOOWR/rr8h+d9pwsSn6T+gP+uuuzb77bcnDz70OHPmzGXu3Hk8+NDj7L//6I7+CPnUEMVvnazsI25Jw4Etgf6Sjix4qx/Qo9zt5U3Mm83Sh2+nz7nXEMuWUj91MvVTJ1M7ak+6/8fRdNv/KOpfeZYld/1plYTtGjCQmPPhp3XNnY0GDET9BxJzZ69QXtV/YEd9JCuzjTb6DB9+OJurr7qEESO24Omnn+PU037MwoWLlh8zZL11mfbupwnk3ps2nSHrrZuUTysofy8ptyLkaFVJFiPuzYGDgQHAIQXbSOD4lk6UNEbSU5KeuuaFdzLoWgXo2Zuaz+3IgnOPY8HZ34Bu3anZbjRL7rqWBRecwMJfnYp69aXbvl9s4uQm0vYGoKbS+Vpe1VRXs+22n+PKK69j+x32Z8GChZzxg/9c4Rg18XceEc2UZ9bVLiUaGoreOlvZA3dE3BERxwIHR8SxBdv3I+JfrZw7NiK2i4jtjt1q/XJ3rSLUbL4NDbNnEJ/Mh4Z66p59kuoNP0vMn5McUFfHsgkPUr3+ZqucG3M/RGustXxfAwYS82Yn5QMGrlDeMG/2KudbPkx7bzrTpk1n4r8nA3Dbbfew7TafW+WYocPWW74/ZOhg3p8+IykfWlA+ZDDvT/+gYzqedzmaKil74Jb0g/TlV9MnF6+wlbu9vGmYM4vqDTaH2u4A1Gy2NQ0z3k3mqVM1I3aifvrbq5xbN/VpaoZvCz17JyP34dtSN/XpJOgvXkTVBpsDULvD3slFS8ulGTNmMW3a+2y22cYA7L33bkyZ8soKx9x99ziO/lryrWzHHUYyf958PvhgJuPGPcrn992DAQP6M2BAfz6/7x6MG/doh3+GXMo+H3fZZLGqZEr686kM6s69hrdfoe6ZJ+j1g19DfQMN773Osn/dR88TzkV9+gOi4b03WHLz5QBUDduE2t0OZMmNv4WFn7Dk/pvp/d+XALDkvpuSC5XA4lt+lywH7NaNupcmUf+S//jz7ORT/4frrv0t3brV8uab73Dct09jzPFHAzD2D9fz93sf4oAD9ublKU+wcNEivv3t0wCYM2cuF/zs14z/1z0AnH/BJcyZM7fTPkeuVMBIuliKCp0A+/j7B1dmx6xTrXHF5M7uglWguqXvtftCz4IfH1V0zOl93k2demEpi1Uld9HCgy4j4tByt2lm1m4VMAVSrCymSi5Kfx4JrAv8Od3/CvBWBu2ZmbVfjqZKyh64I+JRAEk/jYg9Ct66S9Jj5W7PzKwcKmGZX7GyvHNykKSNGnckbQgMyrA9M7PS5Wg5YJZJpk4FHpH0Rrq/AfCdDNszMytdBQTkYmUWuCPiPkmbAsPToqkRsSSr9szM2iVHt7xnmR2wF3Aa8JmIOF7SppI2j4i7s2rTzKxUeXrmZJZz3NcAS4Gd0/1pwPkZtmdmVroczXFnGbg3joj/BZYBRMQimsySZGZWARoait86WZYXJ5dK6kl6M46kjQHPcZtZZaqAkXSxsgzc5wD3AcMk3QDsCnwzw/bMzEq3ugduJUmBp5LcPbkTyRTJyRHxYYsnmpl1kqjv/CmQYmUSuCMiJP0tIkYB92TRhplZWeVoxJ3lxcnxkrbPsH4zs7KJhih6a42kUyW9KOkFSTdK6iFpQ0kTJL0q6WZJ3Urta5aBey/gSUmvS3pO0vOSnsuwPTOz0pVpOaCkIcD3ge0iYiugGjgK+AVwSURsCswBjiu1q1lenDwww7rNzMqrvFPcNUBPScuAXsB0YG/gq+n71wI/AX5fauVZGQy8GBEfA0jqC2wBrPpMLjOzThZ15YncEfGepIuAd4BFwDhgEjA3IurSw6YBQ0ptI8upkt8DnxTsL6DEf13MzDLXUPwmaYykpwq2MY3VSFoDOAzYEFgP6E3TMxAlXw3NcsStKHguWkQ0SMqyPTOzkrUlV0lEjAXGNvP2vsCbETELQNJtwC7AAEk16ah7KPB+qX3NcsT9hqTvS6pNt5OBN1o9y8ysM7RhxN2Kd4CdJPVK72nZB3gJ+AfwxfSYY4A7Su1qloH7BJJ/Zd4jmc/ZERjT4hlmZp2kXMsBI2IC8FfgaeB5kjg7FjgDOE3Sa8BA4OpS+5plPu6ZJEtgzMwqXxlXlUTEOSRpPwq9AexQjvo952xmBixf75EDDtxmZkDkJ1WJA7eZGVDuG3AyVfbALem0lt6PiIvL3aaZWXt1iRG3pH4tnRgR85t5q2/6c3Nge+DOdP8Q4LG2dtDMrCN0icANvEhyZ0/h48Ya9wNYv6mTIuJcAEnjgJEFt7z/BPhL+7tsZlZ+UZ+fJys2G7gjYlg7616f5GHBjZYCG7SzTjOzTHSVEfdyko4CNoqIn0kaCqwTEZNaOe16YKKk20lG6EcA17Wrt2ZmGYmGLjDibiTpMqAW2AP4GbAQuIJk/rpZEXGBpPuA3dKiYyNicvu6a2aWja424t4lIkZKmgwQER8V++SGiJgk6V2gB4Ck9SPindK7a2aWjYj8jLiLyVWyTFIVaQpCSQMpYsWjpEMlvQq8CTya/ry3HX01M8tMNBS/dbZiAvflwK3AIEnnAv8keQRPa35K8oT3VyJiQ5JUh0+U2lEzsyw11KvorbO1OlUSEddJmkQSeAG+FBEvFFH3soiYLalKUlVE/ENSMQHfzKzDdamLk6lqYBnJdEmxqWDnSupDctPNDZJmAjlK42Jmq5M8Be5Wg7CkHwE3kjyCZyjwf5LOKqLuw0hWoJwK3Ae8TnL3pJlZxYkofutsxYy4vw6MioiFAJIuIHnw5c9bOikiFqQvGyTdA8wufJSZmVkl6VIjbpKnshcG+BpaeASZpJ0kPSLpNknbSnoBeAGYIemA9nXXzCwbESp662wtJZm6hGROeyHwoqT70/39SFaWNOcy4IdAf+Bh4MCIGC9pOMmUy31l6ruZWdnUV8BqkWK1NFXSuHLkReCegvLxrdUZEeMAJJ0XEeMBImJq8txMM7PKUwkj6WK1lGSq1AdZFi5PX7RytSXWaWaWqTzNcReTq2Rj4AJgC9Jb1wEiYrNmTtla0nyS9K8909ek+z2aOcfMrFPlaelEMatK/gScD1wEHAgcSwu3vEdEdVl6ZmbWgfI04i5mVUmviLgfICJej4izgb2y7ZaZWceqb6gqeutsxYy4lyi5qvi6pBOA94C1s+2WmVnH6mpTJacCfYDvk8x19we+lWWnzMw6WkNXWFXSKCImpC8/Bo7OtjtmZp2jSywHLHjkWJMi4shMemRm1gm6ylTJZR3Wiyb0+PGlndm8Vah5A37Y2V2wLqpLTJVExEMd2REzs85UCatFilVsPm4zsy4tRzMlRT8UwcysS2sIFb21RtIASX+VNFXSFEk7S1pT0gOSXk1/rlFqX4sO3JK6l9qImVmlK3Na198A90XEcGBrYApwJvBQRGwKPJTul6SYJ+DsIOl54NV0f2tJvy21QTOzStTQhq0lkvoBewBXA0TE0oiYS/JUsGvTw64FDi+1r8WMuC8FDgZmp514Ft/ybmZdTKCiN0ljJD1VsI0pqGojYBZwjaTJkq6S1BtYJyKmA6Q/S74DvZiLk1UR8fZKubTrS23QzKwS1bVhOWBEjAXGNvN2DTAS+F5ETJD0G9oxLdKUYkbc70raAQhJ1ZJOAV4pZyfMzDpbW0bcrZgGTCu46/yvJIF8hqTBAOnPmaX2tZjAfSJwGrA+MAPYKS0zM+syyjXHHREfkAx4N0+L9gFeAu4EjknLjgHuKLWvxeQqmQkcVWoDZmZ5UMRIui2+B9wgqRvJw9WPJRko3yLpOOAd4EulVl7ME3D+QBNr0yNiTBOHm5nlUmsj6baIiGeA7Zp4a59y1F/MxckHC173AI4A3i1H42ZmlaK+vCPuTBUzVXJz4b6k64EHMuuRmVknyNGTy0rKVbIh8Jlyd8TMrDM1dKURt6Q5fDrHXQV8RJnXJJqZdbY8JZlqMXCnz5rcmuQ5kwANEXlKN25mVpxyXpzMWovruNMgfXtE1Kebg7aZdUkNUtFbZyvmBpyJkkZm3hMzs05U34ats7X0zMmaiKgDdgOOl/Q6sAAQyWDcwdzMuoyusqpkIsn99SWnHjQzy4uusqpEABHxegf1xcys0+TpAl5LgXuQpNOaezMiLs6gP2ZmnaKrTJVUA30gR98fzMxKlKflgC0F7ukRcV6H9cTMrBPV52iI2uoct5nZ6qCrjLjLkn7QzCwPukTgjoiPOrIjZmadqQ2PnOx0pWQHNDPrcrrEiNvMbHVSCbeyF8uB28yMrrOO28xsteGpEjOznHHgNjPLma6Sq8TMbLXhOW4zs5zxqhIzs5xpyNFkiQO3mRm+OGlmljv5GW87cJuZAR5xm5nlTp3yM+au6uwOmJlVgmjDVgxJ1ZImS7o73d9Q0gRJr0q6WVK3UvvqwG1mRjJVUuxWpJOBKQX7vwAuiYhNgTnAcaX21YHbzIxkOWCxW2skDQX+A7gq3RewN/DX9JBrgcNL7asDt5kZbZsqkTRG0lMF25iVqvs18AM+HaAPBOZGRF26Pw0YUmpffXHSzIy2rSqJiLHA2Kbek3QwMDMiJkka3VjcVDVt6+GnHLjNzID68q3k3hU4VNJBQA+gH8kIfICkmnTUPRR4v9QGPFViZkb5Lk5GxFkRMTQiNgCOAh6OiK8B/wC+mB52DHBHqX114DYzA6IN/5XoDOA0Sa+RzHlfXWpFnioxMyObOycj4hHgkfT1G8AO5ajXgbsDnP2zi3nsiYmsucYA/vbnKwC46LKrePSJCdTU1jBsyGDO/+Fp9OvbB4A/XHczt919P9VVVZx16onsuuOoVeqc9v4HnH7Ohcyb/zGf3WwTLvzxf1NbW8vSpUs566e/4qWXX2VA/35cdN5ZDBm8Tod+XitNzc4HUTtqLwhomPEOS/52BVXDNqPb/l9H1TXUv/8GS++4EhpWDTE12+xB7Z5HALDs0dupe+YxAKoGb0j3I0+Emm7UvzqZpX+/tkM/U57kKTugp0o6wOEHfZ4rLj5/hbKdt9+W26+/gtuv+z0bDBvCVdffDMDrb77NvQ89yh1/voIrLj6fn150GfX1q2YKvuT3f+ToLx/O32++mn59+3Dr3fcDcNvd4+jXtw/33pK8f/Hv/pj9B7R2U981qN3pABZd8UMWXX46VFVR87ld6X7kd1nyl0tZdPnpxLwPqdlmz1VP7tmb2tFfYNHYs1l05dnUjv4C9OgNQLdDjmPJnX9g0W9OQQMHU73pNh38yfKj3HdOZimzwC1p12LKVgfbbfM5+vfru0LZrjuOoqamGoARWw5nxswPAXj48fEcuM+edOvWjaHrrcv6Q9fj+SmvrHBuRDBh0rPsN3p3AA47aF8efuzJ9PwnOeygfQHYb/TuTJj0DBGV8KtmraqqhtpuUFUFtd2JZUugbhkxezoA9a8/T/UWq37Trt5ka+pffx4WLYDFC5LjNt0a9RmAuvek4d1XAah75jGqh2/XoR8pT+qIorfOluWI+7dFlq32br9nHLvtvD0AM2fNZt11Bi1/b52112LmrA9XOH7uvPn07dN7eeBfZ9BazJw1+9Pz114LgJqaavr07sXcefM74mNYO8THc1j2xN30Ou1yep1+BSxeSP0LT0J1NVXrbQRAzRY7UtV/4Crnqt+axPzZn9Y1/yPUb820/KNVyq1pHXBxsmzKPsctaWdgF2CQpNMK3uoHVLdy7hhgDMDvfnU+3/7GV8rdvYpz5bU3Ul1dzcH77QXQ5C+FVlq739QIOrmjtuX3rIL16E3N8FEsvOR7sHgh3b98CtUjdmPJXy6l24HfgOoa6l9/jmhYddqsyb/dZr9ldX7QqVSre1rXbkCftO7C+YH5fLqGsUmFdyMt+/CNLv8bdsffH+CxJyZy1aU/Xx5c1xm0Fh/MmLX8mBkzP2TQoBVHWWsM6M/Hnyygrq6emppqZsz6kEFrJSOpddZeiw9mfsi6aw+irq6eTxYsXGWaxipP9cZb0TBnFiz8GID6lyZSvf5mLH3unyy++ifpMSOoGjh4lXMb5n9E9QZbLN9XvzVpeOulVUbYyQh8TrYfJMcqYSRdrLJPlUTEoxFxLrBTRJxbsF0cEa+Wu728+uf4p7j6hr/w21+cQ88ePZaX77XbTtz70KMsXbqUae9/wDvT3udzn91shXMlscPIEYx75HEA7vj7g+y9+87Lz7/j7w8CMO6Rx9lx1NYecedAzJtN9bBNkjluoGqjrWiY9R707pccUF1D7e6HsuzfD65ybv1rz1K9yYjkgmSP3lRvMoL6154lPplLLF1M1dBNgGTlSf3UpzrsM+VNBtkBM6NyX7iS9OuIOEXSXTTxvSwiDi2mnq404j79nAv59+TnmDt3PgPXHMB3jzuaq66/maXLljGgX/I/5ogth3POD74HJNMnt989jprqas44+Tvsns5/n/hf/8O5Z57C2oMG8u570wuWA27MhT8+nW7durFkyVLO+ukvmfLK6/Tv15dfnnsmw4asOkrLq6WX/rCzu5CZ2r2+SM1WO0NDAw3T32LJHVdSu8+Xqdl8JEgs+/cD1D15LwBV621Ezfb7svSOJF1Gzbajqd0jSTa37LHbqZv86PLjuh9xItR2o/7VZ1h6zzWd8tmy1vu8m9o9Ovn6Z44sOub8+e3bOnU0lEXgHpUmV2li3VIyIi+mnq4UuK18unLgttKVI3B/9TNHFB1z/u/t2zs1cJd9jjsiJqU/iwrQZmaVIE9z3FmsKnmeFi5dR8SIcrdpZtZelTB3XawsVpUcnP48Kf15ffrza8DCDNozM2u3PN3ynsVUyduQ3CUZEYV3Sp4p6QngvHK3aWbWXnmaKsnyzsneknZr3JG0C9A7w/bMzEpWH1H01tmyzA54HPBHSf1J5rznAd/KsD0zs5Kt1lMljdLVJVtL6key7HBeVm2ZmbVXni5OZpkdcB1JVwM3R8Q8SVtIOi6r9szM2iNPSaaynOP+E3A/sF66/wpwSobtmZmVrIEoeutsWQbutSLiFtJvIOmTjVdNbWZmVgEiouits2V5cXKBpIGkN+NI2onkAqWZWcWpr4CRdLGyDNynAXcCG6frtwfRSlpXM7POUglTIMXKJHBLqgJ6AHsCm5Pken85IpZl0Z6ZWXtVwhRIsTIJ3BHRIOlXEbEz8GIWbZiZlVOeRtxZXpwcJ+kLchZ/M8uBPC0HzHqOuzdQJ2kxyXRJRES/DNs0MytJJdzKXqws75z0gw7NLDdW66kSSftLWmX1iKSvSvp8udszMyuH1f0GnHOBpp5+8zBO6WpmFWp1vwGnV0TMWrkwIj6Q5LSuZlaRKmEkXawsRtw9JK3yD4KkWqBnBu2ZmbVbuVaVSBom6R+Spkh6UdLJafmakh6Q9Gr6c41S+5pF4L4N+EPh6Dp9fUX6nplZxamPhqK3VtQB/xURnwV2Ak6StAVwJvBQRGwKPJTulySLwH02MAN4W9IkSZOAt4BZ6XtmZhWnXHPcETE9Ip5OX38MTAGGAIcB16aHXQscXmpfs3jmZB3J8yXPBTZJi1+LiEXlbsvMrFzaMsctaQwwpqBobESMbeK4DYBtgQnAOhExHZLgLmntUvua5TruRcDzWdVvZlZObbkjMg3SqwTqQpL6ALcCp0TE/HLeRJ7lnZNmZrnRUMZlfulijFuBGyKi8dreDEmD09H2YGBmqfVnmavEzCw3yriqRMDVwJSIuLjgrTuBY9LXxwB3lNrXso+4JY1s6f3GSXszs0pSxGqRYu0KHA08L+mZtOyHwIXALemzd98BvlRqA1lMlfwq/dkD2A54liTB1AiSCfrdMmjTzKxdyjVVEhH/JIl5TdmnHG2UfaokIvaKiL2At4GREbFdRIwiubL6WrnbMzMrB6d1TQyPiOWrSiLiBUnbZNiemVnJynlxMmtZBu4pkq4C/kzywOCvkyxENzOrOJUwki5WloH7WOBE4OR0/zHg9xm2Z2ZWsvqo7+wuFC3LG3AWS7oceJBkxO2HBZtZxaqEdK3FyixwSxpNcj/+WyRXWIdJOiYiHsuqTTOzUuUprWuWUyW/AvaLiJcBJG0G3AiMyrBNM7OSeMSdqG0M2gAR8Up6G6iZWcXxqpLEU5KuBq5P978GTMqwPTOzknlVSeJE4CTg+yRz3I8Bv8uwPTOzkpXxlvfMZbmqZImky4AH8KoSM6twnuPGq0rMLF88x53wqhIzyw2PuBNeVWJmueF13AmvKjGz3PCIO+FVJWaWG15VQrKqBLg43czMKlqeLk6W/UEKkg6TdFLB/gRJb6RbyY/qMTPLUkQUvXW2LB4W/AOSh2I26g5sD4wGTsigPTOzdlvdn4DTLSLeLdj/Z0TMBmZL6p1Be2Zm7VYJI+liZRG41yjciYj/LNgdlEF7Zmbtlqc5bpX7XxlJNwCPRMQfVir/DjA6Ir5S1gZXA5LGRMTYzu6HVRb/Xqy+sgjcawN/A5YAT6fFo0jmug+PiBllbXA1IOmpiNius/thlcW/F6uvsk+VRMRMYBdJewNbpsX3RMTD5W7LzGx1lOU67ocBB2szszLLYjmglZ/nMa0p/r1YTZV9jtvMzLLlEbeZWc44cJuZ5YwDdztJqpf0jKRnJT0taZc2nv8TSf+dVf/SNkZLujt9faikM9PXh0vaIsu2VxeS1pH0f2lOnkmSnpR0RJnqfkTSKsv+JNVKulDSq5JekDRR0oHtaGcDSS+kr7eTdGn6enRbf68tW1mmdV1dLIqIbQAk7Q/8HNgz60YlieQaRZtyUUbEnXyaS+Zw4G7gpTJ3b7WS/l38Dbg2Ir6aln0GODTjpn8KDAa2Sp/xug5N/O5Jqo6I+rZUHBFPAU+lu6OBT4B/ta+7Vi4ecZdXP2AOgKQ+kh5KR+HPSzqs8SBJP5L0sqQHgc2bqigdwd2ejuSflbRLOiKaIul3JDc3DZO0Xzq6e1rSXyT1Sc8/QNJUSf8Ejiyo95uSLktHUIcCv0y/MWyc2Z9K17c3sDQirmgsiIi3I+K3AJJ6SARpASIAAAX2SURBVLom/T2YLGmvVsp7SrpJ0nOSbgZ6rtygpF7A8cD30hTKRMSMiLglff8TSedJmgDsLGmUpEfTbwP3SxqcHjcq/f16kiR/fmP9oyXdLWkDkuRwp6a/J7uX/4/P2soj7vbrKekZoAfJ6GfvtHwxcEREzJe0FjBe0p3ASOAoYFuSP/+nafrJQJcCj0bEEZKqgT4keWA2B46NiO+m9Z4N7BsRCySdAZwm6X+BP6R9eQ24eeXKI+JfaX/ujoi/luePYrW1JZ/eJdyUkwAi4nOShgPjlDyDtbnyE4GFETFC0ohm6t4EeCci5jfTZm/ghYj4sZJHBj4KHBYRsyR9GbgA+BZwDUnwf1TSL1euJCLeknQF8ElEXNTqn4R1CAfu9iucKtkZuE7SViRP/fmZpD2ABmAIsA6wO3B7RCxMz7mz6WrZG/gGQPo1d56kNYC3I2J8esxOwBbAE8m3dboBTwLDgTcj4tW0jT8DY8r6qa1Zki4HdiMZhW+fvv4tQERMlfQ2sFkL5XuQ/MNNRDwn6bkSulEP3Jq+3hzYCngg/T2pBqZL6g8MiIhH0+OuB0qeI7eO48BdRhHxZDoKHgQclP4cFRHLJL1FMioH2pXQd0HBawEPrJy4S9I27WzD2uZF4AuNOxFxUvp70DhHrGbOa64cWv/7ew1YX1LfiPi4ifcXF8xrC3gxInZeoXFpQBHtWAXyHHcZpV93q4HZQH9gZhq09wI+kx72GHBEOo/ZFzikmeoeIvnKjKRqSf2aOGY8sKukTdLjeqVftacCGxbMWzeXkfFjoG+bPqQ15WGgh6QTC8p6Fbx+jORh2aR/P+sDLxdZvhUwYuUG029sVwOXSuqWHjtY0teb6N/LwKD0G2HjapQtI2IuyTe53dLjvtbM5/PvSYVx4G6/nulFm2dI5pKPSUc6NwDbSXqK5H+IqQAR8XR63DMkX2Ufb6bek4G9JD1PMge+5coHRMQs4JvAjenX6fHA8IhYTDI1ck96cfLtZtq4CTg9vTDmi5MliuT248OBPSW9KWkicC1wRnrI74Dq9O/yZuCb6QXF5sp/D/RJ/05/AExspumzgVnAS0qW8f0t3V+5f0uBLwK/kPQsye9e4/K+Y4HL04uTi5pp5y6SwYYvTlYI3/JuZpYzHnGbmeWMA7eZWc44cJuZ5YwDt5lZzjhwm5nljAO3tUifZj98Ic2F0qv1s5qtq8kshc0cO0DSd0too8lsi82Vr3TMnyR9sQ1tLc+mZ9aRHLitNYsiYpuI2ApYSpJwaDkl2vx7FBF3RsSFLRwyAGhz4DZbHThwW1s8DmyiMmYpTF+vkgkRuBDYOB3t/zI97nRJ/1aSNe/cgrpazbZYSNLxaT3PSrp1pW8R+0p6XNIrkg5Oj6+W9MuCtr/T3j9Is/Zw4LaiSKohSUD0fFq0OXBdRGxLkj+lMUvhSJIcHadJ6kGSpfAQkuRa6zZTfWMmxK1Jsie+CJwJvJ6O9k+XtB+wKbADsA0wStIekkbxabbFI4Hti/g4t0XE9ml7U4DjCt7bgCSn9X8AV6Sf4ThgXpowanvgeEkbFtGOWSacZMpa05i2FpIR99XAepQ3S2FzmRAL7Zduk9P9PiSBvC/FZVsstJWk80mmY/oA9xe8d0v6cIpXJb2Rfob9gBEF89/907ZfKaIts7Jz4LbWLE9b2ygNzh2dpVDAzyPiypXaOKWENv4EHB4Rz0r6JskTXhqtXFekbX8vIgoDPEoeMmDW4TxVYuXQ3iyFTWVCXDkj3f3AtwrmzodIWpvisy0W6kuSj7qWVTPifUlSVdrnjUgy690PnJgej6TNJPUuoh2zTHjEbe2WPlXlmyRZCrunxWdHxCuSGrMUfgj8kySh/8pOBsZKOo7kAQAnprnNn0iX292bznN/FngyHfF/Anw9Ip5W8nivZ0iyIDaXbbHQ/wAT0uOfZ8V/IF4meVrMOsAJEbFY0lUkc99PK2l8Fkk2QLNO4eyAZmY546kSM7OcceA2M8sZB24zs5xx4DYzyxkHbjOznHHgNjPLGQduM7Oc+f/9t8yQ3SIqVAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_cm( y_pred_df.actual,y_pred_df.predicted_using_cost )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}