Classification Model Evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification Model Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import itertools\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold, cross_val_score\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Selection Process"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Cleaning and Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>148</td>\n",
       "      <td>72</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>33.6</td>\n",
       "      <td>0.627</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>85</td>\n",
       "      <td>66</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>26.6</td>\n",
       "      <td>0.351</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin   BMI  \\\n",
       "0            6      148             72             35        0  33.6   \n",
       "1            1       85             66             29        0  26.6   \n",
       "\n",
       "   DiabetesPedigreeFunction  Age  Outcome  \n",
       "0                     0.627   50        1  \n",
       "1                     0.351   31        0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read data\n",
    "diabetes = pd.read_csv('datasets/diabetes.csv')\n",
    "diabetes.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(768, 9)\n"
     ]
    }
   ],
   "source": [
    "# Dimensions of the data set\n",
    "print(diabetes.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(724, 9)\n"
     ]
    }
   ],
   "source": [
    "# Remove unusual rows of data\n",
    "diabetes_mod = diabetes[(diabetes.BloodPressure != 0) & (diabetes.BMI != 0) & (diabetes.Glucose != 0)]\n",
    "\n",
    "# Dimensions of data set after cleansing\n",
    "print(diabetes_mod.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Features/Response\n",
    "feature_names = ['Pregnancies', 'Glucose', 'BMI', 'DiabetesPedigreeFunction']\n",
    "X = diabetes_mod[feature_names]\n",
    "y = diabetes_mod.Outcome"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg = LogisticRegression(C=1, multi_class='ovr', penalty='l2', solver='liblinear')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Evaluation Procedures"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 01. Train/Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy 0.7955801104972375\n"
     ]
    }
   ],
   "source": [
    "# Split the data into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, stratify = y, random_state = 0)\n",
    "\n",
    "logreg.fit(X_train, y_train)\n",
    "y_pred = logreg.predict(X_test)\n",
    "\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "\n",
    "print(\"Accuracy {}\".format(accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 02. K-Fold Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy 0.7805877119643279\n"
     ]
    }
   ],
   "source": [
    "accuracy = cross_val_score(logreg, X, y, cv = 10, scoring='accuracy').mean()\n",
    "\n",
    "print(\"Accuracy {}\".format(accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Evaluation Metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 01. Classification Accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "We already discussed how classification accuracy works in the previous section."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 02. Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Method to plot the confusion matrix\n",
    "def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues):\n",
    "\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    fmt = '.2f' if normalize else 'd'\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, format(cm[i, j], fmt),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[108  11]\n",
      " [ 26  36]]\n"
     ]
    }
   ],
   "source": [
    "confusion = confusion_matrix(y_test, y_pred)\n",
    "print(confusion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix, without normalization\n",
      "[[108  11]\n",
      " [ 26  36]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVkAAAEmCAYAAADIhuPPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xe8FOW9x/HP99AEARURYjcqYLtiixpbMKJGI7ZYErF7bYklGmNP1OSqpKvRxGCMouYajb3FEm7sigLBXlCMFYVjQQVFyu/+Mc/R9QB79hx2zu4cvm9f+2J3Znbmtyv85tnfPPM8igjMzCwfDbUOwMysI3OSNTPLkZOsmVmOnGTNzHLkJGtmliMnWTOzHDnJWk1J6i7pVknTJP19IfYzXNLd1YytViRtKemFWsdh1SH3k7VKSNoHOB5YA/gImACcHREPLuR+9wOOBjaLiNkLHWidkxTAgIh4qdaxWPtwS9ZaJOl44DzgHKA/sBLwB2CXKux+ZeDFRSHBVkJS51rHYFUWEX74scAHsATwMbBnmW26kSXht9LjPKBbWjcEeAP4ETAFmAwclNadBXwGzErHOAQ4E7iqZN+rAAF0Tq8PBCaRtaZfAYaXLH+w5H2bAY8D09Kfm5Wsuxf4OfBQ2s/dQN8FfLam+E8siX9XYEfgReA94NSS7TcGHgE+SNteCHRN6+5Pn2V6+rx7l+z/JOBt4MqmZek9q6VjbJBeLwc0AkNq/XfDj8oebslaS74OLAbcWGab04BNgfWAwWSJ5vSS9V8hS9bLkyXSiyQtFRFnkLWOr4mInhFxablAJC0OXADsEBG9yBLphPls1we4PW27NPBb4HZJS5dstg9wENAP6AqcUObQXyH7DpYHfgpcAuwLbAhsCfxU0qpp2znAcUBfsu9uG+D7ABGxVdpmcPq815Tsvw9Zq/6w0gNHxMtkCfivknoAlwGXR8S9ZeK1OuIkay1ZGmiM8j/nhwM/i4gpETGVrIW6X8n6WWn9rIi4g6wVN6iN8cwF1pHUPSImR8Qz89nm28DEiLgyImZHxNXA88Cwkm0ui4gXI+IT4FqyE8SCzCKrP88C/kaWQM+PiI/S8Z8B1gWIiHER8Wg67n+APwHfqOAznRERM1M8XxIRlwATgTHAsmQnNSsIJ1lrybtA3xZqhcsBr5a8fjUt+3wfzZL0DKBnawOJiOlkP7GPACZLul3SGhXE0xTT8iWv325FPO9GxJz0vCkJvlOy/pOm90saKOk2SW9L+pCspd63zL4BpkbEpy1scwmwDvD7iJjZwrZWR5xkrSWPAJ+S1SEX5C2yn7pNVkrL2mI60KPk9VdKV0bEXRGxLVmL7nmy5NNSPE0xvdnGmFrjj2RxDYiI3sCpgFp4T9kuPpJ6ktW5LwXOTOUQKwgnWSsrIqaR1SEvkrSrpB6SukjaQdIv02ZXA6dLWkZS37T9VW085ARgK0krSVoCOKVphaT+knZOtdmZZGWHOfPZxx3AQEn7SOosaW9gLeC2NsbUGr2AD4GPUyv7yGbr3wFWnedd5Z0PjIuI/yarNV+80FFau3GStRZFxG/J+sieDkwFXgeOAm5Km/wPMBZ4EngKGJ+WteVY9wDXpH2N48uJsYGsl8JbZFfcv0G6qNRsH+8CO6Vt3yXrGbBTRDS2JaZWOoHsotpHZK3sa5qtPxMYJekDSXu1tDNJuwDfIiuRQPb/YQNJw6sWseXKNyOYmeXILVkzsxw5yZqZ5chJ1swsR06yZmY58mAUNaDO3UNde9U6jEXS+muuVOsQFlmvvvofGhsbW+ozXLFOvVeOmD3PDXJfEp9MvSsivlWtY7aFk2wNqGsvug1qsfeO5eChMRfWOoRF1uabbFTV/cXsT1r8d/TphItautsud06yZlZMEjR0qnUULXKSNbPiUv1fVnKSNbPiUtVKvLlxkjWzgnK5wMwsP8LlAjOz/MjlAjOzXLlcYGaWF7lcYGaWG+FygZlZfgQN9Z/C6j9CM7MFaaj/lmz9FzTMzOanqQtXuUclu5H+ImmKpKdLlvWRdI+kienPpdJySbpA0kuSnpS0QUv7d5I1s4JKNyOUe1TmcrJ51EqdDIyOiAHA6PQaYAdgQHocRjY7cVlOsmZWXFL5RwUi4n6yiTlL7QKMSs9HAbuWLL8iMo8CS0pattz+XZM1s+JquSTQV9LYktcjI2JkBXvuHxGTASJisqR+afnyZLM1N3kjLZu8oB05yZpZMVU21GFjRFRzINv5NY/LTvntcoGZFVcVygUL8E5TGSD9OSUtfwNYsWS7FYC3yu3ISdbMCkpV6V2wALcAB6TnBwA3lyzfP/Uy2BSY1lRWWBCXC8ysmERVxi6QdDUwhKx++wZwBjACuFbSIcBrwJ5p8zuAHYGXgBnAQS3t30nWzAqqOmMXRMT3FrBqm/lsG8APWrN/J1kzKy6PXWBmliMPdWhmlhN5qEMzs1ypwUnWzCwX2XCyrsmameVDzP/+qzrjJGtmBSUaXC4wM8uPywVmZjlykjUzy4kkVIDpZ5xkzayw3JI1M8uRk6yZWV6EywVmZnlyS9bMLCdCTrJmZnlyucDMLC9yucDMLFdOsmZmOVFBxi6o/witXV18xnBeHX0uY/9+6ufLlurdg9v+eBRP3fxTbvvjUSzZqzsAvXsuxnXnHc6Ya05m3HWnsd/Om9Yq7A7n8P8+mJWW68eG663z+bLrr/s7Gwxemx5dGxg3dmwNo6sjauFRB5xk7UuuvPVRdvnBRV9adsJB23LvYy/wX7v8jHsfe4ETDtoOgMP32ornJ73NJnuPYPtDz2fE8bvRpXP9TwdSBPsdcCA333bnl5atvfY6/O3aG9hiy61qFFWdSTXZco964CRrX/LQ+Jd5b9qMLy3baci6XHXrGACuunUMw7ZeF4AAei7eDYDFu3fj/WkzmD1nbrvG21FtseVW9OnT50vL1lhzTQYOGlSjiOpTQ0ND2Uc9cE3WWtRv6V683fghAG83fsgyfXoBcPHf7uO68w5n0t1n02vxxdjvpL+QzZhs1k7qo7FaVm6pXlJI+k3J6xMknVmlfc+RNEHSM5KekHS8lM2oJmkjSRe08P4DJV3YymOe2uz1w62PvGPZdrM1efKFN1h1u9PY5Lvn8ruT96TX4ovVOixbhCzq5YKZwO6S+uaw708iYr2IWBvYFtgROAMgIsZGxDE5HPNLSTYiNsvhGHVpyrsf8ZW+vQH4St/eTH3vIwD223lTbv6/JwCY9Hoj/3nzXQat0r9mcdqiRVIhygV5RjEbGAkc13yFpJUljZb0ZPpzpbT8ckkXSHpY0iRJe7R0kIiYAhwGHKXMEEm3pf1tnPb17/RnaUFrRUl3SnpB0hklse0r6bHUUv6TpE6SRgDd07K/pu0+LnnPiZKeSq3qEW37uurX7fc9xb7DNgFg32GbcNu9TwLw+tvvM2Tj7Cvt16cXA1fpzytvNtYsTlv0FKElm3dN9iLgSUm/bLb8QuCKiBgl6WDgAmDXtG5ZYAtgDeAW4LqWDhIRk1K5oF+zVc8DW0XEbElDgXOA76R1GwPrADOAxyXdDkwH9gY2j4hZkv4ADI+IkyUdFRHrNT+2pB1S7JtExAxJfZpvk7Y7jOxkAF16tvSRambUuQey5YYD6LtkT1668+f8/OI7+PVl93DVLw7mgF2/zuuT32f4iZcCMOKSOxl51r48fu2pSHDa+Tfz7gfTa/wJOob99/0eD9x3L42Njay2ygr85KdnsVSfPhz/w6NpnDqV3Xf5NusOXo9b77ir1qHWVn3k0bJyTbIR8aGkK4BjgE9KVn0d2D09vxIoTcI3RcRc4FlJrfntOb+vewlglKQBZBfDu5Ssuyci3gWQdANZYp8NbEiWdAG6A1NaOO5Q4LKImAEQEe/Nb6OIGEnWsqehR7+6vTp0wCmXz3f5jkf8fp5lk6dOY9j3L5rP1rawrrjq6vku32XX3do5kjom6qYkUE579C44DxgPXFZmm9KkM7PkeUXnKUmrAnPIEuKaJat+DvwrInaTtApw7wKO2fRawKiIOKWS45bEWLdJ06yjElAnFYGycj8NpJbdtcAhJYsfBr6bng8HHmzr/iUtA1wMXBjz9h9aAngzPT+w2bptJfWR1J3s5/5DwGhgD0n90r77SFo5bT9LUhfmdTdwsKQeTe9p62cxs9YoX4+tl5pse7W1fwOU9jI4BjhI0pPAfsCxrdxf00WoZ4B/kiW6s+az3S+BcyU9BDS/FelBslLFBOD61CvhWeB04O4U2z1kNWLIfuo/2XThq0lE3ElWOx4raQJwQis/i5m1UUODyj4qIem41B30aUlXS1pM0lcljZE0UdI1krq2NUa583j7a+jRL7oN2qvWYSyS3n+8Vd2jrYo232Qjxo0bW7Xm5WLLDoxVDpj3WkGpF37xrXERsdGC1ktanqzBtVZEfCLpWuAOsm6hN0TE3yRdDDwREX9sS5z1XzU2M5sPUZ2WLNm1qe6SOgM9gMnAN/miZ9Movuj91Gq+rdbMCquCRNpXUumQZSNTTx8AIuJNSb8GXiPrAXU3MA74ICJmp83eAJZva4xOsmZWTKqod0FjC+WCpYBdgK8CHwB/B3aYz6Ztrqs6yZpZIWVduBa6xDsUeCUipsLnfeY3A5aU1Dm1ZlcA3mrrAVyTNbOCKl+PrbAm+xqwqaQeyjL2NsCzwL+Aptv6DwBubmuUTrJmVlgL2082IsaQXeAaDzxFlhNHAicBx0t6CVgauLStMbpcYGbFVFlNtkURcQZpFL8Sk8jGN1loTrJmVkhNXbjqnZOsmRVWvdw6W46TrJkVVgFyrJOsmRWT5HKBmVmO6mekrXKcZM2ssNySNTPLS5W6cOXNSdbMCqlKt9XmzknWzArL5QIzsxy5JWtmlhfXZM3M8iNaNftBzSwwyUrqXe6NEfFh9cMxM6tcQwGasuVass+QjQZe+imaXgewUo5xmZm1qAA5dsFJNiJWbM9AzMxaQ4JOBSgXVDRot6TvSjo1PV9B0ob5hmVm1rKFHbS7PbSYZCVdCGwN7JcWzQAuzjMoM7NKSOUf9aCS3gWbRcQGkv4NEBHvSeqac1xmZmUJ6FQvmbSMSpLsLEkNpClxJS0NzM01KjOzltRRSaCcSmqyFwHXA8tIOgt4EPhFrlGZmVWgQ5QLIuIKSePI5icH2DMins43LDOz8kQxehdUesdXJ2AWWcnA04ibWV3oEOUCSacBVwPLASsA/yvplLwDMzMrp6VSQb3k30pasvsCG0bEDABJZwPjgHPzDMzMrCUdpXfBq8226wxMyiccM7PKFaFcUG6AmN+R1WBnAM9Iuiu93o6sh4GZWc0IKMB1r7It2aYeBM8At5csfzS/cMzMKqSCD3UYEZe2ZyBmZq1V6HJBE0mrAWcDawGLNS2PiIE5xmVmVlZRygWV9Hm9HLiM7DPtAFwL/C3HmMzMKtIglX3Ug0qSbI+IuAsgIl6OiNPJRuUyM6sZqeMk2ZnKCh8vSzpC0jCgX85xmZm1qBo3I0haUtJ1kp6X9Jykr0vqI+keSRPTn0u1NcZKkuxxQE/gGGBz4FDg4LYe0MysWhoaVPZRofOBOyNiDWAw8BxwMjA6IgYAo9PrNqlkgJgx6elHfDFwt5lZTYmFLwmkCWO3Ag4EiIjPgM8k7QIMSZuNAu4FTmrLMcrdjHAjaQzZ+YmI3dtyQIO1BqzA9f/4Za3DWCSNe+X9WoewyJr+2Zzq7rCykkBfSWNLXo+MiJElr1cFpgKXSRpMNmTAsUD/iJgMEBGTJbW5RFquJXthW3dqZtYeKhi7oDEiNiqzvjOwAXB0RIyRdD4LURpY0AHmKyJGV/NAZmbVJKpyM8IbwBslZdHryJLsO5KWTa3YZYEpbT2Ax4Y1s8JqUPlHSyLibeB1SYPSom2AZ4FbgAPSsgOAm9saY6WDdpuZ1RWpajMjHA38NU0QOwk4iKwBeq2kQ4DXgD3buvOKk6ykbhExs60HMjOrtmrk2IiYAMyvbrvNwu+9spkRNpb0FDAxvR4s6ffVOLiZ2cIowswIldRkLwB2At4FiIgn8G21ZlZjAjpLZR/1oJJyQUNEvNrsKl6VO7yZmbVeneTRsipJsq9L2hgISZ3IisQv5huWmVl5qqNBYMqpJMkeSVYyWAl4B/hnWmZmVlOdCtAJtZKxC6YA322HWMzMKpYN2t0BWrKSLmE+YxhExGG5RGRmVgl1kJYsWXmgyWLAbsDr+YRjZlY50QFashFxTelrSVcC9+QWkZlZBYoyx1dbbqv9KrBytQMxM2utKt1Wm6tKarLv80VNtgF4jyoPBWZm1lodoiWb5vYaDLyZFs2NiAUO5G1m1m7q6NbZcsom2YgISTdGxIbtFZCZWSUEdC5AU7aSDhCPSdog90jMzFqpCAPElJvjq3NEzAa2AA6V9DIwnewEEhHhxGtmNSQaCt6F6zGyuW92badYzMwqpg5wM4IAIuLldorFzKxVin5b7TKSjl/Qyoj4bQ7xmJlVJJtIsdZRtKxcku0E9IQCFD3MbJFU9JsRJkfEz9otEjOzVhDFmG67xZqsmVldUjZwd70rl2SrMlOjmVkeBHQqcpKNiPfaMxAzs9aq/xTbtlG4zMzqQgEask6yZlZMQsUuF5iZ1buiX/gyM6tr9Z9inWTNrKCkgvcuMDOrdy4XmJnlqP5TbDHuSjMzm0fTzQjlHhXvS+ok6d+SbkuvvyppjKSJkq6R1LWtcTrJmllhVXFmhGOB50pe/wL4XUQMAN4HDmlrjE6yZlZQavG/ivYirQB8G/hzei3gm8B1aZNRLMTkBa7JmlkhVTh2QV9JY0tej4yIkc22OQ84EeiVXi8NfJCm3wJ4A1i+rXE6yZpZMVVWEmiMiI0WuAtpJ2BKRIyTNOSLPc8j2hQjTrJWxuQ33+CkYw6lcco7NDQ0sNe+B7H/oT8A4MpL/8hfL/sTnTt15htDt+fHPzm7xtF2LDNnfspR++zEZ5/NZM6c2Wy9/c4ccuwpRAQjf3c2/7rzZjo1NLDrPgez5/6H1zrcmqlCD67NgZ0l7QgsBvQma9kuWTKZ7ArAW209gJOsLVCnzp046YxzWHvd9fn444/4zvZbsNlW36SxcQr/d9dt3DJ6DF27dePdxim1DrXD6dq1G+dfcRM9Fu/J7FmzOPJ7O7DJN4by6ssvMmXym/zvnWNoaGjg/Xen1jrUmqnGUIcRcQpwCkBqyZ4QEcMl/R3YA/gbcABwc1uP4QtftkD9+i/L2uuuD0DPnr1YbcAg3nn7Lf426s8cetSP6NqtGwBL9+1XyzA7JEn0WLwnALNnz2LO7NlI4qb/vYyDjvoxDQ3ZP92lll6mlmHWXDUufC3AScDxkl4iq9Fe2tYdOclaRd54/VWee+oJBm/wNf4zaSJjxzzMXjt+g313256nJoyrdXgd0pw5czhw560Y9vVBbLT5ENYevBFvvv4Ko++4kUN2/yY/OmRPXv/Poj2ZdBW7cBER90bETun5pIjYOCJWj4g9I2JmW2MsZJKVNEfSBEnPSHpC0vGSGtK6jSRd0ML7D5R0YSuPeWqz1w+3PvJimj79Y445ZB9O+dkv6dmrN3Nmz+bDaR9wze33cuJPz+aHh+1HRJuvC9gCdOrUictvuZ8b7n+a554cz6QXn2XWZ5/RtWs3Lr3h/9h5r/0595Sjax1mzVTzZoQ8FTLJAp9ExHoRsTawLbAjcAZARIyNiGNyOOaXkmxEbJbDMerOrFmzOOaQfRi2+95s9+1dAOi/7PJsu+POSGLd9TdKtcHGGkfacfXqvQTrb7w5jz4wmmX6L8eQ7XcGYKvtduLlF56pcXS1VJ1+snkrapL9XERMAQ4DjlJmSMmtcRtLejjdLvewpEElb11R0p2SXpB0RtNCSftKeiy1lP+UbrcbAXRPy/6atvu45D0nSnoqtapHtM8nz19EcPrxR7LagEEcdMQX562h3xrGmAfvA+CVlycya9ZnLLV031qF2SG9/14jH304DYCZn37C2IfvY+VVB7Ll0B0Z9+j9APz7sYdYcZXVaxlmbbVQKqiThmzH6F0QEZNSuaD5FZjnga0iYrakocA5wHfSuo2BdYAZwOOSbgemA3sDm0fELEl/AIZHxMmSjoqI9ZofW9IOZHeDbBIRMyT1yeVD1sD4xx7h5uuuZuCaa7Pr0E0BOO6UM9n9e/tz2nFHMGzIRnTp0pUR548sxGhIRfLulHc4+6TvM3fuHObOncs3d9iVzbfennU33JSf/egwrr38j3TvsTgnnX1+rUOtmcJPpFhA8/u2lwBGSRpA1pm4S8m6eyLiXQBJNwBbALOBDcmSLkB3oKX+SUOByyJiBix4AkpJh5G1uFlu+RUr/Ei1teEmm/H85OnzXferi/7SztEsWlZfY20uu/m+eZb36r0Ev7rkmhpEVJ/qP8V2kCQraVVgDllCXLNk1c+Bf0XEbpJWAe4tWdf8Sk2Q/T8blfrOVXz4+exrHulWvpEA6wzewFeJzKqgCL+gCl+TlbQMcDFwYcx7iXsJ4M30/MBm67aV1EdSd7Kf+w8Bo4E9JPVL++4jaeW0/SxJXZjX3cDBkno0vWdhP5OZVaYINdmiJtmmi1DPAP8kS3RnzWe7XwLnSnoI6NRs3YPAlcAE4PrUK+FZ4HTgbklPAvcAy6btRwJPNl34ahIRdwK3AGMlTQBOqMonNLMWqYVHPShkuSAimifM0nX3ksoCEfEIMLBk9U/S8suByxfw/muAeYpeEXES2V0gTa97ljwfAXSYXgVmRSCKUS4oZJI1M6twFK6ac5I1s8IqQI51kjWzopLLBWZmeSpAjnWSNbNiqqceBOU4yZpZYblcYGaWowLkWCdZMyuuAuRYJ1kzKyi5XGBmlpvsjq9aR9EyJ1kzK6wC5FgnWTMrLpcLzMxyVIAc6yRrZsVVgBzrJGtmxeShDs3M8uShDs3M8lWAHOska2ZF5aEOzcxyVYAc6yRrZsXkoQ7NzHJWhHJBUacENzNDKv9o+f1aUdK/JD0n6RlJx6blfSTdI2li+nOptsboJGtmhaUWHhWYDfwoItYENgV+IGkt4GRgdEQMAEan123iJGtmxZSGOiz3aElETI6I8en5R8BzwPLALsCotNkoYNe2humarJkVUrWHOpS0CrA+MAboHxGTIUvEkvq1db9OsmZWWBXk2L6Sxpa8HhkRI+fZj9QTuB74YUR8WM0Lak6yZlZYDS0nw8aI2KjcBpK6kCXYv0bEDWnxO5KWTa3YZYEpbY6xrW80M6u5hbzypazJeinwXET8tmTVLcAB6fkBwM1tDdEtWTMrrCr8qN8c2A94StKEtOxUYARwraRDgNeAPdt6ACdZMyskqaJyQVkR8SALztXbLNTOEydZMyuu+r/hy0nWzIqrADnWSdbMikoLXS5oD06yZlZI1b4ZIS/uwmVmliO3ZM2ssFwuMDPLiydSNDPLT1Fqsk6yZlZYKkAnLidZMysst2TNzHLkJGtmlqMilAsUEbWOYZEjaSrwaq3jaKO+QGOtg1iEFfn7XzkilqnWziTdSfZ9lNMYEd+q1jHbwknWWkXS2JYGQbb8+PsvHt/xZWaWIydZM7McOclaa80zCZ21K3//BeOarJlZjtySNTPLkZOsmVmOnGTNzHLkJGvtLs113/S8Sy1jMcubk6y1K0mKdLVV0n7ADyR1qnFYHUazE9hitYzFMk6y1i5K/vErvf4+cBxwR0TMqVlgHUizE9gBwDH+pVB7TrLWXtYCiIi5kpYEtgL2jogXJXmgooUwnxPYkcAPgZsiYlbNAjPASdZypkwn4BxJSwNExAdAJ2BrSQ0RMTttu5WkxWsYblGtC5+fwHoDmwPD0wnMLdkac5K13KVywG7AupL+mhb/A1gR2AxA0t7AiYCTbOudJmkZgIj4kKxFO1RS56aWrKSt0y8Ia2dOspYbSf344u/YusCjwJaSRkTEX4DZwImSbgVOBU6JiCm1iba4ImIvYM30PQLcDiwHbAkgaS+yE1i32kS4aPNttZYbSbuQ1QZvBnaMiO0kLQGMB66NiFNSCWEg8EpEvF3DcAtFUn9gWkR8Kmkj4N/Ac8DdEXGUpJOBTcgS67LA/hHxVO0iXnQ5yVquJN0PbAwMjogX0rLewOPAmIjYv5bxFZWkbwE/BW4CtouIoan+OgH4V0q0vYA1gNci4p0ahrtIc5K1qirtRpReH0T2s3UTYLOImJaW9wLuBPYA3g7/RWw1SXcBQ4BNImJCWtYFGAtMiojdahieJe46Y1XTrJ/mHmQ9CCZGxGWSLiSrya4paXdgVkRsXsNwC6f5CQwYBbwAXC9pi4iYHBGzJG0I3ClpeeAtn8Bqy0nWqqYkwf4I2ImsB8GPJZ2Vfr5eJukJIIDhNQy1cJqdwPYCugOvR8QxkmYADwGrphNYl4gYWsNwrYSTrFVVuiCzbkRsnZLtVODulCQOkvQ14I2ImFzbSIulJMEeDwwDbgGOlXRORJwsaXFJTwOzgH1rGKo14yRrCyXdTDC3ZNGn2WLdRPb3a+f0E3Z/SQ9HxOO1ibT4Ul/YddIJ7FhgCnBr6g97tKT1yOrb7qVRR9xP1tpMUqemBCtpkKSV04Wth4FlgHNSgj0QOAmYWbtoi0dS83+fM7LFuh3YFtgpImYC+0paMyImOMHWHydZaxNJawAnp+eHA1cCt0g6iezv1U3AbyT9BTgB2CsiXq9VvEWTWqdNJ7C1JK0eEdOBB4BewK8jYnYaCObHwPQahmtluAuXtYmkHclulZ0GDAB2BwYD25CVCa4lu0W2F/BqRLxRo1ALR9IgYJ+IOEPSocBhZDcV/B14D1iM7PueBKxPNtDOM7WK18pzTdba6m6yXgK7ko1BEBExXtJnwB+AcRFxdy0DLLCVgP6SfgesStbHeCCwI7AkcA1wa3r+RkS8VatArWUuF1jFSgeETiNn3QVcAUwGTpW0WEQ8DTwJrN78PVax0cB1QFey77F7RDwP3EZWi107Il6MiMecYOufW7JWkWb9NA8iu9FgTrrR4FfAfsANkm4kG2rvfPii65GVV/r9piELRwPvA8sDJ0n6TRq6cDywSvP3WP1ykrWKlCTYY8luhT0H+F26IHOapE+A35Al2D0i4uXaRVsszU5g/01Wf50bEX+ZaReKAAAIMElEQVSUNAI4CLhR0vVkt9F+F3wCKwqXC6wsSRtKavrpPwDYAtiO7ILLRGBIamWNIett8GMn2NYpSbA/BPYHXiSbOua8iHiUrMYdwEZkJ7AXaxastZpbsrZAkrqRTRtzj6SeETFR0tHA1mR9NDeTNAy4VtLMiDi1pgEXTBqi8NOIeFrSV8kucG1PNjzki8B/SfpTRByehi58JSIaaxiytYFbsrZAETEzIq4EliLrA/tfqbN7F+CfabPewJnAn2sTZTFJ6gqsBjRKWiIiXiFLrlsBwyJiF+BcYHhq0T7uBFtMTrI2D0mrpTLBlpK6k7WqRgNnSVoLeJWsTHAJ8HOyAbgn1TDkwomIzyLiGrKT1C2SNkljvjYA96XN+gE/Ay6oUZhWBb4Zwb5E0reBs8i6ZfUku7q9I/A2cBSwKXA40IOs7+akiJhYm2iLJ9W1+5KNojU+Ij5IA+lsSXbC+hi4hKzevQ3wTZ/Ais1J1j6XRts/i+zi1f1p2a+BXchqha+QjUGwHfCDiHiuVrEWUTqB/Zzsl0BPYBDZkJAvAkeQJdWjyEbSGkh2o8FLtYnWqsUXvgz4/FbOO8imMrk/3VjwaUSckAYquRXYALiU7D75j2oYbuGkE9hPgOMi4r607Ayy7/XbEXFeum/jSuDYiLi3VrFadbkla5+TdBXZOARDIuKTpkSb7tq6FzgqIp5Ko2/NqWmwBSKpD9BINuzjbU3fa1p3JtmNHIPJ7vDaA/iHB9PpOHzhy5DUCSAi9gWeBcZK6pYSbJfUj/Mjsp+xOMG2TkS8RzbQ9rmSlk7fa7e07kzgNWBg2u5SJ9iOxUnWiIg5JYn2IOAxYHxKtLMk7Q80tcasDSLiduBE4DFJS0XETGWTHgJ8iE9gHZaTrAELTLT3p3EKjgQOdz/NhRMR/yC7sDU2JdqmE9hXAE/Z3UG5JrsIms+UMU3LBTQ0taZSjXZvYHBEPNvOYXZYknYAfkl2u+x+wGFp9DLrgJxkF2FpWpj+wGsRcXVa1jzR9ouIKbWLsmOStBNwA7C+B9zu2FwuWERJ2pusRjgN+L2yWVCbBiuJktKBE2wOIuI2YEkn2I7P/WQXQZK2Ibu54JCIeETSA8B9kuZGxHnzKyVY9UXEjFrHYPlzkl0ENI1XWjJu6bpko2ttIen5iHhG0hDgSUmzIuKimgZs1oG4JtvBNRsQehAwOSI+lLQH2S2d1wH3p2Vrks124PFKzarELdkOriTBfh84GJiY7kAaRnaH0e5AV0n3eCwCs+rzha8OSlKvkudbkk0rvQdwAPAS8ADZrKdPkg3C7Z80Zjlwku2AJK0G/ETS19Ki94GHI+I/wKyI+AHZiFrDIuI84IyI+Lg20Zp1bE6yHdMSwFxgN0nrAe8B20vaqWTyvbfIZjxourfezHLgC18diKQlI+KD9HxtsllNuwO/BlYHbiSbUbYT8B3gu77IZZYvt2Q7CElDyQYfOT+VCd4DLiIbaf9YsjrstmS1117AcCdYs/y5JdtBpLLAo8BnwKlkifUXwBrAVLL5os7zMHpm7ctduDqIiJggaQOySfg+JJsiZmtgQ7Ia7XpAg6STyC5++exq1g7cku1gUqngn2RTmFyexiAYTJZ0b3ZfWLP25STbAaVEezdwWkT8odbxmC3KXC7ogCLi8XQh7HFJn0bEX2odk9miyi3ZDkzS+sCMiHih1rGYLaqcZM3McuR+smZmOXKSNTPLkZOsmVmOnGTNzHLkJGtmliMnWcuVpDmSJkh6WtLfJfVYiH0NkXRber6zpJPLbLtkmg2itcc4U9IJlS5vts3laVqfSo+1iqSnWxujFYuTrOXtk4hYLyLWIRu85ojSlcq0+u9hRNwSESPKbLIk0Ooka1ZtTrLWnh4AVk8tuOck/QEYD6woaTtJj0gan1q8PQEkfUvS85IeJJuPjLT8QEkXpuf9Jd0o6Yn02AwYAayWWtG/Stv9WNLjkp6UdFbJvk6T9IKkfwKDWvoQkg5N+3lC0vXNWudDJT0g6UVJO6XtO0n6VcmxD1/YL9KKw0nW2oWkzsAOwFNp0SDgiohYH5gOnA4MjYgNgLHA8ZIWAy4hm/RxS+ArC9j9BcB9ETEY2AB4BjgZeDm1on8saTtgALAx2YhkG0raStKGZIObr0+WxL823yN82Q0R8bV0vOeAQ0rWrQJ8A/g2cHH6DIcA0yLia2n/h0r6agXHsQ7AYxdY3rpLmpCePwBcCiwHvBoRj6blmwJrAQ9JgmwW3UfIxsJ9JSImAki6imxCyOa+CewPEBFzgGmSlmq2zXbp8e/0uidZ0u0F3BgRM9IxbqngM60j6X/IShI9gbtK1l0bEXPJZgWelD7DdsC6JfXaJdKxPWj6IsBJ1vL2SUSsV7ogJdLppYuAeyLie822W4/qzaIr4NyI+FOzY/ywDce4HNg1Ip6QdCAwpGRd831FOvbREVGajJG0SiuPawXkcoHVg0eBzSWtDiCph6SBwPPAV9PsuwDfW8D7RwNHpvd2ktQb+IisldrkLuDgklrv8pL6AfeTTTjZPU2jPqyCeHsBkyV1AYY3W7enpIYU86rAC+nYR6btkTRQ0uIVHMc6ALdkreYiYmpqEV4tqVtafHpEvCjpMOB2SY3Ag8A689nFscBISYcAc4AjI+IRSQ+lLlL/SHXZNYFHUkv6Y2DfiBgv6RpgAvAqWUmjJT8BxqTtn+LLyfwFstkp+gNHRMSnkv5MVqsdr+zgU4FdK/t2rOg8CpeZWY5cLjAzy5GTrJlZjpxkzcxy5CRrZpYjJ1kzsxw5yZqZ5chJ1swsR/8P1MRw989yZnIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_confusion_matrix(confusion, classes=['Non Diabetic', 'Diabetic'], title='Confusion matrix')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# True Positives\n",
    "TP = confusion[1, 1]\n",
    "\n",
    "# True Negatives\n",
    "TN = confusion[0, 0] \n",
    "\n",
    "# False Positives\n",
    "FP = confusion[0, 1] \n",
    "\n",
    "# False Negatives\n",
    "FN = confusion[1, 0] "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### 03. Metrics computed from the Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import recall_score, precision_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Classification accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7955801104972375\n",
      "0.7955801104972375\n"
     ]
    }
   ],
   "source": [
    "print((TP + TN) / float(TP + TN + FP + FN))\n",
    "print(accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sensitivity/Recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5806451612903226\n",
      "0.5806451612903226\n"
     ]
    }
   ],
   "source": [
    "print(TP / float(TP + FN))\n",
    "print(recall_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Specificity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.907563025210084\n"
     ]
    }
   ],
   "source": [
    "print(TN / float(TN + FP))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### False Positive Rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.09243697478991597\n"
     ]
    }
   ],
   "source": [
    "print(FP / float(TN + FP))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7659574468085106\n",
      "0.7659574468085106\n"
     ]
    }
   ],
   "source": [
    "print(TP / float(TP + FP))\n",
    "print(precision_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adjusting Classification Threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 1, 1, 0, 0, 0, 0, 0], dtype=int64)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# print the first 10 predicted responses\n",
    "logreg.predict(X_test)[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.80085377, 0.19914623],\n",
       "       [0.70450063, 0.29549937],\n",
       "       [0.81765257, 0.18234743],\n",
       "       [0.45582948, 0.54417052],\n",
       "       [0.32622925, 0.67377075],\n",
       "       [0.79674719, 0.20325281],\n",
       "       [0.74752722, 0.25247278],\n",
       "       [0.64438483, 0.35561517],\n",
       "       [0.62830673, 0.37169327],\n",
       "       [0.69703945, 0.30296055]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# print the first 10 predicted probabilities of class membership\n",
    "logreg.predict_proba(X_test)[0:10, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# store the predicted probabilities for class 1 (diabetic)\n",
    "y_pred_prob = logreg.predict_proba(X_test)[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Frequency')"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEWCAYAAABi5jCmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAHlhJREFUeJzt3XmcXFWd9/HPNwlLgEDABMSwNIYAAo8ChkXRERF5IcjmsA4KURQRZxAXhoiOwqPOxEEH9dFHiIAgKARQIAMqYiQEGLYQtrBJDAEiCAGBBBAQ+M0f53RSt1LVfbvTVbe6832/XvXqu5x77++eqq5fnXPrnlJEYGZm1m1Y1QGYmVlncWIwM7MCJwYzMytwYjAzswInBjMzK3BiMDOzAieGQUjSvZJ2qzqOKkk6UNJjkl6QtH0Fx58p6ZN5+ghJv2vDMbskhaQRrT5WPl5I2ryf2y6QtEeTde+V9GCjspJOlnRWD/ttS12v7JwYOkyjfyhJkyTd0D0fEdtExMxe9tPWN5EKfAf454hYKyLuqDKQiPh5ROzZWzlJp0i6oB0xdbKIuD4itmyy7t8jojvhLvcaLlvXtmKcGKxfOiDhbArcOxA76oBzabuV8ZytPCeGQaiu6b2TpNmSFkt6UtJ/5WKz8t/ncnfLuyQNk/RVSY9IekrSzyStU7PfI/O6ZyT9W91xTpF0qaQLJC0GJuVj3yTpOUlPSPqhpFVr9heSjpP0kKQlkr4haXzeZrGki2vL151jw1glrSbpBWA4cJekPzXZPiQdL2m+pKclnSZpWF43SdKNkk6X9FfglLz8E5Lul/SspKslbVqzvw9KekDS85J+CKhmXaFFJ2kbSddI+mt+Tk6WtBdwMnBofj7uymXXkXR2rr8/S/qmpOF53XBJ38nxzwf2KfG6+LKk+/I5/FTS6nndbpIWSjpJ0l+An+bln5I0L8c6XdJb6na7d5M6HC/pD/m18rSkn0saXbftjj3F0uQcaltVjV7D9XW9VU1dPyjpkJp1e+fjL8l1+6We6s9qRIQfHfQAFgB71C2bBNzQqAxwE/CxPL0WsEue7gICGFGz3SeAecBbc9lfAefndVsDLwDvAVYlddX8veY4p+T5A0gfKEYC7wR2AUbk490PnFBzvACmA2sD2wCvADPy8dcB7gOOalIPTWOt2ffmPdRjANcC6wGbAH8EPllTn68B/5JjH5nPax7wtrzsq8D/5PJjgMXAQcAqwOfz9p+sf36AUcATwBeB1fP8zjV1eEFdnJcDZwJrAusDtwKfzuuOBR4ANs7ncW39c9rgtTO3pvyNwDfzut1yzN8GVsvnvDvwNLBDXvb/gFkl63Bz4IN5u7GkN/Hv9SGWhU1ez0vriMav4dq6XhN4DPh4fs52yOezTV7/BPDePL0usEPV/9+D5VF5AH7UPSHpn+QF4Lmax0s0TwyzgFOBMXX7afRPNQM4rmZ+S9Kb/Qjga8CFNevWAF6t+4ed1UvsJwCX1cwHsGvN/O3ASTXz3619M6nbV9NYa/bdW2LYq2b+OGBGnp4EPFpX/jfA0TXzw3K9bwocCdxcs07AQhonhsOBO5rEtPRNL89vQEqWI2uWHQ5cm6f/ABxbs27P+ue0wWuntvzewJ/y9G75+Vy9Zv3ZwH/WzK+V67irtzpscOwDas+7RCwDkRgOBa6vi+NM4Ot5+lHg08Da7fr/HSoPdyV1pgMiYnT3g/QP2czRwBbAA5Juk/ThHsq+BXikZv4RUlLYIK97rHtFRLwEPFO3/WO1M5K2kHSlpL/k7qV/J326rvVkzfTfGsyv1Y9Yy6qN95G8z0brICWA7+duseeAv5ISwDiWr5tosH23jYGG3VsNbEpqgTxRc9wzSS0H6o9LsT6a6emcF0XEyzXzhTqOiBdIz/m43vYnaX1JF+UumsXABSz/3PcUy0DYFNi5u+5y/R0BvDmv/0dSQnpE0nWS3jXAxx+ynBgGuYh4KCIOJ72ZfBu4VNKapE9a9R4n/TN124TUvfAkqdm9UfcKSSOBN9Ufrm7+x6SujgkRsTapD10MjJ5iLWvjuu0fr5mvP5fHSF04o2seIyPif0h1s3RfklS37/r9jG+yrtExXyG19rqPuXZEbJPXF46bz6E3fTnnQh3n182bgD+X2N9/5P29PT/3H2X5576nWMrobejnx4Dr6p6ztSLiMwARcVtE7E/637gcuLiPx19pOTEMcpI+KmlsRLxB6nYCeB1YBLxB6qPvdiHweUmbSVqL9Al/WkS8BlwK7Cvp3UoXhE+l9zf5UaS+9xckbQV8ZsBOrOdYyzpR0rqSNgY+B0zroewZwJclbQNLLwofnNddBWwj6SNK3+Y5nmWfSutdCbxZ0glKF8pHSdo5r3sS6Oq+gBsRTwC/A74raW2lC+7jJb0vl78YOF7SRpLWBSaXOOfP5vLrkRJ1T+f8C+DjkraTtBqpjm+JiAU1ZZrV4Shyl6ekccCJKxhLI41ew7WuBLaQ9DFJq+THjpLeJmlVpXse1omIv5Nep6/38fgrLSeGwW8v4F6lb+p8HzgsIl7OXUHfAm7MzexdgHOA80nXJR4GXiZdgCUi7s3TF5E+qS4BniJ9om3mS8A/5bI/oe//+D1pGmsfXEG6rnEn6c397GYFI+IyUovrotw1Mhf4UF73NHAwMIXU1TKBdDG10X6WkC7K7gv8BXgIeH9efUn++4ykOXn6SNLF/vuAZ0kJesO87ifA1cBdwBzSBfje/IKUbObnxzd7OOcZwL8BvyQ95+OBw+qKNavDU0kXe5/PyxvFVjqWJvE1eg3Xrl9Cuu5yGKk18heWXVwH+BiwID+fx5JaNVaC8kUas4L8Kf05UjfRw1XH01eSghT7vKpjaRdJC0gXxH9fdSw2uLnFYEtJ2lfSGrmv+TvAPaRvjJjZSsSJwWrtT2qSP07qLjks3KQ0W+m4K8nMzArcYjAzs4JBMZDWmDFjoqurq+owzMwGldtvv/3piBjb1+0GRWLo6upi9uzZVYdhZjaoSCpzt/xy3JVkZmYFLW0x5O9VLyHdcfhaREzMd0FOIw2QtQA4JCKebWUcZmZWXjtaDO+PiO0iYmKen0waoXECaQTNMrf5m5lZm1TRlbQ/cF6ePo80XK+ZmXWIVieGAH4n6XZJx+RlG+TBw7oHEVu/0YaSjlH6ZbLZixYtanGYZmbWrdXfSto1Ih6XtD5wjaQHym4YEVOBqQATJ070XXhmZm3S0hZDRDye/z4FXAbsBDwpaUOA/PepVsZgZmZ907LEIGlNSaO6p0nD484l/QbwUbnYUaRhfc3MrEO0sitpA+Cy9GNXjAB+ERG/lXQbcLGko0m/yXpwD/swM7M2a1liiIj5wDsaLH8G+ECrjjtYdU2+quoQlrNgyj5Vh2BmFfCdz2ZmVuDEYGZmBU4MZmZW4MRgZmYFTgxmZlbgxGBmZgVODGZmVuDEYGZmBU4MZmZW4MRgZmYFTgxmZlbgxGBmZgVODGZmVuDEYGZmBU4MZmZW4MRgZmYFTgxmZlbgxGBmZgVODGZmVuDEYGZmBU4MZmZW4MRgZmYFTgxmZlbgxGBmZgVODGZmVuDEYGZmBU4MZmZW4MRgZmYFTgxmZlbgxGBmZgVODGZmVuDEYGZmBU4MZmZW4MRgZmYFLU8MkoZLukPSlXl+M0m3SHpI0jRJq7Y6BjMzK68dLYbPAffXzH8bOD0iJgDPAke3IQYzMyuppYlB0kbAPsBZeV7A7sCluch5wAGtjMHMzPqm1S2G7wH/CryR598EPBcRr+X5hcC4RhtKOkbSbEmzFy1a1OIwzcysW8sSg6QPA09FxO21ixsUjUbbR8TUiJgYERPHjh3bkhjNzGx5I1q4712B/STtDawOrE1qQYyWNCK3GjYCHm9hDGZm1kctazFExJcjYqOI6AIOA/4QEUcA1wIH5WJHAVe0KgYzM+u7Ku5jOAn4gqR5pGsOZ1cQg5mZNdHKrqSlImImMDNPzwd2asdxzcys73zns5mZFTgxmJlZgRODmZkVODGYmVmBE4OZmRU4MZiZWYETg5mZFTgxmJlZgRODmZkVODGYmVmBE4OZmRU4MZiZWYETg5mZFTgxmJlZgRODmZkVtOX3GDpN1+Srqg7BzKxjucVgZmYFTgxmZlbgxGBmZgVODGZmVuDEYGZmBU4MZmZW4MRgZmYFTgxmZlbgxGBmZgVODGZmVuDEYGZmBU4MZmZW4MRgZmYFTgxmZlZQKjFI2rbVgZiZWWco22I4Q9Ktko6TNLqlEZmZWaVK/VBPRLxH0gTgE8BsSbcCP42Ia1oanVWqE3/QaMGUfaoOwWzIK32NISIeAr4KnAS8D/iBpAckfaRReUmr51bGXZLulXRqXr6ZpFskPSRpmqRVB+JEzMxsYJS9xvB2SacD9wO7A/tGxNvy9OlNNnsF2D0i3gFsB+wlaRfg28DpETEBeBY4egXPwczMBlDZFsMPgTnAOyLisxExByAiHie1IpYTyQt5dpX8CFIyuTQvPw84oJ+xm5lZC5S6xgDsDfwtIl4HkDQMWD0iXoqI85ttJGk4cDuwOfAj4E/AcxHxWi6yEBjXZNtjgGMANtlkk5JhmpnZiirbYvg9MLJmfo28rEcR8XpEbAdsBOwEvK1RsSbbTo2IiRExcezYsSXDNDOzFVU2Maxe0y1Enl6j7EEi4jlgJrALMFpSd0tlI+DxsvsxM7PWK5sYXpS0Q/eMpHcCf+tpA0lju+95kDQS2IN08fpa4KBc7Cjgir4GbWZmrVP2GsMJwCWSuj/dbwgc2ss2GwLn5esMw4CLI+JKSfcBF0n6JnAHcHY/4jYzsxYpe4PbbZK2ArYEBDwQEX/vZZu7ge0bLJ9Put5gZmYdqGyLAWBHoCtvs70kIuJnLYnKzMwqUyoxSDofGA/cCbyeFwfgxGBmNsSUbTFMBLaOiIZfLTUzs6Gj7LeS5gJvbmUgZmbWGcq2GMYA9+VRVV/pXhgR+7UkKjMzq0zZxHBKK4MwM7POUfbrqtdJ2hSYEBG/l7QGMLy1oZmZWRXKDrv9KdKIqGfmReOAy1sVlJmZVafsxefPArsCi2Hpj/as36qgzMysOmUTwysR8Wr3TB4Ez19dNTMbgsomhusknQyMlPRB4BLgv1sXlpmZVaVsYpgMLALuAT4N/Jomv9xmZmaDW9lvJb0B/CQ/zMxsCCs7VtLDNLimEBFvHfCIzMysUn0ZK6nb6sDBwHoDH46ZmVWt1DWGiHim5vHniPgesHuLYzMzswqU7UraoWZ2GKkFMaolEZmZWaXKdiV9t2b6NWABcMiAR2NmZpUr+62k97c6EDMz6wxlu5K+0NP6iPivgQnHzMyq1pdvJe0ITM/z+wKzgMdaEZSZmVWnLz/Us0NELAGQdApwSUR8slWBmZlZNcoOibEJ8GrN/KtA14BHY2ZmlSvbYjgfuFXSZaQ7oA8EftayqMzMrDJlv5X0LUm/Ad6bF308Iu5oXVhmZlaVsl1JAGsAiyPi+8BCSZu1KCYzM6tQ2Z/2/DpwEvDlvGgV4IJWBWVmZtUp22I4ENgPeBEgIh7HQ2KYmQ1JZRPDqxER5KG3Ja3ZupDMzKxKZRPDxZLOBEZL+hTwe/yjPWZmQ1LZbyV9J//W82JgS+BrEXFNSyMzM7NK9JoYJA0Hro6IPQAnAzOzIa7XrqSIeB14SdI6bYjHzMwqVvbO55eBeyRdQ/5mEkBEHN9sA0kbk+6OfjPwBjA1Ir4vaT1gGmlIjQXAIRHxbL+iNzOzAVc2MVyVH33xGvDFiJgjaRRwe04sk4AZETFF0mRgMukeCTMz6wA9JgZJm0TEoxFxXl93HBFPAE/k6SWS7gfGAfsDu+Vi5wEzcWIwM+sYvV1juLx7QtIv+3sQSV3A9sAtwAY5aXQnj/WbbHOMpNmSZi9atKi/hzYzsz7qLTGoZvqt/TmApLWAXwInRMTisttFxNSImBgRE8eOHdufQ5uZWT/0lhiiyXQpklYhJYWfR8Sv8uInJW2Y128IPNXX/ZqZWev0lhjeIWmxpCXA2/P0YklLJPX46V+SgLOB++t+E3o6cFSePgq4or/Bm5nZwOvx4nNEDF+Bfe8KfIz0Ndc787KTgSmkITaOBh4FDl6BY5iZ2QAr+3XVPouIGyheo6j1gVYd14a2rsl9/dZ06y2Ysk/VIZgNqL78UI+Zma0EnBjMzKzAicHMzApado3BbGXRadc9fM3DVpRbDGZmVuDEYGZmBU4MZmZW4MRgZmYFTgxmZlbgxGBmZgVODGZmVuDEYGZmBU4MZmZW4MRgZmYFTgxmZlbgxGBmZgVODGZmVuDEYGZmBU4MZmZW4MRgZmYF/qEesyGm0344CPzjQYONWwxmZlbgxGBmZgVODGZmVuDEYGZmBU4MZmZW4MRgZmYFTgxmZlbgxGBmZgVODGZmVuDEYGZmBU4MZmZW4MRgZmYFLUsMks6R9JSkuTXL1pN0jaSH8t91W3V8MzPrn1a2GM4F9qpbNhmYERETgBl53szMOkjLEkNEzAL+Wrd4f+C8PH0ecECrjm9mZv3T7msMG0TEEwD57/ptPr6ZmfWiYy8+SzpG0mxJsxctWlR1OGZmK412J4YnJW0IkP8+1axgREyNiIkRMXHs2LFtC9DMbGXX7sQwHTgqTx8FXNHm45uZWS9a+XXVC4GbgC0lLZR0NDAF+KCkh4AP5nkzM+sgI1q144g4vMmqD7TqmGZmtuI69uKzmZlVw4nBzMwKnBjMzKygZdcYzMw6Wdfkq6oOYTkLpuxTdQiAWwxmZlbHicHMzAqcGMzMrMDXGMys5TqxP9+ac4vBzMwKnBjMzKzAicHMzAqcGMzMrMCJwczMCpwYzMyswInBzMwKnBjMzKzAicHMzAqcGMzMrMCJwczMCpwYzMyswInBzMwKnBjMzKzAicHMzAqcGMzMrMCJwczMCpwYzMyswInBzMwKnBjMzKzAicHMzAqcGMzMrMCJwczMCpwYzMyswInBzMwKnBjMzKygksQgaS9JD0qaJ2lyFTGYmVljbU8MkoYDPwI+BGwNHC5p63bHYWZmjVXRYtgJmBcR8yPiVeAiYP8K4jAzswZGVHDMccBjNfMLgZ3rC0k6Bjgmz74iaW4bYhsMxgBPVx1Eh3BdLOO6WGbQ1oW+PeC73LI/G1WRGNRgWSy3IGIqMBVA0uyImNjqwAYD18UyrotlXBfLuC6WkTS7P9tV0ZW0ENi4Zn4j4PEK4jAzswaqSAy3ARMkbSZpVeAwYHoFcZiZWQNt70qKiNck/TNwNTAcOCci7u1ls6mtj2zQcF0s47pYxnWxjOtimX7VhSKW6943M7OVmO98NjOzAicGMzMr6KjE0NtQGZJWkzQtr79FUlf7o2y9EvXwBUn3Sbpb0gxJm1YRZzuUHT5F0kGSQtKQ/ZpimbqQdEh+bdwr6RftjrFdSvyPbCLpWkl35P+TvauIsx0knSPpqWb3ein5Qa6ruyXt0OtOI6IjHqQL0X8C3gqsCtwFbF1X5jjgjDx9GDCt6rgrqof3A2vk6c8MxXooWxe53ChgFnAzMLHquCt8XUwA7gDWzfPrVx13hXUxFfhMnt4aWFB13C2sj38AdgDmNlm/N/Ab0j1kuwC39LbPTmoxlBkqY3/gvDx9KfABSY1umBvMeq2HiLg2Il7KszeT7gUZisoOn/IN4D+Bl9sZXJuVqYtPAT+KiGcBIuKpNsfYLmXqIoC18/Q6DOF7pSJiFvDXHorsD/wskpuB0ZI27GmfnZQYGg2VMa5ZmYh4DXgeeFNbomufMvVQ62jSp4GhqNe6kLQ9sHFEXNnOwCpQ5nWxBbCFpBsl3Sxpr7ZF115l6uIU4KOSFgK/Bv6lPaF1pL6+p1QyJEYzZYbKKDWcxiBX+hwlfRSYCLyvpRFVp8e6kDQMOB2Y1K6AKlTmdTGC1J20G6kVeb2kbSPiuRbH1m5l6uJw4NyI+K6kdwHn57p4o/XhdZw+v292UouhzFAZS8tIGkFqIvbUhBqMSg0ZImkP4CvAfhHxSptia7fe6mIUsC0wU9ICUv/p9CF6Abrs/8cVEfH3iHgYeJCUKIaaMnVxNHAxQETcBKxOGlxvZdTnYYg6KTGUGSpjOnBUnj4I+EPkqytDSK/1kLtPziQlhaHajwy91EVEPB8RYyKiKyK6SNdb9ouIfg0c1uHK/H9cTvpiApLGkLqW5rc1yvYoUxePAh8AkPQ2UmJY1NYoO8d04Mj87aRdgOcj4omeNuiYrqRoMlSGpP8LzI6I6cDZpCbhPFJL4bDqIm6NkvVwGrAWcEm+9v5oROxXWdAtUrIuVgol6+JqYE9J9wGvAydGxDPVRd0aJevii8BPJH2e1G0yaQh+iARA0oWk7sMx+ZrK14FVACLiDNI1lr2BecBLwMd73ecQrSszM+unTupKMjOzDuDEYGZmBU4MZmZW4MRgZmYFTgxmZlbgxLCSk/S6pDslzZV0iaQ1VmBfu0m6Mk/v18toqKMlHdePY5wi6Uv9jbGH/S6NvQ/bLMj3C9QvP1bSkXn6XEkH5emzJG2dp08eiLjzvo6XdL+kn/dSbmb3zX+Sfi1pdC/lX+hjHAd0n58Nbk4M9reI2C4itgVeBY6tXZlviunz6yQipkfElB6KjCaNlts2+W75louIMyLiZw2WfzIi7suzA5YYSPW4d0QcUXaDiNi7BUNlHEAaydQGOScGq3U9sLmkrvwJ9P8Dc4CNJe0p6SZJc3LLYi1YOi7+A5JuAD7SvSNJkyT9ME9vIOkySXflx7uBKcD43Fo5LZc7UdJtecz4U2v29RWlsfd/D2zZKPD8yfwMSddL+qOkD9fEcYmk/wZ+lxPdabmFdI+kQ2t2s3aO8768r2F5Hz+WNFvpNw5OrTv0iZJuzY/Nc/mGrZruT+ySpgAj87n/XNI3JH2upty3JB3fYPsv5LjnSjohLzuDNPz09HwzV235kZIuyvU5DRhZs25pa0fS5ZJuz+d3TN0+vpuf8xmSxuZl4yX9Nm9zvaSt8nO6H3BaPq/xjcrl7Q/O53CXpFmNnk+rWNVjiftR7QN4If8dAVxB+n2HLuANYJe8bgzp9w7WzPMnAV8jDTPwGGk8HpHGprkyl5kE/DBPTwNOyNPDSWNcdVEzfjywJ2kMfZE+sFxJGmf+ncA9wBqkYZTnAV9qcB7nAr/N204gjQ+zeo5jIbBeLvePwDU5jg1IQydsSLpz9GXSm+zwXOagvM16NbHPBN6e5xcAX8nTR9ac+yndMea4uvczk/x7Ed31nqe7gDl5ehjptwbeVHd+3fWwJumu93uB7WviGNOgTr5AuisY4O3AazXHX7pNzfmNBOZ2H5t0x/ARefprNc/nDGBCnt6ZNDRN4Vx7KXcPMC5Pj676f8CP5R8dMySGVWakpDvz9PWkYUfeAjwSaex2SIPTbQ3cqDQEx6rATcBWwMMR8RCApAuAwifObHfSGycR8TrwvKR168rsmR935Pm1SG/wo4DLIv/+hKSehsG4ONLomQ9Jmp/jA7gmIroHW3wPcGGO40lJ1wE7AouBWyNifj7OhbnspcAh+ZP0CFIS2Rq4O+/vwpq/p/cQW1MRsUDSM0pjYG0A3BHLD2XxHlI9vJjj+xXwXpbVVyP/APwgH+NuSXc3KXe8pAPz9Maken+G9OFgWl5+AfCr3FJ8N8uGYwFYrX6HvZS7EThX0sXAr3qI3yrixGB/i4jtahfkf+QXaxeR3lwPryu3HQM37LmA/4iIM+uOcUIfjlFfrnu+/lxKby9pM+BLwI4R8aykc0ktkUbbrEhdnEVq3bwZOKfB+v7+IFWPMUnaDdgDeFdEvCRpJsXzq9/XMOC5+tdMA03LRcSxknYG9gHulLRdg0RoFfI1BivjZmDXmj70NSRtATwAbCZpfC53eJPtZ5C6qJA0XNLawBJSa6Db1cAntOzaxThJ65O6sA7M/eWjgH17iPNgScNyPG8lDTtdbxZwaI5jLOlT9a153U5KI3YOAw4FbiB1X71IauVsAHyobn+H1vy9qYfY6v1d0io185cBe5FaL1c3ifuAXPdrAgeSWng9mQUcASBpW1J3Ur11gGdzUtiK1DrsNow0ijHAPwE3RMRi4GFJB+f9StI7cpmlz2lP5SSNj4hbIuJrwNMUh4S2DuAWg/UqIhZJmgRcKKm7O+CrEfHH3MVylaSnSW+k2zbYxeeAqZKOJo36+ZmIuEnpl8bmAr+JiBOVhke+KbdYXgA+GhFz8oXTO4FH6PnN8EHgOlJ3zLER8bKW/+XXy4B3kX4nOIB/jYi/5DfFm0gXxf8P6U31soh4Q9IdpD79+aRukFqrSbqF9CbaLDE2MhW4W9KciDgiIl6VdC3pU/br9YVzPZzLsiR2VkT01I0E8GPgp7kL6c6abWv9Fjg2l3mQ9CGg24vANpJuJ/1aYncSPAL4saSvkkbxvIhUnxeRRjQ9npRQmpU7TVL3dakZeZl1EI+uakNCftO8MiIurTqW/sitlDnAwd3XbMyq4q4ks4op3RQ2D5jhpGCdwC0GMzMrcIvBzMwKnBjMzKzAicHMzAqcGMzMrMCJwczMCv4XOjLxFR3UjyEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# histogram of predicted probabilities\n",
    "plt.hist(y_pred_prob, bins=8, linewidth=1.2)\n",
    "plt.xlim(0, 1)\n",
    "plt.title('Histogram of predicted probabilities')\n",
    "plt.xlabel('Predicted probability of diabetes')\n",
    "plt.ylabel('Frequency')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Decrease the threshold for predicting diabetes in order to increase the sensitivity of the classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# predict diabetes if the predicted probability is greater than 0.3\n",
    "from sklearn.preprocessing import binarize\n",
    "\n",
    "y_pred_class = binarize([y_pred_prob], 0.3)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[108  11]\n",
      " [ 26  36]]\n"
     ]
    }
   ],
   "source": [
    "# previous confusion matrix (default threshold of 0.5)\n",
    "print(confusion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[82 37]\n",
      " [ 8 54]]\n"
     ]
    }
   ],
   "source": [
    "# new confusion matrix (threshold of 0.3)\n",
    "confusion_new = confusion_matrix(y_test, y_pred_class)\n",
    "print(confusion_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# True Positives\n",
    "TP = confusion_new[1, 1]\n",
    "\n",
    "# True Negatives\n",
    "TN = confusion_new[0, 0] \n",
    "\n",
    "# False Positives\n",
    "FP = confusion_new[0, 1] \n",
    "\n",
    "# False Negatives\n",
    "FN = confusion_new[1, 0] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8709677419354839\n",
      "0.8709677419354839\n"
     ]
    }
   ],
   "source": [
    "# sensitivity has increased\n",
    "print(TP / float(TP + FN))\n",
    "print(recall_score(y_test, y_pred_class))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6890756302521008\n"
     ]
    }
   ],
   "source": [
    "# specificity has decreased\n",
    "print(TN / float(TN + FP))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 04. ROC Curves and Area Under the Curve (AUC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve, roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3XmcXFWd9/HPN2EVCIiJGhJCWIOBYZNVGW0FFBkFh0EICoqiUWfQB7cHRnwcZFyZwQVBJSoE0AjiRoQoIkODMgkB2STRSIBIQqKsAYIBSfJ7/jin6ZtK1e3bTd/uSvf3/XrVK3XX+tWpdP3qnHPvOYoIzMzMWhkx2AGYmVl7c6IwM7NSThRmZlbKicLMzEo5UZiZWSknCjMzK+VEYUOOkoskPS5pbk2vsUjSofn5JyV9p+Jx0yV9to6Y6iCpU9J7azr3BEkrJI3Myy+TdKOkpySd05tytXptMNgBWP+QtAh4GbAaWAH8EjglIlYU9nkV8FlgP2ANcCNwWkTML+wzCjgLOBrYGvgLcBXw2Yh4ZEDezAt3MHAYMD4inq77xSLi83W/BqQkAyyJiE8NxOvVLSIeADYvrJoKPAKMCt/g1VZcoxha3hIRmwN7AXsD/961QdJBwK+AK4FtgO2BO4GbJO2Q99kIuA7YDTgcGAW8CngU2L+uoCX19w+W7YBFfUkSNcRi1W0HzH+hSSLXKP3d1p8iwo8h8AAWAYcWls8Gri4s/wb4RpPjfgFckp+/F/grsHkvXnc34FrgsXzsJ/P66aRaSNd+HaRfw8V4TwPuAp4FPgX8qOHcXwPOzc+3BL4LLAMeJNWMRjaJ52TgGbprVp/J698HLMxxzgS2KRwTwL8B9wD3t3ifJwJ/JiXNM4rlDZwJfK+w7xWkmtgTpFrbboVt04Fv5TJ7CrgB2K6wfddCeS4Ajs3rpwLPAX/P7+vnef02wI+Bh4H7gQ8XzrU/cCvwZP5svlzyOR4F3JH3vRc4PK/vBN6bn+8I/E8ug0eA7wNbFc5xWv5snsqxH1IWBzAxl/0GuVyK7+/QJuV6IPC/wHLSj5yOwrZO4HPATcBKYKfB/pscSo9BD8CPfvog1/7iGg/8HvhaXn4R6YvzdU2OezewLD+/DLi4F6+5BemL+2PAJnn5gLxtOj0nijuAbYFNSb8m/0ZqdgAYmc99YF7+GXABsBnwUmAu8P4WcZ0E/Law/Pr8xbYPsDHwdeDGwvYgfTlvDWza5HyT85fXa/LxXwZW0TpRvCeXxcbAV4E7Ctum5y/SrnN9rSvW/N4W589kgxzvI+RE06RMRwC/Az4NbATsANwHvDFvnw2cmJ9v3lWWTd7f/qSkdlg+5zhg17ytk+5EsVPeZ2NgDCkJfjVvm5Rj3yYvTwR2LIuDQqJo8f6eL9cc06PAETnGw/LymEKcD5B+uGwAbDjYf5ND6eHq2dDyM0lPkf5gHwL+I6/fmvTHtazJMcuA0fn5S1rs08qbgb9ExDkR8UxEPBURN/fi+HMjYnFErIyIPwO3AW/N214P/C0i5kh6GfAm4NSIeDoiHgK+Akyp+DrvAC6MiNsi4llSk9xBkiYW9vlCRDwWESubHH8McFVE3JiP/3+kPp6mIuLCXBbPkr7s9pS0ZWGXqwvnOiPHsi2pPBdFxEURsSoibiPVFo5p8VL7kb4oz4qIv0fEfcC36S6X54CdJI2OiBURMafFeU7O5XNtRKyJiAcj4o9N3tfCvM+zEfEwKWG+Nm9eTUogkyVtGBGLIuLeXsZR5gRgVkTMyjFeS6qlHFHYZ3pEzMtl91wfXsNacKIYWt4aEVuQfr3vSncCeJz0xTa2yTFjSb9aIf1Ca7ZPK9uSmin6anHD8gzg+Pz87XkZUm1jQ2CZpOWSlpNqFy+t+DrbkJqNAIjUwf8o6Vdqq1gaj39+e6S+j0eb7ShppKQvSrpX0pOkmhN0fxZrvVaO5bH8GtsBB3S9x/w+3wG8vEVc2wHbNOz/SdJFDZASwC7AHyXdIunNLc5T6XOU9FJJl0l6ML+373W9r4hYCJxKSowP5f226WUcZbYD3tbwXg9m7f+vZZ+hvQBOFENQRNxAqsb/d15+mlT9f1uT3Y8ldWAD/Bp4o6TNKr7UYlK7dTNPk5q8ujT7smvstLwC6JA0HvhnuhPFYlI/xuiI2Co/RkXEbhXjXEr6ogEgv7+XkNrTW8VStIz0Zdp1/Ivy8c28ndTefyipX2Vi12GFfYrn2pxU41tKep83FN7jVhGxeUR8sEWMi0l9KsX9t4iIIwAi4p6IOJ6UUL8E/KjFZ1v2ORZ9IcewR0SMIv3Kf/59RcSMiDiYVNaRX7M3cZRZDFza8F43i4gvFvbxlVI1caIYur4KHCZpr7x8OvAuSR+WtIWkF+fr+Q8CPpP3uZT0B/ljSbtKGiHpJfl69iPWfQmuAl4u6VRJG+fzHpC33QEcIWlrSS8n/doslZszOoGLSF+Af8jrl5Gu2DpH0qgc146SXtv6bGuZAbxb0l6SNgY+D9wcEYsqHv8j4M2SDs5Xhp1F67+dLUhJ7VFSomx26ewRhXP9Z45lMak8d5F0oqQN82M/Sa/Ix/2V1A/RZS7wpKTTJG2aazO7S9oPQNIJksZExBpSBzCkJqJG3yWVzyG5bMdJ2rXFe1sBLJc0DvhE1wZJkyS9PpfvM6QO5dW9jKPM94C3SHpjfp+bSOr6UWE1c6IYovKX7iWk9nQi4rfAG0n3RywjNcXsDRwcEffkfZ4l/RL+I6lz90nSl9FoYJ2+h4h4itSp+BbSVT73AK/Lmy8lXZmyiPQlf3nF0GfkGGY0rH8nqcN2Pqkp7UdUbCaLiOtI5fBj0nvfker9G0TEPNJVUTPy8Y8DS1rsfgmpbB/MsTZrj59B6j96DHglqXmpqzzfkGNbSirTL5Ha/iF9oU/OTS8/i4jVpLLfi3TF0yPAd0g1GUiXOM+TtILUaT4lIp5p8v7mkjrQv0Lq1L6BQg2s4DOkDvYngKuBnxS2bQx8McfwF1Lt4ZO9iaNMTqRH5XM+TPpB8wn8HTYgFOHampmZteZsbGZmpWpLFJIulPSQpLtbbJekcyUtlHSXpH3qisXMzPquzhrFdFLbZCtvAnbOj6nAN2uMxczM+qi2RBERN5I661o5ijR0ROQbcLaS1Jtr+M3MbAAM5gBo41j7Bpkled06dwZLmkqqdbDJJpu8csKECQMSYLtbs2YNI0a4mwnWz7L4y9Nr+Ptq2Ghk/543WPumjeHMZdHtyQcXPhIRY/py7GAmimafX9NLsCJiGjANYNKkSbFgwYI641pvdHZ20tHRMdhhtIX1sSyOu2A2AJe//6B+Pe/6WBZ1cVl0k/TnnvdqbjB/gi2hcIcqaSC7pYMUi5mZtTCYNYqZwCmSLgMOAJ7Id+CaDVkzbn6AK+9II4fMX/Ykk8eOGuSIzHpWW6KQ9APS4HSjJS0h3Ym6IUBEfAuYRRr5cSFpeOl31xWLWbu48o4Hn08Qk8eO4qi9xvV8kNkgqy1R5EHAyrZ3TRZjNqxMHjuq3/slzOrkaR/N+qDYhNQbbm6y9dH6dT2hWZvoakLqLTc32frINQqzPnITkg0XrlGYmVkp1yjMKvKlrTZcuUZhVlGxX8J9DTacuEZh1gvul7DhyInCrAddTU5ubrLhyk1PZj0oJgk3N9lw5BqFWQVucrLhzDUKMzMr5URhZmalnCjMzKyUE4WZmZVyZ7ZZE74L26ybaxRmTfgubLNurlGYteBLYs0S1yjMzKyUaxQ2bPRmVjr3S5h1c43Cho3ezErnfgmzbq5R2LDifgez3nOisD7pTTPOQFi+fCXfXDC7dB83J5n1jZuerE9604zTLtycZNY3rlFYn7VTM05nZycdHe0Ri9lQ4xqFmZmVco3C1lGl/8Ht/WbDh2sUto4q/Q9u7zcbPlyjsKbaqf/BzAaXE8Uw1FPTkpuVzKyotOlJ0n6SvibpNknLJN0naaak90vaYqCCtP7VU9OSm5XMrKhljULSVcCjwJXAOcBDwCbALsDrgKslnR0RVw1EoNa/3LRkZlWVNT2dHBF/bVj3DDA3P74k6aW1RWZmZm2hZdNTV5KQ9AFJW7bY56G6AjMzs/ZQ5fLYicBtkmZIOrTmeMzMrM30mCgi4nRgZ+D7wAck3SPpLEkTa47NzMzaQKXLYyNijaRFwCLgH4CxwJWSZkXEv7c6TtLhwNeAkcB3IuKLDdsnABcDW+V9To+IWX14H8Nab0dy9eWvZtYbPdYoJP2rpLmkL/zfAXtExPuAvYHjSo4bCZwPvAmYDBwvaXLDbp8CfhgRewNTgG/06V0Mc70dydWXv5pZb1SpUYwHpkTEfcWVuZZxZMlx+wMLu46TdBlwFDC/eBqg66ftlsDSqoHb2ny5q5nVpUqi2KYxSUiaHhEnRcTdJceNAxYXlpcABzTscybwK0kfAjYDmnaWS5oKTAUYM2YMnZ2dFcIe+lasWEFnZyfLl68EGNbl0lUW5rIocln0jyqJYo/igqQRwH4VjlOTddGwfDwwPSLOkXQQcKmk3SNizVoHRUwDpgFMmjQpOjo6Krz80Hfmpdfyhwc3ZunKZ5k8dtSwno8hzUfRMdhhtAWXRTeXRf9o2Uch6TRJjwN7SHosPx4HHgGqdDgvAbYtLI9n3aalk4EfAkTEbNKd36N7Ef+wNnvpquc7pt3nYGZ1KatRnE0auuMLwOldKyNidcVz3wLsLGl74EFSZ/XbG/Z5ADgEmC7pFaRE8XDF8xvumzCz+pUlip0i4h5JlwK7da2UUotSRNxVduKIWCXpFOAa0qWvF0bEPElnAbdGxEzgY8C3JX2E1Cx1UkQ0Nk8Ne60uf33gqTVstdUgBGRmw0pZojid1DR0fpNtAbymp5PneyJmNaz7dOH5fODVlSIdxrouf22892HCFiPc5GRmtWuZKCLi5PzvPw5cONZKsyamzs5OOg6YMEgRmdlwUeWGu9skfULSdgMRkJmZtZcqgwK+DdgQmClptqRTJbm9w8xsmKgyKOC9EfH5iNgTeA/wSuDPtUdmZmZtodKggJLGA8eSxnbaADijzqDMzKx99JgoJN0EbAFcAZwYEX+qPSpb65JYj/ZqZoOpSo3i/T2M6WQ1KF4S6zuvzWwwtUwUko6PiB8Ar5f0+sbtEXFurZGZ77o2s7ZQVqN4cf53TJNtvnvazGyYKLvhrmsSoasjYk5xm6QDa41qmHK/hJm1oyr3UTSbda7ZsB72AhVnqnO/hJm1i7I+iv2Bg4Axkj5c2DSKdAOe1cD9EmbWbsr6KDYjzQ2xAWv3UzxFulvb+klXk5Obm8ysHZX1UVwPXC/posapUK1/FZOEm5vMrN2UNT2dExEfA86RtM5VThFxdK2RDTNucjKzdlXW9HR5/ve8gQjEzMzaU1nT09z873Vd6yRtCYzLEw6ZmdkwUGU+iuskjZL0YuD3wAxJ/1V/aGZm1g6q3EexdUQ8CRwNXBwRewFvrDcsMzNrF1USxQaSxpAuif15zfGYmVmbqZIoPgfcADwQEXMl7QDcX29YZmbWLnocZjwiLgMuKyzfBxxVZ1BmZtY+qkxcNJo0BerE4v4RMbW+sMzMrF1UmbjoSmAO8Ftgdb3hmJlZu6mSKDbLd2ibmdkwVKUz+xeS3lB7JGZm1paqJIoPAL+UtELSY5Iel/RY3YGZmVl7qNL0NLr2KIaw4qx1rXh4cTNrZz3WKCJiNelmu9Py87HAXnUHNlQUZ61rxcOLm1k7q3J57HmkGe1eA3we+BvwLWC/ekMbOjyEuJmtz6o0Pb0qIvaRdDtARDwmaaOa4zIzszZRpTP7OUkjgACQ9BJgTa1RmZlZ26iSKM4HfgyMkfQZ0o13X6o1KjMzaxtVxnq6RNLvgEPzqrdFxN31hmVmZu2iZY1C0iaSRgJExDzgalKT0w5VTy7pcEkLJC2UdHqLfY6VNF/SPEkzehl/W5px8wMcd8Fsjrtgdo9XPJmZtbuypqdrgB0BJO0IzAUmAx+V9LmeTpyTzPnAm/Jxx0ua3LDPzsC/A6+OiN2AU/vyJtpN8ZJYX/pqZuu7sqanrSPiT/n5u4DLIuJfJW0M3Aqc0cO59wcW5mHJkXQZaXjy4nzb7wPOj4jHASLioT68h7bkS2LNbKgoSxRReP564ByAiHhWUpWrnsYBiwvLS4ADGvbZBUDSTcBI4MyI+GXjiSRNBaYCjBkzhs7OzgovPzA6Fz/H7KWr1lr3wFNrmLDFiNrjXLFiRVuVxWByWXRzWXRzWfSPskQxT9IXgQdJX+i/ApC0JaAK5262TzQsbwDsDHQA44HfSNo9IpavdVDENGAawKRJk6Kjo6PCyw+Mb14wm6Ur1x6CY6ut4Ki9xtFxwIRaX7uzs5N2KovB5LLo5rLo5rLoH2WJ4r3AR4BdgcMj4um8fnfgyxXOvQTYtrA8HljaZJ85EfEccL+kBaTEcUuF87cNNzOZ2VDWMlHkxPDZJutvAm6qcO5bgJ0lbU+qlUwB3t6wz8+A44HpeSa9XYD7qoVuZmYDoWWikPQz4ALg2ohY1bBtO1IH95KIuLDZ8RGxStIppKunRgIXRsQ8SWcBt0bEzLztDZLmk2bP+0REPNofb6y/tRoF1iO/mtlQV9b09G/Ax4DzJf0VeBjYhHQfxQOkq5V+XHbyiJgFzGpY9+nC8wA+mh9treuS18ak4MtfzWyoK2t6epD8JS5pJ9Lw4iuBBRHx1ADF11bcF2Fmw1GV0WOJiIXAwppjMTOzNlRlUEAzMxvGnCjMzKxUpUQhaaPcT2FmZsNMj4lC0j8Bvweuzct7Sfpp3YGZmVl7qFKjOIs0RtNygIi4A3DtwsxsmKg0FWrj2EusO2aTmZkNUVUuj/2DpGOBEXk4jv8DzKk3rPZQvBvbd2Cb2XBVpUZxCvBK0ux2PwGeISWLIc8TEJmZVatRvDEiTgNO61oh6WhS0hjyfDe2mQ13VWoUn2qyrqfZ7czMbIgoGz32jcDhwDhJxfknRpGaoczMbBgoa3p6CLib1Ccxr7D+KeD0OoMyM7P2UTZ67O3A7ZK+HxHPDGBMZmbWRqp0Zo+T9DlgMmk+CgAiYpfaojIzs7ZRpTN7OnARIOBNwA+By2qMyczM2kiVRPGiiLgGICLujYhPAa+rNywzM2sXVZqenpUk4F5JHwAeBF5ab1hmZtYuqiSKjwCbAx8GPgdsCbynzqDMzKx99JgoIuLm/PQp4EQASePrDMrMzNpHaR+FpP0kvVXS6Ly8m6RLGCaDApqZWfmd2V8A/gW4E/hUnqzo/wBfAj4wMOENPI8Ya2a2trKmp6OAPSNipaStgaV5ecHAhDY4ukaMnTx2lEeMNTOjPFE8ExErASLiMUl/HOpJootHjDUz61aWKHaQ1DWUuICJhWUi4uhaIzMzs7ZQlij+pWH5vDoDMTOz9lQ2KOB1AxmImZm1pypDeJiZ2TDmRGFmZqUqJwpJG9cZiJmZtaceE4Wk/SX9HrgnL+8p6eu1R2ZmZm2hSo3iXODNwKMAEXEnHmbczGzYqDJ67IiI+HMaafx5q2uKZ9B0Dd3hYTvMzNZWJVEslrQ/EJJGAh8C/lRvWAOvmCQ8bIeZWbcqTU8fBD4KTAD+ChyY1/VI0uGSFkhaKOn0kv2OkRSS9q1y3rp0Dd3x9gMmDGYYZmZtpUqNYlVETOntiXPt43zgMGAJcIukmRExv2G/LUiTIt287lnMzGywValR3CJplqR35S/1qvYHFkbEfRHxd+Ay0oi0jf4TOBt4phfnNjOzAVJlhrsdJb0KmAJ8RtIdwGURcVkPh44DFheWlwAHFHeQtDewbURcJenjrU4kaSowFWDMmDF0dnb2FHavLV++EqCWc9dlxYoV61W8dXJZdHNZdHNZ9I8qTU9ExP8C/yvpTOCrwPdJNYQyarIunt8ojQC+ApxU4fWnAdMAJk2aFB0dHVXC7pVvLpgNQEfH+jO8eGdnJ3WUxfrIZdHNZdHNZdE/ekwUkjYnNRlNAV4BXAm8qsK5lwDbFpbHkyY/6rIFsDvQmS+9fTkwU9KREXFrpehfIM9mZ2bWsyo1iruBnwNnR8RvenHuW4CdJW0PPEhKNG/v2hgRTwCju5YldQIfH6gkAZ7NzsysiiqJYoeIWNPbE0fEKkmnANcAI4ELI2KepLOAWyNiZm/PWQfPZmdmVq5lopB0TkR8DPixpGjcXmWGu4iYBcxqWPfpFvt29BitmZkNuLIaxeX5X89sZ2Y2jJXNcDc3P31FRKyVLHKTkmfAMzMbBqrccPeeJutO7u9AzMysPZX1URxHulJpe0k/KWzaAlhed2BmZtYeyvoo5pLmoBhPGrOpy1PA7XUGZWZm7aOsj+J+4H7g1wMXjpmZtZuypqcbIuK1kh6nMPQGaWiOiIita4+uBr4b28ysd8qanrqmOx1dss96x3djm5n1TlnTU9fd2NsCSyPi75IOBvYAvgc8OQDx1cJ3Y5uZVVfl8tifkaZB3RG4hDQw4IxaozIzs7ZRJVGsiYjngKOBr0bEh0hzTZiZ2TBQJVGskvQ24ETgqrxuw/pCMjOzdlL1zuzXkYYZvy8PG/6DesMyM7N2UWUq1LslfRjYSdKupHmwP1d/aP3Hl8SamfVdjzUKSf8ILAS+C1wI/EnSq+sOrD91XRIL+JJYM7NeqjJx0VeAIyJiPoCkVwCXAvvWGVh/8yWxZmZ9U6WPYqOuJAEQEX8ANqovJDMzaydVahS3SbqAVIsAeAceFNDMbNiokig+AHwY+L+kcZ5uBL5eZ1BmZtY+ShOFpH8AdgR+GhFnD0xIZmbWTlr2UUj6JGn4jncA10pqNtOdmZkNcWU1incAe0TE05LGALNIl8eamdkwUnbV07MR8TRARDzcw75mZjZEldUodijMlS1gx+Lc2RFxdK2RmZlZWyhLFP/SsHxenYGYmVl7Kpu46LqBDMTMzNqT+x3MzKxUlRvu1kseMdbMrH9UrlFI2rjOQPqbR4w1M+sfPdYoJO1PGmJ8S2CCpD2B9+YpUduaR4w1M3vhqtQozgXeDDwKEBF3kma8MzOzYaBKH8WIiPizpOK61TXF84J19U24X8LMrH9USRSLc/NTSBoJfAj4U71h9V0xSbhfwszshauSKD5Ian6aAPwV+HVe17bcN2Fm1n967KOIiIciYkpEjM6PKRHxSJWTSzpc0gJJCyWd3mT7RyXNl3SXpOskbdeXN2FmZvWpctXTt4FoXB8RU3s4biRwPnAYsAS4RdLM4rSqpJny9o2Iv0n6IHA2cFwv4jczs5pVaXr6deH5JsA/A4srHLc/sDAi7gOQdBlwFFCcf/v6wv5zgBMqnNfMzAZQj4kiIi4vLku6FLi2wrnHsXZCWQIcULL/ycAvmm2QNBWYCjBmzBg6OztbnmT58pUApfsMFStWrBgW77MKl0U3l0U3l0X/6MsQHtsDVfoS1GTdOk1YAJJOAPYFXttse0RMA6YBTJo0KTo6OtYaoqNo6cpnmTx2FB0dQ78zu7Ozk46OjsEOoy24LLq5LLq5LPpHlT6Kx+n+gh8BPAas0zHdxBJg28LyeGBpk/MfCpwBvDYinq1wXoCW90r4slgzs/5VmiiU7rLbE+j66b4mIprWCpq4BdhZ0vb5+CnA2xvOvzdwAXB4RDzUm8DBl8GamQ2E0stjc1L4aUSszo+qSYKIWAWcAlwD/AH4YUTMk3SWpCPzbv8FbA5cIekOSTP79jbMzKwuVfoo5kraJyJu6+3JI2IWMKth3acLzw/t7TnNzGxgtUwUkjbItYKDgfdJuhd4mtRJHRGxzwDFaGZmg6isRjEX2Ad46wDFYmZmbagsUQggIu4doFjMzKwNlSWKMZI+2mpjRHy5hnjMzKzNlCWKkaQrkprdOGdmZsNEWaJYFhFnDVgkZmbWlsruo3BNwszMShPFIQMWhZmZta2WiSIiHhvIQMzMrD31OMOdmZkNb04UZmZWyonCzMxKOVGYmVkpJwozMyvlRGFmZqWcKMzMrJQThZmZlXKiMDOzUk4UZmZWyonCzMxKOVGYmVkpJwozMyvlRGFmZqXWu0Txl6fXcNwFs5m/7MnBDsXMbFhY7xLF31enfyePHcVRe40b3GDMzIaBsjmz29JGI+Hy9x802GGYmQ0b612NwszMBpYThZmZlXKiMDOzUk4UZmZWyonCzMxKOVGYmVkpJwozMyvlRGFmZqWcKMzMrFStiULS4ZIWSFoo6fQm2zeWdHnefrOkiXXGY2ZmvVdbopA0EjgfeBMwGThe0uSG3U4GHo+InYCvAF+qKx4zM+ubOmsU+wMLI+K+iPg7cBlwVMM+RwEX5+c/Ag6RpBpjMjOzXqpzUMBxwOLC8hLggFb7RMQqSU8ALwEeKe4kaSowNS8+K+nuWiJe/4ymoayGMZdFN5dFN5dFt0l9PbDORNGsZhB92IeImAZMA5B0a0Ts+8LDW/+5LLq5LLq5LLq5LLpJurWvx9bZ9LQE2LawPB5Y2mofSRsAWwKP1RiTmZn1Up2J4hZgZ0nbS9oImALMbNhnJvCu/PwY4H8iYp0ahZmZDZ7amp5yn8MpwDXASODCiJgn6Szg1oiYCXwXuFTSQlJNYkqFU0+rK+b1kMuim8uim8uim8uiW5/LQv4Bb2ZmZXxntpmZlXKiMDOzUm2bKDz8R7cKZfFRSfMl3SXpOknbDUacA6Gnsijsd4ykkDRkL42sUhaSjs3/N+ZJmjHQMQ6UCn8jEyRdL+n2/HdyxGDEWTdJF0p6qNW9ZkrOzeV0l6R9Kp04ItruQer8vhfYAdgIuBOY3LDPvwLfys+nAJcPdtyDWBavA16Un39wOJdF3m8L4EZgDrDvYMc9iP8vdgZuB16cl1862HEPYllMAz6Yn08GFg123DWVxWuAfYC7W2w/AvgF6R62A4Gbq5y3XWsUHv6jW49lERHXR8Tf8uIc0j0rQ1GV/xcA/wmcDTwzkMENsCpl8T7g/Ih4HCAiHhrgGAdKlbIIYFR+viXr3tM1JETEjZTfi3YUcEkkc4CtJI3t6bztmiiaDf8xrtU+EbEK6Br+Y6ipUhZFJ5N+MQxFPZaFpL2BbSPiqoEMbBBU+X+xC7CLpJskzZF0+IBFN7CqlMWZwAmSlgCzgA8NTGhtp7ffJ0C9Q3i8EP12D0DCAAAHkklEQVQ2/McQUPl9SjoB2Bd4ba0RDZ7SspA0gjQK8UkDFdAgqvL/YgNS81MHqZb5G0m7R8TymmMbaFXK4nhgekScI+kg0v1bu0fEmvrDayt9+t5s1xqFh//oVqUskHQocAZwZEQ8O0CxDbSeymILYHegU9IiUhvszCHaoV31b+TKiHguIu4HFpASx1BTpSxOBn4IEBGzgU1IAwYON5W+Txq1a6Lw8B/deiyL3NxyASlJDNV2aOihLCLiiYgYHRETI2Iiqb/myIjo82BobazK38jPSBc6IGk0qSnqvgGNcmBUKYsHgEMAJL2ClCgeHtAo28NM4J356qcDgSciYllPB7Vl01PUN/zHeqdiWfwXsDlwRe7PfyAijhy0oGtSsSyGhYplcQ3wBknzgdXAJyLi0cGLuh4Vy+JjwLclfYTU1HLSUPxhKekHpKbG0bk/5j+ADQEi4luk/pkjgIXA34B3VzrvECwrMzPrR+3a9GRmZm3CicLMzEo5UZiZWSknCjMzK+VEYWZmpZwohilJqyXdUXhMLNl3YqvRKHv5mp15hM8787ASk/pwjg9Iemd+fpKkbQrbviNpcj/HeYukvSocc6qkF/Xhtb4q6TX5+Sl5VM/I9z309lyTcux3SPqDpH6d3U3SkV0js0oaozRq8+2S/lHSLElblRzb8nMrOebXkl7cf+/A+mywRzv0Y3AewIpe7DuRFqNR9vI1O8mjuQJTgZn9db5+LptinO8Grq1wzCJgdC9fZ2tgTmF571zWvT5XPv4a4KjC8j/U+P9nCnBxnZ8b6YbaM+p6D35Uf7hGYc/LNYffSLotP17VZJ/dJM3Nv1rvkrRzXn9CYf0Fkkb28HI3AjvlYw/Jv0x/rzSe/sZ5/RfVPc/Gf+d1Z0r6uKRjSONafT+/5qb51/S+kj4o6exCzCdJ+nof45xNYdA0Sd+UdKvS/A6fyes+DGwDXC/p+rzuDZJm53K8QtLmTc59DPDLroWIuD0iFvUQT5mxpCEaus73+xzLSZKulPTLXFP6j8L7aVoeSvM73JZrVdcVznNermGdDRxRKPtFXbUgSe/Mn9mdki7N61p9bv8k6aeFeA6T9JO8OJM0RpMNtsHOVH4MzoN0p+4d+fHTvO5FwCb5+c6ku1qhUKMAvg68Iz/fCNgUeAXwc2DDvP4bwDubvGYn3b/UPwFcThpKYTGwS15/CXAq6df2ArpvCt0q/3sm8PHG8xWXgTGkYae71v8COLiPcZ4KfL6wbev878i83x55eRG5FkAaQ+hGYLO8fBrw6SavczHwlibrnz9XLz/Td5NGUf4F8JFCmZ0ELCONrrwpcHcup6blkctvMbB9w3s+CTiv8XkxZmC3/LmNbji26edGGqTuj8CYvDyjWCbAPcBLBvvvZbg/2nIIDxsQKyOise19Q6DrF+Nq0thAjWYDZ0gaD/wkIu6RdAjwSuAWpSFENgVajTn1fUkrSV8sHwImAfdHxJ/y9ouBfwPOI80n8R1JVwOVhw2PiIcl3ac0ls09+TVuyuftTZybkRJCcRawYyVNJQ1/M5Y0Cc5dDccemNfflF9nI1K5NRpLP443FBEXSboGOJw078D7Je2ZN18befiO/Iv9YGAVzcvjQODGSAMJEhG9GWzz9cCPIuKRKsdGRORaxwmSLgIOIiWrLg+RamtDbuiR9YkThRV9BPgrsCfpQod1Jv6JiBmSbgb+CbhG0ntJvwovjoh/r/Aa74jCIH2Sms4hEmn8nv1JA7lNAU4hfQlVdTlwLOnX6k/zF1Kv4iTNlPZF4HzgaEnbAx8H9ouIxyVNJ9WIGon0xdxTs8nKFse3lL9M9waWRsQ603lGxFLgQuBCpQsQdu/a1LgrLT43SUc22b9yiH049iJSzeYZ4IpI88t02YRUTjaI3EdhRVsCyyKN0X8i6df0WiTtANwXEeeS2pD3AK4DjpH00rzP1qo+b/cfgYmSdsrLJwI35Db9LSNiFqn5p9mVR0+RhhZv5ifAW0lt3Jfndb2KMyKeAz4FHKg04ugo4GngCUkvA97UIpY5wKu73pOkF0lqVjv7A7mfpqqIeHdE7NUsSeR+hQ3z85eTmpoezJsPy+93U1K53ETr8pgNvDYnRiRt3YsQryPVul5Scuxan1tObktJZT298H4EvJxU+7RB5ERhRd8A3iVpDqnZ6ekm+xwH3C3pDmBX0rSK80l/5L+SdBdwLalZpUcR8Qypbf0KSb8H1gDfIn2RXJXPdwOpttNoOvCtrg7VhvM+DswHtouIuXldr+OMiJXAOaT29TtJc1DPI/1qv6mw6zTgF5Kuj4iHSW34P8ivM4dUVo2uJo30CaROcaURP8cDd0n6TllsTbyB9NncSboC6hMR8Ze87bfApaQ+qR9HxK2tyiPHPxX4ST7X5Y0v1EpEzAM+R0r2dwJfbrLbdNb93L4PLM4xdXkl6aqwVY0nsIHl0WPNBpGk3wJvjhpnnZN0Eqnz+JS6XuOFknQecHtEfLew7mukS6ivG7zIDFyjMBtsHwMmDHYQg0nS70hNmN9r2HS3k0R7cI3CzMxKuUZhZmalnCjMzKyUE4WZmZVyojAzs1JOFGZmVur/A+uL0y2nM7obAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fpr, tpr, thresholds = roc_curve(y_test, y_pred_prob)\n",
    "plt.plot(fpr, tpr)\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.0])\n",
    "plt.title('ROC curve for diabetes classifier')\n",
    "plt.xlabel('False Positive Rate (1 - Specificity)')\n",
    "plt.ylabel('True Positive Rate (Sensitivity)')\n",
    "plt.grid(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define a function that accepts a threshold and prints sensitivity and specificity\n",
    "def evaluate_threshold(threshold):\n",
    "    print('Sensitivity:', tpr[thresholds > threshold][-1])\n",
    "    print('Specificity:', 1 - fpr[thresholds > threshold][-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sensitivity: 0.8709677419354839\n",
      "Specificity: 0.7058823529411764\n"
     ]
    }
   ],
   "source": [
    "evaluate_threshold(0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sensitivity: 0.5806451612903226\n",
      "Specificity: 0.907563025210084\n"
     ]
    }
   ],
   "source": [
    "evaluate_threshold(0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8587693141772839\n"
     ]
    }
   ],
   "source": [
    "print(roc_auc_score(y_test, y_pred_prob))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8374308510638298"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calculate cross-validated AUC\n",
    "cross_val_score(logreg, X, y, cv=10, scoring='roc_auc').mean()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}