Logistic-Regression/Recommender.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ad recommender:\n",
    "In this notebook, I have implemented a logistic regression algorithm(pretty much from the scratch) for predicting whether a person will buy a magazine or not, based on their gender, age, salary, and earlier purchases."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1028,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports:\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "# For interactive plots, change 'inline' to 'notebook'.\n",
    "%matplotlib inline\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading the dataset:\n",
    "Dataset obtained from Kaggle."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1029,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User ID</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>EstimatedSalary</th>\n",
       "      <th>Purchased</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15624510</td>\n",
       "      <td>Male</td>\n",
       "      <td>19</td>\n",
       "      <td>19000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15810944</td>\n",
       "      <td>Male</td>\n",
       "      <td>35</td>\n",
       "      <td>20000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15668575</td>\n",
       "      <td>Female</td>\n",
       "      <td>26</td>\n",
       "      <td>43000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15603246</td>\n",
       "      <td>Female</td>\n",
       "      <td>27</td>\n",
       "      <td>57000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15804002</td>\n",
       "      <td>Male</td>\n",
       "      <td>19</td>\n",
       "      <td>76000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>395</th>\n",
       "      <td>15691863</td>\n",
       "      <td>Female</td>\n",
       "      <td>46</td>\n",
       "      <td>41000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>15706071</td>\n",
       "      <td>Male</td>\n",
       "      <td>51</td>\n",
       "      <td>23000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>397</th>\n",
       "      <td>15654296</td>\n",
       "      <td>Female</td>\n",
       "      <td>50</td>\n",
       "      <td>20000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398</th>\n",
       "      <td>15755018</td>\n",
       "      <td>Male</td>\n",
       "      <td>36</td>\n",
       "      <td>33000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>399</th>\n",
       "      <td>15594041</td>\n",
       "      <td>Female</td>\n",
       "      <td>49</td>\n",
       "      <td>36000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>400 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      User ID  Gender  Age  EstimatedSalary  Purchased\n",
       "0    15624510    Male   19            19000          0\n",
       "1    15810944    Male   35            20000          0\n",
       "2    15668575  Female   26            43000          0\n",
       "3    15603246  Female   27            57000          0\n",
       "4    15804002    Male   19            76000          0\n",
       "..        ...     ...  ...              ...        ...\n",
       "395  15691863  Female   46            41000          1\n",
       "396  15706071    Male   51            23000          1\n",
       "397  15654296  Female   50            20000          1\n",
       "398  15755018    Male   36            33000          0\n",
       "399  15594041  Female   49            36000          1\n",
       "\n",
       "[400 rows x 5 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = pd.read_csv('data.csv')\n",
    "display(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data preprocessing:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1030,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 'User ID' is a useless feature.\n",
    "df = df.drop(columns = ['User ID'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1031,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Gender             0\n",
       "Age                0\n",
       "EstimatedSalary    0\n",
       "Purchased          0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 1031,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No null values are present."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1032,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    257\n",
       "1    143\n",
       "Name: Purchased, dtype: int64"
      ]
     },
     "execution_count": 1032,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"Purchased\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A value of 1 means that they have purchased the magazine."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1033,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We need numeric data.\n",
    "df[\"Gender\"].replace('Male', 0, inplace=True) # 0 is for male\n",
    "df[\"Gender\"].replace('Female', 1, inplace=True) # 1 is for female"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, we have 3 features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1034,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>EstimatedSalary</th>\n",
       "      <th>x_0</th>\n",
       "      <th>Gender</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1.779569</td>\n",
       "      <td>-1.488183</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.253270</td>\n",
       "      <td>-1.458854</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.111813</td>\n",
       "      <td>-0.784307</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.016419</td>\n",
       "      <td>-0.373714</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.779569</td>\n",
       "      <td>0.183521</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>395</th>\n",
       "      <td>0.796060</td>\n",
       "      <td>-0.842964</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>1.273028</td>\n",
       "      <td>-1.370870</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>397</th>\n",
       "      <td>1.177635</td>\n",
       "      <td>-1.458854</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398</th>\n",
       "      <td>-0.157877</td>\n",
       "      <td>-1.077589</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>399</th>\n",
       "      <td>1.082241</td>\n",
       "      <td>-0.989604</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>400 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Age  EstimatedSalary  x_0  Gender\n",
       "0   -1.779569        -1.488183  1.0       0\n",
       "1   -0.253270        -1.458854  1.0       0\n",
       "2   -1.111813        -0.784307  1.0       1\n",
       "3   -1.016419        -0.373714  1.0       1\n",
       "4   -1.779569         0.183521  1.0       0\n",
       "..        ...              ...  ...     ...\n",
       "395  0.796060        -0.842964  1.0       1\n",
       "396  1.273028        -1.370870  1.0       0\n",
       "397  1.177635        -1.458854  1.0       1\n",
       "398 -0.157877        -1.077589  1.0       0\n",
       "399  1.082241        -0.989604  1.0       1\n",
       "\n",
       "[400 rows x 4 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X = df[[\"Age\", \"EstimatedSalary\"]] \n",
    "y = df[[\"Purchased\"]] # Dependent variable.\n",
    "\n",
    "# Feature scaling: Standard scaling/ Mean normalization\n",
    "X = (X - X.mean())/X.std()\n",
    "\n",
    "X = X.assign(x_0 = pd.Series(np.ones(X.shape[0])).values) # Adding the dummy column filled with ones.\n",
    "X = X.assign(Gender = df[[\"Gender\"]]) # Adding the gender column.\n",
    "display(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have added the dummy feature column filled with ones. (To take care of $\\theta _0$)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Splitting the dataset:\n",
    "We split the dataset into training and validation datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1035,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42) # 3:1 Ratio split.\n",
    "\n",
    "m = X_train.shape[0] # The number of datapoints/rows.\n",
    "n = X_train.shape[1] # The number of columns."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1036,
   "metadata": {},
   "outputs": [],
   "source": [
    "theta = np.zeros(n) # Initial value for the parameters.\n",
    "#theta_temp = theta # Temporary theta required for simultaneous upgrading."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper functions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1037,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The logistic function/ sigmoid function:\n",
    "def sigmoid(z):\n",
    "    return np.float32(1/(1 + np.exp(-z)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1038,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The cost function:\n",
    "def cost(y, X, theta):\n",
    "    s = 0\n",
    "    for i in range(m):\n",
    "        s = s - (1/m)*(y[i]*np.log(h(X[i], theta)) + (1 - y[i])*np.log(1 - h(X[i], theta)))\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1039,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Our hypothesis function:\n",
    "def h(x, theta): # Here x is a datapoint.\n",
    "    return sigmoid(np.dot(x,theta))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1040,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculates the gradient with respect to a parameter.\n",
    "def gradient(X, y, j):\n",
    "    s = 0\n",
    "    for i in range(m):\n",
    "        s = s + (1/m)*(h(X[i], theta) - y[i])*X[i][j]\n",
    "    return s"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gradient Descent:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1041,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Converting dataframes to numpy arrays:\n",
    "X_train = X_train.values \n",
    "y_train = y_train.values\n",
    "X_test = X_test.values \n",
    "y_test = y_test.values\n",
    "y_test = y_test.reshape(y_test.shape[0],)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1042,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GRADIENT DESCENT PARAMETERS:\n",
    "N_ITER = 100 # The number of iterations.\n",
    "alpha = 0.3 # The learning rate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1043,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GRADIENT DESCENT:\n",
    "costY = []\n",
    "for i in range(N_ITER):\n",
    "    costY.append(cost(y_train, X_train, theta))\n",
    "    # Theta gets updated simultaneously in each iteration and then gets updated as a whole.\n",
    "    theta = [(theta[j] - alpha*gradient(X_train, y_train, j))[0] for j in range(n)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1044,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The parameters are:  [1.7936195485665132, 0.9876229126705455, -0.8318873857464645, -0.2331764692016877]\n"
     ]
    }
   ],
   "source": [
    "print(\"The parameters are: \", theta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1045,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de3Rc5Xnv8e8zo9H9bklYlmQsG9vYXIxBMQSSQGkghiSQNCkF0hZWm5BeCE1zaUnPOU0L55yml5SkLU1LCSntCXEIyWkcQssBwiXhEiyD7WAbYyGMLdtg2ZJlybrO6Dl/zJYYhGSNbMkj7fl91po1s9/97plnr23/ZvTud/aYuyMiIuEVyXQBIiIysxT0IiIhp6AXEQk5Bb2ISMgp6EVEQi4n0wWMVVVV5YsWLcp0GSIic8rGjRsPunv1eOtmXdAvWrSI5ubmTJchIjKnmNnrE63T0I2ISMilFfRmttbMdphZi5ndOs76O8xsU3B7xcwOp6y7wcx2BrcbprN4ERGZ3KRDN2YWBe4ELgPagA1mtt7dt430cfc/TOn/GWB18LgS+DLQBDiwMdi2c1r3QkREJpTOJ/o1QIu7t7r7ILAOuPoY/a8DvhM8/gDwiLt3BOH+CLD2RAoWEZGpSSfo64A9KcttQds7mNmpQCPwk6lsa2Y3mVmzmTW3t7enU7eIiKQpnaC3cdomuhLatcAD7p6Yyrbufpe7N7l7U3X1uLODRETkOKUT9G1AQ8pyPbBvgr7X8tawzVS3FRGRGZBO0G8AlppZo5nlkgzz9WM7mdlyoAJ4NqX5YeByM6swswrg8qBt2nX1DfH1R3eyec/hyTuLiGSRSWfduHvczG4mGdBR4B5332pmtwHN7j4S+tcB6zzlAvfu3mFmt5N8swC4zd07pncXkszgjkdfIT8WYVVD+Uy8hIjInJTWN2Pd/SHgoTFtfzpm+c8m2PYe4J7jrC9tpfkxygtj7O7onemXEhGZU0L1zdiGikL2dPZlugwRkVklVEG/sLKQPfpELyLyNqEK+vrKAvZ29jE8rN/BFREZEaqgX1hZyGBimDe7+zNdiojIrBGqoG+oKARg9yEN34iIjAhX0Fcmg14nZEVE3hKqoK8rL8AMTbEUEUkRqqDPzYlQW5pPm4JeRGRUqIIeksM3+kQvIvKWUAb9nk4FvYjIiPAFfUUhbx4ZoH8oMXlnEZEsELqgXzivAIA2zbwREQFCGPQjc+k1fCMikhS6oF84MpdeJ2RFRIAQBn11SR55OREFvYhIIHRBb2bJmTcdGqMXEYEQBj1AQ0WB5tKLiATCGfTBdelTftVQRCRrhTLoF1YW0j0Qp6tvKNOliIhkXCiDvn5kiqXG6UVEwhn0o1MsNZdeRCScQd9Qmfx27Ov6ARIRkfSC3szWmtkOM2sxs1sn6HONmW0zs61mdl9Ke8LMNgW39dNV+LGU5MeoKs5j18GjJ+PlRERmtZzJOphZFLgTuAxoAzaY2Xp335bSZynwJeAid+80s5qUp+hz93Omue5JLa4u4tX2npP9siIis046n+jXAC3u3urug8A64OoxfT4F3OnunQDufmB6y5y6JdXFCnoREdIL+jpgT8pyW9CWahmwzMyeNrPnzGxtyrp8M2sO2j8y3guY2U1Bn+b29vYp7cBEllQX0dk7RMfRwWl5PhGRuSqdoLdx2sZ+EykHWApcAlwH3G1m5cG6he7eBFwPfM3Mlrzjydzvcvcmd2+qrq5Ou/hjWVJdDECrPtWLSJZLJ+jbgIaU5Xpg3zh9fujuQ+7+GrCDZPDj7vuC+1bgCWD1CdaclpGg1/CNiGS7dIJ+A7DUzBrNLBe4Fhg7e+Y/gF8CMLMqkkM5rWZWYWZ5Ke0XAds4CeoqCsjNidDarpk3IpLdJp114+5xM7sZeBiIAve4+1Yzuw1odvf1wbrLzWwbkAC+6O6HzOxC4J/NbJjkm8pXUmfrzKRoxFhcpZk3IiKTBj2Auz8EPDSm7U9THjvwueCW2ucZ4KwTL/P4LK4uYvv+7ky9vIjIrBDKb8aOWFJdzO6OXgbjw5kuRUQkY0If9IlhZ3eHxulFJHuFPugBWg4o6EUke4U66BuriwBNsRSR7BbqoC/Oy2F+ab6mWIpIVgt10AMsqdEUSxHJbqEP+sVVyYub6fdjRSRbhT7ol1QX0d0f52CPLm4mItkp/EFfo2veiEh2C3/Q6+JmIpLlQh/080vzKcyN0nJAQS8i2Sn0QR+JGEtPKeFlXfNGRLJU6IMeYGVtCdvfOKKZNyKSlbIi6FfUlnK4d4g3jvRnuhQRkZMua4IeYPv+IxmuRETk5MuKoD99fgmArk0vIlkpK4K+JD9GQ2UB2/SJXkSyUFYEPcDK2lIN3YhIVsqaoF9RW8qug0fpG0xkuhQRkZMqq4J+2GHHmxqnF5HskjVBv1Izb0QkS2VN0NdXFFCSl6OgF5GskzVBb2acXluioBeRrJNW0JvZWjPbYWYtZnbrBH2uMbNtZrbVzO5Lab/BzHYGtxumq/DjsaK2lJf3d+tSCCKSVSYNejOLAncCVwArgevMbOWYPkuBLwEXufsZwGeD9krgy8D5wBrgy2ZWMa17MAUrakvpHojT1tmXqRJERE66dD7RrwFa3L3V3QeBdcDVY/p8CrjT3TsB3P1A0P4B4BF37wjWPQKsnZ7Sp27kUgj64pSIZJN0gr4O2JOy3Ba0pVoGLDOzp83sOTNbO4VtMbObzKzZzJrb29vTr36Klp9SQsQ080ZEsks6QW/jtI0d5M4BlgKXANcBd5tZeZrb4u53uXuTuzdVV1enUdLxKciN0lhVxEt7FfQikj3SCfo2oCFluR7YN06fH7r7kLu/BuwgGfzpbHtSrWooZ9OewzohKyJZI52g3wAsNbNGM8sFrgXWj+nzH8AvAZhZFcmhnFbgYeByM6sITsJeHrRlzOqFFRzsGdAJWRHJGpMGvbvHgZtJBvR24H5332pmt5nZVUG3h4FDZrYNeBz4orsfcvcO4HaSbxYbgNuCtoxZ3VAOwKY9hzNZhojISZOTTid3fwh4aEzbn6Y8duBzwW3stvcA95xYmdNn+fwS8nIivLj7MB9etSDT5YiIzLis+WbsiFg0wtn1ZWza05npUkREToqsC3qAcxrKeWnfEQbjw5kuRURkxmVl0K9eWMFgfFjz6UUkK2Rl0J+jE7IikkWyMuhry/KpKcnjxd0apxeR8MvKoDczVi8s1yd6EckKWRn0AOc0VLDrUC+dRwczXYqIyIzK4qDXOL2IZIesDfqz68uIGLyooBeRkMvaoC/Ky2H5/FJeeF0nZEUk3LI26AHOb6xk4+ud+uKUiIRaVgf9BYvn0TeUYHObhm9EJLyyOujPb6zEDJ579VCmSxERmTFZHfQVRbmcPr+UZ1sV9CISXlkd9ADvXjyPja93MhBPZLoUEZEZoaBfMo+B+DAv7tY4vYiEU9YH/ZrGSiIGz2qcXkRCKuuDvqwgxhkLyjROLyKhlfVBD8nhm027D9M/pHF6EQkfBT3JE7KDiWF9S1ZEQklBDzQtqiAaMQ3fiEgoKeiBkvwYZ9aV8XTLwUyXIiIy7dIKejNba2Y7zKzFzG4dZ/2NZtZuZpuC2ydT1iVS2tdPZ/HT6eJl1Wzac1jXpxeR0Jk06M0sCtwJXAGsBK4zs5XjdP2uu58T3O5Oae9Lab9qesqefpeeXsOww5OvtGe6FBGRaZXOJ/o1QIu7t7r7ILAOuHpmyzr5zq4ro6o4l5+8fCDTpYiITKt0gr4O2JOy3Ba0jfUxM9tiZg+YWUNKe76ZNZvZc2b2kfFewMxuCvo0t7dn5hN1JGJcsryGJ3YcIJ7QZYtFJDzSCXobp83HLP8IWOTuZwOPAvemrFvo7k3A9cDXzGzJO57M/S53b3L3purq6jRLn36Xnl7Dkf44L+hyCCISIukEfRuQ+gm9HtiX2sHdD7n7QLD4L8B5Kev2BfetwBPA6hOod0a9d2kVORHjsZffzHQpIiLTJp2g3wAsNbNGM8sFrgXeNnvGzGpTFq8CtgftFWaWFzyuAi4Ctk1H4TOhJD/GmsZKHtc4vYiEyKRB7+5x4GbgYZIBfr+7bzWz28xsZBbNLWa21cw2A7cANwbtK4DmoP1x4CvuPmuDHpLDN6+82cOejt5MlyIiMi3Mfexwe2Y1NTV5c3Nzxl6/tb2HS7/6JLddfQa/+e5FGatDRGQqzGxjcD70HfTN2DEWVxfTWFXEY9s1fCMi4aCgH8f7V9TwzKsH6eobynQpIiInTEE/jg+evYChhPP/tr6R6VJERE6Ygn4cq+rLaKgs4MEt+zNdiojICVPQj8PM+OBZC/hZy0E6dJEzEZnjFPQT+NDZtSSGnf96ScM3IjK3KegncMaCUhqrinhwy77JO4uIzGIK+gmYGR86u5bnWg/R3j0w+QYiIrOUgv4YPrxqAcMO//mSTsqKyNyloD+GZaeUsOyUYn60WcM3IjJ3KegncdWqBWzY1cnrh45muhQRkeOioJ/Ex86rJ2Jwf/OeyTuLiMxCCvpJ1JYVcMnyGr7X3KZfnhKROUlBn4Zfe1cDB7oHeGKHfjhcROYeBX0aLj29hqriPNZt0PCNiMw9Cvo0xKIRPn5ePY/vOMCbR/ozXY6IyJQo6NP0a+9qIDHsPLCxLdOliIhMiYI+TY1VRZzfWMl3N+xheHh2/SqXiMixKOin4PrzF7K7o5fHd+jXp0Rk7lDQT8GVZ9VSW5bP3T99LdOliIikTUE/BbFohBsvXMSzrYd4aW9XpssREUmLgn6Krl2zkMLcKPf8TJ/qRWRuSCvozWytme0wsxYzu3Wc9TeaWbuZbQpun0xZd4OZ7QxuN0xn8ZlQVhDjmqYG1m/exxtdmmopIrPfpEFvZlHgTuAKYCVwnZmtHKfrd939nOB2d7BtJfBl4HxgDfBlM6uYtuoz5LcuaiThzr89uyvTpYiITCqdT/RrgBZ3b3X3QWAdcHWaz/8B4BF373D3TuARYO3xlTp7LJxXyAdWzufbP9/N0YF4pssRETmmdIK+Dkj97n9b0DbWx8xsi5k9YGYNU9nWzG4ys2Yza25vnxvXk/n0xYvp6hvi3md3ZboUEZFjSifobZy2sd8Y+hGwyN3PBh4F7p3Ctrj7Xe7e5O5N1dXVaZSUeasXVnDJ8mrueqqVHn2qF5FZLJ2gbwMaUpbrgbf95JK7H3L3kR9W/RfgvHS3ncs++/5lHO4d4t5ndmW6FBGRCaUT9BuApWbWaGa5wLXA+tQOZlabsngVsD14/DBwuZlVBCdhLw/aQuGchnJ++fQa7nqqle7+oUyXIyIyrkmD3t3jwM0kA3o7cL+7bzWz28zsqqDbLWa21cw2A7cANwbbdgC3k3yz2ADcFrSFxmffv4yuviG+9fSuTJciIjIuc59dF+hqamry5ubmTJcxJZ+8t5nnXzvET//oUsoKY5kuR0SykJltdPem8dbpm7HT4HOXLaN7IM7f/2RnpksREXkHBf00WLmglGvOa+DeZ3fx2sGjmS5HRORtFPTT5PMfWEZuNML/fmj75J1FRE4iBf00qSnJ5/cvPY1Htr3J0y0HM12OiMgoBf00+q2LGqmvKOD2B7eR0K9QicgsoaCfRvmxKH9y5QpefqNbFzwTkVlDQT/NrjhzPhcvq+avH97B3sN9mS5HRERBP93MjP/10TMB+O//9xfMtu8piEj2UdDPgPqKQj5/+XIe39HOj7bsz3Q5IpLlFPQz5MYLF7Gqvow/X7+VzqODmS5HRLKYgn6GRCPGVz52Nkf6h/gTDeGISAYp6GfQitpSvnD5cv7zpTe4v3nP5BuIiMwABf0M+9R7F3Phknn82fpttLb3ZLocEclCCvoZFokYX71mFbk5ET773U0MJYYzXZKIZBkF/UlQW1bAX37sLLa0dfEXD72c6XJEJMso6E+StWfWcuOFi7jn6df44aa9mS5HRLKIgv4k+m8fXMGaRZX88fe3sH3/kUyXIyJZQkF/EsWiEf7hE6spK4jx6X/fyOFeza8XkZmnoD/Jakry+cdPnMf+rj5+5/9sZCCeyHRJIhJyCvoMOO/UCv7646t4rrWDP35gi75MJSIzKifTBWSrj6yuY+/hPv764R3UVRTwxQ+cnumSRCSkFPQZ9HuXLKGts5c7H3+V+aX5/Ma7F2W6JBEJIQV9BpkZt199Ju3dA/yPH24lLxblmqaGTJclIiGT1hi9ma01sx1m1mJmtx6j38fNzM2sKVheZGZ9ZrYpuP3TdBUeFjnRCP9w/bm8d2kVf/z9LZpjLyLTbtKgN7MocCdwBbASuM7MVo7TrwS4Bfj5mFWvuvs5we13pqHm0MmPRbnrN5pYs6iSz92/mR/rGvYiMo3S+US/Bmhx91Z3HwTWAVeP0+924K+A/mmsL2sU5Eb55o3v4pyGcj7znRd4YGNbpksSkZBIJ+jrgNRr7LYFbaPMbDXQ4O4PjrN9o5m9aGZPmtl7x3sBM7vJzJrNrLm9vT3d2kOnOC+Hf/utNbx7yTy+8L3N3PvMrkyXJCIhkE7Q2zhtoxO/zSwC3AF8fpx++4GF7r4a+Bxwn5mVvuPJ3O9y9yZ3b6qurk6v8pAqysvhmze8i8tWnsKX12/l64/u1Dx7ETkh6QR9G5A6FaQe2JeyXAKcCTxhZruAC4D1Ztbk7gPufgjA3TcCrwLLpqPwMMuPRfnHT5zLr5xbxx2PvsIXH9jCYFyXNxaR45PO9MoNwFIzawT2AtcC14+sdPcuoGpk2cyeAL7g7s1mVg10uHvCzBYDS4HWaaw/tGLRCF/91VU0VBTy9cd2su9wH9/49fMoK4hlujQRmWMm/UTv7nHgZuBhYDtwv7tvNbPbzOyqSTZ/H7DFzDYDDwC/4+4dJ1p0tjAz/vCyZfztNavYsKuDj975NDvf7M50WSIyx9hsG/9tamry5ubmTJcx6zz/Wge/9+2N9A0m+JtfXcUVZ9VmuiQRmUXMbKO7N423Thc1myPWNFby4Gfey7L5Jfzut1/gLx7arp8lFJG0KOjnkPll+ay76QJ+/YKF/PNTrXz8G8/w+qGjmS5LRGY5Bf0ck5cT5X9+5Cz+8RPn8trBo3zw737GD15o0xRMEZmQgn6OuvKsWv7zs+9jRW0Jn7t/M5/+940c6NaXkkXknRT0c1hdeQHrbno3X7ridJ54pZ3L73iKH27aq0/3IvI2Cvo5LhoxPn3xEh665T0smlfEH6zbxG/e8zy7DmrsXkSSFPQhcVpNCd//3Qv5sw+v5MXdh7n8a0/x9Ud30j+k36QVyXYK+hCJRowbL2rksc9fzGUrT+GOR1/hl7/6JA9u2afhHJEspqAPoVNK87nz+nO571PnU1oQ4+b7XuRX/+lZNr6uLyWLZCMFfYhduKSKBz/zHr7yK2ex61AvH/vGs3zy3g28/MaRTJcmIieRLoGQJXoH43zr6V3805Ov0jMQ58qzavnMpadx+vx3XDVaROagY10CQUGfZQ73DvIvP23l3mdep2cgztoz5vO7lyxhVUN5pksTkROgoJd3ONw7yLee3sU9T79Gd3+cCxZX8un3LeGS5dWYjfdbMyIymynoZUI9A3HWPb+bb/7sNfZ39bOkuogbLlzEr5xbT3FeOj9XICKzgYJeJjUYH+bHv9jHvz69i81tXZTk5fDRc+u4/vyFGscXmQMU9DIlL+7u5N5ndvHQS28wGB/m3IXl/Nq7GrjyrFpK8vULVyKzkYJejkvn0UG+/0Ib33l+N6+2HyU/FuGKM2v56Oo6Llwyj5yoZueKzBYKejkh7s6mPYd5YGMb6zfvo7s/TlVxHh86u5YPr1rA6oZyIhGdwBXJJAW9TJv+oQRP7Ghn/ea9PLr9AIPxYeaX5rP2zPlcceZ8mhZVElXoi5x0CnqZEUf6h/jJ9gP8+Bf7efKVdgbjw1QW5XLp6TW8f8UpvGdplWbuiJwkCnqZcT0DcZ7c0c4j297gsZcP0N0fJzcaYU1jJZcsr+biZdWcVlOsOfoiM0RBLyfVUGKY5l2dPL7jAD95+QAtB3oAqC3L5z2nVXHRaVVcuGQeNaX5Ga5UJDxOOOjNbC3wdSAK3O3uX5mg38eB7wHvcvfmoO1LwG8DCeAWd3/4WK+loA+fts5efrrzID/d2c7TLYfo6hsC4LSaYs5vrOT8xfO4oLFSwS9yAk4o6M0sCrwCXAa0ARuA69x925h+JcCPgVzgZndvNrOVwHeANcAC4FFgmbtP+GsYCvpwSww72/cf4emWgzzbeojmXZ30DMQBaKgs4F2nVnLeogrOXVjBslNKdGJXJE3HCvp0zpStAVrcvTV4snXA1cC2Mf1uB/4K+EJK29XAOncfAF4zs5bg+Z6d2i5IWEQjxpl1ZZxZV8anL15CPDHMtv1HeP61Dpp3dfLUznZ+8OJeAIpyo6xqKE/e6ss4u76c2rJ8jfOLTFE6QV8H7ElZbgPOT+1gZquBBnd/0My+MGbb58ZsWzf2BczsJuAmgIULF6ZXuYRCTjTC2fXlnF1fziffm5yz//qhXl7c08mLuw/z4u7D3P3TVoYSyb88q4pzOWNBGWfVlXHGglJWLiiloaJQ8/hFjiGdoB/vf9DoeI+ZRYA7gBunuu1og/tdwF2QHLpJoyYJKTNjUVURi6qK+OjqeiA5d3/7/iNsaeviF3u7eGlvFz9rOUhiOPlPpTgvh+XzS1g+v4QV80tYPr+UZacUU16Ym8ldEZk10gn6NqAhZbke2JeyXAKcCTwR/Ek9H1hvZlelsa3IpPJjUVYvrGD1worRtv6hBK+82c22fUfYtv8IL+/v5sHN+7jv5/HRPjUleSw9pZilNSUsqSlmSXURp1UXU12Sp+EfySrpBP0GYKmZNQJ7gWuB60dWunsXUDWybGZPAF8ITsb2AfeZ2d+SPBm7FHh++sqXbJUfi44O+Yxwd/Z19fPKm9288kY3O97s5tUDPXyveQ9HB986/1+cl8Pi6iIWzSuisSp5O3VeIafOK6KiMKY3AQmdSYPe3eNmdjPwMMnplfe4+1Yzuw1odvf1x9h2q5ndT/LEbRz4/WPNuBE5EWZGXXkBdeUF/NLymtF2d2d/Vz+t7Ud5tb2H1vYeWg8e5YXdnfxoyz5SJ56V5OWwcF4hCysLaagspKGigPrgvq68kILcaAb2TOTE6AtTktX6hxK0dfay62Avuw4dZXdH7+itraOPwcTw2/rPK8qlrqKABWUFLCgvYEF5PrVlBdSW51Nblk91cZ6u6ikZcaLTK0VCKz8W5bSaEk6rKXnHuuFhp71ngLbOXvZ09LH3cB9tncn7lvYentrZTu/g2/9AjRhUl+QxvzSfU0ZvedSU5FNdmkdNSfJxZVGuviMgJ42CXmQCkYiNhvV5p75zvbvT1TfE/q5+9nf1sb+rnze7+tnf1c8bR/p5/VAvz+/q4HDv0Duf22BecR5VxXlUFeeO3s8rzmNeUXK5siiXyqJc5hXnUpir/6py/PSvR+Q4mRnlhbmUF+ayonbin1vsH0rQ3j3Age4B2rv7ae8eGF0+2DNAe88gre1HOdgzwEB8eNznyI9FqCzMpSII//LCXCoLY5QV5lJRGKOiMJeywhjlBbFkTQUxSgti+qtBAAW9yIzLj0WTJ3YrC4/Zz93pHUxwsGeAQ0cH6egZ5NDRATqODtGRct/ZO8Sejl46jg5ypD9+zOcsycuhtCBGWUGM0oKc5H1+8k2gND9GSX5OcItRGtyX5OdQnJ9DcV4O+TGdfA4DBb3ILGFmFOXlUJSXw6nzitLaJjGcHD7q7B2kq2+Irt6Ux31DHO4d4kj/EEeC5V0HeznSn3w89vzCeHKjkdHQL8rLoTgvmvI4Z7TeotwohcH6wtwcinJzKMyLUpgbpTD21uOCWFTTVzNAQS8yh0UjNjqWP1XxxDA9A3GO9MU50j9Ed3+c7v4hegbi9AzEg+U4RwdG2hMcHYjT3jPA64d66RlIrjuaxhvGCDMoiCUDvyD3rfAvCO4Lc5N/RRTkRsjPSbbnx5K3gliU/FgkWE6uz3tbW5S8nMjofU7E9KYSUNCLZKmcaGT0HMOJGB52+uOJIPgT9A7G6R1MLvcNJugdfKutdzBBX/C4bzDB0cE4fUPD9A8maO8ZoH+oj77BBH1DCfqHkvfHOwM8YoyGfl5OlLxYhLycCLkjy6OPI+TmRMmNvrU8sm6kLTcnQixlfSyaXBcb7WPEopHRW3JdalvycabefBT0InJCIhGjMDcnOTPonbNUT4i7MxAfZmBomL6ht94Akrdh+uMJBoaGGYgn2wbiw8n7oeG3HseT6wfjw/QPDTOYSC4PDCX/ohkI2gbjw2/rO5gYPu43mWNJDf3cnAg5keBNIRLhjLoy/v661dP+mgp6EZm1zGx0WKaM2El9bXcnPuzJ0I+/9WYwlEg+Hoo7g4nkG0k84aN9hhLBcmKYeGKYwWBdPDHM0LAH64cZSukz0r+homBG9kVBLyIyDjMb/fRdlJfpak6MvqstIhJyCnoRkZBT0IuIhJyCXkQk5BT0IiIhp6AXEQk5Bb2ISMgp6EVEQm7W/ZSgmbUDr09xsyrg4AyUM5tl4z5Ddu53Nu4zZOd+n8g+n+ru1eOtmHVBfzzMrHmi30oMq2zcZ8jO/c7GfYbs3O+Z2mcN3YiIhJyCXkQk5MIS9HdluoAMyMZ9huzc72zcZ8jO/Z6RfQ7FGL2IiEwsLJ/oRURkAgp6EZGQm9NBb2ZrzWyHmbWY2a2ZrmemmFmDmT1uZtvNbKuZ/UHQXmlmj5jZzuC+ItO1Tjczi5rZi2b2YLDcaGY/D/b5u2Z2Yj94OsuYWbmZPWBmLwfH+91Zcpz/MPi3/ZKZfcfM8sN4rM3sHjM7YGYvpbSNe3wt6e+CfNtiZuce7+vO2aA3syhwJ3AFsBK4zsxWZraqGRMHPu/uK4ALgN8P9vVW4DF3Xwo8FiyHzR8A21OW/xK4I9jnTuC3M1LVzPk68F/ufjqwiuS+h/o4m1kdcAvQ5O5nAlHgWsJ5rP8VWDumbaLjewWwNLjdBHzjeF90zgY9sAZocfdWdx8E1gFXZ7imGeHu+939heBxN8n//HUk9/feoNu9wEcyU+HMMLN64IPA3cGyAZcCDwRdQrXPZlYKvA/4JoC7D7r7YeEfvewAAAJDSURBVEJ+nAM5QIGZ5QCFwH5CeKzd/SmgY0zzRMf3auDfPOk5oNzMao/ndedy0NcBe1KW24K2UDOzRcBq4OfAKe6+H5JvBkBN5iqbEV8D/ggYDpbnAYfdPR4sh+2YLwbagW8Fw1V3m1kRIT/O7r4X+BtgN8mA7wI2Eu5jnWqi4zttGTeXg97GaQv1XFEzKwa+D3zW3Y9kup6ZZGYfAg64+8bU5nG6humY5wDnAt9w99XAUUI2TDOeYEz6aqARWAAUkRy2GCtMxzod0/bvfS4HfRvQkLJcD+zLUC0zzsxiJEP+2+7+g6D5zZE/5YL7A5mqbwZcBFxlZrtIDstdSvITfnnw5z2E75i3AW3u/vNg+QGSwR/m4wzwfuA1d2939yHgB8CFhPtYp5ro+E5bxs3loN8ALA3OzOeSPHmzPsM1zYhgbPqbwHZ3/9uUVeuBG4LHNwA/PNm1zRR3/5K717v7IpLH9ifu/gngceDjQbew7fMbwB4zWx40/TKwjRAf58Bu4AIzKwz+rY/sd2iP9RgTHd/1wG8Gs28uALpGhnimzN3n7A24EngFeBX4b5muZwb38z0k/2TbAmwKbleSHLN+DNgZ3FdmutYZ2v9LgAeDx4uB54EW4HtAXqbrm+Z9PQdoDo71fwAV2XCcgT8HXgZeAv4dyAvjsQa+Q/I8xBDJT+y/PdHxJTl0c2eQb78gOSvpuF5Xl0AQEQm5uTx0IyIiaVDQi4iEnIJeRCTkFPQiIiGnoBcRCTkFvYhIyCnoRURC7v8DVfc17cn6G1cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting the cost function for better visualization: \n",
    "plt.plot(np.linspace(1, N_ITER, N_ITER), costY)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1046,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions1 = [h(X_test[i], theta) for i in range(X_test.shape[0])]\n",
    "# Cut-off set as 0.5\n",
    "predictions = np.round(predictions1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1047,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Gives us some measure of the accuracy of our model.\n",
    "def score(a, b):\n",
    "    return ((np.sum(a==b))/y_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1048,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The accuracy is: 0.88\n"
     ]
    }
   ],
   "source": [
    "print(\"The accuracy is: {:.2f}\".format(score(predictions, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1049,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[61  2]\n",
      " [10 27]]\n",
      "Accuracy: 0.88\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "cnf_matrix = metrics.confusion_matrix(y_test, predictions)\n",
    "print(cnf_matrix)\n",
    "print(\"Accuracy:\",metrics.accuracy_score(y_test, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1050,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9631059631059632\n"
     ]
    }
   ],
   "source": [
    "# Another way to test the accuracy of our classifier:\n",
    "auc = metrics.roc_auc_score(y_test, predictions1) # Note that this accepts probability predictions and not binary integer classes.\n",
    "print(auc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "AUC score 1 represents perfect classifier, and 0.5 represents a worthless classifier."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}