machine_learning_algorithms/logistic_regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.datasets import load_breast_cancer\n",
    "from sklearn.linear_model import LogisticRegression \n",
    "from sklearn.model_selection import train_test_split "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "   mean radius  mean texture  mean perimeter  mean area  mean smoothness  \\\n0        17.99         10.38          122.80     1001.0          0.11840   \n1        20.57         17.77          132.90     1326.0          0.08474   \n2        19.69         21.25          130.00     1203.0          0.10960   \n3        11.42         20.38           77.58      386.1          0.14250   \n4        20.29         14.34          135.10     1297.0          0.10030   \n\n   mean compactness  mean concavity  mean concave points  mean symmetry  \\\n0           0.27760          0.3001              0.14710         0.2419   \n1           0.07864          0.0869              0.07017         0.1812   \n2           0.15990          0.1974              0.12790         0.2069   \n3           0.28390          0.2414              0.10520         0.2597   \n4           0.13280          0.1980              0.10430         0.1809   \n\n   mean fractal dimension  ...  worst texture  worst perimeter  worst area  \\\n0                 0.07871  ...          17.33           184.60      2019.0   \n1                 0.05667  ...          23.41           158.80      1956.0   \n2                 0.05999  ...          25.53           152.50      1709.0   \n3                 0.09744  ...          26.50            98.87       567.7   \n4                 0.05883  ...          16.67           152.20      1575.0   \n\n   worst smoothness  worst compactness  worst concavity  worst concave points  \\\n0            0.1622             0.6656           0.7119                0.2654   \n1            0.1238             0.1866           0.2416                0.1860   \n2            0.1444             0.4245           0.4504                0.2430   \n3            0.2098             0.8663           0.6869                0.2575   \n4            0.1374             0.2050           0.4000                0.1625   \n\n   worst symmetry  worst fractal dimension  target  \n0          0.4601                  0.11890       0  \n1          0.2750                  0.08902       0  \n2          0.3613                  0.08758       0  \n3          0.6638                  0.17300       0  \n4          0.2364                  0.07678       0  \n\n[5 rows x 31 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>mean radius</th>\n      <th>mean texture</th>\n      <th>mean perimeter</th>\n      <th>mean area</th>\n      <th>mean smoothness</th>\n      <th>mean compactness</th>\n      <th>mean concavity</th>\n      <th>mean concave points</th>\n      <th>mean symmetry</th>\n      <th>mean fractal dimension</th>\n      <th>...</th>\n      <th>worst texture</th>\n      <th>worst perimeter</th>\n      <th>worst area</th>\n      <th>worst smoothness</th>\n      <th>worst compactness</th>\n      <th>worst concavity</th>\n      <th>worst concave points</th>\n      <th>worst symmetry</th>\n      <th>worst fractal dimension</th>\n      <th>target</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>17.99</td>\n      <td>10.38</td>\n      <td>122.80</td>\n      <td>1001.0</td>\n      <td>0.11840</td>\n      <td>0.27760</td>\n      <td>0.3001</td>\n      <td>0.14710</td>\n      <td>0.2419</td>\n      <td>0.07871</td>\n      <td>...</td>\n      <td>17.33</td>\n      <td>184.60</td>\n      <td>2019.0</td>\n      <td>0.1622</td>\n      <td>0.6656</td>\n      <td>0.7119</td>\n      <td>0.2654</td>\n      <td>0.4601</td>\n      <td>0.11890</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>20.57</td>\n      <td>17.77</td>\n      <td>132.90</td>\n      <td>1326.0</td>\n      <td>0.08474</td>\n      <td>0.07864</td>\n      <td>0.0869</td>\n      <td>0.07017</td>\n      <td>0.1812</td>\n      <td>0.05667</td>\n      <td>...</td>\n      <td>23.41</td>\n      <td>158.80</td>\n      <td>1956.0</td>\n      <td>0.1238</td>\n      <td>0.1866</td>\n      <td>0.2416</td>\n      <td>0.1860</td>\n      <td>0.2750</td>\n      <td>0.08902</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>19.69</td>\n      <td>21.25</td>\n      <td>130.00</td>\n      <td>1203.0</td>\n      <td>0.10960</td>\n      <td>0.15990</td>\n      <td>0.1974</td>\n      <td>0.12790</td>\n      <td>0.2069</td>\n      <td>0.05999</td>\n      <td>...</td>\n      <td>25.53</td>\n      <td>152.50</td>\n      <td>1709.0</td>\n      <td>0.1444</td>\n      <td>0.4245</td>\n      <td>0.4504</td>\n      <td>0.2430</td>\n      <td>0.3613</td>\n      <td>0.08758</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>11.42</td>\n      <td>20.38</td>\n      <td>77.58</td>\n      <td>386.1</td>\n      <td>0.14250</td>\n      <td>0.28390</td>\n      <td>0.2414</td>\n      <td>0.10520</td>\n      <td>0.2597</td>\n      <td>0.09744</td>\n      <td>...</td>\n      <td>26.50</td>\n      <td>98.87</td>\n      <td>567.7</td>\n      <td>0.2098</td>\n      <td>0.8663</td>\n      <td>0.6869</td>\n      <td>0.2575</td>\n      <td>0.6638</td>\n      <td>0.17300</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>20.29</td>\n      <td>14.34</td>\n      <td>135.10</td>\n      <td>1297.0</td>\n      <td>0.10030</td>\n      <td>0.13280</td>\n      <td>0.1980</td>\n      <td>0.10430</td>\n      <td>0.1809</td>\n      <td>0.05883</td>\n      <td>...</td>\n      <td>16.67</td>\n      <td>152.20</td>\n      <td>1575.0</td>\n      <td>0.1374</td>\n      <td>0.2050</td>\n      <td>0.4000</td>\n      <td>0.1625</td>\n      <td>0.2364</td>\n      <td>0.07678</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 31 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 108
    }
   ],
   "source": [
    "# loading the data set\n",
    "dataset = load_breast_cancer(as_frame=True)\n",
    "df= pd.DataFrame(data= dataset.data)\n",
    "df[\"target\"] = dataset.target\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Seperating to X and Y \n",
    "X = df.iloc[:, :-1]\n",
    "y = df.iloc[:, -1]\n",
    "\n",
    "# splitting training and test\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.75, shuffle=True, random_state=24) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "def param_init(X): \n",
    "    \"\"\"\n",
    "    Initialize parameters\n",
    "    __________________ \n",
    "    Input(s)\n",
    "    X: Training data\n",
    "    __________________\n",
    "    Output(s)\n",
    "    params: Dictionary containing coefficients\n",
    "    \"\"\"\n",
    "    params = {} # initialize dictionary \n",
    "    _, n_features = X.shape # shape of training data\n",
    "\n",
    "    # initializing coefficents to 0 \n",
    "    params[\"W\"] = np.zeros(n_features)\n",
    "    params[\"b\"] = 0\n",
    "    return params "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_z(X, W, b): \n",
    "    \"\"\"\n",
    "    Calculates Linear Function\n",
    "    __________________\n",
    "    Input(s)\n",
    "    X: Training data\n",
    "    W: Weight coefficients\n",
    "    b: bias coefficients\n",
    "    __________________\n",
    "    Output(s)\n",
    "    z: a Linear function\n",
    "    \"\"\"\n",
    "    z = np.dot(X, W) + b\n",
    "    return z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(z):\n",
    "    \"\"\"\n",
    "    Logit model\n",
    "    _________________\n",
    "    Input(s)\n",
    "    z: Linear model \n",
    "    _________________\n",
    "    Output(s)\n",
    "    g: Logit function applied to linear model\n",
    "    \"\"\"\n",
    "    g = 1 / (1 + np.exp(-z))\n",
    "    return g "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient_descent(X, y, params, lr, n_iter): \n",
    "    \"\"\"\n",
    "    Gradient descent to minimize cost function\n",
    "    __________________ \n",
    "    Input(s)\n",
    "    X: Training data\n",
    "    y: Labels\n",
    "    params: Dictionary contatining coefficients\n",
    "    lr: learning rate\n",
    "    __________________\n",
    "    Output(s)\n",
    "    params: Dictionary containing optimized coefficients\n",
    "    \"\"\"\n",
    "    W = params[\"W\"] \n",
    "    b = params[\"b\"]\n",
    "    m = X.shape[0] # number of training instances \n",
    "\n",
    "    for _ in range(n_iter): \n",
    "        # prediction with random weights\n",
    "        g = sigmoid(get_z(X, W, b))\n",
    "        # calculate the loss\n",
    "        loss = -1/m * np.sum(y * np.log(g)) + (1 - y) * np.log(1 - g)\n",
    "        # partial derivative of weights \n",
    "        dW = 1/m * np.dot(X.T, (g - y))\n",
    "        db = 1/m * np.sum(g - y)\n",
    "        # updates to coefficients\n",
    "        W -= lr * dW\n",
    "        b -= lr * db \n",
    "    \n",
    "    params[\"W\"] = W\n",
    "    params[\"b\"] = b\n",
    "    return params "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(X, y, lr=0.01, n_iter=1000):\n",
    "    \"\"\"\n",
    "    Train Linear Regression model with Gradient decent\n",
    "    __________________ \n",
    "    Input(s)\n",
    "    X: Training data\n",
    "    y: Labels\n",
    "    lr: learning rate\n",
    "    n_iter: Number of iterations \n",
    "    __________________\n",
    "    Output(s)\n",
    "    params: Dictionary containing optimized coefficients\n",
    "    \"\"\" \n",
    "    init_params = param_init(X)\n",
    "    params = gradient_descent(X, y, init_params, lr, n_iter)\n",
    "    return params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(X_test, params):\n",
    "    \"\"\"\n",
    "    Train Linear Regression model with Gradient decent\n",
    "    __________________ \n",
    "    Input(s)\n",
    "    X: Unseen data\n",
    "    params: Dictionary contianing optimized weights from training\n",
    "    __________________\n",
    "    Output(s)\n",
    "    prediction of model\n",
    "    \"\"\"  \n",
    "    z = np.dot(X_test, params[\"W\"]) + params[\"b\"]\n",
    "    y_pred = sigmoid(z) >= 0.5\n",
    "    return y_pred.astype(\"int\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = train(X_train, y_train) # train model\n",
    "y_pred = predict(X_test, params) # inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = LogisticRegression(C=0.01)\n",
    "lr.fit(X_train, y_train)\n",
    "sklearn_y_pred = lr.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "My Implementation: 0.9300699300699301\nSklearn Implementation: 0.9300699300699301\n"
    }
   ],
   "source": [
    "print(f\"My Implementation: {accuracy_score(y_test, y_pred)}\\nSklearn Implementation: {accuracy_score(y_test, sklearn_y_pred)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Object Oriented Programming (OOB)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LogReg(): \n",
    "    \"\"\"\n",
    "    Custom made Logistic Regression class\n",
    "    \"\"\"\n",
    "    def __init__(self, lr=0.01, n_iter= 1000): \n",
    "        self.lr = lr\n",
    "        self.n_iter = n_iter \n",
    "        self.params = {}\n",
    "    \n",
    "    def param_init(self, X_train): \n",
    "        \"\"\"\n",
    "        Initialize parameters for linear regression model\n",
    "        __________________ \n",
    "        Input(s)\n",
    "        X: Training data\n",
    "        \"\"\"\n",
    "        _, n_features = self.X.shape # shape of training data\n",
    "\n",
    "        # initializing coefficents to 0 \n",
    "        self.params[\"W\"] = np.zeros(n_features)\n",
    "        self.params[\"b\"] = 0\n",
    "        return self\n",
    "\n",
    "    def get_z(X, W, b): \n",
    "        \"\"\"\n",
    "        Calculates Linear Function\n",
    "        __________________\n",
    "        Input(s)\n",
    "        X: Training data\n",
    "        W: Weight coefficients\n",
    "        b: bias coefficients\n",
    "        __________________\n",
    "        Output(s)\n",
    "        z: a Linear function\n",
    "        \"\"\"\n",
    "        z = np.dot(X, W) + b\n",
    "        return z\n",
    "        \n",
    "    def sigmoid(z):\n",
    "        \"\"\"\n",
    "        Logit model\n",
    "        _________________\n",
    "        Input(s)\n",
    "        z: Linear model \n",
    "        _________________\n",
    "        Output(s)\n",
    "        g: Logit function applied to linear model\n",
    "        \"\"\"\n",
    "        g = 1 / (1 + np.exp(-z))\n",
    "        return g \n",
    "        \n",
    "\n",
    "    def gradient_descent(self, X_train, y_train): \n",
    "        \"\"\"\n",
    "        Gradient descent to minimize cost function\n",
    "        __________________ \n",
    "        Input(s)\n",
    "        X: Training data\n",
    "        y: Labels\n",
    "        params: Dictionary contatining random coefficients\n",
    "        alpha: Model learning rate\n",
    "        __________________\n",
    "        Output(s)\n",
    "        params: Dictionary containing optimized coefficients\n",
    "        \"\"\"\n",
    "        W = self.params[\"W\"] \n",
    "        b = self.params[\"b\"] \n",
    "        m = X_train.shape[0]\n",
    "\n",
    "        for _ in range(self.n_iter): \n",
    "            # prediction with random weights\n",
    "            g = sigmoid(get_z(X, W, b))\n",
    "            # calculate the loss\n",
    "            loss = -1/m * np.sum(y * np.log(g)) + (1 - y) * np.log(1 - g)\n",
    "            # partial derivative of weights \n",
    "            dW = 1/m * np.dot(X.T, (g - y))\n",
    "            db = 1/m * np.sum(g - y)\n",
    "            # updates to coefficients\n",
    "            W -= self.lr * dW\n",
    "            b -= self.lr * db \n",
    "        \n",
    "        self.params[\"W\"] = W\n",
    "        self.params[\"b\"] = b\n",
    "        return self\n",
    "\n",
    "    def train(self, X_train, y_train):\n",
    "        \"\"\"\n",
    "        Train Linear Regression model with Gradient decent\n",
    "        __________________ \n",
    "        Input(s)\n",
    "        X: Training data\n",
    "        y: Labels\n",
    "        alpha: Model learning rate\n",
    "        n_iter: Number of iterations \n",
    "        __________________\n",
    "        Output(s)\n",
    "        params: Dictionary containing optimized coefficients\n",
    "        \"\"\" \n",
    "        self.params = param_init(X_train)\n",
    "        gradient_descent(X_train, y_train, self.params , self.lr, self.n_iter)\n",
    "        return self \n",
    "\n",
    "    def predict(self, X_test):\n",
    "        \"\"\"\n",
    "        Train Linear Regression model with Gradient decent\n",
    "        __________________ \n",
    "        Input(s)\n",
    "        X: Unseen data\n",
    "        params: Dictionary contianing optimized weights from training\n",
    "        __________________\n",
    "        Output(s)\n",
    "        y_preds: Predictions of model\n",
    "        \"\"\"  \n",
    "        g = sigmoid(np.dot(X_test, self.params[\"W\"]) + self.params[\"b\"])\n",
    "        return g "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg = LogReg()\n",
    "logreg.train(X_train, y_train)\n",
    "oop_y_pred = logreg.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "array([ True,  True,  True,  True,  True,  True,  True,  True,  True,\n        True,  True,  True,  True,  True,  True,  True,  True,  True,\n        True,  True,  True,  True,  True,  True,  True,  True,  True,\n        True,  True,  True,  True,  True,  True,  True,  True,  True,\n        True,  True,  True,  True,  True,  True,  True,  True,  True,\n        True,  True,  True,  True,  True,  True,  True,  True,  True,\n        True,  True,  True,  True,  True,  True,  True,  True,  True,\n        True,  True,  True,  True,  True,  True,  True,  True,  True,\n        True,  True,  True,  True,  True,  True,  True,  True,  True,\n        True,  True,  True,  True,  True,  True,  True,  True,  True,\n        True,  True,  True,  True,  True,  True,  True,  True,  True,\n        True,  True,  True,  True,  True,  True,  True,  True,  True,\n        True,  True,  True,  True,  True,  True,  True,  True,  True,\n        True,  True,  True,  True,  True,  True,  True,  True,  True,\n        True,  True,  True,  True,  True,  True,  True,  True,  True,\n        True,  True,  True,  True,  True,  True,  True,  True])"
     },
     "metadata": {},
     "execution_count": 98
    }
   ],
   "source": [
    "oop_y_pred == y_preds "
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python38364bitcondab8eeb421859c4b2b97a8c102771f8a36",
   "display_name": "Python 3.8.3 64-bit (conda)"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}