SupportVectorMachine.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyOHb/QGcqWTX0Gi959Y2gI6",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/solmvz/MLActivities/blob/main/SupportVectorMachine.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Support Vector Machine Implementation"
      ],
      "metadata": {
        "id": "sgIdmFQYEVBL"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "In this session I implemented SVM using the cost function hinge loss and stochastic gradient descent. In SVM, the objective is to find a hyperplane that separates +ve and -ve examples with the largest margin while keeping the misclassification as low as possible. The dataset that I'll be using is a breast cancer diagnostic dataset available on [kaggle](https://www.kaggle.com/datasets/uciml/breast-cancer-wisconsin-data). The features in the dataset are computed from a digitized image of a fine needle aspirate (FNA) of a breast mass. They describe the characteristics of the cell nuclei present in the image. Based on these features we will train our SVM model to detect if the mass is benign B (generally harmless) or malignant M (cancerous)."
      ],
      "metadata": {
        "id": "mkZ4M_O33y0D"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "irF_AoF50TyW"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import statsmodels.api as sm\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import accuracy_score, recall_score, precision_score\n",
        "from sklearn.utils import shuffle"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class SVM():\n",
        "  def __init__(self, X, y, reg=10000, lr=0.000001):\n",
        "    self.X = X\n",
        "    self.y = y\n",
        "    self.reg = reg\n",
        "    self.lr = lr\n",
        "\n",
        "  def compute_cost(self, X, y, W):\n",
        "    # Calculate Hinge Loss\n",
        "    N = X.shape[0]\n",
        "    distances = 1 - y*(np.dot(X, W))\n",
        "    distances[distances < 0] = 0\n",
        "    cost = 1/2*(np.dot(W, W)) + self.reg*(np.sum(distances)/N)\n",
        "    return cost\n",
        "\n",
        "  def compute_gradient(self, W, X_batch, y_batch):\n",
        "\n",
        "    y_batch = np.array([y_batch])\n",
        "    X_batch = np.array([X_batch])\n",
        "\n",
        "    distance = 1 - (y_batch*np.dot(X_batch,W))\n",
        "    dw = np.zeros(len(W))\n",
        "\n",
        "    for i, d in enumerate(distance):\n",
        "      if max(0, d) == 0:\n",
        "        di = W\n",
        "      else:\n",
        "        di = W - (self.reg*y_batch[i]*X_batch[i])\n",
        "      dw += di\n",
        "\n",
        "    dw = dw/len(Y)\n",
        "    return dw\n",
        "\n",
        "  def gradient_descent(self):\n",
        "    max_epochs = 5000\n",
        "    weights = np.zeros(self.X.shape[1])\n",
        "    for epoch in range(1, max_epochs):\n",
        "      for i, x in enumerate(self.X):\n",
        "        dw = self.compute_gradient(weights, x, self.y[i])\n",
        "        weights = weights - (self.lr*dw)\n",
        "    return weights\n"
      ],
      "metadata": {
        "id": "JRR5HOYa1Ain"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = pd.read_csv('/content/data.csv')\n",
        "diagnosis_map = {'M':1, 'B':-1}\n",
        "data['diagnosis'] = data['diagnosis'].map(diagnosis_map)\n",
        "data.drop(data.columns[[-1, 0]], axis=1, inplace=True)\n",
        "\n",
        "Y = data.loc[:, 'diagnosis']\n",
        "X = data.iloc[:, 1:]\n",
        "# Normalise Data\n",
        "X_norm = MinMaxScaler().fit_transform(X.values)\n",
        "X = pd.DataFrame(X_norm)\n",
        "\n",
        "# Insert 1 in each row for intercept b\n",
        "X.insert(loc=len(X.columns), column='intercept', value=1)\n",
        "# Split train and test \n",
        "X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.2, random_state=42)"
      ],
      "metadata": {
        "id": "0X8pavaL1QPi"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = SVM(X_train.to_numpy(), y_train.to_numpy())\n",
        "print(\"training started...\")\n",
        "W = model.gradient_descent()\n",
        "print(\"training finished.\")\n",
        "print(\"weights are: {}\".format(W))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VyMsCWrecQGU",
        "outputId": "4b371b81-cc91-4505-b7e0-798d68fb0949"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "training started...\n",
            "training finished.\n",
            "weights are: [ 0.42474349  0.22987926  0.47739123  0.60566123 -0.19579285  0.46226968\n",
            "  1.01518923  1.28085585 -0.14721568 -0.59429363  0.47925372 -0.30330219\n",
            "  0.39870367  0.40360628 -0.30615331 -0.21558165 -0.17821916 -0.20511514\n",
            " -0.26319069 -0.31715063  0.80118522  0.44786878  0.78480886  0.77019176\n",
            "  0.24604272  0.57781986  0.75480969  1.19301864  0.35981256  0.12602673\n",
            " -2.76282531]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y_pred = np.array([])\n",
        "for i in range(X_test.shape[0]):\n",
        "  y_p = np.sign(np.dot(W, X_test.to_numpy()[i]))\n",
        "  y_pred = np.append(y_pred, y_p)\n",
        "\n",
        "print(\"accuracy on test dataset: {}\".format(accuracy_score(y_test.to_numpy(), y_pred)))\n",
        "print(\"recall on test dataset: {}\".format(recall_score(y_test.to_numpy(), y_pred)))\n",
        "print(\"precision on test dataset: {}\".format(recall_score(y_test.to_numpy(), y_pred)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ScKUFgGkClBz",
        "outputId": "ed048720-a393-4cda-8f73-c690846268ba"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "accuracy on test dataset: 0.956140350877193\n",
            "recall on test dataset: 0.9069767441860465\n",
            "precision on test dataset: 0.9069767441860465\n"
          ]
        }
      ]
    }
  ]
}