svm on mnist.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ssvakil/Machine-Learning-in-Python-Workshop/blob/master/svm%20on%20mnist.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "QkdOGoJkdl9m",
        "outputId": "8c90b201-2c58-496d-8f41-18dcaa6df7d2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
            "11490434/11490434 [==============================] - 0s 0us/step\n",
            "60000 train samples\n",
            "10000 test samples\n",
            "grid search\n",
            "grid.fit\n",
            "Fitting 5 folds for each of 8 candidates, totalling 40 fits\n",
            "[CV 1/5] END ......C=1, gamma=0.001, kernel=rbf;, score=0.509 total time=   9.8s\n",
            "[CV 2/5] END ......C=1, gamma=0.001, kernel=rbf;, score=0.509 total time=   4.3s\n",
            "[CV 3/5] END ......C=1, gamma=0.001, kernel=rbf;, score=0.509 total time=   4.4s\n",
            "[CV 4/5] END ......C=1, gamma=0.001, kernel=rbf;, score=0.509 total time=   4.9s\n",
            "[CV 5/5] END ......C=1, gamma=0.001, kernel=rbf;, score=0.508 total time=   4.2s\n",
            "[CV 1/5] END .....C=1, gamma=0.0001, kernel=rbf;, score=0.594 total time=   3.0s\n",
            "[CV 2/5] END .....C=1, gamma=0.0001, kernel=rbf;, score=0.594 total time=   4.1s\n",
            "[CV 3/5] END .....C=1, gamma=0.0001, kernel=rbf;, score=0.596 total time=   3.0s\n",
            "[CV 4/5] END .....C=1, gamma=0.0001, kernel=rbf;, score=0.594 total time=   3.0s\n",
            "[CV 5/5] END .....C=1, gamma=0.0001, kernel=rbf;, score=0.594 total time=   3.0s\n",
            "[CV 1/5] END .....C=10, gamma=0.001, kernel=rbf;, score=0.509 total time=   4.9s\n",
            "[CV 2/5] END .....C=10, gamma=0.001, kernel=rbf;, score=0.509 total time=   4.1s\n",
            "[CV 3/5] END .....C=10, gamma=0.001, kernel=rbf;, score=0.509 total time=   4.1s\n",
            "[CV 4/5] END .....C=10, gamma=0.001, kernel=rbf;, score=0.509 total time=   4.9s\n",
            "[CV 5/5] END .....C=10, gamma=0.001, kernel=rbf;, score=0.508 total time=   4.0s\n",
            "[CV 1/5] END ....C=10, gamma=0.0001, kernel=rbf;, score=0.599 total time=   2.8s\n",
            "[CV 2/5] END ....C=10, gamma=0.0001, kernel=rbf;, score=0.602 total time=   3.9s\n",
            "[CV 3/5] END ....C=10, gamma=0.0001, kernel=rbf;, score=0.606 total time=   3.0s\n",
            "[CV 4/5] END ....C=10, gamma=0.0001, kernel=rbf;, score=0.602 total time=   2.8s\n",
            "[CV 5/5] END ....C=10, gamma=0.0001, kernel=rbf;, score=0.601 total time=   2.8s\n",
            "[CV 1/5] END ....C=100, gamma=0.001, kernel=rbf;, score=0.509 total time=   5.2s\n",
            "[CV 2/5] END ....C=100, gamma=0.001, kernel=rbf;, score=0.509 total time=   4.1s\n",
            "[CV 3/5] END ....C=100, gamma=0.001, kernel=rbf;, score=0.509 total time=   4.1s\n",
            "[CV 4/5] END ....C=100, gamma=0.001, kernel=rbf;, score=0.509 total time=   5.7s\n",
            "[CV 5/5] END ....C=100, gamma=0.001, kernel=rbf;, score=0.508 total time=   4.3s\n",
            "[CV 1/5] END ...C=100, gamma=0.0001, kernel=rbf;, score=0.599 total time=   2.8s\n",
            "[CV 2/5] END ...C=100, gamma=0.0001, kernel=rbf;, score=0.602 total time=   3.7s\n",
            "[CV 3/5] END ...C=100, gamma=0.0001, kernel=rbf;, score=0.606 total time=   3.2s\n",
            "[CV 4/5] END ...C=100, gamma=0.0001, kernel=rbf;, score=0.602 total time=   2.8s\n",
            "[CV 5/5] END ...C=100, gamma=0.0001, kernel=rbf;, score=0.601 total time=   2.8s\n",
            "[CV 1/5] END ...C=1000, gamma=0.001, kernel=rbf;, score=0.509 total time=   5.3s\n",
            "[CV 2/5] END ...C=1000, gamma=0.001, kernel=rbf;, score=0.509 total time=   4.1s\n",
            "[CV 3/5] END ...C=1000, gamma=0.001, kernel=rbf;, score=0.509 total time=   4.1s\n",
            "[CV 4/5] END ...C=1000, gamma=0.001, kernel=rbf;, score=0.509 total time=   5.2s\n",
            "[CV 5/5] END ...C=1000, gamma=0.001, kernel=rbf;, score=0.508 total time=   4.1s\n",
            "[CV 1/5] END ..C=1000, gamma=0.0001, kernel=rbf;, score=0.599 total time=   2.8s\n",
            "[CV 2/5] END ..C=1000, gamma=0.0001, kernel=rbf;, score=0.602 total time=   2.8s\n",
            "[CV 3/5] END ..C=1000, gamma=0.0001, kernel=rbf;, score=0.606 total time=   3.9s\n",
            "[CV 4/5] END ..C=1000, gamma=0.0001, kernel=rbf;, score=0.602 total time=   2.8s\n",
            "[CV 5/5] END ..C=1000, gamma=0.0001, kernel=rbf;, score=0.601 total time=   2.8s\n",
            "grid done\n",
            "{'C': 10, 'gamma': 0.0001, 'kernel': 'rbf'}\n",
            "training svm\n",
            "svm done\n",
            "Testing\n",
            "score:  0.6286\n"
          ]
        }
      ],
      "source": [
        "from sklearn.model_selection import GridSearchCV\n",
        "\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.decomposition import PCA\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "#import dataset\n",
        "# the data, shuffled and split between train and test sets\n",
        "from keras.datasets import mnist\n",
        "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
        "\n",
        "#                         PREPROCESSING DATA\n",
        "\n",
        "# Change from matrix to array --> dimension 28x28 to array of dimention 784\n",
        "x_train = x_train.reshape(60000, 784)\n",
        "x_test = x_test.reshape(10000, 784)\n",
        "\n",
        "# Change to float datatype\n",
        "x_train = x_train.astype('float32')\n",
        "x_test = x_test.astype('float32')\n",
        "\n",
        "# Scale the data to lie between -1 to 1\n",
        "x_train = x_train / 255.0*100 - 50\n",
        "x_test = x_test / 255.0*100 - 50\n",
        "print(x_train.shape[0], 'train samples')\n",
        "print(x_test.shape[0], 'test samples')\n",
        "\n",
        "#I mod2 the label sets to have a result of 1 or 0 , for odds and evens respectively\n",
        "y_train =y_train % 2\n",
        "y_test =y_test % 2\n",
        "\n",
        "# PCA\n",
        "pca = PCA(n_components=50)\n",
        "x_train = pca.fit_transform(x_train)\n",
        "x_test = pca.transform(x_test)\n",
        "\n",
        "#                         GRID SEARCH FOR PARAMETER OPTIMIZING\n",
        "\n",
        "svm = SVC()\n",
        "parameters = [{'kernel': ['rbf'], 'gamma': [1e-3, 1e-4],\n",
        "                     'C': [1, 10, 100, 1000]}]\n",
        "                    #{'kernel': ['linear'], 'C': [1, 10, 100, 1000]}]\n",
        "print(\"grid search\")\n",
        "grid = GridSearchCV(svm, parameters, verbose=3)\n",
        "print(\"grid.fit\")\n",
        "grid.fit(x_train[0:7000], y_train[0:7000]) #grid search learning the best parameters\n",
        "print(\"grid done\")\n",
        "\n",
        "print (grid.best_params_)\n",
        "\n",
        "#Now we train the best estimator in the full dataset\n",
        "print(\"training svm\")\n",
        "best_svm = grid.best_estimator_\n",
        "best_svm.fit(x_train , y_train)\n",
        "print(\"svm done\")\n",
        "\n",
        "\n",
        "print(\"Testing\")\n",
        "print(\"score: \", best_svm.score(x_test, y_test,))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eo3snFkYd-dh"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vBnI1iPud-g1"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NYyzKkZwd-jv"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iXB-n49Bd-mg"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ipr4mdvGd-pH"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "g-7FRESed-rz"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QLQPUPMod-u3"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyOzHpCBjlrz83bAfMpr7AeI",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}