Hypothesis_Testing_for_Evaluating_ML_Classifier.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Hypothesis_Testing_for_Evaluating_ML_Classifier.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "Aflqi7lkJYy6"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv('https://raw.githubusercontent.com/venky14/Machine-Learning-with-Iris-Dataset/master/Iris.csv')\n",
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 468
        },
        "id": "DakNaR8iRLBG",
        "outputId": "95090712-55a5-4390-cfc5-b70e41fb2c13"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "      Id  SepalLengthCm  SepalWidthCm  PetalLengthCm  PetalWidthCm  \\\n",
              "0      1            5.1           3.5            1.4           0.2   \n",
              "1      2            4.9           3.0            1.4           0.2   \n",
              "2      3            4.7           3.2            1.3           0.2   \n",
              "3      4            4.6           3.1            1.5           0.2   \n",
              "4      5            5.0           3.6            1.4           0.2   \n",
              "..   ...            ...           ...            ...           ...   \n",
              "145  146            6.7           3.0            5.2           2.3   \n",
              "146  147            6.3           2.5            5.0           1.9   \n",
              "147  148            6.5           3.0            5.2           2.0   \n",
              "148  149            6.2           3.4            5.4           2.3   \n",
              "149  150            5.9           3.0            5.1           1.8   \n",
              "\n",
              "            Species  \n",
              "0       Iris-setosa  \n",
              "1       Iris-setosa  \n",
              "2       Iris-setosa  \n",
              "3       Iris-setosa  \n",
              "4       Iris-setosa  \n",
              "..              ...  \n",
              "145  Iris-virginica  \n",
              "146  Iris-virginica  \n",
              "147  Iris-virginica  \n",
              "148  Iris-virginica  \n",
              "149  Iris-virginica  \n",
              "\n",
              "[150 rows x 6 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-92884d95-c199-4567-bf93-93d1e8229118\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Id</th>\n",
              "      <th>SepalLengthCm</th>\n",
              "      <th>SepalWidthCm</th>\n",
              "      <th>PetalLengthCm</th>\n",
              "      <th>PetalWidthCm</th>\n",
              "      <th>Species</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>5.1</td>\n",
              "      <td>3.5</td>\n",
              "      <td>1.4</td>\n",
              "      <td>0.2</td>\n",
              "      <td>Iris-setosa</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>4.9</td>\n",
              "      <td>3.0</td>\n",
              "      <td>1.4</td>\n",
              "      <td>0.2</td>\n",
              "      <td>Iris-setosa</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>4.7</td>\n",
              "      <td>3.2</td>\n",
              "      <td>1.3</td>\n",
              "      <td>0.2</td>\n",
              "      <td>Iris-setosa</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>4.6</td>\n",
              "      <td>3.1</td>\n",
              "      <td>1.5</td>\n",
              "      <td>0.2</td>\n",
              "      <td>Iris-setosa</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>5.0</td>\n",
              "      <td>3.6</td>\n",
              "      <td>1.4</td>\n",
              "      <td>0.2</td>\n",
              "      <td>Iris-setosa</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>145</th>\n",
              "      <td>146</td>\n",
              "      <td>6.7</td>\n",
              "      <td>3.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>2.3</td>\n",
              "      <td>Iris-virginica</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>146</th>\n",
              "      <td>147</td>\n",
              "      <td>6.3</td>\n",
              "      <td>2.5</td>\n",
              "      <td>5.0</td>\n",
              "      <td>1.9</td>\n",
              "      <td>Iris-virginica</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>147</th>\n",
              "      <td>148</td>\n",
              "      <td>6.5</td>\n",
              "      <td>3.0</td>\n",
              "      <td>5.2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>Iris-virginica</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>148</th>\n",
              "      <td>149</td>\n",
              "      <td>6.2</td>\n",
              "      <td>3.4</td>\n",
              "      <td>5.4</td>\n",
              "      <td>2.3</td>\n",
              "      <td>Iris-virginica</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>149</th>\n",
              "      <td>150</td>\n",
              "      <td>5.9</td>\n",
              "      <td>3.0</td>\n",
              "      <td>5.1</td>\n",
              "      <td>1.8</td>\n",
              "      <td>Iris-virginica</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>150 rows × 6 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-92884d95-c199-4567-bf93-93d1e8229118')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-92884d95-c199-4567-bf93-93d1e8229118 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-92884d95-c199-4567-bf93-93d1e8229118');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.svm import SVC\n",
        "\n",
        "from sklearn.model_selection import RepeatedStratifiedKFold\n",
        "from sklearn.model_selection import cross_val_score"
      ],
      "metadata": {
        "id": "cwqjnYVxSNux"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X = df[[\"SepalLengthCm\",\"SepalWidthCm\",\"PetalLengthCm\",\"PetalWidthCm\"]].values\n",
        "y = df['Species'].values"
      ],
      "metadata": {
        "id": "eGmZWOCTT0L6"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## The Intuition behind 5x2 fold approach"
      ],
      "metadata": {
        "id": "4VSBTcNkWjeH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Spot-Check Algorithms\n",
        "models = []\n",
        "models.append(('LR', LogisticRegression(max_iter=1000))) \n",
        "models.append(('LDA', LinearDiscriminantAnalysis())) \n",
        "models.append(('KNN', KNeighborsClassifier()))\n",
        "models.append(('DSC', DecisionTreeClassifier(random_state = 1, max_depth=2)))\n",
        "models.append(('SVM', SVC()))\n",
        "# evaluate each model in turn\n",
        "results = []\n",
        "names = []\n",
        "for name, model in models:\n",
        "    kfold = RepeatedStratifiedKFold(n_splits=10, n_repeats = 3, random_state=1)\n",
        "    cv_results = cross_val_score(model, X, y, cv=kfold, scoring='accuracy') \n",
        "    results.append(cv_results)\n",
        "    names.append(name)\n",
        "    msg = \"%s: %.2f (%.3f)\" % (name, cv_results.mean(), cv_results.std())\n",
        "    print(msg)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Lo9dCjHMRO_U",
        "outputId": "a2c74993-6c35-452b-c3e7-f4cfe4c26c1c"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "LR: 0.96 (0.041)\n",
            "LDA: 0.98 (0.031)\n",
            "KNN: 0.96 (0.037)\n",
            "DSC: 0.94 (0.051)\n",
            "SVM: 0.96 (0.045)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "It seems that LR, KNN and SVM **has same mean with slightly different standard deviation.** However, LDA shows a higher performance, in the other hands, DTC shows the lowest performance over the rest of algorithms. "
      ],
      "metadata": {
        "id": "AIqhu0hRUEzS"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Steps to Hypothesis Testing"
      ],
      "metadata": {
        "id": "SaV_A3fSVRam"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The first step would be to to state the null hypothesis statement.\n",
        "\n",
        "- H0 : Both models have the same performance on the dataset.\n",
        "- H1 : Both models doesn’t have the same performance on the dataset.\n",
        "\n",
        "Significance level is 0.05"
      ],
      "metadata": {
        "id": "rq1m-0fFVVa0"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "let’s assume a significance threshold of α=0.05 for rejecting the null hypothesis"
      ],
      "metadata": {
        "id": "FbcfE8uZVlmm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from statistics import mean, stdev\n",
        "# evaluate model 1\n",
        "model1 = LinearDiscriminantAnalysis()\n",
        "cv1 = RepeatedStratifiedKFold(n_splits = 10, n_repeats = 3, random_state = 1)\n",
        "scores1 = cross_val_score(model1, X, y, scoring = 'accuracy', cv = cv1, n_jobs = -1)\n",
        "print('LDA Mean Accuracy: %.1f%% +/-(%.3f)' % (mean(scores1*100), stdev(scores1)))\n",
        "\n",
        "# evaluate model 2\n",
        "model2 = DecisionTreeClassifier(random_state = 1, max_depth=2)\n",
        "cv2 = RepeatedStratifiedKFold(n_splits = 10, n_repeats = 3, random_state = 1)\n",
        "scores2 = cross_val_score(model2, X, y, scoring = 'accuracy', cv = cv2, n_jobs = -1)\n",
        "print('DecisionTreeClassifier Mean Accuracy: %.1f%% +/-(%.3f)' % (mean(scores3*100), stdev(scores3)))\n",
        "\n",
        "# plot the results\n",
        "plt.boxplot([scores1, scores2], labels=['LDA', 'DTC'], showmeans=True)\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "DQ2mX_pnVQCz",
        "outputId": "ef12426b-e561-48ef-bef5-0ed7e09fff40"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "LDA Mean Accuracy: 98.0% +/-(0.031)\n",
            "DecisionTreeClassifier Mean Accuracy: 93.8% +/-(0.052)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAW40lEQVR4nO3df5CV1X3H8fcnKJpWMSibjGVRSEImEE1Jc4uZlkSNo0Gaij/SxI1GyVBpmuIfiWaKg62Ulkk7Y2PHqZViMQQnQhjT6M7EhPyCdOhoy0V+KDLoilPZxSnXEZN0tLq43/5xz9LH68I+hLt7d/d8XjN39nnOc86555m9cz/3+XHvUURgZmb5eUerB2BmZq3hADAzy5QDwMwsUw4AM7NMOQDMzDJ1UqsHcDwmTZoUU6dObfUwzMxGlW3btr0UEW2N5aMqAKZOnUq1Wm31MMzMRhVJ/zVQuU8BmZllygFgZpYpB4CZWaYcAGZmmXIAmJllqlQASLpf0kFJTx1luyTdLalL0i5Jv1PYdqOkZ9PjxkL5RyU9mdrcLUknvjtmZlZW2SOANcDcY2y/HJieHouAewEknQncAVwAzAbukDQxtbkXuKnQ7lj9m5lZk5UKgIj4N+DlY1SZD6yNuseBd0k6G/gU8OOIeDkiDgE/BuambRMi4vGo/x71WuDKE9oTMzM7Ls36IthkYH9hvTuVHau8e4Dyt5G0iPpRBeecc06ThjsGLTtjmJ7nF8PzPDZmnHnmmRw6dGhIn2PixIm8/PKxPqPaQEb8N4EjYhWwCqBSqXj2mqPQX/2SoZ7cRxKxbEifwsagQ4cODctr045fs+4C6gGmFNbbU9mxytsHKDczs2HSrADoBG5IdwN9DPhFRLwIbAQukzQxXfy9DNiYtv1S0sfS3T83AI80aSxmZlZCqVNAktYBFwGTJHVTv7PnZICIWAk8CswDuoBXgS+mbS9L+mtga+pqeUT0n6j7MvW7i94J/CA9zMxsmGg0TQpfqVTCvwY6MEnDcw1gFL1ebGTwa7P1JG2LiEpjub8JbGaWKQeAmVmmHABmZplyAJiZZcoBYGaWKQeAmVmmHABmZplyAJiZZcoBYGaWKQeAmVmmHABmZplyAGSq9mqNBT9cwEuvvdTqoZhZizgAMrVy10qe+O8nWLlzZauHYmYt4gDIUO3VGo90PUIQPNz1sI8CzDLlAMjQyl0r6Ys+APqiz0cBZplyAGSm/9N/b18vAL19vT4KMMtUqQCQNFfSXkldkpYMsP1cST+VtEvSZkntqfxiSTsKj/+VdGXatkbS84Vts5q7azaQ4qf/fj4KMMvToFNCShoH3ANcCnQDWyV1RsTThWp3Amsj4luSPgl8HfhCRGwCZqV+zqQ+ZeSPCu2+FhEPNWdXrIydB3ce+fTfr7evlx0Hd7RoRGbWKmXmBJ4NdEXEPgBJ64H5QDEAZgJfTcubgIcH6OczwA8i4tVff7h2oh66wnlrZnVlTgFNBvYX1rtTWdFO4Oq0fBVwuqSzGupcC6xrKFuRThvdJemUgZ5c0iJJVUnVWq1WYrhmZlZGsy4C3wpcKGk7cCHQA7zZv1HS2cD5wMZCm9uADwK/C5wJ/PlAHUfEqoioRESlra2tScM1M7Myp4B6gCmF9fZUdkREHCAdAUg6DbgmIl4pVPks8L2I6C20eTEtvi7pm9RDxMzMhkmZI4CtwHRJ0ySNp34qp7NYQdIkSf193Qbc39BHBw2nf9JRAZIEXAk8dfzDNzOzX9egARARh4HF1E/f7AE2RMRuScslXZGqXQTslfQM8B5gRX97SVOpH0H8vKHrb0t6EngSmAT8zQntiZmZHRdFRKvHUFqlUolqtdrqYYxIkhjq/+VwPIeNPX5ttp6kbRFRaSz3N4HNzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMlQoASXMl7ZXUJWnJANvPlfRTSbskbZbUXtj2pqQd6dFZKJ8m6T9Sn99J002amdkwGTQAJI0D7gEuB2YCHZJmNlS7E1gbER8GlgNfL2x7LSJmpccVhfK/A+6KiPcDh4CFJ7AfZmZ2nMocAcwGuiJiX0S8AawH5jfUmQn8LC1vGmD7W6SJ4D8JPJSKvkV9YngzMxsmZQJgMrC/sN6dyop2Alen5auA0yWdldZPlVSV9Lik/jf5s4BX0oTzR+sTAEmLUvtqrVYrMVwzMyujWReBbwUulLQduBDoAd5M285NkxF/HvgHSe87no4jYlVEVCKi0tbW1qThmpnZSSXq9ABTCuvtqeyIiDhAOgKQdBpwTUS8krb1pL/7JG0GPgJ8F3iXpJPSUcDb+jQzs6FV5ghgKzA93bUzHrgW6CxWkDRJUn9ftwH3p/KJkk7prwP8PvB0RAT1awWfSW1uBB450Z0xM7PyBg2A9Al9MbAR2ANsiIjdkpZL6r+r5yJgr6RngPcAK1L5DKAqaSf1N/y/jYin07Y/B74qqYv6NYHVTdonMzMrQfUP46NDpVKJarXa6mGMSJIY6v/lcDyHjT1+bbaepG3pWuxb+JvAZmaZcgCYmWXKAWBmlikHgJmNKLVXayz44QJeeu2lVg9lzHMAmNmIsnLXSp747ydYuXNlq4cy5jkAzGzEqL1a45GuRwiCh7se9lHAEHMAmNmIsXLXSvqiD4C+6PNRwBBzAJjZiND/6b+3rxeA3r5eHwUMMQeAmY0IxU///XwUMLQcAGY2Iuw8uPPIp/9+vX297Di4o0UjGvvK/BqojRL1eXaGzsSJE4e0fxub4o4JsOyMQes9dLQNz78ATxy7fdwx4fgHZg6AscK/g2Ij1rJfHHcT/7bP8PApIDOzTDkAzMwy5QAwM8tUqQCQNFfSXkldkpYMsP1cST+VtEvSZkntqXyWpMck7U7bPldos0bS85J2pMes5u2WmZkNZtAAkDQOuAe4HJgJdEia2VDtTmBtRHwYWA58PZW/CtwQER8C5lKfFP5dhXZfi4hZ6eF7vczMhlGZI4DZQFdE7IuIN4D1wPyGOjOBn6XlTf3bI+KZiHg2LR8ADgJtzRi4mZmdmDIBMBnYX1jvTmVFO4Gr0/JVwOmSzipWkDQbGA88VyhekU4N3dU/eXwjSYskVSVVa7VaieGamVkZzboIfCtwoaTtwIVAD/Bm/0ZJZwMPAF+MOPJd79uADwK/C5xJfZL4t4mIVRFRiYhKW5sPHszMmqXMF8F6gCmF9fZUdkQ6vXM1gKTTgGsi4pW0PgH4PrA0Ih4vtHkxLb4u6ZvUQ8TMzIZJmSOArcB0SdMkjQeuBTqLFSRNktTf123A/al8PPA96heIH2poc3b6K+BK4KkT2REzMzs+gwZARBwGFgMbgT3AhojYLWm5pCtStYuAvZKeAd4DrEjlnwU+ASwY4HbPb0t6EngSmAT8TbN2yszMBqfR9HsblUolqtVqq4dhZkPMvwXUXJK2RUSlsdzfBDYzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDJVKgAkzZW0V1KXpCUDbD9X0k8l7ZK0WVJ7YduNkp5NjxsL5R+V9GTq8+40NaSZmQ2TQQNA0jjgHuByYCbQIWlmQ7U7qc/7+2FgOfD11PZM4A7gAmA2cIekianNvcBNwPT0mHvCe2NmZqWVOQKYDXRFxL6IeANYD8xvqDMT+Fla3lTY/ingxxHxckQcAn4MzE0Twk+IiMejPu/bWuoTw5uZ2TApEwCTgf2F9e5UVrQTuDotXwWcLumsY7SdnJaP1ScAkhZJqkqq1mq1EsM1M7MymnUR+FbgQknbgQuBHuDNZnQcEasiohIRlba2tmZ0aWZmwEkl6vQAUwrr7ansiIg4QDoCkHQacE1EvCKpB7iooe3m1L69ofwtfZqZ2dAqcwSwFZguaZqk8cC1QGexgqRJkvr7ug24Py1vBC6TNDFd/L0M2BgRLwK/lPSxdPfPDcAjTdgfMzMradAAiIjDwGLqb+Z7gA0RsVvScklXpGoXAXslPQO8B1iR2r4M/DX1ENkKLE9lAF8G/gXoAp4DftCsnTIzs8GpfhPO6FCpVKJarbZ6GGY2xCQxmt6bRjpJ2yKi0ljubwKbmWXKAWBmlikHgJlZphwAZmaZcgCYmWXKAWBmlikHgJlZphwAZmaZcgCYmWXKAWBmlikHgJlZphwAZmaZcgCYmWXKAWBmlikHgJlZphwAZmaZKhUAkuZK2iupS9KSAbafI2mTpO2Sdkmal8qvk7Sj8OiTNCtt25z67N/27ubumpmZHcugk8JLGgfcA1wKdANbJXVGxNOFardTnyryXkkzgUeBqRHxbeDbqZ/zgYcjYkeh3XUR4Sm+zMxaoMwRwGygKyL2RcQbwHpgfkOdACak5TOAAwP005HampnZCFAmACYD+wvr3amsaBlwvaRu6p/+bx6gn88B6xrKvplO//yFJA305JIWSapKqtZqtRLDNTOzMpp1EbgDWBMR7cA84AFJR/qWdAHwakQ8VWhzXUScD3w8Pb4wUMcRsSoiKhFRaWtra9JwzcysTAD0AFMK6+2prGghsAEgIh4DTgUmFbZfS8On/4joSX9/BTxI/VSTmZkNkzIBsBWYLmmapPHU38w7G+q8AFwCIGkG9QCopfV3AJ+lcP5f0kmSJqXlk4FPA09hZmbDZtC7gCLisKTFwEZgHHB/ROyWtByoRkQncAtwn6SvUL8gvCAiInXxCWB/ROwrdHsKsDG9+Y8DfgLc17S9MjOzQen/36dHvkqlEtWq7xo1G+skMZrem0Y6SdsiotJY7m8Cm5llygFgZpapQa8BmJkNlaN8/eeY23xqqHkcAGbWMn4zby2fAjIzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMlQoASXMl7ZXUJWnJANvPkbRJ0nZJuyTNS+VTJb2WJn7fIWlloc1HJT2Z+rz7aJPCm5nZ0Bg0ACSNA+4BLgdmAh2SZjZUux3YEBEfoT5l5D8Vtj0XEbPS40uF8nuBm4Dp6TH3198NMzM7XmWOAGYDXRGxLyLeoD637/yGOgFMSMtnAAeO1aGks4EJEfF4mjpyLXDlcY3czMxOSJkAmAzsL6x3p7KiZcD1krqBR4GbC9umpVNDP5f08UKf3YP0CYCkRZKqkqq1Wq3EcM3MrIxmXQTuANZERDswD3hA0juAF4Fz0qmhrwIPSppwjH7eJiJWRUQlIiptbW1NGq6ZmZWZEKYHmFJYb09lRQtJ5/Aj4jFJpwKTIuIg8Hoq3ybpOeADqX37IH2amdkQKnMEsBWYLmmapPHUL/J2NtR5AbgEQNIM4FSgJqktXURG0nupX+zdFxEvAr+U9LF0988NwCNN2SMzMytl0COAiDgsaTGwERgH3B8RuyUtB6oR0QncAtwn6SvULwgviIiQ9AlguaReoA/4UkS8nLr+MrAGeCfwg/QwM7NhotE0J2elUolqtdrqYZiZjSqStkVEpbHc3wQ2M8uUA8DMLFMOADOzTDkAzMwy5QAwM8uUA8DMLFMOADOzTDkAzMwy5QAwM8uUA8DMLFMOADOzTDkAzMwy5QAwM8uUA8DMLFMOADOzTDkAzMwyVSoAJM2VtFdSl6QlA2w/R9ImSdsl7ZI0L5VfKmmbpCfT308W2mxOfe5Ij3c3b7fMzGwwg04Jmeb0vQe4FOgGtkrqjIinC9VuBzZExL2SZgKPAlOBl4A/jIgDks6jPq3k5EK76yLCU3yZmbVAmSOA2UBXROyLiDeA9cD8hjoBTEjLZwAHACJie0QcSOW7gXdKOuXEh21mZieqTABMBvYX1rt566d4gGXA9ZK6qX/6v3mAfq4BnoiI1wtl30ynf/5CkgZ6ckmLJFUlVWu1WonhmplZGc26CNwBrImIdmAe8ICkI31L+hDwd8CfFNpcFxHnAx9Pjy8M1HFErIqISkRU2tramjRcMzMrEwA9wJTCensqK1oIbACIiMeAU4FJAJLage8BN0TEc/0NIqIn/f0V8CD1U01mZjZMygTAVmC6pGmSxgPXAp0NdV4ALgGQNIN6ANQkvQv4PrAkIv69v7KkkyT1B8TJwKeBp050Z8zMrLxBAyAiDgOLqd/Bs4f63T67JS2XdEWqdgtwk6SdwDpgQUREavd+4C8bbvc8BdgoaRewg/oRxX3N3jkzMzs61d+nR4dKpRLVqu8aNTM7HpK2RUSlsdzfBDYzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDJVKgAkzZW0V1KXpCUDbD9H0iZJ2yXtkjSvsO221G6vpE+V7dPM8rNu3TrOO+88xo0bx3nnnce6detaPaQx7aTBKkgaB9wDXAp0A1sldUbE04Vqt1OfKvJeSTOBR4Gpafla4EPAbwE/kfSB1GawPs0sI+vWrWPp0qWsXr2aOXPmsGXLFhYuXAhAR0dHi0c3NpU5ApgNdEXEvoh4A1gPzG+oE8CEtHwGcCAtzwfWR8TrEfE80JX6K9OnmWVkxYoVrF69mosvvpiTTz6Ziy++mNWrV7NixYpWD23MKhMAk4H9hfXuVFa0DLheUjf1T/83D9K2TJ8ASFokqSqpWqvVSgzXzEajPXv2MGfOnLeUzZkzhz179rRoRGNfsy4CdwBrIqIdmAc8IKkpfUfEqoioRESlra2tGV2a2Qg0Y8YMtmzZ8payLVu2MGPGjBaNaOwr8ybdA0wprLensqKFwAaAiHgMOBWYdIy2Zfo0s4wsXbqUhQsXsmnTJnp7e9m0aRMLFy5k6dKlrR7amDXoRWBgKzBd0jTqb9LXAp9vqPMCcAmwRtIM6gFQAzqBByV9g/pF4OnAfwIq0aeZZaT/Qu/NN9/Mnj17mDFjBitWrPAF4CE0aABExGFJi4GNwDjg/ojYLWk5UI2ITuAW4D5JX6F+QXhBRASwW9IG4GngMPBnEfEmwEB9DsH+mdko0tHR4Tf8YaT6+/ToUKlUolqttnoYZmajiqRtEVFpLPc3gc3MMuUAMDPLlAPAzCxTDgAzs0yNqovAkmrAf7V6HGPIJOClVg/CbAB+bTbXuRHxtm/SjqoAsOaSVB3ozgCzVvNrc3j4FJCZWaYcAGZmmXIA5G1VqwdgdhR+bQ4DXwMwM8uUjwDMzDLlADAzy5QDYIyS9D8DlC2T1CNph6RnJf1rmre5WGeWpJA0d/hGazmR9GZ6De6WtFPSLZLeIelTqXyHpP+RtDctr5V0mqR/lvScpG2SNku6oNX7MtqVmQ/Axpa7IuJOAEmfA34m6fyI6J9vswPYkv7+sEVjtLHttYiYBSDp3cCDwISIuIP6T8QjaTNwa0RU0/p64HlgekT0pblEZg7UuZXnI4CMRcR3gB+RJuORJOCPgAXApZJObd3oLAcRcRBYBCxOr7+3kfQ+4ALg9ojoS+2ej4jvD99IxyYHgD0BfDAt/x7wfEQ8B2wG/qBVg7J8RMQ+6hNDvfsoVT4E7OifTMqaxwFgxU9dHcD6tLw+rZvZGOVrAPYRoCppHHANMF/SUurBcJak0yPiVy0doY1pkt4LvAkcPEqV3cBvSxrno4Dm8hFAxiRdA1wGrAMuAXZFxJSImBoR5wLfBa5q5RhtbJPUBqwE/jGO8q3UdEqyCvxV/3UCSVMl+RTlCfIRwNj1G5K6C+vfSH+/Iul64DeBp4BPRkRNUgfwvYY+vgv8KbB2yEdrOXmnpB3AycBh4AH+//V5NH8M/D3QJek16j8V/bUhHWUG/FMQZmaZ8ikgM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy9T/AbPQCcAQSixdAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Seems that **LDA** has a better performance over the DTCwhere LDAhas a higher accuracy."
      ],
      "metadata": {
        "id": "umBhpcdzWSdS"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 5 by 2 CV using MLxtend package"
      ],
      "metadata": {
        "id": "oHlhPvEKWq8Q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from mlxtend.evaluate import paired_ttest_5x2cv\n",
        "# check if difference between algorithms is real\n",
        "t, p = paired_ttest_5x2cv(estimator1=model1, \n",
        "                          estimator2=model2, \n",
        "                          X=X, \n",
        "                          y=y, \n",
        "                          scoring='accuracy', \n",
        "                          random_seed=1)\n",
        "# summarize\n",
        "print(f'The P-value is = {p:.3f}')\n",
        "print(f'The t-statistics is = {t:.3f}')\n",
        "# interpret the result\n",
        "if p <= 0.05:\n",
        "    print('Since p<0.05, We can reject the null-hypothesis that both models perform equally well on this dataset. We may conclude that the two algorithms are significantly different.')\n",
        "else:\n",
        "    print('Since p>0.05, we cannot reject the null hypothesis and may conclude that the performance of the two algorithms is not significantly different.')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cNSwoAvuVoyq",
        "outputId": "5e7fa200-bc45-4f1f-b583-c1737a6169da"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The P-value is = 0.027\n",
            "The t-statistics is = 3.101\n",
            "Since p<0.05, We can reject the null-hypothesis that both models perform equally well on this dataset. We may conclude that the two algorithms are significantly different.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now you have a strong argument about why picking LDA over the DTC."
      ],
      "metadata": {
        "id": "PWhyr5yDXYFq"
      }
    }
  ]
}