ML Algorithms in Spark.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# **LAB 03 - Cài đặt Spark và thực hiện cài đặt các thuật toán máy học**\n"
      ],
      "metadata": {
        "id": "l13ddSL2RMIF"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **Cài đặt Spark**"
      ],
      "metadata": {
        "id": "8gDkfWXTTcUY"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W28Khy_kwuId"
      },
      "source": [
        "Cài đặt môi trường Java-8"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QeyAKBgwo05o"
      },
      "outputs": [],
      "source": [
        "# !apt-get install openjdk-8-jdk-headless -qq > /dev/null"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s5dxc2tOxK3x"
      },
      "source": [
        "Tải gói cài đặt Apache Spark (version 3.2.1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ehQVcFjFxZ4t"
      },
      "outputs": [],
      "source": [
        "# !wget -q https://dlcdn.apache.org/spark/spark-3.2.1/spark-3.2.1-bin-hadoop3.2.tgz"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KQ2NcnPfxgVL"
      },
      "source": [
        "Giải nén gói cài đặt Apache Spark"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VTRRiDNrp-s9"
      },
      "outputs": [],
      "source": [
        "# !tar xzf spark-3.2.1-bin-hadoop3.2.tgz\n",
        "# !pip install findspark"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nFJwlRWFy2jq"
      },
      "source": [
        "Đặt đường dẫn của môi trường Java và gói đặt Spark"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Un07WvpcqOyM"
      },
      "outputs": [],
      "source": [
        "# import os\n",
        "# os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64/\"\n",
        "# os.environ[\"SPARK_HOME\"] = \"/content/spark-3.2.1-bin-hadoop3.2\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oc7PSR-6zVMT"
      },
      "source": [
        "Cài đặt gói hỗ trợ Spark Python API - PySpark"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ktKgWqqirVGe"
      },
      "outputs": [],
      "source": [
        "# !pip install pyspark"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Import và khởi tạo Spark Session"
      ],
      "metadata": {
        "id": "6_i8FsGlVTtG"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1ywH5jP2yz89"
      },
      "outputs": [],
      "source": [
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.builder.master(\"local[*]\").getOrCreate()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Đọc tập dữ liệu Absenteeism at work từ link github hướng dẫn được cung cấp"
      ],
      "metadata": {
        "id": "2WPdmnn-Umwo"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fU9_AUawtlaz"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "data = pd.read_csv('https://raw.githubusercontent.com/Ruthvicp/CS5590_BigDataProgramming/master/Lab/Lab4/Source/Absenteeism_at_work.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 540
        },
        "id": "oKQ20Iu4ukTd",
        "outputId": "1e3e788d-a59b-4d13-a96c-e573e3a7279a"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     ID  Reason for absence  Month of absence  Day of the week  Seasons  \\\n",
              "0    11                  26                 7                3        1   \n",
              "1    36                   0                 7                3        1   \n",
              "2     3                  23                 7                4        1   \n",
              "3     7                   7                 7                5        1   \n",
              "4    11                  23                 7                5        1   \n",
              "..   ..                 ...               ...              ...      ...   \n",
              "735  11                  14                 7                3        1   \n",
              "736   1                  11                 7                3        1   \n",
              "737   4                   0                 0                3        1   \n",
              "738   8                   0                 0                4        2   \n",
              "739  35                   0                 0                6        3   \n",
              "\n",
              "     Transportation expense  Distance from Residence to Work  Service time  \\\n",
              "0                       289                               36            13   \n",
              "1                       118                               13            18   \n",
              "2                       179                               51            18   \n",
              "3                       279                                5            14   \n",
              "4                       289                               36            13   \n",
              "..                      ...                              ...           ...   \n",
              "735                     289                               36            13   \n",
              "736                     235                               11            14   \n",
              "737                     118                               14            13   \n",
              "738                     231                               35            14   \n",
              "739                     179                               45            14   \n",
              "\n",
              "     Age  Work load Average/day   ...  Disciplinary_failure  Education  Son  \\\n",
              "0     33                 239.554  ...                     0          1    2   \n",
              "1     50                 239.554  ...                     1          1    1   \n",
              "2     38                 239.554  ...                     0          1    0   \n",
              "3     39                 239.554  ...                     0          1    2   \n",
              "4     33                 239.554  ...                     0          1    2   \n",
              "..   ...                     ...  ...                   ...        ...  ...   \n",
              "735   33                 264.604  ...                     0          1    2   \n",
              "736   37                 264.604  ...                     0          3    1   \n",
              "737   40                 271.219  ...                     0          1    1   \n",
              "738   39                 271.219  ...                     0          1    2   \n",
              "739   53                 271.219  ...                     0          1    1   \n",
              "\n",
              "     Social drinker  Social smoker  Pet  Weight  Height  Body mass index  \\\n",
              "0                 1              0    1      90     172               30   \n",
              "1                 1              0    0      98     178               31   \n",
              "2                 1              0    0      89     170               31   \n",
              "3                 1              1    0      68     168               24   \n",
              "4                 1              0    1      90     172               30   \n",
              "..              ...            ...  ...     ...     ...              ...   \n",
              "735               1              0    1      90     172               30   \n",
              "736               0              0    1      88     172               29   \n",
              "737               1              0    8      98     170               34   \n",
              "738               1              0    2     100     170               35   \n",
              "739               0              0    1      77     175               25   \n",
              "\n",
              "     Absenteeism_time_in_hours  \n",
              "0                            4  \n",
              "1                            0  \n",
              "2                            2  \n",
              "3                            4  \n",
              "4                            2  \n",
              "..                         ...  \n",
              "735                          8  \n",
              "736                          4  \n",
              "737                          0  \n",
              "738                          0  \n",
              "739                          0  \n",
              "\n",
              "[740 rows x 21 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-39ce320c-348f-4cfc-909c-6c9dd594439d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ID</th>\n",
              "      <th>Reason for absence</th>\n",
              "      <th>Month of absence</th>\n",
              "      <th>Day of the week</th>\n",
              "      <th>Seasons</th>\n",
              "      <th>Transportation expense</th>\n",
              "      <th>Distance from Residence to Work</th>\n",
              "      <th>Service time</th>\n",
              "      <th>Age</th>\n",
              "      <th>Work load Average/day</th>\n",
              "      <th>...</th>\n",
              "      <th>Disciplinary_failure</th>\n",
              "      <th>Education</th>\n",
              "      <th>Son</th>\n",
              "      <th>Social drinker</th>\n",
              "      <th>Social smoker</th>\n",
              "      <th>Pet</th>\n",
              "      <th>Weight</th>\n",
              "      <th>Height</th>\n",
              "      <th>Body mass index</th>\n",
              "      <th>Absenteeism_time_in_hours</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>11</td>\n",
              "      <td>26</td>\n",
              "      <td>7</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>289</td>\n",
              "      <td>36</td>\n",
              "      <td>13</td>\n",
              "      <td>33</td>\n",
              "      <td>239.554</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>90</td>\n",
              "      <td>172</td>\n",
              "      <td>30</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>36</td>\n",
              "      <td>0</td>\n",
              "      <td>7</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>118</td>\n",
              "      <td>13</td>\n",
              "      <td>18</td>\n",
              "      <td>50</td>\n",
              "      <td>239.554</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>98</td>\n",
              "      <td>178</td>\n",
              "      <td>31</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>23</td>\n",
              "      <td>7</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>179</td>\n",
              "      <td>51</td>\n",
              "      <td>18</td>\n",
              "      <td>38</td>\n",
              "      <td>239.554</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>89</td>\n",
              "      <td>170</td>\n",
              "      <td>31</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>279</td>\n",
              "      <td>5</td>\n",
              "      <td>14</td>\n",
              "      <td>39</td>\n",
              "      <td>239.554</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>68</td>\n",
              "      <td>168</td>\n",
              "      <td>24</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>11</td>\n",
              "      <td>23</td>\n",
              "      <td>7</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>289</td>\n",
              "      <td>36</td>\n",
              "      <td>13</td>\n",
              "      <td>33</td>\n",
              "      <td>239.554</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>90</td>\n",
              "      <td>172</td>\n",
              "      <td>30</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>735</th>\n",
              "      <td>11</td>\n",
              "      <td>14</td>\n",
              "      <td>7</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>289</td>\n",
              "      <td>36</td>\n",
              "      <td>13</td>\n",
              "      <td>33</td>\n",
              "      <td>264.604</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>90</td>\n",
              "      <td>172</td>\n",
              "      <td>30</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>736</th>\n",
              "      <td>1</td>\n",
              "      <td>11</td>\n",
              "      <td>7</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>235</td>\n",
              "      <td>11</td>\n",
              "      <td>14</td>\n",
              "      <td>37</td>\n",
              "      <td>264.604</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>88</td>\n",
              "      <td>172</td>\n",
              "      <td>29</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>737</th>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>118</td>\n",
              "      <td>14</td>\n",
              "      <td>13</td>\n",
              "      <td>40</td>\n",
              "      <td>271.219</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>8</td>\n",
              "      <td>98</td>\n",
              "      <td>170</td>\n",
              "      <td>34</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>738</th>\n",
              "      <td>8</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "      <td>231</td>\n",
              "      <td>35</td>\n",
              "      <td>14</td>\n",
              "      <td>39</td>\n",
              "      <td>271.219</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>100</td>\n",
              "      <td>170</td>\n",
              "      <td>35</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>739</th>\n",
              "      <td>35</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>3</td>\n",
              "      <td>179</td>\n",
              "      <td>45</td>\n",
              "      <td>14</td>\n",
              "      <td>53</td>\n",
              "      <td>271.219</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>77</td>\n",
              "      <td>175</td>\n",
              "      <td>25</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>740 rows × 21 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-39ce320c-348f-4cfc-909c-6c9dd594439d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-39ce320c-348f-4cfc-909c-6c9dd594439d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-39ce320c-348f-4cfc-909c-6c9dd594439d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ],
      "source": [
        "data"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Tạo một Spark dataframe từ pandas dataframe đã tạo"
      ],
      "metadata": {
        "id": "IlJGPAuZVfgo"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sjNoTd8LvFhU"
      },
      "outputs": [],
      "source": [
        "data = spark.createDataFrame(data)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Lưu ý:** Các bước cài đặt Spark cần được chạy một lần sau khi truy cập Google Colab để cài đặt và khởi tạo Spark trên Google Colab, các lần chạy sau cần #comment các cell cài đặt để tránh tải lại các gói cài đặt Spark."
      ],
      "metadata": {
        "id": "KUESIE9TXvX2"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **Chạy các thuật toán máy học**\n",
        "Sử dụng tập dữ liệu sẳn có trong đường dẫn github đã cung cấp"
      ],
      "metadata": {
        "id": "4CwImrd3Wokv"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Un6SFytS0g6s"
      },
      "source": [
        "### Decision Tree"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GilOXUbYsBaF",
        "outputId": "a033ccc7-ff05-4332-97b9-b53c7788c4a9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------+------------+------------+\n",
            "|prediction|indexedLabel|    features|\n",
            "+----------+------------+------------+\n",
            "|       1.0|         1.0|[172.0,11.0]|\n",
            "|       1.0|         1.0|[172.0,11.0]|\n",
            "|       1.0|         1.0|[172.0,11.0]|\n",
            "|       1.0|         1.0|[172.0,11.0]|\n",
            "|       1.0|         1.0|[172.0,11.0]|\n",
            "+----------+------------+------------+\n",
            "only showing top 5 rows\n",
            "\n",
            "DecisionTreeClassificationModel: uid=DecisionTreeClassifier_46f71cf64313, depth=5, numNodes=23, numClasses=14, numFeatures=2\n",
            "Decision Tree - Test Accuracy = 0.966527\n",
            "Decision Tree - Test Error = 0.0334728\n",
            "The Confusion Matrix for Decision Tree Model is :\n",
            "[[0 0 0 ... 0 0 0]\n",
            " [0 0 0 ... 0 0 0]\n",
            " [0 0 0 ... 0 0 0]\n",
            " ...\n",
            " [2 0 0 ... 0 0 0]\n",
            " [0 0 0 ... 0 0 0]\n",
            " [5 0 0 ... 0 0 0]]\n",
            "The precision score for Decision Tree Model is: 0.02972972972972973\n",
            "The recall score for Decision Tree Model is: 0.02972972972972973\n"
          ]
        }
      ],
      "source": [
        "from pyspark.ml import Pipeline\n",
        "from pyspark.ml.classification import DecisionTreeClassifier\n",
        "from pyspark.ml.feature import StringIndexer, VectorIndexer, VectorAssembler\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import precision_score\n",
        "from sklearn.metrics import recall_score\n",
        "import scipy\n",
        "\n",
        "from pyspark.python.pyspark.shell import spark\n",
        "\n",
        "data1 = data.withColumn(\"MOA\", data[\"Month of absence\"] - 0).withColumn(\"label\", data['Height'] - 0). \\\n",
        "    withColumn(\"ROA\", data[\"Reason for absence\"] - 0). \\\n",
        "    withColumn(\"distance\", data[\"Distance from Residence to Work\"] - 0). \\\n",
        "    withColumn(\"BMI\", data[\"Body mass index\"] - 0)\n",
        "\n",
        "assem = VectorAssembler(inputCols=[\"label\", \"distance\"], outputCol='features')\n",
        "data1 = assem.transform(data1)\n",
        "\n",
        "# Index labels, adding metadata to the label column.\n",
        "# Fit on whole dataset to include all labels in index.\n",
        "labelIndexer = StringIndexer(inputCol=\"label\", outputCol=\"indexedLabel\").fit(data1)\n",
        "# Automatically identify categorical features, and index them.\n",
        "# We specify maxCategories so features with > 4 distinct values are treated as continuous.\n",
        "featureIndexer =\\\n",
        "    VectorIndexer(inputCol=\"features\", outputCol=\"indexedFeatures\", maxCategories=4).fit(data1)\n",
        "\n",
        "# Split the data into training and test sets (30% held out for testing)\n",
        "(trainingData, testData) = data1.randomSplit([0.7, 0.3])\n",
        "\n",
        "# Train a DecisionTree model.\n",
        "dt = DecisionTreeClassifier(labelCol=\"indexedLabel\", featuresCol=\"indexedFeatures\")\n",
        "\n",
        "# Chain indexers and tree in a Pipeline\n",
        "pipeline = Pipeline(stages=[labelIndexer, featureIndexer, dt])\n",
        "\n",
        "# Train model.  This also runs the indexers.\n",
        "model = pipeline.fit(trainingData)\n",
        "\n",
        "# Make predictions.\n",
        "predictions = model.transform(testData)\n",
        "\n",
        "# Select example rows to display.\n",
        "predictions.select(\"prediction\", \"indexedLabel\", \"features\").show(5)\n",
        "\n",
        "# Select (prediction, true label) and compute test error\n",
        "evaluator = MulticlassClassificationEvaluator(\n",
        "    labelCol=\"indexedLabel\", predictionCol=\"prediction\", metricName=\"accuracy\")\n",
        "\n",
        "accuracy = evaluator.evaluate(predictions)\n",
        "\n",
        "y_true = data1.select(\"BMI\").rdd.flatMap(lambda x: x).collect()\n",
        "y_pred = data1.select(\"ROA\").rdd.flatMap(lambda x: x).collect()\n",
        "\n",
        "confusionmatrix = confusion_matrix(y_true, y_pred)\n",
        "\n",
        "precision = precision_score(y_true, y_pred, average='micro')\n",
        "\n",
        "recall = recall_score(y_true, y_pred, average='micro')\n",
        "\n",
        "treeModel = model.stages[2]\n",
        "# summary only\n",
        "print(treeModel)\n",
        "print(\"Decision Tree - Test Accuracy = %g\" % (accuracy))\n",
        "print(\"Decision Tree - Test Error = %g\" % (1.0 - accuracy))\n",
        "\n",
        "print(\"The Confusion Matrix for Decision Tree Model is :\\n\" + str(confusionmatrix))\n",
        "\n",
        "print(\"The precision score for Decision Tree Model is: \" + str(precision))\n",
        "\n",
        "print(\"The recall score for Decision Tree Model is: \" + str(recall))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7iXPG29g0rCh"
      },
      "source": [
        "### Naive Bayesian"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FxKDNGG205vh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8cdcf9db-62e0-4ecb-bdf1-ee21af5b97cb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Naive Bayes - Test set accuracy = 0.06756756756756757\n",
            "The Confusion Matrix for Naive Bayes Model is :\n",
            "[[0 0 0 ... 0 0 0]\n",
            " [0 0 0 ... 0 0 0]\n",
            " [0 0 0 ... 0 0 0]\n",
            " ...\n",
            " [2 0 0 ... 0 0 0]\n",
            " [0 0 0 ... 0 0 0]\n",
            " [5 0 0 ... 0 0 0]]\n",
            "The precision score for Naive Bayes Model is: 0.02972972972972973\n",
            "The recall score for Naive Bayes Model is: 0.02972972972972973\n"
          ]
        }
      ],
      "source": [
        "from pyspark.ml.feature import VectorAssembler\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import precision_score\n",
        "from sklearn.metrics import recall_score\n",
        "import scipy\n",
        "\n",
        "from pyspark.ml.classification import NaiveBayes\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "\n",
        "from pyspark.ml.linalg import SparseVector\n",
        "from pyspark.python.pyspark.shell import spark\n",
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.builder.getOrCreate()\n",
        "\n",
        "data2 = data.withColumn(\"MOA\", data[\"Month of absence\"] - 0).withColumn(\"label\", data['Seasons'] - 0). \\\n",
        "    withColumn(\"ROA\", data[\"Reason for absence\"] - 0). \\\n",
        "    withColumn(\"distance\", data[\"Distance from Residence to Work\"] - 0). \\\n",
        "    withColumn(\"BMI\", data[\"Body mass index\"] - 0)\n",
        "\n",
        "assem = VectorAssembler(inputCols=[\"label\", \"MOA\"], outputCol='features')\n",
        "\n",
        "data2 = assem.transform(data2)\n",
        "# Split the data into train and test\n",
        "splits = data2.randomSplit([0.7, 0.3], 1000)\n",
        "train = splits[0]\n",
        "test = splits[1]\n",
        "\n",
        "# create the trainer and set its parameters\n",
        "nb = NaiveBayes(smoothing=1.0, modelType=\"multinomial\")\n",
        "\n",
        "# train the model\n",
        "model = nb.fit(train)\n",
        "\n",
        "# select example rows to display.\n",
        "predictions = model.transform(test)\n",
        "\n",
        "# compute accuracy on the test set\n",
        "evaluator = MulticlassClassificationEvaluator(labelCol=\"label\", predictionCol=\"prediction\",\n",
        "                                              metricName=\"accuracy\")\n",
        "\n",
        "y_true = data2.select(\"BMI\").rdd.flatMap(lambda x: x).collect()\n",
        "y_pred = data2.select(\"ROA\").rdd.flatMap(lambda x: x).collect()\n",
        "\n",
        "\n",
        "accuracy = evaluator.evaluate(predictions)\n",
        "\n",
        "confusionmatrix = confusion_matrix(y_true, y_pred)\n",
        "\n",
        "precision = precision_score(y_true, y_pred, average='micro')\n",
        "\n",
        "recall = recall_score(y_true, y_pred, average='micro')\n",
        "\n",
        "\n",
        "print(\"Naive Bayes - Test set accuracy = \" + str(accuracy))\n",
        "\n",
        "print(\"The Confusion Matrix for Naive Bayes Model is :\\n\" + str(confusionmatrix))\n",
        "\n",
        "print(\"The precision score for Naive Bayes Model is: \" + str(precision))\n",
        "\n",
        "print(\"The recall score for Naive Bayes Model is: \" + str(recall))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4XcuXMeXwquo"
      },
      "source": [
        "### Random Forest"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SjuSsSEF06Oh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "72fe7b53-ee7d-4962-ae46-e7975697ed03"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------+-----+------------+\n",
            "|predictedLabel|label|    features|\n",
            "+--------------+-----+------------+\n",
            "|           172|  172|[172.0,11.0]|\n",
            "|           172|  172|[172.0,11.0]|\n",
            "|           172|  172|[172.0,11.0]|\n",
            "|           172|  172|[172.0,11.0]|\n",
            "|           172|  172|[172.0,11.0]|\n",
            "+--------------+-----+------------+\n",
            "only showing top 5 rows\n",
            "\n",
            "RandomForestClassificationModel: uid=RandomForestClassifier_54fba69b98d7, numTrees=10, numClasses=14, numFeatures=2\n",
            "Random Forest - Test Accuracy = 0.967442\n",
            "Random Forest - Test Error = 0.0325581\n",
            "The Confusion Matrix for Random Forest Model is :\n",
            "[[0 0 0 ... 0 0 0]\n",
            " [0 0 0 ... 0 0 0]\n",
            " [0 0 0 ... 0 0 0]\n",
            " ...\n",
            " [2 0 0 ... 0 0 0]\n",
            " [0 0 0 ... 0 0 0]\n",
            " [5 0 0 ... 0 0 0]]\n",
            "The precision score for Random Forest Model is: 0.02972972972972973\n",
            "The recall score for Random Forest Model is: 0.02972972972972973\n"
          ]
        }
      ],
      "source": [
        "\n",
        "from pyspark.ml import Pipeline\n",
        "from pyspark.ml.classification import RandomForestClassifier\n",
        "from pyspark.ml.feature import IndexToString, StringIndexer, VectorIndexer, VectorAssembler\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
        "\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import precision_score\n",
        "from sklearn.metrics import recall_score\n",
        "\n",
        "\n",
        "from pyspark.python.pyspark.shell import spark\n",
        "\n",
        "data3 = data.withColumn(\"MOA\", data[\"Month of absence\"] - 0).withColumn(\"label\", data['Height'] - 0). \\\n",
        "    withColumn(\"ROA\", data[\"Reason for absence\"] - 0). \\\n",
        "    withColumn(\"distance\", data[\"Distance from Residence to Work\"] - 0). \\\n",
        "    withColumn(\"BMI\", data[\"Body mass index\"] - 0)\n",
        "\n",
        "assem = VectorAssembler(inputCols=[\"label\", \"distance\"], outputCol='features')\n",
        "\n",
        "data3 = assem.transform(data3)\n",
        "\n",
        "labelIndexer = StringIndexer(inputCol=\"label\", outputCol=\"indexedLabel\").fit(data3)\n",
        "\n",
        "# Automatically identify categorical features, and index them.\n",
        "# Set maxCategories so features with > 4 distinct values are treated as continuous.\n",
        "featureIndexer =\\\n",
        "    VectorIndexer(inputCol=\"features\", outputCol=\"indexedFeatures\", maxCategories=4).fit(data3)\n",
        "\n",
        "# Split the data into training and test sets (30% held out for testing)\n",
        "(trainingData, testData) = data3.randomSplit([0.7, 0.3])\n",
        "\n",
        "# Train a RandomForest model.\n",
        "rf = RandomForestClassifier(labelCol=\"indexedLabel\", featuresCol=\"indexedFeatures\", numTrees=10)\n",
        "\n",
        "# Convert indexed labels back to original labels.\n",
        "labelConverter = IndexToString(inputCol=\"prediction\", outputCol=\"predictedLabel\",\n",
        "                               labels=labelIndexer.labels)\n",
        "\n",
        "y_true = data3.select(\"BMI\").rdd.flatMap(lambda x: x).collect()\n",
        "y_pred = data3.select(\"ROA\").rdd.flatMap(lambda x: x).collect()\n",
        "\n",
        "# Chain indexers and forest in a Pipeline\n",
        "pipeline = Pipeline(stages=[labelIndexer, featureIndexer, rf, labelConverter])\n",
        "\n",
        "# Train model.  This also runs the indexers.\n",
        "model = pipeline.fit(trainingData)\n",
        "\n",
        "# Make predictions.\n",
        "predictions = model.transform(testData)\n",
        "\n",
        "# Select example rows to display.\n",
        "predictions.select(\"predictedLabel\", \"label\", \"features\").show(5)\n",
        "\n",
        "# Select (prediction, true label) and compute test error\n",
        "evaluator = MulticlassClassificationEvaluator(\n",
        "    labelCol=\"indexedLabel\", predictionCol=\"prediction\", metricName=\"accuracy\")\n",
        "\n",
        "accuracy = evaluator.evaluate(predictions)\n",
        "\n",
        "confusionmatrix = confusion_matrix(y_true, y_pred)\n",
        "\n",
        "precision = precision_score(y_true, y_pred, average='micro')\n",
        "\n",
        "recall = recall_score(y_true, y_pred, average='micro')\n",
        "\n",
        "rfModel = model.stages[2]\n",
        "print(rfModel)  # summary only\n",
        "print(\"Random Forest - Test Accuracy = %g\" % (accuracy))\n",
        "print(\"Random Forest - Test Error = %g\" % (1.0 - accuracy))\n",
        "\n",
        "print(\"The Confusion Matrix for Random Forest Model is :\\n\" + str(confusionmatrix))\n",
        "\n",
        "print(\"The precision score for Random Forest Model is: \" + str(precision))\n",
        "\n",
        "print(\"The recall score for Random Forest Model is: \" + str(recall))"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **Chạy các thuật toán trên tập dữ liệu khác**\n",
        "Sử dụng tập dữ liệu **hoa Iris** để chạy lại trên các mô hình đã triển khai"
      ],
      "metadata": {
        "id": "xa16vGOate34"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Đọc tập dữ liệu từ trang github website và tạo một spark dataframe"
      ],
      "metadata": {
        "id": "xM0J5mXjXKYa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "testdata = pd.read_csv('https://gist.githubusercontent.com/curran/a08a1080b88344b0c8a7/raw/0e7a9b0a5d22642a06d3d5b9bcbad9890c8ee534/iris.csv')\n",
        "testdata = spark.createDataFrame(testdata)"
      ],
      "metadata": {
        "id": "ADCXAmuVOMoH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "testdata.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KR22mCu-OZ-Z",
        "outputId": "c2d7f22e-fc60-4981-a497-994b5b029d5c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------+-----------+------------+-----------+-------+\n",
            "|sepal_length|sepal_width|petal_length|petal_width|species|\n",
            "+------------+-----------+------------+-----------+-------+\n",
            "|         5.1|        3.5|         1.4|        0.2| setosa|\n",
            "|         4.9|        3.0|         1.4|        0.2| setosa|\n",
            "|         4.7|        3.2|         1.3|        0.2| setosa|\n",
            "|         4.6|        3.1|         1.5|        0.2| setosa|\n",
            "|         5.0|        3.6|         1.4|        0.2| setosa|\n",
            "|         5.4|        3.9|         1.7|        0.4| setosa|\n",
            "|         4.6|        3.4|         1.4|        0.3| setosa|\n",
            "|         5.0|        3.4|         1.5|        0.2| setosa|\n",
            "|         4.4|        2.9|         1.4|        0.2| setosa|\n",
            "|         4.9|        3.1|         1.5|        0.1| setosa|\n",
            "|         5.4|        3.7|         1.5|        0.2| setosa|\n",
            "|         4.8|        3.4|         1.6|        0.2| setosa|\n",
            "|         4.8|        3.0|         1.4|        0.1| setosa|\n",
            "|         4.3|        3.0|         1.1|        0.1| setosa|\n",
            "|         5.8|        4.0|         1.2|        0.2| setosa|\n",
            "|         5.7|        4.4|         1.5|        0.4| setosa|\n",
            "|         5.4|        3.9|         1.3|        0.4| setosa|\n",
            "|         5.1|        3.5|         1.4|        0.3| setosa|\n",
            "|         5.7|        3.8|         1.7|        0.3| setosa|\n",
            "|         5.1|        3.8|         1.5|        0.3| setosa|\n",
            "+------------+-----------+------------+-----------+-------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Decision Tree"
      ],
      "metadata": {
        "id": "nGMFueEoOgRy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "testdata1 = testdata.withColumn(\"SL\", testdata[\"sepal_length\"] - 0). \\\n",
        "    withColumn(\"SW\", testdata[\"sepal_width\"] - 0). \\\n",
        "    withColumn(\"PL\", testdata[\"petal_length\"] - 0). \\\n",
        "    withColumn(\"PW\", testdata[\"petal_width\"] - 0)\n",
        "\n",
        "assem = VectorAssembler(inputCols=[\"SL\", \"SW\", \"PL\", \"PW\"], outputCol='features')\n",
        "testdata1 = assem.transform(testdata1)\n",
        "\n",
        "# Index labels, adding metadata to the label column.\n",
        "# Fit on whole dataset to include all labels in indexL\n",
        "labelIndexer = StringIndexer(inputCol=\"species\", outputCol=\"indexedLabel\").fit(testdata1)\n",
        "# Automatically identify categorical features, and index them.\n",
        "# We specify maxCategories so features with > 4 distinct values are treated as continuous.\n",
        "featureIndexer =\\\n",
        "    VectorIndexer(inputCol=\"features\", outputCol=\"indexedFeatures\", maxCategories=4).fit(testdata1)\n",
        "\n",
        "# Split the data into training and test sets (30% held out for testing)\n",
        "(trainingData, testData) = testdata1.randomSplit([0.7, 0.3])\n",
        "\n",
        "# Train a DecisionTree model.\n",
        "dt = DecisionTreeClassifier(labelCol=\"indexedLabel\", featuresCol=\"indexedFeatures\")\n",
        "\n",
        "# Chain indexers and tree in a Pipeline\n",
        "pipeline = Pipeline(stages=[labelIndexer, featureIndexer, dt])\n",
        "\n",
        "# Train model.  This also runs the indexers.\n",
        "model = pipeline.fit(trainingData)\n",
        "\n",
        "# Make predictions.\n",
        "predictions = model.transform(testdata1)\n",
        "\n",
        "# Select example rows to display.\n",
        "predictions.select(\"prediction\", \"indexedLabel\", \"features\").show(5)\n",
        "\n",
        "# Select (prediction, true label) and compute test error\n",
        "evaluator = MulticlassClassificationEvaluator(\n",
        "    labelCol=\"indexedLabel\", predictionCol=\"prediction\", metricName=\"accuracy\")\n",
        "\n",
        "accuracy = evaluator.evaluate(predictions)\n",
        "\n",
        "y_true = predictions.select(\"indexedLabel\").rdd.flatMap(lambda x: x).collect()\n",
        "y_pred = predictions.select(\"prediction\").rdd.flatMap(lambda x: x).collect()\n",
        "\n",
        "confusionmatrix = confusion_matrix(y_true, y_pred)\n",
        "\n",
        "precision = precision_score(y_true, y_pred, average='micro')\n",
        "\n",
        "recall = recall_score(y_true, y_pred, average='micro')\n",
        "\n",
        "treeModel = model.stages[2]\n",
        "print(treeModel)\n",
        "print(\"Decision Tree - Test Accuracy = %g\" % (accuracy))\n",
        "print(\"Decision Tree - Test Error = %g\" % (1.0 - accuracy))\n",
        "\n",
        "print(\"The Confusion Matrix for Decision Tree Model is :\\n\" + str(confusionmatrix))\n",
        "\n",
        "print(\"The precision score for Decision Tree Model is: \" + str(precision))\n",
        "\n",
        "print(\"The recall score for Decision Tree Model is: \" + str(recall))"
      ],
      "metadata": {
        "id": "Iklf5CHZLyZr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e967d7ed-e2a6-43e7-b069-54233727ebdc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------+------------+-----------------+\n",
            "|prediction|indexedLabel|         features|\n",
            "+----------+------------+-----------------+\n",
            "|       0.0|         0.0|[5.1,3.5,1.4,0.2]|\n",
            "|       0.0|         0.0|[4.9,3.0,1.4,0.2]|\n",
            "|       0.0|         0.0|[4.7,3.2,1.3,0.2]|\n",
            "|       0.0|         0.0|[4.6,3.1,1.5,0.2]|\n",
            "|       0.0|         0.0|[5.0,3.6,1.4,0.2]|\n",
            "+----------+------------+-----------------+\n",
            "only showing top 5 rows\n",
            "\n",
            "DecisionTreeClassificationModel: uid=DecisionTreeClassifier_b2a64f065afe, depth=4, numNodes=11, numClasses=3, numFeatures=4\n",
            "Decision Tree - Test Accuracy = 0.98\n",
            "Decision Tree - Test Error = 0.02\n",
            "The Confusion Matrix for Decision Tree Model is :\n",
            "[[50  0  0]\n",
            " [ 0 48  2]\n",
            " [ 0  1 49]]\n",
            "The precision score for Decision Tree Model is: 0.98\n",
            "The recall score for Decision Tree Model is: 0.98\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Naive Bayes"
      ],
      "metadata": {
        "id": "UzsW6eK8E86r"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "testdata2 = testdata.withColumn(\"SL\", testdata[\"sepal_length\"] - 0). \\\n",
        "    withColumn(\"SW\", testdata[\"sepal_width\"] - 0). \\\n",
        "    withColumn(\"PL\", testdata[\"petal_length\"] - 0). \\\n",
        "    withColumn(\"PW\", testdata[\"petal_width\"] - 0)\n",
        "\n",
        "assem = VectorAssembler(inputCols=[\"SL\", \"SW\", \"PL\", \"PW\"], outputCol='features')\n",
        "testdata2 = assem.transform(testdata2)\n",
        "\n",
        "labelIndexer = StringIndexer(inputCol=\"species\", outputCol=\"label\").fit(testdata2)\n",
        "featureIndexer =\\\n",
        "    VectorIndexer(inputCol=\"features\", outputCol=\"indexedFeatures\", maxCategories=3).fit(testdata2)\n",
        "\n",
        "# Split the data into train and test\n",
        "splits = testdata2.randomSplit([0.7, 0.3], 1000)\n",
        "train = splits[0]\n",
        "test = splits[1]\n",
        "\n",
        "# create the trainer and set its parameters\n",
        "nb = NaiveBayes(smoothing=1.0, modelType=\"multinomial\")\n",
        "\n",
        "pipeline = Pipeline(stages=[labelIndexer, featureIndexer, nb])\n",
        "\n",
        "model = pipeline.fit(train)\n",
        "\n",
        "# select example rows to display.\n",
        "predictions = model.transform(test)\n",
        "\n",
        "# compute accuracy on the test set\n",
        "evaluator = MulticlassClassificationEvaluator(labelCol=\"label\", predictionCol=\"prediction\",\n",
        "                                              metricName=\"accuracy\")\n",
        "\n",
        "y_true = predictions.select(\"label\").rdd.flatMap(lambda x: x).collect()\n",
        "y_pred = predictions.select(\"prediction\").rdd.flatMap(lambda x: x).collect()\n",
        "\n",
        "\n",
        "accuracy = evaluator.evaluate(predictions)\n",
        "\n",
        "confusionmatrix = confusion_matrix(y_true, y_pred)\n",
        "\n",
        "precision = precision_score(y_true, y_pred, average='micro')\n",
        "\n",
        "recall = recall_score(y_true, y_pred, average='micro')\n",
        "\n",
        "\n",
        "print(\"Naive Bayes - Test set accuracy = \" + str(accuracy))\n",
        "\n",
        "print(\"The Confusion Matrix for Naive Bayes Model is :\\n\" + str(confusionmatrix))\n",
        "\n",
        "print(\"The precision score for Naive Bayes Model is: \" + str(precision))\n",
        "\n",
        "print(\"The recall score for Naive Bayes Model is: \" + str(recall))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p38gtjGnFBU6",
        "outputId": "532758bf-5207-41eb-f4b7-1405683a1ebe"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Naive Bayes - Test set accuracy = 0.8409090909090909\n",
            "The Confusion Matrix for Naive Bayes Model is :\n",
            "[[15  0  0]\n",
            " [ 0 13  0]\n",
            " [ 0  7  9]]\n",
            "The precision score for Naive Bayes Model is: 0.8409090909090909\n",
            "The recall score for Naive Bayes Model is: 0.8409090909090909\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Random Forest"
      ],
      "metadata": {
        "id": "5W3VrD38O0fB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "testdata3 = testdata.withColumn(\"SL\", testdata[\"sepal_length\"] - 0). \\\n",
        "    withColumn(\"SW\", testdata[\"sepal_width\"] - 0). \\\n",
        "    withColumn(\"PL\", testdata[\"petal_length\"] - 0). \\\n",
        "    withColumn(\"PW\", testdata[\"petal_width\"] - 0)\n",
        "\n",
        "assem = VectorAssembler(inputCols=[\"SL\", \"SW\", \"PL\", \"PW\"], outputCol='features')\n",
        "testdata3 = assem.transform(testdata3)\n",
        "\n",
        "labelIndexer = StringIndexer(inputCol=\"species\", outputCol=\"indexedLabel\").fit(testdata3)\n",
        "\n",
        "# Automatically identify categorical features, and index them.\n",
        "# Set maxCategories so features with > 4 distinct values are treated as continuous.\n",
        "featureIndexer =\\\n",
        "    VectorIndexer(inputCol=\"features\", outputCol=\"indexedFeatures\", maxCategories=3).fit(testdata3)\n",
        "\n",
        "# Split the data into training and test sets (30% held out for testing)\n",
        "(trainingData, testData) = testdata3.randomSplit([0.7, 0.3])\n",
        "\n",
        "# Train a RandomForest model.\n",
        "rf = RandomForestClassifier(labelCol=\"indexedLabel\", featuresCol=\"indexedFeatures\", numTrees=10)\n",
        "\n",
        "# Convert indexed labels back to original labels.\n",
        "labelConverter = IndexToString(inputCol=\"prediction\", outputCol=\"predictedLabel\",\n",
        "                               labels=labelIndexer.labels)\n",
        "\n",
        "# Chain indexers and forest in a Pipeline\n",
        "pipeline = Pipeline(stages=[labelIndexer, featureIndexer, rf, labelConverter])\n",
        "\n",
        "# Train model.  This also runs the indexers.\n",
        "model = pipeline.fit(trainingData)\n",
        "\n",
        "# Make predictions.\n",
        "predictions = model.transform(testData)\n",
        "\n",
        "# Select (prediction, true label) and compute test error\n",
        "evaluator = MulticlassClassificationEvaluator(\n",
        "    labelCol=\"indexedLabel\", predictionCol=\"prediction\", metricName=\"accuracy\")\n",
        "\n",
        "accuracy = evaluator.evaluate(predictions)\n",
        "\n",
        "y_true = testData.select(\"species\").rdd.flatMap(lambda x: x).collect()\n",
        "y_pred = predictions.select(\"predictedLabel\").rdd.flatMap(lambda x: x).collect()\n",
        "\n",
        "confusionmatrix = confusion_matrix(y_true, y_pred)\n",
        "\n",
        "precision = precision_score(y_true, y_pred, average='micro')\n",
        "\n",
        "recall = recall_score(y_true, y_pred, average='micro')\n",
        "\n",
        "rfModel = model.stages[2]\n",
        "print(rfModel)  # summary only\n",
        "print(\"Random Forest - Test Accuracy = %g\" % (accuracy))\n",
        "print(\"Random Forest - Test Error = %g\" % (1.0 - accuracy))\n",
        "\n",
        "print(\"The Confusion Matrix for Random Forest Model is :\\n\" + str(confusionmatrix))\n",
        "\n",
        "print(\"The precision score for Random Forest Model is: \" + str(precision))\n",
        "\n",
        "print(\"The recall score for Random Forest Model is: \" + str(recall))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2KOpDwJoO3Yd",
        "outputId": "c435b6c2-2f75-4949-bd01-637a57109454"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "RandomForestClassificationModel: uid=RandomForestClassifier_9bfc3860d025, numTrees=10, numClasses=3, numFeatures=4\n",
            "Random Forest - Test Accuracy = 0.947368\n",
            "Random Forest - Test Error = 0.0526316\n",
            "The Confusion Matrix for Random Forest Model is :\n",
            "[[12  0  0]\n",
            " [ 0 11  1]\n",
            " [ 0  1 13]]\n",
            "The precision score for Random Forest Model is: 0.9473684210526315\n",
            "The recall score for Random Forest Model is: 0.9473684210526315\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predictions.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B5aiMIJXQetV",
        "outputId": "d10c3e7b-6967-426e-98eb-8da16f26242d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------+-----------+------------+-----------+----------+---+---+---+---+-----------------+------------+-----------------+--------------+--------------------+----------+--------------+\n",
            "|sepal_length|sepal_width|petal_length|petal_width|   species| SL| SW| PL| PW|         features|indexedLabel|  indexedFeatures| rawPrediction|         probability|prediction|predictedLabel|\n",
            "+------------+-----------+------------+-----------+----------+---+---+---+---+-----------------+------------+-----------------+--------------+--------------------+----------+--------------+\n",
            "|         4.4|        3.2|         1.3|        0.2|    setosa|4.4|3.2|1.3|0.2|[4.4,3.2,1.3,0.2]|         0.0|[4.4,3.2,1.3,0.2]|[10.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|        setosa|\n",
            "|         4.6|        3.2|         1.4|        0.2|    setosa|4.6|3.2|1.4|0.2|[4.6,3.2,1.4,0.2]|         0.0|[4.6,3.2,1.4,0.2]|[10.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|        setosa|\n",
            "|         4.8|        3.0|         1.4|        0.1|    setosa|4.8|3.0|1.4|0.1|[4.8,3.0,1.4,0.1]|         0.0|[4.8,3.0,1.4,0.1]|[10.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|        setosa|\n",
            "|         4.8|        3.4|         1.9|        0.2|    setosa|4.8|3.4|1.9|0.2|[4.8,3.4,1.9,0.2]|         0.0|[4.8,3.4,1.9,0.2]| [9.0,1.0,0.0]|       [0.9,0.1,0.0]|       0.0|        setosa|\n",
            "|         5.0|        3.0|         1.6|        0.2|    setosa|5.0|3.0|1.6|0.2|[5.0,3.0,1.6,0.2]|         0.0|[5.0,3.0,1.6,0.2]|[10.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|        setosa|\n",
            "|         5.0|        3.4|         1.6|        0.4|    setosa|5.0|3.4|1.6|0.4|[5.0,3.4,1.6,0.4]|         0.0|[5.0,3.4,1.6,0.4]|[10.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|        setosa|\n",
            "|         5.0|        3.5|         1.6|        0.6|    setosa|5.0|3.5|1.6|0.6|[5.0,3.5,1.6,0.6]|         0.0|[5.0,3.5,1.6,0.6]|[10.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|        setosa|\n",
            "|         5.1|        3.3|         1.7|        0.5|    setosa|5.1|3.3|1.7|0.5|[5.1,3.3,1.7,0.5]|         0.0|[5.1,3.3,1.7,0.5]|[10.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|        setosa|\n",
            "|         5.1|        3.7|         1.5|        0.4|    setosa|5.1|3.7|1.5|0.4|[5.1,3.7,1.5,0.4]|         0.0|[5.1,3.7,1.5,0.4]|[10.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|        setosa|\n",
            "|         5.2|        2.7|         3.9|        1.4|versicolor|5.2|2.7|3.9|1.4|[5.2,2.7,3.9,1.4]|         1.0|[5.2,2.7,3.9,1.4]|[0.0,10.0,0.0]|       [0.0,1.0,0.0]|       1.0|    versicolor|\n",
            "|         5.4|        3.4|         1.7|        0.2|    setosa|5.4|3.4|1.7|0.2|[5.4,3.4,1.7,0.2]|         0.0|[5.4,3.4,1.7,0.2]|[10.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|        setosa|\n",
            "|         5.4|        3.9|         1.7|        0.4|    setosa|5.4|3.9|1.7|0.4|[5.4,3.9,1.7,0.4]|         0.0|[5.4,3.9,1.7,0.4]|[10.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|        setosa|\n",
            "|         5.6|        2.9|         3.6|        1.3|versicolor|5.6|2.9|3.6|1.3|[5.6,2.9,3.6,1.3]|         1.0|[5.6,2.9,3.6,1.3]|[0.0,10.0,0.0]|       [0.0,1.0,0.0]|       1.0|    versicolor|\n",
            "|         5.7|        3.8|         1.7|        0.3|    setosa|5.7|3.8|1.7|0.3|[5.7,3.8,1.7,0.3]|         0.0|[5.7,3.8,1.7,0.3]|[10.0,0.0,0.0]|       [1.0,0.0,0.0]|       0.0|        setosa|\n",
            "|         6.1|        2.8|         4.0|        1.3|versicolor|6.1|2.8|4.0|1.3|[6.1,2.8,4.0,1.3]|         1.0|[6.1,2.8,4.0,1.3]|[0.0,10.0,0.0]|       [0.0,1.0,0.0]|       1.0|    versicolor|\n",
            "|         6.1|        2.9|         4.7|        1.4|versicolor|6.1|2.9|4.7|1.4|[6.1,2.9,4.7,1.4]|         1.0|[6.1,2.9,4.7,1.4]|[0.0,10.0,0.0]|       [0.0,1.0,0.0]|       1.0|    versicolor|\n",
            "|         6.6|        2.9|         4.6|        1.3|versicolor|6.6|2.9|4.6|1.3|[6.6,2.9,4.6,1.3]|         1.0|[6.6,2.9,4.6,1.3]|[0.0,10.0,0.0]|       [0.0,1.0,0.0]|       1.0|    versicolor|\n",
            "|         6.7|        3.1|         4.4|        1.4|versicolor|6.7|3.1|4.4|1.4|[6.7,3.1,4.4,1.4]|         1.0|[6.7,3.1,4.4,1.4]|[0.0,10.0,0.0]|       [0.0,1.0,0.0]|       1.0|    versicolor|\n",
            "|         4.9|        2.5|         4.5|        1.7| virginica|4.9|2.5|4.5|1.7|[4.9,2.5,4.5,1.7]|         2.0|[4.9,2.5,4.5,1.7]| [0.0,8.9,1.1]|[0.0,0.89,0.11000...|       1.0|    versicolor|\n",
            "|         5.5|        2.5|         4.0|        1.3|versicolor|5.5|2.5|4.0|1.3|[5.5,2.5,4.0,1.3]|         1.0|[5.5,2.5,4.0,1.3]|[0.0,10.0,0.0]|       [0.0,1.0,0.0]|       1.0|    versicolor|\n",
            "+------------+-----------+------------+-----------+----------+---+---+---+---+-----------------+------------+-----------------+--------------+--------------------+----------+--------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "colab": {
      "name": "Lab03.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}