rule_based_difficulty_skill_prediction_data_2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "rule_based_difficulty_skill_prediction_data_2.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 554
        },
        "id": "F271l2E3jUZ6",
        "outputId": "40135e49-b504-4e91-df1e-73737b3b9db7"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-fbe92ea6-78c1-464b-99d2-ddbaa969433a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Question</th>\n",
              "      <th>Answer</th>\n",
              "      <th>DifficultyFromAnswerer</th>\n",
              "      <th>question_answer</th>\n",
              "      <th>difficulty_label</th>\n",
              "      <th>skill_label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Is the dialect spoken in Jeju located in fact ...</td>\n",
              "      <td>The dialect spoken in Jeju is in fact classifi...</td>\n",
              "      <td>hard</td>\n",
              "      <td>Is the dialect spoken in Jeju located in fact ...</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>What cello manufacturer should I buy from if I...</td>\n",
              "      <td>Luis &amp; Clark</td>\n",
              "      <td>hard</td>\n",
              "      <td>What cello manufacturer should I buy from if I...</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Does it have a border with Norway?</td>\n",
              "      <td>yes</td>\n",
              "      <td>medium</td>\n",
              "      <td>Does it have a border with Norway? yes</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>How many people use the bus network daily?</td>\n",
              "      <td>More than 2.78 million people.</td>\n",
              "      <td>easy</td>\n",
              "      <td>How many people use the bus network daily? Mor...</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Who founded Montevideo?</td>\n",
              "      <td>The Spanish.</td>\n",
              "      <td>medium</td>\n",
              "      <td>Who founded Montevideo? The Spanish.</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2763</th>\n",
              "      <td>Did he become a professor before the revolutio...</td>\n",
              "      <td>yes</td>\n",
              "      <td>hard</td>\n",
              "      <td>Did he become a professor before the revolutio...</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2764</th>\n",
              "      <td>Does Vietnamese borrow from Latin and Greek?</td>\n",
              "      <td>No, Vietnamese does not borrow from Latin and ...</td>\n",
              "      <td>medium</td>\n",
              "      <td>Does Vietnamese borrow from Latin and Greek? N...</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2765</th>\n",
              "      <td>Where is San Francisco?</td>\n",
              "      <td>San Francisco is in California.</td>\n",
              "      <td>medium</td>\n",
              "      <td>Where is San Francisco? San Francisco is in Ca...</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2766</th>\n",
              "      <td>What is the primary item in an otter's diet?</td>\n",
              "      <td>fish</td>\n",
              "      <td>medium</td>\n",
              "      <td>What is the primary item in an otter's diet? fish</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2767</th>\n",
              "      <td>Where are turtle eggs layed?</td>\n",
              "      <td>Turtles lay eggs on land.</td>\n",
              "      <td>hard</td>\n",
              "      <td>Where are turtle eggs layed? Turtles lay eggs ...</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2768 rows × 6 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fbe92ea6-78c1-464b-99d2-ddbaa969433a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-fbe92ea6-78c1-464b-99d2-ddbaa969433a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-fbe92ea6-78c1-464b-99d2-ddbaa969433a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                               Question  ... skill_label\n",
              "0     Is the dialect spoken in Jeju located in fact ...  ...           3\n",
              "1     What cello manufacturer should I buy from if I...  ...           3\n",
              "2                    Does it have a border with Norway?  ...           2\n",
              "3            How many people use the bus network daily?  ...           2\n",
              "4                               Who founded Montevideo?  ...           2\n",
              "...                                                 ...  ...         ...\n",
              "2763  Did he become a professor before the revolutio...  ...           3\n",
              "2764       Does Vietnamese borrow from Latin and Greek?  ...           3\n",
              "2765                            Where is San Francisco?  ...           3\n",
              "2766       What is the primary item in an otter's diet?  ...           2\n",
              "2767                       Where are turtle eggs layed?  ...           3\n",
              "\n",
              "[2768 rows x 6 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "import pandas as pd\n",
        "data = pd.read_csv(\"train_qdiff_data_2_soft_labeled.csv\")\n",
        "data"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "ENlWBg16lDuT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9exlBELH5oq9"
      },
      "source": [
        "!cp \"/content/drive/My Drive/research_skill_name_prediction/label_encoder_difficulty_Lstm\"  /content"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pUqcGtXwBIyl"
      },
      "source": [
        "!cp \"/content/drive/MyDrive/research_skill_name_prediction/label_encoder_skill_lstm\" /content"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-OBarOLBz2nO"
      },
      "source": [
        "def get_labels(LE, prediction):\n",
        "    predicted_label =  LE.inverse_transform([prediction])\n",
        "    return predicted_label[0]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import joblib\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "\n",
        "LE_diff = LabelEncoder()"
      ],
      "metadata": {
        "id": "0bcRAnCGy1uZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data[\"difficulty_label\"] = LE_diff.fit_transform(data[\"DifficultyFromAnswerer\"].values)"
      ],
      "metadata": {
        "id": "7oWcDHZ4yt_-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 554
        },
        "id": "YdjrLp0vy6hf",
        "outputId": "926e4924-5eee-4890-f636-2e1881020f55"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-b3db12bc-791b-4a06-8114-05965d5bc145\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Question</th>\n",
              "      <th>Answer</th>\n",
              "      <th>DifficultyFromAnswerer</th>\n",
              "      <th>question_answer</th>\n",
              "      <th>difficulty_label</th>\n",
              "      <th>skill_label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Is the dialect spoken in Jeju located in fact ...</td>\n",
              "      <td>The dialect spoken in Jeju is in fact classifi...</td>\n",
              "      <td>hard</td>\n",
              "      <td>Is the dialect spoken in Jeju located in fact ...</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>What cello manufacturer should I buy from if I...</td>\n",
              "      <td>Luis &amp; Clark</td>\n",
              "      <td>hard</td>\n",
              "      <td>What cello manufacturer should I buy from if I...</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Does it have a border with Norway?</td>\n",
              "      <td>yes</td>\n",
              "      <td>medium</td>\n",
              "      <td>Does it have a border with Norway? yes</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>How many people use the bus network daily?</td>\n",
              "      <td>More than 2.78 million people.</td>\n",
              "      <td>easy</td>\n",
              "      <td>How many people use the bus network daily? Mor...</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Who founded Montevideo?</td>\n",
              "      <td>The Spanish.</td>\n",
              "      <td>medium</td>\n",
              "      <td>Who founded Montevideo? The Spanish.</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2763</th>\n",
              "      <td>Did he become a professor before the revolutio...</td>\n",
              "      <td>yes</td>\n",
              "      <td>hard</td>\n",
              "      <td>Did he become a professor before the revolutio...</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2764</th>\n",
              "      <td>Does Vietnamese borrow from Latin and Greek?</td>\n",
              "      <td>No, Vietnamese does not borrow from Latin and ...</td>\n",
              "      <td>medium</td>\n",
              "      <td>Does Vietnamese borrow from Latin and Greek? N...</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2765</th>\n",
              "      <td>Where is San Francisco?</td>\n",
              "      <td>San Francisco is in California.</td>\n",
              "      <td>medium</td>\n",
              "      <td>Where is San Francisco? San Francisco is in Ca...</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2766</th>\n",
              "      <td>What is the primary item in an otter's diet?</td>\n",
              "      <td>fish</td>\n",
              "      <td>medium</td>\n",
              "      <td>What is the primary item in an otter's diet? fish</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2767</th>\n",
              "      <td>Where are turtle eggs layed?</td>\n",
              "      <td>Turtles lay eggs on land.</td>\n",
              "      <td>hard</td>\n",
              "      <td>Where are turtle eggs layed? Turtles lay eggs ...</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2768 rows × 6 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b3db12bc-791b-4a06-8114-05965d5bc145')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b3db12bc-791b-4a06-8114-05965d5bc145 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b3db12bc-791b-4a06-8114-05965d5bc145');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                               Question  ... skill_label\n",
              "0     Is the dialect spoken in Jeju located in fact ...  ...           3\n",
              "1     What cello manufacturer should I buy from if I...  ...           3\n",
              "2                    Does it have a border with Norway?  ...           2\n",
              "3            How many people use the bus network daily?  ...           2\n",
              "4                               Who founded Montevideo?  ...           2\n",
              "...                                                 ...  ...         ...\n",
              "2763  Did he become a professor before the revolutio...  ...           3\n",
              "2764       Does Vietnamese borrow from Latin and Greek?  ...           3\n",
              "2765                            Where is San Francisco?  ...           3\n",
              "2766       What is the primary item in an otter's diet?  ...           2\n",
              "2767                       Where are turtle eggs layed?  ...           3\n",
              "\n",
              "[2768 rows x 6 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import joblib\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "\n",
        "# LE = LabelEncoder()\n",
        "# LE = joblib.load('label_encoder_difficulty_Lstm')\n",
        "\n",
        "get_labels(LE_diff, 0)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "p1Q1k-VolOZJ",
        "outputId": "bc24f9f5-f585-4dc3-9216-d02424147fb5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'easy'"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "LE_skill = LabelEncoder()\n",
        "LE_skill = joblib.load('label_encoder_skill_lstm')\n",
        "\n",
        "get_labels(LE_skill, 0)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "id": "Z4cGpAnqlVQC",
        "outputId": "d11b943b-3c86-4021-913b-82e6b2cbf10b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/base.py:338: UserWarning: Trying to unpickle estimator LabelEncoder from version 0.22.2.post1 when using version 1.0.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
            "https://scikit-learn.org/stable/modules/model_persistence.html#security-maintainability-limitations\n",
            "  UserWarning,\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Analysing'"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "LE_diff.classes_"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rhxjcnIulgCh",
        "outputId": "a705eecd-24e5-40d7-bc96-ef8d864b2771"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['easy', 'hard', 'medium'], dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "LE_skill.classes_"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JpHUBQqvlj2X",
        "outputId": "09b86f8d-8f54-4432-f456-f9c91dc40020"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['Analysing', 'Applying', 'Knowledge & understanding',\n",
              "       'Remembering', 'Understanding'], dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data[data[\"difficulty_label\"]==0][\"skill_label\"].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MZKtAG6OllI1",
        "outputId": "7d9cdf74-3e92-4def-cf41-0b31e18c676a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3    704\n",
              "2    340\n",
              "4     36\n",
              "1     18\n",
              "0      1\n",
              "Name: skill_label, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data[data[\"difficulty_label\"]==1][\"skill_label\"].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G_QSNi2Jmico",
        "outputId": "7c89881b-1263-44eb-c6b9-2d68386d8904"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3    417\n",
              "2    189\n",
              "4     54\n",
              "1      4\n",
              "0      4\n",
              "Name: skill_label, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data[data[\"difficulty_label\"]==2][\"skill_label\"].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4yIaUu_lmkZQ",
        "outputId": "e39ee54d-3255-40e5-db70-13a0897488eb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3    692\n",
              "2    272\n",
              "4     30\n",
              "1      7\n",
              "Name: skill_label, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "skill_list = data[data[\"difficulty_label\"]==0][\"skill_label\"].value_counts().values.tolist()\n",
        "understanding_count = skill_list[0]+skill_list[1]"
      ],
      "metadata": {
        "id": "vcbtccnjoxqf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "zZC9XXE7mkry"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "diff_dict = {}\n",
        "diff_dict[\"easy\"] = {\n",
        "    \"3\": skill_list[0],\n",
        "    \"2\": skill_list[1],\n",
        "    \"4\": skill_list[2],\n",
        "    \"1\": skill_list[3],\n",
        "    \"0\":skill_list[4]\n",
        "}"
      ],
      "metadata": {
        "id": "HkKDRisfl7h6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "skill_list = data[data[\"difficulty_label\"]==1][\"skill_label\"].value_counts().values.tolist()\n",
        "skill_list"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u14NBRH7p9_E",
        "outputId": "85c1644f-3743-46c8-bc5b-fda4a0c7c446"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[417, 189, 54, 4, 4]"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "diff_dict[\"hard\"] = {\n",
        "    \"3\": skill_list[0],\n",
        "    \"2\": skill_list[1],\n",
        "    \"4\": skill_list[2],\n",
        "    \"1\": skill_list[3],\n",
        "    \"0\":skill_list[4]\n",
        "}"
      ],
      "metadata": {
        "id": "B6EePNy_p7at"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "skill_list = data[data[\"difficulty_label\"]==2][\"skill_label\"].value_counts().values.tolist()\n",
        "skill_list"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "999df316-d45d-4b55-aa3a-fe4c1de97f8e",
        "id": "dj8hN1v4qKYy"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[692, 272, 30, 7]"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "diff_dict[\"medium\"] = {\n",
        "    \"3\": skill_list[0],\n",
        "    \"2\": skill_list[1],\n",
        "    \"4\": skill_list[2],\n",
        "    \"1\": skill_list[3],\n",
        "    \"0\":0\n",
        "}"
      ],
      "metadata": {
        "id": "UIOeifMAqKY0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "diff_dict"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B3wO4gB_qd9s",
        "outputId": "2bca28fe-c42a-48a9-8d69-9b80411cefde"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'easy': {'0': 1, '1': 18, '2': 340, '3': 704, '4': 36},\n",
              " 'hard': {'0': 4, '1': 4, '2': 189, '3': 417, '4': 54},\n",
              " 'medium': {'0': 0, '1': 7, '2': 272, '3': 692, '4': 30}}"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data[data[\"skill_label\"]==0][\"difficulty_label\"].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "91bd28af-6161-47be-b443-d20b226699a7",
        "id": "TbMqvHFerKng"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1    4\n",
              "0    1\n",
              "Name: difficulty_label, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data[data[\"skill_label\"]==1][\"difficulty_label\"].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2b639ebd-80af-469b-a842-186643638ba0",
        "id": "_qXwxx78rKnj"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    18\n",
              "2     7\n",
              "1     4\n",
              "Name: difficulty_label, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data[data[\"skill_label\"]==2][\"difficulty_label\"].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "acecfe1f-85a7-468b-b04f-33e2fc8911ba",
        "id": "lQHawroorKnk"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    340\n",
              "2    272\n",
              "1    189\n",
              "Name: difficulty_label, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data[data[\"skill_label\"]==3][\"difficulty_label\"].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EvqNZCCBrXZZ",
        "outputId": "c2ec854f-5377-43d2-e6c1-00d94857986d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    704\n",
              "2    692\n",
              "1    417\n",
              "Name: difficulty_label, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data[data[\"skill_label\"]==4][\"difficulty_label\"].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7CTzFRfXrcwp",
        "outputId": "fc51919a-6b49-4afe-8c07-2a6b24882792"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1    54\n",
              "0    36\n",
              "2    30\n",
              "Name: difficulty_label, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "diff_list = data[data[\"skill_label\"]==0][\"difficulty_label\"].value_counts().values.tolist()"
      ],
      "metadata": {
        "id": "0eKrL5HNr1o2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "skill_dict = {}\n",
        "# array(['Analysing', 'Applying', 'Knowledge & understanding',\n",
        "    #    'Remembering', 'Understanding'], dtype=object)\n",
        "skill_dict[\"Analysing\"] = {\n",
        "    \"1\": diff_list[0],\n",
        "    \"0\": diff_list[1],\n",
        "    \"2\": 0\n",
        "}"
      ],
      "metadata": {
        "id": "ejCJn_BcrjXb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "diff_list = data[data[\"skill_label\"]==1][\"difficulty_label\"].value_counts().values.tolist()"
      ],
      "metadata": {
        "id": "Sgg-su_bsA7n"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# array(['Analysing', 'Applying', 'Knowledge & understanding',\n",
        "    #    'Remembering', 'Understanding'], dtype=object)\n",
        "skill_dict[\"Applying\"] = {\n",
        "    \"0\": diff_list[0],\n",
        "    \"2\": diff_list[1],\n",
        "    \"1\": diff_list[2]\n",
        "}"
      ],
      "metadata": {
        "id": "o4K7HhbFsA7o"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "diff_list = data[data[\"skill_label\"]==2][\"difficulty_label\"].value_counts().values.tolist()"
      ],
      "metadata": {
        "id": "Rrj1fB8usgnK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# array(['Analysing', 'Applying', 'Knowledge & understanding',\n",
        "    #    'Remembering', 'Understanding'], dtype=object)\n",
        "skill_dict[\"Knowledge & understanding\"] = {\n",
        "    \"0\": diff_list[0],\n",
        "    \"2\": diff_list[1],\n",
        "    \"1\": diff_list[2]\n",
        "}"
      ],
      "metadata": {
        "id": "0ipV8dWCsgnM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "diff_list = data[data[\"skill_label\"]==3][\"difficulty_label\"].value_counts().values.tolist()"
      ],
      "metadata": {
        "id": "jJHp03dKspHs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# skill_dict = {}\n",
        "# array(['Analysing', 'Applying', 'Knowledge & understanding',\n",
        "    #    'Remembering', 'Understanding'], dtype=object)\n",
        "skill_dict[\"Remembering\"] = {\n",
        "    \"0\": diff_list[0],\n",
        "    \"2\": diff_list[1],\n",
        "    \"1\": diff_list[2]\n",
        "}"
      ],
      "metadata": {
        "id": "xoig6FHSspHs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "diff_list = data[data[\"skill_label\"]==4][\"difficulty_label\"].value_counts().values.tolist()"
      ],
      "metadata": {
        "id": "5c-qwrdrspes"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# skill_dict = {}\n",
        "# array(['Analysing', 'Applying', 'Knowledge & understanding',\n",
        "    #    'Remembering', 'Understanding'], dtype=object)\n",
        "skill_dict[\"Understanding\"] = {\n",
        "    \"1\": diff_list[0],\n",
        "    \"0\": diff_list[1],\n",
        "    \"2\": diff_list[2]\n",
        "}"
      ],
      "metadata": {
        "id": "bD3Fd45zspes"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "skill_dict"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rkv1e69js0d0",
        "outputId": "3f0c4597-4e5d-4b28-b907-7fd99d1aeac9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'Analysing': {'0': 1, '1': 4, '2': 0},\n",
              " 'Applying': {'0': 18, '1': 4, '2': 7},\n",
              " 'Knowledge & understanding': {'0': 340, '1': 189, '2': 272},\n",
              " 'Remembering': {'0': 704, '1': 417, '2': 692},\n",
              " 'Understanding': {'0': 36, '1': 54, '2': 30}}"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "list(skill_dict[\"Applying\"].keys())[list(skill_dict[\"Applying\"].values()).index(max(skill_dict[\"Applying\"].values()))]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "-PjT49pJtRy0",
        "outputId": "f2d1d679-d993-4ef2-c211-04ea65926382"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'0'"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test = pd.read_csv(\"test_qdiff_data_2_soft_labeled.csv\")\n",
        "test"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 520
        },
        "id": "jDCaBJL-qmHe",
        "outputId": "3e0ae934-37eb-4c7e-9341-14c6fea4caa0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-915b88e6-b043-46f1-8888-d9d8564e082d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Question</th>\n",
              "      <th>Answer</th>\n",
              "      <th>DifficultyFromAnswerer</th>\n",
              "      <th>difficulty_label</th>\n",
              "      <th>question_answer</th>\n",
              "      <th>skill_label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>How are western-style xylophones characterised?</td>\n",
              "      <td>by a bright, sharp tone and high register</td>\n",
              "      <td>medium</td>\n",
              "      <td>2</td>\n",
              "      <td>How are western-style xylophones characterised...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Is Nairobi the capital of Kenya?</td>\n",
              "      <td>Yes</td>\n",
              "      <td>easy</td>\n",
              "      <td>0</td>\n",
              "      <td>Is Nairobi the capital of Kenya? Yes</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>How many sister cities does the City of Melbou...</td>\n",
              "      <td>six</td>\n",
              "      <td>medium</td>\n",
              "      <td>2</td>\n",
              "      <td>How many sister cities does the City of Melbou...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Is the electric eel a true eel?</td>\n",
              "      <td>No</td>\n",
              "      <td>easy</td>\n",
              "      <td>0</td>\n",
              "      <td>Is the electric eel a true eel? No</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Does Swedish use the perfect participle to for...</td>\n",
              "      <td>No.</td>\n",
              "      <td>easy</td>\n",
              "      <td>0</td>\n",
              "      <td>Does Swedish use the perfect participle to for...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>337</th>\n",
              "      <td>Where was there a vast swarm of butterflies?</td>\n",
              "      <td>In Kyoto there was a vast swarm of butterflies.</td>\n",
              "      <td>medium</td>\n",
              "      <td>2</td>\n",
              "      <td>Where was there a vast swarm of butterflies? I...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>338</th>\n",
              "      <td>What is the most common romanization standard ...</td>\n",
              "      <td>Hanyu Pinyin</td>\n",
              "      <td>medium</td>\n",
              "      <td>2</td>\n",
              "      <td>What is the most common romanization standard ...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>339</th>\n",
              "      <td>Is Jakarta the 12th largest city in the world?</td>\n",
              "      <td>yes</td>\n",
              "      <td>medium</td>\n",
              "      <td>2</td>\n",
              "      <td>Is Jakarta the 12th largest city in the world?...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>340</th>\n",
              "      <td>What sort of turtles are ectothermic?</td>\n",
              "      <td>all of them</td>\n",
              "      <td>medium</td>\n",
              "      <td>2</td>\n",
              "      <td>What sort of turtles are ectothermic? all of them</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>341</th>\n",
              "      <td>Was Gellu Naum the leader of the surrealist mo...</td>\n",
              "      <td>Yes</td>\n",
              "      <td>easy</td>\n",
              "      <td>0</td>\n",
              "      <td>Was Gellu Naum the leader of the surrealist mo...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>342 rows × 6 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-915b88e6-b043-46f1-8888-d9d8564e082d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-915b88e6-b043-46f1-8888-d9d8564e082d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-915b88e6-b043-46f1-8888-d9d8564e082d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                              Question  ... skill_label\n",
              "0      How are western-style xylophones characterised?  ...           2\n",
              "1                     Is Nairobi the capital of Kenya?  ...           3\n",
              "2    How many sister cities does the City of Melbou...  ...           3\n",
              "3                      Is the electric eel a true eel?  ...           3\n",
              "4    Does Swedish use the perfect participle to for...  ...           3\n",
              "..                                                 ...  ...         ...\n",
              "337       Where was there a vast swarm of butterflies?  ...           3\n",
              "338  What is the most common romanization standard ...  ...           3\n",
              "339     Is Jakarta the 12th largest city in the world?  ...           2\n",
              "340              What sort of turtles are ectothermic?  ...           2\n",
              "341  Was Gellu Naum the leader of the surrealist mo...  ...           3\n",
              "\n",
              "[342 rows x 6 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predicted_diff_labels =[]\n",
        "for index, row in test.iterrows():\n",
        "    if skill_dict[get_labels(LE_skill,row[\"skill_label\"])]:\n",
        "        # dict_entry = skill_dict[get_labels(LE_skill,row[\"skill_label\"])]\n",
        "        skill_label_name = get_labels(LE_skill,row[\"skill_label\"])\n",
        "        print(skill_label_name)\n",
        "        diff_label = list(skill_dict[skill_label_name].keys())[list(skill_dict[skill_label_name].values()).index(max(skill_dict[skill_label_name].values()))]\n",
        "        predicted_diff_labels.append(int(diff_label))\n",
        "    "
      ],
      "metadata": {
        "id": "0fduwH71qfHL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7c3c3805-601b-466c-de48-ce42c0f72a70"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Analysing\n",
            "Remembering\n",
            "Understanding\n",
            "Remembering\n",
            "Understanding\n",
            "Knowledge & understanding\n",
            "Understanding\n",
            "Understanding\n",
            "Knowledge & understanding\n",
            "Understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Understanding\n",
            "Knowledge & understanding\n",
            "Understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Understanding\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Applying\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Remembering\n",
            "Knowledge & understanding\n",
            "Knowledge & understanding\n",
            "Remembering\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "tiKgTB_fvXPo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zFediYEjlKjX"
      },
      "source": [
        "def get_confusion_matrix(predicted,actual):\n",
        "    conf_matrix = np.zeros((3, 3))\n",
        "    for pred,act in zip(predicted,actual):\n",
        "        conf_matrix[act,pred]+=1\n",
        "    return conf_matrix\n",
        "        \n",
        "def get_TP(confusion_matrix,label):\n",
        "    tp = confusion_matrix[label][label]\n",
        "    return tp\n",
        "\n",
        "def get_FN(confusion_matrix,label):\n",
        "    row = confusion_matrix[label,]\n",
        "    row_truepositives = row[label]\n",
        "    fn = row.sum() - row_truepositives\n",
        "    return fn\n",
        "\n",
        "def get_FP(confusion_matrix,tag):\n",
        "    col = confusion_matrix[:,tag]\n",
        "    col_tp = col[tag]\n",
        "    #  sum of all values in column except tp\n",
        "    fp = col.sum() - col_tp\n",
        "    return fp\n",
        "def Precision(conf_matrix):\n",
        "    precision = 0.0\n",
        "    for label in [0,1,2]:\n",
        "        dividor= get_TP(conf_matrix,label)+get_FP(conf_matrix,label)\n",
        "        if dividor != 0.0:\n",
        "            precision += (get_TP(conf_matrix,label))/dividor\n",
        "    return (precision / 3)\n",
        "\n",
        "def Recall(conf_matrix):\n",
        "    recall = 0.0\n",
        "    for label in [0,1,2]:\n",
        "        dividor=get_TP(conf_matrix,label)+get_FN(conf_matrix,label)\n",
        "        if dividor != 0.0:\n",
        "            recall += (get_TP(conf_matrix,label))/dividor\n",
        "    return (recall / 3)\n",
        "\n",
        "def F1(precision,recall):\n",
        "    return (2*precision*recall)/(precision+recall)\n",
        "def accuracy_per_class(preds_flat, labels_flat):\n",
        "\n",
        "    for label in np.unique(labels_flat):\n",
        "        y_preds = preds_flat[labels_flat==label]\n",
        "        y_true = labels_flat[labels_flat==label]\n",
        "        print(f'Class: {label}')\n",
        "        print(f'Accuracy: {len(y_preds[y_preds==label])}/{len(y_true)}\\n')\n",
        "def print_metrics(predictions,test_labels):\n",
        "    conf_matrix = get_confusion_matrix(predictions,test_labels)\n",
        "    precision = Precision(conf_matrix)\n",
        "    recall = Recall(conf_matrix)\n",
        "    f1_score = F1(precision,recall)\n",
        "    print(\"Macro : Precision:{}, Recall: {}, F1: {}\".format(precision,recall,f1_score))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aFEi23ejPfOm"
      },
      "source": [
        "def Precision_macro_weighted(conf_matrix,test_samples):\n",
        "    accum =0\n",
        "    label_wise_precision = dict()\n",
        "    for label in [0,1,2]:\n",
        "        true_sample = [sample for sample in test_samples if sample==label ]\n",
        "        if (get_TP(conf_matrix,label)+get_FP(conf_matrix,label))!=0:\n",
        "            accum+= float(len(true_sample)) *(get_TP(conf_matrix,label)/(get_TP(conf_matrix,label)+get_FP(conf_matrix,label)))\n",
        "            label_wise_precision[label] = get_TP(conf_matrix,label)/(get_TP(conf_matrix,label)+get_FP(conf_matrix,label))\n",
        "\n",
        "    \n",
        "    precision =  accum/len(test_samples)\n",
        "            \n",
        "    return precision\n",
        "\n",
        "\n",
        "def Recall_macro_weighted(conf_matrix,test_samples):\n",
        "    accum =0\n",
        "    label_wise_recall = dict()\n",
        "    for label in [0,1,2]:\n",
        "        true_sample = [sample for sample in test_samples if sample==label ]\n",
        "\n",
        "        if (get_TP(conf_matrix,label)+get_FN(conf_matrix,label))!=0:\n",
        "            accum+= float(len(true_sample)) * (get_TP(conf_matrix,label)/(get_TP(conf_matrix,label)+get_FN(conf_matrix,label)))\n",
        "            label_wise_recall[label] = get_TP(conf_matrix,label)/(get_TP(conf_matrix,label)+get_FN(conf_matrix,label))\n",
        "\n",
        "    \n",
        "    recall =  accum/len(test_samples)\n",
        "    return recall\n",
        "def print_weighted_metrics(predictions,test_labels):\n",
        "    conf_matrix = get_confusion_matrix(predictions,test_labels)\n",
        "    precision = Precision_macro_weighted(conf_matrix,test_labels)\n",
        "    recall = Recall_macro_weighted(conf_matrix,test_labels)\n",
        "    f1_score = F1(precision,recall)\n",
        "    print(\"Macro : Precision:{}, Recall: {}, F1: {}\".format(precision,recall,f1_score))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print_metrics(np.array(predicted_diff_labels), test[\"difficulty_label\"].values)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7b3glHBfvoRs",
        "outputId": "09758bf7-df63-462f-cc86-99dcc9b9d228"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Macro : Precision:0.2360685071574642, Recall: 0.3405033655253146, F1: 0.2788277576137621\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print_weighted_metrics(np.array(predicted_diff_labels), test[\"difficulty_label\"].values)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gDwkG6rGvoox",
        "outputId": "6616007d-ec78-45d0-a7b7-706677070ddf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Macro : Precision:0.23271064291608365, Recall: 0.391812865497076, F1: 0.2919954896951523\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "-TZgBRBpvxpx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Now skill prediction"
      ],
      "metadata": {
        "id": "gaiT9W87wX4h"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "diff_dict"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_qnZ2ZvRwZfQ",
        "outputId": "e742e216-ec37-460d-897b-806553a7ccfd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'easy': {'0': 1, '1': 18, '2': 340, '3': 704, '4': 36},\n",
              " 'hard': {'0': 4, '1': 4, '2': 189, '3': 417, '4': 54},\n",
              " 'medium': {'0': 0, '1': 7, '2': 272, '3': 692, '4': 30}}"
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predicted_skill_labels =[]\n",
        "for index, row in test.iterrows():\n",
        "    if diff_dict[get_labels(LE_diff,row[\"difficulty_label\"])]:\n",
        "        # dict_entry = skill_dict[get_labels(LE_skill,row[\"skill_label\"])]\n",
        "        diff_label_name = get_labels(LE_diff,row[\"difficulty_label\"])\n",
        "        print(diff_label_name)\n",
        "        skill_label = list(diff_dict[diff_label_name].keys())[list(diff_dict[diff_label_name].values()).index(max(diff_dict[diff_label_name].values()))]\n",
        "        predicted_skill_labels.append(int(skill_label))\n",
        "    "
      ],
      "metadata": {
        "id": "Lan2fECxwvsL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "eda35221-1e76-4d15-b5c6-6c50aa80da28"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "medium\n",
            "easy\n",
            "medium\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "hard\n",
            "medium\n",
            "hard\n",
            "medium\n",
            "medium\n",
            "hard\n",
            "easy\n",
            "easy\n",
            "hard\n",
            "hard\n",
            "hard\n",
            "easy\n",
            "medium\n",
            "medium\n",
            "easy\n",
            "medium\n",
            "hard\n",
            "hard\n",
            "hard\n",
            "medium\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "medium\n",
            "medium\n",
            "medium\n",
            "easy\n",
            "easy\n",
            "medium\n",
            "hard\n",
            "easy\n",
            "hard\n",
            "medium\n",
            "hard\n",
            "medium\n",
            "medium\n",
            "easy\n",
            "medium\n",
            "easy\n",
            "medium\n",
            "medium\n",
            "easy\n",
            "medium\n",
            "medium\n",
            "hard\n",
            "easy\n",
            "medium\n",
            "medium\n",
            "hard\n",
            "easy\n",
            "easy\n",
            "medium\n",
            "easy\n",
            "hard\n",
            "easy\n",
            "easy\n",
            "medium\n",
            "easy\n",
            "medium\n",
            "easy\n",
            "easy\n",
            "medium\n",
            "medium\n",
            "medium\n",
            "medium\n",
            "easy\n",
            "easy\n",
            "medium\n",
            "medium\n",
            "easy\n",
            "medium\n",
            "easy\n",
            "medium\n",
            "medium\n",
            "medium\n",
            "medium\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "medium\n",
            "easy\n",
            "hard\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "medium\n",
            "medium\n",
            "easy\n",
            "hard\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "hard\n",
            "hard\n",
            "medium\n",
            "medium\n",
            "easy\n",
            "medium\n",
            "medium\n",
            "easy\n",
            "medium\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "hard\n",
            "easy\n",
            "medium\n",
            "medium\n",
            "hard\n",
            "hard\n",
            "hard\n",
            "hard\n",
            "medium\n",
            "hard\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "hard\n",
            "hard\n",
            "easy\n",
            "easy\n",
            "hard\n",
            "easy\n",
            "easy\n",
            "hard\n",
            "hard\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "medium\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "medium\n",
            "easy\n",
            "medium\n",
            "easy\n",
            "easy\n",
            "hard\n",
            "easy\n",
            "medium\n",
            "medium\n",
            "medium\n",
            "medium\n",
            "medium\n",
            "easy\n",
            "hard\n",
            "medium\n",
            "medium\n",
            "easy\n",
            "medium\n",
            "medium\n",
            "medium\n",
            "hard\n",
            "hard\n",
            "medium\n",
            "medium\n",
            "hard\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "medium\n",
            "hard\n",
            "hard\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "hard\n",
            "medium\n",
            "easy\n",
            "medium\n",
            "medium\n",
            "medium\n",
            "hard\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "medium\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "hard\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "medium\n",
            "easy\n",
            "easy\n",
            "medium\n",
            "hard\n",
            "easy\n",
            "medium\n",
            "hard\n",
            "hard\n",
            "easy\n",
            "easy\n",
            "medium\n",
            "medium\n",
            "easy\n",
            "medium\n",
            "medium\n",
            "easy\n",
            "easy\n",
            "medium\n",
            "medium\n",
            "medium\n",
            "hard\n",
            "medium\n",
            "medium\n",
            "easy\n",
            "hard\n",
            "medium\n",
            "hard\n",
            "medium\n",
            "hard\n",
            "medium\n",
            "hard\n",
            "medium\n",
            "hard\n",
            "easy\n",
            "easy\n",
            "hard\n",
            "medium\n",
            "medium\n",
            "hard\n",
            "easy\n",
            "easy\n",
            "easy\n",
            "hard\n",
            "medium\n",
            "easy\n",
            "easy\n",
            "medium\n",
            "easy\n",
            "medium\n",
            "easy\n",
            "easy\n",
            "medium\n",
            "hard\n",
            "hard\n",
            "medium\n",
            "medium\n",
            "easy\n",
            "medium\n",
            "easy\n",
            "medium\n",
            "medium\n",
            "hard\n",
            "medium\n",
            "medium\n",
            "medium\n",
            "easy\n",
            "hard\n",
            "medium\n",
            "easy\n",
            "hard\n",
            "hard\n",
            "medium\n",
            "medium\n",
            "easy\n",
            "medium\n",
            "hard\n",
            "easy\n",
            "medium\n",
            "hard\n",
            "easy\n",
            "hard\n",
            "hard\n",
            "medium\n",
            "easy\n",
            "hard\n",
            "hard\n",
            "hard\n",
            "hard\n",
            "medium\n",
            "hard\n",
            "easy\n",
            "medium\n",
            "easy\n",
            "easy\n",
            "medium\n",
            "easy\n",
            "hard\n",
            "easy\n",
            "hard\n",
            "hard\n",
            "easy\n",
            "medium\n",
            "easy\n",
            "hard\n",
            "medium\n",
            "easy\n",
            "easy\n",
            "hard\n",
            "easy\n",
            "hard\n",
            "medium\n",
            "hard\n",
            "hard\n",
            "hard\n",
            "medium\n",
            "medium\n",
            "hard\n",
            "medium\n",
            "easy\n",
            "medium\n",
            "hard\n",
            "easy\n",
            "hard\n",
            "hard\n",
            "easy\n",
            "easy\n",
            "hard\n",
            "easy\n",
            "easy\n",
            "hard\n",
            "medium\n",
            "medium\n",
            "medium\n",
            "hard\n",
            "easy\n",
            "hard\n",
            "hard\n",
            "medium\n",
            "medium\n",
            "medium\n",
            "medium\n",
            "medium\n",
            "easy\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UDmycMYMwfKE"
      },
      "source": [
        "def get_confusion_matrix(predicted,actual):\n",
        "    conf_matrix = np.zeros((5, 5))\n",
        "    for pred,act in zip(predicted,actual):\n",
        "        conf_matrix[act,pred]+=1\n",
        "    return conf_matrix\n",
        "        \n",
        "def get_TP(confusion_matrix,label):\n",
        "    tp = confusion_matrix[label][label]\n",
        "    return tp\n",
        "\n",
        "def get_FN(confusion_matrix,label):\n",
        "    row = confusion_matrix[label,]\n",
        "    row_truepositives = row[label]\n",
        "    fn = row.sum() - row_truepositives\n",
        "    return fn\n",
        "\n",
        "def get_FP(confusion_matrix,tag):\n",
        "    col = confusion_matrix[:,tag]\n",
        "    col_tp = col[tag]\n",
        "    #  sum of all values in column except tp\n",
        "    fp = col.sum() - col_tp\n",
        "    return fp\n",
        "def Precision(conf_matrix):\n",
        "    precision = 0.0\n",
        "    for label in [0,1,2,3,4]:\n",
        "        dividor= get_TP(conf_matrix,label)+get_FP(conf_matrix,label)\n",
        "        if dividor != 0.0:\n",
        "            precision += (get_TP(conf_matrix,label))/dividor\n",
        "    return (precision / 5)\n",
        "\n",
        "def Recall(conf_matrix):\n",
        "    recall = 0.0\n",
        "    for label in [0,1,2,3,4]:\n",
        "        dividor=get_TP(conf_matrix,label)+get_FN(conf_matrix,label)\n",
        "        if dividor != 0.0:\n",
        "            recall += (get_TP(conf_matrix,label))/dividor\n",
        "    return (recall / 5)\n",
        "\n",
        "def F1(precision,recall):\n",
        "    return (2*precision*recall)/(precision+recall)\n",
        "def accuracy_per_class(preds_flat, labels_flat):\n",
        "\n",
        "    for label in np.unique(labels_flat):\n",
        "        y_preds = preds_flat[labels_flat==label]\n",
        "        y_true = labels_flat[labels_flat==label]\n",
        "        print(f'Class: {label}')\n",
        "        print(f'Accuracy: {len(y_preds[y_preds==label])}/{len(y_true)}\\n')\n",
        "def print_metrics(predictions,test_labels):\n",
        "    conf_matrix = get_confusion_matrix(predictions,test_labels)\n",
        "    precision = Precision(conf_matrix)\n",
        "    recall = Recall(conf_matrix)\n",
        "    f1_score = F1(precision,recall)\n",
        "    print(\"Macro : Precision:{}, Recall: {}, F1: {}\".format(precision,recall,f1_score))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nJ_3uYGDwfKG"
      },
      "source": [
        "def Precision_macro_weighted(conf_matrix,test_samples):\n",
        "    accum =0\n",
        "    label_wise_precision = dict()\n",
        "    for label in [0,1,2,3,4]:\n",
        "        true_sample = [sample for sample in test_samples if sample==label ]\n",
        "        if (get_TP(conf_matrix,label)+get_FP(conf_matrix,label))!=0:\n",
        "            accum+= float(len(true_sample)) *(get_TP(conf_matrix,label)/(get_TP(conf_matrix,label)+get_FP(conf_matrix,label)))\n",
        "            label_wise_precision[label] = get_TP(conf_matrix,label)/(get_TP(conf_matrix,label)+get_FP(conf_matrix,label))\n",
        "\n",
        "    \n",
        "    precision =  accum/len(test_samples)\n",
        "            \n",
        "    return precision\n",
        "\n",
        "\n",
        "def Recall_macro_weighted(conf_matrix,test_samples):\n",
        "    accum =0\n",
        "    label_wise_recall = dict()\n",
        "    for label in [0,1,2,3,4]:\n",
        "        true_sample = [sample for sample in test_samples if sample==label ]\n",
        "\n",
        "        if (get_TP(conf_matrix,label)+get_FN(conf_matrix,label))!=0:\n",
        "            accum+= float(len(true_sample)) * (get_TP(conf_matrix,label)/(get_TP(conf_matrix,label)+get_FN(conf_matrix,label)))\n",
        "            label_wise_recall[label] = get_TP(conf_matrix,label)/(get_TP(conf_matrix,label)+get_FN(conf_matrix,label))\n",
        "\n",
        "    \n",
        "    recall =  accum/len(test_samples)\n",
        "    return recall\n",
        "def print_weighted_metrics(predictions,test_labels):\n",
        "    conf_matrix = get_confusion_matrix(predictions,test_labels)\n",
        "    precision = Precision_macro_weighted(conf_matrix,test_labels)\n",
        "    recall = Recall_macro_weighted(conf_matrix,test_labels)\n",
        "    f1_score = F1(precision,recall)\n",
        "    print(\"Macro : Precision:{}, Recall: {}, F1: {}\".format(precision,recall,f1_score))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print_metrics(np.array(predicted_skill_labels), test[\"skill_label\"].values)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2281662d-c87c-4fc7-acdf-ba5b45fde6c7",
        "id": "mzacPYBbxUHI"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Macro : Precision:0.1309941520467836, Recall: 0.2, F1: 0.15830388692579503\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print_weighted_metrics(np.array(predicted_skill_labels), test[\"skill_label\"].values)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8df0ec3c-0b8e-4bf1-e2e9-18f63840a9d8",
        "id": "DykqPmJfxUHL"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Macro : Precision:0.42898669676139667, Recall: 0.6549707602339181, F1: 0.518422085838861\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "8PosEu77wbOk"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}