Work_Allocation.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m5tUoHe9FRhs"
      },
      "source": [
        "## Import Libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Dr7BCHS-nIRW",
        "scrolled": true,
        "outputId": "4f8108cf-1602-4c98-aa89-cb2e0388bf7c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.30.2-py3-none-any.whl (7.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.2/7.2 MB\u001b[0m \u001b[31m61.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.0)\n",
            "Collecting huggingface-hub<1.0,>=0.14.1 (from transformers)\n",
            "  Downloading huggingface_hub-0.15.1-py3-none-any.whl (236 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m236.8/236.8 kB\u001b[0m \u001b[31m29.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.22.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.27.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m97.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers)\n",
            "  Downloading safetensors-0.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m67.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (2023.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Installing collected packages: tokenizers, safetensors, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.15.1 safetensors-0.3.1 tokenizers-0.13.3 transformers-4.30.2\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import torch\n",
        "from torch.nn import BCEWithLogitsLoss, BCELoss\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import MultiLabelBinarizer\n",
        "from sklearn.metrics import classification_report, confusion_matrix, multilabel_confusion_matrix, f1_score, accuracy_score\n",
        "import pickle\n",
        "#from transformers import *\n",
        "from tqdm import tqdm, trange\n",
        "from ast import literal_eval"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UhjnJEwKnISB",
        "outputId": "beabd055-2d23-4c54-b481-111ce954b9c1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found GPU at: /device:GPU:0\n"
          ]
        }
      ],
      "source": [
        "device_name = tf.test.gpu_device_name()\n",
        "if device_name != '/device:GPU:0':\n",
        "  raise SystemError('GPU device not found')\n",
        "print('Found GPU at: {}'.format(device_name))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "uorMX_zrnISM",
        "scrolled": true,
        "outputId": "c17a34f7-3df3-4fab-9327-583a1e055120"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Tesla T4'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "n_gpu = torch.cuda.device_count()\n",
        "torch.cuda.get_device_name(0)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NHcY8kii7Qot"
      },
      "source": [
        "# Data Exploration"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 734
        },
        "id": "62QeEJcKtWm3",
        "outputId": "f1afda6e-2ce6-4aad-8b80-617d7d320af7"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                           sentence1  \\\n",
              "0  \" They are trying to turn him into a martyr , ...   \n",
              "1  Of the 23.5 million high-speed lines , 16.3 mi...   \n",
              "2  The news sent Cisco shares on a tear in after-...   \n",
              "3  The technology-laced Nasdaq Composite Index .I...   \n",
              "4  Doctors would face fines and up to two years i...   \n",
              "\n",
              "                                           sentence2  label   idx  \\\n",
              "0  \" We need to take these threats seriously , \" ...      0  1245   \n",
              "1  A total of 16.3 million lines provided advance...      0  2804   \n",
              "2  Cisco shares gained $ 1.28 in after-hours trad...      0  3641   \n",
              "3  The Standard & Poor 's 500 Index .SPX gained 1...      0  3296   \n",
              "4  Physicians who perform the procedure would fac...      0  1018   \n",
              "\n",
              "                                                text  \\\n",
              "0  Sentence1: \" They are trying to turn him into ...   \n",
              "1  Sentence1: Of the 23.5 million high-speed line...   \n",
              "2  Sentence1: The news sent Cisco shares on a tea...   \n",
              "3  Sentence1: The technology-laced Nasdaq Composi...   \n",
              "4  Sentence1: Doctors would face fines and up to ...   \n",
              "\n",
              "                                             prompt1  \\\n",
              "0  Please label if the following two sentences ar...   \n",
              "1  Please label if the following two sentences ar...   \n",
              "2  Please label if the following two sentences ar...   \n",
              "3  Please label if the following two sentences ar...   \n",
              "4  Please label if the following two sentences ar...   \n",
              "\n",
              "                                             prompt2  \\\n",
              "0  Sentence1: \" They are trying to turn him into ...   \n",
              "1  Sentence1: Of the 23.5 million high-speed line...   \n",
              "2  Sentence1: The news sent Cisco shares on a tea...   \n",
              "3  Sentence1: The technology-laced Nasdaq Composi...   \n",
              "4  Sentence1: Doctors would face fines and up to ...   \n",
              "\n",
              "                                             prompt3  \\\n",
              "0  Given the following two sentences, please clas...   \n",
              "1  Given the following two sentences, please clas...   \n",
              "2  Given the following two sentences, please clas...   \n",
              "3  Given the following two sentences, please clas...   \n",
              "4  Given the following two sentences, please clas...   \n",
              "\n",
              "                                             prompt4  \\\n",
              "0  Is it true that the following two sentences ar...   \n",
              "1  Is it true that the following two sentences ar...   \n",
              "2  Is it true that the following two sentences ar...   \n",
              "3  Is it true that the following two sentences ar...   \n",
              "4  Is it true that the following two sentences ar...   \n",
              "\n",
              "                                             prompt5  ... pred2 pred3 pred4  \\\n",
              "0  Is it true that the following two sentences ar...  ...     1     0     0   \n",
              "1  Is it true that the following two sentences ar...  ...     1     1     1   \n",
              "2  Is it true that the following two sentences ar...  ...     1     1     0   \n",
              "3  Is it true that the following two sentences ar...  ...     0     0     0   \n",
              "4  Is it true that the following two sentences ar...  ...     1     1     1   \n",
              "\n",
              "  pred5 pred6 pred7 pred8 pred9   entropy MV  \n",
              "0     0     0     0     0     0  0.503258  0  \n",
              "1     1     1     1     1     1 -0.000000  1  \n",
              "2     1     1     0     1     1  0.764205  1  \n",
              "3     0     0     0     0     0 -0.000000  0  \n",
              "4     1     1     1     1     0  0.503258  1  \n",
              "\n",
              "[5 rows x 34 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0f368c40-61af-48b9-8449-e8f8c253c462\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence1</th>\n",
              "      <th>sentence2</th>\n",
              "      <th>label</th>\n",
              "      <th>idx</th>\n",
              "      <th>text</th>\n",
              "      <th>prompt1</th>\n",
              "      <th>prompt2</th>\n",
              "      <th>prompt3</th>\n",
              "      <th>prompt4</th>\n",
              "      <th>prompt5</th>\n",
              "      <th>...</th>\n",
              "      <th>pred2</th>\n",
              "      <th>pred3</th>\n",
              "      <th>pred4</th>\n",
              "      <th>pred5</th>\n",
              "      <th>pred6</th>\n",
              "      <th>pred7</th>\n",
              "      <th>pred8</th>\n",
              "      <th>pred9</th>\n",
              "      <th>entropy</th>\n",
              "      <th>MV</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>\" They are trying to turn him into a martyr , ...</td>\n",
              "      <td>\" We need to take these threats seriously , \" ...</td>\n",
              "      <td>0</td>\n",
              "      <td>1245</td>\n",
              "      <td>Sentence1: \" They are trying to turn him into ...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: \" They are trying to turn him into ...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.503258</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Of the 23.5 million high-speed lines , 16.3 mi...</td>\n",
              "      <td>A total of 16.3 million lines provided advance...</td>\n",
              "      <td>0</td>\n",
              "      <td>2804</td>\n",
              "      <td>Sentence1: Of the 23.5 million high-speed line...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: Of the 23.5 million high-speed line...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.000000</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>The news sent Cisco shares on a tear in after-...</td>\n",
              "      <td>Cisco shares gained $ 1.28 in after-hours trad...</td>\n",
              "      <td>0</td>\n",
              "      <td>3641</td>\n",
              "      <td>Sentence1: The news sent Cisco shares on a tea...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: The news sent Cisco shares on a tea...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.764205</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>The technology-laced Nasdaq Composite Index .I...</td>\n",
              "      <td>The Standard &amp; Poor 's 500 Index .SPX gained 1...</td>\n",
              "      <td>0</td>\n",
              "      <td>3296</td>\n",
              "      <td>Sentence1: The technology-laced Nasdaq Composi...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: The technology-laced Nasdaq Composi...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.000000</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Doctors would face fines and up to two years i...</td>\n",
              "      <td>Physicians who perform the procedure would fac...</td>\n",
              "      <td>0</td>\n",
              "      <td>1018</td>\n",
              "      <td>Sentence1: Doctors would face fines and up to ...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: Doctors would face fines and up to ...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.503258</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 34 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0f368c40-61af-48b9-8449-e8f8c253c462')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-0f368c40-61af-48b9-8449-e8f8c253c462 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-0f368c40-61af-48b9-8449-e8f8c253c462');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "import pandas as pd\n",
        "data=pd.read_csv('mrpc_train(entropy).csv')\n",
        "data.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 472
        },
        "id": "fEYeBp_u5E11",
        "outputId": "53d1cb54-1b5b-4715-d310-acdaa8f91055"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkEAAAHHCAYAAAC4BYz1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA1JUlEQVR4nO3deXxU1f3/8fdkZctCgCSmhEUWIbLVUCGKIhAIEikICgJCwFBAA7IICHVBwAKiImqjuAZsBRS/Lq2sEQSspCKbIiKLgEFJAhayEErW+/vDR+bXIUHJZJJJOK/n4zEPmXPP3Ps5xyF5c+69MzbLsiwBAAAYxsPdBQAAALgDIQgAABiJEAQAAIxECAIAAEYiBAEAACMRggAAgJEIQQAAwEiEIAAAYCRCEAAAMBIhCLiKNGvWTKNHj3Z3GVe9p59+Wtdee608PT3VqVOnKjvu8uXLZbPZdOLEiSo7JnA1IwQB1VTJL7xdu3aVuf22225Tu3btKnycdevW6YknnqjwfkyxadMmzZw5UzfffLOSkpK0YMGCUn0KCgrUsGFDdevW7bL7sSxL4eHhuuGGGyqzXAC/wsvdBQBwnUOHDsnDo3z/tlm3bp0SExMJQldoy5Yt8vDw0BtvvCEfH58y+3h7e+vuu+/WK6+8oh9++EFNmzYt1Wf79u368ccfNXXq1MouGcBlsBIEXEV8fX3l7e3t7jLKJTc3190llMvp06dVu3btywagEiNGjJBlWVq1alWZ21euXCkPDw/dc889lVEmgCtACAKuIpdeE1RQUKC5c+eqVatWqlWrlho0aKBu3bopOTlZkjR69GglJiZKkmw2m/1RIjc3Vw899JDCw8Pl6+ur6667Ts8884wsy3I47n//+189+OCDatiwofz8/PTHP/5RP/30k2w2m8MK0xNPPCGbzaZvv/1Ww4cPV/369e2njL7++muNHj1a1157rWrVqqXQ0FDdd999+s9//uNwrJJ9HD58WPfee68CAgLUqFEjPfbYY7IsSydPntSAAQPk7++v0NBQPfvss1c0d4WFhZo/f75atGghX19fNWvWTH/+85+Vl5dn72Oz2ZSUlKTc3Fz7XC1fvrzM/d18881q1qyZVq5cWWpbQUGB3nvvPfXo0UNhYWFXPPayXDrHJcq6PiwzM1NTpkyx//9s2bKlnnrqKRUXFzv0W716tSIjI+Xn5yd/f3+1b99ezz///G/WAtQ0nA4DqrmsrCz9/PPPpdoLCgp+87VPPPGEFi5cqLFjx+rGG29Udna2du3apT179qh3794aP368Tp06peTkZP3tb39zeK1lWfrjH/+oTz/9VPHx8erUqZM2btyoGTNm6KefftJzzz1n7zt69Gi9++67GjlypLp27apt27YpNjb2snXdfffdatWqlRYsWGAPVMnJyTp27JjGjBmj0NBQHThwQK+++qoOHDigf//73w7hTJKGDh2qtm3batGiRVq7dq2efPJJBQUF6ZVXXlHPnj311FNP6e2339b06dP1hz/8QbfeeuuvztXYsWO1YsUK3XXXXXrooYf0xRdfaOHChTp48KA++OADSdLf/vY3vfrqq9q5c6def/11SdJNN91U5v5sNpuGDx+uBQsW6MCBA7r++uvt2zZs2KCzZ89qxIgRTo3dGRcuXFD37t31008/afz48WrSpIl27Nih2bNnKy0tTUuXLrXXMmzYMPXq1UtPPfWUJOngwYP6/PPPNXny5ArXAVQrFoBqKSkpyZL0q4/rr7/e4TVNmza14uLi7M87duxoxcbG/upxEhISrLJ+FHz44YeWJOvJJ590aL/rrrssm81mHT161LIsy9q9e7clyZoyZYpDv9GjR1uSrDlz5tjb5syZY0myhg0bVup4Fy5cKNW2atUqS5K1ffv2UvsYN26cva2wsNBq3LixZbPZrEWLFtnbz507Z9WuXdthTsqyb98+S5I1duxYh/bp06dbkqwtW7bY2+Li4qy6dev+6v5KHDhwwJJkzZ4926H9nnvusWrVqmVlZWVZlnXlYy95Txw/ftzedukcl7j0vTB//nyrbt261uHDhx36zZo1y/L09LRSU1Mty7KsyZMnW/7+/lZhYeEVjRGoyTgdBlRziYmJSk5OLvXo0KHDb742MDBQBw4c0JEjR8p93HXr1snT01MPPvigQ/tDDz0ky7K0fv16Sb+sakjSAw884NBv0qRJl933hAkTSrXVrl3b/ueLFy/q559/VteuXSVJe/bsKdV/7Nix9j97enqqc+fOsixL8fHx9vbAwEBdd911Onbs2GVrkX4ZqyRNmzbNof2hhx6SJK1du/ZXX385ERER+v3vf6/Vq1fb23Jzc/WPf/xDd9xxh/z9/SWVf+zOWLNmjW655RbVr19fP//8s/0RHR2toqIibd++XdIvc5abm2s/ZQpczQhBQDV34403Kjo6utSjfv36v/naefPmKTMzU61bt1b79u01Y8YMff3111d03B9++EFhYWHy8/NzaG/btq19e8l/PTw81Lx5c4d+LVu2vOy+L+0rSWfPntXkyZMVEhKi2rVrq1GjRvZ+WVlZpfo3adLE4XlAQIBq1aqlhg0blmo/d+7cZWv53zFcWnNoaKgCAwPtY3XGiBEjdPz4ce3YsUOS9OGHH+rChQv2U2FS+cfujCNHjmjDhg1q1KiRwyM6OlrSLxd8S7+E2datW+v2229X48aNdd9999mDLnC14Zog4Cp266236vvvv9dHH32kTZs26fXXX9dzzz2nZcuWOaykVLX/XfkoMWTIEO3YsUMzZsxQp06dVK9ePRUXF6tv376lLtyVfln9uZI2SaUu5L4cV1x7c6lhw4Zp5syZWrlypW666SatXLlS9evXV79+/ex9yjv2K1FUVOTwvLi4WL1799bMmTPL7N+6dWtJUnBwsPbt26eNGzdq/fr1Wr9+vZKSkjRq1CitWLHCqVqA6ooQBFzlgoKCNGbMGI0ZM0bnz5/XrbfeqieeeMIegi73i79p06b65JNPlJOT47Aa9N1339m3l/y3uLhYx48fV6tWrez9jh49esU1njt3Tps3b9bcuXP1+OOP29udOY3njJIxHDlyxL7SJUkZGRnKzMws83N+rlRYWJh69OihNWvW6LHHHlNycrJGjx5tv8W+omOvX7++MjMzHdry8/OVlpbm0NaiRQudP3/evvLza3x8fNS/f3/1799fxcXFeuCBB/TKK6/oscce+9UVPqCm4XQYcBW79BbrevXqqWXLlg63fdetW1eSSv0i7devn4qKivTXv/7Vof25556TzWbT7bffLkmKiYmRJL300ksO/V588cUrrrNkBefSFZuSO5YqW8mqzKXHW7JkiST96p1uV2LEiBE6ffq0xo8fr4KCAodTYRUde4sWLezX85R49dVXS60EDRkyRCkpKdq4cWOpfWRmZqqwsFBS6feMh4eH/fqz/33fAFcDVoKAq1hERIRuu+02RUZGKigoSLt27dJ7772niRMn2vtERkZKkh588EHFxMTI09NT99xzj/r3768ePXrokUce0YkTJ9SxY0dt2rRJH330kaZMmaIWLVrYXz948GAtXbpU//nPf+y3yB8+fFjSlZ1i8vf316233qrFixeroKBAv/vd77Rp0yYdP368EmaltI4dOyouLk6vvvqqMjMz1b17d+3cuVMrVqzQwIED1aNHjwrtf/DgwXrggQf00UcfKTw83OF2/YqOfezYsZowYYIGDx6s3r1766uvvtLGjRtLXRs1Y8YM+wXZo0ePVmRkpHJzc7V//3699957OnHihBo2bKixY8fq7Nmz6tmzpxo3bqwffvhBL774ojp16uSwSgZcFdx6bxqAyyq5HfrLL78sc3v37t1/8xb5J5980rrxxhutwMBAq3bt2labNm2sv/zlL1Z+fr69T2FhoTVp0iSrUaNGls1mc7hdPicnx5o6daoVFhZmeXt7W61atbKefvppq7i42OG4ubm5VkJCghUUFGTVq1fPGjhwoHXo0CFLksMt6yW3t585c6bUeH788UfrzjvvtAIDA62AgADr7rvvtk6dOnXZ2+wv3cflbl0va57KUlBQYM2dO9dq3ry55e3tbYWHh1uzZ8+2Ll68eEXH+S133323JcmaOXNmqW1XOvaybpEvKiqyHn74Yathw4ZWnTp1rJiYGOvo0aOl3guW9cv/z9mzZ1stW7a0fHx8rIYNG1o33XST9cwzz9jfE++9957Vp08fKzg42PLx8bGaNGlijR8/3kpLSyv3mIHqzmZZV3jFIACUw759+/T73/9ef//73x1O/wBAdcE1QQAq7L///W+ptqVLl8rDw+M3P6kZANyFa4IAVNjixYu1e/du9ejRQ15eXvZbq8eNG6fw8HB3lwcAZeJ0GIAKS05O1ty5c/Xtt9/q/PnzatKkiUaOHKlHHnlEXl78WwtA9UQIAgAARuKaIAAAYCRCEAAAMBIn6/XLd+qcOnVKfn5+lfLdQQAAwPUsy1JOTo7CwsLk4VH+dR1CkKRTp05xBwsAADXUyZMn1bhx43K/jhAk2b8c8uTJk/L393dzNQAA4EpkZ2crPDzc4Uuey4MQpP//3Ub+/v6EIAAAahhnL2XhwmgAAGAkQhAAADASIQgAABiJEAQAAIxECAIAAEYiBAEAACMRggAAgJEIQQAAwEiEIAAAYCRCEAAAMBIhCAAAGIkQBAAAjEQIAgAARiIEAQAAIxGCAACAkbzcXcDVrtmstW457olFsW45LgAANQUrQQAAwEiEIAAAYCRCEAAAMBIhCAAAGIkQBAAAjEQIAgAARiIEAQAAIxGCAACAkQhBAADASIQgAABgJEIQAAAwEiEIAAAYiRAEAACMRAgCAABGIgQBAAAjEYIAAICRCEEAAMBIhCAAAGAkQhAAADASIQgAABiJEAQAAIxECAIAAEYiBAEAACMRggAAgJEIQQAAwEiEIAAAYCRCEAAAMBIhCAAAGIkQBAAAjEQIAgAARiIEAQAAIxGCAACAkQhBAADASIQgAABgJEIQAAAwEiEIAAAYiRAEAACMRAgCAABGIgQBAAAjEYIAAICRCEEAAMBIhCAAAGCkahOCFi1aJJvNpilTptjbLl68qISEBDVo0ED16tXT4MGDlZGR4fC61NRUxcbGqk6dOgoODtaMGTNUWFhYxdUDAICaplqEoC+//FKvvPKKOnTo4NA+depU/fOf/9SaNWu0bds2nTp1SoMGDbJvLyoqUmxsrPLz87Vjxw6tWLFCy5cv1+OPP17VQwAAADWM20PQ+fPnNWLECL322muqX7++vT0rK0tvvPGGlixZop49eyoyMlJJSUnasWOH/v3vf0uSNm3apG+//VZ///vf1alTJ91+++2aP3++EhMTlZ+f764hAQCAGsDtISghIUGxsbGKjo52aN+9e7cKCgoc2tu0aaMmTZooJSVFkpSSkqL27dsrJCTE3icmJkbZ2dk6cODAZY+Zl5en7OxshwcAADCLlzsPvnr1au3Zs0dffvllqW3p6eny8fFRYGCgQ3tISIjS09Ptff43AJVsL9l2OQsXLtTcuXMrWD0AAKjJ3LYSdPLkSU2ePFlvv/22atWqVaXHnj17trKysuyPkydPVunxAQCA+7ktBO3evVunT5/WDTfcIC8vL3l5eWnbtm164YUX5OXlpZCQEOXn5yszM9PhdRkZGQoNDZUkhYaGlrpbrOR5SZ+y+Pr6yt/f3+EBAADM4rYQ1KtXL+3fv1/79u2zPzp37qwRI0bY/+zt7a3NmzfbX3Po0CGlpqYqKipKkhQVFaX9+/fr9OnT9j7Jycny9/dXRERElY8JAADUHG67JsjPz0/t2rVzaKtbt64aNGhgb4+Pj9e0adMUFBQkf39/TZo0SVFRUerataskqU+fPoqIiNDIkSO1ePFipaen69FHH1VCQoJ8fX2rfEwAAKDmcOuF0b/lueeek4eHhwYPHqy8vDzFxMTopZdesm/39PTUxx9/rPvvv19RUVGqW7eu4uLiNG/ePDdWDQAAagKbZVmWu4twt+zsbAUEBCgrK8vl1wc1m7XWpfu7UicWxbrluAAAVJWK/v52++cEAQAAuAMhCAAAGIkQBAAAjEQIAgAARiIEAQAAIxGCAACAkQhBAADASIQgAABgJEIQAAAwEiEIAAAYiRAEAACMRAgCAABGIgQBAAAjEYIAAICRCEEAAMBIhCAAAGAkQhAAADASIQgAABiJEAQAAIxECAIAAEYiBAEAACMRggAAgJEIQQAAwEiEIAAAYCRCEAAAMBIhCAAAGIkQBAAAjEQIAgAARiIEAQAAIxGCAACAkQhBAADASIQgAABgJEIQAAAwEiEIAAAYiRAEAACMRAgCAABGIgQBAAAjEYIAAICRCEEAAMBIhCAAAGAkQhAAADASIQgAABiJEAQAAIxECAIAAEYiBAEAACMRggAAgJEIQQAAwEiEIAAAYCRCEAAAMBIhCAAAGIkQBAAAjEQIAgAARiIEAQAAIxGCAACAkQhBAADASIQgAABgJEIQAAAwEiEIAAAYiRAEAACMRAgCAABGIgQBAAAjEYIAAICRCEEAAMBIhCAAAGAkQhAAADCSW0PQyy+/rA4dOsjf31/+/v6KiorS+vXr7dsvXryohIQENWjQQPXq1dPgwYOVkZHhsI/U1FTFxsaqTp06Cg4O1owZM1RYWFjVQwEAADWMW0NQ48aNtWjRIu3evVu7du1Sz549NWDAAB04cECSNHXqVP3zn//UmjVrtG3bNp06dUqDBg2yv76oqEixsbHKz8/Xjh07tGLFCi1fvlyPP/64u4YEAABqCJtlWZa7i/hfQUFBevrpp3XXXXepUaNGWrlype666y5J0nfffae2bdsqJSVFXbt21fr163XHHXfo1KlTCgkJkSQtW7ZMDz/8sM6cOSMfH58rOmZ2drYCAgKUlZUlf39/l46n2ay1Lt3flTqxKNYtx0XV4b0FwHQV/f1dba4JKioq0urVq5Wbm6uoqCjt3r1bBQUFio6Otvdp06aNmjRpopSUFElSSkqK2rdvbw9AkhQTE6Ps7Gz7alJZ8vLylJ2d7fAAAABmcXsI2r9/v+rVqydfX19NmDBBH3zwgSIiIpSeni4fHx8FBgY69A8JCVF6erokKT093SEAlWwv2XY5CxcuVEBAgP0RHh7u2kEBAIBqz+0h6LrrrtO+ffv0xRdf6P7771dcXJy+/fbbSj3m7NmzlZWVZX+cPHmyUo8HAACqHy93F+Dj46OWLVtKkiIjI/Xll1/q+eef19ChQ5Wfn6/MzEyH1aCMjAyFhoZKkkJDQ7Vz506H/ZXcPVbSpyy+vr7y9fV18UgAAEBN4vaVoEsVFxcrLy9PkZGR8vb21ubNm+3bDh06pNTUVEVFRUmSoqKitH//fp0+fdreJzk5Wf7+/oqIiKjy2gEAQM3h1pWg2bNn6/bbb1eTJk2Uk5OjlStXauvWrdq4caMCAgIUHx+vadOmKSgoSP7+/po0aZKioqLUtWtXSVKfPn0UERGhkSNHavHixUpPT9ejjz6qhIQEVnoAAMCvcmsIOn36tEaNGqW0tDQFBASoQ4cO2rhxo3r37i1Jeu655+Th4aHBgwcrLy9PMTExeumll+yv9/T01Mcff6z7779fUVFRqlu3ruLi4jRv3jx3DQkAANQQ1e5zgtyBzwlCTcR7C4DprprPCQIAAKhKhCAAAGAkQhAAADASIQgAABjJqRB07NgxV9cBAABQpZwKQS1btlSPHj3097//XRcvXnR1TQAAAJXOqRC0Z88edejQQdOmTVNoaKjGjx9f6usrAAAAqjOnQlCnTp30/PPP69SpU3rzzTeVlpambt26qV27dlqyZInOnDnj6joBAABcqkIXRnt5eWnQoEFas2aNnnrqKR09elTTp09XeHi4/ZOgAQAAqqMKhaBdu3bpgQce0DXXXKMlS5Zo+vTp+v7775WcnKxTp05pwIABrqoTAADApZz67rAlS5YoKSlJhw4dUr9+/fTWW2+pX79+8vD4JVM1b95cy5cvV7NmzVxZKwAAgMs4FYJefvll3XfffRo9erSuueaaMvsEBwfrjTfeqFBxAAAAlcWpEHTkyJHf7OPj46O4uDhndg8AAFDpnLomKCkpSWvWrCnVvmbNGq1YsaLCRQEAAFQ2p0LQwoUL1bBhw1LtwcHBWrBgQYWLAgAAqGxOhaDU1FQ1b968VHvTpk2Vmppa4aIAAAAqm1MhKDg4WF9//XWp9q+++koNGjSocFEAAACVzakQNGzYMD344IP69NNPVVRUpKKiIm3ZskWTJ0/WPffc4+oaAQAAXM6pu8Pmz5+vEydOqFevXvLy+mUXxcXFGjVqFNcEAQCAGsGpEOTj46N33nlH8+fP11dffaXatWurffv2atq0qavrAwAAqBROhaASrVu3VuvWrV1VCwAAQJVxKgQVFRVp+fLl2rx5s06fPq3i4mKH7Vu2bHFJcQAAAJXFqRA0efJkLV++XLGxsWrXrp1sNpur6wIAAKhUToWg1atX691331W/fv1cXQ8AAECVcOoWeR8fH7Vs2dLVtQAAAFQZp0LQQw89pOeff16WZbm6HgAAgCrh1Omwf/3rX/r000+1fv16XX/99fL29nbY/v7777ukOAAAgMriVAgKDAzUnXfe6epaAAAAqoxTISgpKcnVdQAAAFQpp64JkqTCwkJ98skneuWVV5STkyNJOnXqlM6fP++y4gAAACqLUytBP/zwg/r27avU1FTl5eWpd+/e8vPz01NPPaW8vDwtW7bM1XUCAAC4lFMrQZMnT1bnzp117tw51a5d295+5513avPmzS4rDgAAoLI4tRL02WefaceOHfLx8XFob9asmX766SeXFAYAAFCZnFoJKi4uVlFRUan2H3/8UX5+fhUuCgAAoLI5FYL69OmjpUuX2p/bbDadP39ec+bM4as0AABAjeDU6bBnn31WMTExioiI0MWLFzV8+HAdOXJEDRs21KpVq1xdIwAAgMs5FYIaN26sr776SqtXr9bXX3+t8+fPKz4+XiNGjHC4UBoAAKC6cioESZKXl5fuvfdeV9YCAABQZZwKQW+99davbh81apRTxQAAAFQVp0LQ5MmTHZ4XFBTowoUL8vHxUZ06dQhBAACg2nPq7rBz5845PM6fP69Dhw6pW7duXBgNAABqBKe/O+xSrVq10qJFi0qtEgEAAFRHLgtB0i8XS586dcqVuwQAAKgUTl0T9I9//MPhuWVZSktL01//+lfdfPPNLikMAACgMjkVggYOHOjw3GazqVGjRurZs6eeffZZV9QFAABQqZwKQcXFxa6uAwAAoEq59JogAACAmsKplaBp06Zdcd8lS5Y4cwgAAIBK5VQI2rt3r/bu3auCggJdd911kqTDhw/L09NTN9xwg72fzWZzTZUAAAAu5lQI6t+/v/z8/LRixQrVr19f0i8foDhmzBjdcssteuihh1xaJAAAgKs5dU3Qs88+q4ULF9oDkCTVr19fTz75JHeHAQCAGsGpEJSdna0zZ86Uaj9z5oxycnIqXBQAAEBlcyoE3XnnnRozZozef/99/fjjj/rxxx/1f//3f4qPj9egQYNcXSMAAIDLOXVN0LJlyzR9+nQNHz5cBQUFv+zIy0vx8fF6+umnXVogAABAZXAqBNWpU0cvvfSSnn76aX3//feSpBYtWqhu3bouLQ4AAKCyVOjDEtPS0pSWlqZWrVqpbt26sizLVXUBAABUKqdC0H/+8x/16tVLrVu3Vr9+/ZSWliZJio+P5/Z4AABQIzgVgqZOnSpvb2+lpqaqTp069vahQ4dqw4YNLisOAACgsjh1TdCmTZu0ceNGNW7c2KG9VatW+uGHH1xSGAAAQGVyaiUoNzfXYQWoxNmzZ+Xr61vhogAAACqbUyHolltu0VtvvWV/brPZVFxcrMWLF6tHjx4uKw4AAKCyOHU6bPHixerVq5d27dql/Px8zZw5UwcOHNDZs2f1+eefu7pGAAAAl3NqJahdu3Y6fPiwunXrpgEDBig3N1eDBg3S3r171aJFC1fXCAAA4HLlXgkqKChQ3759tWzZMj3yyCOVURMAAEClK/dKkLe3t77++uvKqAUAAKDKOHU67N5779Ubb7xR4YMvXLhQf/jDH+Tn56fg4GANHDhQhw4dcuhz8eJFJSQkqEGDBqpXr54GDx6sjIwMhz6pqamKjY1VnTp1FBwcrBkzZqiwsLDC9QEAgKuXUxdGFxYW6s0339Qnn3yiyMjIUt8ZtmTJkivaz7Zt25SQkKA//OEPKiws1J///Gf16dNH3377rX2fU6dO1dq1a7VmzRoFBARo4sSJGjRokP0C7KKiIsXGxio0NFQ7duxQWlqaRo0aJW9vby1YsMCZ4QEAAAPYrHJ84dexY8fUrFkz9erV6/I7tNm0ZcsWp4o5c+aMgoODtW3bNt16663KyspSo0aNtHLlSt11112SpO+++05t27ZVSkqKunbtqvXr1+uOO+7QqVOnFBISIumXb7l/+OGHdebMGfn4+PzmcbOzsxUQEKCsrCz5+/s7VfvlNJu11qX7u1InFsW65bioOry3AJiuor+/y7US1KpVK6WlpenTTz+V9MvXZLzwwgv28FFRWVlZkqSgoCBJ0u7du1VQUKDo6Gh7nzZt2qhJkyb2EJSSkqL27ds71BATE6P7779fBw4c0O9///tSx8nLy1NeXp79eXZ2tkvqBwAANUe5rgm6dNFo/fr1ys3NdUkhxcXFmjJlim6++Wa1a9dOkpSeni4fHx8FBgY69A0JCVF6erq9z6UhrOR5SZ9LLVy4UAEBAfZHeHi4S8YAAABqDqcujC5RjjNpvykhIUHffPONVq9e7bJ9Xs7s2bOVlZVlf5w8ebLSjwkAAKqXcp0Os9lsstlspdoqauLEifr444+1fft2hy9lDQ0NVX5+vjIzMx1WgzIyMhQaGmrvs3PnTof9ldw9VtLnUr6+vnzHGQAAhitXCLIsS6NHj7YHiIsXL2rChAml7g57//33r3h/kyZN0gcffKCtW7eqefPmDtsjIyPl7e2tzZs3a/DgwZKkQ4cOKTU1VVFRUZKkqKgo/eUvf9Hp06cVHBwsSUpOTpa/v78iIiLKMzwAAGCQcoWguLg4h+f33ntvhQ6ekJCglStX6qOPPpKfn5/9Gp6AgADVrl1bAQEBio+P17Rp0xQUFCR/f39NmjRJUVFR6tq1qySpT58+ioiI0MiRI7V48WKlp6fr0UcfVUJCAqs9AADgssoVgpKSklx68JdfflmSdNttt5U6zujRoyVJzz33nDw8PDR48GDl5eUpJiZGL730kr2vp6enPv74Y91///2KiopS3bp1FRcXp3nz5rm0VgAAcHVx6sMSXeVKLqyuVauWEhMTlZiYeNk+TZs21bp161xZGgAAuMpV6O4wAACAmooQBAAAjEQIAgAARiIEAQAAIxGCAACAkQhBAADASIQgAABgJEIQAAAwEiEIAAAYiRAEAACMRAgCAABGIgQBAAAjEYIAAICRCEEAAMBIhCAAAGAkQhAAADASIQgAABiJEAQAAIxECAIAAEYiBAEAACMRggAAgJEIQQAAwEiEIAAAYCRCEAAAMBIhCAAAGIkQBAAAjEQIAgAARiIEAQAAIxGCAACAkQhBAADASIQgAABgJEIQAAAwEiEIAAAYiRAEAACMRAgCAABGIgQBAAAjEYIAAICRCEEAAMBIhCAAAGAkQhAAADASIQgAABiJEAQAAIxECAIAAEYiBAEAACMRggAAgJEIQQAAwEiEIAAAYCRCEAAAMBIhCAAAGIkQBAAAjEQIAgAARiIEAQAAIxGCAACAkQhBAADASIQgAABgJEIQAAAwEiEIAAAYycvdBQBAddZs1lq3HPfEoli3HBcwCStBAADASIQgAABgJEIQAAAwEiEIAAAYiRAEAACMRAgCAABGIgQBAAAjuTUEbd++Xf3791dYWJhsNps+/PBDh+2WZenxxx/XNddco9q1ays6OlpHjhxx6HP27FmNGDFC/v7+CgwMVHx8vM6fP1+FowAAADWRW0NQbm6uOnbsqMTExDK3L168WC+88IKWLVumL774QnXr1lVMTIwuXrxo7zNixAgdOHBAycnJ+vjjj7V9+3aNGzeuqoYAAABqKLd+YvTtt9+u22+/vcxtlmVp6dKlevTRRzVgwABJ0ltvvaWQkBB9+OGHuueee3Tw4EFt2LBBX375pTp37ixJevHFF9WvXz8988wzCgsLq7KxAACAmqXaXhN0/PhxpaenKzo62t4WEBCgLl26KCUlRZKUkpKiwMBAewCSpOjoaHl4eOiLL76o8poBAEDNUW2/Oyw9PV2SFBIS4tAeEhJi35aenq7g4GCH7V5eXgoKCrL3KUteXp7y8vLsz7Ozs11VNgAAqCGq7UpQZVq4cKECAgLsj/DwcHeXBAAAqli1DUGhoaGSpIyMDIf2jIwM+7bQ0FCdPn3aYXthYaHOnj1r71OW2bNnKysry/44efKki6sHAADVXbUNQc2bN1doaKg2b95sb8vOztYXX3yhqKgoSVJUVJQyMzO1e/due58tW7aouLhYXbp0uey+fX195e/v7/AAAABmces1QefPn9fRo0ftz48fP659+/YpKChITZo00ZQpU/Tkk0+qVatWat68uR577DGFhYVp4MCBkqS2bduqb9+++tOf/qRly5apoKBAEydO1D333MOdYQAA4Fe5NQTt2rVLPXr0sD+fNm2aJCkuLk7Lly/XzJkzlZubq3HjxikzM1PdunXThg0bVKtWLftr3n77bU2cOFG9evWSh4eHBg8erBdeeKHKxwIAAGoWt4ag2267TZZlXXa7zWbTvHnzNG/evMv2CQoK0sqVKyujPAAAcBWrttcEAQAAVCZCEAAAMBIhCAAAGIkQBAAAjEQIAgAARiIEAQAAIxGCAACAkQhBAADASIQgAABgJEIQAAAwEiEIAAAYiRAEAACMRAgCAABGIgQBAAAjEYIAAICRCEEAAMBIhCAAAGAkQhAAADCSl7sLAACU1mzWWrcc98SiWLccF3AHVoIAAICRCEEAAMBInA4DANhxGg4mYSUIAAAYiRAEAACMRAgCAABGIgQBAAAjEYIAAICRCEEAAMBIhCAAAGAkQhAAADASIQgAABiJEAQAAIxECAIAAEYiBAEAACMRggAAgJEIQQAAwEiEIAAAYCRCEAAAMBIhCAAAGIkQBAAAjEQIAgAARiIEAQAAIxGCAACAkQhBAADASIQgAABgJEIQAAAwEiEIAAAYiRAEAACMRAgCAABGIgQBAAAjEYIAAICRCEEAAMBIhCAAAGAkQhAAADASIQgAABiJEAQAAIxECAIAAEYiBAEAACMRggAAgJEIQQAAwEiEIAAAYCRCEAAAMBIhCAAAGIkQBAAAjEQIAgAARrpqQlBiYqKaNWumWrVqqUuXLtq5c6e7SwIAANXYVRGC3nnnHU2bNk1z5szRnj171LFjR8XExOj06dPuLg0AAFRTV0UIWrJkif70pz9pzJgxioiI0LJly1SnTh29+eab7i4NAABUU17uLqCi8vPztXv3bs2ePdve5uHhoejoaKWkpLixMgAAytZs1lq3HPfEoli3HLe6qvEh6Oeff1ZRUZFCQkIc2kNCQvTdd9+V+Zq8vDzl5eXZn2dlZUmSsrOzXV5fcd4Fl+/zSlTGWFC98N6qGu6aZ9PwvqoaTaaucctxv5kbUyn7LXnfWJbl1OtrfAhyxsKFCzV37txS7eHh4W6opnIELHV3Bbha8d5CZeB9dXWr7P+/OTk5CggIKPfranwIatiwoTw9PZWRkeHQnpGRodDQ0DJfM3v2bE2bNs3+vLi4WGfPnlWDBg1ks9kqtd4rlZ2drfDwcJ08eVL+/v7uLsctmAPmQGIOTB+/xBxIzIFU9hxYlqWcnByFhYU5tc8aH4J8fHwUGRmpzZs3a+DAgZJ+CTWbN2/WxIkTy3yNr6+vfH19HdoCAwMruVLn+Pv7G/uGL8EcMAcSc2D6+CXmQGIOpNJz4MwKUIkaH4Ikadq0aYqLi1Pnzp114403aunSpcrNzdWYMWPcXRoAAKimrooQNHToUJ05c0aPP/640tPT1alTJ23YsKHUxdIAAAAlrooQJEkTJ0687OmvmsjX11dz5swpddrOJMwBcyAxB6aPX2IOJOZAqpw5sFnO3lcGAABQg10VnxgNAABQXoQgAABgJEIQAAAwEiEIAAAYiRDkRomJiWrWrJlq1aqlLl26aOfOnb/af82aNWrTpo1q1aql9u3ba926dVVUaeUpzxy89tpruuWWW1S/fn3Vr19f0dHRvzlnNUF53wclVq9eLZvNZv+Q0JqqvOPPzMxUQkKCrrnmGvn6+qp169Y1/u9Ceedg6dKluu6661S7dm2Fh4dr6tSpunjxYhVV63rbt29X//79FRYWJpvNpg8//PA3X7N161bdcMMN8vX1VcuWLbV8+fJKr7OylHf877//vnr37q1GjRrJ399fUVFR2rhxY9UUW0mceQ+U+Pzzz+Xl5aVOnTqV+7iEIDd55513NG3aNM2ZM0d79uxRx44dFRMTo9OnT5fZf8eOHRo2bJji4+O1d+9eDRw4UAMHDtQ333xTxZW7TnnnYOvWrRo2bJg+/fRTpaSkKDw8XH369NFPP/1UxZW7TnnnoMSJEyc0ffp03XLLLVVUaeUo7/jz8/PVu3dvnThxQu+9954OHTqk1157Tb/73e+quHLXKe8crFy5UrNmzdKcOXN08OBBvfHGG3rnnXf05z//uYord53c3Fx17NhRiYmJV9T/+PHjio2NVY8ePbRv3z5NmTJFY8eOrbFBoLzj3759u3r37q1169Zp9+7d6tGjh/r376+9e/dWcqWVp7xzUCIzM1OjRo1Sr169nDuwBbe48cYbrYSEBPvzoqIiKywszFq4cGGZ/YcMGWLFxsY6tHXp0sUaP358pdZZmco7B5cqLCy0/Pz8rBUrVlRWiZXOmTkoLCy0brrpJuv111+34uLirAEDBlRBpZWjvON/+eWXrWuvvdbKz8+vqhIrXXnnICEhwerZs6dD27Rp06ybb765UuusKpKsDz744Ff7zJw507r++usd2oYOHWrFxMRUYmVV40rGX5aIiAhr7ty5ri/IDcozB0OHDrUeffRRa86cOVbHjh3LfSxWgtwgPz9fu3fvVnR0tL3Nw8ND0dHRSklJKfM1KSkpDv0lKSYm5rL9qztn5uBSFy5cUEFBgYKCgiqrzErl7BzMmzdPwcHBio+Pr4oyK40z4//HP/6hqKgoJSQkKCQkRO3atdOCBQtUVFRUVWW7lDNzcNNNN2n37t32U2bHjh3TunXr1K9fvyqpuTq42n4eVlRxcbFycnJq7M9CZyUlJenYsWOaM2eO0/u4aj4xuib5+eefVVRUVOprPUJCQvTdd9+V+Zr09PQy+6enp1danZXJmTm41MMPP6ywsLBSPwxrCmfm4F//+pfeeOMN7du3rwoqrFzOjP/YsWPasmWLRowYoXXr1uno0aN64IEHVFBQUKEfhO7izBwMHz5cP//8s7p16ybLslRYWKgJEybU6NNh5XW5n4fZ2dn673//q9q1a7upMvd45plndP78eQ0ZMsTdpVSZI0eOaNasWfrss8/k5eV8lGElCDXSokWLtHr1an3wwQeqVauWu8upEjk5ORo5cqRee+01NWzY0N3luEVxcbGCg4P16quvKjIyUkOHDtUjjzyiZcuWubu0KrN161YtWLBAL730kvbs2aP3339fa9eu1fz5891dGtxg5cqVmjt3rt59910FBwe7u5wqUVRUpOHDh2vu3Llq3bp1hfbFSpAbNGzYUJ6ensrIyHBoz8jIUGhoaJmvCQ0NLVf/6s6ZOSjxzDPPaNGiRfrkk0/UoUOHyiyzUpV3Dr7//nudOHFC/fv3t7cVFxdLkry8vHTo0CG1aNGicot2IWfeA9dcc428vb3l6elpb2vbtq3S09OVn58vHx+fSq3Z1ZyZg8cee0wjR47U2LFjJUnt27dXbm6uxo0bp0ceeUQeHlf/v20v9/PQ39/fqFWg1atXa+zYsVqzZk2NXRF3Rk5Ojnbt2qW9e/favzO0uLhYlmXJy8tLmzZtUs+ePa9oX1f/35ZqyMfHR5GRkdq8ebO9rbi4WJs3b1ZUVFSZr4mKinLoL0nJycmX7V/dOTMHkrR48WLNnz9fGzZsUOfOnaui1EpT3jlo06aN9u/fr3379tkff/zjH+13yISHh1dl+RXmzHvg5ptv1tGjR+3hT5IOHz6sa665psYFIMm5Obhw4UKpoFMSCi1Dvgryavt56IxVq1ZpzJgxWrVqlWJjY91dTpXy9/cv9bNwwoQJuu6667Rv3z516dLlyndW7kup4RKrV6+2fH19reXLl1vffvutNW7cOCswMNBKT0+3LMuyRo4cac2aNcve//PPP7e8vLysZ555xjp48KA1Z84cy9vb29q/f7+7hlBh5Z2DRYsWWT4+PtZ7771npaWl2R85OTnuGkKFlXcOLlXT7w4r7/hTU1MtPz8/a+LEidahQ4esjz/+2AoODraefPJJdw2hwso7B3PmzLH8/PysVatWWceOHbM2bdpktWjRwhoyZIi7hlBhOTk51t69e629e/dakqwlS5ZYe/futX744QfLsixr1qxZ1siRI+39jx07ZtWpU8eaMWOGdfDgQSsxMdHy9PS0NmzY4K4hVEh5x//2229bXl5eVmJiosPPwszMTHcNocLKOweXcvbuMEKQG7344otWkyZNLB8fH+vGG2+0/v3vf9u3de/e3YqLi3Po/+6771qtW7e2fHx8rOuvv95au3ZtFVfseuWZg6ZNm1qSSj3mzJlT9YW7UHnfB/+rpocgyyr/+Hfs2GF16dLF8vX1ta699lrrL3/5i1VYWFjFVbtWeeagoKDAeuKJJ6wWLVpYtWrVssLDw60HHnjAOnfuXNUX7iKffvppmX+3S8YdFxdnde/evdRrOnXqZPn4+FjXXnutlZSUVOV1u0p5x9+9e/df7V8TOfMe+F/OhiCbZRmyfgoAAPA/uCYIAAAYiRAEAACMRAgCAABGIgQBAAAjEYIAAICRCEEAAMBIhCAAAGAkQhCAq9Ztt92mKVOmuLsMANUUIQhAtdS/f3/17du3zG2fffaZbDabvv766yquCsDVhBAEoFqKj49XcnKyfvzxx1LbkpKS1LlzZ3Xo0MENlQG4WhCCAFRLd9xxhxo1aqTly5c7tJ8/f15r1qzRwIEDNWzYMP3ud79TnTp11L59e61atepX92mz2fThhx86tAUGBjoc4+TJkxoyZIgCAwMVFBSkAQMG6MSJE64ZFIBqhRAEoFry8vLSqFGjtHz5cv3vVxyuWbNGRUVFuvfeexUZGam1a9fqm2++0bhx4zRy5Ejt3LnT6WMWFBQoJiZGfn5++uyzz/T555+rXr166tu3r/Lz810xLADVCCEIQLV133336fvvv9e2bdvsbUlJSRo8eLCaNm2q6dOnq1OnTrr22ms1adIk9e3bV++++67Tx3vnnXdUXFys119/Xe3bt1fbtm2VlJSk1NRUbd261QUjAlCdEIIAVFtt2rTRTTfdpDfffFOSdPToUX322WeKj49XUVGR5s+fr/bt2ysoKEj16tXTxo0blZqa6vTxvvrqKx09elR+fn6qV6+e6tWrp6CgIF28eFHff/+9q4YFoJrwcncBAPBr4uPjNWnSJCUmJiopKUktWrRQ9+7d9dRTT+n555/X0qVL1b59e9WtW1dTpkz51dNWNpvN4dSa9MspsBLnz59XZGSk3n777VKvbdSokesGBaBaIAQBqNaGDBmiyZMna+XKlXrrrbd0//33y2az6fPPP9eAAQN07733SpKKi4t1+PBhRUREXHZfjRo1Ulpamv35kSNHdOHCBfvzG264Qe+8846Cg4Pl7+9feYMCUC1wOgxAtVavXj0NHTpUs2fPVlpamkaPHi1JatWqlZKTk7Vjxw4dPHhQ48ePV0ZGxq/uq2fPnvrrX/+qvXv3ateuXZowYYK8vb3t20eMGKGGDRtqwIAB+uyzz3T8+HFt3bpVDz74YJm36gOo2QhBAKq9+Ph4nTt3TjExMQoLC5MkPfroo7rhhhsUExOj2267TaGhoRo4cOCv7ufZZ59VeHi4brnlFg0fPlzTp09XnTp17Nvr1Kmj7du3q0mTJho0aJDatm2r+Ph4Xbx4kZUh4Cpksy49QQ4AAGAAVoIAAICRCEEAAMBIhCAAAGAkQhAAADASIQgAABiJEAQAAIxECAIAAEYiBAEAACMRggAAgJEIQQAAwEiEIAAAYCRCEAAAMNL/A+5bb+ct4yzJAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.hist(data['entropy'], bins=15, align='left')\n",
        "\n",
        "# Add axis labels and a title\n",
        "plt.xlabel('Value')\n",
        "plt.ylabel('Frequency')\n",
        "plt.title('Histogram of Values')\n",
        "# Display the plot\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dTMiGKjD7YGF",
        "outputId": "d710aba5-d931-4a2f-b3c9-c2d152856be1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "-0.0: 462\n",
            "0.5032583347756457: 303\n",
            "0.7642045065086203: 117\n",
            "0.9182958340544896: 68\n",
            "0.9910760598382222: 64\n",
            "1.224394445405986: 2\n",
            "1.3921472236645345: 1\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-10-707faf383544>:5: FutureWarning: iteritems is deprecated and will be removed in a future version. Use .items instead.\n",
            "  for index, count in counts.iteritems():\n"
          ]
        }
      ],
      "source": [
        "# count the unique values in the 'fruit' column\n",
        "counts = data['entropy'].value_counts()\n",
        "\n",
        "# print the unique values and their count\n",
        "for index, count in counts.iteritems():\n",
        "    print(f\"{index}: {count}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sw0vHm8j2wWy",
        "outputId": "58f792e0-22f3-4de8-f239-444a9588f3a7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 1017 entries, 0 to 1016\n",
            "Data columns (total 34 columns):\n",
            " #   Column     Non-Null Count  Dtype  \n",
            "---  ------     --------------  -----  \n",
            " 0   sentence1  1017 non-null   object \n",
            " 1   sentence2  1017 non-null   object \n",
            " 2   label      1017 non-null   int64  \n",
            " 3   idx        1017 non-null   int64  \n",
            " 4   text       1017 non-null   object \n",
            " 5   prompt1    1017 non-null   object \n",
            " 6   prompt2    1017 non-null   object \n",
            " 7   prompt3    1017 non-null   object \n",
            " 8   prompt4    1017 non-null   object \n",
            " 9   prompt5    1017 non-null   object \n",
            " 10  prompt6    1017 non-null   object \n",
            " 11  prompt7    1017 non-null   object \n",
            " 12  prompt8    1017 non-null   object \n",
            " 13  prompt9    1017 non-null   object \n",
            " 14  label1     1017 non-null   object \n",
            " 15  label2     1017 non-null   object \n",
            " 16  label3     1017 non-null   object \n",
            " 17  label4     1017 non-null   object \n",
            " 18  label5     1017 non-null   object \n",
            " 19  label6     1017 non-null   object \n",
            " 20  label7     1017 non-null   object \n",
            " 21  label8     1017 non-null   object \n",
            " 22  label9     1017 non-null   object \n",
            " 23  pred1      1017 non-null   int64  \n",
            " 24  pred2      1017 non-null   int64  \n",
            " 25  pred3      1017 non-null   int64  \n",
            " 26  pred4      1017 non-null   int64  \n",
            " 27  pred5      1017 non-null   int64  \n",
            " 28  pred6      1017 non-null   int64  \n",
            " 29  pred7      1017 non-null   int64  \n",
            " 30  pred8      1017 non-null   int64  \n",
            " 31  pred9      1017 non-null   int64  \n",
            " 32  entropy    1017 non-null   float64\n",
            " 33  MV         1017 non-null   int64  \n",
            "dtypes: float64(1), int64(12), object(21)\n",
            "memory usage: 270.3+ KB\n"
          ]
        }
      ],
      "source": [
        "data.info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 876
        },
        "id": "GDXsD2fEhJvf",
        "outputId": "056b9bd0-4dcd-4d0a-eddf-10c6ad12d9a9"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-c105583f-3479-4ffe-83e3-890223bfd8c2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "      <th>MV</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>Sentence1: Board Chancellor Robert Bennett declined to comment on personnel matters Tuesday , as did Mills .\\nSentence2: Mr. Mills declined to comment yesterday , saying that he never discussed personnel matters .</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>Sentence1: Britain has said it will oppose articles calling for closer coordination of tax and economic policy , to uphold the national right of veto .\\nSentence2: Britain has said it wants to haggle over articles of the draft calling for closer coordination of tax and economic policy .</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>Sentence1: An evaluation suggested there was no overall net clinical benefit in patients receiving the drug in the study , the companies said in a statement distributed by Business Wire .\\nSentence2: An evaluation of 240 arthritis patients suggested there was no overall net clinical benefit from taking the medicine , the companies said in a statement distributed by Business Wire .</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>Sentence1: At 5 p.m. EDT , Henri had maximum sustained winds near 50 mph , with some gusts reaching 60 mph .\\nSentence2: At 8 p.m. Friday , Henri was becoming disorganized , but still had maximum sustained winds near 50 mph , with stronger gusts .</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>Sentence1: The Institute for Supply Management said its manufacturing index stood at 53.7 in September .\\nSentence2: The Institute for Supply Management 's manufacturing index dipped to 53.7 from 54.7 in August .</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>Sentence1: He plans to visit an AIDS clinic in Uganda and meet with infected mothers in Nigeria .\\nSentence2: From there he plans to visit South Africa and make stops in Botswana , Uganda and Nigeria .</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>Sentence1: On Friday , the Food and Drug Administration approved the use of silicone breast implants with conditions .\\nSentence2: The drug was approved for use Friday by the Food and Drug Administration .</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>Sentence1: Analysts say Davis , who faces a historic recall election in October , could get a boost in the polls with a budget plan in place .\\nSentence2: Analysts say Davis , a Democrat , could get a boost in the polls if the 29-day-old budget crisis is resolved without further delay .</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>64</th>\n",
              "      <td>Sentence1: The long-running newsmagazine Dateline NBC has been cut from three to two airings per week and is searching for a replacement for retiring co-anchor Jane Pauley .\\nSentence2: The long-running newsmagazine Dateline NBC has been cut from three to two airings per week , losing its Tuesday berth .</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65</th>\n",
              "      <td>Sentence1: For the first quarter , HP pulled in $ 2.94 billion and captured 27.9 percent of the market .\\nSentence2: H-P came in second with nearly $ 3.32 billion in sales and 26 percent of the market 's revenue .</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>71</th>\n",
              "      <td>Sentence1: Neighboring Canada has just decided to legalize same-sex marriage , and they have high hopes that Massachusetts ' supreme court will take a similar step soon .\\nSentence2: Canada has just decided to legalize same-sex marriage , and they have high hope that Massachusetts ' high court will take a similar step within a few weeks .</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>74</th>\n",
              "      <td>Sentence1: Officials with North Carolina Speedway at Rockingham could not be reached .\\nSentence2: NASCAR officials could not be reached for comment Tuesday .</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>82</th>\n",
              "      <td>Sentence1: The single currency rose as far as 135.13 yen , matching a record high hit shortly after the euro 's launch in January 1999 .\\nSentence2: The single currency rose as far as 135.26 yen , its highest level since the introduction of the single currency in January 1999 .</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>96</th>\n",
              "      <td>Sentence1: PowderJect is roughly 20 percent owned by CEO Drayson , a high-profile UK businessman , and his family .\\nSentence2: PowderJect , roughly 20 percent owned by Chief Executive Paul Drayson and his family , reported surging annual profit early this month .</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>108</th>\n",
              "      <td>Sentence1: Despite such optimism , Taiwan 's SARS crisis threatened to whirl out of control this month .\\nSentence2: Despite such optimism , Taiwan 's SARS crisis threatened to whirl out of control in the last week of May .</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c105583f-3479-4ffe-83e3-890223bfd8c2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-c105583f-3479-4ffe-83e3-890223bfd8c2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-c105583f-3479-4ffe-83e3-890223bfd8c2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                                                                                                                                                                                                                                                                                                                                                                                text  \\\n",
              "18                                                                                                                                                                             Sentence1: Board Chancellor Robert Bennett declined to comment on personnel matters Tuesday , as did Mills .\\nSentence2: Mr. Mills declined to comment yesterday , saying that he never discussed personnel matters .   \n",
              "28                                                                                                   Sentence1: Britain has said it will oppose articles calling for closer coordination of tax and economic policy , to uphold the national right of veto .\\nSentence2: Britain has said it wants to haggle over articles of the draft calling for closer coordination of tax and economic policy .   \n",
              "30   Sentence1: An evaluation suggested there was no overall net clinical benefit in patients receiving the drug in the study , the companies said in a statement distributed by Business Wire .\\nSentence2: An evaluation of 240 arthritis patients suggested there was no overall net clinical benefit from taking the medicine , the companies said in a statement distributed by Business Wire .   \n",
              "36                                                                                                                                           Sentence1: At 5 p.m. EDT , Henri had maximum sustained winds near 50 mph , with some gusts reaching 60 mph .\\nSentence2: At 8 p.m. Friday , Henri was becoming disorganized , but still had maximum sustained winds near 50 mph , with stronger gusts .   \n",
              "38                                                                                                                                                                              Sentence1: The Institute for Supply Management said its manufacturing index stood at 53.7 in September .\\nSentence2: The Institute for Supply Management 's manufacturing index dipped to 53.7 from 54.7 in August .   \n",
              "39                                                                                                                                                                                         Sentence1: He plans to visit an AIDS clinic in Uganda and meet with infected mothers in Nigeria .\\nSentence2: From there he plans to visit South Africa and make stops in Botswana , Uganda and Nigeria .   \n",
              "43                                                                                                                                                                                     Sentence1: On Friday , the Food and Drug Administration approved the use of silicone breast implants with conditions .\\nSentence2: The drug was approved for use Friday by the Food and Drug Administration .   \n",
              "47                                                                                                   Sentence1: Analysts say Davis , who faces a historic recall election in October , could get a boost in the polls with a budget plan in place .\\nSentence2: Analysts say Davis , a Democrat , could get a boost in the polls if the 29-day-old budget crisis is resolved without further delay .   \n",
              "64                                                                                 Sentence1: The long-running newsmagazine Dateline NBC has been cut from three to two airings per week and is searching for a replacement for retiring co-anchor Jane Pauley .\\nSentence2: The long-running newsmagazine Dateline NBC has been cut from three to two airings per week , losing its Tuesday berth .   \n",
              "65                                                                                                                                                                             Sentence1: For the first quarter , HP pulled in $ 2.94 billion and captured 27.9 percent of the market .\\nSentence2: H-P came in second with nearly $ 3.32 billion in sales and 26 percent of the market 's revenue .   \n",
              "71                                              Sentence1: Neighboring Canada has just decided to legalize same-sex marriage , and they have high hopes that Massachusetts ' supreme court will take a similar step soon .\\nSentence2: Canada has just decided to legalize same-sex marriage , and they have high hope that Massachusetts ' high court will take a similar step within a few weeks .   \n",
              "74                                                                                                                                                                                                                                    Sentence1: Officials with North Carolina Speedway at Rockingham could not be reached .\\nSentence2: NASCAR officials could not be reached for comment Tuesday .   \n",
              "82                                                                                                            Sentence1: The single currency rose as far as 135.13 yen , matching a record high hit shortly after the euro 's launch in January 1999 .\\nSentence2: The single currency rose as far as 135.26 yen , its highest level since the introduction of the single currency in January 1999 .   \n",
              "96                                                                                                                          Sentence1: PowderJect is roughly 20 percent owned by CEO Drayson , a high-profile UK businessman , and his family .\\nSentence2: PowderJect , roughly 20 percent owned by Chief Executive Paul Drayson and his family , reported surging annual profit early this month .   \n",
              "108                                                                                                                                                                  Sentence1: Despite such optimism , Taiwan 's SARS crisis threatened to whirl out of control this month .\\nSentence2: Despite such optimism , Taiwan 's SARS crisis threatened to whirl out of control in the last week of May .   \n",
              "\n",
              "     label  MV  \n",
              "18       1   1  \n",
              "28       0   0  \n",
              "30       1   1  \n",
              "36       0   1  \n",
              "38       0   0  \n",
              "39       0   0  \n",
              "43       0   0  \n",
              "47       0   0  \n",
              "64       0   1  \n",
              "65       1   0  \n",
              "71       1   0  \n",
              "74       1   1  \n",
              "82       1   1  \n",
              "96       0   1  \n",
              "108      1   1  "
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#qualitative study of those with high entropy\n",
        "pd.set_option('display.max_colwidth', None)\n",
        "df_high=data[data['entropy']>0.8]\n",
        "df_high.head(15)[['text','label','MV']]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kP2v4s8j79ru",
        "outputId": "574e4ce2-11ba-4aa2-f839-f843ef5b60f8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "266\n"
          ]
        }
      ],
      "source": [
        "#length calculation for cost computation\n",
        "df_low=data[data['entropy']<0.2]\n",
        "print(df_low.shape[0])\n",
        "df=df_low\n",
        "df['length1']=df['prompt1'].apply(lambda x:len(x.split()))\n",
        "df['length2']=df['prompt2'].apply(lambda x:len(x.split()))\n",
        "df['length3']=df['prompt3'].apply(lambda x:len(x.split()))\n",
        "df['length4']=df['prompt4'].apply(lambda x:len(x.split()))\n",
        "df['length5']=df['prompt5'].apply(lambda x:len(x.split()))\n",
        "df['length6']=df['prompt6'].apply(lambda x:len(x.split()))\n",
        "df['length7']=df['prompt7'].apply(lambda x:len(x.split()))\n",
        "df['length8']=df['prompt8'].apply(lambda x:len(x.split()))\n",
        "df['length9']=df['prompt9'].apply(lambda x:len(x.split()))\n",
        "df['length10']=df['prompt10'].apply(lambda x:len(x.split()))\n",
        "df['length11']=df['prompt11'].apply(lambda x:len(x.split()))\n",
        "df['length12']=df['prompt12'].apply(lambda x:len(x.split()))\n",
        "df['length13']=df['prompt13'].apply(lambda x:len(x.split()))\n",
        "df['length']=df['length1']+df['length2']+df['length3']+df['length4']+df['length5']+df['length6']+df['length7']+df['length8']+df['length9']+df['length10']+df['length11']+df['length12']+df['length13']\n",
        "#sum(df['length'].tolist())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lljHhLxuaChh",
        "outputId": "efa3eca0-33ee-4f92-e36d-4aa983b54376"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[\"Title: Wall St. Bears Claw Back Into the Black Description:  NEW YORK (Reuters) - Short-sellers, Wall Street's dwindling  band of ultra-cynics, are seeing green again.\",\n",
              " \"Title: Rescuing an Old Saver Description: If you think you may need to help your elderly relatives with their finances, don't be shy about having the money talk -- soon.\",\n",
              " 'Title: Veteran inventor in market float Description: Trevor Baylis, the veteran inventor famous for creating the Freeplay clockwork radio, is planning to float his company on the stock market.',\n",
              " 'Title: Quality Gets Swept Away Description: Quality Distribution is hammered after reporting a large loss for the second quarter.',\n",
              " \"Title: What's Cool In the Pool ... ... And Hot On the Deck Description: Americans are spending more on tricking out the places where they swim. Here's a look at the new wave of accessories\"]"
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_low['text'].tolist()[:5]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-wavQMyy9SoW"
      },
      "outputs": [],
      "source": [
        "def calculate_alignment(threshold, df_input):\n",
        "  df_low=df_input[df_input['entropy']<=threshold]\n",
        "  print(df_low.shape[0])\n",
        "  df_low_correct1=df_low[df_low['MV']==df_low['label']]\n",
        "  return df_low_correct1.shape[0]/df_low.shape[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qYvGm6fUBM5N",
        "outputId": "014ce84e-f079-48b1-e42e-3a2cf74abb8c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "770\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "0.7584415584415585"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "calculate_alignment(0.6, data)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "06lGS9fgKLjv",
        "outputId": "21a2eb99-d865-4dce-e4bc-e595726477da"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "462\n",
            "462\n",
            "462\n",
            "765\n",
            "882\n",
            "1014\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[0.7965367965367965,\n",
              " 0.7965367965367965,\n",
              " 0.7965367965367965,\n",
              " 0.7620915032679738,\n",
              " 0.7290249433106576,\n",
              " 0.7120315581854043]"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "x = [0, 0.2, 0.4, 0.6, 0.8, 1]\n",
        "y = [calculate_alignment(i, data) for i in x]\n",
        "y"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 539
        },
        "id": "-FYfmfNwB8-8",
        "outputId": "3538a8d9-7ad5-49b9-ddbb-c8ef836147d6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "462\n",
            "462\n",
            "462\n",
            "765\n",
            "882\n",
            "1014\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGfCAYAAAB8wYmvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAnzUlEQVR4nO3df1xVVb7/8Tc/5KAmx5QEMdKpJrWGkVLhalrWMIOTWc5MhpnI9abm+GPmStdJM2X6iTVdL41STkU/h8KctGvl4A8mbSyLLuiUZTqmI4wJ6lSAaICc9f1jf6FOonJIDovj6/l47Ie5WHvz2Utyv11773WCjDFGAAAAFgtu6wIAAABOh8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFiPwAIAAKwX2pKdsrOz9bvf/U5lZWUaMGCAlixZooSEhJP2z8rK0uOPP66SkhJFRkbqpptuUmZmpsLDw1t8zG/yeDz67LPP1KVLFwUFBbXklAAAgJ8ZY1RVVaWYmBgFB59mDsX4KC8vz4SFhZmnn37afPTRR2bKlCmma9eupry8vMn+ubm5xuVymdzcXLN3716zdu1a07NnTzN79uwWH/PbSktLjSQ2NjY2Nja2driVlpae9lofZIxvH36YmJiowYMHa+nSpZKc2Y3Y2FjNmjVLc+fOPaH/zJkztWPHDhUUFDS23XHHHXrvvfe0efPmFh3z2yoqKtS1a1eVlpYqIiLCl9MBAABtpLKyUrGxsfryyy/ldrtP2denW0K1tbUqKirSvHnzGtuCg4OVlJSkLVu2NLnP0KFD9cc//lGFhYVKSEjQnj17tGbNGqWmprb4mDU1NaqpqWn8fVVVlSQpIiKCwAIAQDvTnMc5fAoshw8fVn19vaKiorzao6Ki9MknnzS5z/jx43X48GENGzZMxhgdP35c06ZN01133dXiY2ZmZuqee+7xpXQAANCOtfpbQhs3btSDDz6oxx57TMXFxVq5cqXeeOMN3XfffS0+5rx581RRUdG4lZaWnsGKAQCAbXyaYYmMjFRISIjKy8u92svLyxUdHd3kPgsWLFBqaqomT54sSYqLi1N1dbWmTp2q+fPnt+iYLpdLLpfLl9IBAEA75tMMS1hYmAYOHOj1AK3H41FBQYGGDBnS5D5Hjx494VWlkJAQSZIxpkXHBAAAZxef12FJT09XWlqaBg0apISEBGVlZam6ulqTJk2SJE2cOFG9evVSZmamJGn06NFavHixLr/8ciUmJmr37t1asGCBRo8e3RhcTndMAABwdvM5sKSkpOjQoUNauHChysrKFB8fr/z8/MaHZktKSrxmVO6++24FBQXp7rvv1v79+3Xeeedp9OjReuCBB5p9TAAAcHbzeR0WG1VWVsrtdquiouLMvdbs8UgbN0q7dzu//973pGuvlf7/rBDOEGOkoiLpb3+Tamulnj2l5GSpY8e2rizw7N4tvf22VF0tnXuu9JOfSN27t3VVAM5ivly/W7Q0f0Crq5N+/3spO1vau1dqeDfcGKl3b+mXv5Rmz5bCwtq2zvbOGOmll6TFi53AIjlhsL7euZj+x39I8+ZxQT0TNm6UMjOldeuc34eGSsePS+Hh0rhx0vz50sUXt2mJAHA6fPjhN331lXTjjdLcudKwYdI77zh/sR8/LhUWOjMsCxZIP/2pdPRoW1fbfhkjpadLt94qnXee9NprzuxKXZ20a5d0223SU09JQ4dK//xnW1fbvj31lPSjH0n/+pf07LPSkSPOWB84IGVkSBs2SImJzs83AFiMW0LflJYmvfyytHq19OMfN91n0ybpuuukUaOcvvDd4sXSHXdIS5dKM2Y03Wf3budC63ZL778v8Rq779audcL1tGnSkiVN38784gvp+uudoLh1q3T++f6vE8BZy5frN4Glwa5dUt++0h/+IE2deuq+zz/vhJsPPpDi4lr2/c5Wx445F8Vx45zbbqeybZt0+eVSbq40frxfygsoQ4ZIHTo4t4RO9Smo//qXc0to6lTpoYf8Vh4A+HL95pZQg2XLnOclJk48fd9bbpGio6XHHmv9ugLNyy9Ln3/u3BI6nfh4Z5bldMEGJyoult591xnn031ke/fu0qRJUk6Oc1sUACxEYGmwfr30i184DyKeTocO0s03O/f/4Zv166V/+zfpooua13/8eOdZourq1q0r0GzYIJ1zjnO7pznGj3dmWrZta9WyAKClCCwNqqqkyMjm94+MlCorW6+eQNWScW7YD81XVeW8bRXazBcBzzvP+ZWfaQCWIrA0cLulb32e0SkdPOjsA9+0ZJwl6Uytr3O2cLudW291dc3r3/Bnws80AEsRWBqMHCm98krzXleuqZGWL3fewIBvRo503vr55JPm9X/hBWnECKlTp1YtK+AkJzu30V59tXn9X3hB6tHDecgZACxEYGlw++1SRYXz4OHpPP+8dOiQs4gcfPOLXzi3H/77v0/ft7BQeustxrkl4uKk4cOdV8iPHz913/Jy6bnnpMmTWRARgLUILA0uvFCaMsVZH+S1107eb+1aadYsKTVV6tfPf/UFCpfLWbDsqaekhx92FpFrykcfSWPGSAMHOr/Cd/fc48xmTZ588ltDhw456wp17izNnOnf+gDABwSWb1qyRBo92lnt9uc/d960OHLEmVp/803nzaBRo5xF5Z58sq2rbb+mT5fuvlu6807pqqukvDxnduvYMWfxsl/+UkpIcB64ff11/tXfUtdc48wG5uZKAwY4r4cfPOjc0vz0U+fP4LLLpP37pT//2fkcJwCwFAvHfVt9vfOv/yVLnH/lf1P//s7KrLff3vy3L3Byq1c7tyw2bfJu79nTWcQsPZ2Hbc+E995zZrP+93+dn+8GXbo4CyD+5jdSbGzb1QfgrMVKt2eCMc4zFJ9+6vz3977nrBza8GGIOHN27HBWDa6tdRbkGzHCWesGZ9b+/dKWLc6MYbduzgzMOee0dVUAzmIEFgAAYD2W5gcAAAGFwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHotCizZ2dnq06ePwsPDlZiYqMLCwpP2HTFihIKCgk7YRo0a1djnyJEjmjlzps4//3x17NhRl156qZYtW9aS0gAAQADyObAsX75c6enpysjIUHFxsQYMGKDk5GQdPHiwyf4rV67UgQMHGrft27crJCREY8eObeyTnp6u/Px8/fGPf9SOHTv0n//5n5o5c6ZWr17d8jMDAAABw+fAsnjxYk2ZMkWTJk1qnAnp1KmTnn766Sb7d+vWTdHR0Y3b+vXr1alTJ6/A8s477ygtLU0jRoxQnz59NHXqVA0YMOCkMzc1NTWqrKz02gAAQODyKbDU1taqqKhISUlJXx8gOFhJSUnasmVLs46Rk5OjcePGqXPnzo1tQ4cO1erVq7V//34ZY/Tmm29q165d+slPftLkMTIzM+V2uxu32NhYX04DAAC0Mz4FlsOHD6u+vl5RUVFe7VFRUSorKzvt/oWFhdq+fbsmT57s1b5kyRJdeumlOv/88xUWFqaRI0cqOztbV111VZPHmTdvnioqKhq30tJSX04DAAC0M6H+/GY5OTmKi4tTQkKCV/uSJUv07rvvavXq1erdu7feeustzZgxQzExMV6zOQ1cLpdcLpe/ygYAAG3Mp8ASGRmpkJAQlZeXe7WXl5crOjr6lPtWV1crLy9P9957r1f7sWPHdNddd2nVqlWNbw798Ic/1LZt2/TII480GVgAAMDZxadbQmFhYRo4cKAKCgoa2zwejwoKCjRkyJBT7rtixQrV1NRowoQJXu11dXWqq6tTcLB3KSEhIfJ4PL6UBwAAApTPt4TS09OVlpamQYMGKSEhQVlZWaqurtakSZMkSRMnTlSvXr2UmZnptV9OTo7GjBmj7t27e7VHRETo6quv1pw5c9SxY0f17t1bmzZt0vPPP6/Fixd/h1MDAACBwufAkpKSokOHDmnhwoUqKytTfHy88vPzGx/ELSkpOWG2ZOfOndq8ebPWrVvX5DHz8vI0b9483Xrrrfr888/Vu3dvPfDAA5o2bVoLTgkAAASaIGOMaesivqvKykq53W5VVFQoIiKircsBAADN4Mv1m88SAgAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsF9rWBQBAQCkpkfLzpYoK6ZxzpGuukfr1a+uqgHaPwAIAZ8LWrdI990ivveb8vksXqbpaOn7cCS133y1de23b1gi0Y9wSAoDvKj9fuvJKadcu6bHHnNmVL7+UjhyRcnOd4PLjH0vPPNPWlQLtVpAxxrR1Ed9VZWWl3G63KioqFBER0dblADib7NghDRrkzKKsWCF17Hhin/p6afp06amnpA0bnL4AfLp+M8MCAN/FI49I3btLL7/cdFiRpJAQZ+Zl0CDpgQf8Wx8QIAgsANBSX3whvfiiNG2a1KnTqfuGhEi//rVUUCB98ol/6gMCCIEFAFrqvfekr76Sxo1rXv9f/MIJLm++2bp1AQGIwAIALXXkiPPruec2r7/LJXXu/PV+AJqNwAIALdW1q/NrWVnz+ldVOWGlYT8AzUZgAYCWGjpUcrul555rXv/cXOfXkSNbryYgQBFYAKClOnWSJk1yXlc+dOjUfY8dkx59VLrhBik21j/1AQGEwAIA30V6uvMg7ahRJw8tR49KY8dK+/ZJCxf6tz4gQLQosGRnZ6tPnz4KDw9XYmKiCgsLT9p3xIgRCgoKOmEbNWqUV78dO3bohhtukNvtVufOnTV48GCVlJS0pDwA8J/YWOnPf3bCSP/+0p13Sh99JH3+ubR7t3T//VLfvtLGjdKqVdLll7d1xUC75HNgWb58udLT05WRkaHi4mINGDBAycnJOnjwYJP9V65cqQMHDjRu27dvV0hIiMaOHdvY59NPP9WwYcPUr18/bdy4UR988IEWLFig8PDwlp8ZAPjLFVdIRUXSxInSE09IP/iBs5jc978vPfig88zK++9LycltXSnQbvm8NH9iYqIGDx6spUuXSpI8Ho9iY2M1a9YszZ0797T7Z2VlaeHChTpw4IA6d+4sSRo3bpw6dOigF154oQWnwNL8ACxy9Kj0zjvOZwmdc46UmNj8156Bs0yrLc1fW1uroqIiJSUlfX2A4GAlJSVpy5YtzTpGTk6Oxo0b1xhWPB6P3njjDV1yySVKTk5Wjx49lJiYqFdfffWkx6ipqVFlZaXXBgBW6NRJSkqSbrrJmVkhrABnhE+B5fDhw6qvr1dUVJRXe1RUlMqasQ5BYWGhtm/frsmTJze2HTx4UEeOHNGiRYs0cuRIrVu3Tj/72c/085//XJs2bWryOJmZmXK73Y1bLE/cAwAQ0Pz6llBOTo7i4uKUkJDQ2ObxeCRJN954o2bPnq34+HjNnTtX119/vZYtW9bkcebNm6eKiorGrbS01C/1AwCAtuFTYImMjFRISIjKy8u92svLyxUdHX3Kfaurq5WXl6fbbrvthGOGhobq0ksv9Wrv37//Sd8ScrlcioiI8NoAAEDg8imwhIWFaeDAgSooKGhs83g8Kigo0JAhQ06574oVK1RTU6MJEyaccMzBgwdr586dXu27du1S7969fSkPAAAEqFBfd0hPT1daWpoGDRqkhIQEZWVlqbq6WpMmTZIkTZw4Ub169VJmZqbXfjk5ORozZoy6d+9+wjHnzJmjlJQUXXXVVbrmmmuUn5+v1157TRs3bmzZWQEAgIDic2BJSUnRoUOHtHDhQpWVlSk+Pl75+fmND+KWlJQoONh74mbnzp3avHmz1q1b1+Qxf/azn2nZsmXKzMzUr371K/Xt21evvPKKhg0b1oJTAgAAgcbndVhsxDosAAC0P622DgsAAEBbILAAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgvRYFluzsbPXp00fh4eFKTExUYWHhSfuOGDFCQUFBJ2yjRo1qsv+0adMUFBSkrKyslpQGAAACkM+BZfny5UpPT1dGRoaKi4s1YMAAJScn6+DBg032X7lypQ4cONC4bd++XSEhIRo7duwJfVetWqV3331XMTExvp8JAAAIWD4HlsWLF2vKlCmaNGmSLr30Ui1btkydOnXS008/3WT/bt26KTo6unFbv369OnXqdEJg2b9/v2bNmqXc3Fx16NDhlDXU1NSosrLSawMAAIHLp8BSW1uroqIiJSUlfX2A4GAlJSVpy5YtzTpGTk6Oxo0bp86dOze2eTwepaamas6cObrssstOe4zMzEy53e7GLTY21pfTAAAA7YxPgeXw4cOqr69XVFSUV3tUVJTKyspOu39hYaG2b9+uyZMne7U/9NBDCg0N1a9+9atm1TFv3jxVVFQ0bqWlpc0/CQAA0O6E+vOb5eTkKC4uTgkJCY1tRUVFevTRR1VcXKygoKBmHcflcsnlcrVWmQAAwDI+zbBERkYqJCRE5eXlXu3l5eWKjo4+5b7V1dXKy8vTbbfd5tX+17/+VQcPHtQFF1yg0NBQhYaGat++fbrjjjvUp08fX8oDAAAByqfAEhYWpoEDB6qgoKCxzePxqKCgQEOGDDnlvitWrFBNTY0mTJjg1Z6amqoPPvhA27Zta9xiYmI0Z84crV271pfyAABAgPL5llB6errS0tI0aNAgJSQkKCsrS9XV1Zo0aZIkaeLEierVq5cyMzO99svJydGYMWPUvXt3r/bu3buf0NahQwdFR0erb9++vpYHAAACkM+BJSUlRYcOHdLChQtVVlam+Ph45efnNz6IW1JSouBg74mbnTt3avPmzVq3bt2ZqRoAAJxVgowxpq2L+K4qKyvldrtVUVGhiIiIti4HAAA0gy/Xbz5LCAAAWI/AAgAArEdgAQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1iOwAAAA6xFYAACA9QgsAID25e9/l+64Qzr/fCksTOrSRRoxQlq+XKqtbevq0EpC27oAAACaxeOR7rpLeughqVs3aeJE6fvfl44elV5/XRo3TrrwQue/+/dv62pxhhFYAADtw3/9l5SVJWVmSr/+tdSxo/fXPvxQuuUW6eqrpXffdcILAga3hAAA9nv7bel//kd69FFp7lzvsNIgLk7auNG5RTR9ut9LROsisAAA7Jed7dz+mTHj1P0iI6WFC6W1a6Xdu/1TG/yCwAIAsNuRI9Kf/iTdfrsU3IzL1s03S+eeKz3/fOvXBr8hsAAA7FZeLtXVSZdf3rz+HTs6D92WlLRuXfArAgsAwG4Nsyr19c3fp75eCglpnXrQJggsAAC79ezpPEi7aVPz+n/+ufS3v0n9+rVuXfArAgsAwG7h4VJamvTkk81bGO7ZZ50ZlrS0Vi8N/kNgAQDYb/p06fBhZ4VbY07eb8cO6f77nUXkevTwX31odQQWAID9+veXHn9cWrpUSk098ZXl2lrpxRel4cOlXr2k3/++bepEq2GlWwBA+zB1qtSpk7PKbW6udO210sUXS8eOOeuuHDwojRrlvM7ctWtbV4szjBkWAED7MWGC9M9/Os+phIVJ//d/0qefSikp0kcfOZ8j1K1bW1eJVhBkzKluBrYPlZWVcrvdqqioUERERFuXAwAAmsGX6zczLAAAwHoEFgAAYD0CCwAAsB6BBQAAWI/AAgAArEdgAQAA1iOwAAAA6xFYAACA9QgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPVaFFiys7PVp08fhYeHKzExUYWFhSftO2LECAUFBZ2wjRo1SpJUV1enO++8U3FxcercubNiYmI0ceJEffbZZy07IwAAEHB8DizLly9Xenq6MjIyVFxcrAEDBig5OVkHDx5ssv/KlSt14MCBxm379u0KCQnR2LFjJUlHjx5VcXGxFixYoOLiYq1cuVI7d+7UDTfc8N3ODAAABIwgY4zxZYfExEQNHjxYS5culSR5PB7FxsZq1qxZmjt37mn3z8rK0sKFC3XgwAF17ty5yT7vv/++EhIStG/fPl1wwQWnPWZlZaXcbrcqKioUERHhy+kAAIA24sv126cZltraWhUVFSkpKenrAwQHKykpSVu2bGnWMXJycjRu3LiThhVJqqioUFBQkLp27drk12tqalRZWem1AQCAwOVTYDl8+LDq6+sVFRXl1R4VFaWysrLT7l9YWKjt27dr8uTJJ+3z1Vdf6c4779Qtt9xy0rSVmZkpt9vduMXGxvpyGgAAoJ3x61tCOTk5iouLU0JCQpNfr6ur08033yxjjB5//PGTHmfevHmqqKho3EpLS1urZAAAYIFQXzpHRkYqJCRE5eXlXu3l5eWKjo4+5b7V1dXKy8vTvffe2+TXG8LKvn379Je//OWU97JcLpdcLpcvpQMAgHbMpxmWsLAwDRw4UAUFBY1tHo9HBQUFGjJkyCn3XbFihWpqajRhwoQTvtYQVv7+979rw4YN6t69uy9lAQCAAOfTDIskpaenKy0tTYMGDVJCQoKysrJUXV2tSZMmSZImTpyoXr16KTMz02u/nJwcjRkz5oQwUldXp5tuuknFxcV6/fXXVV9f3/g8TLdu3RQWFtbScwMAAAHC58CSkpKiQ4cOaeHChSorK1N8fLzy8/MbH8QtKSlRcLD3xM3OnTu1efNmrVu37oTj7d+/X6tXr5YkxcfHe33tzTff1IgRI3wtEQAABBif12GxEeuwAADQ/rTaOiwAAABtgcACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoEFgAAYD0CCwAAsB6BBQAAWC+0rQsAAACWqquTNmyQSkqk4GCpb19p+HApKMjvpRBYAACAt2PHpN/9TvrDH6TPPnPCijHO1revNHOmNH260+4n3BICAABfq6iQfvQjKTNTuv56aetWZ6alrk7atEmKj5d+/WspJcVp8xNmWAAAgMMYJ4js2CG99ZY0eLD316+6ytluuUW66SYpPV1assQvpTHDAgAAHO+8I61dKz377Ilh5ZtuvFF68EFp2TLnlpEfEFgAAIDjscekiy+WRo8+fd+pUyWXS3riidavSwQWAADQYP1653ZPcx6mdbudZ1w2bGj9ukRgAQAADaqqpMjI5vePjJQqK1uvnm8gsAAAAIfbLZWXN7//wYPOPn5AYAEAAI6RI6UXX5Tq60/f94svpNdek37609avSwQWAADQ4Je/lP7xD+mVV07fNzvbCTa33dbqZUkEFgAA0CAhQbrhBieE/PWvJ+/30ktSRoazgFxUlF9KI7AAAABHUJCUm+uswXLttdKtt0qbN0tHjzoP5K5Z47wZNH68NGGCtGiR30pjpVsAAPC1c86R8vOlpUuddVlefNH765dfLj3zjJSW5tcPQQwyxhi/fbdWUllZKbfbrYqKCkVERLR1OQAABAaPR3r77a8/rfmSS6QrrjhjQcWX6zczLAAAoGnBwdLw4W1dhSSeYQEAAO0AgQUAAFiPwAIAAKxHYAEAANYjsAAAAOsRWAAAgPUILAAAwHoBsQ5Lw9p3lZWVbVwJAABorobrdnPWsA2IwFJVVSVJio2NbeNKAACAr6qqquR2u0/ZJyCW5vd4PPrss8/UpUsXBZ3hzzWorKxUbGysSktLWfa/FTHO/sE4+w9j7R+Ms3+01jgbY1RVVaWYmBgFB5/6KZWAmGEJDg7W+eef36rfIyIigv8Z/IBx9g/G2X8Ya/9gnP2jNcb5dDMrDXjoFgAAWI/AAgAArEdgOQ2Xy6WMjAy5XK62LiWgMc7+wTj7D2PtH4yzf9gwzgHx0C0AAAhszLAAAADrEVgAAID1CCwAAMB6BBYAAGA9AgsAALAegUVSdna2+vTpo/DwcCUmJqqwsPCU/VesWKF+/fopPDxccXFxWrNmjZ8qbd98Gecnn3xSw4cP17nnnqtzzz1XSUlJp/1zgcPXn+cGeXl5CgoK0pgxY1q3wADh6zh/+eWXmjFjhnr27CmXy6VLLrmEvzuaydexzsrKUt++fdWxY0fFxsZq9uzZ+uqrr/xUbfvz1ltvafTo0YqJiVFQUJBeffXV0+6zceNGXXHFFXK5XLr44ov17LPPtnqdMme5vLw8ExYWZp5++mnz0UcfmSlTppiuXbua8vLyJvu//fbbJiQkxDz88MPm448/Nnfffbfp0KGD+fDDD/1cefvi6ziPHz/eZGdnm61bt5odO3aYf//3fzdut9v885//9HPl7Yuv49xg7969plevXmb48OHmxhtv9E+x7Ziv41xTU2MGDRpkrrvuOrN582azd+9es3HjRrNt2zY/V97++DrWubm5xuVymdzcXLN3716zdu1a07NnTzN79mw/V95+rFmzxsyfP9+sXLnSSDKrVq06Zf89e/aYTp06mfT0dPPxxx+bJUuWmJCQEJOfn9+qdZ71gSUhIcHMmDGj8ff19fUmJibGZGZmNtn/5ptvNqNGjfJqS0xMNLfffnur1tne+TrO33b8+HHTpUsX89xzz7VWiQGhJeN8/PhxM3ToUPPUU0+ZtLQ0Aksz+DrOjz/+uLnwwgtNbW2tv0oMGL6O9YwZM8y1117r1Zaenm6uvPLKVq0zUDQnsPzmN78xl112mVdbSkqKSU5ObsXKjDmrbwnV1taqqKhISUlJjW3BwcFKSkrSli1bmtxny5YtXv0lKTk5+aT90bJx/rajR4+qrq5O3bp1a60y272WjvO9996rHj166LbbbvNHme1eS8Z59erVGjJkiGbMmKGoqCj94Ac/0IMPPqj6+np/ld0utWSshw4dqqKiosbbRnv27NGaNWt03XXX+aXms0FbXQcD4tOaW+rw4cOqr69XVFSUV3tUVJQ++eSTJvcpKytrsn9ZWVmr1dnetWScv+3OO+9UTEzMCf+T4GstGefNmzcrJydH27Zt80OFgaEl47xnzx795S9/0a233qo1a9Zo9+7dmj59uurq6pSRkeGPstulloz1+PHjdfjwYQ0bNkzGGB0/flzTpk3TXXfd5Y+Szwonuw5WVlbq2LFj6tixY6t837N6hgXtw6JFi5SXl6dVq1YpPDy8rcsJGFVVVUpNTdWTTz6pyMjIti4noHk8HvXo0UNPPPGEBg4cqJSUFM2fP1/Lli1r69ICzsaNG/Xggw/qscceU3FxsVauXKk33nhD9913X1uXhu/orJ5hiYyMVEhIiMrLy73ay8vLFR0d3eQ+0dHRPvVHy8a5wSOPPKJFixZpw4YN+uEPf9iaZbZ7vo7zp59+qn/84x8aPXp0Y5vH45EkhYaGaufOnbroootat+h2qCU/zz179lSHDh0UEhLS2Na/f3+VlZWptrZWYWFhrVpze9WSsV6wYIFSU1M1efJkSVJcXJyqq6s1depUzZ8/X8HB/Dv9uzrZdTAiIqLVZleks3yGJSwsTAMHDlRBQUFjm8fjUUFBgYYMGdLkPkOGDPHqL0nr168/aX+0bJwl6eGHH9Z9992n/Px8DRo0yB+ltmu+jnO/fv304Ycfatu2bY3bDTfcoGuuuUbbtm1TbGysP8tvN1ry83zllVdq9+7djYFQknbt2qWePXsSVk6hJWN99OjRE0JJQ1A0fNbvGdFm18FWfaS3HcjLyzMul8s8++yz5uOPPzZTp041Xbt2NWVlZcYYY1JTU83cuXMb+7/99tsmNDTUPPLII2bHjh0mIyOD15qbwddxXrRokQkLCzN/+tOfzIEDBxq3qqqqtjqFdsHXcf423hJqHl/HuaSkxHTp0sXMnDnT7Ny507z++uumR48e5v7772+rU2g3fB3rjIwM06VLF/PSSy+ZPXv2mHXr1pmLLrrI3HzzzW11CtarqqoyW7duNVu3bjWSzOLFi83WrVvNvn37jDHGzJ0716Smpjb2b3itec6cOWbHjh0mOzub15r9ZcmSJeaCCy4wYWFhJiEhwbz77ruNX7v66qtNWlqaV/+XX37ZXHLJJSYsLMxcdtll5o033vBzxe2TL+Pcu3dvI+mELSMjw/+FtzO+/jx/E4Gl+Xwd53feecckJiYal8tlLrzwQvPAAw+Y48eP+7nq9smXsa6rqzO//e1vzUUXXWTCw8NNbGysmT59uvniiy/8X3g78eabbzb5923DuKalpZmrr776hH3i4+NNWFiYufDCC80zzzzT6nUGGcMcGQAAsNtZ/QwLAABoHwgsAADAegQWAABgPQILAACwHoEFAABYj8ACAACsR2ABAADWI7AAAADrEVgAAID1CCwAAMB6BBYAAGC9/wc+MlbbX+IHzwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "x = [0, 0.2, 0.4, 0.6, 0.8, 1]\n",
        "y = [calculate_alignment(i, data) for i in x]\n",
        "\n",
        "plt.scatter(x, y, s=80, facecolors='none', edgecolors='r')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bH-186UG76UP"
      },
      "outputs": [],
      "source": [
        "data['MV_yes']=data['MV'].apply(lambda x:1 if x==1 else 0)\n",
        "data['MV_no']=data['MV'].apply(lambda x:1 if x==0 else 0)\n",
        "data['label_yes']=data['label'].apply(lambda x:1 if x==1 else 0)\n",
        "data['label_no']=data['label'].apply(lambda x:1 if x==0 else 0)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 751
        },
        "id": "xlCZ12ECSYt2",
        "outputId": "5864b3b6-5f7a-4087-aa18-7634beb62887"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "610\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                           sentence1  \\\n",
              "0  Bush said that he ordered the Treasury Departm...   \n",
              "1  Strong spokeswoman Stephanie Truog said , \" We...   \n",
              "2  Many of the victims of Sunday 's attack had be...   \n",
              "3  \" The bank requires growth from elsewhere in t...   \n",
              "4  Tyco later said the loan had not been forgiven...   \n",
              "\n",
              "                                           sentence2  label   idx  \\\n",
              "0  Bush said in a statement that he ordered the U...      1  3154   \n",
              "1  Strong Financial released a statement Wednesda...      0  3402   \n",
              "2  Many of the victims had been headed home for R...      0  3958   \n",
              "3  The Bank of England \" requires growth from els...      1  1410   \n",
              "4  Tyco has said the loan was not forgiven , but ...      1   142   \n",
              "\n",
              "                                                text  \\\n",
              "0  Sentence1: Bush said that he ordered the Treas...   \n",
              "1  Sentence1: Strong spokeswoman Stephanie Truog ...   \n",
              "2  Sentence1: Many of the victims of Sunday 's at...   \n",
              "3  Sentence1: \" The bank requires growth from els...   \n",
              "4  Sentence1: Tyco later said the loan had not be...   \n",
              "\n",
              "                                             prompt1  \\\n",
              "0  Please label if the following two sentences ar...   \n",
              "1  Please label if the following two sentences ar...   \n",
              "2  Please label if the following two sentences ar...   \n",
              "3  Please label if the following two sentences ar...   \n",
              "4  Please label if the following two sentences ar...   \n",
              "\n",
              "                                             prompt2  \\\n",
              "0  Sentence1: Bush said that he ordered the Treas...   \n",
              "1  Sentence1: Strong spokeswoman Stephanie Truog ...   \n",
              "2  Sentence1: Many of the victims of Sunday 's at...   \n",
              "3  Sentence1: \" The bank requires growth from els...   \n",
              "4  Sentence1: Tyco later said the loan had not be...   \n",
              "\n",
              "                                             prompt3  \\\n",
              "0  Given the following two sentences, please clas...   \n",
              "1  Given the following two sentences, please clas...   \n",
              "2  Given the following two sentences, please clas...   \n",
              "3  Given the following two sentences, please clas...   \n",
              "4  Given the following two sentences, please clas...   \n",
              "\n",
              "                                             prompt4  \\\n",
              "0  Is it true that the following two sentences ar...   \n",
              "1  Is it true that the following two sentences ar...   \n",
              "2  Is it true that the following two sentences ar...   \n",
              "3  Is it true that the following two sentences ar...   \n",
              "4  Is it true that the following two sentences ar...   \n",
              "\n",
              "                                             prompt5  ... pred7 pred8 pred9  \\\n",
              "0  Is it true that the following two sentences ar...  ...     1     1     1   \n",
              "1  Is it true that the following two sentences ar...  ...     1     1     1   \n",
              "2  Is it true that the following two sentences ar...  ...     1     1     1   \n",
              "3  Is it true that the following two sentences ar...  ...     1     1     1   \n",
              "4  Is it true that the following two sentences ar...  ...     1     1     1   \n",
              "\n",
              "  entropy MV MV_yes MV_no label_yes label_no allocate  \n",
              "0    -0.0  1      1     0         1        0        1  \n",
              "1    -0.0  1      1     0         0        1        1  \n",
              "2    -0.0  1      1     0         0        1        1  \n",
              "3    -0.0  1      1     0         1        0        1  \n",
              "4    -0.0  1      1     0         1        0        1  \n",
              "\n",
              "[5 rows x 39 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d7fa3eb7-c89b-44a7-af00-4f27bba6a76d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence1</th>\n",
              "      <th>sentence2</th>\n",
              "      <th>label</th>\n",
              "      <th>idx</th>\n",
              "      <th>text</th>\n",
              "      <th>prompt1</th>\n",
              "      <th>prompt2</th>\n",
              "      <th>prompt3</th>\n",
              "      <th>prompt4</th>\n",
              "      <th>prompt5</th>\n",
              "      <th>...</th>\n",
              "      <th>pred7</th>\n",
              "      <th>pred8</th>\n",
              "      <th>pred9</th>\n",
              "      <th>entropy</th>\n",
              "      <th>MV</th>\n",
              "      <th>MV_yes</th>\n",
              "      <th>MV_no</th>\n",
              "      <th>label_yes</th>\n",
              "      <th>label_no</th>\n",
              "      <th>allocate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Bush said that he ordered the Treasury Departm...</td>\n",
              "      <td>Bush said in a statement that he ordered the U...</td>\n",
              "      <td>1</td>\n",
              "      <td>3154</td>\n",
              "      <td>Sentence1: Bush said that he ordered the Treas...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: Bush said that he ordered the Treas...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Strong spokeswoman Stephanie Truog said , \" We...</td>\n",
              "      <td>Strong Financial released a statement Wednesda...</td>\n",
              "      <td>0</td>\n",
              "      <td>3402</td>\n",
              "      <td>Sentence1: Strong spokeswoman Stephanie Truog ...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: Strong spokeswoman Stephanie Truog ...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Many of the victims of Sunday 's attack had be...</td>\n",
              "      <td>Many of the victims had been headed home for R...</td>\n",
              "      <td>0</td>\n",
              "      <td>3958</td>\n",
              "      <td>Sentence1: Many of the victims of Sunday 's at...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: Many of the victims of Sunday 's at...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>\" The bank requires growth from elsewhere in t...</td>\n",
              "      <td>The Bank of England \" requires growth from els...</td>\n",
              "      <td>1</td>\n",
              "      <td>1410</td>\n",
              "      <td>Sentence1: \" The bank requires growth from els...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: \" The bank requires growth from els...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Tyco later said the loan had not been forgiven...</td>\n",
              "      <td>Tyco has said the loan was not forgiven , but ...</td>\n",
              "      <td>1</td>\n",
              "      <td>142</td>\n",
              "      <td>Sentence1: Tyco later said the loan had not be...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: Tyco later said the loan had not be...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 39 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d7fa3eb7-c89b-44a7-af00-4f27bba6a76d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d7fa3eb7-c89b-44a7-af00-4f27bba6a76d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d7fa3eb7-c89b-44a7-af00-4f27bba6a76d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 149
        }
      ],
      "source": [
        "import pandas as pd\n",
        "df=data\n",
        "df=df.sort_values(by='entropy',ignore_index=True)\n",
        "# Assuming your dataframe is called 'df'\n",
        "total_rows = df.shape[0]\n",
        "num_rows_20_percent = int(total_rows * 0.6) #can change this to 0.2, 0.4, 0.6, 0.8\n",
        "print(num_rows_20_percent)\n",
        "\n",
        "df['allocate'] = 0  # Initialize the new column with 0\n",
        "df.loc[:num_rows_20_percent-1, 'allocate'] = 1  # Assign 1 to the first 20% of rows\n",
        "\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 734
        },
        "id": "WxS4YXfA730N",
        "outputId": "b173d5b1-ee33-450a-ddb0-233c3c7f1ed4"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                           sentence1  \\\n",
              "0  Bush said that he ordered the Treasury Departm...   \n",
              "1  Strong spokeswoman Stephanie Truog said , \" We...   \n",
              "2  Many of the victims of Sunday 's attack had be...   \n",
              "3  \" The bank requires growth from elsewhere in t...   \n",
              "4  Tyco later said the loan had not been forgiven...   \n",
              "\n",
              "                                           sentence2  label   idx  \\\n",
              "0  Bush said in a statement that he ordered the U...      1  3154   \n",
              "1  Strong Financial released a statement Wednesda...      0  3402   \n",
              "2  Many of the victims had been headed home for R...      0  3958   \n",
              "3  The Bank of England \" requires growth from els...      1  1410   \n",
              "4  Tyco has said the loan was not forgiven , but ...      1   142   \n",
              "\n",
              "                                                text  \\\n",
              "0  Sentence1: Bush said that he ordered the Treas...   \n",
              "1  Sentence1: Strong spokeswoman Stephanie Truog ...   \n",
              "2  Sentence1: Many of the victims of Sunday 's at...   \n",
              "3  Sentence1: \" The bank requires growth from els...   \n",
              "4  Sentence1: Tyco later said the loan had not be...   \n",
              "\n",
              "                                             prompt1  \\\n",
              "0  Please label if the following two sentences ar...   \n",
              "1  Please label if the following two sentences ar...   \n",
              "2  Please label if the following two sentences ar...   \n",
              "3  Please label if the following two sentences ar...   \n",
              "4  Please label if the following two sentences ar...   \n",
              "\n",
              "                                             prompt2  \\\n",
              "0  Sentence1: Bush said that he ordered the Treas...   \n",
              "1  Sentence1: Strong spokeswoman Stephanie Truog ...   \n",
              "2  Sentence1: Many of the victims of Sunday 's at...   \n",
              "3  Sentence1: \" The bank requires growth from els...   \n",
              "4  Sentence1: Tyco later said the loan had not be...   \n",
              "\n",
              "                                             prompt3  \\\n",
              "0  Given the following two sentences, please clas...   \n",
              "1  Given the following two sentences, please clas...   \n",
              "2  Given the following two sentences, please clas...   \n",
              "3  Given the following two sentences, please clas...   \n",
              "4  Given the following two sentences, please clas...   \n",
              "\n",
              "                                             prompt4  \\\n",
              "0  Is it true that the following two sentences ar...   \n",
              "1  Is it true that the following two sentences ar...   \n",
              "2  Is it true that the following two sentences ar...   \n",
              "3  Is it true that the following two sentences ar...   \n",
              "4  Is it true that the following two sentences ar...   \n",
              "\n",
              "                                             prompt5  ... pred9 entropy MV  \\\n",
              "0  Is it true that the following two sentences ar...  ...     1    -0.0  1   \n",
              "1  Is it true that the following two sentences ar...  ...     1    -0.0  1   \n",
              "2  Is it true that the following two sentences ar...  ...     1    -0.0  1   \n",
              "3  Is it true that the following two sentences ar...  ...     1    -0.0  1   \n",
              "4  Is it true that the following two sentences ar...  ...     1    -0.0  1   \n",
              "\n",
              "  MV_yes MV_no label_yes label_no allocate yes no  \n",
              "0      1     0         1        0        1   1  0  \n",
              "1      1     0         0        1        1   1  0  \n",
              "2      1     0         0        1        1   1  0  \n",
              "3      1     0         1        0        1   1  0  \n",
              "4      1     0         1        0        1   1  0  \n",
              "\n",
              "[5 rows x 41 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-3fb0a350-0f84-43ee-b937-6e7608deec04\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence1</th>\n",
              "      <th>sentence2</th>\n",
              "      <th>label</th>\n",
              "      <th>idx</th>\n",
              "      <th>text</th>\n",
              "      <th>prompt1</th>\n",
              "      <th>prompt2</th>\n",
              "      <th>prompt3</th>\n",
              "      <th>prompt4</th>\n",
              "      <th>prompt5</th>\n",
              "      <th>...</th>\n",
              "      <th>pred9</th>\n",
              "      <th>entropy</th>\n",
              "      <th>MV</th>\n",
              "      <th>MV_yes</th>\n",
              "      <th>MV_no</th>\n",
              "      <th>label_yes</th>\n",
              "      <th>label_no</th>\n",
              "      <th>allocate</th>\n",
              "      <th>yes</th>\n",
              "      <th>no</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Bush said that he ordered the Treasury Departm...</td>\n",
              "      <td>Bush said in a statement that he ordered the U...</td>\n",
              "      <td>1</td>\n",
              "      <td>3154</td>\n",
              "      <td>Sentence1: Bush said that he ordered the Treas...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: Bush said that he ordered the Treas...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Strong spokeswoman Stephanie Truog said , \" We...</td>\n",
              "      <td>Strong Financial released a statement Wednesda...</td>\n",
              "      <td>0</td>\n",
              "      <td>3402</td>\n",
              "      <td>Sentence1: Strong spokeswoman Stephanie Truog ...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: Strong spokeswoman Stephanie Truog ...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Many of the victims of Sunday 's attack had be...</td>\n",
              "      <td>Many of the victims had been headed home for R...</td>\n",
              "      <td>0</td>\n",
              "      <td>3958</td>\n",
              "      <td>Sentence1: Many of the victims of Sunday 's at...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: Many of the victims of Sunday 's at...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>\" The bank requires growth from elsewhere in t...</td>\n",
              "      <td>The Bank of England \" requires growth from els...</td>\n",
              "      <td>1</td>\n",
              "      <td>1410</td>\n",
              "      <td>Sentence1: \" The bank requires growth from els...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: \" The bank requires growth from els...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Tyco later said the loan had not been forgiven...</td>\n",
              "      <td>Tyco has said the loan was not forgiven , but ...</td>\n",
              "      <td>1</td>\n",
              "      <td>142</td>\n",
              "      <td>Sentence1: Tyco later said the loan had not be...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: Tyco later said the loan had not be...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 41 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3fb0a350-0f84-43ee-b937-6e7608deec04')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-3fb0a350-0f84-43ee-b937-6e7608deec04 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-3fb0a350-0f84-43ee-b937-6e7608deec04');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 150
        }
      ],
      "source": [
        "#allocation\n",
        "yes=[]\n",
        "no=[]\n",
        "chatgpt_count=0\n",
        "#df=data\n",
        "for index, row in df.iterrows():\n",
        "    if row['allocate']==1: #edit the threshold here: I tried <-0.2(all human), ==0, <0.2, <0.4, <0.6, <0.8, <1(all chatgpt)\n",
        "      yes.append(row['MV_yes'])\n",
        "      no.append(row['MV_no'])\n",
        "    else:\n",
        "      yes.append(row['label_yes'])\n",
        "      no.append(row['label_no'])\n",
        "df['yes']=yes\n",
        "df['no']=no\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dLcetMjZFjSH"
      },
      "source": [
        "# Fine-tuning\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2dal0ggBcYdD"
      },
      "source": [
        "Dataset will be tokenized then split into training and validation sets. The validation set will be used to monitor training. For testing a separate test set will be loaded for analysis."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0ecREc7GnISW",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "#df = data #jigsaw-toxic-comment-classification-challenge"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UVI59S9VaAfB",
        "outputId": "8b193bb7-979b-4f81-b65d-f5d330186a63"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Label columns:  ['yes', 'no']\n"
          ]
        }
      ],
      "source": [
        "#combined\n",
        "cols = df.columns\n",
        "label_cols = list(cols[39:41])\n",
        "num_labels = len(label_cols)\n",
        "print('Label columns: ', label_cols)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xzgA5qQgYIBZ",
        "outputId": "6a89aea9-97bf-469b-f30d-a925b4ff9b80"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Count of 1 per label: \n",
            " yes    632\n",
            "no     385\n",
            "dtype: int64 \n",
            "\n",
            "Count of 0 per label: \n",
            " yes    385\n",
            "no     632\n",
            "dtype: int64\n"
          ]
        }
      ],
      "source": [
        "print('Count of 1 per label: \\n', df[label_cols].sum(), '\\n') # Label counts, may need to downsample or upsample\n",
        "print('Count of 0 per label: \\n', df[label_cols].eq(0).sum())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 734
        },
        "id": "0DF3ddjej5vd",
        "outputId": "16841558-5bfa-45bb-d501-3a469bf1cd7e"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                           sentence1  \\\n",
              "0  Bush said that he ordered the Treasury Departm...   \n",
              "1  Strong spokeswoman Stephanie Truog said , \" We...   \n",
              "2  Many of the victims of Sunday 's attack had be...   \n",
              "3  \" The bank requires growth from elsewhere in t...   \n",
              "4  Tyco later said the loan had not been forgiven...   \n",
              "\n",
              "                                           sentence2  label   idx  \\\n",
              "0  Bush said in a statement that he ordered the U...      1  3154   \n",
              "1  Strong Financial released a statement Wednesda...      0  3402   \n",
              "2  Many of the victims had been headed home for R...      0  3958   \n",
              "3  The Bank of England \" requires growth from els...      1  1410   \n",
              "4  Tyco has said the loan was not forgiven , but ...      1   142   \n",
              "\n",
              "                                                text  \\\n",
              "0  Sentence1: Bush said that he ordered the Treas...   \n",
              "1  Sentence1: Strong spokeswoman Stephanie Truog ...   \n",
              "2  Sentence1: Many of the victims of Sunday 's at...   \n",
              "3  Sentence1: \" The bank requires growth from els...   \n",
              "4  Sentence1: Tyco later said the loan had not be...   \n",
              "\n",
              "                                             prompt1  \\\n",
              "0  Please label if the following two sentences ar...   \n",
              "1  Please label if the following two sentences ar...   \n",
              "2  Please label if the following two sentences ar...   \n",
              "3  Please label if the following two sentences ar...   \n",
              "4  Please label if the following two sentences ar...   \n",
              "\n",
              "                                             prompt2  \\\n",
              "0  Sentence1: Bush said that he ordered the Treas...   \n",
              "1  Sentence1: Strong spokeswoman Stephanie Truog ...   \n",
              "2  Sentence1: Many of the victims of Sunday 's at...   \n",
              "3  Sentence1: \" The bank requires growth from els...   \n",
              "4  Sentence1: Tyco later said the loan had not be...   \n",
              "\n",
              "                                             prompt3  \\\n",
              "0  Given the following two sentences, please clas...   \n",
              "1  Given the following two sentences, please clas...   \n",
              "2  Given the following two sentences, please clas...   \n",
              "3  Given the following two sentences, please clas...   \n",
              "4  Given the following two sentences, please clas...   \n",
              "\n",
              "                                             prompt4  \\\n",
              "0  Is it true that the following two sentences ar...   \n",
              "1  Is it true that the following two sentences ar...   \n",
              "2  Is it true that the following two sentences ar...   \n",
              "3  Is it true that the following two sentences ar...   \n",
              "4  Is it true that the following two sentences ar...   \n",
              "\n",
              "                                             prompt5  ... entropy MV MV_yes  \\\n",
              "0  Is it true that the following two sentences ar...  ...    -0.0  1      1   \n",
              "1  Is it true that the following two sentences ar...  ...    -0.0  1      1   \n",
              "2  Is it true that the following two sentences ar...  ...    -0.0  1      1   \n",
              "3  Is it true that the following two sentences ar...  ...    -0.0  1      1   \n",
              "4  Is it true that the following two sentences ar...  ...    -0.0  1      1   \n",
              "\n",
              "  MV_no label_yes label_no allocate yes no one_hot_labels  \n",
              "0     0         1        0        1   1  0         [1, 0]  \n",
              "1     0         0        1        1   1  0         [1, 0]  \n",
              "2     0         0        1        1   1  0         [1, 0]  \n",
              "3     0         1        0        1   1  0         [1, 0]  \n",
              "4     0         1        0        1   1  0         [1, 0]  \n",
              "\n",
              "[5 rows x 42 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e76107dc-d6a6-47d7-b1aa-2d1f61dfc851\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence1</th>\n",
              "      <th>sentence2</th>\n",
              "      <th>label</th>\n",
              "      <th>idx</th>\n",
              "      <th>text</th>\n",
              "      <th>prompt1</th>\n",
              "      <th>prompt2</th>\n",
              "      <th>prompt3</th>\n",
              "      <th>prompt4</th>\n",
              "      <th>prompt5</th>\n",
              "      <th>...</th>\n",
              "      <th>entropy</th>\n",
              "      <th>MV</th>\n",
              "      <th>MV_yes</th>\n",
              "      <th>MV_no</th>\n",
              "      <th>label_yes</th>\n",
              "      <th>label_no</th>\n",
              "      <th>allocate</th>\n",
              "      <th>yes</th>\n",
              "      <th>no</th>\n",
              "      <th>one_hot_labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Bush said that he ordered the Treasury Departm...</td>\n",
              "      <td>Bush said in a statement that he ordered the U...</td>\n",
              "      <td>1</td>\n",
              "      <td>3154</td>\n",
              "      <td>Sentence1: Bush said that he ordered the Treas...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: Bush said that he ordered the Treas...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[1, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Strong spokeswoman Stephanie Truog said , \" We...</td>\n",
              "      <td>Strong Financial released a statement Wednesda...</td>\n",
              "      <td>0</td>\n",
              "      <td>3402</td>\n",
              "      <td>Sentence1: Strong spokeswoman Stephanie Truog ...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: Strong spokeswoman Stephanie Truog ...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[1, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Many of the victims of Sunday 's attack had be...</td>\n",
              "      <td>Many of the victims had been headed home for R...</td>\n",
              "      <td>0</td>\n",
              "      <td>3958</td>\n",
              "      <td>Sentence1: Many of the victims of Sunday 's at...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: Many of the victims of Sunday 's at...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[1, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>\" The bank requires growth from elsewhere in t...</td>\n",
              "      <td>The Bank of England \" requires growth from els...</td>\n",
              "      <td>1</td>\n",
              "      <td>1410</td>\n",
              "      <td>Sentence1: \" The bank requires growth from els...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: \" The bank requires growth from els...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[1, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Tyco later said the loan had not been forgiven...</td>\n",
              "      <td>Tyco has said the loan was not forgiven , but ...</td>\n",
              "      <td>1</td>\n",
              "      <td>142</td>\n",
              "      <td>Sentence1: Tyco later said the loan had not be...</td>\n",
              "      <td>Please label if the following two sentences ar...</td>\n",
              "      <td>Sentence1: Tyco later said the loan had not be...</td>\n",
              "      <td>Given the following two sentences, please clas...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>Is it true that the following two sentences ar...</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[1, 0]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 42 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e76107dc-d6a6-47d7-b1aa-2d1f61dfc851')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e76107dc-d6a6-47d7-b1aa-2d1f61dfc851 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e76107dc-d6a6-47d7-b1aa-2d1f61dfc851');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 153
        }
      ],
      "source": [
        "df['one_hot_labels'] = list(df[label_cols].values)\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MlhHifh5bW7e"
      },
      "outputs": [],
      "source": [
        "labels = list(df.one_hot_labels.values)\n",
        "comments = list(df.text.values)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IlMHfElhGJzc"
      },
      "source": [
        "Load the pretrained tokenizer that corresponds to your choice in model. e.g.,\n",
        "\n",
        "```\n",
        "BERT:\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
        "\n",
        "XLNet:\n",
        "tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased', do_lower_case=False)\n",
        "\n",
        "RoBERTa:\n",
        "tokenizer = RobertaTokenizer.from_pretrained('roberta-base', do_lower_case=False)\n",
        "```\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IhVr8SziL_PY"
      },
      "source": [
        "In order to avoid memory issues with Google Colab, I enforce a max_length of 100 tokens. Note that some sentences may not adequately represent each label because of this."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HNsEu-vUur-4",
        "outputId": "3c687927-aa76-46c0-d4f6-0bce0f08d5d7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2377: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tokenizer outputs:  dict_keys(['input_ids', 'attention_mask'])\n"
          ]
        }
      ],
      "source": [
        "from transformers import RobertaTokenizer\n",
        "max_length = 100\n",
        "tokenizer = RobertaTokenizer.from_pretrained('roberta-base', do_lower_case=False) # tokenizer\n",
        "encodings = tokenizer.batch_encode_plus(comments,max_length=max_length,pad_to_max_length=True) # tokenizer's encoding method\n",
        "print('tokenizer outputs: ', encodings.keys())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nG9vOTp92yZE",
        "outputId": "4258abef-382f-4559-96f3-e04a9869a03d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "dict_keys(['input_ids', 'attention_mask'])"
            ]
          },
          "execution_count": 62,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "encodings.keys()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "l6CCLSjfur-9"
      },
      "outputs": [],
      "source": [
        "input_ids = encodings['input_ids'] # tokenized and encoded sentences\n",
        "#token_type_ids = encodings['token_type_ids'] # token type ids\n",
        "attention_masks = encodings['attention_mask'] # attention masks"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vSOFbThlYcpb",
        "outputId": "7ed1ae93-ab55-4e47-e18d-eaede42a24f3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "df label indices with only one instance:  []\n"
          ]
        }
      ],
      "source": [
        "# Identifying indices of 'one_hot_labels' entries that only occur once - this will allow us to stratify split our training data later\n",
        "label_counts = df.one_hot_labels.astype(str).value_counts()\n",
        "one_freq = label_counts[label_counts==1].keys()\n",
        "one_freq_idxs = sorted(list(df[df.one_hot_labels.astype(str).isin(one_freq)].index), reverse=True)\n",
        "print('df label indices with only one instance: ', one_freq_idxs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CQQ7CoOag_r7"
      },
      "outputs": [],
      "source": [
        "# Gathering single instance inputs to force into the training set after stratified split\n",
        "one_freq_input_ids = [input_ids.pop(i) for i in one_freq_idxs]\n",
        "#one_freq_token_types = [token_type_ids.pop(i) for i in one_freq_idxs]\n",
        "one_freq_attention_masks = [attention_masks.pop(i) for i in one_freq_idxs]\n",
        "one_freq_labels = [labels.pop(i) for i in one_freq_idxs]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r9PxAt48HRRj"
      },
      "source": [
        "Be sure to handle all classes during validation using \"stratify\" during train/validation split:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WPFaq4ufnIT2"
      },
      "outputs": [],
      "source": [
        "# Use train_test_split to split our data into train and validation sets\n",
        "\n",
        "train_inputs, validation_inputs, train_labels, validation_labels, train_masks, validation_masks = train_test_split(input_ids, labels, attention_masks,\n",
        "                                                            random_state=20, test_size=0.10, stratify = labels)\n",
        "\n",
        "# Add one frequency data to train data\n",
        "train_inputs.extend(one_freq_input_ids)\n",
        "train_labels.extend(one_freq_labels)\n",
        "train_masks.extend(one_freq_attention_masks)\n",
        "#train_token_types.extend(one_freq_token_types)\n",
        "\n",
        "# Convert all of our data into torch tensors, the required datatype for our model\n",
        "train_inputs = torch.tensor(train_inputs)\n",
        "train_labels = torch.tensor(train_labels)\n",
        "train_masks = torch.tensor(train_masks)\n",
        "#train_token_types = torch.tensor(train_token_types)\n",
        "\n",
        "validation_inputs = torch.tensor(validation_inputs)\n",
        "validation_labels = torch.tensor(validation_labels)\n",
        "validation_masks = torch.tensor(validation_masks)\n",
        "#validation_token_types = torch.tensor(validation_token_types)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZRnuLna-nIT4"
      },
      "outputs": [],
      "source": [
        "# Select a batch size for training. For fine-tuning with XLNet, the authors recommend a batch size of 32, 48, or 128. We will use 32 here to avoid memory issues.\n",
        "batch_size = 16\n",
        "\n",
        "# Create an iterator of our data with torch DataLoader. This helps save on memory during training because, unlike a for loop,\n",
        "# with an iterator the entire dataset does not need to be loaded into memory\n",
        "\n",
        "train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
        "train_sampler = RandomSampler(train_data)\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)\n",
        "validation_sampler = SequentialSampler(validation_data)\n",
        "validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iiFRnP_ZTBFa"
      },
      "outputs": [],
      "source": [
        "torch.save(validation_dataloader,'validation_data_loader')\n",
        "torch.save(train_dataloader,'train_data_loader')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ncGteBuSFuZM"
      },
      "source": [
        "## Load Model & Set Params"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z0dL-Bz_NrGj"
      },
      "source": [
        "Load the appropriate model below, each model already contains a single dense layer for classification on top.\n",
        "\n",
        "\n",
        "\n",
        "```\n",
        "BERT:\n",
        "model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\", num_labels=num_labels)\n",
        "\n",
        "XLNet:\n",
        "model = XLNetForSequenceClassification.from_pretrained(\"xlnet-base-cased\", num_labels=num_labels)\n",
        "\n",
        "RoBERTa:\n",
        "model = RobertaForSequenceClassification.from_pretrained('roberta-base', num_labels=num_labels)\n",
        "```\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ujk4k16DnIT6",
        "outputId": "6d052091-943d-4757-cccc-bff64848704f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.layer_norm.bias', 'lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.weight', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RobertaForSequenceClassification(\n",
              "  (roberta): RobertaModel(\n",
              "    (embeddings): RobertaEmbeddings(\n",
              "      (word_embeddings): Embedding(50265, 768, padding_idx=1)\n",
              "      (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
              "      (token_type_embeddings): Embedding(1, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): RobertaEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0-11): 12 x RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (classifier): RobertaClassificationHead(\n",
              "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "    (out_proj): Linear(in_features=768, out_features=2, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 162
        }
      ],
      "source": [
        "# Load model, the pretrained model will include a single linear classification layer on top for classification.\n",
        "from transformers import RobertaForSequenceClassification\n",
        "model = RobertaForSequenceClassification.from_pretrained('roberta-base', num_labels=num_labels)\n",
        "model.cuda()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jGE4gv9qfhRG"
      },
      "source": [
        "Setting custom optimization parameters for the AdamW optimizer https://huggingface.co/transformers/main_classes/optimizer_schedules.html"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GsV8zwWYnIT9"
      },
      "outputs": [],
      "source": [
        "# setting custom optimization parameters. You may implement a scheduler here as well.\n",
        "param_optimizer = list(model.named_parameters())\n",
        "no_decay = ['bias', 'gamma', 'beta']\n",
        "optimizer_grouped_parameters = [\n",
        "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
        "     'weight_decay_rate': 0.01},\n",
        "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
        "     'weight_decay_rate': 0.0}\n",
        "]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aOomZIEIoHOL",
        "outputId": "af4e9db3-3237-4427-cd0d-28cae2b766ac"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "from transformers import AdamW\n",
        "optimizer = AdamW(optimizer_grouped_parameters,lr=2e-5 ,correct_bias=True)\n",
        "# optimizer = AdamW(model.parameters(),lr=2e-5)  # Default optimization"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JRQQZ8zIFzLW"
      },
      "source": [
        "## Train Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ho6xx8cQfhvK"
      },
      "outputs": [],
      "source": [
        "def is_max(array):\n",
        "  max_val=max(array)\n",
        "  res=[i==max_val for i in array]\n",
        "  return res"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uDLZmEC_oKo3",
        "outputId": "7efb3818-4eb8-4737-ab26-664cc9fa9f1a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:   0%|          | 0/10 [00:00<?, ?it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train loss: 0.6403988234955689\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  10%|█         | 1/10 [00:15<02:17, 15.25s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  70.58823529411765\n",
            "Flat Validation Accuracy:  70.58823529411765\n",
            "Train loss: 0.4949221536517143\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  20%|██        | 2/10 [00:31<02:04, 15.60s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  71.56862745098039\n",
            "Flat Validation Accuracy:  71.56862745098039\n",
            "Train loss: 0.35384158212049255\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  30%|███       | 3/10 [00:46<01:49, 15.66s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  77.45098039215686\n",
            "Flat Validation Accuracy:  77.45098039215686\n",
            "Train loss: 0.21234087273478508\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  40%|████      | 4/10 [01:02<01:34, 15.75s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  74.50980392156863\n",
            "Flat Validation Accuracy:  74.50980392156863\n",
            "Train loss: 0.10146555986961928\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  50%|█████     | 5/10 [01:18<01:19, 15.84s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  72.54901960784314\n",
            "Flat Validation Accuracy:  72.54901960784314\n",
            "Train loss: 0.044939427411761775\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  60%|██████    | 6/10 [01:34<01:03, 15.87s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  74.50980392156863\n",
            "Flat Validation Accuracy:  74.50980392156863\n",
            "Train loss: 0.10822694938116033\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  70%|███████   | 7/10 [01:50<00:47, 15.86s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  74.50980392156863\n",
            "Flat Validation Accuracy:  74.50980392156863\n",
            "Train loss: 0.07492344775879435\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  80%|████████  | 8/10 [02:06<00:31, 15.85s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  73.5294117647059\n",
            "Flat Validation Accuracy:  73.52941176470588\n",
            "Train loss: 0.025043121514970373\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  90%|█████████ | 9/10 [02:22<00:15, 15.93s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  71.56862745098039\n",
            "Flat Validation Accuracy:  71.56862745098039\n",
            "Train loss: 0.04611270974293865\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch: 100%|██████████| 10/10 [02:38<00:00, 15.84s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  72.54901960784314\n",
            "Flat Validation Accuracy:  72.54901960784314\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "# Store our loss and accuracy for plotting\n",
        "train_loss_set = []\n",
        "\n",
        "# Number of training epochs (authors recommend between 2 and 4)\n",
        "epochs = 10\n",
        "\n",
        "# trange is a tqdm wrapper around the normal python range\n",
        "for _ in trange(epochs, desc=\"Epoch\"):\n",
        "\n",
        "  # Training\n",
        "\n",
        "  # Set our model to training mode (as opposed to evaluation mode)\n",
        "  model.train()\n",
        "\n",
        "  # Tracking variables\n",
        "  tr_loss = 0 #running loss\n",
        "  nb_tr_examples, nb_tr_steps = 0, 0\n",
        "\n",
        "  # Train the data for one epoch\n",
        "  for step, batch in enumerate(train_dataloader):\n",
        "    # Add batch to GPU\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    # Unpack the inputs from our dataloader\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    # Clear out the gradients (by default they accumulate)\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    # # Forward pass for multiclass classification\n",
        "    # outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n",
        "    # loss = outputs[0]\n",
        "    # logits = outputs[1]\n",
        "\n",
        "    # Forward pass for multilabel classification\n",
        "    outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "    logits = outputs[0]\n",
        "    loss_func = BCEWithLogitsLoss()\n",
        "    loss = loss_func(logits.view(-1,num_labels),b_labels.type_as(logits).view(-1,num_labels)) #convert labels to float for calculation\n",
        "    # loss_func = BCELoss()\n",
        "    # loss = loss_func(torch.sigmoid(logits.view(-1,num_labels)),b_labels.type_as(logits).view(-1,num_labels)) #convert labels to float for calculation\n",
        "    train_loss_set.append(loss.item())\n",
        "\n",
        "    # Backward pass\n",
        "    loss.backward()\n",
        "    # Update parameters and take a step using the computed gradient\n",
        "    optimizer.step()\n",
        "    # scheduler.step()\n",
        "    # Update tracking variables\n",
        "    tr_loss += loss.item()\n",
        "    nb_tr_examples += b_input_ids.size(0)\n",
        "    nb_tr_steps += 1\n",
        "\n",
        "  print(\"Train loss: {}\".format(tr_loss/nb_tr_steps))\n",
        "\n",
        "###############################################################################\n",
        "\n",
        "  # Validation\n",
        "\n",
        "  # Put model in evaluation mode to evaluate loss on the validation set\n",
        "  model.eval()\n",
        "\n",
        "  # Variables to gather full output\n",
        "  logit_preds,true_labels,pred_labels,tokenized_texts = [],[],[],[]\n",
        "\n",
        "  # Predict\n",
        "  for i, batch in enumerate(validation_dataloader):\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    # Unpack the inputs from our dataloader\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    with torch.no_grad():\n",
        "      # Forward pass\n",
        "      outs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "      b_logit_pred = outs[0]\n",
        "      pred_label = torch.sigmoid(b_logit_pred)\n",
        "\n",
        "      b_logit_pred = b_logit_pred.detach().cpu().numpy()\n",
        "      pred_label = pred_label.to('cpu').numpy()\n",
        "      b_labels = b_labels.to('cpu').numpy()\n",
        "\n",
        "    tokenized_texts.append(b_input_ids)\n",
        "    logit_preds.append(b_logit_pred)\n",
        "    true_labels.append(b_labels)\n",
        "    pred_labels.append(pred_label)\n",
        "\n",
        "  # Flatten outputs\n",
        "  pred_labels = [item for sublist in pred_labels for item in sublist]\n",
        "  true_labels = [item for sublist in true_labels for item in sublist]\n",
        "\n",
        "  # Calculate Accuracy\n",
        "  #threshold = 0.30\n",
        "  #pred_bools = [pl>threshold for pl in pred_labels]\n",
        "  #print(pred_labels)\n",
        "  #max_val = max(pred_labels)\n",
        "  pred_bools = [is_max(arr) for arr in pred_labels]\n",
        "  true_bools = [tl==1 for tl in true_labels]\n",
        "  val_f1_accuracy = f1_score(true_bools,pred_bools,average='micro')*100\n",
        "  val_flat_accuracy = accuracy_score(true_bools, pred_bools)*100\n",
        "\n",
        "  print('F1 Validation Accuracy: ', val_f1_accuracy)\n",
        "  print('Flat Validation Accuracy: ', val_flat_accuracy)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l6_zh67P9NF2"
      },
      "source": [
        "# Evaluation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_7dd2GE3F4yK"
      },
      "source": [
        "## Load and Preprocess Test Data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        },
        "id": "G5Q7hC4GFOLJ",
        "outputId": "f776084e-5df3-40f4-84be-13471881e648"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                             sentence1  \\\n",
              "403  Their contract will expire at 12 : 01 a.m. Wed...   \n",
              "404  But plaque volume increased by 2.7 percent in ...   \n",
              "405  Today in the US , the book - kept under wraps ...   \n",
              "406  The S & P / TSX composite rose 87.74 points on...   \n",
              "407  Ex-KGB agent Putin added that the Beatles were...   \n",
              "\n",
              "                                             sentence2  label   idx  yes  no  \\\n",
              "403  \" It has outraged the membership , \" said Rian...      0  4023    0   1   \n",
              "404  The volume of plaque in Pravachol patients ' a...      1  4028    1   0   \n",
              "405  Tomorrow the book , kept under wraps by G. P. ...      1  4040    1   0   \n",
              "406  On the week , the Dow Jones industrial average...      0  4049    0   1   \n",
              "407  In Soviet times the Beatles ' music \" was cons...      1  4053    1   0   \n",
              "\n",
              "                                                  text  \n",
              "403  Sentence1: Their contract will expire at 12 : ...  \n",
              "404  Sentence1: But plaque volume increased by 2.7 ...  \n",
              "405  Sentence1: Today in the US , the book - kept u...  \n",
              "406  Sentence1: The S & P / TSX composite rose 87.7...  \n",
              "407  Sentence1: Ex-KGB agent Putin added that the B...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-07c0605c-b98c-4a43-8c18-d71704d4be72\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence1</th>\n",
              "      <th>sentence2</th>\n",
              "      <th>label</th>\n",
              "      <th>idx</th>\n",
              "      <th>yes</th>\n",
              "      <th>no</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>403</th>\n",
              "      <td>Their contract will expire at 12 : 01 a.m. Wed...</td>\n",
              "      <td>\" It has outraged the membership , \" said Rian...</td>\n",
              "      <td>0</td>\n",
              "      <td>4023</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>Sentence1: Their contract will expire at 12 : ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>404</th>\n",
              "      <td>But plaque volume increased by 2.7 percent in ...</td>\n",
              "      <td>The volume of plaque in Pravachol patients ' a...</td>\n",
              "      <td>1</td>\n",
              "      <td>4028</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>Sentence1: But plaque volume increased by 2.7 ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>405</th>\n",
              "      <td>Today in the US , the book - kept under wraps ...</td>\n",
              "      <td>Tomorrow the book , kept under wraps by G. P. ...</td>\n",
              "      <td>1</td>\n",
              "      <td>4040</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>Sentence1: Today in the US , the book - kept u...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>406</th>\n",
              "      <td>The S &amp; P / TSX composite rose 87.74 points on...</td>\n",
              "      <td>On the week , the Dow Jones industrial average...</td>\n",
              "      <td>0</td>\n",
              "      <td>4049</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>Sentence1: The S &amp; P / TSX composite rose 87.7...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>407</th>\n",
              "      <td>Ex-KGB agent Putin added that the Beatles were...</td>\n",
              "      <td>In Soviet times the Beatles ' music \" was cons...</td>\n",
              "      <td>1</td>\n",
              "      <td>4053</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>Sentence1: Ex-KGB agent Putin added that the B...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-07c0605c-b98c-4a43-8c18-d71704d4be72')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-07c0605c-b98c-4a43-8c18-d71704d4be72 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-07c0605c-b98c-4a43-8c18-d71704d4be72');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ],
      "source": [
        "test_df = pd.read_csv('mrpc_valid.csv')\n",
        "#test_labels_df = pd.read_csv('test_labels.csv')\n",
        "#test_df = test_df.merge(test_labels_df, on='id', how='left')\n",
        "# test_df['text']=test_df['sentence']\n",
        "test_df['yes']=test_df['label'].apply(lambda x:1 if x==1 else 0)\n",
        "test_df['no']=test_df['label'].apply(lambda x:1 if x==0 else 0)\n",
        "test_df['text']='Sentence1: '+test_df['sentence1']+'Sentence2: '+test_df['sentence2']\n",
        "#print('Null values: ', test_df.isnull().values.any()) #should not be any null sentences or labels\n",
        "#print('Same columns between train and test: ', label_cols == test_label_cols) #columns should be the same\n",
        "test_df.tail()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j_PZ1mVOhkK8",
        "outputId": "b73b1e36-2832-4e87-90a9-1611071dfffb"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['yes', 'no']"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ],
      "source": [
        "test_label_cols = list(test_df.columns[4:6])\n",
        "test_label_cols"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        },
        "id": "77rjCrMGpYxz",
        "outputId": "19339d22-4c42-4e3e-938f-554858403878"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                           sentence1  \\\n",
              "0  He said the foodservice pie business doesn 't ...   \n",
              "1  Magnarelli said Racicot hated the Iraqi regime...   \n",
              "2  The dollar was at 116.92 yen against the yen ,...   \n",
              "3  The AFL-CIO is waiting until October to decide...   \n",
              "4  No dates have been set for the civil or the cr...   \n",
              "\n",
              "                                           sentence2  label  idx  yes  no  \\\n",
              "0  \" The foodservice pie business does not fit ou...      1    9    1   0   \n",
              "1  His wife said he was \" 100 percent behind Geor...      0   18    0   1   \n",
              "2  The dollar was at 116.78 yen JPY = , virtually...      0   25    0   1   \n",
              "3  The AFL-CIO announced Wednesday that it will d...      1   32    1   0   \n",
              "4  No dates have been set for the criminal or civ...      0   33    0   1   \n",
              "\n",
              "                                                text one_hot_labels  \n",
              "0  Sentence1: He said the foodservice pie busines...         [1, 0]  \n",
              "1  Sentence1: Magnarelli said Racicot hated the I...         [0, 1]  \n",
              "2  Sentence1: The dollar was at 116.92 yen agains...         [0, 1]  \n",
              "3  Sentence1: The AFL-CIO is waiting until Octobe...         [1, 0]  \n",
              "4  Sentence1: No dates have been set for the civi...         [0, 1]  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f52f0c50-4e24-4905-a49b-8d148f619931\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence1</th>\n",
              "      <th>sentence2</th>\n",
              "      <th>label</th>\n",
              "      <th>idx</th>\n",
              "      <th>yes</th>\n",
              "      <th>no</th>\n",
              "      <th>text</th>\n",
              "      <th>one_hot_labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>He said the foodservice pie business doesn 't ...</td>\n",
              "      <td>\" The foodservice pie business does not fit ou...</td>\n",
              "      <td>1</td>\n",
              "      <td>9</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>Sentence1: He said the foodservice pie busines...</td>\n",
              "      <td>[1, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Magnarelli said Racicot hated the Iraqi regime...</td>\n",
              "      <td>His wife said he was \" 100 percent behind Geor...</td>\n",
              "      <td>0</td>\n",
              "      <td>18</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>Sentence1: Magnarelli said Racicot hated the I...</td>\n",
              "      <td>[0, 1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>The dollar was at 116.92 yen against the yen ,...</td>\n",
              "      <td>The dollar was at 116.78 yen JPY = , virtually...</td>\n",
              "      <td>0</td>\n",
              "      <td>25</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>Sentence1: The dollar was at 116.92 yen agains...</td>\n",
              "      <td>[0, 1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>The AFL-CIO is waiting until October to decide...</td>\n",
              "      <td>The AFL-CIO announced Wednesday that it will d...</td>\n",
              "      <td>1</td>\n",
              "      <td>32</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>Sentence1: The AFL-CIO is waiting until Octobe...</td>\n",
              "      <td>[1, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>No dates have been set for the civil or the cr...</td>\n",
              "      <td>No dates have been set for the criminal or civ...</td>\n",
              "      <td>0</td>\n",
              "      <td>33</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>Sentence1: No dates have been set for the civi...</td>\n",
              "      <td>[0, 1]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f52f0c50-4e24-4905-a49b-8d148f619931')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f52f0c50-4e24-4905-a49b-8d148f619931 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f52f0c50-4e24-4905-a49b-8d148f619931');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ],
      "source": [
        "test_df = test_df[~test_df[test_label_cols].eq(-1).any(axis=1)] #remove irrelevant rows/comments with -1 values\n",
        "test_df['one_hot_labels'] = list(test_df[test_label_cols].values)\n",
        "test_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1a41OmU2i7qp"
      },
      "outputs": [],
      "source": [
        "# Gathering input data\n",
        "test_labels = list(test_df.one_hot_labels.values)\n",
        "test_comments = list(test_df.text.values)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "amySMO8EQzf2",
        "outputId": "56931eb8-6d2c-4289-fdc7-63da92cff5c8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2377: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "# Encoding input data\n",
        "test_encodings = tokenizer.batch_encode_plus(test_comments,max_length=max_length,pad_to_max_length=True)\n",
        "test_input_ids = test_encodings['input_ids']\n",
        "#test_token_type_ids = test_encodings['token_type_ids']\n",
        "test_attention_masks = test_encodings['attention_mask']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hqOfi9fkRaRN"
      },
      "outputs": [],
      "source": [
        "# Make tensors out of data\n",
        "test_inputs = torch.tensor(test_input_ids)\n",
        "test_labels = torch.tensor(test_labels)\n",
        "test_masks = torch.tensor(test_attention_masks)\n",
        "#test_token_types = torch.tensor(test_token_type_ids)\n",
        "# Create test dataloader\n",
        "test_data = TensorDataset(test_inputs, test_masks, test_labels)\n",
        "#test_data = TensorDataset(test_inputs, test_masks)\n",
        "test_sampler = SequentialSampler(test_data)\n",
        "test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=batch_size)\n",
        "# Save test dataloader\n",
        "torch.save(test_dataloader,'test_data_loader')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PFTWxCA_GBau"
      },
      "source": [
        "## Prediction and Metics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NPvrL6OFSQvf"
      },
      "outputs": [],
      "source": [
        "# Test\n",
        "\n",
        "# Put model in evaluation mode to evaluate loss on the validation set\n",
        "model.eval()\n",
        "\n",
        "#track variables\n",
        "logit_preds,true_labels,pred_labels,tokenized_texts = [],[],[],[]\n",
        "\n",
        "# Predict\n",
        "for i, batch in enumerate(test_dataloader):\n",
        "  batch = tuple(t.to(device) for t in batch)\n",
        "  # Unpack the inputs from our dataloader\n",
        "  b_input_ids, b_input_mask, b_labels = batch\n",
        "  #b_input_ids, b_input_mask = batch\n",
        "  with torch.no_grad():\n",
        "    # Forward pass\n",
        "    outs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "    b_logit_pred = outs[0]\n",
        "    pred_label = torch.sigmoid(b_logit_pred)\n",
        "\n",
        "    b_logit_pred = b_logit_pred.detach().cpu().numpy()\n",
        "    pred_label = pred_label.to('cpu').numpy()\n",
        "    b_labels = b_labels.to('cpu').numpy()\n",
        "\n",
        "  tokenized_texts.append(b_input_ids)\n",
        "  logit_preds.append(b_logit_pred)\n",
        "  true_labels.append(b_labels)\n",
        "  pred_labels.append(pred_label)\n",
        "\n",
        "# Flatten outputs\n",
        "tokenized_texts = [item for sublist in tokenized_texts for item in sublist]\n",
        "pred_labels = [item for sublist in pred_labels for item in sublist]\n",
        "true_labels = [item for sublist in true_labels for item in sublist]\n",
        "# Converting flattened binary values to boolean values\n",
        "true_bools = [tl==1 for tl in true_labels]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bQeGWqeMzAoZ"
      },
      "source": [
        "We need to threshold our sigmoid function outputs which range from [0, 1]. Below I use 0.50 as a threshold."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined4 (0.2)\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Micro F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Macro F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Weighted F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mssu0iGZIQ_M",
        "outputId": "97391cc4-3427-4ed9-a99f-412cdf9ee025"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Micro F1 Accuracy:  0.8235294117647058\n",
            "Macro F1 Accuracy:  0.7999782111341105\n",
            "Weighted F1 Accuracy:  0.825211640381177\n",
            "Test Flat Accuracy:  0.8235294117647058 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         yes       0.88      0.85      0.87       279\n",
            "          no       0.71      0.76      0.73       129\n",
            "\n",
            "   micro avg       0.82      0.82      0.82       408\n",
            "   macro avg       0.79      0.81      0.80       408\n",
            "weighted avg       0.83      0.82      0.83       408\n",
            " samples avg       0.82      0.82      0.82       408\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined4 (0.4)\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Micro F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Macro F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Weighted F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dOQOmpGMNIfv",
        "outputId": "1b6d2837-8519-450f-cbf6-fd28deea790f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Micro F1 Accuracy:  0.8259803921568627\n",
            "Macro F1 Accuracy:  0.7983614430994759\n",
            "Weighted F1 Accuracy:  0.8257974852094628\n",
            "Test Flat Accuracy:  0.8259803921568627 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         yes       0.87      0.87      0.87       279\n",
            "          no       0.73      0.72      0.72       129\n",
            "\n",
            "   micro avg       0.83      0.83      0.83       408\n",
            "   macro avg       0.80      0.80      0.80       408\n",
            "weighted avg       0.83      0.83      0.83       408\n",
            " samples avg       0.83      0.83      0.83       408\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined4 (0.6)\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Micro F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Macro F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Weighted F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DBbC5mY-U92c",
        "outputId": "cbc81fda-2ffb-40ef-9d3e-68c9e0f82885"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Micro F1 Accuracy:  0.8063725490196079\n",
            "Macro F1 Accuracy:  0.7662704944780026\n",
            "Weighted F1 Accuracy:  0.8018640340119717\n",
            "Test Flat Accuracy:  0.8063725490196079 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         yes       0.84      0.89      0.86       279\n",
            "          no       0.73      0.62      0.67       129\n",
            "\n",
            "   micro avg       0.81      0.81      0.81       408\n",
            "   macro avg       0.78      0.76      0.77       408\n",
            "weighted avg       0.80      0.81      0.80       408\n",
            " samples avg       0.81      0.81      0.81       408\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined4 (0.8)\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Micro F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Macro F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Weighted F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P52rjx5JP7u5",
        "outputId": "bd935e3b-8075-4070-cab3-b90de7d71082"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Micro F1 Accuracy:  0.7843137254901961\n",
            "Macro F1 Accuracy:  0.7307334633268336\n",
            "Weighted F1 Accuracy:  0.7748930200548796\n",
            "Test Flat Accuracy:  0.7843137254901961 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         yes       0.81      0.90      0.85       279\n",
            "          no       0.71      0.53      0.61       129\n",
            "\n",
            "   micro avg       0.78      0.78      0.78       408\n",
            "   macro avg       0.76      0.72      0.73       408\n",
            "weighted avg       0.78      0.78      0.77       408\n",
            " samples avg       0.78      0.78      0.78       408\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined4 (1)\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Micro F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Macro F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Weighted F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qr5DAskVThGe",
        "outputId": "13cfb01a-88fd-4740-c955-58f3796bf5e6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Micro F1 Accuracy:  0.7573529411764706\n",
            "Macro F1 Accuracy:  0.6591478696741855\n",
            "Weighted F1 Accuracy:  0.7264116172784904\n",
            "Test Flat Accuracy:  0.7573529411764706 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         yes       0.76      0.95      0.84       279\n",
            "          no       0.75      0.35      0.48       129\n",
            "\n",
            "   micro avg       0.76      0.76      0.76       408\n",
            "   macro avg       0.75      0.65      0.66       408\n",
            "weighted avg       0.76      0.76      0.73       408\n",
            " samples avg       0.76      0.76      0.76       408\n",
            "\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}