Random_allocation_agnews.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m5tUoHe9FRhs"
      },
      "source": [
        "## Import Libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Dr7BCHS-nIRW",
        "outputId": "dece085e-900d-45df-89bf-462c3f202c57",
        "scrolled": true
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.30.2-py3-none-any.whl (7.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.2/7.2 MB\u001b[0m \u001b[31m62.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.0)\n",
            "Collecting huggingface-hub<1.0,>=0.14.1 (from transformers)\n",
            "  Downloading huggingface_hub-0.15.1-py3-none-any.whl (236 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m236.8/236.8 kB\u001b[0m \u001b[31m27.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.22.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.27.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m88.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers)\n",
            "  Downloading safetensors-0.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m79.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (2023.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Installing collected packages: tokenizers, safetensors, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.15.1 safetensors-0.3.1 tokenizers-0.13.3 transformers-4.30.2\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import torch\n",
        "from torch.nn import BCEWithLogitsLoss, BCELoss\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import MultiLabelBinarizer\n",
        "from sklearn.metrics import classification_report, confusion_matrix, multilabel_confusion_matrix, f1_score, accuracy_score\n",
        "import pickle\n",
        "#from transformers import *\n",
        "from tqdm import tqdm, trange\n",
        "from ast import literal_eval"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zB5Dij6JuItX"
      },
      "outputs": [],
      "source": [
        "# from google.colab import drive\n",
        "# drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UhjnJEwKnISB",
        "outputId": "692de56d-43c8-4e15-fd08-a5812fa481de"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found GPU at: /device:GPU:0\n"
          ]
        }
      ],
      "source": [
        "device_name = tf.test.gpu_device_name()\n",
        "if device_name != '/device:GPU:0':\n",
        "  raise SystemError('GPU device not found')\n",
        "print('Found GPU at: {}'.format(device_name))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "uorMX_zrnISM",
        "outputId": "648e9018-e419-4209-e4d1-5f40d1d3b6cd",
        "scrolled": true
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Tesla T4'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "n_gpu = torch.cuda.device_count()\n",
        "torch.cuda.get_device_name(0)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 699
        },
        "id": "62QeEJcKtWm3",
        "outputId": "f232cf66-bb2e-4c93-9f99-b68ff99da393"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Class Index                                               text  \\\n",
              "0            3  Title: Wall St. Bears Claw Back Into the Black...   \n",
              "1            3  Title: Carlyle Looks Toward Commercial Aerospa...   \n",
              "2            3  Title: Oil and Economy Cloud Stocks' Outlook (...   \n",
              "3            3  Title: Iraq Halts Oil Exports from Main Southe...   \n",
              "4            3  Title: Oil prices soar to all-time record, pos...   \n",
              "\n",
              "            label1    label2                                         label3  \\\n",
              "0  topic: business  business                                topic: business   \n",
              "1         business  business                                topic: business   \n",
              "2  topic: business  business  the topic of this news article is \"business\".   \n",
              "3         business  business                                       business   \n",
              "4         business  business                                topic: business   \n",
              "\n",
              "  label4                                             label5 label6  \\\n",
              "0  true.                                              FALSE  true.   \n",
              "1  true.                                             false.   TRUE   \n",
              "2   TRUE                                             false.  true.   \n",
              "3   TRUE  false. the news title and description belong t...  true.   \n",
              "4  true.                                              FALSE   TRUE   \n",
              "\n",
              "                                              label7  \\\n",
              "0  false. this news title and description belongs...   \n",
              "1                                              FALSE   \n",
              "2                                              FALSE   \n",
              "3                                               TRUE   \n",
              "4                                               TRUE   \n",
              "\n",
              "                     label8  ... sci6 world7 sports7 business7 sci7  \\\n",
              "0                  business  ...    0      1       0         1    0   \n",
              "1                  business  ...    0      1       0         1    0   \n",
              "2  the topic is \"business\".  ...    0      1       0         1    0   \n",
              "3  the topic is \"business\".  ...    0      1       0         1    0   \n",
              "4                  business  ...    0      1       0         1    0   \n",
              "\n",
              "   entropy_world  entropy_sports  entropy_business  entropy_sci   entropy  \n",
              "0       0.863121             0.0               0.0     0.000000  0.215780  \n",
              "1       0.863121             0.0               0.0     0.000000  0.215780  \n",
              "2       0.863121             0.0               0.0     0.000000  0.215780  \n",
              "3       0.863121             0.0               0.0     0.591673  0.363698  \n",
              "4       0.863121             0.0               0.0     0.591673  0.363698  \n",
              "\n",
              "[5 rows x 54 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-69fadd77-3371-4464-bda0-0c9d3a771119\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Class Index</th>\n",
              "      <th>text</th>\n",
              "      <th>label1</th>\n",
              "      <th>label2</th>\n",
              "      <th>label3</th>\n",
              "      <th>label4</th>\n",
              "      <th>label5</th>\n",
              "      <th>label6</th>\n",
              "      <th>label7</th>\n",
              "      <th>label8</th>\n",
              "      <th>...</th>\n",
              "      <th>sci6</th>\n",
              "      <th>world7</th>\n",
              "      <th>sports7</th>\n",
              "      <th>business7</th>\n",
              "      <th>sci7</th>\n",
              "      <th>entropy_world</th>\n",
              "      <th>entropy_sports</th>\n",
              "      <th>entropy_business</th>\n",
              "      <th>entropy_sci</th>\n",
              "      <th>entropy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Wall St. Bears Claw Back Into the Black...</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>business</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>true.</td>\n",
              "      <td>FALSE</td>\n",
              "      <td>true.</td>\n",
              "      <td>false. this news title and description belongs...</td>\n",
              "      <td>business</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.863121</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.215780</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Carlyle Looks Toward Commercial Aerospa...</td>\n",
              "      <td>business</td>\n",
              "      <td>business</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>true.</td>\n",
              "      <td>false.</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>FALSE</td>\n",
              "      <td>business</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.863121</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.215780</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Oil and Economy Cloud Stocks' Outlook (...</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>business</td>\n",
              "      <td>the topic of this news article is \"business\".</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>false.</td>\n",
              "      <td>true.</td>\n",
              "      <td>FALSE</td>\n",
              "      <td>the topic is \"business\".</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.863121</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.215780</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Iraq Halts Oil Exports from Main Southe...</td>\n",
              "      <td>business</td>\n",
              "      <td>business</td>\n",
              "      <td>business</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>false. the news title and description belong t...</td>\n",
              "      <td>true.</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>the topic is \"business\".</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.863121</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.591673</td>\n",
              "      <td>0.363698</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Oil prices soar to all-time record, pos...</td>\n",
              "      <td>business</td>\n",
              "      <td>business</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>true.</td>\n",
              "      <td>FALSE</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>business</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.863121</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.591673</td>\n",
              "      <td>0.363698</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 54 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-69fadd77-3371-4464-bda0-0c9d3a771119')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-69fadd77-3371-4464-bda0-0c9d3a771119 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-69fadd77-3371-4464-bda0-0c9d3a771119');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "import pandas as pd\n",
        "data=pd.read_csv('agnews_train(entropy).csv')\n",
        "data.head()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data2=pd.read_csv('agnews_train2.csv')\n",
        "data2['entropy']=data['entropy']"
      ],
      "metadata": {
        "id": "kBy1Yf12ERWm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data2.info()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GFgYXPTYFJMl",
        "outputId": "b71c3027-ca20-4ce2-d15e-529f1e6977ef"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 1154 entries, 0 to 1153\n",
            "Columns: 134 entries, Class Index to 4label9\n",
            "dtypes: float64(9), int64(33), object(92)\n",
            "memory usage: 1.2+ MB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#length calculation for cost computation\n",
        "df=data2\n",
        "df['length1']=df['prompt1'].apply(lambda x:len(x.split()))\n",
        "df['length2']=df['prompt2'].apply(lambda x:len(x.split()))\n",
        "df['length3']=df['prompt3'].apply(lambda x:len(x.split()))\n",
        "df['length4']=df['prompt4'].apply(lambda x:len(x.split()))\n",
        "df['length5']=df['prompt5'].apply(lambda x:len(x.split()))\n",
        "df['length6']=df['prompt6'].apply(lambda x:len(x.split()))\n",
        "df['length7']=df['prompt7'].apply(lambda x:len(x.split()))\n",
        "df['length8']=df['prompt8'].apply(lambda x:len(x.split()))\n",
        "df['length9']=df['prompt9'].apply(lambda x:len(x.split()))\n",
        "df['length10']=df['prompt10'].apply(lambda x:len(x.split()))\n",
        "df['length11']=df['prompt10'].apply(lambda x:len(x.split()))\n",
        "df['length12']=df['prompt10'].apply(lambda x:len(x.split()))\n",
        "df['length13']=df['prompt10'].apply(lambda x:len(x.split()))\n",
        "\n",
        "df['length']=df['length1']+df['length2']+df['length3']+df['length4']+df['length5']+df['length6']+df['length7']+df['length8']+df['length9']+df['length10']+df['length11']+df['length12']+df['length13']\n",
        "\n",
        "df=df.sort_values(by='entropy',ignore_index=True)\n",
        "# Assuming your dataframe is called 'df'\n",
        "total_rows = df.shape[0]\n",
        "num_rows_20_percent = int(total_rows * 0.8) #can change this to 0.2, 0.4, 0.6, 0.8\n",
        "print(1154-num_rows_20_percent)\n",
        "df['allocate'] = 0  # Initialize the new column with 0\n",
        "df.loc[0:num_rows_20_percent-1, 'allocate'] = 1  # Assign 1 to the first 20% of rows\n",
        "df2=df[df['allocate']==0]\n",
        "df1=df[df['allocate']==1]\n",
        "sum(df1['length'].tolist())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_QwSSSa7EWrA",
        "outputId": "4e3f5b0c-c847-4632-eefd-0a724fd4d3cc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "231\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "799747"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#length calculation for cost computation\n",
        "df=data2\n",
        "df['length1']=df['prompt1'].apply(lambda x:len(x.split()))\n",
        "df['length2']=df['prompt2'].apply(lambda x:len(x.split()))\n",
        "df['length3']=df['prompt3'].apply(lambda x:len(x.split()))\n",
        "df['length4']=df['prompt4'].apply(lambda x:len(x.split()))\n",
        "df['length5']=df['prompt5'].apply(lambda x:len(x.split()))\n",
        "df['length6']=df['prompt6'].apply(lambda x:len(x.split()))\n",
        "df['length7']=df['prompt7'].apply(lambda x:len(x.split()))\n",
        "df['length8']=df['prompt8'].apply(lambda x:len(x.split()))\n",
        "df['length9']=df['prompt9'].apply(lambda x:len(x.split()))\n",
        "df['length10']=df['prompt10'].apply(lambda x:len(x.split()))\n",
        "df['length11']=df['prompt10'].apply(lambda x:len(x.split()))\n",
        "df['length12']=df['prompt10'].apply(lambda x:len(x.split()))\n",
        "df['length13']=df['prompt10'].apply(lambda x:len(x.split()))\n",
        "\n",
        "df['length']=df['length1']+df['length2']+df['length3']+df['length4']+df['length5']+df['length6']+df['length7']+df['length8']+df['length9']+df['length10']+df['length11']+df['length12']+df['length13']\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "# Assuming you have a DataFrame named 'df'\n",
        "\n",
        "# Set the seed for reproducibility\n",
        "np.random.seed(42)\n",
        "\n",
        "# Calculate the number of rows for each allocation value\n",
        "total_rows = len(df)\n",
        "allocation_1_count = int(1 * total_rows)\n",
        "allocation_0_count = total_rows - allocation_1_count\n",
        "\n",
        "# Create an array of 80% ones and 20% zeros\n",
        "allocation_values = np.concatenate((np.ones(allocation_1_count), np.zeros(allocation_0_count)))\n",
        "\n",
        "# Shuffle the allocation values randomly\n",
        "np.random.shuffle(allocation_values)\n",
        "\n",
        "# Assign the allocation values to the 'allocation' column in the DataFrame\n",
        "df['allocation'] = allocation_values.astype(int)\n",
        "#df2=df[df['allocate']==0]\n",
        "df1=df[df['allocation']==1]\n",
        "print(1154-df1.shape[0])\n",
        "sum(df1['length'].tolist())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7QGZGD-ZFtRU",
        "outputId": "c89acc12-84bc-408f-9548-0b3d75dbe613"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "989297"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "# Assuming you have a DataFrame named 'df'\n",
        "\n",
        "# Set the seed for reproducibility\n",
        "np.random.seed(42)\n",
        "\n",
        "# Calculate the number of rows for each allocation value\n",
        "df=data\n",
        "total_rows = len(df)\n",
        "allocation_1_count = int(0.8 * total_rows)\n",
        "allocation_0_count = total_rows - allocation_1_count\n",
        "\n",
        "# Create an array of 80% ones and 20% zeros\n",
        "allocation_values = np.concatenate((np.ones(allocation_1_count), np.zeros(allocation_0_count)))\n",
        "\n",
        "# Shuffle the allocation values randomly\n",
        "np.random.shuffle(allocation_values)\n",
        "\n",
        "# Assign the allocation values to the 'allocation' column in the DataFrame\n",
        "df['allocation'] = allocation_values.astype(int)"
      ],
      "metadata": {
        "id": "DzV5_MCjGXhw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data['gold_label']=data['Class Index']\n",
        "def calculate_alignment(threshold, df_input):\n",
        "  df_low=df_input[df_input['entropy']<=threshold]\n",
        "  print(df_low.shape[0])\n",
        "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
        "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
        "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
        "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
        "  df_low_correct1=df_low[df_low['MV_world']==df_low['world_gold']]\n",
        "  df_low_correct2=df_low[df_low['MV_sports']==df_low['gold_sports']]\n",
        "  df_low_correct3=df_low[df_low['MV_business']==df_low['gold_business']]\n",
        "  df_low_correct4=df_low[df_low['MV_sci']==df_low['gold_sci']]\n",
        "  return (df_low_correct1.shape[0]+df_low_correct2.shape[0]+df_low_correct3.shape[0]+df_low_correct4.shape[0])/4/df_low.shape[0]"
      ],
      "metadata": {
        "id": "0gzmqBWZGWIk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "calculate_alignment(1, data)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H1XFXF6eHNtJ",
        "outputId": "a8439a24-0e81-4cbd-a448-729006c41a04"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1154\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9113951473136915"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x = [0, 0.2, 0.4, 0.6, 0.8, 1]\n",
        "y = [calculate_alignment(i, data) for i in x]\n",
        "y"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jnj9O3_2HAZ5",
        "outputId": "d27ee389-757a-41c8-9f67-5e6cc385216f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "141\n",
            "266\n",
            "832\n",
            "1064\n",
            "1146\n",
            "1154\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-9-cc88c7172cb6>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-9-cc88c7172cb6>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-9-cc88c7172cb6>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-9-cc88c7172cb6>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-9-cc88c7172cb6>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-9-cc88c7172cb6>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-9-cc88c7172cb6>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-9-cc88c7172cb6>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-9-cc88c7172cb6>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-9-cc88c7172cb6>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-9-cc88c7172cb6>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-9-cc88c7172cb6>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-9-cc88c7172cb6>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-9-cc88c7172cb6>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-9-cc88c7172cb6>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-9-cc88c7172cb6>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-9-cc88c7172cb6>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-9-cc88c7172cb6>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-9-cc88c7172cb6>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-9-cc88c7172cb6>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.9858156028368794,\n",
              " 0.9793233082706767,\n",
              " 0.9456129807692307,\n",
              " 0.9224624060150376,\n",
              " 0.912521815008726,\n",
              " 0.9113951473136915]"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.hist(data['4label1'], bins=15, align='left')\n",
        "\n",
        "# Add axis labels and a title\n",
        "plt.xlabel('Value')\n",
        "plt.ylabel('Frequency')\n",
        "plt.title('Histogram of Values')\n",
        "# Display the plot\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 472
        },
        "id": "AXG26rU75seo",
        "outputId": "344118e7-f0c4-4088-86d9-8a03cbb5573c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj8AAAHHCAYAAABQhTneAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8EklEQVR4nO3deVhV5f7+8XszOrFBVECOOOSQkpqFJ91pg0aikkeTBssMDbMMzSG1OJmmdsSs1OpoNoKdMst+jZoDWWlHydK0TM3Z0Bi0FFA8zOv3hxf72xYs2AIbWO/Xda2r1rOetdbneSS5W8PeFsMwDAEAAJiEm6sLAAAAqE6EHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEH6AOad26tUaOHOnqMuq8Z555Rpdddpnc3d3VrVu3ajtvYmKiLBaLjh49Wm3nBOoiwg9QQ5X8otu2bVuZ22+88UZ17tz5ks/z2Wef6cknn7zk45jF+vXrNW3aNPXq1UsJCQmaO3duqT4FBQVq2rSpevfufdHjGIahkJAQXX311VVZLoAyeLi6AACVZ9++fXJzq9j/03z22WdavHgxAaicvvjiC7m5uen111+Xl5dXmX08PT11++236+WXX9Yvv/yiVq1aleqzadMmHT9+XJMmTarqkgFcgCs/QB3i7e0tT09PV5dRITk5Oa4uoUJOnDih+vXrXzT4lBg+fLgMw9A777xT5vbly5fLzc1Nw4YNq4oyAfwJwg9Qh1z4zE9BQYFmzZql9u3bq169emrSpIl69+6tpKQkSdLIkSO1ePFiSZLFYrEvJXJycvTII48oJCRE3t7euvzyy/Xss8/KMAyH8/7vf//Tww8/rKZNm8rHx0f/+Mc/9Ouvv8pisThcUXryySdlsVi0Z88e3X333WrcuLH91tCPP/6okSNH6rLLLlO9evUUFBSk++67T7///rvDuUqOsX//ft1zzz3y9fVVs2bN9MQTT8gwDB07dkyDBw+W1WpVUFCQnnvuuXLNXWFhoebMmaO2bdvK29tbrVu31j//+U/l5eXZ+1gsFiUkJCgnJ8c+V4mJiWUer1evXmrdurWWL19ealtBQYHef/999enTR8HBweUee1kunOMSZT3/lZmZqYkTJ9r/PNu1a6enn35axcXFDv1WrFihsLAw+fj4yGq1qkuXLnr++ef/shagtuC2F1DDZWVl6bfffivVXlBQ8Jf7Pvnkk4qPj9fo0aN1zTXXKDs7W9u2bdP333+vm2++WQ888IBSU1OVlJSk//znPw77Goahf/zjH/ryyy8VExOjbt26ad26dZo6dap+/fVXLVy40N535MiReu+99zRixAj17NlTGzduVGRk5EXruv3229W+fXvNnTvXHqSSkpJ0+PBhjRo1SkFBQdq9e7deeeUV7d69W998841DKJOkO++8U506ddK8efO0evVqPfXUU/L399fLL7+svn376umnn9bbb7+tKVOm6O9//7uuv/76P52r0aNHa9myZbrtttv0yCOPaOvWrYqPj9fevXv14YcfSpL+85//6JVXXtG3336r1157TZJ07bXXlnk8i8Wiu+++W3PnztXu3bt1xRVX2LetXbtWp06d0vDhw50auzPOnTunG264Qb/++qseeOABtWzZUlu2bFFcXJzS0tK0aNEiey133XWXbrrpJj399NOSpL1792rz5s2aMGHCJdcB1AgGgBopISHBkPSnyxVXXOGwT6tWrYzo6Gj7+pVXXmlERkb+6XliY2ONsv4q+OijjwxJxlNPPeXQfttttxkWi8U4ePCgYRiGsX37dkOSMXHiRId+I0eONCQZM2fOtLfNnDnTkGTcddddpc537ty5Um3vvPOOIcnYtGlTqWOMGTPG3lZYWGi0aNHCsFgsxrx58+ztp0+fNurXr+8wJ2XZuXOnIckYPXq0Q/uUKVMMScYXX3xhb4uOjjYaNmz4p8crsXv3bkOSERcX59A+bNgwo169ekZWVpZhGOUfe8nPxJEjR+xtF85xiQt/FubMmWM0bNjQ2L9/v0O/xx57zHB3dzdSUlIMwzCMCRMmGFar1SgsLCzXGIHaiNteQA23ePFiJSUllVq6du36l/v6+flp9+7dOnDgQIXP+9lnn8nd3V0PP/ywQ/sjjzwiwzC0Zs0aSeevYkjSQw895NBv/PjxFz32gw8+WKqtfv369n/Pzc3Vb7/9pp49e0qSvv/++1L9R48ebf93d3d3de/eXYZhKCYmxt7u5+enyy+/XIcPH75oLdL5sUrS5MmTHdofeeQRSdLq1av/dP+LCQ0N1VVXXaUVK1bY23JycvTJJ5/olltukdVqlVTxsTtj5cqVuu6669S4cWP99ttv9iU8PFxFRUXatGmTpPNzlpOTY781CtRFhB+ghrvmmmsUHh5eamncuPFf7jt79mxlZmaqQ4cO6tKli6ZOnaoff/yxXOf95ZdfFBwcLB8fH4f2Tp062beX/NPNzU1t2rRx6NeuXbuLHvvCvpJ06tQpTZgwQYGBgapfv76aNWtm75eVlVWqf8uWLR3WfX19Va9ePTVt2rRU++nTpy9ayx/HcGHNQUFB8vPzs4/VGcOHD9eRI0e0ZcsWSdJHH32kc+fO2W95SRUfuzMOHDigtWvXqlmzZg5LeHi4pPMPckvnQ2yHDh00YMAAtWjRQvfdd5894AJ1Bc/8AHXY9ddfr0OHDunjjz/W+vXr9dprr2nhwoVaunSpw5WT6vbHKx0l7rjjDm3ZskVTp05Vt27d1KhRIxUXF6t///6lHsiVzl/tKU+bpFIPaF9MZTxbc6G77rpL06ZN0/Lly3Xttddq+fLlaty4sQYOHGjvU9Gxl0dRUZHDenFxsW6++WZNmzatzP4dOnSQJAUEBGjnzp1at26d1qxZozVr1ighIUH33nuvli1b5lQtQE1D+AHqOH9/f40aNUqjRo3S2bNndf311+vJJ5+0h5+L/cJv1aqVPv/8c505c8bh6s/PP/9s317yz+LiYh05ckTt27e39zt48GC5azx9+rQ2bNigWbNmacaMGfZ2Z27XOaNkDAcOHLBf2ZKkjIwMZWZmlvk5PeUVHBysPn36aOXKlXriiSeUlJSkkSNH2l+Vv9SxN27cWJmZmQ5t+fn5SktLc2hr27atzp49a7/S82e8vLw0aNAgDRo0SMXFxXrooYf08ssv64knnvjTK3pAbcFtL6AOu/BV6UaNGqldu3YOr283bNhQkkr9Ah04cKCKior073//26F94cKFslgsGjBggCQpIiJCkrRkyRKHfi+++GK56yy5YnPhFZqSN5CqWslVmAvPt2DBAkn60zfXymP48OE6ceKEHnjgARUUFDjc8rrUsbdt29b+vE6JV155pdSVnzvuuEPJyclat25dqWNkZmaqsLBQUumfGTc3N/vzZX/8uQFqM678AHVYaGiobrzxRoWFhcnf31/btm3T+++/r3Hjxtn7hIWFSZIefvhhRUREyN3dXcOGDdOgQYPUp08fPf744zp69KiuvPJKrV+/Xh9//LEmTpyotm3b2vePiorSokWL9Pvvv9tfdd+/f7+k8t1Kslqtuv766zV//nwVFBTob3/7m9avX68jR45UwayUduWVVyo6OlqvvPKKMjMzdcMNN+jbb7/VsmXLNGTIEPXp0+eSjh8VFaWHHnpIH3/8sUJCQhxeu7/UsY8ePVoPPvigoqKidPPNN+uHH37QunXrSj37NHXqVPuD1iNHjlRYWJhycnK0a9cuvf/++zp69KiaNm2q0aNH69SpU+rbt69atGihX375RS+++KK6devmcFUMqNVc+q4ZgIsqea35u+++K3P7DTfc8Jevuj/11FPGNddcY/j5+Rn169c3OnbsaPzrX/8y8vPz7X0KCwuN8ePHG82aNTMsFovDa+9nzpwxJk2aZAQHBxuenp5G+/btjWeeecYoLi52OG9OTo4RGxtr+Pv7G40aNTKGDBli7Nu3z5Dk8Op5yWvqJ0+eLDWe48ePG7feeqvh5+dn+Pr6GrfffruRmpp60dflLzzGxV5BL2ueylJQUGDMmjXLaNOmjeHp6WmEhIQYcXFxRm5ubrnO81duv/12Q5Ixbdq0UtvKO/ayXnUvKioyHn30UaNp06ZGgwYNjIiICOPgwYOlfhYM4/yfZ1xcnNGuXTvDy8vLaNq0qXHttdcazz77rP1n4v333zf69etnBAQEGF5eXkbLli2NBx54wEhLS6vwmIGaymIY5XwSEAAqYOfOnbrqqqv01ltvOdzmAQBX45kfAJfsf//7X6m2RYsWyc3N7S8/WRkAqhvP/AC4ZPPnz9f27dvVp08feXh42F+RHjNmjEJCQlxdHgA44LYXgEuWlJSkWbNmac+ePTp79qxatmypESNG6PHHH5eHB/+PBaBmIfwAAABT4ZkfAABgKoQfAABgKtyM1/nvvElNTZWPj0+VfLcPAACofIZh6MyZMwoODpabW/mv5xB+JKWmpvJGCgAAtdSxY8fUokWLcvcn/Ej2L208duyYrFari6sBAADlkZ2drZCQEIcvXy4Pwo/+77uHrFYr4QcAgFqmoo+s8MAzAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFZeGn9atW8tisZRaYmNjJUm5ubmKjY1VkyZN1KhRI0VFRSkjI8PhGCkpKYqMjFSDBg0UEBCgqVOnqrCw0BXDAQAAtYBLw893332ntLQ0+5KUlCRJuv322yVJkyZN0qeffqqVK1dq48aNSk1N1dChQ+37FxUVKTIyUvn5+dqyZYuWLVumxMREzZgxwyXjAQAANV+N+lb3iRMnatWqVTpw4ICys7PVrFkzLV++XLfddpsk6eeff1anTp2UnJysnj17as2aNbrllluUmpqqwMBASdLSpUv16KOP6uTJk/Ly8irXebOzs+Xr66usrCw+5wcAgFrC2d/fNeaZn/z8fL311lu67777ZLFYtH37dhUUFCg8PNzep2PHjmrZsqWSk5MlScnJyerSpYs9+EhSRESEsrOztXv37mofAwAAqPlqzCc8f/TRR8rMzNTIkSMlSenp6fLy8pKfn59Dv8DAQKWnp9v7/DH4lGwv2XYxeXl5ysvLs69nZ2dXwggAAEBtUGOu/Lz++usaMGCAgoODq/xc8fHx8vX1tS98qSkAAOZRI8LPL7/8os8//1yjR4+2twUFBSk/P1+ZmZkOfTMyMhQUFGTvc+HbXyXrJX3KEhcXp6ysLPty7NixShoJAACo6WpE+ElISFBAQIAiIyPtbWFhYfL09NSGDRvsbfv27VNKSopsNpskyWazadeuXTpx4oS9T1JSkqxWq0JDQy96Pm9vb/uXmPJlpgAAmIvLn/kpLi5WQkKCoqOj5eHxf+X4+voqJiZGkydPlr+/v6xWq8aPHy+bzaaePXtKkvr166fQ0FCNGDFC8+fPV3p6uqZPn67Y2Fh5e3u7akgAAKAGc3n4+fzzz5WSkqL77ruv1LaFCxfKzc1NUVFRysvLU0REhJYsWWLf7u7urlWrVmns2LGy2Wxq2LChoqOjNXv27OocAgAAqEVq1Of8uAqf8wMAqE6tH1vtkvMenRf5151qkVr/OT8AAADVgfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMxeXh59dff9U999yjJk2aqH79+urSpYu2bdtm324YhmbMmKHmzZurfv36Cg8P14EDBxyOcerUKQ0fPlxWq1V+fn6KiYnR2bNnq3soAACgFnBp+Dl9+rR69eolT09PrVmzRnv27NFzzz2nxo0b2/vMnz9fL7zwgpYuXaqtW7eqYcOGioiIUG5urr3P8OHDtXv3biUlJWnVqlXatGmTxowZ44ohAQCAGs5iGIbhqpM/9thj2rx5s77++usytxuGoeDgYD3yyCOaMmWKJCkrK0uBgYFKTEzUsGHDtHfvXoWGhuq7775T9+7dJUlr167VwIEDdfz4cQUHB/9lHdnZ2fL19VVWVpasVmvlDRAAgDK0fmy1S857dF6kS85bVZz9/e3SKz+ffPKJunfvrttvv10BAQG66qqr9Oqrr9q3HzlyROnp6QoPD7e3+fr6qkePHkpOTpYkJScny8/Pzx58JCk8PFxubm7aunVr9Q0GAADUCi4NP4cPH9ZLL72k9u3ba926dRo7dqwefvhhLVu2TJKUnp4uSQoMDHTYLzAw0L4tPT1dAQEBDts9PDzk7+9v73OhvLw8ZWdnOywAAMAcPFx58uLiYnXv3l1z586VJF111VX66aeftHTpUkVHR1fZeePj4zVr1qwqOz4AAKi5XHrlp3nz5goNDXVo69Spk1JSUiRJQUFBkqSMjAyHPhkZGfZtQUFBOnHihMP2wsJCnTp1yt7nQnFxccrKyrIvx44dq5TxAACAms+l4adXr17at2+fQ9v+/fvVqlUrSVKbNm0UFBSkDRs22LdnZ2dr69atstlskiSbzabMzExt377d3ueLL75QcXGxevToUeZ5vb29ZbVaHRYAAGAOLr3tNWnSJF177bWaO3eu7rjjDn377bd65ZVX9Morr0iSLBaLJk6cqKeeekrt27dXmzZt9MQTTyg4OFhDhgyRdP5KUf/+/XX//fdr6dKlKigo0Lhx4zRs2LByvekFAADMxaXh5+9//7s+/PBDxcXFafbs2WrTpo0WLVqk4cOH2/tMmzZNOTk5GjNmjDIzM9W7d2+tXbtW9erVs/d5++23NW7cON10001yc3NTVFSUXnjhBVcMCQAA1HAu/ZyfmoLP+QEAVCc+56dy1MrP+QEAAKhuhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqhB8AAGAqLg0/Tz75pCwWi8PSsWNH+/bc3FzFxsaqSZMmatSokaKiopSRkeFwjJSUFEVGRqpBgwYKCAjQ1KlTVVhYWN1DAQAAtYSHqwu44oor9Pnnn9vXPTz+r6RJkyZp9erVWrlypXx9fTVu3DgNHTpUmzdvliQVFRUpMjJSQUFB2rJli9LS0nTvvffK09NTc+fOrfaxAACAms/l4cfDw0NBQUGl2rOysvT6669r+fLl6tu3ryQpISFBnTp10jfffKOePXtq/fr12rNnjz7//HMFBgaqW7dumjNnjh599FE9+eST8vLyqu7hAACAGs7lz/wcOHBAwcHBuuyyyzR8+HClpKRIkrZv366CggKFh4fb+3bs2FEtW7ZUcnKyJCk5OVldunRRYGCgvU9ERISys7O1e/fu6h0IAACoFVx65adHjx5KTEzU5ZdfrrS0NM2aNUvXXXedfvrpJ6Wnp8vLy0t+fn4O+wQGBio9PV2SlJ6e7hB8SraXbLuYvLw85eXl2dezs7MraUQAAKCmc2n4GTBggP3fu3btqh49eqhVq1Z67733VL9+/So7b3x8vGbNmlVlxwcAADWXy297/ZGfn586dOiggwcPKigoSPn5+crMzHTok5GRYX9GKCgoqNTbXyXrZT1HVCIuLk5ZWVn25dixY5U7EAAAUGPVqPBz9uxZHTp0SM2bN1dYWJg8PT21YcMG+/Z9+/YpJSVFNptNkmSz2bRr1y6dOHHC3icpKUlWq1WhoaEXPY+3t7esVqvDAgAAzMGlt72mTJmiQYMGqVWrVkpNTdXMmTPl7u6uu+66S76+voqJidHkyZPl7+8vq9Wq8ePHy2azqWfPnpKkfv36KTQ0VCNGjND8+fOVnp6u6dOnKzY2Vt7e3q4cGgAAqKFcGn6OHz+uu+66S7///ruaNWum3r1765tvvlGzZs0kSQsXLpSbm5uioqKUl5eniIgILVmyxL6/u7u7Vq1apbFjx8pms6lhw4aKjo7W7NmzXTUkAABQw1kMwzBcXYSrZWdny9fXV1lZWdwCAwBUudaPrXbJeY/Oi3TJeauKs7+/a9QzPwAAAFWN8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEzFqfBz+PDhyq4DAACgWjgVftq1a6c+ffrorbfeUm5ubmXXBAAAUGWcCj/ff/+9unbtqsmTJysoKEgPPPCAvv3228quDQAAoNI5FX66deum559/XqmpqXrjjTeUlpam3r17q3PnzlqwYIFOnjxZ2XUCAABUikt64NnDw0NDhw7VypUr9fTTT+vgwYOaMmWKQkJCdO+99yotLa2y6gQAAKgUlxR+tm3bpoceekjNmzfXggULNGXKFB06dEhJSUlKTU3V4MGDK6tOAACASuHhzE4LFixQQkKC9u3bp4EDB+rNN9/UwIED5eZ2Pku1adNGiYmJat26dWXWCgAAcMmcCj8vvfSS7rvvPo0cOVLNmzcvs09AQIBef/31SyoOAACgsjkVfg4cOPCXfby8vBQdHe3M4QEAAKqMU8/8JCQkaOXKlaXaV65cqWXLljlVyLx582SxWDRx4kR7W25urmJjY9WkSRM1atRIUVFRysjIcNgvJSVFkZGRatCggQICAjR16lQVFhY6VQMAAKj7nAo/8fHxatq0aan2gIAAzZ07t8LH++677/Tyyy+ra9euDu2TJk3Sp59+qpUrV2rjxo1KTU3V0KFD7duLiooUGRmp/Px8bdmyRcuWLVNiYqJmzJhR8UEBAABTcCr8pKSkqE2bNqXaW7VqpZSUlAod6+zZsxo+fLheffVVNW7c2N6elZWl119/XQsWLFDfvn0VFhamhIQEbdmyRd98840kaf369dqzZ4/eeustdevWTQMGDNCcOXO0ePFi5efnOzM0AABQxzkVfgICAvTjjz+Wav/hhx/UpEmTCh0rNjZWkZGRCg8Pd2jfvn27CgoKHNo7duyoli1bKjk5WZKUnJysLl26KDAw0N4nIiJC2dnZ2r17d4XqAAAA5uDUA8933XWXHn74Yfn4+Oj666+XJG3cuFETJkzQsGHDyn2cFStW6Pvvv9d3331Xalt6erq8vLzk5+fn0B4YGKj09HR7nz8Gn5LtJdsuJi8vT3l5efb17OzsctcMAABqN6fCz5w5c3T06FHddNNN8vA4f4ji4mLde++95X7m59ixY5owYYKSkpJUr149Z8pwWnx8vGbNmlWt5wQAADWDU7e9vLy89O677+rnn3/W22+/rQ8++ECHDh3SG2+8IS8vr3IdY/v27Tpx4oSuvvpqeXh4yMPDQxs3btQLL7wgDw8PBQYGKj8/X5mZmQ77ZWRkKCgoSJIUFBRU6u2vkvWSPmWJi4tTVlaWfTl27FgFRg8AAGozp678lOjQoYM6dOjg1L433XSTdu3a5dA2atQodezYUY8++qhCQkLk6empDRs2KCoqSpK0b98+paSkyGazSZJsNpv+9a9/6cSJEwoICJAkJSUlyWq1KjQ09KLn9vb2lre3t1N1AwCA2s2p8FNUVKTExERt2LBBJ06cUHFxscP2L7744i+P4ePjo86dOzu0NWzYUE2aNLG3x8TEaPLkyfL395fVatX48eNls9nUs2dPSVK/fv0UGhqqESNGaP78+UpPT9f06dMVGxtLuAEAAGVyKvxMmDBBiYmJioyMVOfOnWWxWCq7LknSwoUL5ebmpqioKOXl5SkiIkJLliyxb3d3d9eqVas0duxY2Ww2NWzYUNHR0Zo9e3aV1AMAAGo/i2EYRkV3atq0qf3LTOuC7Oxs+fr6KisrS1ar1dXlAADquNaPrXbJeY/Oi3TJeauKs7+/nX7guV27ds7sCgAA4FJOhZ9HHnlEzz//vJy4aAQAAOBSTj3z89///ldffvml1qxZoyuuuEKenp4O2z/44INKKQ4AAKCyORV+/Pz8dOutt1Z2LQAAAFXOqfCTkJBQ2XUAAABUC6ee+ZGkwsJCff7553r55Zd15swZSVJqaqrOnj1bacUBAABUNqeu/Pzyyy/q37+/UlJSlJeXp5tvvlk+Pj56+umnlZeXp6VLl1Z2nQAAAJXCqSs/EyZMUPfu3XX69GnVr1/f3n7rrbdqw4YNlVYcAABAZXPqys/XX3+tLVu2lPoS09atW+vXX3+tlMIAAACqglNXfoqLi1VUVFSq/fjx4/Lx8bnkogAAAKqKU+GnX79+WrRokX3dYrHo7NmzmjlzZp35ygsAAFA3OXXb67nnnlNERIRCQ0OVm5uru+++WwcOHFDTpk31zjvvVHaNAAAAlcap8NOiRQv98MMPWrFihX788UedPXtWMTExGj58uMMD0AAAADWNU+FHkjw8PHTPPfdUZi0AAABVzqnw8+abb/7p9nvvvdepYgAAAKqaU+FnwoQJDusFBQU6d+6cvLy81KBBA8IPAACosZx62+v06dMOy9mzZ7Vv3z717t2bB54BAECN5vR3e12offv2mjdvXqmrQgAAADVJpYUf6fxD0KmpqZV5SAAAgErl1DM/n3zyicO6YRhKS0vTv//9b/Xq1atSCgMAAKgKToWfIUOGOKxbLBY1a9ZMffv21XPPPVcZdQEAAFQJp8JPcXFxZdcBAABQLSr1mR8AAICazqkrP5MnTy533wULFjhzCgAAgCrhVPjZsWOHduzYoYKCAl1++eWSpP3798vd3V1XX321vZ/FYqmcKgEAACqJU+Fn0KBB8vHx0bJly9S4cWNJ5z/4cNSoUbruuuv0yCOPVGqRAAAAlcWpZ36ee+45xcfH24OPJDVu3FhPPfUUb3sBAIAazanwk52drZMnT5ZqP3nypM6cOXPJRQEAAFQVp8LPrbfeqlGjRumDDz7Q8ePHdfz4cf2///f/FBMTo6FDh1Z2jQAAAJXGqWd+li5dqilTpujuu+9WQUHB+QN5eCgmJkbPPPNMpRYIAABQmZwKPw0aNNCSJUv0zDPP6NChQ5Kktm3bqmHDhpVaHAAAQGW7pA85TEtLU1pamtq3b6+GDRvKMIzKqgsAAKBKOBV+fv/9d910003q0KGDBg4cqLS0NElSTEwMr7kDAIAazanwM2nSJHl6eiolJUUNGjSwt995551au3ZtpRUHAABQ2Zx65mf9+vVat26dWrRo4dDevn17/fLLL5VSGAAAQFVw6spPTk6OwxWfEqdOnZK3t/clFwUAAFBVnAo/1113nd588037usViUXFxsebPn68+ffpUWnEAAACVzanbXvPnz9dNN92kbdu2KT8/X9OmTdPu3bt16tQpbd68ubJrBAAAqDROXfnp3Lmz9u/fr969e2vw4MHKycnR0KFDtWPHDrVt27ayawQAAKg0Fb7yU1BQoP79+2vp0qV6/PHHq6ImAACAKlPhKz+enp768ccfK+XkL730krp27Sqr1Sqr1SqbzaY1a9bYt+fm5io2NlZNmjRRo0aNFBUVpYyMDIdjpKSkKDIyUg0aNFBAQICmTp2qwsLCSqkPAADUPU7d9rrnnnv0+uuvX/LJW7RooXnz5mn79u3atm2b+vbtq8GDB2v37t2Szn+e0KeffqqVK1dq48aNSk1Ndfji1KKiIkVGRio/P19btmzRsmXLlJiYqBkzZlxybQAAoG6yGE58J8X48eP15ptvqn379goLCyv1nV4LFixwuiB/f38988wzuu2229SsWTMtX75ct912myTp559/VqdOnZScnKyePXtqzZo1uuWWW5SamqrAwEBJ57909dFHH9XJkyfl5eVVrnNmZ2fL19dXWVlZslqtTtcOAEB5tH5stUvOe3RepEvOW1Wc/f1doSs/hw8fVnFxsX766SddffXV8vHx0f79+7Vjxw77snPnzorWLun8VZwVK1YoJydHNptN27dvV0FBgcLDw+19OnbsqJYtWyo5OVmSlJycrC5dutiDjyRFREQoOzvbfvUIAADgjyr0wHP79u2VlpamL7/8UtL5r7N44YUXHMJHRe3atUs2m025ublq1KiRPvzwQ4WGhmrnzp3y8vKSn5+fQ//AwEClp6dLktLT00udu2S9pE9Z8vLylJeXZ1/Pzs52un4AAFC7VOjKz4V3yNasWaOcnJxLKuDyyy/Xzp07tXXrVo0dO1bR0dHas2fPJR3zr8THx8vX19e+hISEVOn5AABAzeHUA88lnHhcqBQvLy+1a9dOYWFhio+P15VXXqnnn39eQUFBys/PV2ZmpkP/jIwMBQUFSZKCgoJKvf1Vsl7SpyxxcXHKysqyL8eOHbvkcQAAgNqhQuHHYrHIYrGUaqtMxcXFysvLU1hYmDw9PbVhwwb7tn379iklJUU2m02SZLPZtGvXLp04ccLeJykpSVarVaGhoRc9h7e3t/31+pIFAACYQ4We+TEMQyNHjrR/eWlubq4efPDBUm97ffDBB+U6XlxcnAYMGKCWLVvqzJkzWr58ub766iutW7dOvr6+iomJ0eTJk+Xv7y+r1arx48fLZrOpZ8+ekqR+/fopNDRUI0aM0Pz585Wenq7p06crNjaWL1gFAABlqlD4iY6Odli/5557LunkJ06c0L333qu0tDT5+vqqa9euWrdunW6++WZJ0sKFC+Xm5qaoqCjl5eUpIiJCS5Ysse/v7u6uVatWaezYsbLZbGrYsKGio6M1e/bsS6oLAADUXU59zk9dw+f8AACqE5/zUzmq5XN+AAAAajvCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMJUKfbcXAACovfhajfO48gMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEzFpeEnPj5ef//73+Xj46OAgAANGTJE+/btc+iTm5ur2NhYNWnSRI0aNVJUVJQyMjIc+qSkpCgyMlINGjRQQECApk6dqsLCwuocCgAAqCVcGn42btyo2NhYffPNN0pKSlJBQYH69eunnJwce59Jkybp008/1cqVK7Vx40alpqZq6NCh9u1FRUWKjIxUfn6+tmzZomXLlikxMVEzZsxwxZAAAEANZzEMw3B1ESVOnjypgIAAbdy4Uddff72ysrLUrFkzLV++XLfddpsk6eeff1anTp2UnJysnj17as2aNbrllluUmpqqwMBASdLSpUv16KOP6uTJk/Ly8vrL82ZnZ8vX11dZWVmyWq1VOkYAAFo/ttrVJVSro/Miq+S4zv7+rlHP/GRlZUmS/P39JUnbt29XQUGBwsPD7X06duyoli1bKjk5WZKUnJysLl262IOPJEVERCg7O1u7d++uxuoBAEBt4OHqAkoUFxdr4sSJ6tWrlzp37ixJSk9Pl5eXl/z8/Bz6BgYGKj093d7nj8GnZHvJtrLk5eUpLy/Pvp6dnV1ZwwAAADVcjbnyExsbq59++kkrVqyo8nPFx8fL19fXvoSEhFT5OQEAQM1QI8LPuHHjtGrVKn355Zdq0aKFvT0oKEj5+fnKzMx06J+RkaGgoCB7nwvf/ipZL+lzobi4OGVlZdmXY8eOVeJoAABATebS8GMYhsaNG6cPP/xQX3zxhdq0aeOwPSwsTJ6entqwYYO9bd++fUpJSZHNZpMk2Ww27dq1SydOnLD3SUpKktVqVWhoaJnn9fb2ltVqdVgAAIA5uPSZn9jYWC1fvlwff/yxfHx87M/o+Pr6qn79+vL19VVMTIwmT54sf39/Wa1WjR8/XjabTT179pQk9evXT6GhoRoxYoTmz5+v9PR0TZ8+XbGxsfL29nbl8AAAQA3k0vDz0ksvSZJuvPFGh/aEhASNHDlSkrRw4UK5ubkpKipKeXl5ioiI0JIlS+x93d3dtWrVKo0dO1Y2m00NGzZUdHS0Zs+eXV3DAAAAtUiN+pwfV+FzfgAA1YnP+akcdeJzfgAAAKoa4QcAAJgK4QcAAJgK4QcAAJgK4QcAAJgK4QcAAJgK4QcAAJgK4QcAAJgK4QcAAJgK4QcAAJgK4QcAAJgK4QcAAJgK4QcAAJgK4QcAAJgK4QcAAJgK4QcAAJgK4QcAAJgK4QcAAJgK4QcAAJgK4QcAAJiKh6sLAADAVVo/ttrVJcAFuPIDAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMhfADAABMxaXhZ9OmTRo0aJCCg4NlsVj00UcfOWw3DEMzZsxQ8+bNVb9+fYWHh+vAgQMOfU6dOqXhw4fLarXKz89PMTExOnv2bDWOAgAA1CYuDT85OTm68sortXjx4jK3z58/Xy+88IKWLl2qrVu3qmHDhoqIiFBubq69z/Dhw7V7924lJSVp1apV2rRpk8aMGVNdQwAAALWMhytPPmDAAA0YMKDMbYZhaNGiRZo+fboGDx4sSXrzzTcVGBiojz76SMOGDdPevXu1du1afffdd+revbsk6cUXX9TAgQP17LPPKjg4uNrGAgAAaoca+8zPkSNHlJ6ervDwcHubr6+vevTooeTkZElScnKy/Pz87MFHksLDw+Xm5qatW7dWe80AAKDmc+mVnz+Tnp4uSQoMDHRoDwwMtG9LT09XQECAw3YPDw/5+/vb+5QlLy9PeXl59vXs7OzKKhsAANRwNfbKT1WKj4+Xr6+vfQkJCXF1SQAAoJrU2PATFBQkScrIyHBoz8jIsG8LCgrSiRMnHLYXFhbq1KlT9j5liYuLU1ZWln05duxYJVcPAABqqhobftq0aaOgoCBt2LDB3padna2tW7fKZrNJkmw2mzIzM7V9+3Z7ny+++ELFxcXq0aPHRY/t7e0tq9XqsAAAAHNw6TM/Z8+e1cGDB+3rR44c0c6dO+Xv76+WLVtq4sSJeuqpp9S+fXu1adNGTzzxhIKDgzVkyBBJUqdOndS/f3/df//9Wrp0qQoKCjRu3DgNGzaMN70AAECZXBp+tm3bpj59+tjXJ0+eLEmKjo5WYmKipk2bppycHI0ZM0aZmZnq3bu31q5dq3r16tn3efvttzVu3DjddNNNcnNzU1RUlF544YVqHwsAAKgdLIZhGK4uwtWys7Pl6+urrKwsboEBgIm0fmy1q0swhaPzIqvkuM7+/q6xz/wAAABUBcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFcIPAAAwFQ9XFwAAKK31Y6tdct6j8yJdcl6gOnHlBwAAmApXfgAAdlxxghnUmfCzePFiPfPMM0pPT9eVV16pF198Uddcc42ry0I14S9sAEB51YnbXu+++64mT56smTNn6vvvv9eVV16piIgInThxwtWlAQCAGqZOhJ8FCxbo/vvv16hRoxQaGqqlS5eqQYMGeuONN1xdGgAAqGFq/W2v/Px8bd++XXFxcfY2Nzc3hYeHKzk52YWVwQxcdbtNct0tN1eO2RW4tQnUPbU+/Pz2228qKipSYGCgQ3tgYKB+/vnnMvfJy8tTXl6efT0rK0uSlJ2dXen1dZ65rtKPWR4/zYpwyXldpTjvnKtLqHZV8fNaHmaba+a5ejDPdVtV/fmWHNcwjArtV+vDjzPi4+M1a9asUu0hISEuqKZq+C5ydQWoavwZVw/muXowz3VbVf/5njlzRr6+vuXuX+vDT9OmTeXu7q6MjAyH9oyMDAUFBZW5T1xcnCZPnmxfLy4u1qlTp9SkSRNZLJYqrbe8srOzFRISomPHjslqtbq6nDqLea4+zHX1YJ6rB/NcPf5qng3D0JkzZxQcHFyh49b68OPl5aWwsDBt2LBBQ4YMkXQ+zGzYsEHjxo0rcx9vb295e3s7tPn5+VVxpc6xWq38h1UNmOfqw1xXD+a5ejDP1ePP5rkiV3xK1PrwI0mTJ09WdHS0unfvrmuuuUaLFi1STk6ORo0a5erSAABADVMnws+dd96pkydPasaMGUpPT1e3bt20du3aUg9BAwAA1InwI0njxo276G2u2sjb21szZ84sdXsOlYt5rj7MdfVgnqsH81w9qmqeLUZF3w8DAACoxerEJzwDAACUF+EHAACYCuEHAACYCuEHAACYCuHHhRYvXqzWrVurXr166tGjh7799ts/7b9y5Up17NhR9erVU5cuXfTZZ59VU6W1W0Xm+dVXX9V1112nxo0bq3HjxgoPD//LPxf8n4r+TJdYsWKFLBaL/YNK8ecqOs+ZmZmKjY1V8+bN5e3trQ4dOvD3RzlUdJ4XLVqkyy+/XPXr11dISIgmTZqk3Nzcaqq2dtq0aZMGDRqk4OBgWSwWffTRR3+5z1dffaWrr75a3t7eateunRITEyt+YgMusWLFCsPLy8t44403jN27dxv333+/4efnZ2RkZJTZf/PmzYa7u7sxf/58Y8+ePcb06dMNT09PY9euXdVcee1S0Xm+++67jcWLFxs7duww9u7da4wcOdLw9fU1jh8/Xs2V1z4VnesSR44cMf72t78Z1113nTF48ODqKbYWq+g85+XlGd27dzcGDhxo/Pe//zWOHDlifPXVV8bOnTurufLapaLz/Pbbbxve3t7G22+/bRw5csRYt26d0bx5c2PSpEnVXHnt8tlnnxmPP/648cEHHxiSjA8//PBP+x8+fNho0KCBMXnyZGPPnj3Giy++aLi7uxtr166t0HkJPy5yzTXXGLGxsfb1oqIiIzg42IiPjy+z/x133GFERkY6tPXo0cN44IEHqrTO2q6i83yhwsJCw8fHx1i2bFlVlVhnODPXhYWFxrXXXmu89tprRnR0NOGnHCo6zy+99JJx2WWXGfn5+dVVYp1Q0XmOjY01+vbt69A2efJko1evXlVaZ11SnvAzbdo044orrnBou/POO42IiIgKnYvbXi6Qn5+v7du3Kzw83N7m5uam8PBwJScnl7lPcnKyQ39JioiIuGh/ODfPFzp37pwKCgrk7+9fVWXWCc7O9ezZsxUQEKCYmJjqKLPWc2aeP/nkE9lsNsXGxiowMFCdO3fW3LlzVVRUVF1l1zrOzPO1116r7du322+NHT58WJ999pkGDhxYLTWbRWX9Lqwzn/Bcm/z2228qKioq9fUbgYGB+vnnn8vcJz09vcz+6enpVVZnbefMPF/o0UcfVXBwcKn/2ODImbn+73//q9dff107d+6shgrrBmfm+fDhw/riiy80fPhwffbZZzp48KAeeughFRQUaObMmdVRdq3jzDzffffd+u2339S7d28ZhqHCwkI9+OCD+uc//1kdJZvGxX4XZmdn63//+5/q169fruNw5Qe4iHnz5mnFihX68MMPVa9ePVeXU6ecOXNGI0aM0KuvvqqmTZu6upw6rbi4WAEBAXrllVcUFhamO++8U48//riWLl3q6tLqlK+++kpz587VkiVL9P333+uDDz7Q6tWrNWfOHFeXhjJw5ccFmjZtKnd3d2VkZDi0Z2RkKCgoqMx9goKCKtQfzs1ziWeffVbz5s3T559/rq5du1ZlmXVCRef60KFDOnr0qAYNGmRvKy4uliR5eHho3759atu2bdUWXQs58zPdvHlzeXp6yt3d3d7WqVMnpaenKz8/X15eXlVac23kzDw/8cQTGjFihEaPHi1J6tKli3JycjRmzBg9/vjjcnPjWkNluNjvQqvVWu6rPhJXflzCy8tLYWFh2rBhg72tuLhYGzZskM1mK3Mfm83m0F+SkpKSLtofzs2zJM2fP19z5szR2rVr1b179+ootdar6Fx37NhRu3bt0s6dO+3LP/7xD/Xp00c7d+5USEhIdZZfazjzM92rVy8dPHjQHi4laf/+/WrevDnB5yKcmedz586VCjglgdPgKzQrTaX9LqzYs9ioLCtWrDC8vb2NxMREY8+ePcaYMWMMPz8/Iz093TAMwxgxYoTx2GOP2ftv3rzZ8PDwMJ599llj7969xsyZM3nVvRwqOs/z5s0zvLy8jPfff99IS0uzL2fOnHHVEGqNis71hXjbq3wqOs8pKSmGj4+PMW7cOGPfvn3GqlWrjICAAOOpp55y1RBqhYrO88yZMw0fHx/jnXfeMQ4fPmysX7/eaNu2rXHHHXe4agi1wpkzZ4wdO3YYO3bsMCQZCxYsMHbs2GH88ssvhmEYxmOPPWaMGDHC3r/kVfepU6cae/fuNRYvXsyr7rXNiy++aLRs2dLw8vIyrrnmGuObb76xb7vhhhuM6Ohoh/7vvfee0aFDB8PLy8u44oorjNWrV1dzxbVTRea5VatWhqRSy8yZM6u/8Fqooj/Tf0T4Kb+KzvOWLVuMHj16GN7e3sZll11m/Otf/zIKCwuruerapyLzXFBQYDz55JNG27ZtjXr16hkhISHGQw89ZJw+fbr6C69FvvzyyzL/zi2Z2+joaOOGG24otU+3bt0MLy8v47LLLjMSEhIqfF6LYXA9DgAAmAfP/AAAAFMh/AAAAFMh/AAAAFMh/AAAAFMh/AAAAFMh/AAAAFMh/AAAAFMh/ACos2688UZNnDjR1WUAqGEIPwBqpEGDBql///5lbvv6669lsVj0448/VnNVAOoCwg+AGikmJkZJSUk6fvx4qW0JCQnq3r27unbt6oLKANR2hB8ANdItt9yiZs2aKTEx0aH97NmzWrlypYYMGaK77rpLf/vb39SgQQN16dJF77zzzp8e02Kx6KOPPnJo8/PzczjHsWPHdMcdd8jPz0/+/v4aPHiwjh49WjmDAlAjEH4A1EgeHh669957lZiYqD9+BeHKlStVVFSke+65R2FhYVq9erV++uknjRkzRiNGjNC3337r9DkLCgoUEREhHx8fff3119q8ebMaNWqk/v37Kz8/vzKGBaAGIPwAqLHuu+8+HTp0SBs3brS3JSQkKCoqSq1atdKUKVPUrVs3XXbZZRo/frz69++v9957z+nzvfvuuyouLtZrr72mLl26qFOnTkpISFBKSoq++uqrShgRgJqA8AOgxurYsaOuvfZavfHGG5KkgwcP6uuvv1ZMTIyKioo0Z84cdenSRf7+/mrUqJHWrVunlJQUp8/3ww8/6ODBg/Lx8VGjRo3UqFEj+fv7Kzc3V4cOHaqsYQFwMQ9XFwAAfyYmJkbjx4/X4sWLlZCQoLZt2+qGG27Q008/reeff16LFi1Sly5d1LBhQ02cOPFPb09ZLBaHW2jS+VtdJc6ePauwsDC9/fbbpfZt1qxZ5Q0KgEsRfgDUaHfccYcmTJig5cuX680339TYsWNlsVi0efNmDR48WPfcc48kqbi4WPv371doaOhFj9WsWTOlpaXZ1w8cOKBz587Z16+++mq9++67CggIkNVqrbpBAXApbnsBqNEaNWqkO++8U3FxcUpLS9PIkSMlSe3bt1dSUpK2bNmivXv36oEHHlBGRsafHqtv377697//rR07dmjbtm168MEH5enpad8+fPhwNW3aVIMHD9bXX3+tI0eO6KuvvtLDDz9c5iv3AGonwg+AGi8mJkanT59WRESEgoODJUnTp0/X1VdfrYiICN14440KCgrSkCFD/vQ4zz33nEJCQnTdddfp7rvv1pQpU9SgQQP79gYNGmjTpk1q2bKlhg4dqk6dOikmJka5ublcCQLqEItx4Q1wAACAOowrPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFT+P8gniu3rnuF0AAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.hist(data['5label1'], bins=15, align='left')\n",
        "\n",
        "# Add axis labels and a title\n",
        "plt.xlabel('Value')\n",
        "plt.ylabel('Frequency')\n",
        "plt.title('Histogram of Values')\n",
        "# Display the plot\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 472
        },
        "id": "_0vVbIecxYBG",
        "outputId": "1fb78b20-b5a3-4764-ff41-02d79c1c4309"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj8AAAHHCAYAAABQhTneAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8M0lEQVR4nO3deVyU5f7/8fewujEgKiBHXHJJKczCk5K2aBQqeTRpsdTQKMvQVFKLk2lpR81KbdFsBTtllv1a3VFLO0qapuVRc0kNjUU7CqN02O/fHz2Y7xnBkmlggPv1fDzuR811X3Nfn+sS4929zFgMwzAEAABgEh7uLgAAAKAmEX4AAICpEH4AAICpEH4AAICpEH4AAICpEH4AAICpEH4AAICpEH4AAICpEH4AAICpEH6AeqRt27YaOXKku8uo95599lldcskl8vT0VLdu3Wps3NTUVFksFh07dqzGxgTqI8IPUEuV/6LbsWNHpftvuOEGXX755X96nFWrVunJJ5/808cxi3Xr1mnKlCnq1auXUlJSNGvWrAp9iouL1bx5c/Xu3fuCxzEMQ2FhYbrqqquqs1wAlfBydwEAXOfAgQPy8Kja/9OsWrVKCxcuJABdpI0bN8rDw0NvvvmmfHx8Ku3j7e2t22+/Xa+++qp++ukntWnTpkKfzZs368SJE5o4cWJ1lwzgPJz5AeoRX19feXt7u7uMKsnPz3d3CVVy8uRJNWzY8ILBp9ywYcNkGIbee++9SvcvXbpUHh4eGjp0aHWUCeB3EH6AeuT8e36Ki4v11FNPqWPHjmrQoIGaNWum3r17Ky0tTZI0cuRILVy4UJJksVjsW7n8/Hw98sgjCgsLk6+vry699FI999xzMgzDYdz//ve/evjhh9W8eXP5+fnpb3/7m37++WdZLBaHM0pPPvmkLBaL9u3bp7vvvltNmza1Xxr6/vvvNXLkSF1yySVq0KCBQkJCdO+99+o///mPw1jlxzh48KCGDx8uf39/tWjRQk888YQMw9Dx48c1aNAgWa1WhYSE6Pnnn7+otSspKdHMmTPVvn17+fr6qm3btvr73/+uwsJCex+LxaKUlBTl5+fb1yo1NbXS4/Xq1Utt27bV0qVLK+wrLi7Whx9+qD59+ig0NPSi516Z89e4XGX3f+Xm5mrChAn2P88OHTromWeeUVlZmUO/ZcuWKTIyUn5+frJarYqIiNALL7zwh7UAdQWXvYBaLi8vT7/88kuF9uLi4j9875NPPqnZs2frvvvu09VXXy2bzaYdO3bo22+/1U033aQHHnhAmZmZSktL0z//+U+H9xqGob/97W/64osvlJCQoG7dumnt2rWaPHmyfv75Z82fP9/ed+TIkfrggw80YsQI9ezZU5s2bVJsbOwF67r99tvVsWNHzZo1yx6k0tLSdOTIEY0aNUohISHau3evXnvtNe3du1dff/21QyiTpDvvvFNdunTRnDlztHLlSj399NMKDAzUq6++qr59++qZZ57Ru+++q0mTJumvf/2rrrvuut9dq/vuu09LlizRbbfdpkceeUTbtm3T7NmztX//fn388ceSpH/+85967bXXtH37dr3xxhuSpGuuuabS41ksFt19992aNWuW9u7dq8suu8y+b82aNTp9+rSGDRvm1Nyd8euvv+r666/Xzz//rAceeECtW7fW1q1blZycrKysLC1YsMBey1133aUbb7xRzzzzjCRp//792rJli8aPH/+n6wBqBQNArZSSkmJI+t3tsssuc3hPmzZtjPj4ePvrK664woiNjf3dcRITE43K/lPwySefGJKMp59+2qH9tttuMywWi3H48GHDMAxj586dhiRjwoQJDv1GjhxpSDKmT59ub5s+fbohybjrrrsqjPfrr79WaHvvvfcMScbmzZsrHGP06NH2tpKSEqNVq1aGxWIx5syZY28/c+aM0bBhQ4c1qczu3bsNScZ9993n0D5p0iRDkrFx40Z7W3x8vNG4cePfPV65vXv3GpKM5ORkh/ahQ4caDRo0MPLy8gzDuPi5l/9MHD161N52/hqXO/9nYebMmUbjxo2NgwcPOvR77LHHDE9PTyMjI8MwDMMYP368YbVajZKSkouaI1AXcdkLqOUWLlyotLS0ClvXrl3/8L0BAQHau3evDh06VOVxV61aJU9PTz388MMO7Y888ogMw9Dq1asl/XYWQ5Ieeughh37jxo274LEffPDBCm0NGza0/3tBQYF++eUX9ezZU5L07bffVuh/33332f/d09NT3bt3l2EYSkhIsLcHBATo0ksv1ZEjRy5Yi/TbXCUpKSnJof2RRx6RJK1cufJ3338h4eHhuvLKK7Vs2TJ7W35+vj777DPdcsstslqtkqo+d2csX75c1157rZo2bapffvnFvkVHR6u0tFSbN2+W9Nua5efn2y+NAvUR4Qeo5a6++mpFR0dX2Jo2bfqH750xY4Zyc3PVqVMnRUREaPLkyfr+++8vatyffvpJoaGh8vPzc2jv0qWLfX/5Pz08PNSuXTuHfh06dLjgsc/vK0mnT5/W+PHjFRwcrIYNG6pFixb2fnl5eRX6t27d2uG1v7+/GjRooObNm1doP3PmzAVr+d85nF9zSEiIAgIC7HN1xrBhw3T06FFt3bpVkvTJJ5/o119/tV/ykqo+d2ccOnRIa9asUYsWLRy26OhoSb/dyC39FmI7deqk/v37q1WrVrr33nvtAReoL7jnB6jHrrvuOv3444/69NNPtW7dOr3xxhuaP3++Fi9e7HDmpKb975mOcnfccYe2bt2qyZMnq1u3bmrSpInKysrUr1+/CjfkSr+d7bmYNkkVbtC+EFfcW3O+u+66S1OmTNHSpUt1zTXXaOnSpWratKkGDBhg71PVuV+M0tJSh9dlZWW66aabNGXKlEr7d+rUSZIUFBSk3bt3a+3atVq9erVWr16tlJQU3XPPPVqyZIlTtQC1DeEHqOcCAwM1atQojRo1SufOndN1112nJ5980h5+LvQLv02bNlq/fr3Onj3rcPbnhx9+sO8v/2dZWZmOHj2qjh072vsdPnz4oms8c+aMNmzYoKeeekrTpk2ztztzuc4Z5XM4dOiQ/cyWJOXk5Cg3N7fSz+m5WKGhoerTp4+WL1+uJ554QmlpaRo5cqT9Ufk/O/emTZsqNzfXoa2oqEhZWVkObe3bt9e5c+fsZ3p+j4+PjwYOHKiBAweqrKxMDz30kF599VU98cQTv3tGD6gruOwF1GPnPyrdpEkTdejQweHx7caNG0tShV+gAwYMUGlpqV5++WWH9vnz58tisah///6SpJiYGEnSokWLHPq99NJLF11n+Rmb88/QlD+BVN3Kz8KcP968efMk6XefXLsYw4YN08mTJ/XAAw+ouLjY4ZLXn517+/bt7ffrlHvttdcqnPm54447lJ6errVr11Y4Rm5urkpKSiRV/Jnx8PCw31/2vz83QF3GmR+gHgsPD9cNN9ygyMhIBQYGaseOHfrwww81duxYe5/IyEhJ0sMPP6yYmBh5enpq6NChGjhwoPr06aPHH39cx44d0xVXXKF169bp008/1YQJE9S+fXv7++Pi4rRgwQL95z//sT/qfvDgQUkXdynJarXquuuu09y5c1VcXKy//OUvWrdunY4ePVoNq1LRFVdcofj4eL322mvKzc3V9ddfr+3bt2vJkiUaPHiw+vTp86eOHxcXp4ceekiffvqpwsLCHB67/7Nzv++++/Tggw8qLi5ON910k7777jutXbu2wr1PkydPtt9oPXLkSEVGRio/P1979uzRhx9+qGPHjql58+a67777dPr0afXt21etWrXSTz/9pJdeekndunVzOCsG1GlufdYMwAWVP9b8zTffVLr/+uuv/8NH3Z9++mnj6quvNgICAoyGDRsanTt3Nv7xj38YRUVF9j4lJSXGuHHjjBYtWhgWi8XhsfezZ88aEydONEJDQw1vb2+jY8eOxrPPPmuUlZU5jJufn28kJiYagYGBRpMmTYzBgwcbBw4cMCQ5PHpe/pj6qVOnKsznxIkTxq233moEBAQY/v7+xu23325kZmZe8HH5849xoUfQK1unyhQXFxtPPfWU0a5dO8Pb29sICwszkpOTjYKCgosa54/cfvvthiRjypQpFfZd7Nwre9S9tLTUePTRR43mzZsbjRo1MmJiYozDhw9X+FkwjN/+PJOTk40OHToYPj4+RvPmzY1rrrnGeO655+w/Ex9++KFx8803G0FBQYaPj4/RunVr44EHHjCysrKqPGegtrIYxkXeCQgAVbB7925deeWVeueddxwu8wCAu3HPD4A/7b///W+FtgULFsjDw+MPP1kZAGoa9/wA+NPmzp2rnTt3qk+fPvLy8rI/Ij169GiFhYW5uzwAcMBlLwB/Wlpamp566int27dP586dU+vWrTVixAg9/vjj8vLi/7EA1C6EHwAAYCrc8wMAAEyF8AMAAEyFi/H67TtvMjMz5efnVy3f7QMAAFzPMAydPXtWoaGh8vC4+PM5hB9JmZmZPJECAEAddfz4cbVq1eqi+xN+JPuXNh4/flxWq9XN1QAAgIths9kUFhbm8OXLF4Pwo//77iGr1Ur4AQCgjqnqLSvc8AwAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEyF8AMAAEzFy90FAADQ9rGVbhn32JxYt4wL9+LMDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBXCDwAAMBW3hp+2bdvKYrFU2BITEyVJBQUFSkxMVLNmzdSkSRPFxcUpJyfH4RgZGRmKjY1Vo0aNFBQUpMmTJ6ukpMQd0wEAAHWAW8PPN998o6ysLPuWlpYmSbr99tslSRMnTtTnn3+u5cuXa9OmTcrMzNSQIUPs7y8tLVVsbKyKioq0detWLVmyRKmpqZo2bZpb5gMAAGo/i2EYhruLKDdhwgStWLFChw4dks1mU4sWLbR06VLddtttkqQffvhBXbp0UXp6unr27KnVq1frlltuUWZmpoKDgyVJixcv1qOPPqpTp07Jx8fnosa12Wzy9/dXXl6erFZrtc0PAFC5to+tdMu4x+bEumVcuIazv79rzT0/RUVFeuedd3TvvffKYrFo586dKi4uVnR0tL1P586d1bp1a6Wnp0uS0tPTFRERYQ8+khQTEyObzaa9e/decKzCwkLZbDaHDQAAmEOtCT+ffPKJcnNzNXLkSElSdna2fHx8FBAQ4NAvODhY2dnZ9j7/G3zK95fvu5DZs2fL39/fvoWFhbluIgAAoFarNeHnzTffVP/+/RUaGlrtYyUnJysvL8++HT9+vNrHBAAAtYOXuwuQpJ9++knr16/XRx99ZG8LCQlRUVGRcnNzHc7+5OTkKCQkxN5n+/btDscqfxqsvE9lfH195evr68IZAACAuqJWnPlJSUlRUFCQYmP/78azyMhIeXt7a8OGDfa2AwcOKCMjQ1FRUZKkqKgo7dmzRydPnrT3SUtLk9VqVXh4eM1NAAAA1BluP/NTVlamlJQUxcfHy8vr/8rx9/dXQkKCkpKSFBgYKKvVqnHjxikqKko9e/aUJN18880KDw/XiBEjNHfuXGVnZ2vq1KlKTEzkzA4AAKiU28PP+vXrlZGRoXvvvbfCvvnz58vDw0NxcXEqLCxUTEyMFi1aZN/v6empFStWaMyYMYqKilLjxo0VHx+vGTNm1OQUAABAHVKrPufHXficHwBwLz7nB86o85/zAwAAUBMIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFQIPwAAwFTcHn5+/vlnDR8+XM2aNVPDhg0VERGhHTt22PcbhqFp06apZcuWatiwoaKjo3Xo0CGHY5w+fVrDhg2T1WpVQECAEhISdO7cuZqeCgAAqAPcGn7OnDmjXr16ydvbW6tXr9a+ffv0/PPPq2nTpvY+c+fO1YsvvqjFixdr27Ztaty4sWJiYlRQUGDvM2zYMO3du1dpaWlasWKFNm/erNGjR7tjSgAAoJazGIZhuGvwxx57TFu2bNFXX31V6X7DMBQaGqpHHnlEkyZNkiTl5eUpODhYqampGjp0qPbv36/w8HB988036t69uyRpzZo1GjBggE6cOKHQ0NA/rMNms8nf3195eXmyWq2umyAA4KK0fWylW8Y9NifWLePCNZz9/e3WMz+fffaZunfvrttvv11BQUG68sor9frrr9v3Hz16VNnZ2YqOjra3+fv7q0ePHkpPT5ckpaenKyAgwB58JCk6OloeHh7atm1bpeMWFhbKZrM5bAAAwBzcGn6OHDmiV155RR07dtTatWs1ZswYPfzww1qyZIkkKTs7W5IUHBzs8L7g4GD7vuzsbAUFBTns9/LyUmBgoL3P+WbPni1/f3/7FhYW5uqpAQCAWsqt4aesrExXXXWVZs2apSuvvFKjR4/W/fffr8WLF1fruMnJycrLy7Nvx48fr9bxAABA7eHW8NOyZUuFh4c7tHXp0kUZGRmSpJCQEElSTk6OQ5+cnBz7vpCQEJ08edJhf0lJiU6fPm3vcz5fX19ZrVaHDQAAmINbw0+vXr104MABh7aDBw+qTZs2kqR27dopJCREGzZssO+32Wzatm2boqKiJElRUVHKzc3Vzp077X02btyosrIy9ejRowZmAQAA6hIvdw4+ceJEXXPNNZo1a5buuOMObd++Xa+99ppee+01SZLFYtGECRP09NNPq2PHjmrXrp2eeOIJhYaGavDgwZJ+O1PUr18/++Wy4uJijR07VkOHDr2oJ70AAIC5uDX8/PWvf9XHH3+s5ORkzZgxQ+3atdOCBQs0bNgwe58pU6YoPz9fo0ePVm5urnr37q01a9aoQYMG9j7vvvuuxo4dqxtvvFEeHh6Ki4vTiy++6I4pAQCAWs6tn/NTW/A5PwDgXnzOD5xRJz/nBwAAoKYRfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKkQfgAAgKm4Nfw8+eSTslgsDlvnzp3t+wsKCpSYmKhmzZqpSZMmiouLU05OjsMxMjIyFBsbq0aNGikoKEiTJ09WSUlJTU8FAADUEV7uLuCyyy7T+vXr7a+9vP6vpIkTJ2rlypVavny5/P39NXbsWA0ZMkRbtmyRJJWWlio2NlYhISHaunWrsrKydM8998jb21uzZs2q8bkAAIDaz+3hx8vLSyEhIRXa8/Ly9Oabb2rp0qXq27evJCklJUVdunTR119/rZ49e2rdunXat2+f1q9fr+DgYHXr1k0zZ87Uo48+qieffFI+Pj41PR0AAFDLuf2en0OHDik0NFSXXHKJhg0bpoyMDEnSzp07VVxcrOjoaHvfzp07q3Xr1kpPT5ckpaenKyIiQsHBwfY+MTExstls2rt37wXHLCwslM1mc9gAAIA5uDX89OjRQ6mpqVqzZo1eeeUVHT16VNdee63Onj2r7Oxs+fj4KCAgwOE9wcHBys7OliRlZ2c7BJ/y/eX7LmT27Nny9/e3b2FhYa6dGAAAqLXcetmrf//+9n/v2rWrevTooTZt2uiDDz5Qw4YNq23c5ORkJSUl2V/bbDYCEAAAJuH2y17/KyAgQJ06ddLhw4cVEhKioqIi5ebmOvTJycmx3yMUEhJS4emv8teV3UdUztfXV1ar1WEDAADmUKvCz7lz5/Tjjz+qZcuWioyMlLe3tzZs2GDff+DAAWVkZCgqKkqSFBUVpT179ujkyZP2PmlpabJarQoPD6/x+gEAQO3n1stekyZN0sCBA9WmTRtlZmZq+vTp8vT01F133SV/f38lJCQoKSlJgYGBslqtGjdunKKiotSzZ09J0s0336zw8HCNGDFCc+fOVXZ2tqZOnarExET5+vq6c2oAAKCWcmv4OXHihO666y795z//UYsWLdS7d299/fXXatGihSRp/vz58vDwUFxcnAoLCxUTE6NFixbZ3+/p6akVK1ZozJgxioqKUuPGjRUfH68ZM2a4a0oAAKCWsxiGYbi7CHez2Wzy9/dXXl4e9/8AgBu0fWylW8Y9NifWLePCNZz9/V2r7vkBAACoboQfAABgKoQfAABgKoQfAABgKoQfAABgKoQfAABgKoQfAABgKk6FnyNHjri6DgAAgBrhVPjp0KGD+vTpo3feeUcFBQWurgkAAKDaOBV+vv32W3Xt2lVJSUkKCQnRAw88oO3bt7u6NgAAAJdzKvx069ZNL7zwgjIzM/XWW28pKytLvXv31uWXX6558+bp1KlTrq4TAADAJf7UDc9eXl4aMmSIli9frmeeeUaHDx/WpEmTFBYWpnvuuUdZWVmuqhMAAMAl/lT42bFjhx566CG1bNlS8+bN06RJk/Tjjz8qLS1NmZmZGjRokKvqBAAAcAkvZ940b948paSk6MCBAxowYIDefvttDRgwQB4ev2Wpdu3aKTU1VW3btnVlrQAAAH+aU+HnlVde0b333quRI0eqZcuWlfYJCgrSm2+++aeKAwAAcDWnws+hQ4f+sI+Pj4/i4+OdOTwAAEC1ceqen5SUFC1fvrxC+/Lly7VkyZI/XRQAAEB1cSr8zJ49W82bN6/QHhQUpFmzZv3pogAAAKqLU+EnIyND7dq1q9Depk0bZWRk/OmiAAAAqotT4ScoKEjff/99hfbvvvtOzZo1+9NFAQAAVBenws9dd92lhx9+WF988YVKS0tVWlqqjRs3avz48Ro6dKirawQAAHAZp572mjlzpo4dO6Ybb7xRXl6/HaKsrEz33HMP9/wAAIBazanw4+Pjo/fff18zZ87Ud999p4YNGyoiIkJt2rRxdX0AAAAu5VT4KdepUyd16tTJVbUAAABUO6fCT2lpqVJTU7VhwwadPHlSZWVlDvs3btzokuIAAABczanwM378eKWmpio2NlaXX365LBaLq+sCAACoFk6Fn2XLlumDDz7QgAEDXF0PAABAtXLqUXcfHx916NDB1bUAAABUO6fCzyOPPKIXXnhBhmG4uh4AAIBq5dRlr3/961/64osvtHr1al122WXy9vZ22P/RRx+5pDgAAABXcyr8BAQE6NZbb3V1LQAAANXOqfCTkpLi6joAAABqhFP3/EhSSUmJ1q9fr1dffVVnz56VJGVmZurcuXMuKw4AAMDVnDrz89NPP6lfv37KyMhQYWGhbrrpJvn5+emZZ55RYWGhFi9e7Oo6AQAAXMKpMz/jx49X9+7ddebMGTVs2NDefuutt2rDhg0uKw4AAMDVnAo/X331laZOnSofHx+H9rZt2+rnn392qpA5c+bIYrFowoQJ9raCggIlJiaqWbNmatKkieLi4pSTk+PwvoyMDMXGxqpRo0YKCgrS5MmTVVJS4lQNAACg/nMq/JSVlam0tLRC+4kTJ+Tn51fl433zzTd69dVX1bVrV4f2iRMn6vPPP9fy5cu1adMmZWZmasiQIfb9paWlio2NVVFRkbZu3aolS5YoNTVV06ZNq/qkAACAKTgVfm6++WYtWLDA/tpisejcuXOaPn16lb/y4ty5cxo2bJhef/11NW3a1N6el5enN998U/PmzVPfvn0VGRmplJQUbd26VV9//bUkad26ddq3b5/eeecddevWTf3799fMmTO1cOFCFRUVOTM1AABQzzkVfp5//nlt2bJF4eHhKigo0N13322/5PXMM89U6ViJiYmKjY1VdHS0Q/vOnTtVXFzs0N65c2e1bt1a6enpkqT09HRFREQoODjY3icmJkY2m0179+51ZmoAAKCec+ppr1atWum7777TsmXL9P333+vcuXNKSEjQsGHDHG6A/iPLli3Tt99+q2+++abCvuzsbPn4+CggIMChPTg4WNnZ2fY+/xt8yveX77uQwsJCFRYW2l/bbLaLrhkAANRtToUfSfLy8tLw4cOdHvj48eMaP3680tLS1KBBA6eP44zZs2frqaeeqtExAQBA7eBU+Hn77bd/d/8999zzh8fYuXOnTp48qauuusreVlpaqs2bN+vll1/W2rVrVVRUpNzcXIezPzk5OQoJCZEkhYSEaPv27Q7HLX8arLxPZZKTk5WUlGR/bbPZFBYW9oc1AwCAus+p8DN+/HiH18XFxfr111/l4+OjRo0aXVT4ufHGG7Vnzx6HtlGjRqlz58569NFHFRYWJm9vb23YsEFxcXGSpAMHDigjI0NRUVGSpKioKP3jH//QyZMnFRQUJElKS0uT1WpVeHj4Bcf29fWVr69vleYMAADqB6fCz5kzZyq0HTp0SGPGjNHkyZMv6hh+fn66/PLLHdoaN26sZs2a2dsTEhKUlJSkwMBAWa1WjRs3TlFRUerZs6ek3546Cw8P14gRIzR37lxlZ2dr6tSpSkxMJNwAAIBKOX3Pz/k6duyoOXPmaPjw4frhhx9ccsz58+fLw8NDcXFxKiwsVExMjBYtWmTf7+npqRUrVmjMmDGKiopS48aNFR8frxkzZrhkfAAAUP9YDMMwXHWw3bt367rrrqtzT0/ZbDb5+/srLy9PVqvV3eUAgOm0fWylW8Y9NifWLePCNZz9/e3UmZ/PPvvM4bVhGMrKytLLL7+sXr16OXNIAACAGuFU+Bk8eLDDa4vFohYtWqhv3756/vnnXVEXAABAtXAq/JSVlbm6DgAAgBrh1NdbAAAA1FVOnfn53w8I/CPz5s1zZggAAIBq4VT42bVrl3bt2qXi4mJdeumlkqSDBw/K09PT4RObLRaLa6oEAABwEafCz8CBA+Xn56clS5aoadOmkn774MNRo0bp2muv1SOPPOLSIgEAAFzFqXt+nn/+ec2ePdsefCSpadOmevrpp3naCwAA1GpOhR+bzaZTp05VaD916pTOnj37p4sCAACoLk6Fn1tvvVWjRo3SRx99pBMnTujEiRP6f//v/ykhIUFDhgxxdY0AAAAu49Q9P4sXL9akSZN09913q7i4+LcDeXkpISFBzz77rEsLBAAAcCWnwk+jRo20aNEiPfvss/rxxx8lSe3bt1fjxo1dWhwAAICr/akPOczKylJWVpY6duyoxo0by4XfkQoAAFAtnAo///nPf3TjjTeqU6dOGjBggLKysiRJCQkJPOYOAABqNafCz8SJE+Xt7a2MjAw1atTI3n7nnXdqzZo1LisOAADA1Zy652fdunVau3atWrVq5dDesWNH/fTTTy4pDAAAoDo4deYnPz/f4YxPudOnT8vX1/dPFwUAAFBdnAo/1157rd5++237a4vForKyMs2dO1d9+vRxWXEAAACu5tRlr7lz5+rGG2/Ujh07VFRUpClTpmjv3r06ffq0tmzZ4uoaAQAAXMapMz+XX365Dh48qN69e2vQoEHKz8/XkCFDtGvXLrVv397VNQIAALhMlc/8FBcXq1+/flq8eLEef/zx6qgJAACg2lT5zI+3t7e+//776qgFAACg2jl12Wv48OF68803XV0LAABAtXPqhueSkhK99dZbWr9+vSIjIyt8p9e8efNcUhwAAICrVSn8HDlyRG3bttW///1vXXXVVZKkgwcPOvSxWCyuqw4AAMDFqhR+OnbsqKysLH3xxReSfvs6ixdffFHBwcHVUhwAAICrVemen/O/tX316tXKz893aUEAAADVyakbnsudH4YAAABquyqFH4vFUuGeHu7xAQAAdUmV7vkxDEMjR460f3lpQUGBHnzwwQpPe3300UeuqxAAAMCFqhR+4uPjHV4PHz7cpcUAAABUtyqFn5SUlOqqAwAAoEb8qRueAQAA6hrCDwAAMBXCDwAAMBXCDwAAMBW3hp9XXnlFXbt2ldVqldVqVVRUlFavXm3fX1BQoMTERDVr1kxNmjRRXFyccnJyHI6RkZGh2NhYNWrUSEFBQZo8ebJKSkpqeioAAKCOcGv4adWqlebMmaOdO3dqx44d6tu3rwYNGqS9e/dKkiZOnKjPP/9cy5cv16ZNm5SZmakhQ4bY319aWqrY2FgVFRVp69atWrJkiVJTUzVt2jR3TQkAANRyFqOWfUdFYGCgnn32Wd12221q0aKFli5dqttuu02S9MMPP6hLly5KT09Xz549tXr1at1yyy3KzMy0f7nq4sWL9eijj+rUqVPy8fG5qDFtNpv8/f2Vl5cnq9VabXMDAFSu7WMr3TLusTmxbhkXruHs7+9ac89PaWmpli1bpvz8fEVFRWnnzp0qLi5WdHS0vU/nzp3VunVrpaenS5LS09MVERHh8K3yMTExstls9rNHlSksLJTNZnPYAACAObg9/OzZs0dNmjSRr6+vHnzwQX388ccKDw9Xdna2fHx8FBAQ4NA/ODhY2dnZkqTs7GyH4FO+v3zfhcyePVv+/v72LSwszLWTAgAAtZbbw8+ll16q3bt3a9u2bRozZozi4+O1b9++ah0zOTlZeXl59u348ePVOh4AAKg9qvT1FtXBx8dHHTp0kCRFRkbqm2++0QsvvKA777xTRUVFys3NdTj7k5OTo5CQEElSSEiItm/f7nC88qfByvtUxtfX1/7lrAAAwFzcfubnfGVlZSosLFRkZKS8vb21YcMG+74DBw4oIyNDUVFRkqSoqCjt2bNHJ0+etPdJS0uT1WpVeHh4jdcOAABqP7ee+UlOTlb//v3VunVrnT17VkuXLtWXX36ptWvXyt/fXwkJCUpKSlJgYKCsVqvGjRunqKgo9ezZU5J08803Kzw8XCNGjNDcuXOVnZ2tqVOnKjExkTM7AACgUm4NPydPntQ999yjrKws+fv7q2vXrlq7dq1uuukmSdL8+fPl4eGhuLg4FRYWKiYmRosWLbK/39PTUytWrNCYMWMUFRWlxo0bKz4+XjNmzHDXlAAAQC1X6z7nxx34nB8AcC8+5wfOqPOf8wMAAFATCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUvNxdAOAKbR9b6ZZxj82Jdcu4AADnceYHAACYCuEHAACYCuEHAACYCuEHAACYCuEHAACYCuEHAACYCuEHAACYCuEHAACYCuEHAACYCuEHAACYCuEHAACYCuEHAACYCuEHAACYCuEHAACYCuEHAACYCuEHAACYCuEHAACYCuEHAACYCuEHAACYilvDz+zZs/XXv/5Vfn5+CgoK0uDBg3XgwAGHPgUFBUpMTFSzZs3UpEkTxcXFKScnx6FPRkaGYmNj1ahRIwUFBWny5MkqKSmpyakAAIA6wq3hZ9OmTUpMTNTXX3+ttLQ0FRcX6+abb1Z+fr69z8SJE/X5559r+fLl2rRpkzIzMzVkyBD7/tLSUsXGxqqoqEhbt27VkiVLlJqaqmnTprljSgAAoJazGIZhuLuIcqdOnVJQUJA2bdqk6667Tnl5eWrRooWWLl2q2267TZL0ww8/qEuXLkpPT1fPnj21evVq3XLLLcrMzFRwcLAkafHixXr00Ud16tQp+fj4/OG4NptN/v7+ysvLk9VqrdY5onq0fWylW8Y9NifWLeMC9Q1/h+EMZ39/16p7fvLy8iRJgYGBkqSdO3equLhY0dHR9j6dO3dW69atlZ6eLklKT09XRESEPfhIUkxMjGw2m/bu3VvpOIWFhbLZbA4bAAAwh1oTfsrKyjRhwgT16tVLl19+uSQpOztbPj4+CggIcOgbHBys7Oxse5//DT7l+8v3VWb27Nny9/e3b2FhYS6eDQAAqK1qTfhJTEzUv//9by1btqzax0pOTlZeXp59O378eLWPCQAAagcvdxcgSWPHjtWKFSu0efNmtWrVyt4eEhKioqIi5ebmOpz9ycnJUUhIiL3P9u3bHY5X/jRYeZ/z+fr6ytfX18WzAAAAdYFbz/wYhqGxY8fq448/1saNG9WuXTuH/ZGRkfL29taGDRvsbQcOHFBGRoaioqIkSVFRUdqzZ49Onjxp75OWliar1arw8PCamQgAAKgz3HrmJzExUUuXLtWnn34qPz8/+z06/v7+atiwofz9/ZWQkKCkpCQFBgbKarVq3LhxioqKUs+ePSVJN998s8LDwzVixAjNnTtX2dnZmjp1qhITEzm7AwAAKnBr+HnllVckSTfccINDe0pKikaOHClJmj9/vjw8PBQXF6fCwkLFxMRo0aJF9r6enp5asWKFxowZo6ioKDVu3Fjx8fGaMWNGTU0DAADUIW4NPxfzEUMNGjTQwoULtXDhwgv2adOmjVatWuXK0gAAQD1Va572AgAAqAmEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCqEHwAAYCpuDT+bN2/WwIEDFRoaKovFok8++cRhv2EYmjZtmlq2bKmGDRsqOjpahw4dcuhz+vRpDRs2TFarVQEBAUpISNC5c+dqcBYAAKAucWv4yc/P1xVXXKGFCxdWun/u3Ll68cUXtXjxYm3btk2NGzdWTEyMCgoK7H2GDRumvXv3Ki0tTStWrNDmzZs1evTompoCAACoY7zcOXj//v3Vv3//SvcZhqEFCxZo6tSpGjRokCTp7bffVnBwsD755BMNHTpU+/fv15o1a/TNN9+oe/fukqSXXnpJAwYM0HPPPafQ0NAamwsAAKgbau09P0ePHlV2draio6Ptbf7+/urRo4fS09MlSenp6QoICLAHH0mKjo6Wh4eHtm3bdsFjFxYWymazOWwAAMAcam34yc7OliQFBwc7tAcHB9v3ZWdnKygoyGG/l5eXAgMD7X0qM3v2bPn7+9u3sLAwF1cPAABqq1obfqpTcnKy8vLy7Nvx48fdXRIAAKghtTb8hISESJJycnIc2nNycuz7QkJCdPLkSYf9JSUlOn36tL1PZXx9fWW1Wh02AABgDrU2/LRr104hISHasGGDvc1ms2nbtm2KioqSJEVFRSk3N1c7d+6099m4caPKysrUo0ePGq8ZAADUfm592uvcuXM6fPiw/fXRo0e1e/duBQYGqnXr1powYYKefvppdezYUe3atdMTTzyh0NBQDR48WJLUpUsX9evXT/fff78WL16s4uJijR07VkOHDuVJLwAAUCm3hp8dO3aoT58+9tdJSUmSpPj4eKWmpmrKlCnKz8/X6NGjlZubq969e2vNmjVq0KCB/T3vvvuuxo4dqxtvvFEeHh6Ki4vTiy++WONzAQAAdYPFMAzD3UW4m81mk7+/v/Ly8rj/p45q+9hKt4x7bE6sW8YF6hv+DsMZzv7+rrX3/AAAAFQHwg8AADAVt97zAwAXi8siAFyFMz8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUCD8AAMBUvNxdQH3X9rGVbhn32JxYt4wLAEBtx5kfAABgKoQfAABgKoQfAABgKoQfAABgKoQfAABgKoQfAABgKoQfAABgKoQfAABgKoQfAABgKoQfAABgKoQfAABgKny3FwAAJsH3Tf6GMz8AAMBUCD8AAMBU6k34Wbhwodq2basGDRqoR48e2r59u7tLAgAAtVC9CD/vv/++kpKSNH36dH377be64oorFBMTo5MnT7q7NAAAUMvUi/Azb9483X///Ro1apTCw8O1ePFiNWrUSG+99Za7SwMAALVMnQ8/RUVF2rlzp6Kjo+1tHh4eio6OVnp6uhsrAwAAtVGdf9T9l19+UWlpqYKDgx3ag4OD9cMPP1T6nsLCQhUWFtpf5+XlSZJsNpvL6ysr/NXlx7wY1TGX2ox1rv/4M67f+POtGfVtncuPaxhGld5X58OPM2bPnq2nnnqqQntYWJgbqqke/gvcXYE5sM71H3/G9Rt/vjWjutf57Nmz8vf3v+j+dT78NG/eXJ6ensrJyXFoz8nJUUhISKXvSU5OVlJSkv11WVmZTp8+rWbNmslisVRrvRfLZrMpLCxMx48fl9VqdXc59RbrXHNY65rBOtcM1rlm/NE6G4ahs2fPKjQ0tErHrfPhx8fHR5GRkdqwYYMGDx4s6bcws2HDBo0dO7bS9/j6+srX19ehLSAgoJordY7VauUvVg1gnWsOa10zWOeawTrXjN9b56qc8SlX58OPJCUlJSk+Pl7du3fX1VdfrQULFig/P1+jRo1yd2kAAKCWqRfh584779SpU6c0bdo0ZWdnq1u3blqzZk2Fm6ABAADqRfiRpLFjx17wMldd5Ovrq+nTp1e4PAfXYp1rDmtdM1jnmsE614zqWmeLUdXnwwAAAOqwOv8hhwAAAFVB+AEAAKZC+AEAAKZC+AEAAKZC+HGjhQsXqm3btmrQoIF69Oih7du3/27/5cuXq3PnzmrQoIEiIiK0atWqGqq0bqvKOr/++uu69tpr1bRpUzVt2lTR0dF/+OeC/1PVn+lyy5Ytk8VisX9QKX5fVdc5NzdXiYmJatmypXx9fdWpUyf++3ERqrrOCxYs0KWXXqqGDRsqLCxMEydOVEFBQQ1VWzdt3rxZAwcOVGhoqCwWiz755JM/fM+XX36pq666Sr6+vurQoYNSU1OrPrABt1i2bJnh4+NjvPXWW8bevXuN+++/3wgICDBycnIq7b9lyxbD09PTmDt3rrFv3z5j6tSphre3t7Fnz54arrxuqeo633333cbChQuNXbt2Gfv37zdGjhxp+Pv7GydOnKjhyuueqq51uaNHjxp/+ctfjGuvvdYYNGhQzRRbh1V1nQsLC43u3bsbAwYMMP71r38ZR48eNb788ktj9+7dNVx53VLVdX733XcNX19f49133zWOHj1qrF271mjZsqUxceLEGq68blm1apXx+OOPGx999JEhyfj4449/t/+RI0eMRo0aGUlJSca+ffuMl156yfD09DTWrFlTpXEJP25y9dVXG4mJifbXpaWlRmhoqDF79uxK+99xxx1GbGysQ1uPHj2MBx54oFrrrOuqus7nKykpMfz8/IwlS5ZUV4n1hjNrXVJSYlxzzTXGG2+8YcTHxxN+LkJV1/mVV14xLrnkEqOoqKimSqwXqrrOiYmJRt++fR3akpKSjF69elVrnfXJxYSfKVOmGJdddplD25133mnExMRUaSwue7lBUVGRdu7cqejoaHubh4eHoqOjlZ6eXul70tPTHfpLUkxMzAX7w7l1Pt+vv/6q4uJiBQYGVleZ9YKzaz1jxgwFBQUpISGhJsqs85xZ588++0xRUVFKTExUcHCwLr/8cs2aNUulpaU1VXad48w6X3PNNdq5c6f90tiRI0e0atUqDRgwoEZqNgtX/S6sN5/wXJf88ssvKi0trfD1G8HBwfrhhx8qfU92dnal/bOzs6utzrrOmXU+36OPPqrQ0NAKf9ngyJm1/te//qU333xTu3fvroEK6wdn1vnIkSPauHGjhg0bplWrVunw4cN66KGHVFxcrOnTp9dE2XWOM+t8991365dfflHv3r1lGIZKSkr04IMP6u9//3tNlGwaF/pdaLPZ9N///lcNGza8qONw5ge4gDlz5mjZsmX6+OOP1aBBA3eXU6+cPXtWI0aM0Ouvv67mzZu7u5x6raysTEFBQXrttdcUGRmpO++8U48//rgWL17s7tLqlS+//FKzZs3SokWL9O233+qjjz7SypUrNXPmTHeXhkpw5scNmjdvLk9PT+Xk5Di05+TkKCQkpNL3hISEVKk/nFvncs8995zmzJmj9evXq2vXrtVZZr1Q1bX+8ccfdezYMQ0cONDeVlZWJkny8vLSgQMH1L59++otug5y5me6ZcuW8vb2lqenp72tS5cuys7OVlFRkXx8fKq15rrImXV+4oknNGLECN13332SpIiICOXn52v06NF6/PHH5eHBuQZXuNDvQqvVetFnfSTO/LiFj4+PIiMjtWHDBntbWVmZNmzYoKioqErfExUV5dBfktLS0i7YH86tsyTNnTtXM2fO1Jo1a9S9e/eaKLXOq+pad+7cWXv27NHu3bvt29/+9jf16dNHu3fvVlhYWE2WX2c48zPdq1cvHT582B4uJengwYNq2bIlwecCnFnnX3/9tULAKQ+cBl+h6TIu+11YtXux4SrLli0zfH19jdTUVGPfvn3G6NGjjYCAACM7O9swDMMYMWKE8dhjj9n7b9myxfDy8jKee+45Y//+/cb06dN51P0iVHWd58yZY/j4+BgffvihkZWVZd/Onj3rrinUGVVd6/PxtNfFqeo6Z2RkGH5+fsbYsWONAwcOGCtWrDCCgoKMp59+2l1TqBOqus7Tp083/Pz8jPfee884cuSIsW7dOqN9+/bGHXfc4a4p1Alnz541du3aZezatcuQZMybN8/YtWuX8dNPPxmGYRiPPfaYMWLECHv/8kfdJ0+ebOzfv99YuHAhj7rXNS+99JLRunVrw8fHx7j66quNr7/+2r7v+uuvN+Lj4x36f/DBB0anTp0MHx8f47LLLjNWrlxZwxXXTVVZ5zZt2hiSKmzTp0+v+cLroKr+TP8vws/Fq+o6b9261ejRo4fh6+trXHLJJcY//vEPo6SkpIarrnuqss7FxcXGk08+abRv395o0KCBERYWZjz00EPGmTNnar7wOuSLL76o9L+55WsbHx9vXH/99RXe061bN8PHx8e45JJLjJSUlCqPazEMzscBAADz4J4fAABgKoQfAABgKoQfAABgKoQfAABgKoQfAABgKoQfAABgKoQfAABgKoQfAPXWDTfcoAkTJri7DAC1DOEHQK00cOBA9evXr9J9X331lSwWi77//vsargpAfUD4AVArJSQkKC0tTSdOnKiwLyUlRd27d1fXrl3dUBmAuo7wA6BWuuWWW9SiRQulpqY6tJ87d07Lly/X4MGDddddd+kvf/mLGjVqpIiICL333nu/e0yLxaJPPvnEoS0gIMBhjOPHj+uOO+5QQECAAgMDNWjQIB07dsw1kwJQKxB+ANRKXl5euueee5Samqr//QrC5cuXq7S0VMOHD1dkZKRWrlypf//73xo9erRGjBih7du3Oz1mcXGxYmJi5Ofnp6+++kpbtmxRkyZN1K9fPxUVFbliWgBqAcIPgFrr3nvv1Y8//qhNmzbZ21JSUhQXF6c2bdpo0qRJ6tatmy655BKNGzdO/fr10wcffOD0eO+//77Kysr0xhtvKCIiQl26dFFKSooyMjL05ZdfumBGAGoDwg+AWqtz58665ppr9NZbb0mSDh8+rK+++koJCQkqLS3VzJkzFRERocDAQDVp0kRr165VRkaG0+N99913Onz4sPz8/NSkSRM1adJEgYGBKigo0I8//uiqaQFwMy93FwAAvychIUHjxo3TwoULlZKSovbt2+v666/XM888oxdeeEELFixQRESEGjdurAkTJvzu5SmLxeJwCU367VJXuXPnzikyMlLvvvtuhfe2aNHCdZMC4FaEHwC12h133KHx48dr6dKlevvttzVmzBhZLBZt2bJFgwYN0vDhwyVJZWVlOnjwoMLDwy94rBYtWigrK8v++tChQ/r111/tr6+66iq9//77CgoKktVqrb5JAXArLnsBqNWaNGmiO++8U8nJycrKytLIkSMlSR07dlRaWpq2bt2q/fv364EHHlBOTs7vHqtv3756+eWXtWvXLu3YsUMPPvigvL297fuHDRum5s2ba9CgQfrqq6909OhRffnll3r44YcrfeQeQN1E+AFQ6yUkJOjMmTOKiYlRaGioJGnq1Km66qqrFBMToxtuuEEhISEaPHjw7x7n+eefV1hYmK699lrdfffdmjRpkho1amTf36hRI23evFmtW7fWkCFD1KVLFyUkJKigoIAzQUA9YjHOvwAOAABQj3HmBwAAmArhBwAAmArhBwAAmArhBwAAmArhBwAAmArhBwAAmArhBwAAmArhBwAAmArhBwAAmArhBwAAmArhBwAAmArhBwAAmMr/B39zi2F1mz8rAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.hist(data['entropy'], bins=15, align='left')\n",
        "\n",
        "# Add axis labels and a title\n",
        "plt.xlabel('Value')\n",
        "plt.ylabel('Frequency')\n",
        "plt.title('Histogram of Values')\n",
        "# Display the plot\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 472
        },
        "id": "sfDVLjYFxZcb",
        "outputId": "5f21c519-88be-40c7-c32f-904810889092"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAHHCAYAAABZbpmkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA58klEQVR4nO3deVhWdf7/8dcNAm4sIrKNuOROohaWkUsuJCqZJi3uS5haaCq5xGS5NaJWZtOY1lRqM5JlV8vkjppaSZqmaWruhsWipYLiV2Q5vz+6vH/dgia3wH1zej6u61zD+ZzPfc77c2Li1ecst8UwDEMAAAAm5eLoAgAAAMoSYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcwkXr16mno0KGOLsP0XnrpJd12221ydXVVq1atyu24S5YskcVi0cmTJ8vtmIAZEHYAJ3X1D9vOnTuL3d6xY0c1b978lo+zevVqTZs27Zb381exfv16TZo0SW3bttXixYs1a9asIn3y8vLk5+endu3aXXc/hmEoJCREd955Z1mWC0BSJUcXAKD0HDp0SC4uJftvmNWrV2vBggUEnpu0adMmubi46J133pG7u3uxfdzc3PTII4/ozTff1E8//aS6desW6bN161b9/PPPGj9+fFmXDPzlMbMDmIiHh4fc3NwcXUaJ5OTkOLqEEjl9+rSqVKly3aBz1YABA2QYht5///1ityclJcnFxUV9+/YtizIB/AFhBzCRa+/ZycvL0/Tp09WoUSNVrlxZNWvWVLt27ZScnCxJGjp0qBYsWCBJslgs1uWqnJwcPfPMMwoJCZGHh4eaNGmil19+WYZh2Bz3//7v//T000/Lz89Pnp6eevDBB/XLL7/IYrHYzBhNmzZNFotFBw4cUP/+/VWjRg3rpZ69e/dq6NChuu2221S5cmUFBgbq8ccf12+//WZzrKv7OHz4sAYOHChvb2/VqlVLzz//vAzD0KlTp9SrVy95eXkpMDBQr7zyyk2du/z8fM2cOVMNGjSQh4eH6tWrp7///e/Kzc219rFYLFq8eLFycnKs52rJkiXF7q9t27aqV6+ekpKSimzLy8vTRx99pE6dOik4OPimx16ca8/xVcXdv3X+/HmNGzfO+s+zYcOGmjNnjgoLC236LV++XOHh4fL09JSXl5fCwsL02muv/WktgLPiMhbg5LKysvTrr78Wac/Ly/vTz06bNk2JiYkaPny47r77bmVnZ2vnzp367rvvdP/992vkyJFKS0tTcnKy/vOf/9h81jAMPfjgg/riiy8UGxurVq1aad26dZo4caJ++eUXvfrqq9a+Q4cO1YcffqhBgwbpnnvu0ZYtWxQdHX3duh555BE1atRIs2bNsgan5ORkHT9+XMOGDVNgYKD279+vt956S/v379c333xjE8Ik6bHHHlOzZs00e/ZsrVq1Si+++KJ8fX315ptvqnPnzpozZ46WLVumCRMm6K677lKHDh1ueK6GDx+upUuX6uGHH9Yzzzyj7du3KzExUQcPHtQnn3wiSfrPf/6jt956Szt27NDbb78tSbr33nuL3Z/FYlH//v01a9Ys7d+/X7fffrt129q1a3X27FkNGDDArrHb49KlS7rvvvv0yy+/aOTIkapTp462bdumhIQEpaena/78+dZa+vXrpy5dumjOnDmSpIMHD+rrr7/W2LFjb7kOwCEMAE5p8eLFhqQbLrfffrvNZ+rWrWsMGTLEut6yZUsjOjr6hseJi4szivtXwaeffmpIMl588UWb9ocfftiwWCzG0aNHDcMwjF27dhmSjHHjxtn0Gzp0qCHJmDp1qrVt6tSphiSjX79+RY536dKlIm3vv/++IcnYunVrkX2MGDHC2pafn2/Url3bsFgsxuzZs63t586dM6pUqWJzToqzZ88eQ5IxfPhwm/YJEyYYkoxNmzZZ24YMGWJUq1bthvu7av/+/YYkIyEhwaa9b9++RuXKlY2srCzDMG5+7Fd/J06cOGFtu/YcX3Xt78LMmTONatWqGYcPH7bp9+yzzxqurq5GamqqYRiGMXbsWMPLy8vIz8+/qTECFQGXsQAnt2DBAiUnJxdZWrRo8aef9fHx0f79+3XkyJESH3f16tVydXXV008/bdP+zDPPyDAMrVmzRtLvsxSS9NRTT9n0GzNmzHX3PWrUqCJtVapUsf58+fJl/frrr7rnnnskSd99912R/sOHD7f+7OrqqtatW8swDMXGxlrbfXx81KRJEx0/fvy6tUi/j1WS4uPjbdqfeeYZSdKqVatu+PnrCQ0N1R133KHly5db23JycvS///1PDzzwgLy8vCSVfOz2WLFihdq3b68aNWro119/tS6RkZEqKCjQ1q1bJf1+znJycqyXOgEzIOwATu7uu+9WZGRkkaVGjRp/+tkZM2bo/Pnzaty4scLCwjRx4kTt3bv3po77008/KTg4WJ6enjbtzZo1s26/+r8uLi6qX7++Tb+GDRted9/X9pWks2fPauzYsQoICFCVKlVUq1Yta7+srKwi/evUqWOz7u3trcqVK8vPz69I+7lz565byx/HcG3NgYGB8vHxsY7VHgMGDNCJEye0bds2SdKnn36qS5cuWS9hSSUfuz2OHDmitWvXqlatWjZLZGSkpN9vvJZ+D62NGzdW9+7dVbt2bT3++OPWQAtUVNyzA5hYhw4ddOzYMX322Wdav3693n77bb366qtatGiRzcxIefvjTMZVjz76qLZt26aJEyeqVatWql69ugoLC9WtW7ciN9BKv8/m3EybpCI3VF9Padwbc61+/fpp0qRJSkpK0r333qukpCTVqFFDPXr0sPYp6dhvRkFBgc16YWGh7r//fk2aNKnY/o0bN5Yk+fv7a8+ePVq3bp3WrFmjNWvWaPHixRo8eLCWLl1qVy2AoxF2AJPz9fXVsGHDNGzYMF28eFEdOnTQtGnTrGHnen/g69atqw0bNujChQs2szs//vijdfvV/y0sLNSJEyfUqFEja7+jR4/edI3nzp3Txo0bNX36dL3wwgvWdnsuv9nj6hiOHDlinbmSpMzMTJ0/f77Y9+TcrODgYHXq1EkrVqzQ888/r+TkZA0dOtT66Pqtjr1GjRo6f/68TduVK1eUnp5u09agQQNdvHjROpNzI+7u7urZs6d69uypwsJCPfXUU3rzzTf1/PPP33DGDnBWXMYCTOzaR5erV6+uhg0b2jxOXa1aNUkq8gezR48eKigo0L/+9S+b9ldffVUWi0Xdu3eXJEVFRUmS3njjDZt+r7/++k3XeXVG5toZmKtPCJW1q7Ms1x5v3rx5knTDJ8tuxoABA3T69GmNHDlSeXl5NpewbnXsDRo0sN5vc9Vbb71VZGbn0UcfVUpKitatW1dkH+fPn1d+fr6kor8zLi4u1vvD/vh7A1QkzOwAJhYaGqqOHTsqPDxcvr6+2rlzpz766CONHj3a2ic8PFyS9PTTTysqKkqurq7q27evevbsqU6dOum5557TyZMn1bJlS61fv16fffaZxo0bpwYNGlg/HxMTo/nz5+u3336zPnp++PBhSTd3acjLy0sdOnTQ3LlzlZeXp7/97W9av369Tpw4UQZnpaiWLVtqyJAheuutt3T+/Hndd9992rFjh5YuXarevXurU6dOt7T/mJgYPfXUU/rss88UEhJi8xj8rY59+PDhGjVqlGJiYnT//ffr+++/17p164rcuzRx4kTrjdFDhw5VeHi4cnJytG/fPn300Uc6efKk/Pz8NHz4cJ09e1adO3dW7dq19dNPP+n1119Xq1atbGa9gArFoc+CAbiuq48Zf/vtt8Vuv++++/700fMXX3zRuPvuuw0fHx+jSpUqRtOmTY1//OMfxpUrV6x98vPzjTFjxhi1atUyLBaLzWPoFy5cMMaPH28EBwcbbm5uRqNGjYyXXnrJKCwstDluTk6OERcXZ/j6+hrVq1c3evfubRw6dMiQZPMo+NXHxs+cOVNkPD///LPx0EMPGT4+Poa3t7fxyCOPGGlpadd9fP3afVzvkfDizlNx8vLyjOnTpxv169c33NzcjJCQECMhIcG4fPnyTR3nzzzyyCOGJGPSpElFtt3s2It79LygoMCYPHmy4efnZ1StWtWIiooyjh49WuR3wTB+/+eZkJBgNGzY0HB3dzf8/PyMe++913j55ZetvxMfffSR0bVrV8Pf399wd3c36tSpY4wcOdJIT08v8ZgBZ2ExjJu8cw8ASmDPnj2644479N///tfmsg0AlDfu2QFwy/7v//6vSNv8+fPl4uLyp28uBoCyxj07AG7Z3LlztWvXLnXq1EmVKlWyPrI8YsQIhYSEOLo8AH9xXMYCcMuSk5M1ffp0HThwQBcvXlSdOnU0aNAgPffcc6pUif+mAuBYhB0AAGBq3LMDAABMjbADAABMjYvp+v07Y9LS0uTp6Vkm340DAABKn2EYunDhgoKDg+Xicv35G8KOpLS0NJ4YAQCggjp16pRq16593e2EHcn6JYenTp2Sl5eXg6sBAAA3Izs7WyEhITZfVlwcwo7+/3f3eHl5EXYAAKhg/uwWFG5QBgAApkbYAQAApkbYAQAApkbYAQAApkbYAQAApkbYAQAApkbYAQAApkbYAQAApkbYAQAApkbYAQAApubQsLNw4UK1aNHC+jUNERERWrNmjXV7x44dZbFYbJZRo0bZ7CM1NVXR0dGqWrWq/P39NXHiROXn55f3UAAAgJNy6Hdj1a5dW7Nnz1ajRo1kGIaWLl2qXr16affu3br99tslSU888YRmzJhh/UzVqlWtPxcUFCg6OlqBgYHatm2b0tPTNXjwYLm5uWnWrFnlPh4AAOB8LIZhGI4u4o98fX310ksvKTY2Vh07dlSrVq00f/78YvuuWbNGDzzwgNLS0hQQECBJWrRokSZPnqwzZ87I3d39po6ZnZ0tb29vZWVl8UWgAABUEDf799tp7tkpKCjQ8uXLlZOTo4iICGv7smXL5Ofnp+bNmyshIUGXLl2ybktJSVFYWJg16EhSVFSUsrOztX///nKtHwAAOCeHXsaSpH379ikiIkKXL19W9erV9cknnyg0NFSS1L9/f9WtW1fBwcHau3evJk+erEOHDunjjz+WJGVkZNgEHUnW9YyMjOseMzc3V7m5udb17Ozs0h4WAABwEg4PO02aNNGePXuUlZWljz76SEOGDNGWLVsUGhqqESNGWPuFhYUpKChIXbp00bFjx9SgQQO7j5mYmKjp06eXRvlwEvWeXeWQ456cHe2Q4wIAbp7DL2O5u7urYcOGCg8PV2Jiolq2bKnXXnut2L5t2rSRJB09elSSFBgYqMzMTJs+V9cDAwOve8yEhARlZWVZl1OnTpXGUAAAgBNyeNi5VmFhoc0lpj/as2ePJCkoKEiSFBERoX379un06dPWPsnJyfLy8rJeCiuOh4eH9XH3qwsAADAnh17GSkhIUPfu3VWnTh1duHBBSUlJ2rx5s9atW6djx44pKSlJPXr0UM2aNbV3716NHz9eHTp0UIsWLSRJXbt2VWhoqAYNGqS5c+cqIyNDU6ZMUVxcnDw8PBw5NAAA4CQcGnZOnz6twYMHKz09Xd7e3mrRooXWrVun+++/X6dOndKGDRs0f/585eTkKCQkRDExMZoyZYr1866urlq5cqWefPJJRUREqFq1ahoyZIjNe3kAAMBfm9O9Z8cReM9OxccNygDw11Ph3rMDAABQFgg7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1BwadhYuXKgWLVrIy8tLXl5eioiI0Jo1a6zbL1++rLi4ONWsWVPVq1dXTEyMMjMzbfaRmpqq6OhoVa1aVf7+/po4caLy8/PLeygAAMBJOTTs1K5dW7Nnz9auXbu0c+dOde7cWb169dL+/fslSePHj9fnn3+uFStWaMuWLUpLS1OfPn2sny8oKFB0dLSuXLmibdu2aenSpVqyZIleeOEFRw0JAAA4GYthGIaji/gjX19fvfTSS3r44YdVq1YtJSUl6eGHH5Yk/fjjj2rWrJlSUlJ0zz33aM2aNXrggQeUlpamgIAASdKiRYs0efJknTlzRu7u7jd1zOzsbHl7eysrK0teXl5lNjaUnXrPrnLIcU/OjnbIcQEAN//322nu2SkoKNDy5cuVk5OjiIgI7dq1S3l5eYqMjLT2adq0qerUqaOUlBRJUkpKisLCwqxBR5KioqKUnZ1tnR0qTm5urrKzs20WAABgTg4PO/v27VP16tXl4eGhUaNG6ZNPPlFoaKgyMjLk7u4uHx8fm/4BAQHKyMiQJGVkZNgEnavbr267nsTERHl7e1uXkJCQ0h0UAABwGg4PO02aNNGePXu0fft2PfnkkxoyZIgOHDhQpsdMSEhQVlaWdTl16lSZHg8AADhOJUcX4O7uroYNG0qSwsPD9e233+q1117TY489pitXruj8+fM2szuZmZkKDAyUJAUGBmrHjh02+7v6tNbVPsXx8PCQh4dHKY8EAAA4I4fP7FyrsLBQubm5Cg8Pl5ubmzZu3GjddujQIaWmpioiIkKSFBERoX379un06dPWPsnJyfLy8lJoaGi51w4AAJyPQ2d2EhIS1L17d9WpU0cXLlxQUlKSNm/erHXr1snb21uxsbGKj4+Xr6+vvLy8NGbMGEVEROiee+6RJHXt2lWhoaEaNGiQ5s6dq4yMDE2ZMkVxcXHM3AAAAEkODjunT5/W4MGDlZ6eLm9vb7Vo0ULr1q3T/fffL0l69dVX5eLiopiYGOXm5ioqKkpvvPGG9fOurq5auXKlnnzySUVERKhatWoaMmSIZsyY4aghAQAAJ+N079lxBN6zU/Hxnh0A+OupcO/ZAQAAKAuEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGoODTuJiYm666675OnpKX9/f/Xu3VuHDh2y6dOxY0dZLBabZdSoUTZ9UlNTFR0drapVq8rf318TJ05Ufn5+eQ4FAAA4qUqOPPiWLVsUFxenu+66S/n5+fr73/+url276sCBA6pWrZq13xNPPKEZM2ZY16tWrWr9uaCgQNHR0QoMDNS2bduUnp6uwYMHy83NTbNmzSrX8QAAAOfj0LCzdu1am/UlS5bI399fu3btUocOHaztVatWVWBgYLH7WL9+vQ4cOKANGzYoICBArVq10syZMzV58mRNmzZN7u7uZToGAADg3Jzqnp2srCxJkq+vr037smXL5Ofnp+bNmyshIUGXLl2ybktJSVFYWJgCAgKsbVFRUcrOztb+/fuLPU5ubq6ys7NtFgAAYE4Ondn5o8LCQo0bN05t27ZV8+bNre39+/dX3bp1FRwcrL1792ry5Mk6dOiQPv74Y0lSRkaGTdCRZF3PyMgo9liJiYmaPn16GY0EAAA4E6cJO3Fxcfrhhx/01Vdf2bSPGDHC+nNYWJiCgoLUpUsXHTt2TA0aNLDrWAkJCYqPj7euZ2dnKyQkxL7CAQCAU3OKy1ijR4/WypUr9cUXX6h27do37NumTRtJ0tGjRyVJgYGByszMtOlzdf169/l4eHjIy8vLZgEAAObk0LBjGIZGjx6tTz75RJs2bVL9+vX/9DN79uyRJAUFBUmSIiIitG/fPp0+fdraJzk5WV5eXgoNDS2TugEAQMXh0MtYcXFxSkpK0meffSZPT0/rPTbe3t6qUqWKjh07pqSkJPXo0UM1a9bU3r17NX78eHXo0EEtWrSQJHXt2lWhoaEaNGiQ5s6dq4yMDE2ZMkVxcXHy8PBw5PAAAIATcOjMzsKFC5WVlaWOHTsqKCjIunzwwQeSJHd3d23YsEFdu3ZV06ZN9cwzzygmJkaff/65dR+urq5auXKlXF1dFRERoYEDB2rw4ME27+UBAAB/XQ6d2TEM44bbQ0JCtGXLlj/dT926dbV69erSKgsAAJiIU9ygDAAAUFYIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNTsCjvHjx8v7ToAAADKhF1hp2HDhurUqZP++9//6vLly6VdEwAAQKmxK+x89913atGiheLj4xUYGKiRI0dqx44dpV0bAADALbMr7LRq1Uqvvfaa0tLS9O677yo9PV3t2rVT8+bNNW/ePJ05c6a06wQAALDLLd2gXKlSJfXp00crVqzQnDlzdPToUU2YMEEhISEaPHiw0tPTS6tOAAAAu9xS2Nm5c6eeeuopBQUFad68eZowYYKOHTum5ORkpaWlqVevXqVVJwAAgF0q2fOhefPmafHixTp06JB69Oih9957Tz169JCLy+/ZqX79+lqyZInq1atXmrUCAACUmF1hZ+HChXr88cc1dOhQBQUFFdvH399f77zzzi0VBwAAcKvsCjtHjhz50z7u7u4aMmSIPbsHAAAoNXbds7N48WKtWLGiSPuKFSu0dOnSWy4KAACgtNgVdhITE+Xn51ek3d/fX7NmzbrlogAAAEqLXWEnNTVV9evXL9Jet25dpaam3nJRAAAApcWusOPv76+9e/cWaf/+++9Vs2bNm95PYmKi7rrrLnl6esrf31+9e/fWoUOHbPpcvnxZcXFxqlmzpqpXr66YmBhlZmba9ElNTVV0dLSqVq0qf39/TZw4Ufn5+fYMDQAAmIxdYadfv356+umn9cUXX6igoEAFBQXatGmTxo4dq759+970frZs2aK4uDh98803Sk5OVl5enrp27aqcnBxrn/Hjx+vzzz/XihUrtGXLFqWlpalPnz7W7QUFBYqOjtaVK1e0bds2LV26VEuWLNELL7xgz9AAAIDJWAzDMEr6oStXrmjQoEFasWKFKlX6/YGuwsJCDR48WIsWLZK7u7tdxZw5c0b+/v7asmWLOnTooKysLNWqVUtJSUl6+OGHJUk//vijmjVrppSUFN1zzz1as2aNHnjgAaWlpSkgIECStGjRIk2ePFlnzpy5qVqys7Pl7e2trKwseXl52VU7HKves6scctyTs6MdclwAwM3//bZrZsfd3V0ffPCBfvzxRy1btkwff/yxjh07pnfffdfuoCNJWVlZkiRfX19J0q5du5SXl6fIyEhrn6ZNm6pOnTpKSUmRJKWkpCgsLMwadCQpKipK2dnZ2r9/v921AAAAc7DrPTtXNW7cWI0bNy6VQgoLCzVu3Di1bdtWzZs3lyRlZGTI3d1dPj4+Nn0DAgKUkZFh7fPHoHN1+9VtxcnNzVVubq51PTs7u1TGAAAAnI9dYaegoEBLlizRxo0bdfr0aRUWFtps37RpU4n3GRcXpx9++EFfffWVPSWVSGJioqZPn17mxwEAAI5nV9gZO3aslixZoujoaDVv3lwWi+WWihg9erRWrlyprVu3qnbt2tb2wMBAXblyRefPn7eZ3cnMzFRgYKC1z44dO2z2d/Vprat9rpWQkKD4+HjrenZ2tkJCQm5pDAAAwDnZFXaWL1+uDz/8UD169LilgxuGoTFjxuiTTz7R5s2bi7y7Jzw8XG5ubtq4caNiYmIkSYcOHVJqaqoiIiIkSREREfrHP/6h06dPy9/fX5KUnJwsLy8vhYaGFntcDw8PeXh43FLtAACgYrAr7Li7u6thw4a3fPC4uDglJSXps88+k6enp/UeG29vb1WpUkXe3t6KjY1VfHy8fH195eXlpTFjxigiIkL33HOPJKlr164KDQ3VoEGDNHfuXGVkZGjKlCmKi4sj0AAAAPuexnrmmWf02muvyY6n1m0sXLhQWVlZ6tixo4KCgqzLBx98YO3z6quv6oEHHlBMTIw6dOigwMBAffzxx9btrq6uWrlypVxdXRUREaGBAwdq8ODBmjFjxi3VBgAAzMGu9+w89NBD+uKLL+Tr66vbb79dbm5uNtv/GEYqAt6zU/Hxnh0A+Ou52b/fdl3G8vHx0UMPPWR3cQAAAOXFrrCzePHi0q4DAACgTNh1z44k5efna8OGDXrzzTd14cIFSVJaWpouXrxYasUBAADcKrtmdn766Sd169ZNqampys3N1f333y9PT0/NmTNHubm5WrRoUWnXCQAAYBe7ZnbGjh2r1q1b69y5c6pSpYq1/aGHHtLGjRtLrTgAAIBbZdfMzpdffqlt27YV+dLPevXq6ZdffimVwgAAAEqDXTM7hYWFKigoKNL+888/y9PT85aLAgAAKC12hZ2uXbtq/vz51nWLxaKLFy9q6tSpt/wVEgAAAKXJrstYr7zyiqKiohQaGqrLly+rf//+OnLkiPz8/PT++++Xdo0AAAB2syvs1K5dW99//72WL1+uvXv36uLFi4qNjdWAAQNsblgGAABwNLvCjiRVqlRJAwcOLM1aAAAASp1dYee999674fbBgwfbVQwAAEBpsyvsjB071mY9Ly9Ply5dkru7u6pWrUrYAQAATsOup7HOnTtns1y8eFGHDh1Su3btuEEZAAA4Fbu/G+tajRo10uzZs4vM+gAAADhSqYUd6febltPS0kpzlwAAALfErnt2/ve//9msG4ah9PR0/etf/1Lbtm1LpTAAAIDSYFfY6d27t826xWJRrVq11LlzZ73yyiulURcAAECpsCvsFBYWlnYdAAAAZaJU79kBAABwNnbN7MTHx99033nz5tlzCAAAgFJhV9jZvXu3du/erby8PDVp0kSSdPjwYbm6uurOO++09rNYLKVTJQAAgJ3sCjs9e/aUp6enli5dqho1akj6/UWDw4YNU/v27fXMM8+UapEAAAD2suuenVdeeUWJiYnWoCNJNWrU0IsvvsjTWAAAwKnYFXays7N15syZIu1nzpzRhQsXbrkoAACA0mLXZayHHnpIw4YN0yuvvKK7775bkrR9+3ZNnDhRffr0KdUCATiXes+ucshxT86OdshxAVR8doWdRYsWacKECerfv7/y8vJ+31GlSoqNjdVLL71UqgUCAADcCrvCTtWqVfXGG2/opZde0rFjxyRJDRo0ULVq1Uq1OAAAgFt1Sy8VTE9PV3p6uho1aqRq1arJMIzSqgsAAKBU2BV2fvvtN3Xp0kWNGzdWjx49lJ6eLkmKjY3lsXMAAOBU7Ao748ePl5ubm1JTU1W1alVr+2OPPaa1a9eWWnEAAAC3yq57dtavX69169apdu3aNu2NGjXSTz/9VCqFAQAAlAa7ZnZycnJsZnSuOnv2rDw8PG65KAAAgNJi18xO+/bt9d5772nmzJmSfv8OrMLCQs2dO1edOnUq1QIBFM9R77sBgIrGrrAzd+5cdenSRTt37tSVK1c0adIk7d+/X2fPntXXX39d2jUCAADYza7LWM2bN9fhw4fVrl079erVSzk5OerTp492796tBg0alHaNAAAAdivxzE5eXp66deumRYsW6bnnniuLmgAAAEpNiWd23NzctHfv3rKoBQAAoNTZdRlr4MCBeuedd0q7FgAAgFJn1w3K+fn5evfdd7VhwwaFh4cX+U6sefPmlUpxAAAAt6pEMzvHjx9XYWGhfvjhB915553y9PTU4cOHtXv3buuyZ8+em97f1q1b1bNnTwUHB8tisejTTz+12T506FBZLBabpVu3bjZ9zp49qwEDBsjLy0s+Pj6KjY3VxYsXSzIsAABgYiWa2WnUqJHS09P1xRdfSPr96yH++c9/KiAgwK6D5+TkqGXLlnr88cfVp0+fYvt069ZNixcvtq5f+9LCAQMGKD09XcnJycrLy9OwYcM0YsQIJSUl2VUTAAAwlxKFnWu/1XzNmjXKycmx++Ddu3dX9+7db9jHw8NDgYGBxW47ePCg1q5dq2+//VatW7eWJL3++uvq0aOHXn75ZQUHB9tdGwAAMAe7blC+6trwUxY2b94sf39/NWnSRE8++aR+++0367aUlBT5+PhYg44kRUZGysXFRdu3b7/uPnNzc5WdnW2zAAAAcypR2Ll638y1bWWlW7dueu+997Rx40bNmTNHW7ZsUffu3VVQUCBJysjIkL+/v81nKlWqJF9fX2VkZFx3v4mJifL29rYuISEhZTYGAADgWCW+jDV06FDrfTOXL1/WqFGjijyN9fHHH5dKcX379rX+HBYWphYtWqhBgwbavHmzunTpYvd+ExISFB8fb13Pzs4m8AAAYFIlCjtDhgyxWR84cGCpFvNnbrvtNvn5+eno0aPq0qWLAgMDdfr0aZs++fn5Onv27HXv85F+vw+Ib2cHAOCvoURh549PRTnCzz//rN9++01BQUGSpIiICJ0/f167du1SeHi4JGnTpk0qLCxUmzZtHFkqAABwEna9VLC0XLx4UUePHrWunzhxQnv27JGvr698fX01ffp0xcTEKDAwUMeOHdOkSZPUsGFDRUVFSZKaNWumbt266YknntCiRYuUl5en0aNHq2/fvjyJBQAAJN3i01i3aufOnbrjjjt0xx13SJLi4+N1xx136IUXXpCrq6v27t2rBx98UI0bN1ZsbKzCw8P15Zdf2lyCWrZsmZo2baouXbqoR48eateund566y1HDQkAADgZh87sdOzY8YaPr69bt+5P9+Hr68sLBAEAwHU5dGYHAACgrBF2AACAqTn0MtZfQb1nVznkuCdnRzvkuAAAOBtmdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKlVcnQBMJd6z65ydAkAANhgZgcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJiaQ8PO1q1b1bNnTwUHB8tisejTTz+12W4Yhl544QUFBQWpSpUqioyM1JEjR2z6nD17VgMGDJCXl5d8fHwUGxurixcvluMoAACAM3No2MnJyVHLli21YMGCYrfPnTtX//znP7Vo0SJt375d1apVU1RUlC5fvmztM2DAAO3fv1/JyclauXKltm7dqhEjRpTXEAAAgJOr5MiDd+/eXd27dy92m2EYmj9/vqZMmaJevXpJkt577z0FBATo008/Vd++fXXw4EGtXbtW3377rVq3bi1Jev3119WjRw+9/PLLCg4OLrexAAAA5+S09+ycOHFCGRkZioyMtLZ5e3urTZs2SklJkSSlpKTIx8fHGnQkKTIyUi4uLtq+fft1952bm6vs7GybBQAAmJPThp2MjAxJUkBAgE17QECAdVtGRob8/f1ttleqVEm+vr7WPsVJTEyUt7e3dQkJCSnl6gEAgLNw2rBTlhISEpSVlWVdTp065eiSAABAGXHasBMYGChJyszMtGnPzMy0bgsMDNTp06dttufn5+vs2bPWPsXx8PCQl5eXzQIAAMzJacNO/fr1FRgYqI0bN1rbsrOztX37dkVEREiSIiIidP78ee3atcvaZ9OmTSosLFSbNm3KvWYAAOB8HPo01sWLF3X06FHr+okTJ7Rnzx75+vqqTp06GjdunF588UU1atRI9evX1/PPP6/g4GD17t1bktSsWTN169ZNTzzxhBYtWqS8vDyNHj1affv25UksAAAgycFhZ+fOnerUqZN1PT4+XpI0ZMgQLVmyRJMmTVJOTo5GjBih8+fPq127dlq7dq0qV65s/cyyZcs0evRodenSRS4uLoqJidE///nPch8LAABwTg4NOx07dpRhGNfdbrFYNGPGDM2YMeO6fXx9fZWUlFQW5QEAABNw2nt2AAAASgNhBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmJpTh51p06bJYrHYLE2bNrVuv3z5suLi4lSzZk1Vr15dMTExyszMdGDFAADA2Th12JGk22+/Xenp6dblq6++sm4bP368Pv/8c61YsUJbtmxRWlqa+vTp48BqAQCAs6nk6AL+TKVKlRQYGFikPSsrS++8846SkpLUuXNnSdLixYvVrFkzffPNN7rnnnvKu1QAAOCEnD7sHDlyRMHBwapcubIiIiKUmJioOnXqaNeuXcrLy1NkZKS1b9OmTVWnTh2lpKTcMOzk5uYqNzfXup6dnV2mYwCAkqr37CqHHPfk7GiHHBcoS059GatNmzZasmSJ1q5dq4ULF+rEiRNq3769Lly4oIyMDLm7u8vHx8fmMwEBAcrIyLjhfhMTE+Xt7W1dQkJCynAUAADAkZx6Zqd79+7Wn1u0aKE2bdqobt26+vDDD1WlShW795uQkKD4+HjrenZ2NoEHdnHUf30DAG6eU8/sXMvHx0eNGzfW0aNHFRgYqCtXruj8+fM2fTIzM4u9x+ePPDw85OXlZbMAAABzqlBh5+LFizp27JiCgoIUHh4uNzc3bdy40br90KFDSk1NVUREhAOrBAAAzsSpL2NNmDBBPXv2VN26dZWWlqapU6fK1dVV/fr1k7e3t2JjYxUfHy9fX195eXlpzJgxioiI4EksAABg5dRh5+eff1a/fv3022+/qVatWmrXrp2++eYb1apVS5L06quvysXFRTExMcrNzVVUVJTeeOMNB1cNAACciVOHneXLl99we+XKlbVgwQItWLCgnCoCAAAVTYW6ZwcAAKCkCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUnPoNygCA8lXv2VUOOe7J2dEOOS7+GpjZAQAApkbYAQAApkbYAQAApsY9OwBwA466hwVA6WFmBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBpfBAoAcDhHfeHqydnRDjkuyhczOwAAwNQIOwAAwNQIOwAAwNS4ZwdAheCoezoAVHzM7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFPjPTsAgL8svpPrr8E0MzsLFixQvXr1VLlyZbVp00Y7duxwdEkAAMAJmCLsfPDBB4qPj9fUqVP13XffqWXLloqKitLp06cdXRoAAHAwU1zGmjdvnp544gkNGzZMkrRo0SKtWrVK7777rp599lkHVwcAgHP4q162q/AzO1euXNGuXbsUGRlpbXNxcVFkZKRSUlIcWBkAAHAGFX5m59dff1VBQYECAgJs2gMCAvTjjz8W+5nc3Fzl5uZa17OysiRJ2dnZpV5fYe6lUt/nzSiLsdwMR40XACqSOuNXOLqEclVWf5Ou7tcwjBv2q/Bhxx6JiYmaPn16kfaQkBAHVFM2vOc7ugIAAH5X1n+TLly4IG9v7+tur/Bhx8/PT66ursrMzLRpz8zMVGBgYLGfSUhIUHx8vHW9sLBQZ8+eVc2aNWWxWMq03puVnZ2tkJAQnTp1Sl5eXo4u5y+Fc+8YnHfH4dw7Buf91hmGoQsXLig4OPiG/Sp82HF3d1d4eLg2btyo3r17S/o9vGzcuFGjR48u9jMeHh7y8PCwafPx8SnjSu3j5eXF/wkchHPvGJx3x+HcOwbn/dbcaEbnqgofdiQpPj5eQ4YMUevWrXX33Xdr/vz5ysnJsT6dBQAA/rpMEXYee+wxnTlzRi+88IIyMjLUqlUrrV27tshNywAA4K/HFGFHkkaPHn3dy1YVkYeHh6ZOnVrkchvKHufeMTjvjsO5dwzOe/mxGH/2vBYAAEAFVuFfKggAAHAjhB0AAGBqhB0AAGBqhB0AAGBqhB0HWrBggerVq6fKlSurTZs22rFjxw37r1ixQk2bNlXlypUVFham1atXl1Ol5lOSc//vf/9b7du3V40aNVSjRg1FRkb+6T8rFK+kv/NXLV++XBaLxfriUJRMSc/7+fPnFRcXp6CgIHl4eKhx48b8+8ZOJT338+fPV5MmTVSlShWFhIRo/Pjxunz5cjlVa2IGHGL58uWGu7u78e677xr79+83nnjiCcPHx8fIzMwstv/XX39tuLq6GnPnzjUOHDhgTJkyxXBzczP27dtXzpVXfCU99/379zcWLFhg7N692zh48KAxdOhQw9vb2/j555/LufKKraTn/aoTJ04Yf/vb34z27dsbvXr1Kp9iTaSk5z03N9do3bq10aNHD+Orr74yTpw4YWzevNnYs2dPOVde8ZX03C9btszw8PAwli1bZpw4ccJYt26dERQUZIwfP76cKzcfwo6D3H333UZcXJx1vaCgwAgODjYSExOL7f/oo48a0dHRNm1t2rQxRo4cWaZ1mlFJz/218vPzDU9PT2Pp0qVlVaIp2XPe8/PzjXvvvdd4++23jSFDhhB27FDS875w4ULjtttuM65cuVJeJZpWSc99XFyc0blzZ5u2+Ph4o23btmVa518Bl7Ec4MqVK9q1a5ciIyOtbS4uLoqMjFRKSkqxn0lJSbHpL0lRUVHX7Y/i2XPur3Xp0iXl5eXJ19e3rMo0HXvP+4wZM+Tv76/Y2NjyKNN07Dnv//vf/xQREaG4uDgFBASoefPmmjVrlgoKCsqrbFOw59zfe++92rVrl/VS1/Hjx7V69Wr16NGjXGo2M9O8Qbki+fXXX1VQUFDk6ywCAgL0448/FvuZjIyMYvtnZGSUWZ1mZM+5v9bkyZMVHBxcJHzi+uw571999ZXeeecd7dmzpxwqNCd7zvvx48e1adMmDRgwQKtXr9bRo0f11FNPKS8vT1OnTi2Psk3BnnPfv39//frrr2rXrp0Mw1B+fr5GjRqlv//97+VRsqkxswOUwOzZs7V8+XJ98sknqly5sqPLMa0LFy5o0KBB+ve//y0/Pz9Hl/OXUlhYKH9/f7311lsKDw/XY489pueee06LFi1ydGmmt3nzZs2aNUtvvPGGvvvuO3388cdatWqVZs6c6ejSKjxmdhzAz89Prq6uyszMtGnPzMxUYGBgsZ8JDAwsUX8Uz55zf9XLL7+s2bNna8OGDWrRokVZlmk6JT3vx44d08mTJ9WzZ09rW2FhoSSpUqVKOnTokBo0aFC2RZuAPb/vQUFBcnNzk6urq7WtWbNmysjI0JUrV+Tu7l6mNZuFPef++eef16BBgzR8+HBJUlhYmHJycjRixAg999xzcnFhfsJenDkHcHd3V3h4uDZu3GhtKyws1MaNGxUREVHsZyIiImz6S1JycvJ1+6N49px7SZo7d65mzpyptWvXqnXr1uVRqqmU9Lw3bdpU+/bt0549e6zLgw8+qE6dOmnPnj0KCQkpz/IrLHt+39u2baujR49aw6UkHT58WEFBQQSdErDn3F+6dKlIoLkaOg2+xvLWOPoO6b+q5cuXGx4eHsaSJUuMAwcOGCNGjDB8fHyMjIwMwzAMY9CgQcazzz5r7f/1118blSpVMl5++WXj4MGDxtSpU3n03E4lPfezZ8823N3djY8++shIT0+3LhcuXHDUECqkkp73a/E0ln1Ket5TU1MNT09PY/To0cahQ4eMlStXGv7+/saLL77oqCFUWCU991OnTjU8PT2N999/3zh+/Lixfv16o0GDBsajjz7qqCGYBmHHgV5//XWjTp06hru7u3H33Xcb33zzjXXbfffdZwwZMsSm/4cffmg0btzYcHd3N26//XZj1apV5VyxeZTk3NetW9eQVGSZOnVq+RdewZX0d/6PCDv2K+l537Ztm9GmTRvDw8PDuO2224x//OMfRn5+fjlXbQ4lOfd5eXnGtGnTjAYNGhiVK1c2QkJCjKeeeso4d+5c+RduMhbDYG4MAACYF/fsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsADCtjh07aty4cY4uA4CDEXYAOKWePXuqW7duxW778ssvZbFYtHfv3nKuCkBFRNgB4JRiY2OVnJysn3/+uci2xYsXq3Xr1nz7PICbQtgB4JQeeOAB1apVS0uWLLFpv3jxolasWKHevXurX79++tvf/qaqVasqLCxM77///g33abFY9Omnn9q0+fj42Bzj1KlTevTRR+Xj4yNfX1/16tVLJ0+eLJ1BAXAIwg4Ap1SpUiUNHjxYS5Ys0R+/wm/FihUqKCjQwIEDFR4erlWrVumHH37QiBEjNGjQIO3YscPuY+bl5SkqKkqenp768ssv9fXXX6t69erq1q2brly5UhrDAuAAhB0ATuvxxx/XsWPHtGXLFmvb4sWLFRMTo7p162rChAlq1aqVbrvtNo0ZM0bdunXThx9+aPfxPvjgAxUWFurtt99WWFiYmjVrpsWLFys1NVWbN28uhREBcATCDgCn1bRpU91777169913JUlHjx7Vl19+qdjYWBUUFGjmzJkKCwuTr6+vqlevrnXr1ik1NdXu433//fc6evSoPD09Vb16dVWvXl2+vr66fPmyjh07VlrDAlDOKjm6AAC4kdjYWI0ZM0YLFizQ4sWL1aBBA913332aM2eOXnvtNc2fP19hYWGqVq2axo0bd8PLTRaLxeaSmPT7paurLl68qPDwcC1btqzIZ2vVqlV6gwJQrgg7AJzao48+qrFjxyopKUnvvfeennzySVksFn399dfq1auXBg4cKEkqLCzU4cOHFRoaet191apVS+np6db1I0eO6NKlS9b1O++8Ux988IH8/f3l5eVVdoMCUK64jAXAqVWvXl2PPfaYEhISlJ6erqFDh0qSGjVqpOTkZG3btk0HDx7UyJEjlZmZecN9de7cWf/617+0e/du7dy5U6NGjZKbm5t1+4ABA+Tn56devXrpyy+/1IkTJ7R582Y9/fTTxT4CD6BiIOwAcHqxsbE6d+6coqKiFBwcLEmaMmWK7rzzTkVFRaljx44KDAxU7969b7ifV155RSEhIWrfvr369++vCRMmqGrVqtbtVatW1datW1WnTh316dNHzZo1U2xsrC5fvsxMD1CBWYxrL2ADAACYCDM7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1P4fHuh3lPrr50kAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#df=data[data['4label1']>0.9] #0.940 0.963 0.888 0.910 confidence:good\n",
        "#df=data[data['entropy']<0.2] #0.970 0.989 0.970 0.989 entropy:good\n",
        "df=data[data['5label1']<0.5] #0.931 0.989 0.765 0.802 self-estimate:bad\n",
        "#df=data # 0.932 0.975 0.859 0.878\n",
        "df['world_gold']=df['Class Index'].apply(lambda x:1 if x==4 else 0)\n",
        "df_correct=df[df['world_gold']==df['MV_sci']]\n",
        "df_correct.shape[0]/df.shape[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dbGjILxWxi1W",
        "outputId": "95759208-1ad4-4dba-e94e-b6039d00c744"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-29-e82e91ec586f>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df['world_gold']=df['Class Index'].apply(lambda x:1 if x==4 else 0)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8027210884353742"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "# Assuming you have a DataFrame named 'df'\n",
        "\n",
        "# Set the seed for reproducibility\n",
        "np.random.seed(42)\n",
        "\n",
        "# Calculate the number of rows for each allocation value\n",
        "df=data\n",
        "total_rows = len(df)\n",
        "allocation_1_count = int(0.8 * total_rows)\n",
        "allocation_0_count = total_rows - allocation_1_count\n",
        "\n",
        "# Create an array of 80% ones and 20% zeros\n",
        "allocation_values = np.concatenate((np.ones(allocation_1_count), np.zeros(allocation_0_count)))\n",
        "\n",
        "# Shuffle the allocation values randomly\n",
        "np.random.shuffle(allocation_values)\n",
        "\n",
        "# Assign the allocation values to the 'allocation' column in the DataFrame\n",
        "df['allocate'] = allocation_values.astype(int)"
      ],
      "metadata": {
        "id": "YBqlfXLaxfdo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#allocation\n",
        "df['gold_label']=df['Class Index']\n",
        "world=[]\n",
        "sports=[]\n",
        "business=[]\n",
        "sci=[]\n",
        "chatgpt_count=0\n",
        "for index, row in df.iterrows():\n",
        "    if row['allocate']==1: #edit the threshold here: I tried <-0.2(all human), ==0, <0.2, <0.4, <0.6, <0.8, <1(all chatgpt)\n",
        "      world.append(row['MV_world'])\n",
        "      sports.append(row['MV_sports'])\n",
        "      business.append(row['MV_business'])\n",
        "      sci.append(row['MV_sci'])\n",
        "    else:\n",
        "      if row['gold_label']==1:\n",
        "        world.append(1)\n",
        "        sports.append(0)\n",
        "        business.append(0)\n",
        "        sci.append(0)\n",
        "      elif row['gold_label']==2:\n",
        "        world.append(0)\n",
        "        sports.append(1)\n",
        "        business.append(0)\n",
        "        sci.append(0)\n",
        "      elif row['gold_label']==3:\n",
        "        world.append(0)\n",
        "        sports.append(0)\n",
        "        business.append(1)\n",
        "        sci.append(0)\n",
        "      elif row['gold_label']==4:\n",
        "        world.append(0)\n",
        "        sports.append(0)\n",
        "        business.append(0)\n",
        "        sci.append(1)\n",
        "df['world']=world\n",
        "df['sports']=sports\n",
        "df['business']=business\n",
        "df['sci']=sci\n",
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 699
        },
        "id": "WxS4YXfA730N",
        "outputId": "e7f96750-243b-4b8b-be34-0712ab179135"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Class Index                                               text  \\\n",
              "0            3  Title: Wall St. Bears Claw Back Into the Black...   \n",
              "1            3  Title: Carlyle Looks Toward Commercial Aerospa...   \n",
              "2            3  Title: Oil and Economy Cloud Stocks' Outlook (...   \n",
              "3            3  Title: Iraq Halts Oil Exports from Main Southe...   \n",
              "4            3  Title: Oil prices soar to all-time record, pos...   \n",
              "\n",
              "            label1    label2                                         label3  \\\n",
              "0  topic: business  business                                topic: business   \n",
              "1         business  business                                topic: business   \n",
              "2  topic: business  business  the topic of this news article is \"business\".   \n",
              "3         business  business                                       business   \n",
              "4         business  business                                topic: business   \n",
              "\n",
              "  label4                                             label5 label6  \\\n",
              "0  true.                                              FALSE  true.   \n",
              "1  true.                                             false.   TRUE   \n",
              "2   TRUE                                             false.  true.   \n",
              "3   TRUE  false. the news title and description belong t...  true.   \n",
              "4  true.                                              FALSE   TRUE   \n",
              "\n",
              "                                              label7  \\\n",
              "0  false. this news title and description belongs...   \n",
              "1                                              FALSE   \n",
              "2                                              FALSE   \n",
              "3                                               TRUE   \n",
              "4                                               TRUE   \n",
              "\n",
              "                     label8  ... entropy_sci   entropy allocation gold_label  \\\n",
              "0                  business  ...    0.000000  0.215780          1          3   \n",
              "1                  business  ...    0.000000  0.215780          1          3   \n",
              "2  the topic is \"business\".  ...    0.000000  0.215780          1          3   \n",
              "3  the topic is \"business\".  ...    0.591673  0.363698          1          3   \n",
              "4                  business  ...    0.591673  0.363698          1          3   \n",
              "\n",
              "  world  sports  business  sci  allocate  one_hot_labels  \n",
              "0     0       0         1    0         1    [0, 0, 1, 0]  \n",
              "1     0       0         1    0         1    [0, 0, 1, 0]  \n",
              "2     0       0         1    0         1    [0, 0, 1, 0]  \n",
              "3     0       0         1    0         1    [0, 0, 1, 0]  \n",
              "4     0       0         1    0         1    [0, 0, 1, 0]  \n",
              "\n",
              "[5 rows x 62 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-44b7ad65-0a5a-4234-a1b9-28b857663086\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Class Index</th>\n",
              "      <th>text</th>\n",
              "      <th>label1</th>\n",
              "      <th>label2</th>\n",
              "      <th>label3</th>\n",
              "      <th>label4</th>\n",
              "      <th>label5</th>\n",
              "      <th>label6</th>\n",
              "      <th>label7</th>\n",
              "      <th>label8</th>\n",
              "      <th>...</th>\n",
              "      <th>entropy_sci</th>\n",
              "      <th>entropy</th>\n",
              "      <th>allocation</th>\n",
              "      <th>gold_label</th>\n",
              "      <th>world</th>\n",
              "      <th>sports</th>\n",
              "      <th>business</th>\n",
              "      <th>sci</th>\n",
              "      <th>allocate</th>\n",
              "      <th>one_hot_labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Wall St. Bears Claw Back Into the Black...</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>business</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>true.</td>\n",
              "      <td>FALSE</td>\n",
              "      <td>true.</td>\n",
              "      <td>false. this news title and description belongs...</td>\n",
              "      <td>business</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.215780</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 1, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Carlyle Looks Toward Commercial Aerospa...</td>\n",
              "      <td>business</td>\n",
              "      <td>business</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>true.</td>\n",
              "      <td>false.</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>FALSE</td>\n",
              "      <td>business</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.215780</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 1, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Oil and Economy Cloud Stocks' Outlook (...</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>business</td>\n",
              "      <td>the topic of this news article is \"business\".</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>false.</td>\n",
              "      <td>true.</td>\n",
              "      <td>FALSE</td>\n",
              "      <td>the topic is \"business\".</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.215780</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 1, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Iraq Halts Oil Exports from Main Southe...</td>\n",
              "      <td>business</td>\n",
              "      <td>business</td>\n",
              "      <td>business</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>false. the news title and description belong t...</td>\n",
              "      <td>true.</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>the topic is \"business\".</td>\n",
              "      <td>...</td>\n",
              "      <td>0.591673</td>\n",
              "      <td>0.363698</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 1, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Oil prices soar to all-time record, pos...</td>\n",
              "      <td>business</td>\n",
              "      <td>business</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>true.</td>\n",
              "      <td>FALSE</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>business</td>\n",
              "      <td>...</td>\n",
              "      <td>0.591673</td>\n",
              "      <td>0.363698</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 1, 0]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 62 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-44b7ad65-0a5a-4234-a1b9-28b857663086')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-44b7ad65-0a5a-4234-a1b9-28b857663086 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-44b7ad65-0a5a-4234-a1b9-28b857663086');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 97
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dLcetMjZFjSH"
      },
      "source": [
        "## Load and Preprocess Training Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2dal0ggBcYdD"
      },
      "source": [
        "Dataset will be tokenized then split into training and validation sets. The validation set will be used to monitor training. For testing a separate test set will be loaded for analysis."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0ecREc7GnISW",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "#df = data #jigsaw-toxic-comment-classification-challenge"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UVI59S9VaAfB",
        "outputId": "2f89be17-0ea4-4a0b-90b0-553919e0a6d8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Label columns:  ['world', 'sports', 'business', 'sci']\n"
          ]
        }
      ],
      "source": [
        "#combined\n",
        "cols = df.columns\n",
        "label_cols = list(cols[56:60])\n",
        "num_labels = len(label_cols)\n",
        "print('Label columns: ', label_cols)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xzgA5qQgYIBZ",
        "outputId": "70677f7f-c2ef-44db-a195-cab8d3424f8b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Count of 1 per label: \n",
            " world       266\n",
            "sports      266\n",
            "business    389\n",
            "sci         240\n",
            "dtype: int64 \n",
            "\n",
            "Count of 0 per label: \n",
            " world       888\n",
            "sports      888\n",
            "business    765\n",
            "sci         914\n",
            "dtype: int64\n"
          ]
        }
      ],
      "source": [
        "print('Count of 1 per label: \\n', df[label_cols].sum(), '\\n') # Label counts, may need to downsample or upsample\n",
        "print('Count of 0 per label: \\n', df[label_cols].eq(0).sum())"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df=df.drop(423)"
      ],
      "metadata": {
        "id": "BQQQ30bQmYt-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uFpSd4JzaAae"
      },
      "outputs": [],
      "source": [
        "#df = df.sample(frac=1).reset_index(drop=True) #shuffle rows"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 699
        },
        "id": "0DF3ddjej5vd",
        "outputId": "e8e5e8cf-89df-4b25-a0c2-cdcc96a30c1a"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Class Index                                               text  \\\n",
              "0            3  Title: Wall St. Bears Claw Back Into the Black...   \n",
              "1            3  Title: Carlyle Looks Toward Commercial Aerospa...   \n",
              "2            3  Title: Oil and Economy Cloud Stocks' Outlook (...   \n",
              "3            3  Title: Iraq Halts Oil Exports from Main Southe...   \n",
              "4            3  Title: Oil prices soar to all-time record, pos...   \n",
              "\n",
              "            label1    label2                                         label3  \\\n",
              "0  topic: business  business                                topic: business   \n",
              "1         business  business                                topic: business   \n",
              "2  topic: business  business  the topic of this news article is \"business\".   \n",
              "3         business  business                                       business   \n",
              "4         business  business                                topic: business   \n",
              "\n",
              "  label4                                             label5 label6  \\\n",
              "0  true.                                              FALSE  true.   \n",
              "1  true.                                             false.   TRUE   \n",
              "2   TRUE                                             false.  true.   \n",
              "3   TRUE  false. the news title and description belong t...  true.   \n",
              "4  true.                                              FALSE   TRUE   \n",
              "\n",
              "                                              label7  \\\n",
              "0  false. this news title and description belongs...   \n",
              "1                                              FALSE   \n",
              "2                                              FALSE   \n",
              "3                                               TRUE   \n",
              "4                                               TRUE   \n",
              "\n",
              "                     label8  ... entropy_sci   entropy allocation gold_label  \\\n",
              "0                  business  ...    0.000000  0.215780          1          3   \n",
              "1                  business  ...    0.000000  0.215780          1          3   \n",
              "2  the topic is \"business\".  ...    0.000000  0.215780          1          3   \n",
              "3  the topic is \"business\".  ...    0.591673  0.363698          1          3   \n",
              "4                  business  ...    0.591673  0.363698          1          3   \n",
              "\n",
              "  world  sports  business  sci  allocate  one_hot_labels  \n",
              "0     0       0         1    0         1    [0, 0, 1, 0]  \n",
              "1     0       0         1    0         1    [0, 0, 1, 0]  \n",
              "2     0       0         1    0         1    [0, 0, 1, 0]  \n",
              "3     0       0         1    0         1    [0, 0, 1, 0]  \n",
              "4     0       0         1    0         1    [0, 0, 1, 0]  \n",
              "\n",
              "[5 rows x 62 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f5eabbfd-a3c0-43f1-96fa-b251fb0cdbad\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Class Index</th>\n",
              "      <th>text</th>\n",
              "      <th>label1</th>\n",
              "      <th>label2</th>\n",
              "      <th>label3</th>\n",
              "      <th>label4</th>\n",
              "      <th>label5</th>\n",
              "      <th>label6</th>\n",
              "      <th>label7</th>\n",
              "      <th>label8</th>\n",
              "      <th>...</th>\n",
              "      <th>entropy_sci</th>\n",
              "      <th>entropy</th>\n",
              "      <th>allocation</th>\n",
              "      <th>gold_label</th>\n",
              "      <th>world</th>\n",
              "      <th>sports</th>\n",
              "      <th>business</th>\n",
              "      <th>sci</th>\n",
              "      <th>allocate</th>\n",
              "      <th>one_hot_labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Wall St. Bears Claw Back Into the Black...</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>business</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>true.</td>\n",
              "      <td>FALSE</td>\n",
              "      <td>true.</td>\n",
              "      <td>false. this news title and description belongs...</td>\n",
              "      <td>business</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.215780</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 1, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Carlyle Looks Toward Commercial Aerospa...</td>\n",
              "      <td>business</td>\n",
              "      <td>business</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>true.</td>\n",
              "      <td>false.</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>FALSE</td>\n",
              "      <td>business</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.215780</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 1, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Oil and Economy Cloud Stocks' Outlook (...</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>business</td>\n",
              "      <td>the topic of this news article is \"business\".</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>false.</td>\n",
              "      <td>true.</td>\n",
              "      <td>FALSE</td>\n",
              "      <td>the topic is \"business\".</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.215780</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 1, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Iraq Halts Oil Exports from Main Southe...</td>\n",
              "      <td>business</td>\n",
              "      <td>business</td>\n",
              "      <td>business</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>false. the news title and description belong t...</td>\n",
              "      <td>true.</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>the topic is \"business\".</td>\n",
              "      <td>...</td>\n",
              "      <td>0.591673</td>\n",
              "      <td>0.363698</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 1, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Oil prices soar to all-time record, pos...</td>\n",
              "      <td>business</td>\n",
              "      <td>business</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>true.</td>\n",
              "      <td>FALSE</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>business</td>\n",
              "      <td>...</td>\n",
              "      <td>0.591673</td>\n",
              "      <td>0.363698</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 1, 0]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 62 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f5eabbfd-a3c0-43f1-96fa-b251fb0cdbad')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f5eabbfd-a3c0-43f1-96fa-b251fb0cdbad button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f5eabbfd-a3c0-43f1-96fa-b251fb0cdbad');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 106
        }
      ],
      "source": [
        "df['one_hot_labels'] = list(df[label_cols].values)\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MlhHifh5bW7e"
      },
      "outputs": [],
      "source": [
        "labels = list(df.one_hot_labels.values)\n",
        "comments = list(df.text.values)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IlMHfElhGJzc"
      },
      "source": [
        "Load the pretrained tokenizer that corresponds to your choice in model. e.g.,\n",
        "\n",
        "```\n",
        "BERT:\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
        "\n",
        "XLNet:\n",
        "tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased', do_lower_case=False)\n",
        "\n",
        "RoBERTa:\n",
        "tokenizer = RobertaTokenizer.from_pretrained('roberta-base', do_lower_case=False)\n",
        "```\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IhVr8SziL_PY"
      },
      "source": [
        "In order to avoid memory issues with Google Colab, I enforce a max_length of 100 tokens. Note that some sentences may not adequately represent each label because of this."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HNsEu-vUur-4",
        "outputId": "838eee9c-59ad-44ae-de77-ca361e8c754e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2377: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tokenizer outputs:  dict_keys(['input_ids', 'attention_mask'])\n"
          ]
        }
      ],
      "source": [
        "from transformers import RobertaTokenizer\n",
        "max_length = 100\n",
        "tokenizer = RobertaTokenizer.from_pretrained('roberta-base', do_lower_case=False) # tokenizer\n",
        "encodings = tokenizer.batch_encode_plus(comments,max_length=max_length,pad_to_max_length=True) # tokenizer's encoding method\n",
        "print('tokenizer outputs: ', encodings.keys())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nG9vOTp92yZE",
        "outputId": "4258abef-382f-4559-96f3-e04a9869a03d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "dict_keys(['input_ids', 'attention_mask'])"
            ]
          },
          "execution_count": 62,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "encodings.keys()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "l6CCLSjfur-9"
      },
      "outputs": [],
      "source": [
        "input_ids = encodings['input_ids'] # tokenized and encoded sentences\n",
        "#token_type_ids = encodings['token_type_ids'] # token type ids\n",
        "attention_masks = encodings['attention_mask'] # attention masks"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vSOFbThlYcpb",
        "outputId": "aaa693d4-9c35-4bf3-fa4b-8e6772159a5f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "df label indices with only one instance:  []\n"
          ]
        }
      ],
      "source": [
        "# Identifying indices of 'one_hot_labels' entries that only occur once - this will allow us to stratify split our training data later\n",
        "label_counts = df.one_hot_labels.astype(str).value_counts()\n",
        "one_freq = label_counts[label_counts==1].keys()\n",
        "one_freq_idxs = sorted(list(df[df.one_hot_labels.astype(str).isin(one_freq)].index), reverse=True)\n",
        "print('df label indices with only one instance: ', one_freq_idxs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CQQ7CoOag_r7"
      },
      "outputs": [],
      "source": [
        "# Gathering single instance inputs to force into the training set after stratified split\n",
        "one_freq_input_ids = [input_ids.pop(i) for i in one_freq_idxs]\n",
        "#one_freq_token_types = [token_type_ids.pop(i) for i in one_freq_idxs]\n",
        "one_freq_attention_masks = [attention_masks.pop(i) for i in one_freq_idxs]\n",
        "one_freq_labels = [labels.pop(i) for i in one_freq_idxs]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r9PxAt48HRRj"
      },
      "source": [
        "Be sure to handle all classes during validation using \"stratify\" during train/validation split:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WPFaq4ufnIT2"
      },
      "outputs": [],
      "source": [
        "# Use train_test_split to split our data into train and validation sets\n",
        "\n",
        "train_inputs, validation_inputs, train_labels, validation_labels, train_masks, validation_masks = train_test_split(input_ids, labels, attention_masks,\n",
        "                                                            random_state=20, test_size=0.10, stratify = labels)\n",
        "\n",
        "# Add one frequency data to train data\n",
        "train_inputs.extend(one_freq_input_ids)\n",
        "train_labels.extend(one_freq_labels)\n",
        "train_masks.extend(one_freq_attention_masks)\n",
        "#train_token_types.extend(one_freq_token_types)\n",
        "\n",
        "# Convert all of our data into torch tensors, the required datatype for our model\n",
        "train_inputs = torch.tensor(train_inputs)\n",
        "train_labels = torch.tensor(train_labels)\n",
        "train_masks = torch.tensor(train_masks)\n",
        "#train_token_types = torch.tensor(train_token_types)\n",
        "\n",
        "validation_inputs = torch.tensor(validation_inputs)\n",
        "validation_labels = torch.tensor(validation_labels)\n",
        "validation_masks = torch.tensor(validation_masks)\n",
        "#validation_token_types = torch.tensor(validation_token_types)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZRnuLna-nIT4"
      },
      "outputs": [],
      "source": [
        "# Select a batch size for training. For fine-tuning with XLNet, the authors recommend a batch size of 32, 48, or 128. We will use 32 here to avoid memory issues.\n",
        "batch_size = 16\n",
        "\n",
        "# Create an iterator of our data with torch DataLoader. This helps save on memory during training because, unlike a for loop,\n",
        "# with an iterator the entire dataset does not need to be loaded into memory\n",
        "\n",
        "train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
        "train_sampler = RandomSampler(train_data)\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)\n",
        "validation_sampler = SequentialSampler(validation_data)\n",
        "validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iiFRnP_ZTBFa"
      },
      "outputs": [],
      "source": [
        "torch.save(validation_dataloader,'validation_data_loader')\n",
        "torch.save(train_dataloader,'train_data_loader')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ncGteBuSFuZM"
      },
      "source": [
        "## Load Model & Set Params"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z0dL-Bz_NrGj"
      },
      "source": [
        "Load the appropriate model below, each model already contains a single dense layer for classification on top.\n",
        "\n",
        "\n",
        "\n",
        "```\n",
        "BERT:\n",
        "model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\", num_labels=num_labels)\n",
        "\n",
        "XLNet:\n",
        "model = XLNetForSequenceClassification.from_pretrained(\"xlnet-base-cased\", num_labels=num_labels)\n",
        "\n",
        "RoBERTa:\n",
        "model = RobertaForSequenceClassification.from_pretrained('roberta-base', num_labels=num_labels)\n",
        "```\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ujk4k16DnIT6",
        "outputId": "ccceec98-c38c-4821-85c1-3b52195f02f3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'lm_head.layer_norm.weight', 'lm_head.bias', 'lm_head.layer_norm.bias', 'lm_head.dense.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.weight', 'classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RobertaForSequenceClassification(\n",
              "  (roberta): RobertaModel(\n",
              "    (embeddings): RobertaEmbeddings(\n",
              "      (word_embeddings): Embedding(50265, 768, padding_idx=1)\n",
              "      (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
              "      (token_type_embeddings): Embedding(1, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): RobertaEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0-11): 12 x RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (classifier): RobertaClassificationHead(\n",
              "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "    (out_proj): Linear(in_features=768, out_features=4, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 115
        }
      ],
      "source": [
        "# Load model, the pretrained model will include a single linear classification layer on top for classification.\n",
        "from transformers import RobertaForSequenceClassification\n",
        "model = RobertaForSequenceClassification.from_pretrained('roberta-base', num_labels=num_labels)\n",
        "model.cuda()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jGE4gv9qfhRG"
      },
      "source": [
        "Setting custom optimization parameters for the AdamW optimizer https://huggingface.co/transformers/main_classes/optimizer_schedules.html"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GsV8zwWYnIT9"
      },
      "outputs": [],
      "source": [
        "# setting custom optimization parameters. You may implement a scheduler here as well.\n",
        "param_optimizer = list(model.named_parameters())\n",
        "no_decay = ['bias', 'gamma', 'beta']\n",
        "optimizer_grouped_parameters = [\n",
        "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
        "     'weight_decay_rate': 0.01},\n",
        "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
        "     'weight_decay_rate': 0.0}\n",
        "]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aOomZIEIoHOL",
        "outputId": "dde4215a-5da9-4187-bd33-28b124171f70"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "from transformers import AdamW\n",
        "optimizer = AdamW(optimizer_grouped_parameters,lr=2e-5 ,correct_bias=True)\n",
        "# optimizer = AdamW(model.parameters(),lr=2e-5)  # Default optimization"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JRQQZ8zIFzLW"
      },
      "source": [
        "## Train Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ho6xx8cQfhvK"
      },
      "outputs": [],
      "source": [
        "def is_max(array):\n",
        "  max_val=max(array)\n",
        "  res=[i==max_val for i in array]\n",
        "  return res"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Store our loss and accuracy for plotting\n",
        "train_loss_set = []\n",
        "\n",
        "# Number of training epochs (authors recommend between 2 and 4)\n",
        "epochs = 10\n",
        "\n",
        "# trange is a tqdm wrapper around the normal python range\n",
        "for _ in trange(epochs, desc=\"Epoch\"):\n",
        "\n",
        "  # Training\n",
        "\n",
        "  # Set our model to training mode (as opposed to evaluation mode)\n",
        "  model.train()\n",
        "\n",
        "  # Tracking variables\n",
        "  tr_loss = 0 #running loss\n",
        "  nb_tr_examples, nb_tr_steps = 0, 0\n",
        "\n",
        "  # Train the data for one epoch\n",
        "  for step, batch in enumerate(train_dataloader):\n",
        "    # Add batch to GPU\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    # Unpack the inputs from our dataloader\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    # Clear out the gradients (by default they accumulate)\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    # # Forward pass for multiclass classification\n",
        "    # outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n",
        "    # loss = outputs[0]\n",
        "    # logits = outputs[1]\n",
        "\n",
        "    # Forward pass for multilabel classification\n",
        "    outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "    logits = outputs[0]\n",
        "    loss_func = BCEWithLogitsLoss()\n",
        "    loss = loss_func(logits.view(-1,num_labels),b_labels.type_as(logits).view(-1,num_labels)) #convert labels to float for calculation\n",
        "    # loss_func = BCELoss()\n",
        "    # loss = loss_func(torch.sigmoid(logits.view(-1,num_labels)),b_labels.type_as(logits).view(-1,num_labels)) #convert labels to float for calculation\n",
        "    train_loss_set.append(loss.item())\n",
        "\n",
        "    # Backward pass\n",
        "    loss.backward()\n",
        "    # Update parameters and take a step using the computed gradient\n",
        "    optimizer.step()\n",
        "    # scheduler.step()\n",
        "    # Update tracking variables\n",
        "    tr_loss += loss.item()\n",
        "    nb_tr_examples += b_input_ids.size(0)\n",
        "    nb_tr_steps += 1\n",
        "\n",
        "  print(\"Train loss: {}\".format(tr_loss/nb_tr_steps))\n",
        "\n",
        "###############################################################################\n",
        "\n",
        "  # Validation\n",
        "\n",
        "  # Put model in evaluation mode to evaluate loss on the validation set\n",
        "  model.eval()\n",
        "\n",
        "  # Variables to gather full output\n",
        "  logit_preds,true_labels,pred_labels,tokenized_texts = [],[],[],[]\n",
        "\n",
        "  # Predict\n",
        "  for i, batch in enumerate(validation_dataloader):\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    # Unpack the inputs from our dataloader\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    with torch.no_grad():\n",
        "      # Forward pass\n",
        "      outs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "      b_logit_pred = outs[0]\n",
        "      pred_label = torch.sigmoid(b_logit_pred)\n",
        "\n",
        "      b_logit_pred = b_logit_pred.detach().cpu().numpy()\n",
        "      pred_label = pred_label.to('cpu').numpy()\n",
        "      b_labels = b_labels.to('cpu').numpy()\n",
        "\n",
        "    tokenized_texts.append(b_input_ids)\n",
        "    logit_preds.append(b_logit_pred)\n",
        "    true_labels.append(b_labels)\n",
        "    pred_labels.append(pred_label)\n",
        "\n",
        "  # Flatten outputs\n",
        "  pred_labels = [item for sublist in pred_labels for item in sublist]\n",
        "  true_labels = [item for sublist in true_labels for item in sublist]\n",
        "\n",
        "  # Calculate Accuracy\n",
        "  #threshold = 0.30\n",
        "  #pred_bools = [pl>threshold for pl in pred_labels]\n",
        "  #print(pred_labels)\n",
        "  #max_val = max(pred_labels)\n",
        "  pred_bools = [is_max(arr) for arr in pred_labels]\n",
        "  true_bools = [tl==1 for tl in true_labels]\n",
        "  val_f1_accuracy = f1_score(true_bools,pred_bools,average='micro')*100\n",
        "  val_flat_accuracy = accuracy_score(true_bools, pred_bools)*100\n",
        "\n",
        "  print('F1 Validation Accuracy: ', val_f1_accuracy)\n",
        "  print('Flat Validation Accuracy: ', val_flat_accuracy)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V2x2N2D3iVAD",
        "outputId": "76e08124-c758-4cdc-ee4f-07612bdd26b5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:   0%|          | 0/10 [00:00<?, ?it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train loss: 0.4266913129733159\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  10%|█         | 1/10 [00:18<02:49, 18.85s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  88.79310344827587\n",
            "Flat Validation Accuracy:  88.79310344827587\n",
            "Train loss: 0.1557770162820816\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  20%|██        | 2/10 [00:38<02:34, 19.29s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  91.37931034482759\n",
            "Flat Validation Accuracy:  91.37931034482759\n",
            "Train loss: 0.10357528833242563\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  30%|███       | 3/10 [00:58<02:16, 19.50s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  93.10344827586206\n",
            "Flat Validation Accuracy:  93.10344827586206\n",
            "Train loss: 0.07853862540079998\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  40%|████      | 4/10 [01:17<01:56, 19.35s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  93.10344827586206\n",
            "Flat Validation Accuracy:  93.10344827586206\n",
            "Train loss: 0.05571835685807925\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  50%|█████     | 5/10 [01:36<01:36, 19.20s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  91.37931034482759\n",
            "Flat Validation Accuracy:  91.37931034482759\n",
            "Train loss: 0.05444328627334191\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  60%|██████    | 6/10 [01:55<01:16, 19.15s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  93.10344827586206\n",
            "Flat Validation Accuracy:  93.10344827586206\n",
            "Train loss: 0.04713241747365548\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  70%|███████   | 7/10 [02:14<00:57, 19.18s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  93.10344827586206\n",
            "Flat Validation Accuracy:  93.10344827586206\n",
            "Train loss: 0.03499100056405251\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  80%|████████  | 8/10 [02:33<00:38, 19.20s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  93.10344827586206\n",
            "Flat Validation Accuracy:  93.10344827586206\n",
            "Train loss: 0.02904101888147684\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  90%|█████████ | 9/10 [02:52<00:19, 19.18s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  93.10344827586206\n",
            "Flat Validation Accuracy:  93.10344827586206\n",
            "Train loss: 0.03903886830577483\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch: 100%|██████████| 10/10 [03:12<00:00, 19.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  91.37931034482759\n",
            "Flat Validation Accuracy:  91.37931034482759\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Store our loss and accuracy for plotting\n",
        "train_loss_set = []\n",
        "\n",
        "# Number of training epochs (authors recommend between 2 and 4)\n",
        "epochs = 5\n",
        "\n",
        "# trange is a tqdm wrapper around the normal python range\n",
        "for _ in trange(epochs, desc=\"Epoch\"):\n",
        "\n",
        "  # Training\n",
        "\n",
        "  # Set our model to training mode (as opposed to evaluation mode)\n",
        "  model.train()\n",
        "\n",
        "  # Tracking variables\n",
        "  tr_loss = 0 #running loss\n",
        "  nb_tr_examples, nb_tr_steps = 0, 0\n",
        "\n",
        "  # Train the data for one epoch\n",
        "  for step, batch in enumerate(train_dataloader):\n",
        "    # Add batch to GPU\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    # Unpack the inputs from our dataloader\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    # Clear out the gradients (by default they accumulate)\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    # # Forward pass for multiclass classification\n",
        "    # outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n",
        "    # loss = outputs[0]\n",
        "    # logits = outputs[1]\n",
        "\n",
        "    # Forward pass for multilabel classification\n",
        "    outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "    logits = outputs[0]\n",
        "    loss_func = BCEWithLogitsLoss()\n",
        "    loss = loss_func(logits.view(-1,num_labels),b_labels.type_as(logits).view(-1,num_labels)) #convert labels to float for calculation\n",
        "    # loss_func = BCELoss()\n",
        "    # loss = loss_func(torch.sigmoid(logits.view(-1,num_labels)),b_labels.type_as(logits).view(-1,num_labels)) #convert labels to float for calculation\n",
        "    train_loss_set.append(loss.item())\n",
        "\n",
        "    # Backward pass\n",
        "    loss.backward()\n",
        "    # Update parameters and take a step using the computed gradient\n",
        "    optimizer.step()\n",
        "    # scheduler.step()\n",
        "    # Update tracking variables\n",
        "    tr_loss += loss.item()\n",
        "    nb_tr_examples += b_input_ids.size(0)\n",
        "    nb_tr_steps += 1\n",
        "\n",
        "  print(\"Train loss: {}\".format(tr_loss/nb_tr_steps))\n",
        "\n",
        "###############################################################################\n",
        "\n",
        "  # Validation\n",
        "\n",
        "  # Put model in evaluation mode to evaluate loss on the validation set\n",
        "  model.eval()\n",
        "\n",
        "  # Variables to gather full output\n",
        "  logit_preds,true_labels,pred_labels,tokenized_texts = [],[],[],[]\n",
        "\n",
        "  # Predict\n",
        "  for i, batch in enumerate(validation_dataloader):\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    # Unpack the inputs from our dataloader\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    with torch.no_grad():\n",
        "      # Forward pass\n",
        "      outs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "      b_logit_pred = outs[0]\n",
        "      pred_label = torch.sigmoid(b_logit_pred)\n",
        "\n",
        "      b_logit_pred = b_logit_pred.detach().cpu().numpy()\n",
        "      pred_label = pred_label.to('cpu').numpy()\n",
        "      b_labels = b_labels.to('cpu').numpy()\n",
        "\n",
        "    tokenized_texts.append(b_input_ids)\n",
        "    logit_preds.append(b_logit_pred)\n",
        "    true_labels.append(b_labels)\n",
        "    pred_labels.append(pred_label)\n",
        "\n",
        "  # Flatten outputs\n",
        "  pred_labels = [item for sublist in pred_labels for item in sublist]\n",
        "  true_labels = [item for sublist in true_labels for item in sublist]\n",
        "\n",
        "  # Calculate Accuracy\n",
        "  #threshold = 0.30\n",
        "  #pred_bools = [pl>threshold for pl in pred_labels]\n",
        "  #print(pred_labels)\n",
        "  #max_val = max(pred_labels)\n",
        "  pred_bools = [is_max(arr) for arr in pred_labels]\n",
        "  true_bools = [tl==1 for tl in true_labels]\n",
        "  val_f1_accuracy = f1_score(true_bools,pred_bools,average='micro')*100\n",
        "  val_flat_accuracy = accuracy_score(true_bools, pred_bools)*100\n",
        "\n",
        "  print('F1 Validation Accuracy: ', val_f1_accuracy)\n",
        "  print('Flat Validation Accuracy: ', val_flat_accuracy)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PcqrFElwTMLl",
        "outputId": "d19ae88e-50ec-4a84-a402-0c68c5c1eea3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:   0%|          | 0/5 [00:00<?, ?it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train loss: 0.02341034564977655\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  20%|██        | 1/5 [00:19<01:16, 19.20s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  94.82758620689656\n",
            "Flat Validation Accuracy:  94.82758620689656\n",
            "Train loss: 0.010642064581266963\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  40%|████      | 2/5 [00:39<00:58, 19.64s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  91.37931034482759\n",
            "Flat Validation Accuracy:  91.37931034482759\n",
            "Train loss: 0.00914723970927298\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  60%|██████    | 3/5 [00:58<00:39, 19.62s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  93.96551724137932\n",
            "Flat Validation Accuracy:  93.96551724137932\n",
            "Train loss: 0.008067033307340283\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  80%|████████  | 4/5 [01:17<00:19, 19.36s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  92.24137931034483\n",
            "Flat Validation Accuracy:  92.24137931034483\n",
            "Train loss: 0.007938659345158017\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch: 100%|██████████| 5/5 [01:36<00:00, 19.31s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  91.37931034482759\n",
            "Flat Validation Accuracy:  91.37931034482759\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_7dd2GE3F4yK"
      },
      "source": [
        "## Load and Preprocess Test Data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 372
        },
        "id": "G5Q7hC4GFOLJ",
        "outputId": "891ad743-2049-494c-e2cf-84bce0c9825d"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     Class Index                                         Title  \\\n",
              "495            2                               Pressure points   \n",
              "496            3  Unions protest as overtime rules take effect   \n",
              "497            1              Serb denies siege terror charges   \n",
              "498            2                 11th-hour highlights too late   \n",
              "499            4                             No IE? No Can See   \n",
              "\n",
              "                                           Description  \\\n",
              "495  ATHENS -- The booing went on for nearly 10 min...   \n",
              "496  WASHINGTON -- Hundreds of workers rallied on t...   \n",
              "497  A Bosnian Serb general accused of organising t...   \n",
              "498  NBC's prime-time Olympic coverage is taped and...   \n",
              "499  One thing that #39;s always irritated those wh...   \n",
              "\n",
              "                                                  text  gold_label  world  \\\n",
              "495  Title: Pressure points Description: ATHENS -- ...           2      0   \n",
              "496  Title: Unions protest as overtime rules take e...           3      0   \n",
              "497  Title: Serb denies siege terror charges Descri...           1      1   \n",
              "498  Title: 11th-hour highlights too late Descripti...           2      0   \n",
              "499  Title: No IE? No Can See Description: One thin...           4      0   \n",
              "\n",
              "     sports  business  sci  \n",
              "495       1         0    0  \n",
              "496       0         1    0  \n",
              "497       0         0    0  \n",
              "498       1         0    0  \n",
              "499       0         0    1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0a92f923-ae61-422b-b3a9-37a2ffe32e1e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Class Index</th>\n",
              "      <th>Title</th>\n",
              "      <th>Description</th>\n",
              "      <th>text</th>\n",
              "      <th>gold_label</th>\n",
              "      <th>world</th>\n",
              "      <th>sports</th>\n",
              "      <th>business</th>\n",
              "      <th>sci</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>495</th>\n",
              "      <td>2</td>\n",
              "      <td>Pressure points</td>\n",
              "      <td>ATHENS -- The booing went on for nearly 10 min...</td>\n",
              "      <td>Title: Pressure points Description: ATHENS -- ...</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>496</th>\n",
              "      <td>3</td>\n",
              "      <td>Unions protest as overtime rules take effect</td>\n",
              "      <td>WASHINGTON -- Hundreds of workers rallied on t...</td>\n",
              "      <td>Title: Unions protest as overtime rules take e...</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>497</th>\n",
              "      <td>1</td>\n",
              "      <td>Serb denies siege terror charges</td>\n",
              "      <td>A Bosnian Serb general accused of organising t...</td>\n",
              "      <td>Title: Serb denies siege terror charges Descri...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>498</th>\n",
              "      <td>2</td>\n",
              "      <td>11th-hour highlights too late</td>\n",
              "      <td>NBC's prime-time Olympic coverage is taped and...</td>\n",
              "      <td>Title: 11th-hour highlights too late Descripti...</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499</th>\n",
              "      <td>4</td>\n",
              "      <td>No IE? No Can See</td>\n",
              "      <td>One thing that #39;s always irritated those wh...</td>\n",
              "      <td>Title: No IE? No Can See Description: One thin...</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0a92f923-ae61-422b-b3a9-37a2ffe32e1e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-0a92f923-ae61-422b-b3a9-37a2ffe32e1e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-0a92f923-ae61-422b-b3a9-37a2ffe32e1e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ],
      "source": [
        "test_df = pd.read_csv('agnews_test.csv')\n",
        "#test_labels_df = pd.read_csv('test_labels.csv')\n",
        "#test_df = test_df.merge(test_labels_df, on='id', how='left')\n",
        "# test_df['text']=test_df['sentence']\n",
        "test_df['world']=test_df['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
        "test_df['sports']=test_df['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
        "test_df['business']=test_df['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
        "test_df['sci']=test_df['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
        "#print('Null values: ', test_df.isnull().values.any()) #should not be any null sentences or labels\n",
        "#print('Same columns between train and test: ', label_cols == test_label_cols) #columns should be the same\n",
        "test_df.tail()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j_PZ1mVOhkK8",
        "outputId": "9b742ce9-ca1b-47de-d57c-adbc55910776"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['world', 'sports', 'business', 'sci']"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ],
      "source": [
        "test_label_cols = list(test_df.columns[5:])\n",
        "test_label_cols"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 441
        },
        "id": "77rjCrMGpYxz",
        "outputId": "af684016-0217-4f4b-8ae4-b0cefe6b7972"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Class Index                                              Title  \\\n",
              "0            3                  Fears for T N pension after talks   \n",
              "1            4  The Race is On: Second Private Team Sets Launc...   \n",
              "2            4      Ky. Company Wins Grant to Study Peptides (AP)   \n",
              "3            4      Prediction Unit Helps Forecast Wildfires (AP)   \n",
              "4            4        Calif. Aims to Limit Farm-Related Smog (AP)   \n",
              "\n",
              "                                         Description  \\\n",
              "0  Unions representing workers at Turner   Newall...   \n",
              "1  SPACE.com - TORONTO, Canada -- A second\\team o...   \n",
              "2  AP - A company founded by a chemistry research...   \n",
              "3  AP - It's barely dawn when Mike Fitzpatrick st...   \n",
              "4  AP - Southern California's smog-fighting agenc...   \n",
              "\n",
              "                                                text  gold_label  world  \\\n",
              "0  Title: Fears for T N pension after talks Descr...           3      0   \n",
              "1  Title: The Race is On: Second Private Team Set...           4      0   \n",
              "2  Title: Ky. Company Wins Grant to Study Peptide...           4      0   \n",
              "3  Title: Prediction Unit Helps Forecast Wildfire...           4      0   \n",
              "4  Title: Calif. Aims to Limit Farm-Related Smog ...           4      0   \n",
              "\n",
              "   sports  business  sci one_hot_labels  \n",
              "0       0         1    0   [0, 0, 1, 0]  \n",
              "1       0         0    1   [0, 0, 0, 1]  \n",
              "2       0         0    1   [0, 0, 0, 1]  \n",
              "3       0         0    1   [0, 0, 0, 1]  \n",
              "4       0         0    1   [0, 0, 0, 1]  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-de44c456-441e-4376-955d-ca61a574761e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Class Index</th>\n",
              "      <th>Title</th>\n",
              "      <th>Description</th>\n",
              "      <th>text</th>\n",
              "      <th>gold_label</th>\n",
              "      <th>world</th>\n",
              "      <th>sports</th>\n",
              "      <th>business</th>\n",
              "      <th>sci</th>\n",
              "      <th>one_hot_labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3</td>\n",
              "      <td>Fears for T N pension after talks</td>\n",
              "      <td>Unions representing workers at Turner   Newall...</td>\n",
              "      <td>Title: Fears for T N pension after talks Descr...</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[0, 0, 1, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>4</td>\n",
              "      <td>The Race is On: Second Private Team Sets Launc...</td>\n",
              "      <td>SPACE.com - TORONTO, Canada -- A second\\team o...</td>\n",
              "      <td>Title: The Race is On: Second Private Team Set...</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 0, 1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4</td>\n",
              "      <td>Ky. Company Wins Grant to Study Peptides (AP)</td>\n",
              "      <td>AP - A company founded by a chemistry research...</td>\n",
              "      <td>Title: Ky. Company Wins Grant to Study Peptide...</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 0, 1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>Prediction Unit Helps Forecast Wildfires (AP)</td>\n",
              "      <td>AP - It's barely dawn when Mike Fitzpatrick st...</td>\n",
              "      <td>Title: Prediction Unit Helps Forecast Wildfire...</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 0, 1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>Calif. Aims to Limit Farm-Related Smog (AP)</td>\n",
              "      <td>AP - Southern California's smog-fighting agenc...</td>\n",
              "      <td>Title: Calif. Aims to Limit Farm-Related Smog ...</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 0, 1]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-de44c456-441e-4376-955d-ca61a574761e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-de44c456-441e-4376-955d-ca61a574761e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-de44c456-441e-4376-955d-ca61a574761e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ],
      "source": [
        "test_df = test_df[~test_df[test_label_cols].eq(-1).any(axis=1)] #remove irrelevant rows/comments with -1 values\n",
        "test_df['one_hot_labels'] = list(test_df[test_label_cols].values)\n",
        "test_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1a41OmU2i7qp"
      },
      "outputs": [],
      "source": [
        "# Gathering input data\n",
        "test_labels = list(test_df.one_hot_labels.values)\n",
        "test_comments = list(test_df.text.values)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "amySMO8EQzf2",
        "outputId": "bbb75ef1-88ad-4283-944b-7d1f02948695"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2377: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "# Encoding input data\n",
        "test_encodings = tokenizer.batch_encode_plus(test_comments,max_length=max_length,pad_to_max_length=True)\n",
        "test_input_ids = test_encodings['input_ids']\n",
        "#test_token_type_ids = test_encodings['token_type_ids']\n",
        "test_attention_masks = test_encodings['attention_mask']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hqOfi9fkRaRN"
      },
      "outputs": [],
      "source": [
        "# Make tensors out of data\n",
        "test_inputs = torch.tensor(test_input_ids)\n",
        "test_labels = torch.tensor(test_labels)\n",
        "test_masks = torch.tensor(test_attention_masks)\n",
        "#test_token_types = torch.tensor(test_token_type_ids)\n",
        "# Create test dataloader\n",
        "test_data = TensorDataset(test_inputs, test_masks, test_labels)\n",
        "#test_data = TensorDataset(test_inputs, test_masks)\n",
        "test_sampler = SequentialSampler(test_data)\n",
        "test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=batch_size)\n",
        "# Save test dataloader\n",
        "torch.save(test_dataloader,'test_data_loader')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PFTWxCA_GBau"
      },
      "source": [
        "## Prediction and Metics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NPvrL6OFSQvf"
      },
      "outputs": [],
      "source": [
        "# Test\n",
        "\n",
        "# Put model in evaluation mode to evaluate loss on the validation set\n",
        "model.eval()\n",
        "\n",
        "#track variables\n",
        "logit_preds,true_labels,pred_labels,tokenized_texts = [],[],[],[]\n",
        "\n",
        "# Predict\n",
        "for i, batch in enumerate(test_dataloader):\n",
        "  batch = tuple(t.to(device) for t in batch)\n",
        "  # Unpack the inputs from our dataloader\n",
        "  b_input_ids, b_input_mask, b_labels = batch\n",
        "  #b_input_ids, b_input_mask = batch\n",
        "  with torch.no_grad():\n",
        "    # Forward pass\n",
        "    outs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "    b_logit_pred = outs[0]\n",
        "    pred_label = torch.sigmoid(b_logit_pred)\n",
        "\n",
        "    b_logit_pred = b_logit_pred.detach().cpu().numpy()\n",
        "    pred_label = pred_label.to('cpu').numpy()\n",
        "    b_labels = b_labels.to('cpu').numpy()\n",
        "\n",
        "  tokenized_texts.append(b_input_ids)\n",
        "  logit_preds.append(b_logit_pred)\n",
        "  true_labels.append(b_labels)\n",
        "  pred_labels.append(pred_label)\n",
        "\n",
        "# Flatten outputs\n",
        "tokenized_texts = [item for sublist in tokenized_texts for item in sublist]\n",
        "pred_labels = [item for sublist in pred_labels for item in sublist]\n",
        "true_labels = [item for sublist in true_labels for item in sublist]\n",
        "# Converting flattened binary values to boolean values\n",
        "true_bools = [tl==1 for tl in true_labels]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_INi5MSJtEkQ",
        "outputId": "70a67240-c004-43e8-f563-3a769c7df70e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[array([0.0045191 , 0.12284625, 0.78677535], dtype=float32),\n",
              " array([0.9635983 , 0.04758941, 0.00145089], dtype=float32),\n",
              " array([0.00207093, 0.01049966, 0.99107623], dtype=float32),\n",
              " array([0.01098563, 0.99250317, 0.00175095], dtype=float32),\n",
              " array([0.9887567 , 0.01733083, 0.00118418], dtype=float32),\n",
              " array([0.00506789, 0.00214649, 0.997235  ], dtype=float32),\n",
              " array([0.9978714 , 0.00287712, 0.00215554], dtype=float32),\n",
              " array([0.00341052, 0.996572  , 0.00284842], dtype=float32),\n",
              " array([0.9862281 , 0.00255858, 0.01361227], dtype=float32),\n",
              " array([0.10106141, 0.09945446, 0.10647077], dtype=float32)]"
            ]
          },
          "execution_count": 46,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "pred_labels[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dCiP1ZmTiewr",
        "outputId": "625ff3f1-db93-4b5a-b8ea-e3330f1e3e55"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[False, False, True]"
            ]
          },
          "execution_count": 47,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "is_max(pred_labels[0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GBxAD5B2ikLb",
        "outputId": "943787d7-47ee-4539-d427-b7833bce4250"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "10000"
            ]
          },
          "execution_count": 69,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(true_labels)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bQeGWqeMzAoZ"
      },
      "source": [
        "We need to threshold our sigmoid function outputs which range from [0, 1]. Below I use 0.50 as a threshold."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BZcZUcYOxxmM",
        "outputId": "3bc80000-b883-4485-ab80-012fb035e16f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test F1 Accuracy:  0.88\n",
            "Test Flat Accuracy:  0.88 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       world       0.85      0.92      0.88       122\n",
            "      sports       0.96      0.94      0.95       145\n",
            "    business       0.81      0.76      0.79       106\n",
            "         sci       0.88      0.87      0.87       127\n",
            "\n",
            "   micro avg       0.88      0.88      0.88       500\n",
            "   macro avg       0.87      0.87      0.87       500\n",
            "weighted avg       0.88      0.88      0.88       500\n",
            " samples avg       0.88      0.88      0.88       500\n",
            "\n"
          ]
        }
      ],
      "source": [
        "#human\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined - 0.2\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Macro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Micro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Weighted Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bg9C1KbQfux4",
        "outputId": "b277bfe4-c632-415f-9087-27582f51a543"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Macro Test F1 Accuracy:  0.8793896106883861\n",
            "Micro Test F1 Accuracy:  0.884\n",
            "Weighted Test F1 Accuracy:  0.8848087559052588\n",
            "Test Flat Accuracy:  0.884 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       world       0.93      0.84      0.88       122\n",
            "      sports       0.95      0.97      0.96       145\n",
            "    business       0.76      0.89      0.82       106\n",
            "         sci       0.89      0.83      0.86       127\n",
            "\n",
            "   micro avg       0.88      0.88      0.88       500\n",
            "   macro avg       0.88      0.88      0.88       500\n",
            "weighted avg       0.89      0.88      0.88       500\n",
            " samples avg       0.88      0.88      0.88       500\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined - 0.4\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Macro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Micro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Weighted Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oNjQDgiSjiEk",
        "outputId": "0fcf5702-bbc9-4e56-818c-c8a840d67f70"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Macro Test F1 Accuracy:  0.8583741668153195\n",
            "Micro Test F1 Accuracy:  0.8619999999999999\n",
            "Weighted Test F1 Accuracy:  0.8634604165404576\n",
            "Test Flat Accuracy:  0.862 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       world       0.84      0.89      0.86       122\n",
            "      sports       0.96      0.90      0.93       145\n",
            "    business       0.74      0.87      0.80       106\n",
            "         sci       0.91      0.79      0.84       127\n",
            "\n",
            "   micro avg       0.86      0.86      0.86       500\n",
            "   macro avg       0.86      0.86      0.86       500\n",
            "weighted avg       0.87      0.86      0.86       500\n",
            " samples avg       0.86      0.86      0.86       500\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined - 0.6\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Macro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Micro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Weighted Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rbrh1caKmj9A",
        "outputId": "d1a33f39-43d1-4cda-c62a-89253deef042"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Macro Test F1 Accuracy:  0.7980504439471048\n",
            "Micro Test F1 Accuracy:  0.8000000000000002\n",
            "Weighted Test F1 Accuracy:  0.8024709893378096\n",
            "Test Flat Accuracy:  0.8 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       world       0.70      0.89      0.78       122\n",
            "      sports       0.95      0.81      0.87       145\n",
            "    business       0.68      0.86      0.76       106\n",
            "         sci       0.94      0.66      0.78       127\n",
            "\n",
            "   micro avg       0.80      0.80      0.80       500\n",
            "   macro avg       0.82      0.80      0.80       500\n",
            "weighted avg       0.83      0.80      0.80       500\n",
            " samples avg       0.80      0.80      0.80       500\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined - 0.8\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Macro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Micro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Weighted Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "52TY6kfAxG6c",
        "outputId": "06085f1f-a4a7-416d-a969-d30bad6bd868"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Macro Test F1 Accuracy:  0.8179636996540369\n",
            "Micro Test F1 Accuracy:  0.826\n",
            "Weighted Test F1 Accuracy:  0.8254039052651132\n",
            "Test Flat Accuracy:  0.826 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       world       0.85      0.84      0.85       122\n",
            "      sports       0.96      0.94      0.95       145\n",
            "    business       0.63      0.92      0.75       106\n",
            "         sci       0.94      0.59      0.72       127\n",
            "\n",
            "   micro avg       0.83      0.83      0.83       500\n",
            "   macro avg       0.84      0.83      0.82       500\n",
            "weighted avg       0.86      0.83      0.83       500\n",
            " samples avg       0.83      0.83      0.83       500\n",
            "\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}