UINSUSKA_hasoc2023_svm_notebook.ipynb

{
  "metadata": {
    "kernelspec": {
      "language": "python",
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.10.12",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat_minor": 0,
  "nbformat": 4,
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# **Team name: UINSUSKA-Mandiri**\n",
        "\n",
        "Task 1A and 1B : Hate Speech classification in Sinhala and Gujarati"
      ],
      "metadata": {
        "id": "AJCXmY0GvRvh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# read data from folder\n",
        "\n",
        "import numpy as np\n",
        "import os\n",
        "import pandas as pd\n",
        "from tqdm.auto import tqdm\n",
        "from gensim.models import FastText\n",
        "from nltk.tokenize import word_tokenize\n",
        "import re\n",
        "\n",
        "path_data = 'hasoc2023-uin/'      # adjust the path to your data sources\n",
        "train_setA = \"train_sinhala.csv\"  # file name was changed to our standard convention name:\n",
        "test_setA = \"test_sinhala.csv\"    # train_<language>.csv and test_<language>.csv\n",
        "train_setB = \"train_gujarati.csv\"\n",
        "test_setB = \"test_gujarati.csv\"\n",
        "\n",
        "df_train = pd.read_csv(path_data+train_setA, sep='\\t')   #check the separator of csv file,\n",
        "df_test = pd.read_csv(path_data+test_setA, sep=',')      #there are differences among all files\n",
        "df_train"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:00:22.532890Z",
          "iopub.execute_input": "2023-10-26T01:00:22.533374Z",
          "iopub.status.idle": "2023-10-26T01:00:22.727326Z",
          "shell.execute_reply.started": "2023-10-26T01:00:22.533338Z",
          "shell.execute_reply": "2023-10-26T01:00:22.724613Z"
        },
        "trusted": true,
        "id": "keQaZVixuNM7",
        "outputId": "5b42665b-f010-4ecc-93fc-4ea295a14221"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 4,
          "output_type": "execute_result",
          "data": {
            "text/plain": "                  post_id                                               text  \\\n0      726758237668659201                         @USER @USER  පට්ට පට පට...   \n1      915618589855617026  පරණ කෑල්ල අද වෙනකම් හිටියනම් අදට අවුරුදු 4යි. ...   \n2      925001070430040065  යාළුවා කියලා හිතන් සර් ගේ ඔලුවට රෙද්ද දාලා නෙල...   \n3     1397219745707986955  හොඳ මිතුරියක් කතා කලා. විස්තර කතාකරමින් ඉදලා ම...   \n4      950376113150222336              ඔය බනින්නෙ.. හරකා, මී හරකා කිය කිය...   \n...                   ...                                                ...   \n7495   930270216612872193                    අද උදේ දැක්කා පට්ට ලස්සන හීනයක්   \n7496  1159471424613969921  @USER කසල වෙන් කරලා දෙන්න පුරුදු වුනොත්  වැඩේ ...   \n7497  1073271775583100928  ඒත් පබාගේ පස්ස නම්... ඉස්සෝ කොටුවක් උනත් දාන්න...   \n7498  1131018244733657088  සුදුවෑන්වලට මාරම මාර බයක් තියෙන ඈයෝ  අන්තවාදයට...   \n7499  1386977011198046209  1 වෙනි පෝස්ටුවෙන් පසු... මොකක්ද යකෝ ඔය දාල තිය...   \n\n     label  \n0      NOT  \n1      HOF  \n2      NOT  \n3      HOF  \n4      HOF  \n...    ...  \n7495   NOT  \n7496   NOT  \n7497   HOF  \n7498   HOF  \n7499   HOF  \n\n[7500 rows x 3 columns]",
            "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>post_id</th>\n      <th>text</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>726758237668659201</td>\n      <td>@USER @USER  පට්ට පට පට...</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>915618589855617026</td>\n      <td>පරණ කෑල්ල අද වෙනකම් හිටියනම් අදට අවුරුදු 4යි. ...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>925001070430040065</td>\n      <td>යාළුවා කියලා හිතන් සර් ගේ ඔලුවට රෙද්ද දාලා නෙල...</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1397219745707986955</td>\n      <td>හොඳ මිතුරියක් කතා කලා. විස්තර කතාකරමින් ඉදලා ම...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>950376113150222336</td>\n      <td>ඔය බනින්නෙ.. හරකා, මී හරකා කිය කිය...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>7495</th>\n      <td>930270216612872193</td>\n      <td>අද උදේ දැක්කා පට්ට ලස්සන හීනයක්</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>7496</th>\n      <td>1159471424613969921</td>\n      <td>@USER කසල වෙන් කරලා දෙන්න පුරුදු වුනොත්  වැඩේ ...</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>7497</th>\n      <td>1073271775583100928</td>\n      <td>ඒත් පබාගේ පස්ස නම්... ඉස්සෝ කොටුවක් උනත් දාන්න...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>7498</th>\n      <td>1131018244733657088</td>\n      <td>සුදුවෑන්වලට මාරම මාර බයක් තියෙන ඈයෝ  අන්තවාදයට...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>7499</th>\n      <td>1386977011198046209</td>\n      <td>1 වෙනි පෝස්ටුවෙන් පසු... මොකක්ද යකෝ ඔය දාල තිය...</td>\n      <td>HOF</td>\n    </tr>\n  </tbody>\n</table>\n<p>7500 rows × 3 columns</p>\n</div>"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# rename column, display data-test\n",
        "\n",
        "df_test = df_test.set_axis(['tweet_id', 'tweet'], axis='columns', copy=False)\n",
        "df_test"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:00:43.106767Z",
          "iopub.execute_input": "2023-10-26T01:00:43.107287Z",
          "iopub.status.idle": "2023-10-26T01:00:43.125035Z",
          "shell.execute_reply.started": "2023-10-26T01:00:43.107242Z",
          "shell.execute_reply": "2023-10-26T01:00:43.123842Z"
        },
        "trusted": true,
        "id": "6zRxIrwBuNM8",
        "outputId": "2612dc22-0d1b-494e-eb73-b131f3d666fd"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 5,
          "output_type": "execute_result",
          "data": {
            "text/plain": "              tweet_id                                              tweet\n0        task1a_test_1  ඔව් ඔව්.. තොපි ආදරේට කලොත් කරන්නෙ ඔහොම වැඩක් ත...\n1        task1a_test_2               ඇයි උඹ අලි පු*න්ද? #SriLankanThreats\n2        task1a_test_3                     @USER  @USER පක ටෝක් දෙන්න එපා\n3        task1a_test_4  මම නවය වසරේදී,පංති නායකයා කාලේ පංතියේ කොස්ස නැ...\n4        task1a_test_5  මේ ආන්ඩුවටනම් හෙන 7ම ගහන්න ඔනේ මුන්ට පේන්නෙ නැ...\n...                ...                                                ...\n2495  task1a_test_2496  රටක ආර්ථිකය කෙලෙසපු මොළකරු පට්ට හොරෙක් ! #SriL...\n2496  task1a_test_2497               @USER හරකා මොකා උනත් අවසානෙ මසට යාලු\n2497  task1a_test_2498  @USER තොපි වගේ කාළකන්නි මාධ්‍ය ආයතන තහනම් කරන ...\n2498  task1a_test_2499  රියැදුරු බලපත්‍ර නිකුත් කිරීම ඇතුළු සියලු කටයු...\n2499  task1a_test_2500  නොදකින් අම්මගෙන් බැනුම් අහගෙන පාඩම් නොකර මැච් ...\n\n[2500 rows x 2 columns]",
            "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>tweet_id</th>\n      <th>tweet</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>task1a_test_1</td>\n      <td>ඔව් ඔව්.. තොපි ආදරේට කලොත් කරන්නෙ ඔහොම වැඩක් ත...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>task1a_test_2</td>\n      <td>ඇයි උඹ අලි පු*න්ද? #SriLankanThreats</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>task1a_test_3</td>\n      <td>@USER  @USER පක ටෝක් දෙන්න එපා</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>task1a_test_4</td>\n      <td>මම නවය වසරේදී,පංති නායකයා කාලේ පංතියේ කොස්ස නැ...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>task1a_test_5</td>\n      <td>මේ ආන්ඩුවටනම් හෙන 7ම ගහන්න ඔනේ මුන්ට පේන්නෙ නැ...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2495</th>\n      <td>task1a_test_2496</td>\n      <td>රටක ආර්ථිකය කෙලෙසපු මොළකරු පට්ට හොරෙක් ! #SriL...</td>\n    </tr>\n    <tr>\n      <th>2496</th>\n      <td>task1a_test_2497</td>\n      <td>@USER හරකා මොකා උනත් අවසානෙ මසට යාලු</td>\n    </tr>\n    <tr>\n      <th>2497</th>\n      <td>task1a_test_2498</td>\n      <td>@USER තොපි වගේ කාළකන්නි මාධ්‍ය ආයතන තහනම් කරන ...</td>\n    </tr>\n    <tr>\n      <th>2498</th>\n      <td>task1a_test_2499</td>\n      <td>රියැදුරු බලපත්‍ර නිකුත් කිරීම ඇතුළු සියලු කටයු...</td>\n    </tr>\n    <tr>\n      <th>2499</th>\n      <td>task1a_test_2500</td>\n      <td>නොදකින් අම්මගෙන් බැනුම් අහගෙන පාඩම් නොකර මැච් ...</td>\n    </tr>\n  </tbody>\n</table>\n<p>2500 rows × 2 columns</p>\n</div>"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# display data-train\n",
        "\n",
        "df_train['label'].value_counts(normalize=False)\n"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:01:32.153524Z",
          "iopub.execute_input": "2023-10-26T01:01:32.154000Z",
          "iopub.status.idle": "2023-10-26T01:01:32.166492Z",
          "shell.execute_reply.started": "2023-10-26T01:01:32.153964Z",
          "shell.execute_reply": "2023-10-26T01:01:32.164123Z"
        },
        "trusted": true,
        "id": "J_ggDBdauNM9",
        "outputId": "9f2a8e49-bd87-447a-fd33-18b850e060bf"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 8,
          "output_type": "execute_result",
          "data": {
            "text/plain": "NOT    4324\nHOF    3176\nName: label, dtype: int64"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "df_train = df_train.set_axis(['tweet_id', 'tweet', 'label'], axis='columns', copy=False)   #standardize column name\n",
        "df_train"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:01:36.991594Z",
          "iopub.execute_input": "2023-10-26T01:01:36.992097Z",
          "iopub.status.idle": "2023-10-26T01:01:37.015650Z",
          "shell.execute_reply.started": "2023-10-26T01:01:36.992057Z",
          "shell.execute_reply": "2023-10-26T01:01:37.013288Z"
        },
        "trusted": true,
        "id": "ksOUn-SjuNM9",
        "outputId": "09b67fd3-bef9-44f8-9905-5f3920a7f256"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 9,
          "output_type": "execute_result",
          "data": {
            "text/plain": "                 tweet_id                                              tweet  \\\n0      726758237668659201                         @USER @USER  පට්ට පට පට...   \n1      915618589855617026  පරණ කෑල්ල අද වෙනකම් හිටියනම් අදට අවුරුදු 4යි. ...   \n2      925001070430040065  යාළුවා කියලා හිතන් සර් ගේ ඔලුවට රෙද්ද දාලා නෙල...   \n3     1397219745707986955  හොඳ මිතුරියක් කතා කලා. විස්තර කතාකරමින් ඉදලා ම...   \n4      950376113150222336              ඔය බනින්නෙ.. හරකා, මී හරකා කිය කිය...   \n...                   ...                                                ...   \n7495   930270216612872193                    අද උදේ දැක්කා පට්ට ලස්සන හීනයක්   \n7496  1159471424613969921  @USER කසල වෙන් කරලා දෙන්න පුරුදු වුනොත්  වැඩේ ...   \n7497  1073271775583100928  ඒත් පබාගේ පස්ස නම්... ඉස්සෝ කොටුවක් උනත් දාන්න...   \n7498  1131018244733657088  සුදුවෑන්වලට මාරම මාර බයක් තියෙන ඈයෝ  අන්තවාදයට...   \n7499  1386977011198046209  1 වෙනි පෝස්ටුවෙන් පසු... මොකක්ද යකෝ ඔය දාල තිය...   \n\n     label  \n0      NOT  \n1      HOF  \n2      NOT  \n3      HOF  \n4      HOF  \n...    ...  \n7495   NOT  \n7496   NOT  \n7497   HOF  \n7498   HOF  \n7499   HOF  \n\n[7500 rows x 3 columns]",
            "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>tweet_id</th>\n      <th>tweet</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>726758237668659201</td>\n      <td>@USER @USER  පට්ට පට පට...</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>915618589855617026</td>\n      <td>පරණ කෑල්ල අද වෙනකම් හිටියනම් අදට අවුරුදු 4යි. ...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>925001070430040065</td>\n      <td>යාළුවා කියලා හිතන් සර් ගේ ඔලුවට රෙද්ද දාලා නෙල...</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1397219745707986955</td>\n      <td>හොඳ මිතුරියක් කතා කලා. විස්තර කතාකරමින් ඉදලා ම...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>950376113150222336</td>\n      <td>ඔය බනින්නෙ.. හරකා, මී හරකා කිය කිය...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>7495</th>\n      <td>930270216612872193</td>\n      <td>අද උදේ දැක්කා පට්ට ලස්සන හීනයක්</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>7496</th>\n      <td>1159471424613969921</td>\n      <td>@USER කසල වෙන් කරලා දෙන්න පුරුදු වුනොත්  වැඩේ ...</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>7497</th>\n      <td>1073271775583100928</td>\n      <td>ඒත් පබාගේ පස්ස නම්... ඉස්සෝ කොටුවක් උනත් දාන්න...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>7498</th>\n      <td>1131018244733657088</td>\n      <td>සුදුවෑන්වලට මාරම මාර බයක් තියෙන ඈයෝ  අන්තවාදයට...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>7499</th>\n      <td>1386977011198046209</td>\n      <td>1 වෙනි පෝස්ටුවෙන් පසු... මොකක්ද යකෝ ඔය දාල තිය...</td>\n      <td>HOF</td>\n    </tr>\n  </tbody>\n</table>\n<p>7500 rows × 3 columns</p>\n</div>"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# creating data-dev for validation\n",
        "\n",
        "shuffled_train = df_train.sample(frac=1, random_state=23).reset_index()\n",
        "df_train1, df_dev1 = train_test_split(shuffled_train, test_size=0.1)\n",
        "df_train1"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:03:19.323524Z",
          "iopub.execute_input": "2023-10-26T01:03:19.324618Z",
          "iopub.status.idle": "2023-10-26T01:03:19.341728Z",
          "shell.execute_reply.started": "2023-10-26T01:03:19.324580Z",
          "shell.execute_reply": "2023-10-26T01:03:19.340510Z"
        },
        "trusted": true,
        "id": "wZIgRqatuNM-",
        "outputId": "1190cce7-9a91-40a1-bd81-958ed9652906"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 14,
          "output_type": "execute_result",
          "data": {
            "text/plain": "      index             tweet_id  \\\n6626   7327  1276189330923335681   \n1357   1790   778300947567026176   \n819    6050  1481279802829320198   \n6217    502   618419188156137472   \n1105   2974  1186283920842584064   \n...     ...                  ...   \n5958   1477   627511530980966400   \n3271   5403  1383455672403054594   \n6305   4779   693998649282576384   \n2344    275   798790313066102784   \n5919   4285   717334380784533504   \n\n                                                  tweet label  \n6626  අපි පොඩි කාලෙ අහපු \"ඇම්ඩගෙ කතා\" ඇත්තද එතකොට.? ...   NOT  \n1357  @USER  @USER ටහුඩු, හැපි බර්ත්ඩේ ටෝකා පුතේ. සො...   NOT  \n819                         @USER ඩ්‍රාමා එච්චර අමාරු ද   NOT  \n6217  හැමෝම තමුන්ගේ පාසලේ හිටපු පට්ට ඩෑල් ගැන වර්ණා ...   NOT  \n1105               @USER පහුගිය අවුරැදු හතර හමාරේ වගේ ,   NOT  \n...                                                 ...   ...  \n5958  මේ Max Dura මගුලක් නිසා හෙන වදේ බන්. ඇයි අර මු...   NOT  \n3271  අමුතුවෙන් හිතන්නත් දෙයක් මෙතන නැහැ... මේක තමයි...   HOF  \n6305  තම දියණියට ලිපියක් දුන් සිසුවා පැහැරගත් පියා ස...   NOT  \n2344  @USER හම්බන්තොටයි, මත්තලයි ගැන කතා කරන්න පුලුව...   NOT  \n5919  Wrestlemania එක කොච්චර චාටර් උනත් මේක ආයෙත් බල...   NOT  \n\n[6750 rows x 4 columns]",
            "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>index</th>\n      <th>tweet_id</th>\n      <th>tweet</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>6626</th>\n      <td>7327</td>\n      <td>1276189330923335681</td>\n      <td>අපි පොඩි කාලෙ අහපු \"ඇම්ඩගෙ කතා\" ඇත්තද එතකොට.? ...</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>1357</th>\n      <td>1790</td>\n      <td>778300947567026176</td>\n      <td>@USER  @USER ටහුඩු, හැපි බර්ත්ඩේ ටෝකා පුතේ. සො...</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>819</th>\n      <td>6050</td>\n      <td>1481279802829320198</td>\n      <td>@USER ඩ්‍රාමා එච්චර අමාරු ද</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>6217</th>\n      <td>502</td>\n      <td>618419188156137472</td>\n      <td>හැමෝම තමුන්ගේ පාසලේ හිටපු පට්ට ඩෑල් ගැන වර්ණා ...</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>1105</th>\n      <td>2974</td>\n      <td>1186283920842584064</td>\n      <td>@USER පහුගිය අවුරැදු හතර හමාරේ වගේ ,</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>5958</th>\n      <td>1477</td>\n      <td>627511530980966400</td>\n      <td>මේ Max Dura මගුලක් නිසා හෙන වදේ බන්. ඇයි අර මු...</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>3271</th>\n      <td>5403</td>\n      <td>1383455672403054594</td>\n      <td>අමුතුවෙන් හිතන්නත් දෙයක් මෙතන නැහැ... මේක තමයි...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>6305</th>\n      <td>4779</td>\n      <td>693998649282576384</td>\n      <td>තම දියණියට ලිපියක් දුන් සිසුවා පැහැරගත් පියා ස...</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>2344</th>\n      <td>275</td>\n      <td>798790313066102784</td>\n      <td>@USER හම්බන්තොටයි, මත්තලයි ගැන කතා කරන්න පුලුව...</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>5919</th>\n      <td>4285</td>\n      <td>717334380784533504</td>\n      <td>Wrestlemania එක කොච්චර චාටර් උනත් මේක ආයෙත් බල...</td>\n      <td>NOT</td>\n    </tr>\n  </tbody>\n</table>\n<p>6750 rows × 4 columns</p>\n</div>"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**List of Stopword**"
      ],
      "metadata": {
        "id": "8lVzDgqFuNM-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Stopwords in Sinhala\n",
        "# Stopwords only used in balancing the data_train\n",
        "# For Gujarati we don't remove stopwords\n",
        "\n",
        "stp=['සහ',\n",
        "'සමග',\n",
        "'සමඟ',\n",
        "'අහා',\n",
        "'ආහ්',\n",
        "'ආ',\n",
        "'ඕහෝ',\n",
        "'අනේ',\n",
        "'අඳෝ',\n",
        "'අපොයි',\n",
        "'අපෝ',\n",
        "'අයියෝ',\n",
        "'ආයි',\n",
        "'ඌයි',\n",
        "'චී',\n",
        "'චිහ්',\n",
        "'චික්',\n",
        "'හෝ‍',\n",
        "'දෝ',\n",
        "'දෝහෝ',\n",
        "'මෙන්',\n",
        "'සේ',\n",
        "'වැනි',\n",
        "'බඳු',\n",
        "'වන්',\n",
        "'අයුරු',\n",
        "'අයුරින්',\n",
        "'ලෙස',\n",
        "'වැඩි',\n",
        "'ශ්‍රී',\n",
        "'හා',\n",
        "'ය',\n",
        "'නිසා',\n",
        "'නිසාවෙන්',\n",
        "'බවට',\n",
        "'බව',\n",
        "'බවෙන්',\n",
        "'නම්',\n",
        "'වැඩි',\n",
        "'සිට',\n",
        "'දී',\n",
        "'මහා',\n",
        "'මහ',\n",
        "'පමණ',\n",
        "'පමණින්',\n",
        "'පමන',\n",
        "'වන',\n",
        "'විට',\n",
        "'විටින්',\n",
        "'මේ',\n",
        "'මෙලෙස',\n",
        "'මෙයින්',\n",
        "'ඇති',\n",
        "'ලෙස',\n",
        "'සිදු',\n",
        "'වශයෙන්',\n",
        "'යන',\n",
        "'සඳහා',\n",
        "'මගින්',\n",
        "'හෝ‍',\n",
        "'ඉතා',\n",
        "'ඒ',\n",
        "'එම',\n",
        "'ද',\n",
        "'අතර',\n",
        "'විසින්',\n",
        "'සමග',\n",
        "'පිළිබඳව',\n",
        "'පිළිබඳ',\n",
        "'තුළ',\n",
        "'බව',\n",
        "'වැනි',\n",
        "'මහ',\n",
        "'මෙම',\n",
        "'මෙහි',\n",
        "'මේ',\n",
        "'වෙත',\n",
        "'වෙතින්',\n",
        "'වෙතට',\n",
        "'වෙනුවෙන්',\n",
        "'වෙනුවට',\n",
        "'වෙන',\n",
        "'ගැන',\n",
        "'නෑ',\n",
        "'අනුව',\n",
        "'නව',\n",
        "'පිළිබඳ',\n",
        "'විශේෂ',\n",
        "'දැනට',\n",
        "'එහෙන්',\n",
        "'මෙහෙන්',\n",
        "'එහේ',\n",
        "'මෙහේ',\n",
        "'ම',\n",
        "'තවත්',\n",
        "'තව ',\n",
        "'සහ',\n",
        "'දක්වා',\n",
        "'ට',\n",
        "'ගේ',\n",
        "'එ',\n",
        "'ක',\n",
        "'ක්',\n",
        "'බවත්',\n",
        "'බවද',\n",
        "'මත',\n",
        "'ඇතුලු',\n",
        "'ඇතුළු',\n",
        "'මෙසේ',\n",
        "'වඩා',\n",
        "'වඩාත්ම',\n",
        "'නිති',\n",
        "'නිතිත්',\n",
        "'නිතොර',\n",
        "'නිතර',\n",
        "'ඉක්බිති',\n",
        "'දැන්',\n",
        "'යලි',\n",
        "'පුන',\n",
        "'ඉතින්',\n",
        "'සිට',\n",
        "'සිටන්',\n",
        "'පටන්',\n",
        "'තෙක්',\n",
        "'දක්වා',\n",
        "'සා',\n",
        "'තාක්',\n",
        "'තුවක්',\n",
        "'පවා',\n",
        "'ද',\n",
        "'හෝ‍',\n",
        "'වත්',\n",
        "'විනා',\n",
        "'හැර',\n",
        "'මිස',\n",
        "'මුත්',\n",
        "'කිම',\n",
        "'කිම්',\n",
        "'ඇයි',\n",
        "'මන්ද',\n",
        "'හෙවත්',\n",
        "'නොහොත්',\n",
        "'පතා',\n",
        "'පාසා',\n",
        "'ගානෙ',\n",
        "'තව',\n",
        "'ඉතා',\n",
        "'බොහෝ',\n",
        "'වහා',\n",
        "'සෙද',\n",
        "'සැනින්',\n",
        "'හනික',\n",
        "'එම්බා',\n",
        "'එම්බල',\n",
        "'බොල',\n",
        "'නම්',\n",
        "'වනාහි',\n",
        "'කලී',\n",
        "'ඉඳුරා',\n",
        "'අන්න',\n",
        "'ඔන්න',\n",
        "'මෙන්න',\n",
        "'උදෙසා',\n",
        "'පිණිස',\n",
        "'සඳහා',\n",
        "'අරබයා',\n",
        "'නිසා',\n",
        "'එනිසා',\n",
        "'එබැවින්',\n",
        "'බැවින්',\n",
        "'හෙයින්',\n",
        "'සේක්',\n",
        "'සේක',\n",
        "'ගැන',\n",
        "'අනුව',\n",
        "'පරිදි',\n",
        "'විට',\n",
        "'තෙක්',\n",
        "'මෙතෙක්',\n",
        "'මේතාක්',\n",
        "'තුරු',\n",
        "'තුරා',\n",
        "'තුරාවට',\n",
        "'තුලින්',\n",
        "'නමුත්',\n",
        "'එනමුත්',\n",
        "'වස්',\n",
        "'මෙන්',\n",
        "'ලෙස',\n",
        "'පරිදි',\n",
        "'එහෙත්',\n",
        "]"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:04:58.269444Z",
          "iopub.execute_input": "2023-10-26T01:04:58.270944Z",
          "iopub.status.idle": "2023-10-26T01:04:58.284599Z",
          "shell.execute_reply.started": "2023-10-26T01:04:58.270894Z",
          "shell.execute_reply": "2023-10-26T01:04:58.282981Z"
        },
        "trusted": true,
        "id": "DR08vdULuNNB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **BALANCING DATASET**"
      ],
      "metadata": {
        "id": "4weXb9Cx5h7Z"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "#Step 1 data train\n",
        "dnot = df_train1[df_train1.label == 'NOT']\n",
        "dhof = df_train1[df_train1.label == 'HOF']\n",
        "dhof\n"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:05:16.028882Z",
          "iopub.execute_input": "2023-10-26T01:05:16.029428Z",
          "iopub.status.idle": "2023-10-26T01:05:16.053493Z",
          "shell.execute_reply.started": "2023-10-26T01:05:16.029381Z",
          "shell.execute_reply": "2023-10-26T01:05:16.052395Z"
        },
        "trusted": true,
        "id": "qIrJCwahuNND",
        "outputId": "616170dc-9cfe-4e59-9c38-482b2e6b613e"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 16,
          "output_type": "execute_result",
          "data": {
            "text/plain": "      index             tweet_id  \\\n5197   1132  1285483222550237184   \n1508   2519   743994404269096960   \n3750   4936  1442016935642902528   \n2805   4859  1197761471851245573   \n4174   2816  1249750123548213248   \n...     ...                  ...   \n5969   7240  1101352281155559427   \n3164   2624  1068011964788559872   \n5657   3910   981194683228471297   \n2013   2029  1250450251317522432   \n3271   5403  1383455672403054594   \n\n                                                  tweet label  \n5197  @USER  තමුසෙ NSBM ගිහින් ඩිග්‍රියක් කරනවා කියල...   HOF  \n1508  අඩෝ පපුව පිච්චුනා වගේ. අැස් දෙකට කදුලු ඉනුවා ඔ...   HOF  \n3750      @USER  @USER    තෝ  පලකෝ  සයිබීරියන් කෑහුට්ටී   HOF  \n2805  ඉස්සෙල්ලාම දුන්න පෙම්හසුනට ලැබුන ප්‍රතිචාරය තම...   HOF  \n4174  ටීවි එකේ නිව්ස් වල යනවා දැක්කා. Police officer...   HOF  \n...                                                 ...   ...  \n5969  බලහල්ල රජයේ කැම්පස්  වලට බැනපු උන් මොනවගෙද ඉන්...   HOF  \n3164  @USER  @USER කොහෙන් ගියත් මූ යන්නෙ ගල විදපු තැ...   HOF  \n5657  @USER පිස්සා කිව්වට පිස්සු පාටක් නම් පේන්නෙම නැ..   HOF  \n2013  මොනවද තිරිසනෝ මේ දෙරණේ පෙන්වන ටෙලි නාට්‍ය?  පා...   HOF  \n3271  අමුතුවෙන් හිතන්නත් දෙයක් මෙතන නැහැ... මේක තමයි...   HOF  \n\n[2857 rows x 4 columns]",
            "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>index</th>\n      <th>tweet_id</th>\n      <th>tweet</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>5197</th>\n      <td>1132</td>\n      <td>1285483222550237184</td>\n      <td>@USER  තමුසෙ NSBM ගිහින් ඩිග්‍රියක් කරනවා කියල...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>1508</th>\n      <td>2519</td>\n      <td>743994404269096960</td>\n      <td>අඩෝ පපුව පිච්චුනා වගේ. අැස් දෙකට කදුලු ඉනුවා ඔ...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>3750</th>\n      <td>4936</td>\n      <td>1442016935642902528</td>\n      <td>@USER  @USER    තෝ  පලකෝ  සයිබීරියන් කෑහුට්ටී</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>2805</th>\n      <td>4859</td>\n      <td>1197761471851245573</td>\n      <td>ඉස්සෙල්ලාම දුන්න පෙම්හසුනට ලැබුන ප්‍රතිචාරය තම...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>4174</th>\n      <td>2816</td>\n      <td>1249750123548213248</td>\n      <td>ටීවි එකේ නිව්ස් වල යනවා දැක්කා. Police officer...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>5969</th>\n      <td>7240</td>\n      <td>1101352281155559427</td>\n      <td>බලහල්ල රජයේ කැම්පස්  වලට බැනපු උන් මොනවගෙද ඉන්...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>3164</th>\n      <td>2624</td>\n      <td>1068011964788559872</td>\n      <td>@USER  @USER කොහෙන් ගියත් මූ යන්නෙ ගල විදපු තැ...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>5657</th>\n      <td>3910</td>\n      <td>981194683228471297</td>\n      <td>@USER පිස්සා කිව්වට පිස්සු පාටක් නම් පේන්නෙම නැ..</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>2013</th>\n      <td>2029</td>\n      <td>1250450251317522432</td>\n      <td>මොනවද තිරිසනෝ මේ දෙරණේ පෙන්වන ටෙලි නාට්‍ය?  පා...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>3271</th>\n      <td>5403</td>\n      <td>1383455672403054594</td>\n      <td>අමුතුවෙන් හිතන්නත් දෙයක් මෙතන නැහැ... මේක තමයි...</td>\n      <td>HOF</td>\n    </tr>\n  </tbody>\n</table>\n<p>2857 rows × 4 columns</p>\n</div>"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# step 2: oversampling the HOF class\n",
        "\n",
        "# select tweets which have more than 18 words, not include stop words\n",
        "new_HOF= []\n",
        "\n",
        "i = 0\n",
        "for txt in dhof['tweet'] :\n",
        "    a= txt.split(' ')\n",
        "    if len(a) > 18:\n",
        "        baru = [x for x in a if x not in stp]\n",
        "        new_HOF.append([i+7500,' '.join(baru),'HOF'])\n",
        "        i += 1\n",
        "#new_HOF\n"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:05:21.227390Z",
          "iopub.execute_input": "2023-10-26T01:05:21.227805Z",
          "iopub.status.idle": "2023-10-26T01:05:21.491641Z",
          "shell.execute_reply.started": "2023-10-26T01:05:21.227773Z",
          "shell.execute_reply": "2023-10-26T01:05:21.490791Z"
        },
        "jupyter": {
          "outputs_hidden": true
        },
        "trusted": true,
        "id": "SeqdgpdnuNND"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# convert into pandas dataframe\n",
        "\n",
        "df_HOF_additional = pd.DataFrame(np.array(new_HOF),\n",
        "             columns=['index', 'tweet', 'label'\n",
        "                      ])\n",
        "df_HOF_additional"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:05:31.087583Z",
          "iopub.execute_input": "2023-10-26T01:05:31.088035Z",
          "iopub.status.idle": "2023-10-26T01:05:31.122200Z",
          "shell.execute_reply.started": "2023-10-26T01:05:31.087953Z",
          "shell.execute_reply": "2023-10-26T01:05:31.120388Z"
        },
        "trusted": true,
        "id": "QMnt1pjXuNNE",
        "outputId": "6c53dc57-64d9-4544-823f-70c55e1ddaf6"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 18,
          "output_type": "execute_result",
          "data": {
            "text/plain": "     index                                              tweet label\n0     7500  @USER  තමුසෙ NSBM ගිහින් ඩිග්‍රියක් කරනවා කියල...   HOF\n1     7501  අඩෝ පපුව පිච්චුනා වගේ. අැස් දෙකට කදුලු ඉනුවා ඔ...   HOF\n2     7502  ඉදගන්න යනකොට ඒකි මොනාදෝ කියනවලු.කඩ්ඩ බැරිඋනත් ...   HOF\n3     7503  ලබ්බ බහින සංගමය කියලා වගේ එකක් හැදුවා හරි. Ami...   HOF\n4     7504  @USER  @USER බෝතල් ගොඩක් කුඩුකරලා දැනුත් පේමන්...   HOF\n...    ...                                                ...   ...\n1469  8969  මං තමා එච්චර ගාණක් ෆන්ඩ් කරන්නෙ...නාමල් අයියා ...   HOF\n1470  8970  @USER  @USER     සවේන්ද්‍රයි සයි හින්දා තෝ අද ...   HOF\n1471  8971  බලහල්ල රජයේ කැම්පස්  වලට බැනපු උන් මොනවගෙද ඉන්...   HOF\n1472  8972  @USER  @USER කොහෙන් ගියත් මූ යන්නෙ ගල විදපු තැ...   HOF\n1473  8973  අමුතුවෙන් හිතන්නත් දෙයක් මෙතන නැහැ... මේක තමයි...   HOF\n\n[1474 rows x 3 columns]",
            "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>index</th>\n      <th>tweet</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>7500</td>\n      <td>@USER  තමුසෙ NSBM ගිහින් ඩිග්‍රියක් කරනවා කියල...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>7501</td>\n      <td>අඩෝ පපුව පිච්චුනා වගේ. අැස් දෙකට කදුලු ඉනුවා ඔ...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>7502</td>\n      <td>ඉදගන්න යනකොට ඒකි මොනාදෝ කියනවලු.කඩ්ඩ බැරිඋනත් ...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>7503</td>\n      <td>ලබ්බ බහින සංගමය කියලා වගේ එකක් හැදුවා හරි. Ami...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>7504</td>\n      <td>@USER  @USER බෝතල් ගොඩක් කුඩුකරලා දැනුත් පේමන්...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1469</th>\n      <td>8969</td>\n      <td>මං තමා එච්චර ගාණක් ෆන්ඩ් කරන්නෙ...නාමල් අයියා ...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>1470</th>\n      <td>8970</td>\n      <td>@USER  @USER     සවේන්ද්‍රයි සයි හින්දා තෝ අද ...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>1471</th>\n      <td>8971</td>\n      <td>බලහල්ල රජයේ කැම්පස්  වලට බැනපු උන් මොනවගෙද ඉන්...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>1472</th>\n      <td>8972</td>\n      <td>@USER  @USER කොහෙන් ගියත් මූ යන්නෙ ගල විදපු තැ...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>1473</th>\n      <td>8973</td>\n      <td>අමුතුවෙන් හිතන්නත් දෙයක් මෙතන නැහැ... මේක තමයි...</td>\n      <td>HOF</td>\n    </tr>\n  </tbody>\n</table>\n<p>1474 rows × 3 columns</p>\n</div>"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# get some tweets with class HOF to be added to data-train (choose by index)\n",
        "# no need for Gujarati\n",
        "\n",
        "adds = df_HOF_additional[200:1236]\n",
        "\n",
        "df1 = pd.concat([dnot,adds,dhof])# .reset_index()\n",
        "df1['label'].value_counts()\n"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:06:36.660281Z",
          "iopub.execute_input": "2023-10-26T01:06:36.660914Z",
          "iopub.status.idle": "2023-10-26T01:06:36.676984Z",
          "shell.execute_reply.started": "2023-10-26T01:06:36.660864Z",
          "shell.execute_reply": "2023-10-26T01:06:36.676063Z"
        },
        "trusted": true,
        "id": "XAPDhB9JuNNF",
        "outputId": "2dee0b81-a282-4a38-b6c2-486504fc40ea"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 22,
          "output_type": "execute_result",
          "data": {
            "text/plain": "NOT    3893\nHOF    3893\nName: label, dtype: int64"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# This is balanced training data, use for SVM Training\n",
        "\n",
        "shuffled_train = df1.sample(frac=1, random_state=23)\n",
        "shuffled_train\n"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:06:47.143693Z",
          "iopub.execute_input": "2023-10-26T01:06:47.144104Z",
          "iopub.status.idle": "2023-10-26T01:06:47.165459Z",
          "shell.execute_reply.started": "2023-10-26T01:06:47.144070Z",
          "shell.execute_reply": "2023-10-26T01:06:47.164454Z"
        },
        "trusted": true,
        "id": "D5n22MOwuNNF",
        "outputId": "e20137bf-6495-4cbf-8a98-abed8397ef57"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 24,
          "output_type": "execute_result",
          "data": {
            "text/plain": "     index      tweet_id                                              tweet  \\\n3030  7099  1.145568e+18           @USER ,  @USER  @USER ටහුක්  තෝ ඉන්නවා ද   \n3642  1331  6.756934e+17  ඉස්සර උබ වලියක් නම් ලොවෙත් කියනො ගිහාන් අයියා ...   \n539   4881  1.396944e+18                 නින්ද යන්නේ නැති එක නම් කරුම ලෙඩක්   \n1089  8589           NaN  10  අළු වලින් ගොයම් ගස් පෝෂණය වෙලා හැදීල එන පි...   \n196   1169  1.161323e+18  @USER රිචඩ් ද සොයිසා ප්‍රේමකීර්ති ද අල්විස් රෝ...   \n...    ...           ...                                                ...   \n1237  2408  1.185154e+18  @USER නරකද වී නිෂ්පාදන වියදම් අඩුකරලා ගොවියායි...   \n5752   668  1.053322e+18    කහපාට රෙද්දකින් තඩි පීප්පයක් ඔතා තිබූ ඒ අවස්තාව   \n3387   634  1.166635e+18  ගත මදක් පේන ලෙස උස්සලා අන්දා සිටි බැවින් සරාගි...   \n1145  8645           NaN  අහසේ දෙයියෝ හුත්තෝ දෙයියෝ ඉන්නේ මෙහෙ මෙහෙ උට්ට...   \n1911   995  1.478016e+18                   @USER හිනාවෙන්න තැන් නෑ මේවාට :3   \n\n     label  \n3030   HOF  \n3642   HOF  \n539    NOT  \n1089   HOF  \n196    NOT  \n...    ...  \n1237   NOT  \n5752   NOT  \n3387   NOT  \n1145   HOF  \n1911   NOT  \n\n[7786 rows x 4 columns]",
            "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>index</th>\n      <th>tweet_id</th>\n      <th>tweet</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>3030</th>\n      <td>7099</td>\n      <td>1.145568e+18</td>\n      <td>@USER ,  @USER  @USER ටහුක්  තෝ ඉන්නවා ද</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>3642</th>\n      <td>1331</td>\n      <td>6.756934e+17</td>\n      <td>ඉස්සර උබ වලියක් නම් ලොවෙත් කියනො ගිහාන් අයියා ...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>539</th>\n      <td>4881</td>\n      <td>1.396944e+18</td>\n      <td>නින්ද යන්නේ නැති එක නම් කරුම ලෙඩක්</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>1089</th>\n      <td>8589</td>\n      <td>NaN</td>\n      <td>10  අළු වලින් ගොයම් ගස් පෝෂණය වෙලා හැදීල එන පි...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>196</th>\n      <td>1169</td>\n      <td>1.161323e+18</td>\n      <td>@USER රිචඩ් ද සොයිසා ප්‍රේමකීර්ති ද අල්විස් රෝ...</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1237</th>\n      <td>2408</td>\n      <td>1.185154e+18</td>\n      <td>@USER නරකද වී නිෂ්පාදන වියදම් අඩුකරලා ගොවියායි...</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>5752</th>\n      <td>668</td>\n      <td>1.053322e+18</td>\n      <td>කහපාට රෙද්දකින් තඩි පීප්පයක් ඔතා තිබූ ඒ අවස්තාව</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>3387</th>\n      <td>634</td>\n      <td>1.166635e+18</td>\n      <td>ගත මදක් පේන ලෙස උස්සලා අන්දා සිටි බැවින් සරාගි...</td>\n      <td>NOT</td>\n    </tr>\n    <tr>\n      <th>1145</th>\n      <td>8645</td>\n      <td>NaN</td>\n      <td>අහසේ දෙයියෝ හුත්තෝ දෙයියෝ ඉන්නේ මෙහෙ මෙහෙ උට්ට...</td>\n      <td>HOF</td>\n    </tr>\n    <tr>\n      <th>1911</th>\n      <td>995</td>\n      <td>1.478016e+18</td>\n      <td>@USER හිනාවෙන්න තැන් නෑ මේවාට :3</td>\n      <td>NOT</td>\n    </tr>\n  </tbody>\n</table>\n<p>7786 rows × 4 columns</p>\n</div>"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# validation data\n",
        "\n",
        "df_dev1['label'].value_counts()\n"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:06:57.928780Z",
          "iopub.execute_input": "2023-10-26T01:06:57.929189Z",
          "iopub.status.idle": "2023-10-26T01:06:57.938650Z",
          "shell.execute_reply.started": "2023-10-26T01:06:57.929159Z",
          "shell.execute_reply": "2023-10-26T01:06:57.937493Z"
        },
        "trusted": true,
        "id": "KB8_tn-JuNNF",
        "outputId": "6b841eea-1909-4007-938a-d9ec99d7fa1f"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 25,
          "output_type": "execute_result",
          "data": {
            "text/plain": "NOT    431\nHOF    319\nName: label, dtype: int64"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Processing Tweet text and transforming to word embeddings"
      ],
      "metadata": {
        "id": "56Uu2UlU6kte"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Pre-processing tweet\n",
        "\n",
        "def preprocess_tweet(text):\n",
        "    text = re.sub('@username', 'MENTIONED_NAME ', text)                           # Removes usernames\n",
        "    text = re.sub('@AUTHOR', 'MENTIONED_NAME', text)                              # Removes usernames\n",
        "    text = re.sub('@USER', 'MENTIONED_NAME', text)                                # Removes usernames\n",
        "    text = re.sub('((www\\.[^\\s]+)|(https?://[^\\s]+))', '', text)                  # Remove URLs\n",
        "    text = re.sub(r\"\\d+\", \"\", text)                                               # Removes all digits\n",
        "    text = re.sub('&quot;',\" \", text)                                             # Remove (&quot;)\n",
        "    text = re.sub(r\"\\b[a-zA-Z]\\b\", \"\", str(text))                                 # Removes all single characters\n",
        "    text = re.sub(r'(.)\\1+', r'\\1\\1', text)                                       # Convert more than 2 letter repetitions to 2 letter\n",
        "    text = re.sub(r\"\\t+\", \" \", text)                                              # Replaces tab spaces with single space\n",
        "    text = re.sub(r\"\\s+\", \" \", text)                                              # Replaces double spaces with single space\n",
        "    text = re.sub(r'\\b(\\w+)( \\1\\b)+', r'\\1', text)                                # remove duplicate of consecutives words\n",
        "    return text\n"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:07:22.368683Z",
          "iopub.execute_input": "2023-10-26T01:07:22.369054Z",
          "iopub.status.idle": "2023-10-26T01:07:22.376459Z",
          "shell.execute_reply.started": "2023-10-26T01:07:22.369024Z",
          "shell.execute_reply": "2023-10-26T01:07:22.375565Z"
        },
        "trusted": true,
        "id": "DZijCBSLuNNG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# data-train with balancing\n",
        "# column name = \"cleantweet\" --> standard text preprocessing\n",
        "\n",
        "shuffled_train['cleantweet'] = shuffled_train['tweet'].apply(preprocess_tweet)\n",
        "shuffled_train"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:07:27.527577Z",
          "iopub.execute_input": "2023-10-26T01:07:27.528303Z",
          "iopub.status.idle": "2023-10-26T01:07:28.097449Z",
          "shell.execute_reply.started": "2023-10-26T01:07:27.528266Z",
          "shell.execute_reply": "2023-10-26T01:07:28.096338Z"
        },
        "trusted": true,
        "id": "siFecCXhuNNG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_test['cleantweet'] = df_test['tweet'].apply(preprocess_tweet)\n",
        "df_test"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:07:32.867504Z",
          "iopub.execute_input": "2023-10-26T01:07:32.868403Z",
          "iopub.status.idle": "2023-10-26T01:07:33.058883Z",
          "shell.execute_reply.started": "2023-10-26T01:07:32.868365Z",
          "shell.execute_reply": "2023-10-26T01:07:33.057652Z"
        },
        "trusted": true,
        "id": "8OKWeBK2uNNH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# data_dev for validation\n",
        "\n",
        "df_dev1['cleantweet'] = df_dev1['tweet'].apply(preprocess_tweet)\n",
        "df_dev1"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:07:52.349153Z",
          "iopub.execute_input": "2023-10-26T01:07:52.349708Z",
          "iopub.status.idle": "2023-10-26T01:07:52.426112Z",
          "shell.execute_reply.started": "2023-10-26T01:07:52.349657Z",
          "shell.execute_reply": "2023-10-26T01:07:52.424557Z"
        },
        "trusted": true,
        "id": "eLFfGAx6uNNH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# data-train without balancing\n",
        "\n",
        "df_train1['cleantweet'] = df_train1['tweet'].apply(preprocess_tweet)\n",
        "df_train1"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:08:01.520848Z",
          "iopub.execute_input": "2023-10-26T01:08:01.522107Z",
          "iopub.status.idle": "2023-10-26T01:08:02.005292Z",
          "shell.execute_reply.started": "2023-10-26T01:08:01.522051Z",
          "shell.execute_reply": "2023-10-26T01:08:02.004141Z"
        },
        "trusted": true,
        "id": "YrF_ZuRguNNH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# function for dealing with emoji (see the paper)\n",
        "\n",
        "def detect_emoji(a):\n",
        "    emoji_pattern = re.compile(\"[\\U0001F600-\\U0001F9FF\\U0001F300-\\U0001F5FF\\U0001F680-\\U0001F6FF\\U0001F700-\\U0001F77F\\U0001F780-\\U0001F7FF\\U0001F800-\\U0001F8FF\\U0001F900-\\U0001F9FF\\U0001FA00-\\U0001FA6F\\U0001FA70-\\U0001FAFF]+\")\n",
        "\n",
        "# Find all emojis in the text\n",
        "    emojis = emoji_pattern.findall(a)\n",
        "    emo=[]\n",
        "# Remove spaces from extracted emojis\n",
        "    emojis_cleaned = [emoji.replace(\" \", \"\") for emoji in emojis]\n",
        "    count_emoji = [0]\n",
        "    #print(emojis_cleaned)\n",
        "    new_text = a.split(' ')\n",
        "    for emoji in emojis_cleaned:\n",
        "        new = (re.split(r'', emoji))\n",
        "        for x in new:\n",
        "            #print((x))\n",
        "            new_text.append(x)\n",
        "    new_text1 = ' '.join(new_text)\n",
        "    new_text1 = re.sub(r\"\\s+\", \" \", new_text1)\n",
        "    return new_text1\n",
        "\n"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:08:52.171421Z",
          "iopub.execute_input": "2023-10-26T01:08:52.171951Z",
          "iopub.status.idle": "2023-10-26T01:08:52.184350Z",
          "shell.execute_reply.started": "2023-10-26T01:08:52.171912Z",
          "shell.execute_reply": "2023-10-26T01:08:52.182610Z"
        },
        "trusted": true,
        "id": "K7Cqr9hnuNNH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# column name = \"ok\" --> emojis are treated as special tokens\n",
        "\n",
        "shuffled_train['ok'] = shuffled_train['cleantweet'].apply(detect_emoji)\n",
        "shuffled_train\n"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:12:57.957007Z",
          "iopub.execute_input": "2023-10-26T01:12:57.957671Z",
          "iopub.status.idle": "2023-10-26T01:12:58.171221Z",
          "shell.execute_reply.started": "2023-10-26T01:12:57.957627Z",
          "shell.execute_reply": "2023-10-26T01:12:58.169693Z"
        },
        "trusted": true,
        "id": "lVDgtnoauNNI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# data_test (for HASOC submission)\n",
        "\n",
        "df_test['ok'] = df_test['cleantweet'].apply(detect_emoji)\n",
        "df_test"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:09:05.849770Z",
          "iopub.execute_input": "2023-10-26T01:09:05.850637Z",
          "iopub.status.idle": "2023-10-26T01:09:05.926606Z",
          "shell.execute_reply.started": "2023-10-26T01:09:05.850589Z",
          "shell.execute_reply": "2023-10-26T01:09:05.925641Z"
        },
        "trusted": true,
        "id": "fzxZsgpEuNNI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_dev1['ok'] = df_dev1['cleantweet'].apply(detect_emoji)\n",
        "df_dev1"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:09:10.308017Z",
          "iopub.execute_input": "2023-10-26T01:09:10.308739Z",
          "iopub.status.idle": "2023-10-26T01:09:10.342343Z",
          "shell.execute_reply.started": "2023-10-26T01:09:10.308699Z",
          "shell.execute_reply": "2023-10-26T01:09:10.341140Z"
        },
        "trusted": true,
        "id": "rM8UnLPbuNNI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# data-train for word embeddings fasttext (use all/original data train)\n",
        "\n",
        "df_train['cleantweet'] = df_train['tweet'].apply(preprocess_tweet)\n",
        "df_train['ok'] = df_train['cleantweet'].apply(detect_emoji)\n",
        "df_train"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:09:19.729143Z",
          "iopub.execute_input": "2023-10-26T01:09:19.729595Z",
          "iopub.status.idle": "2023-10-26T01:09:20.417858Z",
          "shell.execute_reply.started": "2023-10-26T01:09:19.729536Z",
          "shell.execute_reply": "2023-10-26T01:09:20.416340Z"
        },
        "trusted": true,
        "id": "1D5rBtNWuNNI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# PROCESSING FASTTEXT EMBEDDING\n",
        "# if concerning emoji, use column ['ok']\n",
        "# if use normal text processing without tokenizing emoji, use column ['cleantweet']\n",
        "#\n",
        "sentences = [word_tokenize(Tweet) for Tweet in tqdm(df_train['ok'])]\n",
        "model = FastText(sentences, vector_size=100, window=3, min_count=2, workers=4, epochs=50)"
      ],
      "metadata": {
        "id": "pzFkRdsiuNNJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Saving model (adjust the file name and path location)\n",
        "# will be needed to alter the FastText language model\n",
        "\n",
        "model.save(\"sinhala_w3c2v100.fasttext\")"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:10:54.959776Z",
          "iopub.execute_input": "2023-10-26T01:10:54.960266Z",
          "iopub.status.idle": "2023-10-26T01:10:55.869517Z",
          "shell.execute_reply.started": "2023-10-26T01:10:54.960225Z",
          "shell.execute_reply": "2023-10-26T01:10:55.868542Z"
        },
        "trusted": true,
        "id": "A0WeoQTHuNNK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Load the model\n",
        "\n",
        "model = FastText.load(\"sinhala_w3c2v100.fasttext\")\n"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-08-29T04:06:08.889004Z",
          "iopub.execute_input": "2023-08-29T04:06:08.889432Z",
          "iopub.status.idle": "2023-08-29T04:06:09.408245Z",
          "shell.execute_reply.started": "2023-08-29T04:06:08.8894Z",
          "shell.execute_reply": "2023-08-29T04:06:09.407125Z"
        },
        "trusted": true,
        "id": "CoBAJAk2uNNK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#model['නායකයා']\n",
        "feat = model.wv\n",
        "feat.vectors"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:11:21.744743Z",
          "iopub.execute_input": "2023-10-26T01:11:21.745192Z",
          "iopub.status.idle": "2023-10-26T01:11:21.754145Z",
          "shell.execute_reply.started": "2023-10-26T01:11:21.745157Z",
          "shell.execute_reply": "2023-10-26T01:11:21.752989Z"
        },
        "trusted": true,
        "id": "2_s39Bw7uNNK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Function for creating sentence vector\n",
        "\n",
        "def norm_vector(sentence, model):\n",
        "    vecs = [model[word] for word in word_tokenize(sentence)]\n",
        "    norm_vecs = [vec / np.linalg.norm(vec) for vec in vecs if np.linalg.norm(vec) > 0]\n",
        "    sent_vecs = np.mean(norm_vecs, axis=0)\n",
        "    return sent_vecs"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:11:54.282498Z",
          "iopub.execute_input": "2023-10-26T01:11:54.282909Z",
          "iopub.status.idle": "2023-10-26T01:11:54.291902Z",
          "shell.execute_reply.started": "2023-10-26T01:11:54.282877Z",
          "shell.execute_reply": "2023-10-26T01:11:54.290364Z"
        },
        "trusted": true,
        "id": "ohDXfCLLuNNK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **TRANSFORM SENTENCE INTO VECTOR**"
      ],
      "metadata": {
        "id": "9fQ1oadF5UrD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vecs_train = [norm_vector(sentence, feat) for sentence in shuffled_train.ok]\n",
        "vecs_train = np.array(vecs_train)\n",
        "vecs_train"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:13:27.266597Z",
          "iopub.execute_input": "2023-10-26T01:13:27.267054Z",
          "iopub.status.idle": "2023-10-26T01:13:33.551436Z",
          "shell.execute_reply.started": "2023-10-26T01:13:27.267019Z",
          "shell.execute_reply": "2023-10-26T01:13:33.550267Z"
        },
        "trusted": true,
        "id": "a7NqoyojuNNR",
        "outputId": "7a0e22dd-2697-45ed-ea51-3908dad5adf0"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 52,
          "output_type": "execute_result",
          "data": {
            "text/plain": "array([[-5.6887668e-02, -6.8822759e-03, -7.6828472e-04, ...,\n         3.2185442e-05, -5.7128023e-02, -1.5850360e-02],\n       [ 3.3731952e-02, -2.4035878e-02, -1.2270814e-02, ...,\n        -3.4085557e-02,  1.5071415e-02, -6.4492261e-04],\n       [ 2.7488681e-02, -1.1688858e-02,  1.7720545e-02, ...,\n        -3.7581824e-02, -2.8778730e-02,  3.1723484e-02],\n       ...,\n       [ 3.4136720e-02, -1.6505117e-02, -3.2315586e-02, ...,\n        -2.9686270e-02, -8.7184804e-03, -1.4336245e-02],\n       [ 2.6732562e-02, -3.2145746e-02, -3.5472650e-02, ...,\n        -1.9729570e-02,  1.7658006e-02,  2.1268060e-02],\n       [ 1.0084289e-02,  2.4283404e-02, -5.6376085e-03, ...,\n         2.8706591e-03,  3.0506453e-02, -2.7896937e-03]], dtype=float32)"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "vecs_dev = [norm_vector(sentence, feat) for sentence in df_dev1.ok]\n",
        "vecs_dev = np.array(vecs_dev)\n",
        "vecs_dev"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:13:37.547002Z",
          "iopub.execute_input": "2023-10-26T01:13:37.547511Z",
          "iopub.status.idle": "2023-10-26T01:13:38.170569Z",
          "shell.execute_reply.started": "2023-10-26T01:13:37.547461Z",
          "shell.execute_reply": "2023-10-26T01:13:38.169558Z"
        },
        "trusted": true,
        "id": "xFyna36euNNR",
        "outputId": "26bc63c8-5ffb-4be7-ff2d-ab6887ef678a"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 53,
          "output_type": "execute_result",
          "data": {
            "text/plain": "array([[ 0.02841851,  0.00067929, -0.04173805, ..., -0.04154312,\n         0.00134731, -0.0218883 ],\n       [-0.01198898, -0.02833401,  0.01888742, ..., -0.03112266,\n        -0.00941231, -0.01677913],\n       [ 0.00467574, -0.02278483, -0.03343655, ..., -0.0515876 ,\n         0.01947294, -0.01766241],\n       ...,\n       [ 0.0391865 ,  0.01585085,  0.00012952, ..., -0.07477746,\n        -0.02469535, -0.03000988],\n       [ 0.04795717, -0.00293585,  0.00259417, ..., -0.02006187,\n         0.01258126,  0.00648162],\n       [-0.01300191, -0.01335822, -0.039092  , ...,  0.01249252,\n         0.0257376 , -0.03880717]], dtype=float32)"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "vecs_test = [norm_vector(sentence, feat) for sentence in df_test.ok]\n",
        "vecs_test = np.array(vecs_test)\n",
        "vecs_test"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:13:50.294911Z",
          "iopub.execute_input": "2023-10-26T01:13:50.295401Z",
          "iopub.status.idle": "2023-10-26T01:13:52.302062Z",
          "shell.execute_reply.started": "2023-10-26T01:13:50.295362Z",
          "shell.execute_reply": "2023-10-26T01:13:52.300465Z"
        },
        "trusted": true,
        "id": "okKlPq4FuNNS",
        "outputId": "a5064ab4-fc2d-42d8-efd0-688d9aba9fcd"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 54,
          "output_type": "execute_result",
          "data": {
            "text/plain": "array([[ 0.02036053,  0.0416871 , -0.03889521, ..., -0.00613656,\n        -0.03478594,  0.03410538],\n       [ 0.05254064,  0.0084663 , -0.06094459, ..., -0.02086362,\n         0.00846146, -0.00919613],\n       [-0.03986843, -0.04553909,  0.06200192, ...,  0.0010208 ,\n         0.08350173, -0.00852834],\n       ...,\n       [ 0.00242339, -0.01687736,  0.01765568, ...,  0.02207897,\n         0.03728187, -0.01602666],\n       [ 0.02332435, -0.0407596 , -0.03102377, ...,  0.00578075,\n         0.07404105, -0.06994012],\n       [ 0.0073469 , -0.00954356, -0.01056082, ...,  0.03843581,\n         0.06405949,  0.02256356]], dtype=float32)"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "vecs_dev.shape"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:15:30.102696Z",
          "iopub.execute_input": "2023-10-26T01:15:30.103685Z",
          "iopub.status.idle": "2023-10-26T01:15:30.112115Z",
          "shell.execute_reply.started": "2023-10-26T01:15:30.103634Z",
          "shell.execute_reply": "2023-10-26T01:15:30.110343Z"
        },
        "trusted": true,
        "id": "gYJmQzpkuNNS",
        "outputId": "f364d98d-f3cd-4e7d-8ebb-48354f620e1a"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 55,
          "output_type": "execute_result",
          "data": {
            "text/plain": "(750, 100)"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**CLASSIFICATION PHASE**"
      ],
      "metadata": {
        "id": "bAoV1iPG0HNj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.svm import SVC\n"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:15:37.782403Z",
          "iopub.execute_input": "2023-10-26T01:15:37.782871Z",
          "iopub.status.idle": "2023-10-26T01:15:37.788440Z",
          "shell.execute_reply.started": "2023-10-26T01:15:37.782837Z",
          "shell.execute_reply": "2023-10-26T01:15:37.787045Z"
        },
        "trusted": true,
        "id": "BxyyAbaNuNNS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Train SVM and validation (Predict data-dev)\n",
        "\n",
        "svc_HS = SVC()\n",
        "svc_HS = svc_HS.fit(vecs_train, shuffled_train.label)\n",
        "result_svc_noscaling = svc_HS.predict(vecs_dev)\n",
        "print(accuracy_score(df_dev1.label, result_svc_noscaling)*100)\n",
        "\n",
        "# jump to cell code evaluation scoring"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:15:56.862285Z",
          "iopub.execute_input": "2023-10-26T01:15:56.862742Z",
          "iopub.status.idle": "2023-10-26T01:16:01.137671Z",
          "shell.execute_reply.started": "2023-10-26T01:15:56.862709Z",
          "shell.execute_reply": "2023-10-26T01:16:01.136434Z"
        },
        "trusted": true,
        "id": "Ts_drZ2QuNNS",
        "outputId": "26b8a84e-a022-46ce-947c-7afc50d19338"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "78.53333333333333\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Evaluation\n",
        "\n",
        "from sklearn.metrics import accuracy_score, recall_score, precision_score, confusion_matrix, classification_report\n",
        "print(classification_report(df_dev1.label, result_svc_noscaling))\n",
        "\n",
        "#Scoring Report"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:18:18.309972Z",
          "iopub.execute_input": "2023-10-26T01:18:18.310451Z",
          "iopub.status.idle": "2023-10-26T01:18:18.351365Z",
          "shell.execute_reply.started": "2023-10-26T01:18:18.310413Z",
          "shell.execute_reply": "2023-10-26T01:18:18.350164Z"
        },
        "trusted": true,
        "id": "uhbUxYt8uNNS",
        "outputId": "1496c003-1706-4bd5-d7c5-294472455462"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "              precision    recall  f1-score   support\n\n         HOF       0.74      0.76      0.75       319\n         NOT       0.82      0.80      0.81       431\n\n    accuracy                           0.79       750\n   macro avg       0.78      0.78      0.78       750\nweighted avg       0.79      0.79      0.79       750\n\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Use minmax scaler for sentence vector\n",
        "\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "scaler = MinMaxScaler()\n",
        "scaler.fit(vecs_train)\n",
        "\n",
        "X_train_scaled = scaler.transform(vecs_train)\n",
        "X_test_scaled = scaler.transform(vecs_dev)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:18:31.774563Z",
          "iopub.execute_input": "2023-10-26T01:18:31.774922Z",
          "iopub.status.idle": "2023-10-26T01:18:31.805924Z",
          "shell.execute_reply.started": "2023-10-26T01:18:31.774895Z",
          "shell.execute_reply": "2023-10-26T01:18:31.804556Z"
        },
        "trusted": true,
        "id": "kRaw8IyquNNT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Train SVM and validation (Predict data-dev)\n",
        "\n",
        "svc_HS = svc_HS.fit(X_train_scaled, shuffled_train.label)\n",
        "result_svc_minmax = svc_HS.predict(X_test_scaled)\n",
        "print(accuracy_score(df_dev1.label, result_svc_minmax)*100)\n",
        "\n",
        "# Accuracy"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:26:52.362787Z",
          "iopub.execute_input": "2023-10-26T01:26:52.363177Z",
          "iopub.status.idle": "2023-10-26T01:26:52.373826Z",
          "shell.execute_reply.started": "2023-10-26T01:26:52.363148Z",
          "shell.execute_reply": "2023-10-26T01:26:52.372146Z"
        },
        "trusted": true,
        "id": "mPlO9DMcuNNT",
        "outputId": "88b7ef21-8ffb-43ff-b3f6-b559f31f8d52"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "77.60000000000001\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Evaluation\n",
        "\n",
        "print(classification_report(df_dev1.label, result_svc_minmax))\n",
        "\n",
        "#Scoring Report"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:27:05.633580Z",
          "iopub.execute_input": "2023-10-26T01:27:05.634018Z",
          "iopub.status.idle": "2023-10-26T01:27:05.673204Z",
          "shell.execute_reply.started": "2023-10-26T01:27:05.633986Z",
          "shell.execute_reply": "2023-10-26T01:27:05.671752Z"
        },
        "trusted": true,
        "id": "hQIJTgOeuNNT",
        "outputId": "5a622cf0-bfe1-46c1-accd-430bd0c0c26b"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "              precision    recall  f1-score   support\n\n         HOF       0.73      0.74      0.74       319\n         NOT       0.81      0.80      0.80       431\n\n    accuracy                           0.78       750\n   macro avg       0.77      0.77      0.77       750\nweighted avg       0.78      0.78      0.78       750\n\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Use robust scaler for sentence vector\n",
        "\n",
        "from sklearn.preprocessing import RobustScaler\n",
        "scaler = RobustScaler()\n",
        "scaler.fit(vecs_train)\n",
        "\n",
        "X_train_scaled = scaler.transform(vecs_train)\n",
        "X_test_scaled = scaler.transform(vecs_dev)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:27:10.789716Z",
          "iopub.execute_input": "2023-10-26T01:27:10.790195Z",
          "iopub.status.idle": "2023-10-26T01:27:10.865203Z",
          "shell.execute_reply.started": "2023-10-26T01:27:10.790160Z",
          "shell.execute_reply": "2023-10-26T01:27:10.863351Z"
        },
        "trusted": true,
        "id": "_D78PtymuNNT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Train SVM and validation (Predict data-dev)\n",
        "\n",
        "svc_HS = svc_HS.fit(X_train_scaled, shuffled_train.label)\n",
        "results_svc_robust = svc_HS.predict(X_test_scaled)\n",
        "print(accuracy_score(df_dev1.label, results_svc_robust)*100)\n",
        "\n",
        "# Accuracy"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:27:17.862895Z",
          "iopub.execute_input": "2023-10-26T01:27:17.863905Z",
          "iopub.status.idle": "2023-10-26T01:27:22.397942Z",
          "shell.execute_reply.started": "2023-10-26T01:27:17.863858Z",
          "shell.execute_reply": "2023-10-26T01:27:22.396837Z"
        },
        "trusted": true,
        "id": "mkMmSHJQuNNU",
        "outputId": "0c5a9305-c32d-47e1-b8bd-e8705d181bb2"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "77.60000000000001\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Evaluation\n",
        "\n",
        "print(classification_report(df_dev1.label, results_svc_robust))"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:27:25.731138Z",
          "iopub.execute_input": "2023-10-26T01:27:25.731629Z",
          "iopub.status.idle": "2023-10-26T01:27:25.774340Z",
          "shell.execute_reply.started": "2023-10-26T01:27:25.731594Z",
          "shell.execute_reply": "2023-10-26T01:27:25.773101Z"
        },
        "trusted": true,
        "id": "XpdeaRB6uNNU",
        "outputId": "89e073e0-0054-4333-a79a-fdd352327397"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "              precision    recall  f1-score   support\n\n         HOF       0.73      0.74      0.74       319\n         NOT       0.81      0.80      0.80       431\n\n    accuracy                           0.78       750\n   macro avg       0.77      0.77      0.77       750\nweighted avg       0.78      0.78      0.78       750\n\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Parameter Tuning"
      ],
      "metadata": {
        "id": "MK2bZxBU743S"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# grid search on parameters using default 5-fold cross validation\n",
        "\n",
        "#default SVC param\n",
        "# C=1.0, kernel='rbf', degree=3, gamma='scale', coef0=0.0, shrinking=True,\n",
        "#probability=False, tol=0.001, cache_size=200, class_weight=None, verbose=False,\n",
        "#max_iter=-1, decision_function_shape='ovr', break_ties=False, random_state=None)\n",
        "\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "parameters = {\n",
        "     #'kernel': ['sigmoid','rbf', 'linear'],\n",
        "     'C': [ 1, 10],\n",
        "    }\n",
        "\n",
        "clf = GridSearchCV(SVC(random_state=42), param_grid=parameters, verbose=1, scoring='accuracy')\n",
        "\n",
        "clf.fit(X_train_scaled, shuffled_train.label)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:28:26.070924Z",
          "iopub.execute_input": "2023-10-26T01:28:26.071317Z",
          "iopub.status.idle": "2023-10-26T01:29:11.225355Z",
          "shell.execute_reply.started": "2023-10-26T01:28:26.071286Z",
          "shell.execute_reply": "2023-10-26T01:29:11.224230Z"
        },
        "trusted": true,
        "id": "m2lUGssvuNNU",
        "outputId": "09639953-d4be-4084-9b74-7721dc2070b2"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "Fitting 5 folds for each of 2 candidates, totalling 10 fits\n",
          "output_type": "stream"
        },
        {
          "execution_count": 77,
          "output_type": "execute_result",
          "data": {
            "text/plain": "GridSearchCV(estimator=SVC(random_state=42), param_grid={'C': [1, 10]},\n             scoring='accuracy', verbose=1)",
            "text/html": "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(estimator=SVC(random_state=42), param_grid={&#x27;C&#x27;: [1, 10]},\n             scoring=&#x27;accuracy&#x27;, verbose=1)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(estimator=SVC(random_state=42), param_grid={&#x27;C&#x27;: [1, 10]},\n             scoring=&#x27;accuracy&#x27;, verbose=1)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: SVC</label><div class=\"sk-toggleable__content\"><pre>SVC(random_state=42)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SVC</label><div class=\"sk-toggleable__content\"><pre>SVC(random_state=42)</pre></div></div></div></div></div></div></div></div></div></div>"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "clf.best_params_"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:30:29.310374Z",
          "iopub.execute_input": "2023-10-26T01:30:29.310774Z",
          "iopub.status.idle": "2023-10-26T01:30:29.317984Z",
          "shell.execute_reply.started": "2023-10-26T01:30:29.310745Z",
          "shell.execute_reply": "2023-10-26T01:30:29.316745Z"
        },
        "trusted": true,
        "id": "5Zk5X8PouNNV",
        "outputId": "2301c31c-3a30-4c17-ea20-614723230055"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 78,
          "output_type": "execute_result",
          "data": {
            "text/plain": "{'C': 10}"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#  Enable the line with the optimal results for assigning which vector to be used as SVC input\n",
        "\n",
        "optimal_vectors_train = vecs_train         # no scaling\n",
        "#optimal_vectors = X_train_scaled          # with scaling\n",
        "\n",
        "optimal_vectors_val = vecs_dev             # no scaling\n",
        "#optimal_vectors_val = X_test_scaled       # with scaling\n",
        "\n",
        "\n",
        "best_model = SVC (kernel='rbf', C=10)\n",
        "best_model.fit(optimal_vectors_train, shuffled_train.label)\n",
        "print('acc for data-train\\n', best_model.score(optimal_vectors_train, shuffled_train.label))\n",
        "\n",
        "final_result = best_model.predict(optimal_vectors_val)\n",
        "print('\\nevaluation reports for data-dev (validation)\\n')\n",
        "print(classification_report(df_dev1.label, final_result))"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:42:10.574045Z",
          "iopub.execute_input": "2023-10-26T01:42:10.575062Z",
          "iopub.status.idle": "2023-10-26T01:42:19.518144Z",
          "shell.execute_reply.started": "2023-10-26T01:42:10.575015Z",
          "shell.execute_reply": "2023-10-26T01:42:19.516961Z"
        },
        "trusted": true,
        "id": "dM-g274SuNNV",
        "outputId": "b0e4d4ec-12a7-47e1-e40c-131d85a0a6ef"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "acc for data-train\n 0.9202414590290264\n\nevaluation reports for data-dev (validation)\n\n              precision    recall  f1-score   support\n\n         HOF       0.70      0.75      0.72       319\n         NOT       0.80      0.76      0.78       431\n\n    accuracy                           0.75       750\n   macro avg       0.75      0.75      0.75       750\nweighted avg       0.76      0.75      0.76       750\n\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# SUBMISSION"
      ],
      "metadata": {
        "id": "o3RhvV4guNNV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# select vectors depend on which scaling is the best\n",
        "\n",
        "X_test_real = vecs_test  #no scaling\n",
        "#X_test_real = scaler.transform(vecs_test)\n",
        "\n",
        "result = svc_HS.predict(X_test_real)\n",
        "result"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-10-26T01:44:33.713393Z",
          "iopub.execute_input": "2023-10-26T01:44:33.713831Z",
          "iopub.status.idle": "2023-10-26T01:44:34.934153Z",
          "shell.execute_reply.started": "2023-10-26T01:44:33.713799Z",
          "shell.execute_reply": "2023-10-26T01:44:34.933053Z"
        },
        "trusted": true,
        "id": "M05buAaQuNNW",
        "outputId": "3ade61ed-e9e8-44d4-b891-fb9a52aaaa6a"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 86,
          "output_type": "execute_result",
          "data": {
            "text/plain": "array(['HOF', 'HOF', 'HOF', ..., 'HOF', 'HOF', 'HOF'], dtype=object)"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "lA7zAGJBuNNc"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}