TeamUNCC_Hopespeech_Detection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hope Speech Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Information\n",
    "\n",
    "[Website](https://competitions.codalab.org/competitions/27653#participate)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Important Dates\n",
    "\n",
    "<br>\n",
    "<font color=\"blue\" size=4>\n",
    "\n",
    "Task announcement: Nov 20, 2020\n",
    "\n",
    "Release of Trail data: Nov 20, 2020\n",
    "\n",
    "Release of Training data: Nov 20, 2020\n",
    "\n",
    "Release of Test data: Jan 2, 2021\n",
    "\n",
    "Run submission deadline: Jan 10, 2021\n",
    "\n",
    "Results declared: Jan 15, 2021\n",
    "\n",
    "Paper submission: Jan 30, 2021\n",
    "\n",
    "Peer review notification: Feb 18, 2021\n",
    "\n",
    "Camera-ready paper due: Mar 1, 2021\n",
    "\n",
    "Workshop Dates: April 19-20, 2021\n",
    "</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting transformers\n",
      "  Downloading transformers-4.0.0-py3-none-any.whl (1.4 MB)\n",
      "Requirement already satisfied: packaging in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from transformers) (20.7)\n",
      "Requirement already satisfied: tqdm>=4.27 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from transformers) (4.54.1)\n",
      "Requirement already satisfied: requests in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from transformers) (2.25.0)\n",
      "Requirement already satisfied: numpy in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from transformers) (1.19.4)\n",
      "Collecting filelock\n",
      "  Downloading filelock-3.0.12-py3-none-any.whl (7.6 kB)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from packaging->transformers) (2.4.7)\n",
      "Collecting regex!=2019.12.17\n",
      "  Downloading regex-2020.11.13-cp38-cp38-win_amd64.whl (270 kB)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from requests->transformers) (1.26.2)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from requests->transformers) (2020.11.8)\n",
      "Requirement already satisfied: idna<3,>=2.5 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from requests->transformers) (2.10)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from requests->transformers) (3.0.4)\n",
      "Collecting sacremoses\n",
      "  Using cached sacremoses-0.0.43.tar.gz (883 kB)\n",
      "Requirement already satisfied: six in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from sacremoses->transformers) (1.15.0)\n",
      "Requirement already satisfied: joblib in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from sacremoses->transformers) (0.17.0)\n",
      "Requirement already satisfied: tqdm>=4.27 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from transformers) (4.54.1)\n",
      "Collecting click\n",
      "  Downloading click-7.1.2-py2.py3-none-any.whl (82 kB)\n",
      "Collecting tokenizers==0.9.4\n",
      "  Downloading tokenizers-0.9.4-cp38-cp38-win_amd64.whl (1.9 MB)\n",
      "Building wheels for collected packages: sacremoses\n",
      "  Building wheel for sacremoses (setup.py): started\n",
      "  Building wheel for sacremoses (setup.py): finished with status 'done'\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.43-py3-none-any.whl size=893259 sha256=2d4f1eb6d0d8b7eccd13dccd9f2008d7b274d66a674999301fdb30b7ba26e3fa\n",
      "  Stored in directory: c:\\users\\e-neo.desktop-m4a5e83\\appdata\\local\\pip\\cache\\wheels\\7b\\78\\f4\\27d43a65043e1b75dbddaa421b573eddc67e712be4b1c80677\n",
      "Successfully built sacremoses\n",
      "Installing collected packages: regex, click, tokenizers, sacremoses, filelock, transformers\n",
      "Successfully installed click-7.1.2 filelock-3.0.12 regex-2020.11.13 sacremoses-0.0.43 tokenizers-0.9.4 transformers-4.0.0\n"
     ]
    }
   ],
   "source": [
    "!pip install --upgrade transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting simpletransformers\n",
      "  Downloading simpletransformers-0.51.0-py3-none-any.whl (224 kB)\n",
      "Requirement already satisfied: tokenizers in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (0.9.4)\n",
      "Requirement already satisfied: transformers>=4.0.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (4.0.0)\n",
      "Requirement already satisfied: tqdm>=4.47.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (4.54.1)\n",
      "Requirement already satisfied: requests in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (2.25.0)\n",
      "Requirement already satisfied: scikit-learn in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (0.23.2)\n",
      "Requirement already satisfied: regex in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (2020.11.13)\n",
      "Requirement already satisfied: scipy in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (1.5.4)\n",
      "Requirement already satisfied: numpy in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (1.19.4)\n",
      "Requirement already satisfied: pandas in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (1.1.4)\n",
      "Requirement already satisfied: pytz>=2017.2 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from pandas->simpletransformers) (2020.4)\n",
      "Requirement already satisfied: numpy in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (1.19.4)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from pandas->simpletransformers) (2.8.1)\n",
      "Requirement already satisfied: six>=1.5 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from python-dateutil>=2.7.3->pandas->simpletransformers) (1.15.0)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from requests->simpletransformers) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from requests->simpletransformers) (2020.11.8)\n",
      "Requirement already satisfied: idna<3,>=2.5 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from requests->simpletransformers) (2.10)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from requests->simpletransformers) (1.26.2)\n",
      "Requirement already satisfied: numpy in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (1.19.4)\n",
      "Requirement already satisfied: scipy in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (1.5.4)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from scikit-learn->simpletransformers) (2.1.0)\n",
      "Requirement already satisfied: joblib>=0.11 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from scikit-learn->simpletransformers) (0.17.0)\n",
      "Requirement already satisfied: numpy in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (1.19.4)\n",
      "Collecting seqeval\n",
      "  Downloading seqeval-1.2.2.tar.gz (43 kB)\n",
      "Requirement already satisfied: numpy in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (1.19.4)\n",
      "Requirement already satisfied: scikit-learn in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (0.23.2)\n",
      "Collecting streamlit\n",
      "  Downloading streamlit-0.72.0-py2.py3-none-any.whl (7.4 MB)\n",
      "Requirement already satisfied: pillow>=6.2.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from streamlit->simpletransformers) (8.0.1)\n",
      "Requirement already satisfied: pandas in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (1.1.4)\n",
      "Requirement already satisfied: click>=7.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from streamlit->simpletransformers) (7.1.2)\n",
      "Requirement already satisfied: numpy in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (1.19.4)\n",
      "Requirement already satisfied: requests in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (2.25.0)\n",
      "Requirement already satisfied: pyarrow in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from streamlit->simpletransformers) (2.0.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from pandas->simpletransformers) (2.8.1)\n",
      "Requirement already satisfied: packaging in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from streamlit->simpletransformers) (20.7)\n",
      "Requirement already satisfied: tornado>=5.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from streamlit->simpletransformers) (6.1)\n",
      "Collecting altair>=3.2.0\n",
      "  Downloading altair-4.1.0-py3-none-any.whl (727 kB)\n",
      "Requirement already satisfied: numpy in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (1.19.4)\n",
      "Requirement already satisfied: pandas in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (1.1.4)\n",
      "Collecting astor\n",
      "  Using cached astor-0.8.1-py2.py3-none-any.whl (27 kB)\n",
      "Collecting base58\n",
      "  Downloading base58-2.0.1-py3-none-any.whl (4.3 kB)\n",
      "Collecting blinker\n",
      "  Downloading blinker-1.4.tar.gz (111 kB)\n",
      "Collecting cachetools>=4.0\n",
      "  Downloading cachetools-4.1.1-py3-none-any.whl (10 kB)\n",
      "Collecting entrypoints\n",
      "  Downloading entrypoints-0.3-py2.py3-none-any.whl (11 kB)\n",
      "Collecting gitpython\n",
      "  Downloading GitPython-3.1.11-py3-none-any.whl (159 kB)\n",
      "Collecting gitdb<5,>=4.0.1\n",
      "  Downloading gitdb-4.0.5-py3-none-any.whl (63 kB)\n",
      "Collecting jinja2\n",
      "  Downloading Jinja2-2.11.2-py2.py3-none-any.whl (125 kB)\n",
      "Collecting jsonschema\n",
      "  Downloading jsonschema-3.2.0-py2.py3-none-any.whl (56 kB)\n",
      "Requirement already satisfied: setuptools in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from jsonschema->altair>=3.2.0->streamlit->simpletransformers) (50.3.2.post20201202)\n",
      "Requirement already satisfied: six>=1.5 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from python-dateutil>=2.7.3->pandas->simpletransformers) (1.15.0)\n",
      "Collecting attrs>=17.4.0\n",
      "  Downloading attrs-20.3.0-py2.py3-none-any.whl (49 kB)\n",
      "Collecting MarkupSafe>=0.23\n",
      "  Downloading MarkupSafe-1.1.1-cp38-cp38-win_amd64.whl (16 kB)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from packaging->streamlit->simpletransformers) (2.4.7)\n",
      "Collecting protobuf!=3.11,>=3.6.0\n",
      "  Downloading protobuf-3.14.0-py2.py3-none-any.whl (173 kB)\n",
      "Requirement already satisfied: six>=1.5 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from python-dateutil>=2.7.3->pandas->simpletransformers) (1.15.0)\n",
      "Requirement already satisfied: numpy in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (1.19.4)\n",
      "Collecting pydeck>=0.1.dev5\n",
      "  Downloading pydeck-0.5.0-py2.py3-none-any.whl (4.5 MB)\n",
      "Requirement already satisfied: numpy in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (1.19.4)\n",
      "Requirement already satisfied: traitlets>=4.3.2 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from pydeck>=0.1.dev5->streamlit->simpletransformers) (5.0.5)\n",
      "Requirement already satisfied: ipykernel>=5.1.2 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from pydeck>=0.1.dev5->streamlit->simpletransformers) (5.3.4)\n",
      "Requirement already satisfied: traitlets>=4.3.2 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from pydeck>=0.1.dev5->streamlit->simpletransformers) (5.0.5)\n",
      "Requirement already satisfied: jupyter-client in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (6.1.7)\n",
      "Requirement already satisfied: tornado>=5.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from streamlit->simpletransformers) (6.1)\n",
      "Requirement already satisfied: ipython>=5.0.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (7.19.0)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from ipython>=5.0.0->ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (3.0.8)\n",
      "Requirement already satisfied: pickleshare in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from ipython>=5.0.0->ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.7.5)\n",
      "Requirement already satisfied: pygments in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from ipython>=5.0.0->ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (2.7.2)\n",
      "Requirement already satisfied: decorator in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from ipython>=5.0.0->ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (4.4.2)\n",
      "Requirement already satisfied: traitlets>=4.3.2 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from pydeck>=0.1.dev5->streamlit->simpletransformers) (5.0.5)\n",
      "Requirement already satisfied: setuptools in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from jsonschema->altair>=3.2.0->streamlit->simpletransformers) (50.3.2.post20201202)\n",
      "Requirement already satisfied: jedi>=0.10 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from ipython>=5.0.0->ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.17.2)\n",
      "Requirement already satisfied: colorama in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from ipython>=5.0.0->ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.4.4)\n",
      "Requirement already satisfied: backcall in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from ipython>=5.0.0->ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.2.0)\n",
      "Collecting ipywidgets>=7.0.0\n",
      "  Downloading ipywidgets-7.5.1-py2.py3-none-any.whl (121 kB)\n",
      "Requirement already satisfied: ipython>=5.0.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (7.19.0)\n",
      "Requirement already satisfied: ipykernel>=5.1.2 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from pydeck>=0.1.dev5->streamlit->simpletransformers) (5.3.4)\n",
      "Requirement already satisfied: traitlets>=4.3.2 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from pydeck>=0.1.dev5->streamlit->simpletransformers) (5.0.5)\n",
      "Requirement already satisfied: parso<0.8.0,>=0.7.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from jedi>=0.10->ipython>=5.0.0->ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.7.0)\n",
      "Requirement already satisfied: jupyter-core>=4.6.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from jupyter-client->ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (4.7.0)\n",
      "Requirement already satisfied: pyzmq>=13 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from jupyter-client->ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (20.0.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from pandas->simpletransformers) (2.8.1)\n",
      "Requirement already satisfied: tornado>=5.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from streamlit->simpletransformers) (6.1)\n",
      "Requirement already satisfied: traitlets>=4.3.2 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from pydeck>=0.1.dev5->streamlit->simpletransformers) (5.0.5)\n",
      "Requirement already satisfied: pywin32>=1.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from jupyter-core>=4.6.0->jupyter-client->ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (227)\n",
      "Requirement already satisfied: traitlets>=4.3.2 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from pydeck>=0.1.dev5->streamlit->simpletransformers) (5.0.5)\n",
      "Collecting nbformat>=4.2.0\n",
      "  Downloading nbformat-5.0.8-py3-none-any.whl (172 kB)\n",
      "Requirement already satisfied: traitlets>=4.3.2 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from pydeck>=0.1.dev5->streamlit->simpletransformers) (5.0.5)\n",
      "Requirement already satisfied: ipython-genutils in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from nbformat>=4.2.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.2.0)\n",
      "Requirement already satisfied: jupyter-core>=4.6.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from jupyter-client->ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (4.7.0)\n",
      "Requirement already satisfied: wcwidth in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython>=5.0.0->ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.2.5)\n",
      "Collecting pyrsistent>=0.14.0\n",
      "  Downloading pyrsistent-0.17.3.tar.gz (106 kB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting smmap<4,>=3.0.1\n",
      "  Downloading smmap-3.0.4-py2.py3-none-any.whl (25 kB)\n",
      "Collecting tensorboardx\n",
      "  Downloading tensorboardX-2.1-py2.py3-none-any.whl (308 kB)\n",
      "Requirement already satisfied: six>=1.5 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from python-dateutil>=2.7.3->pandas->simpletransformers) (1.15.0)\n",
      "Requirement already satisfied: numpy in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (1.19.4)\n",
      "Collecting toml\n",
      "  Downloading toml-0.10.2-py2.py3-none-any.whl (16 kB)\n",
      "Collecting toolz\n",
      "  Downloading toolz-0.11.1-py3-none-any.whl (55 kB)\n",
      "Requirement already satisfied: ipython-genutils in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from nbformat>=4.2.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.2.0)\n",
      "Requirement already satisfied: packaging in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from streamlit->simpletransformers) (20.7)\n",
      "Requirement already satisfied: tokenizers in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (0.9.4)\n",
      "Requirement already satisfied: requests in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (2.25.0)\n",
      "Requirement already satisfied: tqdm>=4.47.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (4.54.1)\n",
      "Requirement already satisfied: regex in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (2020.11.13)\n",
      "Requirement already satisfied: filelock in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from transformers>=4.0.0->simpletransformers) (3.0.12)\n",
      "Requirement already satisfied: numpy in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (1.19.4)\n",
      "Requirement already satisfied: sacremoses in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from transformers>=4.0.0->simpletransformers) (0.0.43)\n",
      "Requirement already satisfied: tqdm>=4.47.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (4.54.1)\n",
      "Requirement already satisfied: six>=1.5 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from python-dateutil>=2.7.3->pandas->simpletransformers) (1.15.0)\n",
      "Requirement already satisfied: click>=7.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from streamlit->simpletransformers) (7.1.2)\n",
      "Requirement already satisfied: joblib>=0.11 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from scikit-learn->simpletransformers) (0.17.0)\n",
      "Requirement already satisfied: regex in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (2020.11.13)\n",
      "Collecting tzlocal\n",
      "  Using cached tzlocal-2.1-py2.py3-none-any.whl (16 kB)\n",
      "Requirement already satisfied: pytz>=2017.2 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from pandas->simpletransformers) (2020.4)\n",
      "Collecting validators\n",
      "  Downloading validators-0.18.1-py3-none-any.whl (19 kB)\n",
      "Requirement already satisfied: six>=1.5 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from python-dateutil>=2.7.3->pandas->simpletransformers) (1.15.0)\n",
      "Requirement already satisfied: decorator in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from ipython>=5.0.0->ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (4.4.2)\n",
      "Collecting wandb\n",
      "  Downloading wandb-0.10.12-py2.py3-none-any.whl (1.8 MB)\n",
      "Requirement already satisfied: six>=1.5 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from python-dateutil>=2.7.3->pandas->simpletransformers) (1.15.0)\n",
      "Requirement already satisfied: psutil>=5.0.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from wandb->simpletransformers) (5.7.3)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from pandas->simpletransformers) (2.8.1)\n",
      "Requirement already satisfied: PyYAML in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from wandb->simpletransformers) (5.3.1)\n",
      "Requirement already satisfied: requests in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from simpletransformers) (2.25.0)\n",
      "Requirement already satisfied: click>=7.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from streamlit->simpletransformers) (7.1.2)\n",
      "Collecting configparser>=3.8.1\n",
      "  Downloading configparser-5.0.1-py3-none-any.whl (22 kB)\n",
      "Collecting docker-pycreds>=0.4.0\n",
      "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
      "Requirement already satisfied: six>=1.5 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from python-dateutil>=2.7.3->pandas->simpletransformers) (1.15.0)\n",
      "Collecting promise<3,>=2.0\n",
      "  Using cached promise-2.3.tar.gz (19 kB)\n",
      "Requirement already satisfied: six>=1.5 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from python-dateutil>=2.7.3->pandas->simpletransformers) (1.15.0)\n",
      "Collecting sentry-sdk>=0.4.0\n",
      "  Downloading sentry_sdk-0.19.4-py2.py3-none-any.whl (128 kB)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from requests->simpletransformers) (1.26.2)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from requests->simpletransformers) (2020.11.8)\n",
      "Collecting shortuuid>=0.5.0\n",
      "  Downloading shortuuid-1.0.1-py3-none-any.whl (7.5 kB)\n",
      "Collecting subprocess32>=3.5.3\n",
      "  Downloading subprocess32-3.5.4.tar.gz (97 kB)\n",
      "Collecting watchdog\n",
      "  Downloading watchdog-0.10.4.tar.gz (98 kB)\n",
      "Collecting pathtools>=0.1.1\n",
      "  Downloading pathtools-0.1.2.tar.gz (11 kB)\n",
      "Collecting widgetsnbextension~=3.5.0\n",
      "  Downloading widgetsnbextension-3.5.1-py2.py3-none-any.whl (2.2 MB)\n",
      "Collecting notebook>=4.4.1\n",
      "  Downloading notebook-6.1.5-py3-none-any.whl (9.5 MB)\n",
      "Requirement already satisfied: pyzmq>=13 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from jupyter-client->ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (20.0.0)\n",
      "Requirement already satisfied: traitlets>=4.3.2 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from pydeck>=0.1.dev5->streamlit->simpletransformers) (5.0.5)\n",
      "Requirement already satisfied: jupyter-client in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (6.1.7)\n",
      "Requirement already satisfied: tornado>=5.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from streamlit->simpletransformers) (6.1)\n",
      "Requirement already satisfied: ipykernel>=5.1.2 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from pydeck>=0.1.dev5->streamlit->simpletransformers) (5.3.4)\n",
      "Requirement already satisfied: ipython-genutils in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from nbformat>=4.2.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.2.0)\n",
      "Requirement already satisfied: jupyter-core>=4.6.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from jupyter-client->ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (4.7.0)\n",
      "Collecting argon2-cffi\n",
      "  Downloading argon2_cffi-20.1.0-cp38-cp38-win_amd64.whl (42 kB)\n",
      "Requirement already satisfied: six>=1.5 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from python-dateutil>=2.7.3->pandas->simpletransformers) (1.15.0)\n",
      "Collecting cffi>=1.0.0\n",
      "  Downloading cffi-1.14.4-cp38-cp38-win_amd64.whl (179 kB)\n",
      "Collecting nbconvert\n",
      "  Downloading nbconvert-6.0.7-py3-none-any.whl (552 kB)\n",
      "Requirement already satisfied: traitlets>=4.3.2 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from pydeck>=0.1.dev5->streamlit->simpletransformers) (5.0.5)\n",
      "Requirement already satisfied: jupyter-core>=4.6.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from jupyter-client->ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (4.7.0)\n",
      "Requirement already satisfied: pygments in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from ipython>=5.0.0->ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (2.7.2)\n",
      "Collecting bleach\n",
      "  Downloading bleach-3.2.1-py2.py3-none-any.whl (145 kB)\n",
      "Requirement already satisfied: six>=1.5 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from python-dateutil>=2.7.3->pandas->simpletransformers) (1.15.0)\n",
      "Requirement already satisfied: packaging in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from streamlit->simpletransformers) (20.7)\n",
      "Collecting defusedxml\n",
      "  Downloading defusedxml-0.6.0-py2.py3-none-any.whl (23 kB)\n",
      "Collecting jupyterlab-pygments\n",
      "  Downloading jupyterlab_pygments-0.1.2-py2.py3-none-any.whl (4.6 kB)\n",
      "Requirement already satisfied: pygments in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from ipython>=5.0.0->ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (2.7.2)\n",
      "Collecting mistune<2,>=0.8.1\n",
      "  Downloading mistune-0.8.4-py2.py3-none-any.whl (16 kB)\n",
      "Collecting nbclient<0.6.0,>=0.5.0\n",
      "  Downloading nbclient-0.5.1-py3-none-any.whl (65 kB)\n",
      "Requirement already satisfied: traitlets>=4.3.2 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from pydeck>=0.1.dev5->streamlit->simpletransformers) (5.0.5)\n",
      "Requirement already satisfied: jupyter-client in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from ipykernel>=5.1.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (6.1.7)\n",
      "Collecting async-generator\n",
      "  Using cached async_generator-1.10-py3-none-any.whl (18 kB)\n",
      "Collecting nest-asyncio\n",
      "  Downloading nest_asyncio-1.4.3-py3-none-any.whl (5.3 kB)\n",
      "Collecting pandocfilters>=1.4.1\n",
      "  Downloading pandocfilters-1.4.3.tar.gz (16 kB)\n",
      "Collecting prometheus-client\n",
      "  Downloading prometheus_client-0.9.0-py2.py3-none-any.whl (53 kB)\n",
      "Collecting pycparser\n",
      "  Downloading pycparser-2.20-py2.py3-none-any.whl (112 kB)\n",
      "Collecting Send2Trash\n",
      "  Downloading Send2Trash-1.5.0-py3-none-any.whl (12 kB)\n",
      "Collecting terminado>=0.8.3\n",
      "  Downloading terminado-0.9.1-py3-none-any.whl (13 kB)\n",
      "Requirement already satisfied: tornado>=5.0 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from streamlit->simpletransformers) (6.1)\n",
      "Collecting pywinpty>=0.5\n",
      "  Downloading pywinpty-0.5.7-cp38-cp38-win_amd64.whl (1.3 MB)\n",
      "Collecting testpath\n",
      "  Downloading testpath-0.4.4-py2.py3-none-any.whl (163 kB)\n",
      "Collecting webencodings\n",
      "  Downloading webencodings-0.5.1-py2.py3-none-any.whl (11 kB)\n",
      "Building wheels for collected packages: seqeval, blinker, pyrsistent, promise, subprocess32, watchdog, pathtools, pandocfilters\n",
      "  Building wheel for seqeval (setup.py): started\n",
      "  Building wheel for seqeval (setup.py): finished with status 'done'\n",
      "  Created wheel for seqeval: filename=seqeval-1.2.2-py3-none-any.whl size=16170 sha256=2efe9dabdb9d406f8abc017c2fc035acba377fb973854b741f5b95625b0800af\n",
      "  Stored in directory: c:\\users\\e-neo.desktop-m4a5e83\\appdata\\local\\pip\\cache\\wheels\\ad\\5c\\ba\\05fa33fa5855777b7d686e843ec07452f22a66a138e290e732\n",
      "  Building wheel for blinker (setup.py): started\n",
      "  Building wheel for blinker (setup.py): finished with status 'done'\n",
      "  Created wheel for blinker: filename=blinker-1.4-py3-none-any.whl size=13450 sha256=98fdbb9e379263d285547cf6bddc3809e97cafbca15581e0af00f3de9a646aeb\n",
      "  Stored in directory: c:\\users\\e-neo.desktop-m4a5e83\\appdata\\local\\pip\\cache\\wheels\\b7\\a5\\68\\fe632054a5eadd531c7a49d740c50eb6adfbeca822b4eab8d4\n",
      "  Building wheel for pyrsistent (setup.py): started\n",
      "  Building wheel for pyrsistent (setup.py): finished with status 'done'\n",
      "  Created wheel for pyrsistent: filename=pyrsistent-0.17.3-cp38-cp38-win_amd64.whl size=69980 sha256=9b7e2c909785aef0f2522344d92b991aa0ca75c285e566821df4b5b7581ffc60\n",
      "  Stored in directory: c:\\users\\e-neo.desktop-m4a5e83\\appdata\\local\\pip\\cache\\wheels\\3d\\22\\08\\7042eb6309c650c7b53615d5df5cc61f1ea9680e7edd3a08d2\n",
      "  Building wheel for promise (setup.py): started\n",
      "  Building wheel for promise (setup.py): finished with status 'done'\n",
      "  Created wheel for promise: filename=promise-2.3-py3-none-any.whl size=21494 sha256=c2e4d937da52585617b8b76c93f535aa3dc4c61f4d4f28e1a02944bb86ea8d66\n",
      "  Stored in directory: c:\\users\\e-neo.desktop-m4a5e83\\appdata\\local\\pip\\cache\\wheels\\54\\aa\\01\\724885182f93150035a2a91bce34a12877e8067a97baaf5dc8\n",
      "  Building wheel for subprocess32 (setup.py): started\n",
      "  Building wheel for subprocess32 (setup.py): finished with status 'done'\n",
      "  Created wheel for subprocess32: filename=subprocess32-3.5.4-py3-none-any.whl size=6488 sha256=efcbeac3eb0846c0442045cc10c63087c49c0506ae55fc35377752bb98561003\n",
      "  Stored in directory: c:\\users\\e-neo.desktop-m4a5e83\\appdata\\local\\pip\\cache\\wheels\\9f\\69\\d1\\50b39b308a87998eaf5c1d9095e5a5bd2ad98501e2b7936d36\n",
      "  Building wheel for watchdog (setup.py): started\n",
      "  Building wheel for watchdog (setup.py): finished with status 'done'\n",
      "  Created wheel for watchdog: filename=watchdog-0.10.4-py3-none-any.whl size=74840 sha256=89e80f7a5484a3a534b95c33cf938d81d44d223413311337fa7135c2e3422103\n",
      "  Stored in directory: c:\\users\\e-neo.desktop-m4a5e83\\appdata\\local\\pip\\cache\\wheels\\0a\\9f\\05\\5628e30bbbeb4f59e31aa98a59735ca3a8ca3ca4082ba83a13\n",
      "  Building wheel for pathtools (setup.py): started\n",
      "  Building wheel for pathtools (setup.py): finished with status 'done'\n",
      "  Created wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8785 sha256=71e9df6ed3f091305e9810f26c883bc34e840ba9f50e6faf9413581fd0160eb3\n",
      "  Stored in directory: c:\\users\\e-neo.desktop-m4a5e83\\appdata\\local\\pip\\cache\\wheels\\4c\\8e\\7e\\72fbc243e1aeecae64a96875432e70d4e92f3d2d18123be004\n",
      "  Building wheel for pandocfilters (setup.py): started\n",
      "  Building wheel for pandocfilters (setup.py): finished with status 'done'\n",
      "  Created wheel for pandocfilters: filename=pandocfilters-1.4.3-py3-none-any.whl size=7991 sha256=845088bb5ae984863dac1e07b4bd0e6620e459fbc2fe35dddf359222ce4b6796\n",
      "  Stored in directory: c:\\users\\e-neo.desktop-m4a5e83\\appdata\\local\\pip\\cache\\wheels\\fc\\39\\52\\8d6f3cec1cca4ceb44d658427c35711b19d89dbc4914af657f\n",
      "Successfully built seqeval blinker pyrsistent promise subprocess32 watchdog pathtools pandocfilters\n",
      "Installing collected packages: pyrsistent, attrs, jsonschema, webencodings, pycparser, nest-asyncio, nbformat, MarkupSafe, async-generator, testpath, pywinpty, pandocfilters, nbclient, mistune, jupyterlab-pygments, jinja2, entrypoints, defusedxml, cffi, bleach, terminado, Send2Trash, prometheus-client, nbconvert, argon2-cffi, notebook, widgetsnbextension, smmap, toolz, pathtools, ipywidgets, gitdb, watchdog, validators, tzlocal, toml, subprocess32, shortuuid, sentry-sdk, pydeck, protobuf, promise, gitpython, docker-pycreds, configparser, cachetools, blinker, base58, astor, altair, wandb, tensorboardx, streamlit, seqeval, simpletransformers\n",
      "Successfully installed MarkupSafe-1.1.1 Send2Trash-1.5.0 altair-4.1.0 argon2-cffi-20.1.0 astor-0.8.1 async-generator-1.10 attrs-20.3.0 base58-2.0.1 bleach-3.2.1 blinker-1.4 cachetools-4.1.1 cffi-1.14.4 configparser-5.0.1 defusedxml-0.6.0 docker-pycreds-0.4.0 entrypoints-0.3 gitdb-4.0.5 gitpython-3.1.11 ipywidgets-7.5.1 jinja2-2.11.2 jsonschema-3.2.0 jupyterlab-pygments-0.1.2 mistune-0.8.4 nbclient-0.5.1 nbconvert-6.0.7 nbformat-5.0.8 nest-asyncio-1.4.3 notebook-6.1.5 pandocfilters-1.4.3 pathtools-0.1.2 prometheus-client-0.9.0 promise-2.3 protobuf-3.14.0 pycparser-2.20 pydeck-0.5.0 pyrsistent-0.17.3 pywinpty-0.5.7 sentry-sdk-0.19.4 seqeval-1.2.2 shortuuid-1.0.1 simpletransformers-0.51.0 smmap-3.0.4 streamlit-0.72.0 subprocess32-3.5.4 tensorboardx-2.1 terminado-0.9.1 testpath-0.4.4 toml-0.10.2 toolz-0.11.1 tzlocal-2.1 validators-0.18.1 wandb-0.10.12 watchdog-0.10.4 webencodings-0.5.1 widgetsnbextension-3.5.1\n"
     ]
    }
   ],
   "source": [
    "!pip install simpletransformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: gputil in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (1.4.0)\n",
      "Requirement already satisfied: psutil in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (5.7.3)\n",
      "Collecting humanize\n",
      "  Downloading humanize-3.1.0-py3-none-any.whl (69 kB)\n",
      "Requirement already satisfied: setuptools in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from humanize) (50.3.2.post20201202)\n",
      "Installing collected packages: humanize\n",
      "Successfully installed humanize-3.1.0\n"
     ]
    }
   ],
   "source": [
    "!pip install gputil\n",
    "!pip install psutil\n",
    "!pip install humanize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting keras\n",
      "  Downloading Keras-2.4.3-py2.py3-none-any.whl (36 kB)\n",
      "Requirement already satisfied: numpy>=1.9.1 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from keras) (1.19.4)\n",
      "Requirement already satisfied: pyyaml in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from keras) (5.3.1)\n",
      "Requirement already satisfied: scipy>=0.14 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from keras) (1.5.4)\n",
      "Collecting h5py\n",
      "  Downloading h5py-3.1.0-cp38-cp38-win_amd64.whl (2.7 MB)\n",
      "Requirement already satisfied: numpy>=1.9.1 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from keras) (1.19.4)\n",
      "Requirement already satisfied: numpy>=1.9.1 in c:\\programdata\\anaconda3\\envs\\torch\\lib\\site-packages (from keras) (1.19.4)\n",
      "Installing collected packages: h5py, keras\n",
      "Successfully installed h5py-3.1.0 keras-2.4.3\n"
     ]
    }
   ],
   "source": [
    "!pip install keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Loading and Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [],
   "source": [
    "eng_train = pd.read_csv('english_hope_train.csv',sep='\\t',names = ['text','labels','del']).drop(columns=['del'])\n",
    "eng_dev = pd.read_csv('english_hope_dev.csv',sep='\\t',names = ['text','labels','del']).drop(columns=['del'])\n",
    "mala_train = pd.read_csv('malayalam_hope_train.csv',sep='\\t',names = ['text','labels','del']).drop(columns=['del'])\n",
    "mala_dev = pd.read_csv('malayalam_hope_dev.csv',sep='\\t',names = ['text','labels','del']).drop(columns=['del'])\n",
    "tamil_train = pd.read_csv('tamil_hope_first_train.csv',sep='\\t',names = ['text','labels','del']).drop(columns=['del'])\n",
    "tamil_dev = pd.read_csv('tamil_hope_first_dev.csv',sep='\\t',names = ['text','labels','del']).drop(columns=['del'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [],
   "source": [
    "eng_test = pd.read_csv('english_hope_test.csv',sep=';')\n",
    "eng_test = eng_test.drop(eng_test.columns[-1], axis=1)\n",
    "\n",
    "mala_test = pd.read_csv('malayalam_hope_test.csv',sep=';')\n",
    "mala_test = mala_test.drop(mala_test.columns[-1], axis=1)\n",
    "\n",
    "tamil_test = pd.read_csv('tamil_hope_first_test.csv',sep=';')\n",
    "tamil_test = tamil_test.drop(tamil_test.columns[-1], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Expolration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Non_hope_speech    20778\n",
       "Hope_speech         1962\n",
       "not-English           22\n",
       "Name: labels, dtype: int64"
      ]
     },
     "execution_count": 256,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eng_train['labels'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Non_hope_speech    7872\n",
       "Hope_speech        6327\n",
       "not-Tamil          1961\n",
       "Name: labels, dtype: int64"
      ]
     },
     "execution_count": 257,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tamil_train['labels'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Non_hope_speech    6205\n",
       "Hope_speech        1668\n",
       "not-malayalam       691\n",
       "Name: labels, dtype: int64"
      ]
     },
     "execution_count": 258,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mala_train['labels'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Note:** There seems to be a clear class imbalance here."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Selecting Language"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Choose a langauge: \n",
      " 1) eng \t 2) tamil \t 3) mala\n",
      " Input: eng\n"
     ]
    }
   ],
   "source": [
    "lang = input('Choose a langauge: \\n 1) eng \\t 2) tamil \\t 3) mala\\n Input: ')\n",
    "\n",
    "if lang == 'eng':\n",
    "    df_train = eng_train\n",
    "    df_dev = eng_dev\n",
    "    df_test = eng_test\n",
    "\n",
    "elif lang == 'tamil':\n",
    "    df_train = tamil_train\n",
    "    df_dev = tamil_dev\n",
    "    df_test = tamil_test\n",
    "    \n",
    "elif lang == 'mala':\n",
    "    df_train = mala_train\n",
    "    df_dev = mala_dev\n",
    "    df_test = mala_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Compute class weights? (Y/N)Y\n"
     ]
    }
   ],
   "source": [
    "use_weights = str(input('Compute class weights? (Y/N) '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>these tiktoks radiate gay chaotic energy and i...</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>@Champions Again He got killed for using false...</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>It's not that all lives don't matter</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Is it really that difficult to understand? Bla...</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Whenever we say black isn't that racists?  Why...</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22757</th>\n",
       "      <td>It's a load of bollocks every life matters sim...</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22758</th>\n",
       "      <td>no say it because all lives matter! deku would...</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22759</th>\n",
       "      <td>God says her life matters</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22760</th>\n",
       "      <td>This video is just shit. A bunch of whiny ass ...</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22761</th>\n",
       "      <td>Mc Fortnut2821 she did 4 months ago in west ch...</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>22762 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text           labels\n",
       "0      these tiktoks radiate gay chaotic energy and i...  Non_hope_speech\n",
       "1      @Champions Again He got killed for using false...  Non_hope_speech\n",
       "2                   It's not that all lives don't matter  Non_hope_speech\n",
       "3      Is it really that difficult to understand? Bla...  Non_hope_speech\n",
       "4      Whenever we say black isn't that racists?  Why...  Non_hope_speech\n",
       "...                                                  ...              ...\n",
       "22757  It's a load of bollocks every life matters sim...  Non_hope_speech\n",
       "22758  no say it because all lives matter! deku would...  Non_hope_speech\n",
       "22759                          God says her life matters  Non_hope_speech\n",
       "22760  This video is just shit. A bunch of whiny ass ...  Non_hope_speech\n",
       "22761  Mc Fortnut2821 she did 4 months ago in west ch...  Non_hope_speech\n",
       "\n",
       "[22762 rows x 2 columns]"
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What do you mean by the word sniped?</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I love this video!! I’m bisexual and it’s just...</td>\n",
       "      <td>Hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ya the irony but then i don't want to come off...</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A PERSON'S CHARACTER MATTERS. PERIOD!!</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>@Blaster of Gasters</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2841</th>\n",
       "      <td>+Ashrenneemakeup I think it's all a deliberate...</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2842</th>\n",
       "      <td>Sheriff David Clarke. This guy is amazing.</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2843</th>\n",
       "      <td>Abandorn Hope Situation</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2844</th>\n",
       "      <td>Sheriff Clarke you are a person of such strong...</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2845</th>\n",
       "      <td>Sanders has no room to talk. If there's one pe...</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2846 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text            label\n",
       "0                  What do you mean by the word sniped?  Non_hope_speech\n",
       "1     I love this video!! I’m bisexual and it’s just...      Hope_speech\n",
       "2     ya the irony but then i don't want to come off...  Non_hope_speech\n",
       "3                A PERSON'S CHARACTER MATTERS. PERIOD!!  Non_hope_speech\n",
       "4                                   @Blaster of Gasters  Non_hope_speech\n",
       "...                                                 ...              ...\n",
       "2841  +Ashrenneemakeup I think it's all a deliberate...  Non_hope_speech\n",
       "2842         Sheriff David Clarke. This guy is amazing.  Non_hope_speech\n",
       "2843                            Abandorn Hope Situation  Non_hope_speech\n",
       "2844  Sheriff Clarke you are a person of such strong...  Non_hope_speech\n",
       "2845  Sanders has no room to talk. If there's one pe...  Non_hope_speech\n",
       "\n",
       "[2846 rows x 2 columns]"
      ]
     },
     "execution_count": 261,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## English hope speech model\n",
    "\n",
    "I use RoBERTA. <br> \n",
    "[Reference blogpost](https://medium.com/swlh/text-classification-using-transformers-pytorch-implementation-5ff9f21bd106)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [],
   "source": [
    "import psutil\n",
    "import humanize\n",
    "import os\n",
    "import GPUtil as GPU\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import warnings\n",
    "warnings.simplefilter('ignore')\n",
    "import gc\n",
    "from scipy.special import softmax\n",
    "from simpletransformers.classification import ClassificationModel\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold, KFold \n",
    "import sklearn\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.metrics import *\n",
    "from sklearn.model_selection import *\n",
    "import re\n",
    "import random\n",
    "import torch\n",
    "\n",
    "\n",
    "#stdz seeds\n",
    "\n",
    "def seed_all(seed_value):\n",
    "    random.seed(seed_value) # Python\n",
    "    np.random.seed(seed_value) # cpu vars\n",
    "    torch.manual_seed(seed_value) # cpu  vars\n",
    "    \n",
    "    if torch.cuda.is_available(): \n",
    "        torch.cuda.manual_seed(seed_value)\n",
    "        torch.cuda.manual_seed_all(seed_value) # gpu vars\n",
    "        torch.backends.cudnn.deterministic = True  #needed\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "SEED = 2\n",
    "seed_all(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Non_hope_speech', 'Hope_speech', 'not-English']"
      ]
     },
     "execution_count": 264,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = list(df_train['labels'].unique())\n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make a one hot encoding vector for these labels\n",
    "df_train ['labels'] = df_train['labels'].astype('category')\n",
    "df_dev ['labels'] = df_dev['labels'].astype('category')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Non_hope_speech    20778\n",
      "Hope_speech         1962\n",
      "not-English           22\n",
      "Name: labels, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "train_y = df_train['labels']\n",
    "print(train_y.value_counts())\n",
    "train_y = train_y.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Non_hope_speech    2569\n",
      "Hope_speech         272\n",
      "not-English           2\n",
      "Name: labels, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "dev_y = df_dev['labels']\n",
    "#dev_y = eng_dev['class'].cat.codes\n",
    "print(dev_y.value_counts())\n",
    "#eng_dev['class'] = dev_y\n",
    "dev_y = dev_y.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[array(['Hope_speech', 'Non_hope_speech', 'not-English'], dtype=object)]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0., 1., 0.],\n",
       "       [0., 1., 0.],\n",
       "       [0., 1., 0.],\n",
       "       ...,\n",
       "       [0., 1., 0.],\n",
       "       [0., 1., 0.],\n",
       "       [0., 1., 0.]])"
      ]
     },
     "execution_count": 268,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# One hot encode\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "enc = OneHotEncoder(handle_unknown='ignore')\n",
    "enc.fit(train_y.reshape(-1,1))\n",
    "print(enc.categories_)\n",
    "train_y = enc.transform(train_y.reshape(-1,1)).toarray()\n",
    "dev_y = enc.transform(dev_y.reshape(-1,1)).toarray()\n",
    "train_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_dic = dict(enumerate(df_train['labels'].cat.categories))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['labels'] = df_train['labels'].cat.codes\n",
    "df_dev['labels'] = df_dev['labels'].cat.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>these tiktoks radiate gay chaotic energy and i...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>@Champions Again He got killed for using false...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>It's not that all lives don't matter</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Is it really that difficult to understand? Bla...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Whenever we say black isn't that racists?  Why...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  labels\n",
       "0  these tiktoks radiate gay chaotic energy and i...       1\n",
       "1  @Champions Again He got killed for using false...       1\n",
       "2               It's not that all lives don't matter       1\n",
       "3  Is it really that difficult to understand? Bla...       1\n",
       "4  Whenever we say black isn't that racists?  Why...       1"
      ]
     },
     "execution_count": 271,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'Hope_speech', 1: 'Non_hope_speech', 2: 'not-English'}"
      ]
     },
     "execution_count": 272,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels_dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2], dtype=int8)"
      ]
     },
     "execution_count": 273,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(df_train['labels'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[3.867142371729528, 0.36516186992652483, 1.0]"
      ]
     },
     "execution_count": 353,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Deal with class imbalance\n",
    "from sklearn.utils import class_weight\n",
    "import numpy as np\n",
    "\n",
    "class_weights = class_weight.compute_class_weight('balanced',\n",
    "                                                 np.unique(df_train['labels']),\n",
    "                                                 df_train['labels'])\n",
    "# Reset non language class weight\n",
    "class_weights[-1] = 1\n",
    "if lang == 'tamil' or lang == 'mala':\n",
    "    class_weights[-1] = 0.5\n",
    "    \n",
    "class_weights = list(class_weights)\n",
    "class_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  3.86714237,   0.36516187, 344.87878788])"
      ]
     },
     "execution_count": 303,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train['labels'].shape[0]/(3*np.bincount(df_train['labels']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {},
   "outputs": [],
   "source": [
    "sweep_config = {\n",
    "    \"method\": \"bayes\",  # grid, random\n",
    "    \"metric\": {\"name\": \"eval_loss\", \"goal\": \"minimize\"},\n",
    "    \"parameters\": {\n",
    "        \"num_train_epochs\": {\"values\": [2, 3, 5, 6]},\n",
    "        \"learning_rate\": {\"min\": 5e-6, \"max\": 4e-3},\n",
    "        'weight_decay':  {\"values\": [0, 0.1, 0.01, 0.001]}\n",
    "    },\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import sklearn\n",
    "import wandb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {},
   "outputs": [],
   "source": [
    "logging.basicConfig(level=logging.INFO)\n",
    "transformers_logger = logging.getLogger(\"transformers\")\n",
    "transformers_logger.setLevel(logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sweep_id = wandb.sweep(sweep_config, project=\"HopeSpeech Sweep\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file https://huggingface.co/roberta-base/resolve/main/config.json from cache at C:\\Users\\E-Neo.DESKTOP-M4A5E83/.cache\\huggingface\\transformers\\733bade19e5f0ce98e6531021dd5180994bb2f7b8bd7e80c7968805834ba351e.35205c6cfc956461d8515139f0f8dd5d207a2f336c0c3a83b4bc8dca3518e37b\n",
      "Model config RobertaConfig {\n",
      "  \"architectures\": [\n",
      "    \"RobertaForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\",\n",
      "    \"2\": \"LABEL_2\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1,\n",
      "    \"LABEL_2\": 2\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-05,\n",
      "  \"max_position_embeddings\": 514,\n",
      "  \"model_type\": \"roberta\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"type_vocab_size\": 1,\n",
      "  \"vocab_size\": 50265\n",
      "}\n",
      "\n",
      "loading weights file https://huggingface.co/roberta-base/resolve/main/pytorch_model.bin from cache at C:\\Users\\E-Neo.DESKTOP-M4A5E83/.cache\\huggingface\\transformers\\51ba668f7ff34e7cdfa9561e8361747738113878850a7d717dbc69de8683aaad.c7efaa30a0d80b2958b876969faa180e485944a849deee4ad482332de65365a7\n",
      "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.out_proj.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "loading file https://huggingface.co/roberta-base/resolve/main/vocab.json from cache at C:\\Users\\E-Neo.DESKTOP-M4A5E83/.cache\\huggingface\\transformers\\d3ccdbfeb9aaa747ef20432d4976c32ee3fa69663b379deb253ccfce2bb1fdc5.d67d6b367eb24ab43b08ad55e014cf254076934f71d832bbab9ad35644a375ab\n",
      "loading file https://huggingface.co/roberta-base/resolve/main/merges.txt from cache at C:\\Users\\E-Neo.DESKTOP-M4A5E83/.cache\\huggingface\\transformers\\cafdecc90fcab17011e12ac813dd574b4b3fea39da6dd817813efa010262ff3f.5d12962c5ee615a4c803841266e9c3be9a691a924f72d395d3a6c6c81157788b\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 2.54 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "err=[]\n",
    "y_pred_tot=[]\n",
    "\n",
    "\n",
    "\n",
    "i=1\n",
    "#\n",
    "\n",
    "if lang == 'eng' and use_weights == 'Y':\n",
    "    model = ClassificationModel('roberta', 'roberta-base', use_cuda=True, num_labels=3, weight = class_weights , args={\n",
    "                                                                     'train_batch_size':8,\n",
    "                                                                     'reprocess_input_data': True,\n",
    "                                                                     #\"weight\":  class_weights,\n",
    "                                                                     'overwrite_output_dir': True,\n",
    "                                                                     'fp16': False,\n",
    "                                                                     'do_lower_case': False,\n",
    "                                                                     'num_train_epochs': 6,\n",
    "                                                                     'max_seq_length': 256,\n",
    "                                                                     'regression': False,\n",
    "                                                                     'manual_seed': SEED,\n",
    "                                                                     \"learning_rate\":2e-5,\n",
    "                                                                     'weight_decay':0,\n",
    "                                                                     \"save_eval_checkpoints\": False,\n",
    "                                                                     \"save_model_every_epoch\": False,\n",
    "                                                                     \"silent\": False,\n",
    "                                                                      \"verbose\": True,\n",
    "                                                                    \"evaluate_during_training\": True,\n",
    "                                                                    \"use_early_stopping\":True,\n",
    "                                                                    \"dataloader_num_workers\": 0,\n",
    "                                                                    'use_multiprocessing': False,\n",
    "                                                                    \"output_dir\":'outputs/'+lang+'/'})\n",
    "#                                                                     \"wandb_project\": \"HopeSpeech Sweep\"}, \n",
    "#                                                                     sweep_config=wandb.config)                                                                  \n",
    "\n",
    "# weight = class_weights\n",
    "elif lang == 'eng' and use_weights == 'N':\n",
    "    model = ClassificationModel('roberta', 'roberta-base', use_cuda=True, num_labels=3, weight = class_weights , args={\n",
    "                                                                     'train_batch_size':8,\n",
    "                                                                     'reprocess_input_data': True,\n",
    "                                                                     'overwrite_output_dir': True,\n",
    "                                                                     'fp16': False,\n",
    "                                                                     'do_lower_case': False,\n",
    "                                                                     'num_train_epochs': 6,\n",
    "                                                                     'max_seq_length': 256,\n",
    "                                                                     'regression': False,\n",
    "                                                                     'manual_seed': SEED,\n",
    "                                                                     \"learning_rate\":2e-5,\n",
    "                                                                     'weight_decay':0,\n",
    "                                                                     \"save_eval_checkpoints\": False,\n",
    "                                                                     \"save_model_every_epoch\": False,\n",
    "                                                                     \"silent\": False,\n",
    "                                                                      \"verbose\": True,\n",
    "                                                                    \"evaluate_during_training\": True,\n",
    "                                                                    \"use_early_stopping\":True,\n",
    "                                                                    \"dataloader_num_workers\": 0,\n",
    "                                                                    'use_multiprocessing': False,\n",
    "                                                                    \"output_dir\":'outputs/'+lang+'/'})\n",
    "#                                                                     \"wandb_project\": \"HopeSpeech Sweep\"}, \n",
    "#                                                                     sweep_config=wandb.config)  \n",
    "\n",
    "elif use_weights == 'Y':\n",
    "        model = ClassificationModel('xlmroberta', 'xlm-roberta-base', use_cuda=True, num_labels=3,weight = class_weights, args={\n",
    "                                                                    'train_batch_size':8,\n",
    "                                                                    'reprocess_input_data': True,\n",
    "                                                                    #\"weight\":  class_weights,\n",
    "                                                                    'overwrite_output_dir': True,\n",
    "                                                                    'fp16': False,\n",
    "                                                                    'do_lower_case': False,\n",
    "                                                                    'num_train_epochs': 6,\n",
    "                                                                    'max_seq_length': 256,\n",
    "                                                                    'regression': False,\n",
    "                                                                    'manual_seed': SEED,\n",
    "                                                                    \"learning_rate\":1e-5,\n",
    "                                                                    'weight_decay':0,\n",
    "                                                                    \"save_eval_checkpoints\": False,\n",
    "                                                                    \"save_model_every_epoch\": False,\n",
    "                                                                    \"silent\": False,\n",
    "                                                                    \"verbose\": True,\n",
    "                                                                    \"dataloader_num_workers\": 0,\n",
    "                                                                    \"evaluate_during_training\": True,\n",
    "                                                                    \"use_early_stopping\":True,\n",
    "                                                                    'use_multiprocessing': False,\n",
    "                                                                    \"output_dir\":'outputs/'+lang+'/',})\n",
    "#                                                                     \"wandb_project\" : \"HopeSpeech Sweep\"}, \n",
    "#                                                                     sweep_config=wandb.config)\n",
    "else:\n",
    "        model = ClassificationModel('xlmroberta', 'xlm-roberta-base', use_cuda=True,num_labels=3, args={\n",
    "                                                                    'train_batch_size':8,\n",
    "                                                                    'reprocess_input_data': True,\n",
    "                                                                    #\"weight\":  class_weights,\n",
    "                                                                    'overwrite_output_dir': True,\n",
    "                                                                    'fp16': False,\n",
    "                                                                    'do_lower_case': False,\n",
    "                                                                    'num_train_epochs': 6,\n",
    "                                                                    'max_seq_length': 256,\n",
    "                                                                    'regression': False,\n",
    "                                                                    'manual_seed': SEED,\n",
    "                                                                    \"learning_rate\":1e-5,\n",
    "                                                                    'weight_decay':0,\n",
    "                                                                    \"save_eval_checkpoints\": False,\n",
    "                                                                    \"save_model_every_epoch\": False,\n",
    "                                                                    \"silent\": False,\n",
    "                                                                    \"verbose\": True,\n",
    "                                                                    \"dataloader_num_workers\": 0,\n",
    "                                                                    \"evaluate_during_training\": True,\n",
    "                                                                    \"use_early_stopping\":True,\n",
    "                                                                    'use_multiprocessing': False,\n",
    "                                                                    \"output_dir\":'outputs/'+lang+'/',})\n",
    "#                                                                     \"wandb_project\" : \"HopeSpeech Sweep\"}, \n",
    "#                                                                     sweep_config=wandb.config)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3.867142371729528, 0.36516186992652483, 1.0]\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "pprint(model.weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:simpletransformers.classification.classification_model: Converting to features started. Cache is not used.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "55d6c42302ab4a73b1fe8ccc407f2780",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=22762.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee5f643922be45868583fdfd410d44bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Epoch'), FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c7821dbf5fc54017a0a91dbc3e57000e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Running Epoch 0 of 1'), FloatProgress(value=0.0, max=2846.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Configuration saved in outputs/eng/checkpoint-2000\\config.json\n",
      "Model weights saved in outputs/eng/checkpoint-2000\\pytorch_model.bin\n",
      "Configuration saved in outputs/best_model\\config.json\n",
      "Model weights saved in outputs/best_model\\pytorch_model.bin\n",
      "INFO:simpletransformers.classification.classification_model: No improvement in eval_loss\n",
      "INFO:simpletransformers.classification.classification_model: Current step: 1\n",
      "INFO:simpletransformers.classification.classification_model: Early stopping patience: 3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Configuration saved in outputs/best_model\\config.json\n",
      "Model weights saved in outputs/best_model\\pytorch_model.bin\n",
      "Configuration saved in outputs/eng/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in outputs/eng/pytorch_model.bin\n",
      "INFO:simpletransformers.classification.classification_model: Training of roberta model complete. Saved to outputs/eng/.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2846,\n",
       " {'global_step': [2000, 2846],\n",
       "  'mcc': [0.5783551114793409, 0.5556837267022606],\n",
       "  'train_loss': [0.1693790704011917, 0.002056456170976162],\n",
       "  'eval_loss': [0.2393372932398624, 0.22055188839041281]})"
      ]
     },
     "execution_count": 344,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.train_model(df_train, eval_df = df_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# def train():\n",
    "#     # Initialize a new wandb run\n",
    "#     wandb.init()\n",
    "\n",
    "\n",
    "#     # Train the model\n",
    "#     model.train_model(train_df, eval_df=eval_df)\n",
    "\n",
    "#     # Evaluate the model\n",
    "#     model.eval_model(eval_df)\n",
    "\n",
    "#     # Sync wandb\n",
    "#     wandb.join()\n",
    "\n",
    "\n",
    "# wandb.agent(sweep_id, train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:simpletransformers.classification.classification_model: Converting to features started. Cache is not used.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a960c4c45694caf841a45bc3cbf6317",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=2843.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30dcf2dfee8948f9897ba8315a209510",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Running Evaluation'), FloatProgress(value=0.0, max=356.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:simpletransformers.classification.classification_model:{'mcc': 0.5556837267022606, 'eval_loss': 0.22055188839041281}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "{'mcc': 0.5556837267022606, 'eval_loss': 0.22055188839041281}\n"
     ]
    }
   ],
   "source": [
    "result, model_outputs, wrong_predictions = model.eval_model(df_dev)\n",
    "print(result)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_outputs_vals = softmax(model_outputs,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.76779329e-03, 9.98045514e-01, 1.86692340e-04],\n",
       "       [1.27517235e-03, 9.98470154e-01, 2.54673536e-04],\n",
       "       [6.02060752e-03, 9.93807900e-01, 1.71492752e-04],\n",
       "       ...,\n",
       "       [5.43064215e-03, 9.94374652e-01, 1.94705606e-04],\n",
       "       [1.40646154e-03, 9.98315811e-01, 2.77727158e-04],\n",
       "       [1.04877254e-02, 9.89357756e-01, 1.54518453e-04]])"
      ]
     },
     "execution_count": 348,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_outputs_vals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log_Loss: 0.21826368068679378\n"
     ]
    }
   ],
   "source": [
    "print(f\"Log_Loss: {log_loss(df_dev['labels'], raw_outputs_vals)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "y_pred = [np.argmax(i) for i in  raw_outputs_vals]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9352796341892368"
      ]
     },
     "execution_count": 351,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sklearn.metrics.f1_score(df_dev['labels'].to_numpy(),y_pred,average = 'micro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.40      0.54       272\n",
      "           1       0.94      0.99      0.97      2569\n",
      "           2       0.00      0.00      0.00         2\n",
      "\n",
      "    accuracy                           0.94      2843\n",
      "   macro avg       0.60      0.46      0.50      2843\n",
      "weighted avg       0.93      0.94      0.92      2843\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(sklearn.metrics.classification_report(df_dev['labels'].to_numpy(),y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test Set Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file outputs/eng/config.json\n",
      "Model config RobertaConfig {\n",
      "  \"_name_or_path\": \"roberta-base\",\n",
      "  \"architectures\": [\n",
      "    \"RobertaForSequenceClassification\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\",\n",
      "    \"2\": \"LABEL_2\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1,\n",
      "    \"LABEL_2\": 2\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-05,\n",
      "  \"max_position_embeddings\": 514,\n",
      "  \"model_type\": \"roberta\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"type_vocab_size\": 1,\n",
      "  \"vocab_size\": 50265\n",
      "}\n",
      "\n",
      "loading weights file outputs/eng/pytorch_model.bin\n",
      "All model checkpoint weights were used when initializing RobertaForSequenceClassification.\n",
      "\n",
      "All the weights of RobertaForSequenceClassification were initialized from the model checkpoint at outputs/eng/.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use RobertaForSequenceClassification for predictions without further training.\n",
      "Model name 'outputs/eng/' not found in model shortcut name list (roberta-base, roberta-large, roberta-large-mnli, distilroberta-base, roberta-base-openai-detector, roberta-large-openai-detector). Assuming 'outputs/eng/' is a path, a model identifier, or url to a directory containing tokenizer files.\n",
      "Didn't find file outputs/eng/added_tokens.json. We won't load it.\n",
      "Didn't find file outputs/eng/tokenizer.json. We won't load it.\n",
      "loading file outputs/eng/vocab.json\n",
      "loading file outputs/eng/merges.txt\n",
      "loading file None\n",
      "loading file outputs/eng/special_tokens_map.json\n",
      "loading file outputs/eng/tokenizer_config.json\n",
      "loading file None\n"
     ]
    }
   ],
   "source": [
    "if lang == 'eng':\n",
    "    model = ClassificationModel('roberta', 'outputs/'+lang+'/')\n",
    "else:\n",
    "    model = ClassificationModel('xlmroberta', 'outputs/'+lang+'/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sents = list(df_test['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:simpletransformers.classification.classification_model: Converting to features started. Cache is not used.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13cd6e557125405cabdf2920338f9749",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=2846.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a17128d809924c02ab039e65ecea4e13",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=356.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "predictions, raw_outputs = model.predict(test_sents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " ...]"
      ]
     },
     "execution_count": 323,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'Hope_speech', 1: 'Non_hope_speech', 2: 'not-English'}"
      ]
     },
     "execution_count": 324,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels_dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " 'Hope_speech',\n",
       " ...]"
      ]
     },
     "execution_count": 325,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions = [labels_dic[i] for i in predictions]\n",
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 326,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test['label_pred'] = predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>label_pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What do you mean by the word sniped?</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "      <td>Hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I love this video!! I’m bisexual and it’s just...</td>\n",
       "      <td>Hope_speech</td>\n",
       "      <td>Hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ya the irony but then i don't want to come off...</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "      <td>Hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A PERSON'S CHARACTER MATTERS. PERIOD!!</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "      <td>Hope_speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>@Blaster of Gasters</td>\n",
       "      <td>Non_hope_speech</td>\n",
       "      <td>Hope_speech</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text            label  \\\n",
       "0               What do you mean by the word sniped?  Non_hope_speech   \n",
       "1  I love this video!! I’m bisexual and it’s just...      Hope_speech   \n",
       "2  ya the irony but then i don't want to come off...  Non_hope_speech   \n",
       "3             A PERSON'S CHARACTER MATTERS. PERIOD!!  Non_hope_speech   \n",
       "4                                @Blaster of Gasters  Non_hope_speech   \n",
       "\n",
       "    label_pred  \n",
       "0  Hope_speech  \n",
       "1  Hope_speech  \n",
       "2  Hope_speech  \n",
       "3  Hope_speech  \n",
       "4  Hope_speech  "
      ]
     },
     "execution_count": 327,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 precision    recall  f1-score   support\n",
      "\n",
      "    Hope_speech       0.09      1.00      0.16       250\n",
      "Non_hope_speech       0.00      0.00      0.00      2593\n",
      "    not-English       0.00      0.00      0.00         3\n",
      "\n",
      "       accuracy                           0.09      2846\n",
      "      macro avg       0.03      0.33      0.05      2846\n",
      "   weighted avg       0.01      0.09      0.01      2846\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(sklearn.metrics.classification_report(df_test['label'].to_numpy(),predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test['correct'] = df_test.apply(lambda row: row['label'] == row ['label_pred'], axis = 1)\n",
    "ids = [lang +'_sent_'+str(i) for i in range(1,len(df_test)+1)]\n",
    "df_test['id'] = ids\n",
    "df_test = df_test.set_index('id')\n",
    "df_test.head()\n",
    "#df_test= df_test.drop(columns=['labels'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "if lang == 'eng':\n",
    "    df_test.to_csv('TeamUNCC_english.tsv',sep='\\t')\n",
    "elif lang == 'tamil':\n",
    "    df_test.to_csv('TeamUNCC_tamil.tsv',sep='\\t')\n",
    "elif lang == 'mala':\n",
    "    df_test.to_csv('TeamUNCC_malayalam.tsv',sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "if lang == 'eng':\n",
    "    df_test.to_csv('TeamUNCC_english.csv')\n",
    "elif lang == 'tamil':\n",
    "    df_test.to_csv('TeamUNCC_tamil.csv')\n",
    "elif lang == 'mala':\n",
    "    df_test.to_csv('TeamUNCC_malayalam.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:torch]",
   "language": "python",
   "name": "torch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}