Exist_2022.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Exist 2022.ipynb",
      "provenance": [],
      "machine_shape": "hm",
      "mount_file_id": "1nJAG-zSxqsuvpDHL3btKet8Bx1ddivfs",
      "authorship_tag": "ABX9TyNikiUF7rlFFpsQpJIvEQck",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/amaanrzv39/Machine-Learning/blob/master/Exist_2022.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "lbsvolwIwHeb"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from nltk.stem import SnowballStemmer\n",
        "import re\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from nltk.corpus import stopwords\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.metrics import classification_report, f1_score, accuracy_score, confusion_matrix\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.svm import SVC\n",
        "#import contractions"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "exist_train = pd.read_csv(\"/content/drive/MyDrive/Data Sets/Exist 2022/EXIST2021_training.tsv\", sep = '\\t')\n",
        "exist_test = pd.read_csv(\"/content/drive/MyDrive/Data Sets/Exist 2022/EXIST2021_test.tsv\", sep = '\\t')"
      ],
      "metadata": {
        "id": "wB8eBueD0E5_"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_en = exist_train[exist_train['language']=='en']\n",
        "test_en = exist_test[exist_test['language']=='en']\n",
        "train_es = exist_train[exist_train['language']=='es']\n",
        "test_es = exist_test[exist_test['language']=='es']"
      ],
      "metadata": {
        "id": "ppnXOe530FES"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train-en : ',exist_train[exist_train['language']=='en'].shape)\n",
        "print('Train-es : ',exist_train[exist_train['language']=='es'].shape)\n",
        "print('Test-en : ',exist_test[exist_test['language']=='en'].shape)\n",
        "print('Test-es : ',exist_test[exist_test['language']=='es'].shape)"
      ],
      "metadata": {
        "id": "vNTi3av_0FLG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d826d77e-7619-4347-b94a-c09b7fd430cd"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train-en :  (3436, 7)\n",
            "Train-es :  (3541, 7)\n",
            "Test-en :  (2208, 7)\n",
            "Test-es :  (2160, 7)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x = train_en[train_en['text'].apply(len)<50]\n",
        "#train_en['text'].apply(len)"
      ],
      "metadata": {
        "id": "re1fie4y0FRC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81
        },
        "id": "xtl-FBR8X25s",
        "outputId": "f7697243-d303-4bc0-e1e4-cf25653005a4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "      test_case    id   source language  \\\n",
              "3228  EXIST2021  3229  twitter       en   \n",
              "\n",
              "                                           text   task1  \\\n",
              "3228  This bint hahaha  https://t.co/erWAc1PoLi  sexist   \n",
              "\n",
              "                             task2  \n",
              "3228  misogyny-non-sexual-violence  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-bf9a5567-6c37-495f-ae07-7da5d4ad5c50\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>test_case</th>\n",
              "      <th>id</th>\n",
              "      <th>source</th>\n",
              "      <th>language</th>\n",
              "      <th>text</th>\n",
              "      <th>task1</th>\n",
              "      <th>task2</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3228</th>\n",
              "      <td>EXIST2021</td>\n",
              "      <td>3229</td>\n",
              "      <td>twitter</td>\n",
              "      <td>en</td>\n",
              "      <td>This bint hahaha  https://t.co/erWAc1PoLi</td>\n",
              "      <td>sexist</td>\n",
              "      <td>misogyny-non-sexual-violence</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bf9a5567-6c37-495f-ae07-7da5d4ad5c50')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-bf9a5567-6c37-495f-ae07-7da5d4ad5c50 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-bf9a5567-6c37-495f-ae07-7da5d4ad5c50');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "exist_train['task2'].value_counts().plot(kind='barh', color = ['b', 'g', 'r', 'm', 'y', 'k'])\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "0q112dOZW1x0",
        "outputId": "7dfc1212-93e2-45fe-8ca8-1954f4a80024"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfsAAAD4CAYAAADraE/IAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de7xd07338c9XEkKkEZI6KN0o0ZRmyw5F8YS6NS3qoEEPCS11OW59PK1WD2nL06q26k44EcUhotKmeiHkIkUSe8tdJFRyXi2OS0skVVHJ7/wxx2Za2ddYWWvt6ft+vdZrjzXmmGP+5sja+a0x5lx7KSIwMzOz4tqg2gGYmZnZ+uVkb2ZmVnBO9mZmZgXnZG9mZlZwTvZmZmYF173aAZi1pF+/flFXV1ftMMzMupSmpqZXI6J/ab2TvdWkuro6Ghsbqx2GmVmXIum/W6r3Mr6ZmVnBOdmbmZkVnJO9mZlZwTnZm5mZFZyTvZmZWcE52ZuZmRWck72ZmVnByV9xa7VIUpd6Yfr3yMxqgaSmiBhSWu+ZvZmZWcE52ZuZmRWck72ZmVnBOdmbmZkVnJO9mZlZwTnZV4mkOkkLWtl2i6SB69BnvaRhuedHSLowlftLmilptqT9JP1O0mbrcIyhkvbJPT9d0kmd7cfMzCrHX3FbgyLia+u4az0wBPhd6mciMDFt+xwwP9f39HU8xlBgJfBYOsaN69iPmZlViGf2FSLpG5IWpMd5qbq7pDslLZJ0r6RNUtupkoak8iGSHpf0pKTxkjZN9XtIekzSXEmzJPUBvg8MlzRH0nBJIyVdK6ke+DFwZNq2saRlkvqlvk6SNC/1dXuqOzy3EvCQpC0l1QGnA+enfvaTNErSBWmfekkzUl8TJPXNnc/lKc4lkvar0LCbmRlO9hUhqQE4GfgMsBdwKtAXGABcHxGfBN4AzizZrx/wXeCgiBgMNALfkLQhMA44NyIGAQcBfwcuBsZFRH1EjGvuJyLmlGz7R+4Yn0rHODD1dW7a9Edgr4jYHbgb+GZELANuBK5M/ZSuDvwC+FZEfBqYD1yS29Y9IvYEziupz5/vaZIaJTW2NZ5mZtY5XsavjH2BCRHxdwBJ9wH7AX+OiEdTmzuAc4Cf5PbbCxgIPCoJYEPgcbI3CS9GxBMAEfFG6nddYjsQGB8Rr6a+/pbqPwaMk7RVOu7StjpJKwubRcS0VHUbMD7X5L70swmoa6mPiBgNjE79+U/SmZmViWf21VWa0EqfC5iUZtH1ETEwIr5aodiuAa6NiN2ArwM9P2B/q9LP1fhNpplZRTnZV8Z04EuSNpHUCzgq1W0nae/U5gSypfO8GcBnJX0CQFIvSTsDi4GtJO2R6ntL6g6sAHp3MrbJwLGStkh9bZ7q+wDPp/KIXPsWjxERy4HXctfjTwSmlbYzM7PKc7KvgIh4EhgLzAJmArcAr5El7bMkLSK7hn/D+3eLV4CRwF2S5pEt4e8SEW8Dw4FrJM0FJpHNvKcAA5tv0OtgbAuBy4Bpqa+fpU2jgPGSmoBXc7v8Bjiq+Qa9ku5GAFekWOvJbhg0M7Mq87fe1SBJ84EjIqLN6+RF1tWu2fv3yMxqgb/1rouQNIns8/Af2kRvZmbl5RulakxEHFztGMzMrFg8szczMys4J3szM7OCc7K3mtTQ0EBEdJmHmVktc7I3MzMrOCd7MzOzgnOyNzMzKzgnezMzs4JzsjczMys4J3szM7OCc7I3MzMrOCd7MzOzgnOyNzMzKzgnezMzs4JzsjczMys4J3szM7OCc7I3MzMrOCd7MzOzgnOyNzMzKzgnezMzs4JzsjczMyu47tUOwKwlK1Y0MXWqqh1Gpw0dGtUOwcxsLZ7Zm5mZFZyTvZmZWcE52ZuZmRWck72ZmVnBOdmbmZkVnJN9FyZppKRrO9F+iKSry9mnmZnVPn/07kMkIhqBxmrHYWZmleWZfZlI6iXpt5LmSlogabikBknTJDVJekDSVpL6SFosaUDa7y5Jp6byylx/x0gam8qHS5opabakhyRt2U4sd0v6Qu752NTfUEn3p7rNJf1K0jxJMyR9uoV++kv6paQn0uOzqX6UpDGSpkp6TtI5uX1OSn3OlXR7W/2YmVllONmXz2HACxExKCJ2Bf4AXAMcExENwBjgsohYDvw7MFbScUDfiLi5nb7/COwVEbsDdwPfbKf9OODLAJI2BD4H/LakzfeA2RHxaeA7wC9a6Ocq4MqI2AM4Grglt20X4FBgT+ASST0kfQr4LnBgRAwCzu1AP++SdJqkRkmNy5e3c4ZmZtZhXsYvn/nATyVdDtwPvAbsCkySBNANeBEgIiZJOha4DhjUgb4/BoyTtBWwIbC0nfa/B66StBHZm5BHIuIfKY5m+5IlXiJisqQtJH2kpJ+DgIG5/T4iadNU/m1ErAJWSXoZ2BI4EBgfEa+mfv/WVj8R8e5KRmo/GhgNMGCA/KfozMzKxMm+TCJiiaTBwDDgUmAysDAi9i5tK2kD4JPAm0Bf4C/N3eSa9cyVrwF+FhETJQ0FRpX01w1oSk8nRsTFkqaSzbyHk60GrIsNyFYU3io5HsCqXNVq2n4ttdiPmZlVhpfxy0TS1sCbEXEHcAXwGaC/pL3T9uZlboDzgUXACcCtknqk+pckfTK9GTgq130f4PlUHlF67IhYHRH16XFxqh4HnAzsR3ZJodR04CsptqHAqxHxRkmbB4Gzc+dY384wTAaOlbRFar/5OvZjZmZl5Jl9+ewGXCFpDfBP4AzgHeBqSX3Ixvrnkt4BvgbsGRErJD1Cdp37EuBCsksAr5DdNd+8ZD4KGC/pNbKEun0H4nkQuB34dUS83cL2UcAYSfPIVhjWehMBnANcl9p0Bx4BTm/tgBGxUNJlwDRJq4HZwMjO9mNmZuWlCF8atdozYIDippuqHUXn+VvvzKyaJDVFxJDSei/jm5mZFZyTvZmZWcE52ZuZmRWcb9CzmtS7dwNDh/ov+5qZlYNn9mZmZgXnZG9mZlZwTvZmZmYF52RvZmZWcE72ZmZmBedkb2ZmVnBO9mZmZgXnZG9mZlZwTvZmZmYF52RvZmZWcE72ZmZmBedkb2ZmVnBO9mZmZgXnZG9mZlZwTvZmZmYF52RvZmZWcE72ZmZmBde92gGYtWRF0wqmamq1w6iooTG02iGYWUF5Zm9mZlZwTvZmZmYF52RvZmZWcE72ZmZmBedkb2ZmVnBlSfaSjpB0YTn6so6TNFTS/Z1ov7Wke8vZp5mZ1b6yfPQuIiYCE8vRl60/EfECcEy14zAzs8pqd2YvqU7S05LGSloi6U5JB0l6VNIzkvaUNFLStan9sZIWSJor6ZFU11PSrZLmS5ot6YBUv4mkeyQ9JWmCpJmShkg6RdLPczGcKunKFMsiSTdLWijpQUkbS9pR0pO59jvln+fqR0q6T9IfUuw/zm07PsW3QNLlufqVki5L5zND0pYt9Nstjc+C1Mf5qX7HdKwmSdMl7SKpu6QnJA1NbX4o6bJUXiapXyoPkbIPmqcxfjyN3WOSBrTzb/YjSWflno+SdEEavwVt/ZuU9NNL0hhJs1KbIzswjodJejKN18Nt9WNmZpXR0WX8TwA/BXZJjxOAfYELgO+UtL0YODQiBgFHpLqzgIiI3YDjgdsk9QTOBF6LiIHAfwANqf09wOGSeqTnJwNjUnkn4LqI+BTwOnB0RPwJWC6pPtf+1lbOpR4YDuwGDJe0raStgcuBA9P2PSR9KbXvBcxI5/MIcGorfW4TEbumc2w+9mjg7IhoSGN1fUS8A4wEbpB0EHAY8L1WYm32NLBfROxONr7/v53244Av555/OdXltfZvkncRMDki9gQOAK6Q1Ct3zqXj2B+4mezfZBBwbAf6eZek0yQ1SmpczvJ2TtHMzDqqo8v4SyNiPoCkhcDDERGS5gN1JW0fBcZKuge4L9XtC1wDEBFPS/pvYOdUf1WqXyBpXiqvlDQZ+KKkRUCPiJgvqS7FMif125Q7/i3AyZK+QZaE9mzlXB6OiOXpXJ4CPg5sAUyNiFdS/Z3A/sCvgLeB5mvYTcDBLfT5HLCDpGuA3wIPStoU2AcYL6m53Ubp/BZKuj31u3dEvN1KrM36kCXjnYAAerTVOCJmS/poehPTn+wN1Z/T+DVr7d8k7xDgCEkXpOc9ge1SuaVx7As8EhFLU79/a6efRSVxjyZ7g8QADYi2ztHMzDquo8l+Va68Jvd8TWkfEXG6pM8AXwCaJDWwbm4hWzV4mvfP0vOxrAY2TuVfApcAk4GmiPirpKNSHcDXWtm/vTH4Z0Q0J57VQHdJ3cgSP8DEiLhY0iDgUOB0spn0ecDrEVG/Vo+Z3chWJj6aq3uH91Zb8rPsHwBTIuKolLCnlnYm6QFgS6AxIr4GjCe7Pv8vrD2r7yiRzdIXlxzrM3RuHFvsx8zMKqPsH72TtGNEzIyIi4FXgG2B6cBX0vadyWZ1i8lWAb6c6geSJUAAImJm2vcE4K72jhsRbwEPADeQ3hxExISIqE+PxjZ2nwX8H0n9UiI/HpjWxrFW5/q9OF1n3yAifgl8FxgcEW8ASyUdm85P6Q0Bkv4V2Jxs9eAaSZulrpfx3qWMo3OH7AM8n8ojW4np0BRP85uaccBxZAl/fAu7tPZvkvcAcLbS0oSk3Vsbk2QGsL+k7VP7zdexHzMzK6P18Tn7K9JNXwuAx4C5wPXABmnZfxwwMiJWpfr+aRn4UmAhvO9i7T3AoxHxWgePfSfZasODnQk4Il4ELgSmpHibIuLXnehiG2CqpDnAHcC3U/1XgK9Kmkt2bkemNwY/Ar4WEUuAa0mXMsiu3V8lqZFsttzsx8APJc2mg6sxEbEQ6A08n86vVGv/Jnk/ILtkMC9dvvlBO8d8BTgNuC+dc/OKQqf6MTOz8tJ7K9RVOHg2i+4REW9J2hF4CBjQfA1b2ee9r4yIhzvY3wVAn4j4j/UWtFXEAA2Im7ip2mFUlL/1zsw+KElNETGktL7aX3G7CTAl3XUv4MyIeDsta88C5nYi0U8AdiS7o97MzMySqib7iFgBrPUOJCJeZ+07w9vr66hyxWVmZlYk/tv4ZmZmBVftZXyzFvVu6M3QxqHVDsPMrBA8szczMys4J3szM7OCc7I3MzMrOCd7MzOzgnOyNzMzKzgnezMzs4JzsjczMys4J3szM7OCc7I3MzMrOCd7MzOzgnOyNzMzKzgnezMzs4JzsjczMys4J3szM7OCc7I3MzMrOCd7MzOzgnOyNzMzK7ju1Q7ArEVNTSBVO4raFVHtCMysC/HM3szMrOCc7M3MzArOyd7MzKzgnOzNzMwKzsnezMys4LpUspd0nqRNav04km6RNLCcMeX6Xlmmfk6XdFI5+jIzs9qm6EIf4ZG0DBgSEa92Yp9uEbF6fR+nUiStjIhNqx3H+jZEisZqB1HLutDvrZlVjqSmiBhSWl+zM3tJvST9VtJcSQskXQJsDUyRNCW1OUTS45KelDRe0qapfpmkyyU9CRzbRrvPSZotab6kMZI2knRO/jiSTpH081xcp0q6UlKdpKcl3SlpkaR7m1cDJE2VNCSVV0q6LJ3HDElbpvod0/P5ki5tbcYuafsU+3xJl+bqJemKNDbzJQ1P9UMlTZP0a0nPSfqRpK9ImpXa7ZjajZJ0QS7ey1ObJZL2S/V1kqancXtS0j65Y0xN59w8Bkrb9pD0WDrfWZJ6S+qWYn1C0jxJXy/Ty8TMzDqgZpM9cBjwQkQMiohdgZ8DLwAHRMQBkvoB3wUOiojBQCPwjdz+f031D7XUTlJPYCwwPCJ2I/sDQ2dExNX54wD3AIdL6pH6PRkYk8oDgOsj4pPAG8CZLZxHL2BGRAwCHgFOTfVXAVelY/+ljXG4CrghtXsxV/+vQD0wCDgIuELSVmnbIOB04JPAicDOEbEncAtwdivH6Z7anAdckupeBg5O4zYcuDrXfvfUdiCwA/BZSRsC44Bz0/keBPwD+CqwPCL2APYATpW0fWkAkk6T1Cip8ZU2BsTMzDqnlpP9fODgNOPcLyKWl2zfiyzRPCppDjAC+Hhu+7h22g0AlkbEktTuNmD/0iAiYiUwGfiipF2AHhExP23+c0Q8msp3APu2cB5vA/enchNQl8p7A+NT+b9aHgIAPgvclcq35+r3Be6KiNUR8RIwjSyRAjwRES9GxCrgT8CDqX5+7vil7mshxh7AzZLmp1jz9yHMioi/RMQaYE7aZwDwYkQ8ARARb0TEO8AhwElp/GcCWwA7lQYQEaMjYkhEDOnfxoCYmVnn1Oyfy42IJZIGA8OASyU9XNJEwKSIOL6VLv7eVjtJgzoRzi3Ad4CngVvzYZaG3cK+/4z3boxYTTtjLuky4AsAEVHfRr9tWZUrr8k9X9PG8Zvb5GM8H3iJbKVgA+CtVo7R3nkJODsiHmg3cjMzK7uandlL2hp4MyLuAK4ABgMrgN6pyQyypeNPpPa9JO3cQlettVsM1DXXky13T0vl/HGIiJnAtsAJvDfLBthO0t6pfALwx06c4gzg6FQ+LnesiyKiPpfoH81t/0pu/+nA8HQ9vD/ZqsSsThy/I/qQzdTXkI1Pt3baLwa2krQHQLpe3x14ADij+VKIpJ0l9SpzrGZm1oqaTfbAbsCstPR7CXApMBr4g6QpEfEKMBK4S9I84HFgl9JOWmsXEW+RXX8fn5ap1wA3pt3ePU6uq3uARyPitVzdYuAsSYuAvsANnTi/88juHZgHfAIovUzR7Nx0jPnANrn6CcA8YC7ZZYZvRsT/dOL4HXE9MELSXLKx/XtbjSPibbJr+9ekfSYBPclWRp4CnpS0ALiJGl5VMjMrmi710btqknQ/cGVEPJye1wH3p5sH16W/TYB/RERIOg44PiKOLFe8XZ0/etcO/96aWQvUykfvPLtqh6TNyJbH5zYn+jJpAK5NH1l7HTiljH2bmZm9yzN7q0me2bfDv7dm1oLWZva1fM3ezMzMysDL+FabGhqg0XN7M7Ny8MzezMys4JzszczMCs7J3szMrOCc7M3MzArOyd7MzKzgnOzNzMwKzsnezMys4JzszczMCs7J3szMrOCc7M3MzArOyd7MzKzgnOzNzMwKzsnezMys4JzszczMCs7J3szMrOCc7M3MzArOyd7MzKzgFBHVjsFsLdpawderHUXXEpf4d9nsw05SU0QMKa33zN7MzKzgnOzNzMwKzsnezMys4JzszczMCs7J3szMrOAKmewlPdZK/VhJx6xjnx9k3yMkXbiO+y6T1K+cfZaTpDpJC1J5iKSrU3mopH2qG52ZmQF0r3YA60NE1FSSiYiJwMRa7/ODiohGoDE9HQqsBFp842VmZpVT1Jn9yvRTkq6VtFjSQ8BHc20aJE2T1CTpAUlbpfp6STMkzZM0QVLfFvr/nKTZkuZLGiNpo1Q/TNLTqc+rJd2f6kdKujaVt0z9zk2PfVL9r9J+CyWd1oFzzPc5Nh3vMUnP5VcgJP0/SU+k8/lerv4iSUsk/VHSXZIuSPVTJQ1J5X6SlqVynaTpkp5Mj7XeUKXZ/P2S6oDTgfMlzZG0n6Slknqkdh/JPzczs/WrkMk+5yhgADAQOAloTqw9gGuAYyKiARgDXJb2+QXwrYj4NDAfuCTfoaSewFhgeETsRrY6ckaqvwn4fOqzfysxXQ1Mi4hBwGBgYao/Je03BDhH0hadPNetgH2BLwI/SrEeAuwE7AnUAw2S9pfUAByX6oYBe3Sg/5eBgyNiMDA8nUeLImIZcCNwZUTUR8R0YCrwhdTkOOC+iPhnfj9Jp0lqlNTImx06ZzMz64BCLuPn7A/cFRGrgRckTU71A4BdgUmSALoBL0rqA2wWEdNSu9uA8SV9DgCWRsSSXJuzyJLZcxGxNNXfBbQ0Qz+Q7I0HKa7lqf4cSUel8rZkSfqvnTjXX0XEGuApSVumukPSY3Z6vmnqtzcwISLeBJDUkcsBPYBrJdUDq4GdOxEbwC3AN4FfAScDp5Y2iIjRwGhIf0HPzMzKoujJvjUCFkbE3u+rzJJ95YORhgIHAXtHxJuSpgI9S9qcxXsJclgL3azKN8/9/GFE3FTS13lthPMO76345GM4H3gJGJS2v9VGH2uJiEfTpYChQLeIWNCZ/c3MbN0VfRn/EWC4pG7pmvwBqX4x0F/S3pAt60v6VEQsB16TtF9qdyIwraTPxUCdpE+UtFkM7JCuV0O21N2Sh4Ez0nG7pTcYfYDXUqLfBdirdKeIuC4tiddHxAsdPP8HgFMkbZqOt42kj6Zx+ZKkjSX1Bg7P7bMMaEjl/KcP+gAvptWDE8lWQ9qygmwFIe8XwH8Bt3YwfjMzK4OiJ/sJwDPAU2SJ5nGAiHibLJFdLmkuMId0PR8YAVwhaR7ZNe3v5zuMiLfIlqHHS5oPrAFujIh/AGcCf5DURJbslrO2c4ED0r5NZPcT/AHoLmkR2fX2GeU4+Yh4kCy5Pp6Ody/QOyKeBMYBc4HfA0/kdvsJ2T0Is4H8R/6uB0ak8doF+Hs7h/8NcFTzDXqp7k6gL9klDjMzqxB/610ZSdo0IlYquxHgOuCZiLiy2nG1R9IoYGVE/GQ9H+cY4MiIOLHdtv7Wu07zt96ZmVr51rsP6zX79eVUSSOADcluirupnfYfGpKuAT5Py/cbmJnZeuSZvdUkz+w7zzN7M2ttZl/0a/ZmZmYfek72ZmZmBedr9laTGrZuoPGSxvYbmplZuzyzNzMzKzgnezMzs4JzsjczMys4J3szM7OCc7I3MzMrOCd7MzOzgnOyNzMzKzgnezMzs4JzsjczMys4J3szM7OCc7I3MzMrOCd7MzOzgnOyNzMzKzgnezMzs4JzsjczMys4J3szM7OCc7I3MzMruO7VDsCsJU1NIFU7CjOzyopYP/16Zm9mZlZwTvZmZmYF52RvZmZWcE72ZmZmBedkb2ZmVnBO9lY2kk6XdFIb24dK2qeSMZmZmT96Z2UUETe202QosBJ4bP1HY2ZmzTyz72Ik1UlaJOlmSQslPShpY0n1kmZImidpgqS+qf1USZdLmiVpiaT9Wun3HElPpf3vTnW9JI1J+86WdGSqv0rSxal8qKRHJG0gaZSkC1rqT1IdcDpwvqQ5rcVhZmbl55l917QTcHxEnCrpHuBo4JvA2RExTdL3gUuA81L77hGxp6Rhqf6gFvq8ENg+IlZJ2izVXQRMjohTUt0sSQ8B3waekDQduBoYFhFr9P6/gvO+/iLidUk3Aisj4ictnZSk04DTsmfbrdPAmJnZ2jyz75qWRsScVG4CdgQ2i4hpqe42YP9c+/tybeta6XMecKekfwPeSXWHABdKmgNMBXoC20XEm8CpwCTg2oj4Uwf7a1NEjI6IIRExBPp3ZBczM+sAJ/uuaVWuvBrYrLWGJe1Xk1ZzJN2altN/l7Z9AbgOGEw2a+8OCDg6IurTY7uIWJTa7wb8Fdi6lWO21J+ZmVWBk30xLAdey10HPxGY1kZ7IuLklMCHSdoA2DYipgDfAvoAmwIPAGcrrc9L2j39/Djwf4Hdgc9L+ky+7zb6WwH0LscJm5lZx3m2VRwjgBslbQI8B5zciX27AXdI6kM2m786XWP/AfBzYF5K4EslHQ78J3BBRLwg6avAWEl7dKC/3wD3phv9zo6I6R/wnM3MrAMU6+srdsw+AGlIQGO1wzAzq6gPmpIlNWX3Pb2fl/HNzMwKzsnezMys4JzszczMCs436FlNamiARl+yNzMrC8/szczMCs7J3szMrOCc7M3MzArOyd7MzKzgnOzNzMwKzsnezMys4JzszczMCs7J3szMrOD8RThWkyStABZXO45O6Ae8Wu0gOqmrxex416+uFi90vZgrEe/HI6J/aaX/gp7VqsUtfXNTrZLU2JXiha4Xs+Ndv7pavND1Yq5mvF7GNzMzKzgnezMzs4JzsrdaNbraAXRSV4sXul7Mjnf96mrxQteLuWrx+gY9MzOzgvPM3szMrOCc7M3MzArOyd5qiqTDJC2W9KykC6sdT56kZZLmS5ojqTHVbS5pkqRn0s++qV6Srk7nMU/S4ArEN0bSy5IW5Oo6HZ+kEan9M5JGVDjeUZKeT2M8R9Kw3LZvp3gXSzo0V1+R14ykbSVNkfSUpIWSzk31tTzGrcVck+MsqaekWZLmpni/l+q3lzQzHXucpA1T/Ubp+bNpe11751GheMdKWpob3/pUX73XRET44UdNPIBuwJ+AHYANgbnAwGrHlYtvGdCvpO7HwIWpfCFweSoPA34PCNgLmFmB+PYHBgML1jU+YHPgufSzbyr3rWC8o4ALWmg7ML0eNgK2T6+TbpV8zQBbAYNTuTewJMVVy2PcWsw1Oc5prDZN5R7AzDR29wDHpfobgTNS+UzgxlQ+DhjX1nlUMN6xwDEttK/aa8Ize6slewLPRsRzEfE2cDdwZJVjas+RwG2pfBvwpVz9LyIzA9hM0lbrM5CIeAT42weM71BgUkT8LSJeAyYBh1Uw3tYcCdwdEasiYinwLNnrpWKvmYh4MSKeTOUVwCJgG2p7jFuLuTVVHec0VivT0x7pEcCBwL2pvnSMm8f+XuBzktTGeVQq3tZU7TXhZG+1ZBvgz7nnf6Ht/5gqLYAHJTVJOi3VbRkRL6by/wBbpnKtnEtn46uFuP89LXGOaV4SbyOuqsSblot3J5vJdYkxLokZanScJXWTNAd4mSzp/Ql4PSLeaeHY78aVti8HtqhmvBHRPL6XpfG9UtJGpfGWxLXe43WyN+u4fSNiMPB54CxJ++c3RrYeV7OfZa31+JIbgB2BeuBF4KfVDWdtkjYFfgmcFxFv5LfV6hi3EHPNjnNErI6IeuBjZLPxXaocUptK45W0K/Btsrj3IFua/1YVQwSc7K22PA9sm3v+sVRXEyLi+fTzZWAC2X9ELzUvz6efL6fmtXIunY2vqnFHxEvpP881wM28t/RaE/FK6kGWNO+MiPtSdU2PcUsx1/o4pxhfB6YAe5Mtdzd/l0v+2O/Glbb3Af5a5XgPS5dPIiJWAbdSA+PrZG+15Algp3Tn7YZkN9xMrHJMAEjqJal3cxk4BFhAFl/znbMjgF+n8kTgpHT37V7A8txSbyV1Nr4HgEMk9U1Lu4ekuoooua/hKLIxbo73uHT39fbATsAsKviaSdeC/2T1IZIAAAEeSURBVBNYFBE/y22q2TFuLeZaHWdJ/SVtlsobAweT3WcwBTgmNSsd4+axPwaYnFZXWjuPSsT7dO7Nn8juL8iPb3VeE+W8288PPz7og+xu1SVk1+kuqnY8ubh2ILu7dy6wsDk2suuDDwPPAA8Bm6d6Adel85gPDKlAjHeRLcn+k+ya31fXJT7gFLIbmp4FTq5wvLeneOaR/ce4Va79RSnexcDnK/2aAfYlW6KfB8xJj2E1PsatxVyT4wx8Gpid4loAXJz7/ZuVxms8sFGq75meP5u279DeeVQo3slpfBcAd/DeHftVe034z+WamZkVnJfxzczMCs7J3szMrOCc7M3MzArOyd7MzKzgnOzNzMwKzsnezMys4JzszczMCu5/AU5hzQ9TGV2WAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "exist_train['task2'].value_counts()"
      ],
      "metadata": {
        "id": "w78NAkbh0FYD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5b50635a-658e-4524-d5bf-92aa53b44571"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "non-sexist                      3600\n",
              "ideological-inequality           866\n",
              "stereotyping-dominance           809\n",
              "misogyny-non-sexual-violence     685\n",
              "sexual-violence                  517\n",
              "objectification                  500\n",
              "Name: task2, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_es['task2'].count()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Sno9KFGjbpkV",
        "outputId": "f2a869e9-b770-4a49-b12a-046cda61bb9f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3541"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install contractions"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pA9ZmtPAbp3g",
        "outputId": "c487c221-9c10-4888-ebd8-894b6d2492b6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: contractions in /usr/local/lib/python3.7/dist-packages (0.1.72)\n",
            "Requirement already satisfied: textsearch>=0.0.21 in /usr/local/lib/python3.7/dist-packages (from contractions) (0.0.21)\n",
            "Requirement already satisfied: pyahocorasick in /usr/local/lib/python3.7/dist-packages (from textsearch>=0.0.21->contractions) (1.4.4)\n",
            "Requirement already satisfied: anyascii in /usr/local/lib/python3.7/dist-packages (from textsearch>=0.0.21->contractions) (0.3.1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_emoji(text):\n",
        "    emoji_pattern = re.compile(\"[\"\n",
        "                           u\"U0001F600-U0001F64F\"  # emoticons\n",
        "                           u\"U0001F300-U0001F5FF\"  # symbols & pictographs\n",
        "                           u\"U0001F680-U0001F6FF\"  # transport & map symbols\n",
        "                           u\"U0001F1E0-U0001F1FF\"  # flags (iOS)\n",
        "                           \"]+\", flags=re.UNICODE)\n",
        "    return emoji_pattern.sub(r' ', text)\n",
        "\n",
        "def remove_extra_space(text):\n",
        "    text = \" \".join(text.split())\n",
        "    return text\n",
        "\n",
        "def remove_emoticons(text):\n",
        "    text = re.sub(r'[^0-9|a-zA-Z|@|!|?|&|$|#|,|.|_|\\']',' ',text)\n",
        "    return text\n",
        "\n",
        "def remove_weblinks(text):\n",
        "    text = re.sub(r'http\\S+', ' ', text, flags = re.MULTILINE)\n",
        "    return text\n",
        "\n",
        "def remove_specialchar(text):\n",
        "    text = re.sub(r'[,|.|?|$|&|!|#|@]', ' ', text)\n",
        "    return text\n",
        "\n",
        "def remove_apostrophe(text):\n",
        "    text = re.sub(\"'\",\" \",text)\n",
        "    return text\n",
        "\n",
        "def remove_number(text):\n",
        "    text = re.sub(r\"\\b[0-9]+\\b\",\" \",text)\n",
        "    return text\n",
        "\n",
        "def fix_contractions(text):\n",
        "    text = contractions.fix(text)\n",
        "    return text\n",
        "\n",
        "def remove_repetitions(text):\n",
        "    text = re.sub(r'\\b([a-zA-Z])\\1{1,}\\b', ' ', text)\n",
        "    text = re.sub(r'\\b(_)\\1{1,}\\b', ' ', text)\n",
        "    return text\n",
        "\n",
        "def remove_singlealphabet(text):\n",
        "    text = re.sub(r'\\b([a-zA-Z])\\b', ' ', text)\n",
        "    return text    "
      ],
      "metadata": {
        "id": "nn_zEEUYbqIr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def preprocess(text):\n",
        "  text = remove_weblinks(text)\n",
        "  text = remove_specialchar(text)\n",
        "  text = fix_contractions(text)\n",
        "  #text = remove_emoji(text)\n",
        "  text = remove_apostrophe(text)\n",
        "  text = remove_number(text)\n",
        "  text = remove_emoticons(text)\n",
        "  text = remove_singlealphabet(text)\n",
        "  text = remove_repetitions(text)\n",
        "  text = remove_extra_space(text)\n",
        "  return text"
      ],
      "metadata": {
        "id": "mA2UEWPPbqR0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_en['clean text'] = ''"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UyCxj61lbqbn",
        "outputId": "47f6dcc2-f74a-44a7-f5a4-e24c827bad9a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_en['clean text'] = train_en['text'].apply(preprocess)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EPbeHpQxbqkG",
        "outputId": "a04ad987-2413-4c12-c88a-fa336c5fb036"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "i = 890\n",
        "print(train_en['clean text'][i])\n",
        "print(train_en['task1'][i])\n",
        "print(train_en['task2'][i])\n",
        "print('Original : ',train_en['text'][i])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OT6b1pCzbqsi",
        "outputId": "a879ff8e-c072-410e-a984-8aed901e8301"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "SaintRap The fact filtered in the snapped pictureMy man Jack ripped over Google like Jack the RipperYoohoo am fixing to murder this brunette bitchGet pumped like flat fixed to become flat fixtureA rap figure to look like you hacked TwitterI show you Beastie Boy\n",
            "sexist\n",
            "sexual-violence\n",
            "Original :  @SaintRap The fact's filtered in the snapped pictureMy man Jack ripped over Google like Jack the RipperYoohoo, I'm finna murder this brunette bitchGet pumped like a flat fixed to become a flat fixtureA rap figure to look like you hacked TwitterI'll show you Beastie Boy\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "tCDhTjNmTeO8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(stopwords.words('english'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kHG6cVXqpOFE",
        "outputId": "c2102238-d5db-4691-de53-1f1d4eedc3d6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\", \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", 'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn', \"hadn't\", 'hasn', \"hasn't\", 'haven', \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn', \"mustn't\", 'needn', \"needn't\", 'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", 'won', \"won't\", 'wouldn', \"wouldn't\"]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#nltk.download('stopwords')\n",
        "#nltk.download('punkt')\n",
        "nltk.tokenize.word_tokenize(\"Aure_LieGuiboud\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Pq3UA9jzqK40",
        "outputId": "6f69cc8f-2da8-4844-c291-698615309308"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Aure_LieGuiboud']"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "stemmer = SnowballStemmer('english', ignore_stopwords = True)"
      ],
      "metadata": {
        "id": "sEOsB0-q0qo0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_en['clean text'] = test_en['text'].apply(preprocess)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IlsWFFBa1G6m",
        "outputId": "2cf43ad3-339c-46b0-9468-95148d7c08b7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tfidf_vectorizer = TfidfVectorizer(analyzer = 'word', ngram_range = (1,2), lowercase = True)\n",
        "X_train_tfidf = tfidf_vectorizer.fit_transform(train_en['clean text'])\n",
        "X_test_tfidf = tfidf_vectorizer.transform(test_en['clean text'])\n",
        "tfidf_feature_names = np.array(tfidf_vectorizer.get_feature_names())\n",
        "importance = np.argsort(np.asarray(X_train_tfidf.sum(axis=0)).ravel())[::-1]\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WORkGD411HGn",
        "outputId": "ca5787be-5b2b-4784-f3dc-a7e869e371cb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(tfidf_feature_names[1000])\n",
        "X_train_tfidf.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mM7VXLho1HNy",
        "outputId": "b849164b-d262-4805-8471-eba84c48afbe"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "actual good\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(3436, 72620)"
            ]
          },
          "metadata": {},
          "execution_count": 116
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tfidf_feature_names[80:100]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PVlZyHNyWgu3",
        "outputId": "c398deb1-d9d6-4c72-9ec5-0b610abf6700"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['1newsnz', '1newsnz agree', '1pckt', '1pckt ____9753', '1s',\n",
              "       '1s like', '1st', '1st amendment', '1st book', '1st church',\n",
              "       '1st customer', '1st ladies', '1st lady', '1st mawm',\n",
              "       '1st minister', '1st thought', '1starshot', '1starshot kimruxton',\n",
              "       '1stderek', '1stderek senseisfuneral'], dtype='<U58')"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "clf = LogisticRegression(penalty='none')\n",
        "clf.fit(X_train_tfidf,train_en['task1'])\n",
        "pred = clf.predict(X_test_tfidf)\n",
        "#print(clf.best_params_)\n",
        "#print(clf.best_score_)\n",
        "print(classification_report(test_en['task1'],pred))\n",
        "print(\"Accuracy : \",np.mean(pred == test_en['task1']))\n",
        "print('Confusion Matrix:',confusion_matrix(test_en['task1'], pred))\n",
        "\n",
        "clf2 = LogisticRegression(C=100, penalty='none')\n",
        "clf2.fit(X_train_tfidf,train_en['task2'])\n",
        "pred = clf2.predict(X_test_tfidf)\n",
        "#print(clf.best_params_)\n",
        "#print(clf.best_score_)\n",
        "print(classification_report(test_en['task2'],pred))\n",
        "print(\"Accuracy : \",np.mean(pred == test_en['task2']))\n",
        "print('Confusion Matrix:',confusion_matrix(test_en['task2'], pred))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aj8X5kG84so3",
        "outputId": "8fbc8756-5296-44ed-edb3-330a29ffa338"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "  non-sexist       0.72      0.68      0.70      1050\n",
            "      sexist       0.72      0.76      0.74      1158\n",
            "\n",
            "    accuracy                           0.72      2208\n",
            "   macro avg       0.72      0.72      0.72      2208\n",
            "weighted avg       0.72      0.72      0.72      2208\n",
            "\n",
            "Accuracy :  0.7196557971014492\n",
            "Confusion Matrix: [[714 336]\n",
            " [283 875]]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:1484: UserWarning: Setting penalty='none' will ignore the C and l1_ratio parameters\n",
            "  \"Setting penalty='none' will ignore the C and l1_ratio parameters\"\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                              precision    recall  f1-score   support\n",
            "\n",
            "      ideological-inequality       0.51      0.50      0.51       333\n",
            "misogyny-non-sexual-violence       0.43      0.29      0.35       215\n",
            "                  non-sexist       0.68      0.75      0.71      1050\n",
            "             objectification       0.40      0.35      0.37       150\n",
            "             sexual-violence       0.47      0.47      0.47       198\n",
            "      stereotyping-dominance       0.43      0.40      0.42       262\n",
            "\n",
            "                    accuracy                           0.58      2208\n",
            "                   macro avg       0.49      0.46      0.47      2208\n",
            "                weighted avg       0.56      0.58      0.57      2208\n",
            "\n",
            "Accuracy :  0.5756340579710145\n",
            "Confusion Matrix: [[166  14 119   2   7  25]\n",
            " [ 22  63  72  18  18  22]\n",
            " [ 70  35 792  38  56  59]\n",
            " [  7  11  38  52  18  24]\n",
            " [ 10  13  60  15  93   7]\n",
            " [ 49  11  88   5   4 105]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "clf = MultinomialNB(alpha = 2)\n",
        "clf.fit(X_train_tfidf,train_en['task1'])\n",
        "pred = clf.predict(X_test_tfidf)\n",
        "#print(clf.best_params_)\n",
        "#print(clf.best_score_)\n",
        "print(classification_report(test_en['task1'],pred))\n",
        "print(\"Accuracy : \",np.mean(pred == test_en['task1']))\n",
        "print('Confusion Matrix:',confusion_matrix(test_en['task1'], pred))\n",
        "\n",
        "clf2 = MultinomialNB()\n",
        "clf2.fit(X_train_tfidf,train_en['task2'])\n",
        "pred = clf2.predict(X_test_tfidf)\n",
        "#print(clf.best_params_)\n",
        "#print(clf.best_score_)\n",
        "print(classification_report(test_en['task2'],pred))\n",
        "print(\"Accuracy : \",np.mean(pred == test_en['task2']))\n",
        "print('Confusion Matrix:',confusion_matrix(test_en['task2'], pred))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9uFen-th4tKA",
        "outputId": "49a5a803-d4fb-43ca-86c6-1efdb2230c18"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "  non-sexist       0.63      0.79      0.70      1050\n",
            "      sexist       0.75      0.57      0.65      1158\n",
            "\n",
            "    accuracy                           0.68      2208\n",
            "   macro avg       0.69      0.68      0.67      2208\n",
            "weighted avg       0.69      0.68      0.67      2208\n",
            "\n",
            "Accuracy :  0.6757246376811594\n",
            "Confusion Matrix: [[832 218]\n",
            " [498 660]]\n",
            "                              precision    recall  f1-score   support\n",
            "\n",
            "      ideological-inequality       0.00      0.00      0.00       333\n",
            "misogyny-non-sexual-violence       0.00      0.00      0.00       215\n",
            "                  non-sexist       0.48      1.00      0.65      1050\n",
            "             objectification       1.00      0.01      0.03       150\n",
            "             sexual-violence       1.00      0.01      0.01       198\n",
            "      stereotyping-dominance       0.78      0.03      0.05       262\n",
            "\n",
            "                    accuracy                           0.48      2208\n",
            "                   macro avg       0.54      0.17      0.12      2208\n",
            "                weighted avg       0.48      0.48      0.32      2208\n",
            "\n",
            "Accuracy :  0.48007246376811596\n",
            "Confusion Matrix: [[   0    0  333    0    0    0]\n",
            " [   0    0  215    0    0    0]\n",
            " [   0    0 1050    0    0    0]\n",
            " [   0    0  146    2    0    2]\n",
            " [   0    0  197    0    1    0]\n",
            " [   0    0  255    0    0    7]]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "clf = SVC(C=10, kernel='rbf',gamma=0.1)\n",
        "clf.fit(X_train_tfidf,train_en['task1'])\n",
        "pred = clf.predict(X_test_tfidf)\n",
        "#print(clf.best_params_)\n",
        "#print(clf.best_score_)\n",
        "print(classification_report(test_en['task1'],pred))\n",
        "print(\"Accuracy : \",np.mean(pred == test_en['task1']))\n",
        "print('Confusion Matrix:',confusion_matrix(test_en['task1'], pred))\n",
        "\n",
        "clf2 = SVC(C=10, kernel='rbf',gamma=0.1)\n",
        "clf2.fit(X_train_tfidf,train_en['task2'])\n",
        "pred = clf2.predict(X_test_tfidf)\n",
        "#print(clf.best_params_)\n",
        "#print(clf.best_score_)\n",
        "print(classification_report(test_en['task2'],pred))\n",
        "print(\"Accuracy : \",np.mean(pred == test_en['task2']))\n",
        "print('Confusion Matrix:',confusion_matrix(test_en['task2'], pred))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0IXON9Sk4tYC",
        "outputId": "65babaca-069f-4fa8-862b-fbb2a63db608"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "  non-sexist       0.71      0.65      0.68      1050\n",
            "      sexist       0.70      0.75      0.73      1158\n",
            "\n",
            "    accuracy                           0.71      2208\n",
            "   macro avg       0.71      0.70      0.70      2208\n",
            "weighted avg       0.71      0.71      0.70      2208\n",
            "\n",
            "Accuracy :  0.7051630434782609\n",
            "Confusion Matrix: [[683 367]\n",
            " [284 874]]\n",
            "                              precision    recall  f1-score   support\n",
            "\n",
            "      ideological-inequality       0.63      0.42      0.51       333\n",
            "misogyny-non-sexual-violence       0.51      0.25      0.33       215\n",
            "                  non-sexist       0.60      0.86      0.71      1050\n",
            "             objectification       0.45      0.25      0.32       150\n",
            "             sexual-violence       0.53      0.39      0.45       198\n",
            "      stereotyping-dominance       0.52      0.31      0.39       262\n",
            "\n",
            "                    accuracy                           0.59      2208\n",
            "                   macro avg       0.54      0.41      0.45      2208\n",
            "                weighted avg       0.57      0.59      0.55      2208\n",
            "\n",
            "Accuracy :  0.5855978260869565\n",
            "Confusion Matrix: [[140   7 163   0   6  17]\n",
            " [ 12  53 118  10  11  11]\n",
            " [ 37  20 904  23  37  29]\n",
            " [  5   8  69  37  13  18]\n",
            " [  6  10  94   9  78   1]\n",
            " [ 21   6 148   4   2  81]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_es['clean text'] = ''"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FBrbmlEzt22r",
        "outputId": "ed6a9564-c0ea-49d5-a583-5f7573c62ad0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_es['clean text'] = train_es['text'].apply(preprocess)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5IqEsfcvXuDe",
        "outputId": "406edd16-5eb6-4720-ea8e-3e0949a34a14"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_es['clean text'] = test_es['text'].apply(preprocess)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r6P6R-VH3LyT",
        "outputId": "62a5d7a3-fde1-4fc9-9fe9-0619c90bad57"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_es['clean text']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WallwVgPXuLA",
        "outputId": "fa24f460-d325-4868-ec05-a5a6df94b134"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3436    Nadie te va tratar tan bien como un hombre que...\n",
              "3437    lindagisela74 Que rica putita obediente afortu...\n",
              "3438    BicireporteraDF Yo lo hice los os por la carre...\n",
              "3439    las cosas sue son indicios de que yo enrealida...\n",
              "3440    Pero la ni le gust desde que lo vi as que me l...\n",
              "                              ...                        \n",
              "6972    Estamos igual sin pareja pero puedes besar tus...\n",
              "6973                                 hijo de re mil putas\n",
              "6974    SEGURAMENTE ESTA CHICA NO COBRA EL DINERO QUE ...\n",
              "6975              safetyaitana mi madre dice va fea agree\n",
              "6976    En vuestras casas tambi ten is esa tradici de ...\n",
              "Name: clean text, Length: 3541, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 122
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tfidf_vectorizer = TfidfVectorizer(analyzer = 'word', ngram_range = (1,2), lowercase = True)\n",
        "X_train_tfidf = tfidf_vectorizer.fit_transform(train_es['clean text'])\n",
        "X_test_tfidf = tfidf_vectorizer.transform(test_es['clean text'])\n",
        "tfidf_feature_names = np.array(tfidf_vectorizer.get_feature_names())\n",
        "importance = np.argsort(np.asarray(X_train_tfidf.sum(axis=0)).ravel())[::-1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VByff1mr2vUW",
        "outputId": "bb69e811-ac76-44bf-e5c5-ffdb9160af0d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "clf = LogisticRegression(penalty='none')\n",
        "clf.fit(X_train_tfidf,train_es['task1'])\n",
        "pred = clf.predict(X_test_tfidf)\n",
        "#print(clf.best_params_)\n",
        "#print(clf.best_score_)\n",
        "print(classification_report(test_es['task1'],pred))\n",
        "print(\"Accuracy : \",np.mean(pred == test_es['task1']))\n",
        "print('Confusion Matrix:',confusion_matrix(test_es['task1'], pred))\n",
        "\n",
        "clf2 = LogisticRegression(C=100, penalty='none')\n",
        "clf2.fit(X_train_tfidf,train_es['task2'])\n",
        "pred = clf2.predict(X_test_tfidf)\n",
        "#print(clf.best_params_)\n",
        "#print(clf.best_score_)\n",
        "print(classification_report(test_es['task2'],pred))\n",
        "print(\"Accuracy : \",np.mean(pred == test_es['task2']))\n",
        "print('Confusion Matrix:',confusion_matrix(test_es['task2'], pred))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L5kYy2d92vgA",
        "outputId": "8683f8ab-14a0-4943-e0ec-e8c6b03dc534"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "  non-sexist       0.69      0.78      0.73      1037\n",
            "      sexist       0.77      0.68      0.72      1123\n",
            "\n",
            "    accuracy                           0.73      2160\n",
            "   macro avg       0.73      0.73      0.73      2160\n",
            "weighted avg       0.73      0.73      0.73      2160\n",
            "\n",
            "Accuracy :  0.7287037037037037\n",
            "Confusion Matrix: [[805 232]\n",
            " [354 769]]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:1484: UserWarning: Setting penalty='none' will ignore the C and l1_ratio parameters\n",
            "  \"Setting penalty='none' will ignore the C and l1_ratio parameters\"\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                              precision    recall  f1-score   support\n",
            "\n",
            "      ideological-inequality       0.67      0.60      0.63       288\n",
            "misogyny-non-sexual-violence       0.63      0.31      0.41       257\n",
            "                  non-sexist       0.64      0.87      0.74      1037\n",
            "             objectification       0.59      0.24      0.34       174\n",
            "             sexual-violence       0.69      0.27      0.39       202\n",
            "      stereotyping-dominance       0.43      0.48      0.45       202\n",
            "\n",
            "                    accuracy                           0.62      2160\n",
            "                   macro avg       0.61      0.46      0.49      2160\n",
            "                weighted avg       0.62      0.62      0.59      2160\n",
            "\n",
            "Accuracy :  0.6208333333333333\n",
            "Confusion Matrix: [[173   4  92   1   0  18]\n",
            " [ 22  79 120   5   9  22]\n",
            " [ 35  23 898  11  11  59]\n",
            " [  6  10  98  41   4  15]\n",
            " [  5   1 120   8  54  14]\n",
            " [ 18   9  76   3   0  96]]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "clf = MultinomialNB(alpha = 2)\n",
        "clf.fit(X_train_tfidf,train_es['task1'])\n",
        "pred = clf.predict(X_test_tfidf)\n",
        "#print(clf.best_params_)\n",
        "#print(clf.best_score_)\n",
        "print(classification_report(test_es['task1'],pred))\n",
        "print(\"Accuracy : \",np.mean(pred == test_es['task1']))\n",
        "print('Confusion Matrix:',confusion_matrix(test_es['task1'], pred))\n",
        "\n",
        "clf2 = MultinomialNB()\n",
        "clf2.fit(X_train_tfidf,train_es['task2'])\n",
        "pred = clf2.predict(X_test_tfidf)\n",
        "#print(clf.best_params_)\n",
        "#print(clf.best_score_)\n",
        "print(classification_report(test_es['task2'],pred))\n",
        "print(\"Accuracy : \",np.mean(pred == test_es['task2']))\n",
        "print('Confusion Matrix:',confusion_matrix(test_es['task2'], pred))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GKB6HC932voP",
        "outputId": "a2341115-ec67-499c-8cf4-5ae4ad17710c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "  non-sexist       0.66      0.81      0.73      1037\n",
            "      sexist       0.78      0.62      0.69      1123\n",
            "\n",
            "    accuracy                           0.71      2160\n",
            "   macro avg       0.72      0.71      0.71      2160\n",
            "weighted avg       0.72      0.71      0.71      2160\n",
            "\n",
            "Accuracy :  0.7092592592592593\n",
            "Confusion Matrix: [[835 202]\n",
            " [426 697]]\n",
            "                              precision    recall  f1-score   support\n",
            "\n",
            "      ideological-inequality       1.00      0.01      0.01       288\n",
            "misogyny-non-sexual-violence       0.00      0.00      0.00       257\n",
            "                  non-sexist       0.48      1.00      0.65      1037\n",
            "             objectification       0.71      0.03      0.06       174\n",
            "             sexual-violence       0.00      0.00      0.00       202\n",
            "      stereotyping-dominance       1.00      0.03      0.07       202\n",
            "\n",
            "                    accuracy                           0.49      2160\n",
            "                   macro avg       0.53      0.18      0.13      2160\n",
            "                weighted avg       0.52      0.49      0.33      2160\n",
            "\n",
            "Accuracy :  0.48657407407407405\n",
            "Confusion Matrix: [[   2    0  286    0    0    0]\n",
            " [   0    0  257    0    0    0]\n",
            " [   0    0 1037    0    0    0]\n",
            " [   0    0  169    5    0    0]\n",
            " [   0    0  202    0    0    0]\n",
            " [   0    0  193    2    0    7]]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "clf = SVC(C=10, kernel='rbf',gamma=0.1)\n",
        "clf.fit(X_train_tfidf,train_es['task1'])\n",
        "pred = clf.predict(X_test_tfidf)\n",
        "#print(clf.best_params_)\n",
        "#print(clf.best_score_)\n",
        "print(classification_report(test_es['task1'],pred))\n",
        "print(\"Accuracy : \",np.mean(pred == test_es['task1']))\n",
        "print('Confusion Matrix:',confusion_matrix(test_es['task1'], pred))\n",
        "\n",
        "clf2 = SVC(C=10, kernel='rbf',gamma=0.1)\n",
        "clf2.fit(X_train_tfidf,train_es['task2'])\n",
        "pred = clf2.predict(X_test_tfidf)\n",
        "#print(clf.best_params_)\n",
        "#print(clf.best_score_)\n",
        "print(classification_report(test_es['task2'],pred))\n",
        "print(\"Accuracy : \",np.mean(pred == test_es['task2']))\n",
        "print('Confusion Matrix:',confusion_matrix(test_es['task2'], pred))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dhFW7HqU2vuu",
        "outputId": "60df0ac9-6e71-490c-8f79-8f37c8dbf145"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "  non-sexist       0.69      0.78      0.73      1037\n",
            "      sexist       0.77      0.67      0.72      1123\n",
            "\n",
            "    accuracy                           0.72      2160\n",
            "   macro avg       0.73      0.72      0.72      2160\n",
            "weighted avg       0.73      0.72      0.72      2160\n",
            "\n",
            "Accuracy :  0.7226851851851852\n",
            "Confusion Matrix: [[806 231]\n",
            " [368 755]]\n",
            "                              precision    recall  f1-score   support\n",
            "\n",
            "      ideological-inequality       0.69      0.53      0.60       288\n",
            "misogyny-non-sexual-violence       0.63      0.28      0.39       257\n",
            "                  non-sexist       0.59      0.91      0.72      1037\n",
            "             objectification       0.66      0.22      0.33       174\n",
            "             sexual-violence       0.83      0.17      0.28       202\n",
            "      stereotyping-dominance       0.56      0.35      0.43       202\n",
            "\n",
            "                    accuracy                           0.61      2160\n",
            "                   macro avg       0.66      0.41      0.46      2160\n",
            "                weighted avg       0.64      0.61      0.57      2160\n",
            "\n",
            "Accuracy :  0.6101851851851852\n",
            "Confusion Matrix: [[154   1 128   0   0   5]\n",
            " [ 16  73 155   2   1  10]\n",
            " [ 29  23 948   8   3  26]\n",
            " [  7  11 105  39   3   9]\n",
            " [  2   1 154   6  34   5]\n",
            " [ 14   7 107   4   0  70]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "ppe3mqR12v0u"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "iGuV8zPQ2v6W"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "fwgbIu-42wAH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "hxB29JIm2wHi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "IBME_SSG2wNN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "dZwq6N4J2wUZ"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}