05 - Preprocessing + stemming.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rb9QuLWBuMjB",
        "outputId": "da1b062d-a97f-490e-8ded-621f440fd376"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package words to /root/nltk_data...\n",
            "[nltk_data]   Package words is already up-to-date!\n"
          ]
        }
      ],
      "source": [
        "import pandas as pd \n",
        "import numpy as np \n",
        "import matplotlib.pyplot as plt \n",
        "import seaborn as sns\n",
        "from nltk.stem.porter import PorterStemmer\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
        "from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import word_tokenize, sent_tokenize\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.metrics import accuracy_score, precision_score, recall_score\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.ensemble import RandomForestClassifier,GradientBoostingClassifier,AdaBoostClassifier,ExtraTreesClassifier\n",
        "from collections import Counter\n",
        "import string\n",
        "import nltk\n",
        "nltk.download('words')\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "7rBGON8LuMjD",
        "outputId": "5185a149-220e-41d7-9559-3becf3283be6"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                             subject  \\\n",
              "0            job posting - apple-iss research center   \n",
              "1  query : letter frequencies for text identifica...   \n",
              "2                                               risk   \n",
              "3                           request book information   \n",
              "4  call for abstracts : optimality in syntactic t...   \n",
              "\n",
              "                                             message  label  \n",
              "0  content - length : 3386 apple-iss research cen...      0  \n",
              "1  i am posting this inquiry for sergei atamas ( ...      0  \n",
              "2  a colleague and i are researching the differin...      0  \n",
              "3  earlier this morning i was on the phone with a...      0  \n",
              "4  content - length : 4437 call for papers is the...      0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-5e357269-e3fc-4655-a143-217ef9f7c6e4\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>subject</th>\n",
              "      <th>message</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>job posting - apple-iss research center</td>\n",
              "      <td>content - length : 3386 apple-iss research cen...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>query : letter frequencies for text identifica...</td>\n",
              "      <td>i am posting this inquiry for sergei atamas ( ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>risk</td>\n",
              "      <td>a colleague and i are researching the differin...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>request book information</td>\n",
              "      <td>earlier this morning i was on the phone with a...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>call for abstracts : optimality in syntactic t...</td>\n",
              "      <td>content - length : 4437 call for papers is the...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5e357269-e3fc-4655-a143-217ef9f7c6e4')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-5e357269-e3fc-4655-a143-217ef9f7c6e4 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-5e357269-e3fc-4655-a143-217ef9f7c6e4');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "df = pd.read_csv(\"Final Dataset.csv\")\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_TylKe99uMjE",
        "outputId": "133ff422-ba52-469d-b7fe-15d66bc32426"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "subject    0\n",
              "message    0\n",
              "label      0\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "df.isnull().sum()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sns.heatmap(df.isnull(),cmap='gist_gray',yticklabels=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 287
        },
        "id": "joUua-S47aQO",
        "outputId": "20952c96-5d7c-477c-ff1a-6b56fc8d3826"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f80a9067c10>"
            ]
          },
          "metadata": {},
          "execution_count": 4
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWEAAAD8CAYAAACmcBX+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAXq0lEQVR4nO3df5BdZX3H8ffHBCIihoQoxiSaOMR2ACuaGHSslhIIQZEwQ4DQFJdOcEFF23GcIYgSDeiAdgpDi9qFLAbkZ0Mra41mkmAcaxWzYIREiFkjNBuDSDYgys91v/3jPouH693cvdm7+xyyn9fMmT3neZ5z7nPv3Xz2yXPOvUcRgZmZ5fGK3B0wMxvNHMJmZhk5hM3MMnIIm5ll5BA2M8vIIWxmlpFD2Mz2S5LmS9oqqUvS0hr175N0n6ReSQur6lokbUtLS6F8lqQH0jGvkaSh9tMhbGb7HUljgGuBk4EjgbMlHVnV7P+Ac4FbqvadCCwDjgXmAMskTUjVXwU+DMxMy/yh9tUhbGb7ozlAV0Rsj4jngduABcUGEfFwRNwP9FXtexKwNiJ6ImIPsBaYL2ky8JqI+HFUPuV2I3DaUDs6dm+Vn//85/1xOjMblGXLlg35v+bAoDNH0vlAa6GoLSLa0voUYEehrpvKyHYwau07JS3dNcqHZK8hbGZWVilw2+o2LDlPR5hZaUTEoJc6dgLTCttTU9lgDLTvzrS+L8cckEPYzEqjr69v0EsdG4GZkmZIOhBYBHQMshtrgHmSJqQTcvOANRGxC/idpHelqyI+BNy1b8/0TxzCZlYazRoJR0QvcCGVQH0QuCMitkhaLulUAEnvlNQNnAH8u6Qtad8e4DIqQb4RWJ7KAD4KXA90Ab8EvjPU5+w5YTMrjWZ+tW5ErAZWV5VdWljfyEunF4rt2oH2GuWdwNFN6yQOYTMrkdH4/eYOYTMrDYewmVlGDmEzs4wGcdXDfschbGal4ZGwmVlGDmEzs4wcwmZmGTmEzcwy8ok5M7OMPBI2M8vIIWxmlpFD2MwsI4ewmVlGDmEzs4x8dYSZWUYeCZuZZeQQNjPLaDSGsO8xZ2al0cS7LSNpvqStkrokLa1RP07S7an+HknTU/liSZsKS5+kY1LdhnTM/rrXDfU5eyRsZqXRrBNzksYA1wInAt3ARkkdEfHzQrMlwJ6IOELSIuBK4KyIuBm4OR3nrcA3I2JTYb/F6V5zTeGRsJmVRhNHwnOArojYHhHPA7cBC6raLABWpvVVwNx0K/uis9O+w8YhbGal0cQQngLsKGx3p7KabSKiF3gSOKyqzVnArVVlN6SpiM/WCO2GOYTNrDQaCWFJrZI6C0trM/si6Vjg6YjYXCheHBFvBd6blnOG+jieEzaz0mjk6oiIaAPaBqjeCUwrbE9NZbXadEsaC4wHdhfqF1E1Co6InennU5JuoTLtceOgO12DR8JmVhpNnI7YCMyUNEPSgVQCtaOqTQfQktYXAndHOrCkVwBnUpgPljRW0qS0fgBwCrCZIfJI2MxKo1lXR0REr6QLgTXAGKA9IrZIWg50RkQHsAK4SVIX0EMlqPu9D9gREdsLZeOANSmAxwDrgOuG2leHsJmVRjM/rBERq4HVVWWXFtafBc4YYN8NwLuqyv4AzGpaBxOHsJmVxmj8xJxD2MxKwyFsZpaRQ9jMLCOHsJlZRv5SdzOzjDwSNjPLyCFsZpaRQ9jMLCOHsJlZRj4xZ2aWkUfCZmYZOYTNzDJyCJuZZeQQNjPLyCFsZpaRr44wM8vII2Ezs4wcwmZmGY3GEPbdls2sNJp4t2UkzZe0VVKXpKU16sdJuj3V3yNpeiqfLukZSZvS8rXCPrMkPZD2uUaShvqcHcJmVhrNCmFJY4BrgZOBI4GzJR1Z1WwJsCcijgCuAq4s1P0yIo5JywWF8q8CHwZmpmX+kJ4wDmEzK5G+vr5BL3XMAboiYntEPA/cBiyoarMAWJnWVwFz9zaylTQZeE1E/DgqfwVuBE7bl+dZ5BA2s9JoZCQsqVVSZ2FpLRxqCrCjsN2dyqjVJiJ6gSeBw1LdDEk/lfR9Se8ttO+uc8yG+cScmZVGIyfmIqINaBuGbuwC3hgRuyXNAr4p6ahheBzAIWxmJdLEqyN2AtMK21NTWa023ZLGAuOB3Wmq4bnUn3sl/RJ4S2o/tc4xG+bpCDMrjSZeHbERmClphqQDgUVAR1WbDqAlrS8E7o6IkPTadGIPSW+mcgJue0TsAn4n6V1p7vhDwF1Dfc4eCZtZaTTrY8sR0SvpQmANMAZoj4gtkpYDnRHRAawAbpLUBfRQCWqA9wHLJb0A9AEXRERPqvso8HXgIOA7aRkSh7CZlUYzP6wREauB1VVllxbWnwXOqLHfncCdAxyzEzi6aZ3EIWxmJTIaPzHnEDaz0nAIm5ll5BA2M8vIIWxmlpG/1N3MLCOPhM3MMnIIm5ll5BA2M8vIIWxmlpFD2MwsI18dYWaWkUfCZmYZOYTNzDJyCJuZZeQQNjPLyCfmzMwy8kjYzCyj0RjCvtGnmZVGE2/0iaT5krZK6pK0tEb9OEm3p/p7JE1P5SdKulfSA+nn8YV9NqRjbkrL64b6nD0SNrPSaNZION0t+VrgRKAb2CipIyJ+Xmi2BNgTEUdIWgRcCZwFPA58MCJ+LeloKjcLnVLYb3G611xTeCRsZqXRxJHwHKArIrZHxPPAbcCCqjYLgJVpfRUwV5Ii4qcR8etUvgU4SNK4Jj3FP+MQNrPS6OvrG/QiqVVSZ2FpLRxqCrCjsN3NS0ezL2kTEb3Ak8BhVW1OB+6LiOcKZTekqYjPStJQn7OnI8ysNBqZjoiINqBtuPoi6SgqUxTzCsWLI2KnpEOAO4FzgBuH8jgeCZtZaTRxOmInMK2wPTWV1WwjaSwwHtidtqcC/wV8KCJ+WejfzvTzKeAWKtMeQ+IQNrPSaGIIbwRmSpoh6UBgEdBR1aYDaEnrC4G7IyIkHQp8G1gaET/sbyxprKRJaf0A4BRg81Cfs6cjzKw0mnV1RET0SrqQypUNY4D2iNgiaTnQGREdwArgJkldQA+VoAa4EDgCuFTSpalsHvAHYE0K4DHAOuC6ofbVIWxmpdHMjy1HxGpgdVXZpYX1Z4Ezaux3OXD5AIed1bQOJg5hMyuN0fiJOYewmZWGQ9jMLCOHsJlZRg5hM7OMHMJmZhn5S93NzDLySNjMLCOHsJlZRg5hM7OMHMJmZhk5hM3MMvLVEWZmGXkkbGaWkUPYzCwjh7CZWUYOYTOzjHxizswso9E4EvaNPs2sNJp4o08kzZe0VVKXpKU16sdJuj3V3yNpeqHu4lS+VdJJgz3mvnAIm1lpNCuEJY0BrgVOBo4EzpZ0ZFWzJcCeiDgCuAq4Mu17JJWbfh4FzAe+ImnMII/ZMIewmZVGE0fCc4CuiNgeEc8DtwELqtosAFam9VXAXElK5bdFxHMR8SugKx1vMMdsmEPYzEqjkRCW1Cqps7C0Fg41BdhR2O5OZdRqExG9wJPAYXvZdzDHbJhPzJlZaTRydUREtAFtw9ebkeEQNrPSaOLVETuBaYXtqamsVptuSWOB8cDuOvvWO2bDPB1hZqXRxDnhjcBMSTMkHUjlRFtHVZsOoCWtLwTujsqBO4BF6eqJGcBM4CeDPGbDPBI2s9Jo1kg4InolXQisAcYA7RGxRdJyoDMiOoAVwE2SuoAeKqFKancH8HOgF/hYRPwRoNYxh9pXh7CZlUYzP6wREauB1VVllxbWnwXOGGDfLwBfGMwxh8ohbGalMRo/MecQNrPS8HdHmJll5JGwmVlGDmEzs4wcwmZmGTmEzcwy8ok5M7OMPBI2M8vIIWxmlpFD2MwsI4ewmVlGDmEzs4x8dYSZWUYeCZuZZeQQNjPLyCFsZpaRQ9jMLKPRGMK+0aeZlUZfX9+gl6GQNFHSWknb0s8JA7RrSW22SWpJZa+S9G1JD0naIumKQvtzJf1W0qa0nFevLw5hMyuNJt5tuZ6lwPqImAmsT9svIWkisAw4FpgDLCuE9T9HxF8CbwfeI+nkwq63R8Qxabm+XkccwmZWGiMYwguAlWl9JXBajTYnAWsjoici9gBrgfkR8XREfC/193ngPmDqvnbEIWxmpdFICEtqldRZWFobeKjDI2JXWn8UOLxGmynAjsJ2dyp7kaRDgQ9SGU33O13S/ZJWSZpWryM+MWdmpdHICDci2oC2geolrQNeX6PqkqrjhKSGh9aSxgK3AtdExPZU/C3g1oh4TtL5VEbZx+/tOA5hMyuNZn5sOSJOGKhO0m8kTY6IXZImA4/VaLYTOK6wPRXYUNhuA7ZFxNWFx9xdqL8e+FK9fno6wsxKYwTnhDuAlrTeAtxVo80aYJ6kCemE3LxUhqTLgfHAPxV3SIHe71TgwXod8UjYzEpjBK8TvgK4Q9IS4BHgTABJs4ELIuK8iOiRdBmwMe2zPJVNpTKl8RBwnySAf0tXQnxC0qlAL9ADnFuvIw5hMyuNkQrhNG0wt0Z5J3BeYbsdaK9q0w1ogONeDFzcSF8cwmZWGqPxE3MOYTMrDYewmVlG/lJ3M7OMPBI2M8vIIWxmlpFD2MwsI4ewmVlGPjFnZpaRR8JmZhk5hM3MMnIIm5ll5BA2M8vIIWxmlpGvjjAzy8gjYTOzjBzCZmYZOYTNzDIajSHsG32aWWmM1I0+JU2UtFbStvRzwgDtWlKbbZJaCuUbJG2VtCktr0vl4yTdLqlL0j2Sptfri0PYzEqjr69v0MsQLQXWR8RMYH3afglJE4FlwLHAHGBZVVgvjohj0vJYKlsC7ImII4CrgCvrdcQhbGalMYK3vF8ArEzrK4HTarQ5CVgbET0RsQdYC8xv4LirgLlKt2MeiEPYzEqjkRCW1Cqps7C0NvBQh0fErrT+KHB4jTZTgB2F7e5U1u+GNBXx2ULQvrhPRPQCTwKH7a0jPjFnZqXRyAg3ItqAtoHqJa0DXl+j6pKq44SkRofWiyNip6RDgDuBc4AbGzwG4BA2sxJp5tUREXHCQHWSfiNpckTskjQZeKxGs53AcYXtqcCGdOyd6edTkm6hMmd8Y9pnGtAtaSwwHti9t356OsLMSmMET8x1AP1XO7QAd9VoswaYJ2lCOiE3D1gjaaykSQCSDgBOATbXOO5C4O6o85fFI2EzK40RvE74CuAOSUuAR4AzASTNBi6IiPMiokfSZcDGtM/yVHYwlTA+ABgDrAOuS21WADdJ6gJ6gEX1OuIQNrPSGKkQjojdwNwa5Z3AeYXtdqC9qs0fgFkDHPdZ4IxG+uIQNrPSGI2fmHMIm1lpOITNzDJyCJuZZeQvdTczy8gjYTOzjBzCZmYZOYTNzDJyCJuZZeQQNjPLyFdHmJll5JGwmVlGDmEzs4wcwmZmGTmEzcwy8ok5M7OMPBI2M8vIIWxmltFoDGHf6NPMSiMiBr0MhaSJktZK2pZ+ThigXUtqs01SSyo7RNKmwvK4pKtT3bmSfluoO6/WcYs8Ejaz0hjBkfBSYH1EXCFpadq+qNhA0kRgGTAbCOBeSR0RsQc4ptDuXuA/C7veHhEXDrYjHgmbWWmM4C3vFwAr0/pK4LQabU4C1kZETwretcD8YgNJbwFeB/xgXzviEDaz0hip6Qjg8IjYldYfBQ6v0WYKsKOw3Z3KihZRGfkWO3S6pPslrZI0rV5HPB1hZqXRSLhKagVaC0VtEdFWqF8HvL7GrpdUPWZI2tdUXwScU9j+FnBrRDwn6Xwqo+zj93YAh7CZlUYjIZwCt20v9ScMVCfpN5ImR8QuSZOBx2o02wkcV9ieCmwoHONtwNiIuLfwmLsL7a8HvlTnaXg6wszKYwSnIzqAlrTeAtxVo80aYJ6kCenqiXmprN/ZwK3FHVKg9zsVeLBeRzwSNrPSGMGrI64A7pC0BHgEOBNA0mzggog4LyJ6JF0GbEz7LI+InsIxzgTeX3XcT0g6FegFeoBz63XEIWxmpTFS3x2Rpg3m1ijvBM4rbLcD7QMc4801yi4GLm6kLw5hMyuN0fiJOYewmZWGQ9jMLCOHsJlZRg5hM7OM/KXuZmYZeSRsZpaRQ9jMLCOHsJlZRg5hM7OMHMJmZhn56ggzs4w8EjYzy8ghbGaWkUPYzCwjh7CZWUYOYTOzjHx1hJlZRh4Jm5llNBpD2HdbNrPSGKm7LUuaKGmtpG3p54QB2n1X0hOS/ruqfIakeyR1Sbpd0oGpfFza7kr10+v1xSFsZqUxgre8Xwqsj4iZwPq0XcuXgXNqlF8JXBURRwB7gCWpfAmwJ5VfldrtlUPYzEqjr69v0MsQLQBWpvWVwGm1GkXEeuCpYpkkAccDq2rsXzzuKmBuaj+gvc4JL1u2bK87l5Gk1ohoy92P/Zlf4+E3Wl/jiBh05khqBVoLRW0NvGaHR8SutP4ocPhgHxc4DHgiInrTdjcwJa1PAXYARESvpCdT+8cHOtj+eGKuFRh1v7wjzK/x8PNrXEcK3AFfI0nrgNfXqLqk6jghKdsZwf0xhM3MiIgTBqqT9BtJkyNil6TJwGMNHHo3cKiksWk0PBXYmep2AtOAbkljgfGp/YA8J2xmo1EH0JLWW4C7BrtjVM4Kfg9YWGP/4nEXAndHnbOI+2MI+79ww8+v8fDzazy8rgBOlLQNOCFtI2m2pOv7G0n6AfAfVE6wdUs6KVVdBHxSUheVOd8VqXwFcFgq/yQDX3XxIo3Gi6PNzMpifxwJm5m9bDiEzcwyelmGsKTPSfpUjfI3SFpVa59BHPNcSW8Yeu/MapP0+zr10yVtbvCYX5e0sH5LK6uXZQgPJCJ+HRH7+gt5LuAQNrMRVZoQlnSwpG9L+pmkzZLOkvSwpEmpfrakDYVd3ibpR+kLOD6c2rw4kpA0RtKXJW2UdL+k8wuPdZGkB9JjXZFGErOBmyVtknTQyD3z4ZVek4fSiOkXkm6WdIKkH6bXbk567dsl/UTSTyUtSPselco2pddwZq33KbW9NL3WmyW19X9UU9I7076b0vtR9/3Z30l6taT1ku5Lv4cLCtVj03v0oKRVkl6V9pkl6fuS7pW0Jl3bavuDRr4wYzgX4HTgusL2eOBhYFLang1sSOufA34GHARMovIxwTcA04HNqU0r8Jm0Pg7oBGYAJwP/C7wq1U1MPzcAs3O/DsPwuk4HeoG3Uvmjey/QDojK59y/CXwR+PvU/lDgF8DBwL8Ci1P5gen1/rP3qfg6pvWbgA+m9c3Au9P6FfXen9yv1zC/F79PP8cCr0nrk4Cu9H5MBwJ4T6prBz4FHJB+Z1+bys8C2tP614GFuZ+bl31fSjMSBh6gct3elZLeGxFP1ml/V0Q8ExGPU7lwek5V/TzgQ5I2AfdQuZZvJpVrAm+IiKcBIqKnqc+inH4VEQ9ERB+whcq3RwWV13w6lddqaXqtNgCvBN4I/Aj4tKSLgDdFxDMM/D79bfrqvgeofLnJUZIOBQ6JiB+lNrcU+jTQ+zMaCPiipPuBdVS+b6D/uwt2RMQP0/o3gL8G/gI4GlibXq/PUPmUlu0HSvOx5Yj4haR3AO8HLpe0nsoIrv8PxSurd6mzLeDjEbHmJYV/uth6NHmusN5X2O6j8jvwR+D0iNhatd+Dku4BPgCslnR+RNxd4336EvAVKv+T2CHpc/z5+1Wt5vszSiwGXgvMiogXJD3Mn16vWr/XArZExLtHros2UkozEk5XJjwdEd+g8h2e76AyHTErNTm9apcFkl4p6TDgOGBjVf0a4COSDkjHf4ukg4G1wD8U5tompvZPAYc09Um9fKwBPl6Yx317+vlmYHtEXEPlY5l/NcD71B8gj0t6NenjnBHxBPCUpGNT/aKqx6z1/owG44HHUgD/LfCmQt0bJfWH7d8B/wNsBV7bXy7pAElHjWiPbdiUZiRMZc7yy5L6gBeAj1CZg1wh6TIq/00uup/KNMQk4LKI+LUq32LfP5K4nsp/te9L4fJb4LSI+K6kY4BOSc8Dq4FPU5lb+5qkZ6jMYT4zTM+zjC4Drgbul/QK4FfAKcCZwDmSXqDydX9fBN5J1fsUEU9Iuo7K/O+jvPQP4hLgutT++0D/9EXN92dYn2V53Ax8K03ddAIPFeq2Ah+T1A78HPhqRDyfTh5fI2k8lX+3V1OZWrKXuf3qY8uSZgH/EhF/k7svViHp1RHx+7S+FJgcEf+YuVtmpVGmkfCQSJpN5cRP3S/MsBH1AUkXU/lde4TK9dhmluxXI2Ezs5eb0pyYMzMbjRzCZmYZOYTNzDJyCJuZZeQQNjPL6P8BZyYYA5YQ0v4AAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        },
        "id": "K0FO26dLuMjE",
        "outputId": "92d964e8-6920-4079-de67-c35fa52fbb23"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                             subject  \\\n",
              "0            job posting - apple-iss research center   \n",
              "1  query : letter frequencies for text identifica...   \n",
              "2                                               risk   \n",
              "3                           request book information   \n",
              "4  call for abstracts : optimality in syntactic t...   \n",
              "\n",
              "                                             message  label  \\\n",
              "0  content - length : 3386 apple-iss research cen...      0   \n",
              "1  i am posting this inquiry for sergei atamas ( ...      0   \n",
              "2  a colleague and i are researching the differin...      0   \n",
              "3  earlier this morning i was on the phone with a...      0   \n",
              "4  content - length : 4437 call for papers is the...      0   \n",
              "\n",
              "                                            sub_mssg  \n",
              "0  job posting - apple-iss research center conten...  \n",
              "1  query : letter frequencies for text identifica...  \n",
              "2  risk a colleague and i are researching the dif...  \n",
              "3  request book information earlier this morning ...  \n",
              "4  call for abstracts : optimality in syntactic t...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1cefee32-4ab6-4434-9d20-53ccba608a29\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>subject</th>\n",
              "      <th>message</th>\n",
              "      <th>label</th>\n",
              "      <th>sub_mssg</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>job posting - apple-iss research center</td>\n",
              "      <td>content - length : 3386 apple-iss research cen...</td>\n",
              "      <td>0</td>\n",
              "      <td>job posting - apple-iss research center conten...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>query : letter frequencies for text identifica...</td>\n",
              "      <td>i am posting this inquiry for sergei atamas ( ...</td>\n",
              "      <td>0</td>\n",
              "      <td>query : letter frequencies for text identifica...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>risk</td>\n",
              "      <td>a colleague and i are researching the differin...</td>\n",
              "      <td>0</td>\n",
              "      <td>risk a colleague and i are researching the dif...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>request book information</td>\n",
              "      <td>earlier this morning i was on the phone with a...</td>\n",
              "      <td>0</td>\n",
              "      <td>request book information earlier this morning ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>call for abstracts : optimality in syntactic t...</td>\n",
              "      <td>content - length : 4437 call for papers is the...</td>\n",
              "      <td>0</td>\n",
              "      <td>call for abstracts : optimality in syntactic t...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1cefee32-4ab6-4434-9d20-53ccba608a29')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-1cefee32-4ab6-4434-9d20-53ccba608a29 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-1cefee32-4ab6-4434-9d20-53ccba608a29');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "df['sub_mssg']=df['subject']+\" \"+df['message']\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "qxjdieBpuMjE"
      },
      "outputs": [],
      "source": [
        "df.drop('subject',axis=1,inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "QsGW-lytuMjF",
        "outputId": "a380b11f-a92b-47d1-d399-c76c90b6cd61"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                             message  label  \\\n",
              "0  content - length : 3386 apple-iss research cen...      0   \n",
              "1  i am posting this inquiry for sergei atamas ( ...      0   \n",
              "2  a colleague and i are researching the differin...      0   \n",
              "3  earlier this morning i was on the phone with a...      0   \n",
              "4  content - length : 4437 call for papers is the...      0   \n",
              "\n",
              "                                            sub_mssg  \n",
              "0  job posting - apple-iss research center conten...  \n",
              "1  query : letter frequencies for text identifica...  \n",
              "2  risk a colleague and i are researching the dif...  \n",
              "3  request book information earlier this morning ...  \n",
              "4  call for abstracts : optimality in syntactic t...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-66d796ee-aede-4739-a924-66e35f709582\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>message</th>\n",
              "      <th>label</th>\n",
              "      <th>sub_mssg</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>content - length : 3386 apple-iss research cen...</td>\n",
              "      <td>0</td>\n",
              "      <td>job posting - apple-iss research center conten...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>i am posting this inquiry for sergei atamas ( ...</td>\n",
              "      <td>0</td>\n",
              "      <td>query : letter frequencies for text identifica...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>a colleague and i are researching the differin...</td>\n",
              "      <td>0</td>\n",
              "      <td>risk a colleague and i are researching the dif...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>earlier this morning i was on the phone with a...</td>\n",
              "      <td>0</td>\n",
              "      <td>request book information earlier this morning ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>content - length : 4437 call for papers is the...</td>\n",
              "      <td>0</td>\n",
              "      <td>call for abstracts : optimality in syntactic t...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-66d796ee-aede-4739-a924-66e35f709582')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-66d796ee-aede-4739-a924-66e35f709582 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-66d796ee-aede-4739-a924-66e35f709582');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ],
      "source": [
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "1ZKrwnS3uMjF"
      },
      "outputs": [],
      "source": [
        "df.drop('message',axis=1,inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "TgG1UCmRuMjG",
        "outputId": "387c3cf4-05b1-4198-c280-edb7b01dbb74"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   label                                           sub_mssg\n",
              "0      0  job posting - apple-iss research center conten...\n",
              "1      0  query : letter frequencies for text identifica...\n",
              "2      0  risk a colleague and i are researching the dif...\n",
              "3      0  request book information earlier this morning ...\n",
              "4      0  call for abstracts : optimality in syntactic t..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-35617b36-9e36-439c-92d9-e4719633f2cf\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>sub_mssg</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>job posting - apple-iss research center conten...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>query : letter frequencies for text identifica...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>risk a colleague and i are researching the dif...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>request book information earlier this morning ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>call for abstracts : optimality in syntactic t...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-35617b36-9e36-439c-92d9-e4719633f2cf')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-35617b36-9e36-439c-92d9-e4719633f2cf button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-35617b36-9e36-439c-92d9-e4719633f2cf');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ],
      "source": [
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "NC67UJRQuMjG"
      },
      "outputs": [],
      "source": [
        "import re"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['sub_mssg'][5]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 218
        },
        "id": "WziwGE-8vzrB",
        "outputId": "eb32fbf6-182a-4fc4-9db9-9825ce6ba4ef"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'m . a . in scandinavian linguistics m . a . in scandinavian linguistics at the university of tromsoe 1995-1996 ( corrected version ) the linguistics section of the institute for language and literature at the university of tromsoe  norway  offers a one year course leading up to a master of arts degree  starting the fall of 1995 . the course is designed to prepare students for research in generative syntax and phonology with special attention to the scandinavian languages . the course is open to students from all countries with a background in linguistics corresponding roughly to a b . a . with a major in linguistics  or a norwegian cand . mag . the course is equivalent to 60 e . c . t . s credits . instruction will be in english . proficiency in a scandinavian language is not required  nor is previous knowledge of scandinavian grammar . those who wish may follow a course in norwegian for foreigners during the whole year . students pay no tuition  only a nominal registration fee . a limited number of grants are available for students from eastern europe or developing countries . the course will consist of lectures  seminars  and tutorials  with examinations at the end of each term ( fall and spring term ) . in addition  the students are required to write two research papers . in the fall of 1995 there will be a lecture series on scandinavian syntax and another one on scandinavian phonology . in addition there will be seminars  tutorials  and a crash course in icelandic grammar . in the spring of1996 there will be a lecture series on historical scandinavian linguistics  plus seminars and tutorials . also during spring term the students will write their two research papers . the teachers are anders holmberg and tarald taraldsen ( syntax )  ove lorenz and curtis rice ( phonology ) . other teachers who will contribute at various points in the course include christer platzack  cecilia falk ( university of lund )  halldor a . sigurdsson ( university of iceland )  tomas riad ( university of stockholm ) . for further information  contact anders holmberg isl  linguistics university of tromsoe n-9037 tromsoe  norway phone : 47-77645616  fax : 47-77645625 e-mail : andersh @ isl . uit . no '"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 218
        },
        "id": "zqkuGULwuMjG",
        "outputId": "5950eb28-49cd-4c4a-aa44-6b673e5d0c91"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'m . a . in scandinavian linguistics m . a . in scandinavian linguistics at the university of tromsoe numbers-numbers ( corrected version ) the linguistics section of the institute for language and literature at the university of tromsoe  norway  offers a one year course leading up to a master of arts degree  starting the fall of numbers . the course is designed to prepare students for research in generative syntax and phonology with special attention to the scandinavian languages . the course is open to students from all countries with a background in linguistics corresponding roughly to a b . a . with a major in linguistics  or a norwegian cand . mag . the course is equivalent to numbers e . c . t . s credits . instruction will be in english . proficiency in a scandinavian language is not required  nor is previous knowledge of scandinavian grammar . those who wish may follow a course in norwegian for foreigners during the whole year . students pay no tuition  only a nominal registration fee . a limited number of grants are available for students from eastern europe or developing countries . the course will consist of lectures  seminars  and tutorials  with examinations at the end of each term ( fall and spring term ) . in addition  the students are required to write two research papers . in the fall of numbers there will be a lecture series on scandinavian syntax and another one on scandinavian phonology . in addition there will be seminars  tutorials  and a crash course in icelandic grammar . in the spring ofnumbers there will be a lecture series on historical scandinavian linguistics  plus seminars and tutorials . also during spring term the students will write their two research papers . the teachers are anders holmberg and tarald taraldsen ( syntax )  ove lorenz and curtis rice ( phonology ) . other teachers who will contribute at various points in the course include christer platzack  cecilia falk ( university of lund )  halldor a . sigurdsson ( university of iceland )  tomas riad ( university of stockholm ) . for further information  contact anders holmberg isl  linguistics university of tromsoe n-numbers tromsoe  norway phone : numbers-numbers  fax : numbers-numbers e-mail : andersh @ isl . uit . no '"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 12
        }
      ],
      "source": [
        "#REPLACING NUMBERS\n",
        "df['sub_mssg']=df['sub_mssg'].str.replace(r'\\d+(\\.\\d+)?', 'numbers')\n",
        "df['sub_mssg'][5]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "5AviUXryuMjH"
      },
      "outputs": [],
      "source": [
        "#CONVRTING EVERYTHING TO LOWERCASE\n",
        "df['sub_mssg']=df['sub_mssg'].str.lower()\n",
        "#REPLACING NEXT LINES BY 'WHITE SPACE'\n",
        "df['sub_mssg']=df['sub_mssg'].str.replace(r'\\n',\" \") \n",
        "# REPLACING EMAIL IDs BY 'MAILID'\n",
        "df['sub_mssg']=df['sub_mssg'].str.replace(r'^.+@[^\\.].*\\.[a-z]{2,}$','MailID')\n",
        "# REPLACING URLs  BY 'Links'\n",
        "df['sub_mssg']=df['sub_mssg'].str.replace(r'^http\\://[a-zA-Z0-9\\-\\.]+\\.[a-zA-Z]{2,3}(/\\S*)?$','Links')\n",
        "# REPLACING CURRENCY SIGNS BY 'MONEY'\n",
        "df['sub_mssg']=df['sub_mssg'].str.replace(r'£|\\$', 'Money')\n",
        "# REPLACING LARGE WHITE SPACE BY SINGLE WHITE SPACE\n",
        "df['sub_mssg']=df['sub_mssg'].str.replace(r'\\s+', ' ')\n",
        "\n",
        "# REPLACING LEADING AND TRAILING WHITE SPACE BY SINGLE WHITE SPACE\n",
        "df['sub_mssg']=df['sub_mssg'].str.replace(r'^\\s+|\\s+?$', '')\n",
        "#REPLACING CONTACT NUMBERS\n",
        "df['sub_mssg']=df['sub_mssg'].str.replace(r'^\\(?[\\d]{3}\\)?[\\s-]?[\\d]{3}[\\s-]?[\\d]{4}$','contact number')\n",
        "#REPLACING SPECIAL CHARACTERS  BY WHITE SPACE \n",
        "df['sub_mssg']=df['sub_mssg'].str.replace(r\"[^a-zA-Z0-9]+\", \" \")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 218
        },
        "id": "_keSi9YbuMjO",
        "outputId": "b945ff04-b36f-47e5-e22d-4836a20af8d4"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'job posting apple iss research center content length numbers apple iss research center a us Money numbers million joint venture between apple computer inc and the institute of systems science of the national university of singapore located in singapore is looking for a senior speech scientist the successful candidate will have research expertise in computational linguistics including natural language processing and english and chinese statistical language modeling knowledge of state of the art corpus based n gram language models cache language models and part of speech language models are required a text to speech project leader the successful candidate will have research expertise expertise in two or more of the following areas computational linguistics including natural language parsing lexical database design and statistical language modeling text tokenization and normalization prosodic analysis substantial knowledge of the phonology syntax and semantics of chinese is required knowledge of acoustic phonetics and or speech signal processing is desirable both candidates will have a phd with at least numbers to numbers years of relevant work experience or a technical msc degree with at least numbers to numbers years of experienc e very strong software engineering skills including design and implementation and productization are required in these positions knowledge of c c and unix are preferred a unix c programmer we are looking for an experienced unix c programmer preferably with good industry experience to join us in breaking new frontiers strong knowledge of unix tools compilers linkers make x windows e mac and experience in matlab required sun and silicon graphic experience is an advantage programmers with less than two years industry experience need not apply these positions include interaction with scientists in the national university of singapore and with apple s speech research and productization efforts located in cupertino california attendance and publication in international scientific engineering conferences is encouraged benefits include an internationally competitive salary housing subsidy and relocation expenses send a complete resume enclosing personal particulars qualifications experience and contact telephone number to mr jean luc lebrun center manager apple iss research center institute of systems science heng mui keng terrace singapore numbers tel numbers numbers numbers fax numbers numbers numbers email jllebrun iss nus sg'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 14
        }
      ],
      "source": [
        "df['sub_mssg'][0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "S4wj3-j-uMjP"
      },
      "outputs": [],
      "source": [
        "# import spacy\n",
        "# nlp = spacy.load(\"en_core_websm\")\n",
        "\n",
        "# doc = nlp(\"Mando talked for 3 hours although talking isn't his thing\")\n",
        "# doc = nlp(\"eating eats eat ate adjustable rafting ability meeting better\")\n",
        "# for token in doc:\n",
        "#     print(token, \" | \", token.lemma)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(df['sub_mssg'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W3eLkIowwT3W",
        "outputId": "12958bb8-431a-4196-97a3-86ff088b569b"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4196"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(df['sub_mssg'][1])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M4T7TyqKyOay",
        "outputId": "3e3dc253-6b63-470e-8949-fe595bbb35f6"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1446"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['sub_mssg'][1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 218
        },
        "id": "VA1qO7XIGKAr",
        "outputId": "560f27a0-1dde-4ae0-ab09-537cecc6b6b2"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'query letter frequencies for text identification i am posting this inquiry for sergei atamas satamas umabnet ab umd edu a research associate at the university of maryland at baltimore his field is molecular biology and his work involves comparing dna strings using various algorithms i do n t understand the details well enough to pass them along at any rate one such algorithm relies upon frequencies with which the letters g a t and c occur in the dna strings he would like to explore the analogous use of letter sound frequencies in natural language texts hence this posting specifically sergei wonders if any linguist subscribers could help steer him to recent literature concerning text identification based on letter frequencies any suggestions could be sent directly to him at the above address or to me and i ll pass them along he would also be interested in collaborative work if this research connects with the work of any linguists or text processing specialists he observes that very often work in one field would actually help work in a far removed field if only people knew what was going on over there george fowler george fowler gfowler indiana edu email dept of slavic languages numbers numbers numbers numbers home try here first ballantine numbers numbers numbers numbers numbers numbers numbers dept indiana university numbers numbers numbers numbers office bloomington in numbers usa numbers numbers numbers numbers dept fax '"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import spacy\n",
        "nlp = spacy.load(\"en_core_web_sm\")"
      ],
      "metadata": {
        "id": "mFzdyh_eEab5"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# sm = \"\"\n",
        "# temp = str(df['sub_mssg'][1])\n",
        "# doc = nlp(temp)\n",
        "# for token in doc:\n",
        "#   sm = sm + token.lemma_ + \" \"\n"
      ],
      "metadata": {
        "id": "2anGvdTmFvcZ"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "words = set(nltk.corpus.words.words())"
      ],
      "metadata": {
        "id": "YrjqvZifVghK"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# sent = \"Io andiamo to the beach with my amico.\"\n",
        "# temp = \" \".join(w for w in nltk.wordpunct_tokenize(sent) if w.lower() in words or not w.isalpha())\n",
        "# temp"
      ],
      "metadata": {
        "id": "9c-lFt6yVo_D"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sm = \"\"\n",
        "uniqueWords = {\"call\"}\n",
        "for i in range(len(df['sub_mssg'])):\n",
        "  sm = \"\"\n",
        "  temp = str(df['sub_mssg'][i])\n",
        "  temp = \" \".join(w for w in nltk.wordpunct_tokenize(temp) if w.lower() in words or not w.isalpha())\n",
        "  doc = nlp(temp)\n",
        "  for token in doc:\n",
        "    sm = sm + token.lemma_ + \" \"\n",
        "    uniqueWords.add(token.lemma_)\n",
        "\n",
        "  df['sub_mssg'][i] = sm\n"
      ],
      "metadata": {
        "id": "gfo45o3GHqat"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# sm"
      ],
      "metadata": {
        "id": "yMbz_r7cGSIg"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# len(sm)"
      ],
      "metadata": {
        "id": "KEXI1pQEGUpq"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# if sm != df['sub_mssg'][1]:\n",
        "#   print(\"ho gayaaaa\")\n",
        "# else:\n",
        "#   print(\"lg gaye\")"
      ],
      "metadata": {
        "id": "139jGliJGdEh"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# sm = \"\"\n",
        "\n",
        "# for i in range(len(df['sub_mssg'])):\n",
        "#   sm= \"\"\n",
        "#   temp = str(df['sub_mssg'][i])\n",
        "#   doc = nlp(temp)\n",
        "#   for token in doc:\n",
        "#     sm = sm + token.lemma_ + \" \"\n",
        "  \n",
        "#   df['sub_mssg'][i] = sm\n",
        "# sm = \"\"\n",
        "# temp = str(df['sub_mssg'][0])\n",
        "# doc = nlp(temp)\n",
        "# for token in doc:\n",
        "#   sm = sm + token.lemma_ + \" \"\n",
        "# len(sm)"
      ],
      "metadata": {
        "id": "YojNuixDwIuJ"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BBEfS1t3uMjP",
        "outputId": "2f6a426e-0267-44e3-cfdd-7c36bc1fcbe9"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "897"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ],
      "source": [
        "len(df['sub_mssg'][7])"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(uniqueWords)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5RxNiiEpMT-b",
        "outputId": "3852b208-84af-4661-f10e-9552f014b583"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "21768"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(uniqueWords)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yBMb329uX-EG",
        "outputId": "c3f1a6b6-72d1-480e-c84d-e6287acb71bc"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "21768"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "LsV2bXm4uMjQ",
        "outputId": "a8caa380-7678-4b42-97d5-a32a67d9a353"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   label                                           sub_mssg\n",
              "0      0  job post apple research center content length ...\n",
              "1      0  query letter for text identification I be post...\n",
              "2      0  risk a colleague and I be the of risk by our h...\n",
              "3      0  request book information this morning I be on ...\n",
              "4      0  call for in syntactic theory content length ca..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-9ef8bfa4-0841-42ac-9f91-b364df3e32a2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>sub_mssg</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>job post apple research center content length ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>query letter for text identification I be post...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>risk a colleague and I be the of risk by our h...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>request book information this morning I be on ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>call for in syntactic theory content length ca...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9ef8bfa4-0841-42ac-9f91-b364df3e32a2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-9ef8bfa4-0841-42ac-9f91-b364df3e32a2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-9ef8bfa4-0841-42ac-9f91-b364df3e32a2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ],
      "source": [
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.to_csv(\"Updated.csv\",index=False)"
      ],
      "metadata": {
        "id": "i-v1TZ2iBPC_"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# file = open('geek.txt', 'r')\n",
        "# # This will print every line one by one in the file\n",
        "# for each in file:\n",
        "#     print (each)"
      ],
      "metadata": {
        "id": "Xw8Fg6NsPPWj"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "file = open(\"unique words.txt\",\"w\")"
      ],
      "metadata": {
        "id": "QQxFmgKoRRhY"
      },
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# uniqueWords"
      ],
      "metadata": {
        "id": "rLjKkguwRevt"
      },
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in uniqueWords:\n",
        "  file.write(i+\" \")"
      ],
      "metadata": {
        "id": "N79IIN4PRiHx"
      },
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "if '' in uniqueWords:\n",
        "  print(\"YEs\")\n",
        "else:\n",
        "  print(\"no\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gLLq4PDGZ1oY",
        "outputId": "d7f40c3e-1c31-43e6-a72b-d9fce5f5f0e3"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "no\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['sub_mssg']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5Sfr-O8vJkpq",
        "outputId": "81d1f943-c1be-40c2-af0c-7bf331e17e39"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0       job post apple research center content length ...\n",
              "1       query letter for text identification I be post...\n",
              "2       risk a colleague and I be the of risk by our h...\n",
              "3       request book information this morning I be on ...\n",
              "4       call for in syntactic theory content length ca...\n",
              "                              ...                        \n",
              "4191    streamline denizen ajar heavens stolid pinch s...\n",
              "4192    penny stock be about time nomad international ...\n",
              "4193    anomaly from crack poliomyelitis axisymmetric ...\n",
              "4194                        slutty to meet you take that \n",
              "4195    important banking alert dear value bank member...\n",
              "Name: sub_mssg, Length: 4196, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "t = df['sub_mssg'][0].split(' ')\n",
        "type(t)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pWlOH5kHOAOl",
        "outputId": "c0121096-828d-4c1b-810a-24b68df163d1"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "list"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "l = []\n",
        "for i in range(len(df['sub_mssg'])):\n",
        "  temp = df['sub_mssg'][i].split(' ')\n",
        "  l.append(temp)"
      ],
      "metadata": {
        "id": "ifo_wL2fKhB3"
      },
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "l = np.array(l)\n",
        "l.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aePACWgbKUFj",
        "outputId": "11c4c540-f62d-46f3-8dfd-7e205e6d8635"
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(4196,)"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import csv \n",
        "with open(\"updated final dataset.txt\",\"w\") as file:\n",
        "  wrt = csv.writer(file)\n",
        "  wrt.writerows(l)"
      ],
      "metadata": {
        "id": "DvKtoBV9OdS3"
      },
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "W9mo-bCKPOur"
      },
      "execution_count": 42,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3.9.1 64-bit",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.1"
    },
    "orig_nbformat": 4,
    "vscode": {
      "interpreter": {
        "hash": "b14d9e9f5e40da09a473c2902f3de164e0e2e0ee3890fe6368f6ed4b7113d70e"
      }
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}